1use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock, Mutex as StdMutex, OnceLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, JobCancellation, Lane};
31use crate::fleet_status::{spawn_fleet_status_dial, FleetStatusClient};
32use crate::jsonc::strip_jsonc;
33use crate::log_ctx;
34use crate::path_identity::ProjectRootId;
35use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
36use crate::response_finalize::{DispatchOutcome, PendingResponse};
37use crate::run_tool_call::{
38 finish_tool_call_response, prepare_tool_call, run_tool_call, strip_agent_preview_arg_owned,
39 PhaseTrace, ToolCallContext, ToolCallOutcome, ToolCallResult,
40};
41use crate::runtime_drain;
42use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
43
44use subc_protocol::manifest::{
45 Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
46 ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
47};
48use subc_protocol::session::{
49 HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
50 MODULE_CONTROL_OP_HEALTH_CHECK,
51};
52use subc_protocol::{
53 ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, MAX_FRAME_BODY_LEN,
54 PROTOCOL_VERSION,
55};
56use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
57use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
58use tokio::net::TcpStream;
59use tokio::sync::{mpsc, oneshot, Notify};
60use tokio::task::JoinHandle;
61
62const AUTH_DEADLINE: Duration = Duration::from_secs(5);
65const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
66const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
67const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
68const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
69
70const HELLO_CORR: u64 = 1;
72
73const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
76
77const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
81
82const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
86
87const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
91
92const WRITER_QUEUE_CAPACITY: usize = 256;
93
94const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
97
98const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
106const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
107 MaintenanceDrainKind::Watcher,
108 MaintenanceDrainKind::Lsp,
109 MaintenanceDrainKind::ConfigureTail,
110 MaintenanceDrainKind::CompletionDrains,
111];
112#[cfg(test)]
113const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
114
115const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
116const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
117
118const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
119const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
120
121const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
124
125const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
129
130const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
132const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
133
134#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
135struct RouteChannel {
136 channel: u16,
137 epoch: u32,
138}
139
140impl fmt::Display for RouteChannel {
141 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
142 write!(f, "{}@{}", self.channel, self.epoch)
143 }
144}
145
146type PushEnvelope = (ProjectRootId, PushFrame);
147type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
148type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
149mod bash;
150mod health;
151mod manifest;
152mod push;
153mod wire;
154
155use self::health::{
156 build_health_report, warn_slow_pending_binds, DispatchPathMetrics, HealthRollupCache,
157 ReapBlockerCensus, ResponseTaskGuard, HEALTH_ROLLUP_TTL,
158};
159use self::manifest::{
160 build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
161 is_subc_agent_core_tool, is_subc_native_plumbing_tool,
162};
163pub use self::wire::SubcError;
164
165pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
169 manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
170}
171use self::wire::{
172 build_error_frame, build_goodbye_frame, build_tool_response_frame,
173 build_tool_response_frame_with_limit, decrement_counted_channel, response_is_fatal_panic,
174 response_message, send_counted_channel, send_frame, send_reliable_writer_frame,
175 send_traced_tool_response_frame, ToolResponseWriteTrace, WriterFrame, WriterSender,
176};
177
178struct DecodedFrame {
179 frame: Frame,
180 phase_trace: PhaseTrace,
181}
182
183struct ToolCallCompletion {
184 text: String,
185 phase_trace: PhaseTrace,
186}
187
188#[derive(Clone)]
189struct ActiveToolCall {
190 root_id: ProjectRootId,
191 cancellation: JobCancellation,
192}
193
194type ActiveToolCalls = Arc<StdMutex<HashMap<(RouteChannel, u64), ActiveToolCall>>>;
195
196struct PendingInspectSetupGuard(Arc<AtomicUsize>);
197
198impl PendingInspectSetupGuard {
199 fn new(count: Arc<AtomicUsize>) -> Self {
200 count.fetch_add(1, Ordering::SeqCst);
201 Self(count)
202 }
203}
204
205impl Drop for PendingInspectSetupGuard {
206 fn drop(&mut self) {
207 self.0.fetch_sub(1, Ordering::SeqCst);
208 }
209}
210
211enum InspectSetupOutcome {
212 Immediate {
213 text: String,
214 phase_trace: PhaseTrace,
215 },
216 Deferred {
217 pending: PendingResponse,
218 surface_downgraded: bool,
219 phase_trace: PhaseTrace,
220 },
221}
222
223struct PendingSubcInspect {
224 route: RouteChannel,
225 corr: u64,
226 flags: Flags,
227 ver: u8,
228 root: ProjectRootId,
229 session_id: String,
230 format_context: crate::subc_format::FormatContext,
231 bind_trust: BindTrust,
232 pending: PendingResponse,
233 surface_downgraded: bool,
234 phase_trace: PhaseTrace,
235}
236
237struct ResolvedSubcInspect {
238 entry: PendingSubcInspect,
239 response: Response,
240}
241
242#[derive(Default)]
243struct PendingSubcInspects {
244 entries: Vec<PendingSubcInspect>,
245}
246
247impl PendingSubcInspects {
248 fn register(&mut self, pending: PendingSubcInspect) {
249 self.entries.retain(|entry| {
250 let keep = entry.route != pending.route || entry.corr != pending.corr;
251 if !keep {
252 if let Some(cancellation) = &entry.pending.cancellation {
253 cancellation.request_cancel();
254 }
255 }
256 keep
257 });
258 self.entries.push(pending);
259 }
260
261 fn poll_ready(&mut self, executor: &Executor) -> Vec<ResolvedSubcInspect> {
262 let mut ready = Vec::new();
263 let mut waiting = Vec::with_capacity(self.entries.len());
264 for mut entry in self.entries.drain(..) {
265 let response = executor
266 .actor_context(&entry.root)
267 .and_then(|ctx| (entry.pending.poll)(&ctx));
268 if let Some(response) = response {
269 ready.push(ResolvedSubcInspect { entry, response });
270 } else {
271 waiting.push(entry);
272 }
273 }
274 self.entries = waiting;
275 ready
276 }
277
278 fn drain_route(
279 &mut self,
280 route: RouteChannel,
281 executor: &Executor,
282 ) -> Vec<ResolvedSubcInspect> {
283 self.drain_matching(executor, |entry| entry.route == route)
284 }
285
286 fn drain_on_shutdown(&mut self, executor: &Executor) -> Vec<ResolvedSubcInspect> {
287 self.drain_matching(executor, |_| true)
288 }
289
290 fn drain_matching(
291 &mut self,
292 executor: &Executor,
293 matches: impl Fn(&PendingSubcInspect) -> bool,
294 ) -> Vec<ResolvedSubcInspect> {
295 let mut resolved = Vec::new();
296 let mut waiting = Vec::with_capacity(self.entries.len());
297 for mut entry in self.entries.drain(..) {
298 if !matches(&entry) {
299 waiting.push(entry);
300 continue;
301 }
302 if let Some(cancellation) = &entry.pending.cancellation {
303 cancellation.request_cancel();
304 }
305 if let Some(ctx) = executor.actor_context(&entry.root) {
306 if let Some(on_shutdown) = entry.pending.on_shutdown.as_mut() {
307 let response = on_shutdown(&ctx);
308 resolved.push(ResolvedSubcInspect { entry, response });
309 }
310 }
311 }
312 self.entries = waiting;
313 resolved
314 }
315
316 fn is_empty(&self) -> bool {
317 self.entries.is_empty()
318 }
319}
320
321#[derive(Clone)]
322struct PushSenders {
323 lossy_tx: mpsc::Sender<LossyPushEnvelope>,
324 reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
325 lossy_overflow: Arc<push::LossyOverflow>,
326 lossy_seq: Arc<AtomicU64>,
327 fleet_status_client: FleetStatusClient,
328}
329
330#[derive(Clone)]
331struct PersistentCancelSignal {
332 inner: Arc<PersistentCancelInner>,
333}
334
335struct PersistentCancelInner {
336 cancelled: AtomicBool,
337 notify: Notify,
338}
339
340impl PersistentCancelSignal {
341 fn new() -> Self {
342 Self {
343 inner: Arc::new(PersistentCancelInner {
344 cancelled: AtomicBool::new(false),
345 notify: Notify::new(),
346 }),
347 }
348 }
349
350 fn cancel(&self) {
351 if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
352 self.inner.notify.notify_waiters();
353 }
354 }
355
356 fn is_cancelled(&self) -> bool {
357 self.inner.cancelled.load(Ordering::SeqCst)
358 }
359
360 async fn cancelled(&self) {
361 loop {
369 let notified = self.inner.notify.notified();
370 tokio::pin!(notified);
371 notified.as_mut().enable();
372 if self.is_cancelled() {
373 return;
374 }
375 notified.await;
376 }
377 }
378}
379
380fn finish_active_tool_call(active: &ActiveToolCalls, route: RouteChannel, corr: u64) {
381 active
382 .lock()
383 .unwrap_or_else(std::sync::PoisonError::into_inner)
384 .remove(&(route, corr));
385}
386
387fn cancel_active_tool_call(
388 active: &ActiveToolCalls,
389 executor: &Executor,
390 route: RouteChannel,
391 corr: u64,
392 reason: &str,
393) -> bool {
394 let call = active
395 .lock()
396 .unwrap_or_else(std::sync::PoisonError::into_inner)
397 .remove(&(route, corr));
398 let Some(call) = call else {
399 return false;
400 };
401 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
402 log::debug!(
403 "subc attach: cancelled active tool call route={route} corr={corr} reason={reason} outcome={outcome:?}"
404 );
405 true
406}
407
408fn cancel_active_tool_calls_for_route(
409 active: &ActiveToolCalls,
410 executor: &Executor,
411 route: RouteChannel,
412 reason: &str,
413) -> usize {
414 let cancelled = {
415 let mut calls = active
416 .lock()
417 .unwrap_or_else(std::sync::PoisonError::into_inner);
418 let mut cancelled = Vec::new();
419 calls.retain(|(call_route, _), call| {
420 if *call_route == route {
421 cancelled.push(call.clone());
422 false
423 } else {
424 true
425 }
426 });
427 cancelled
428 };
429 for call in &cancelled {
430 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
431 log::debug!(
432 "subc attach: cancelled active tool call route={route} reason={reason} outcome={outcome:?}"
433 );
434 }
435 cancelled.len()
436}
437
438fn cancel_all_active_tool_calls(
439 active: &ActiveToolCalls,
440 executor: &Executor,
441 reason: &str,
442) -> usize {
443 let cancelled = {
444 let mut calls = active
445 .lock()
446 .unwrap_or_else(std::sync::PoisonError::into_inner);
447 calls.drain().map(|(_, call)| call).collect::<Vec<_>>()
448 };
449 for call in &cancelled {
450 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
451 log::debug!("subc attach: cancelled active tool call reason={reason} outcome={outcome:?}");
452 }
453 cancelled.len()
454}
455
456#[derive(Debug, Clone, Copy, PartialEq, Eq)]
457pub(crate) enum BindTrust {
458 FirstParty,
459 Untrusted,
460}
461
462impl BindTrust {
463 fn allows_bash_observation(self) -> bool {
464 matches!(self, Self::FirstParty)
465 }
466
467 fn label(self) -> &'static str {
468 match self {
469 Self::FirstParty => "first_party",
470 Self::Untrusted => "untrusted",
471 }
472 }
473
474 fn sandbox_trust(self) -> PrincipalTrust {
475 match self {
476 Self::FirstParty => PrincipalTrust::FirstParty,
477 Self::Untrusted => PrincipalTrust::Untrusted,
478 }
479 }
480}
481
482pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
483 match principal {
484 Some(Principal::Direct) => BindTrust::FirstParty,
485 Some(Principal::Reserved { module_id })
496 if module_id == "llm-runner"
497 || module_id == "aft"
498 || module_id == "broca"
499 || module_id == "alfonso-core"
500 || module_id == "prefrontal"
501 || module_id == "prefrontal-core" =>
502 {
503 BindTrust::FirstParty
504 }
505 Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
506 BindTrust::Untrusted
507 }
508 }
509}
510
511fn harness_forces_untrusted(harness: &str) -> bool {
512 harness.starts_with("fed:")
513}
514
515pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
516 if harness_forces_untrusted(harness) {
517 BindTrust::Untrusted
518 } else {
519 trust_for_principal(principal)
520 }
521}
522
523fn principal_id(principal: &Option<Principal>) -> Option<String> {
524 match principal {
525 Some(Principal::Direct) => Some("direct".to_string()),
526 Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
527 Some(Principal::Unverified) => Some("unverified".to_string()),
528 None => None,
529 }
530}
531
532fn principal_label(principal: &Option<Principal>) -> String {
533 principal_id(principal).unwrap_or_else(|| "absent".to_string())
534}
535
536#[derive(Debug)]
537struct RootMeta {
543 maintenance_pending: bool,
544 maintenance_jobs_in_flight: usize,
545 maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
546 maintenance_last_submitted: Option<Instant>,
547 maintenance_poisoned: bool,
548 last_touched: Instant,
549 diagnostics_on_edit: bool,
550 active_bash_waits: usize,
551 idle_artifacts_evicted: bool,
552 unbound_quiesced: bool,
553 consecutive_missing_sweeps: u8,
554}
555
556#[derive(Debug)]
557struct PendingBind {
558 bind_root_id: ProjectRootId,
559 inserted_new_actor: bool,
560 cancelled: bool,
561 configure_request_id: String,
562 started_at: Instant,
563 warned_half_deadline: bool,
564 deadline_reported: bool,
565 corr: u64,
566 ver: u8,
567 flags: Flags,
568 cancellation: crate::executor::JobCancellation,
573}
574
575struct RouteBindCompletion {
576 route: RouteChannel,
577 identity: RouteIdentity,
578 bind_root_id: ProjectRootId,
579 inserted_new_actor: bool,
580 configure_response: Response,
581 diagnostics_on_edit: bool,
582 ver: u8,
583 corr: u64,
584 flags: Flags,
585}
586
587#[derive(Debug, Clone)]
588struct RouteIdentity(Arc<RouteIdentityData>);
589
590#[derive(Debug)]
591struct RouteIdentityData {
592 root: ProjectRootId,
593 project_root: PathBuf,
594 harness: String,
595 session: String,
596 trust: BindTrust,
597 spawn_principal: AuthenticatedPrincipal,
598 consumer_elicitation_capable: bool,
599}
600
601impl Deref for RouteIdentity {
602 type Target = RouteIdentityData;
603
604 fn deref(&self) -> &Self::Target {
605 &self.0
606 }
607}
608
609#[derive(Debug, Clone)]
610struct RetainedSessionIdentity {
611 harness: String,
612 trust: BindTrust,
613}
614
615#[derive(Clone)]
616struct BgSub {
617 corr: u64,
618 ver: u8,
619 flags: Flags,
620 root: ProjectRootId,
621 session: String,
622}
623
624type BgSubsBySession = HashMap<(ProjectRootId, String), HashSet<RouteChannel>>;
627
628struct MaintenanceCompletion {
629 root_id: ProjectRootId,
630 kind: MaintenanceDrainKind,
631 response: Response,
632 empty_bg_sessions: Vec<(String, u64)>,
633 requeue_kind: Option<MaintenanceDrainKind>,
634}
635
636#[derive(Clone, Copy, Debug, PartialEq, Eq)]
637enum MaintenanceDrainKind {
638 Watcher,
639 Lsp,
640 ConfigureTail,
641 CompletionDrains,
642}
643
644impl MaintenanceDrainKind {
645 fn label(self) -> &'static str {
646 match self {
647 Self::Watcher => "watcher",
648 Self::Lsp => "lsp",
649 Self::ConfigureTail => "configure-tail",
650 Self::CompletionDrains => "completion-drains",
651 }
652 }
653}
654
655#[derive(Debug, Default)]
656struct MaintenanceJobOutcome {
657 empty_bg_sessions: Vec<(String, u64)>,
658 requeue_kind: Option<MaintenanceDrainKind>,
659}
660
661#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
662struct ReverseCorrKey {
663 route: RouteChannel,
664 corr: u64,
665}
666
667struct PendingBashAsk {
668 route: RouteChannel,
669 tool_corr: u64,
670 tool_flags: Flags,
671 tool_ver: u8,
672 root: ProjectRootId,
673 project_root: PathBuf,
674 session_id: String,
675 spawn_principal: AuthenticatedPrincipal,
676 edit_slot_survives: Option<bool>,
677 request_id: String,
678 arguments: Value,
679 format_context: crate::subc_format::FormatContext,
680 cancel: bash::BashWaitCancel,
681 grants: Vec<String>,
682 expires_at: Instant,
683}
684
685impl RootMeta {
686 fn new(now: Instant) -> Self {
687 Self {
688 maintenance_pending: false,
689 maintenance_jobs_in_flight: 0,
690 maintenance_queued_kinds: VecDeque::new(),
691 maintenance_last_submitted: None,
692 maintenance_poisoned: false,
693 last_touched: now,
694 diagnostics_on_edit: false,
695 active_bash_waits: 0,
696 idle_artifacts_evicted: false,
697 unbound_quiesced: false,
698 consecutive_missing_sweeps: 0,
699 }
700 }
701
702 fn note_activity(&mut self) {
703 self.last_touched = Instant::now();
704 }
705
706 fn reactivate_bound(&mut self) {
707 self.note_activity();
708 self.idle_artifacts_evicted = false;
709 self.unbound_quiesced = false;
710 }
711}
712
713fn due_maintenance_jobs(
714 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
715 executor: Option<&Executor>,
716 bg_sub_by_session: &BgSubsBySession,
717 bg_wake_pending: &HashSet<RouteChannel>,
718 budget: usize,
719 pending_bind_roots: &HashSet<ProjectRootId>,
720) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
721 let mut jobs = Vec::new();
722 let mut deferred = false;
723 let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
724 roots.sort_by(|left, right| {
725 let left_last = live_roots
726 .get(left)
727 .and_then(|meta| meta.maintenance_last_submitted);
728 let right_last = live_roots
729 .get(right)
730 .and_then(|meta| meta.maintenance_last_submitted);
731 left_last
732 .cmp(&right_last)
733 .then_with(|| left.as_path().cmp(right.as_path()))
734 });
735
736 for root_id in roots {
737 let Some(meta) = live_roots.get_mut(&root_id) else {
738 continue;
739 };
740 if meta.maintenance_poisoned {
741 continue;
742 }
743
744 if pending_bind_roots.contains(&root_id) {
745 if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
746 deferred = true;
747 }
748 continue;
749 }
750
751 if !meta.maintenance_pending {
752 if jobs.len() >= budget {
753 deferred = true;
754 continue;
755 }
756 let executor_actor_context =
760 executor.and_then(|executor| executor.actor_context(&root_id));
761 let root_has_pending_bg_wake =
762 bg_sub_by_session.iter().any(|((sub_root, _), channels)| {
763 sub_root == &root_id
764 && channels
765 .iter()
766 .any(|channel| bg_wake_pending.contains(channel))
767 });
768 let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
769 Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
770 .into_iter()
771 .filter(|kind| {
772 if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
773 return false;
774 }
775 match kind {
776 MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
777 MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
778 MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
779 MaintenanceDrainKind::CompletionDrains => {
785 root_has_pending_bg_wake || ctx.completion_drains_have_work()
786 }
787 }
788 })
789 .collect(),
790 None if meta.unbound_quiesced => Vec::new(),
791 None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
793 };
794 if kinds_with_work.is_empty() {
795 continue;
796 }
797 meta.maintenance_pending = true;
798 meta.maintenance_queued_kinds.extend(kinds_with_work);
799 }
800
801 while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
802 if jobs.len() >= budget {
803 meta.maintenance_queued_kinds.push_front(kind);
804 deferred = true;
805 break;
806 }
807 meta.maintenance_jobs_in_flight += 1;
808 meta.maintenance_last_submitted = Some(Instant::now());
809 jobs.push((root_id.clone(), kind));
810 }
811
812 meta.maintenance_pending =
813 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
814 }
815
816 (jobs, deferred)
817}
818
819fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
820 match estimate.estimated_bytes {
821 Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
822 None if estimate.status == "busy" => "busy".to_string(),
823 None => "not estimated".to_string(),
824 }
825}
826
827fn optional_memory_label(bytes: Option<u64>) -> String {
828 bytes.map_or_else(
829 || "not estimated".to_string(),
830 |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
831 )
832}
833
834fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
835 format!(
836 "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
837 optional_memory_label(relief.rss_before_bytes),
838 optional_memory_label(relief.rss_after_bytes),
839 optional_memory_label(relief.allocator_before.bytes_in_use),
840 optional_memory_label(relief.allocator_after.bytes_in_use),
841 optional_memory_label(relief.allocator_before.size_allocated),
842 optional_memory_label(relief.allocator_after.size_allocated),
843 optional_memory_label(relief.allocator_before.retained_slack_bytes),
844 optional_memory_label(relief.allocator_after.retained_slack_bytes),
845 relief.bytes_released as f64 / (1024.0 * 1024.0),
846 )
847}
848
849fn idle_root_eviction_message(
850 root_id: &ProjectRootId,
851 memory: &crate::memory::RootMemorySnapshot,
852 pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
853) -> String {
854 let freed_bytes = [
857 &memory.semantic,
858 &memory.trigram,
859 &memory.symbols,
860 &memory.callgraph,
861 &memory.inspect,
862 ]
863 .iter()
864 .filter_map(|estimate| estimate.estimated_bytes)
865 .fold(0u64, u64::saturating_add);
866 let mut message = format!(
867 "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
868 root_id.as_path().display(),
869 freed_bytes as f64 / (1024.0 * 1024.0),
870 eviction_estimate_label(&memory.semantic),
871 eviction_estimate_label(&memory.trigram),
872 eviction_estimate_label(&memory.symbols),
873 eviction_estimate_label(&memory.callgraph),
874 eviction_estimate_label(&memory.inspect),
875 eviction_estimate_label(&memory.bash),
876 eviction_estimate_label(&memory.lsp),
877 eviction_estimate_label(&memory.parser_pool),
878 );
879 if let Some(pressure_relief) = pressure_relief {
880 message.push_str(&pressure_relief_label(pressure_relief));
881 }
882 message
883}
884
885fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
886 !live_roots.is_empty()
887 && live_roots.values().all(|meta| {
888 now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
889 && meta.active_bash_waits == 0
890 && !meta.maintenance_pending
891 && meta.maintenance_queued_kinds.is_empty()
892 })
893}
894
895fn allocator_pressure_relief_after_idle_sweep(
896 now: Instant,
897 live_roots: &HashMap<ProjectRootId, RootMeta>,
898 executor: &Executor,
899) -> Option<crate::memory::AllocatorPressureRelief> {
900 if !process_has_been_idle(now, live_roots)
901 || live_roots.keys().any(|root_id| {
902 executor
903 .actor_context(root_id)
904 .is_some_and(|ctx| ctx.artifact_eviction_blocked())
905 })
906 {
907 return None;
908 }
909
910 #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))]
911 {
912 Some(crate::memory::relieve_allocator_pressure())
913 }
914 #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))]
915 {
916 None
917 }
918}
919
920fn quiesce_unbound_root(
921 root_id: &ProjectRootId,
922 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
923 executor: &Arc<Executor>,
924) {
925 let Some(meta) = live_roots.get_mut(root_id) else {
926 return;
927 };
928
929 let ctx = executor.actor_context(root_id);
930 if let Some(ctx) = ctx.as_ref() {
931 ctx.mark_subc_unbound();
935 }
936 let cancelled = executor.cancel_queued_maintenance(root_id);
937 let discarded = ctx
945 .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
946 .unwrap_or(0);
947 meta.unbound_quiesced = true;
948 meta.maintenance_queued_kinds.clear();
949 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
950 log::info!(
951 "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s)); cause=goodbye_unbound",
952 root_id.as_path().display(),
953 cancelled,
954 discarded
955 );
956}
957
958#[allow(clippy::too_many_arguments)]
959fn quiesce_connection_roots(
960 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
961 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
962 routes: &mut HashMap<RouteChannel, RouteIdentity>,
963 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
964 installed_route_epochs: &mut HashMap<u16, u32>,
965 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
966 active_tool_calls: &ActiveToolCalls,
967 executor: &Arc<Executor>,
968) {
969 cancel_all_active_tool_calls(active_tool_calls, executor, "connection teardown");
970 for cancel in route_bash_cancels.values() {
971 cancel.token.cancel();
972 }
973 route_bash_cancels.clear();
974
975 let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
976 for pending in pending_binds.values_mut() {
977 pending.cancelled = true;
978 roots.insert(pending.bind_root_id.clone());
979 let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
980 }
981
982 for root_id in roots {
986 if live_roots.contains_key(&root_id) {
987 quiesce_unbound_root(&root_id, live_roots, executor);
988 } else if let Some(ctx) = executor.actor_context(&root_id) {
989 ctx.mark_subc_unbound();
990 executor.cancel_queued_maintenance(&root_id);
991 crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
992 }
993 }
994
995 routes.clear();
996 root_channels.clear();
997 installed_route_epochs.clear();
998}
999
1000#[derive(Debug, Default)]
1004struct ReclaimedRoutes {
1005 highest_epoch_by_channel: HashMap<u16, u32>,
1006}
1007
1008impl ReclaimedRoutes {
1009 fn insert(&mut self, route: RouteChannel) {
1010 self.highest_epoch_by_channel
1011 .entry(route.channel)
1012 .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
1013 .or_insert(route.epoch);
1014 }
1015
1016 fn contains(&self, route: RouteChannel) -> bool {
1017 self.highest_epoch_by_channel
1018 .get(&route.channel)
1019 .is_some_and(|epoch| route.epoch <= *epoch)
1020 }
1021}
1022
1023#[derive(Debug, Default)]
1024struct IdleReapOutcome {
1025 evicted: usize,
1026 forgotten_deleted_roots: Vec<ProjectRootId>,
1027}
1028
1029fn reap_idle_roots(
1030 now: Instant,
1031 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1032 pending_binds: &HashMap<RouteChannel, PendingBind>,
1033 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1034 executor: &Arc<Executor>,
1035 metrics: &DispatchPathMetrics,
1036) -> IdleReapOutcome {
1037 let pending_bind_roots = pending_binds
1038 .values()
1039 .map(|pending| pending.bind_root_id.clone())
1040 .collect::<HashSet<_>>();
1041 let mut census = ReapBlockerCensus::default();
1042 let mut candidates = Vec::new();
1043
1044 for (root_id, meta) in live_roots.iter_mut() {
1045 let deleted = !root_id.as_path().exists();
1046 if deleted {
1047 meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
1057 } else {
1058 meta.consecutive_missing_sweeps = 0;
1059 }
1060 let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
1061 let has_bound_route = root_channels
1062 .get(root_id)
1063 .is_some_and(|channels| !channels.is_empty());
1064 let has_pending_bind = pending_bind_roots.contains(root_id);
1065
1066 if deleted {
1067 let mut retained = false;
1068 if !deletion_confirmed {
1069 census.absence_unconfirmed += 1;
1070 retained = true;
1071 }
1072 if meta.active_bash_waits > 0 {
1076 census.bash_waits += 1;
1077 retained = true;
1078 }
1079 if meta.maintenance_pending {
1080 census.maintenance_pending += 1;
1081 retained = true;
1082 }
1083 if !meta.maintenance_queued_kinds.is_empty() {
1084 census.maintenance_queued += 1;
1085 retained = true;
1086 }
1087 if has_pending_bind {
1088 census.pending_binds += 1;
1089 retained = true;
1090 }
1091 match executor.try_actor_is_idle(root_id) {
1092 Some(true) => {}
1093 Some(false) => {
1094 census.actor_busy += 1;
1095 retained = true;
1096 }
1097 None => {
1098 census.actor_state_busy += 1;
1099 retained = true;
1100 }
1101 }
1102 if retained {
1103 census.deleted_retained += 1;
1104 continue;
1105 }
1106 } else {
1107 if has_bound_route
1111 || !meta.unbound_quiesced
1112 || meta.idle_artifacts_evicted
1113 || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
1114 || meta.active_bash_waits > 0
1115 || meta.maintenance_pending
1116 || !meta.maintenance_queued_kinds.is_empty()
1117 || has_pending_bind
1118 || !executor.actor_is_idle(root_id)
1119 {
1120 continue;
1121 }
1122 }
1123 candidates.push((root_id.clone(), deleted));
1124 }
1125
1126 let mut reaped = Vec::new();
1127 let mut forgotten_deleted_roots = Vec::new();
1128 for (root_id, deleted) in candidates {
1129 let Some(ctx) = executor.actor_context(&root_id) else {
1130 if deleted {
1131 census.deleted_retained += 1;
1132 census.actor_busy += 1;
1133 }
1134 continue;
1135 };
1136 if deleted {
1147 ctx.bash_background()
1148 .kill_running_tasks_for_root(root_id.as_path());
1149 }
1150 let taken_pending = Some(ctx.take_pending_reconciliation_state());
1151 if ctx.artifact_eviction_blocked() {
1152 if let Some(pending) = taken_pending {
1153 ctx.restore_pending_reconciliation_state(pending);
1154 }
1155 if deleted {
1156 census.deleted_retained += 1;
1157 census.artifact_eviction_blocked += 1;
1158 }
1159 continue;
1160 }
1161 let memory_before = ctx.memory_root_snapshot();
1162 if !ctx.evict_idle_artifacts() {
1163 if let Some(pending) = taken_pending {
1164 ctx.restore_pending_reconciliation_state(pending);
1165 }
1166 if deleted {
1167 census.deleted_retained += 1;
1168 census.artifact_eviction_failed += 1;
1169 }
1170 continue;
1171 }
1172 drop(taken_pending);
1173 ctx.stop_watcher_runtime_in_background();
1174 ctx.invalidate_artifacts_after_watcher_gap();
1177
1178 if deleted {
1179 if executor.retire_idle_actor_in_background(&root_id) {
1180 live_roots.remove(&root_id);
1181 forgotten_deleted_roots.push(root_id.clone());
1182 } else {
1183 census.deleted_retained += 1;
1184 census.actor_busy += 1;
1185 }
1186 } else {
1187 if let Some(meta) = live_roots.get_mut(&root_id) {
1188 meta.idle_artifacts_evicted = true;
1189 }
1190 ctx.release_idle_reopenable_resources_in_background();
1191 }
1192 reaped.push((root_id, memory_before));
1193 }
1194
1195 metrics.record_reap(census);
1196 if census.deleted_retained > 0 {
1197 log::info!(
1198 "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
1199 census.deleted_retained,
1200 census.blocker_histogram()
1201 );
1202 }
1203
1204 let pressure_relief = (!reaped.is_empty())
1205 .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
1206 .flatten();
1207 for (root_id, memory_before) in &reaped {
1208 log::info!(
1209 "{}",
1210 idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
1211 );
1212 }
1213 IdleReapOutcome {
1214 evicted: reaped.len(),
1215 forgotten_deleted_roots,
1216 }
1217}
1218
1219#[allow(clippy::too_many_arguments)]
1220fn purge_deleted_root_residents(
1221 root_id: &ProjectRootId,
1222 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1223 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1224 installed_route_epochs: &mut HashMap<u16, u32>,
1225 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1226 active_tool_calls: &ActiveToolCalls,
1227 executor: &Executor,
1228 retry_buffer: &mut RetryBuffer,
1229 reclaimed_routes: &mut ReclaimedRoutes,
1230 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1231 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1232 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1233 bg_sub_by_session: &mut BgSubsBySession,
1234 bg_wake_pending: &mut HashSet<RouteChannel>,
1235 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1236 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1237 metrics: &DispatchPathMetrics,
1238) {
1239 let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1240 stale_routes.extend(
1241 routes
1242 .iter()
1243 .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1244 );
1245 stale_routes.extend(
1246 bg_sub_by_session
1247 .iter()
1248 .filter(|((root, _), _)| root == root_id)
1249 .flat_map(|(_, routes)| routes.iter().copied()),
1250 );
1251 stale_routes.extend(
1252 pending_bash_asks
1253 .values()
1254 .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1255 );
1256
1257 for route in stale_routes {
1258 reclaimed_routes.insert(route);
1259 remove_installed_route(installed_route_epochs, route);
1260 remove_route_channel(routes, root_channels, route);
1261 if let Some(cancel) = route_bash_cancels.remove(&route) {
1262 cancel.token.cancel();
1263 }
1264 cancel_active_tool_calls_for_route(active_tool_calls, executor, route, "root reclaim");
1265 retry_buffer.remove(&route);
1266 if let Some(sub) = bg_subs.remove(&route) {
1267 metrics.record_bg_subscription_ended(&sub.root, &sub.session, route, "root-reclaim");
1268 }
1269 bg_wake_pending.remove(&route);
1270 }
1271 root_channels.remove(root_id);
1272 session_identity.retain(|(root, _), _| root != root_id);
1273 push_buffer.retain(|key, _| &key.root != root_id);
1274 bg_wake_epoch.retain(|(root, _), _| root != root_id);
1275 pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1276 bg_sub_by_session.retain(|(root, _), _| root != root_id);
1277
1278 log::info!(
1279 "subc attach: fully forgot deleted root {}; cause=absence_reclaim",
1280 root_id.as_path().display()
1281 );
1282}
1283
1284#[allow(clippy::too_many_arguments)]
1285fn submit_due_maintenance_jobs(
1286 executor: &Arc<Executor>,
1287 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1288 pending_binds: &HashMap<RouteChannel, PendingBind>,
1289 bg_sub_by_session: &BgSubsBySession,
1290 bg_wake_pending: &HashSet<RouteChannel>,
1291 bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1292 maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1293 metrics: &Arc<DispatchPathMetrics>,
1294) {
1295 let pending_bind_roots = pending_binds
1296 .values()
1297 .map(|pending| pending.bind_root_id.clone())
1298 .collect::<HashSet<_>>();
1299 let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1300 live_roots,
1301 Some(executor),
1302 bg_sub_by_session,
1303 bg_wake_pending,
1304 MAINTENANCE_SUBMIT_BUDGET,
1305 &pending_bind_roots,
1306 );
1307 if deferred_jobs {
1308 metrics
1309 .maintenance_budget_deferrals
1310 .fetch_add(1, Ordering::Relaxed);
1311 }
1312 for (root_id, kind) in due_jobs {
1313 let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1314 bg_sub_by_session
1315 .iter()
1316 .filter_map(|((root, session), _)| {
1317 if root == &root_id {
1318 Some((
1319 session.clone(),
1320 bg_wake_epoch
1321 .get(&(root_id.clone(), session.clone()))
1322 .copied()
1323 .unwrap_or(0),
1324 ))
1325 } else {
1326 None
1327 }
1328 })
1329 .collect()
1330 } else {
1331 Vec::new()
1332 };
1333 submit_maintenance_job(
1334 executor,
1335 root_id,
1336 kind,
1337 bg_sessions_to_check,
1338 maintenance_tx,
1339 metrics,
1340 );
1341 }
1342}
1343
1344fn should_requiesce_after_maintenance(
1345 meta: &RootMeta,
1346 completed_kind: MaintenanceDrainKind,
1347 bind_pending: bool,
1348) -> bool {
1349 meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1350}
1351
1352fn note_maintenance_completion(
1353 meta: &mut RootMeta,
1354 requeue_kind: Option<MaintenanceDrainKind>,
1355 fatal: bool,
1356 defer_requeue: bool,
1357) {
1358 if fatal {
1359 meta.maintenance_poisoned = true;
1360 }
1361
1362 if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1363 meta.maintenance_queued_kinds.push_back(kind);
1364 }
1365
1366 meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1367 meta.maintenance_pending =
1368 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1369}
1370
1371fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1372 RouteChannel { channel, epoch }
1373}
1374
1375fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1376 if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1377 installed_epochs.remove(&route.channel);
1378 }
1379}
1380
1381fn ingress_route_should_be_processed(
1382 installed_epochs: &HashMap<u16, u32>,
1383 reclaimed_routes: &ReclaimedRoutes,
1384 frame: &Frame,
1385) -> bool {
1386 if frame.header.channel == 0
1387 || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1388 {
1389 return true;
1390 }
1391
1392 frame.header.ty == FrameType::Request
1397 && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1398}
1399
1400fn bash_elicitation_timeout() -> Duration {
1401 if cfg!(debug_assertions) {
1402 if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1403 if let Ok(ms) = raw.parse::<u64>() {
1404 if ms > 0 {
1405 return Duration::from_millis(ms);
1406 }
1407 }
1408 }
1409 }
1410 BASH_ELICITATION_TIMEOUT
1411}
1412
1413fn allocate_reverse_corr(
1414 pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1415 route: RouteChannel,
1416 next_corr: &mut u64,
1417) -> u64 {
1418 loop {
1419 let corr = *next_corr;
1420 *next_corr = (*next_corr).wrapping_add(1).max(1);
1421 if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1422 return corr;
1423 }
1424 }
1425}
1426
1427fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1428 match kind {
1429 crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1430 crate::bash_permissions::PermissionKind::Bash => "bash",
1431 }
1432}
1433
1434fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1435 let mut patterns = Vec::new();
1436 let mut seen = HashSet::new();
1437 for ask in asks {
1438 for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1439 if seen.insert(pattern.clone()) {
1440 patterns.push(pattern.clone());
1441 }
1442 }
1443 }
1444 patterns
1445}
1446
1447fn bash_elicitation_message(
1448 command: &str,
1449 asks: &[crate::bash_permissions::PermissionAsk],
1450) -> String {
1451 let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1452 let patterns = bash_elicitation_patterns(asks);
1453 let pattern_text = if patterns.is_empty() {
1454 "no matched permission patterns".to_string()
1455 } else {
1456 patterns.join(", ")
1457 };
1458 let ask_kinds = asks
1459 .iter()
1460 .map(|ask| bash_permission_kind_label(&ask.kind))
1461 .collect::<HashSet<_>>()
1462 .into_iter()
1463 .collect::<Vec<_>>()
1464 .join(", ");
1465 if ask_kinds.is_empty() {
1466 format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1467 } else {
1468 format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1469 }
1470}
1471
1472fn bash_elicitation_request_body(
1473 command: &str,
1474 asks: &[crate::bash_permissions::PermissionAsk],
1475) -> Value {
1476 json!({
1477 "method": BASH_ELICITATION_CREATE_METHOD,
1478 "params": {
1479 "mode": "form",
1480 "message": bash_elicitation_message(command, asks),
1481 "requestedSchema": {
1482 "type": "object",
1483 "properties": {
1484 "decision": {
1485 "type": "string",
1486 "enum": ["allow", "deny"],
1487 "description": "Choose allow to run this bash command once, or deny to block it."
1488 }
1489 },
1490 "required": ["decision"],
1491 "additionalProperties": false
1492 },
1493 "_meta": {
1494 "aft": {
1495 "tool": "bash",
1496 "command": command,
1497 "asks": asks
1498 }
1499 }
1500 }
1501 })
1502}
1503
1504fn build_bash_elicitation_request_frame(
1505 ver: u8,
1506 route: RouteChannel,
1507 corr: u64,
1508 flags: Flags,
1509 command: &str,
1510 asks: &[crate::bash_permissions::PermissionAsk],
1511) -> Result<Frame, SubcError> {
1512 let body = bash_elicitation_request_body(command, asks);
1513 Frame::build_with_version(
1514 ver,
1515 FrameType::Request,
1516 flags,
1517 route.channel,
1518 route.epoch,
1519 corr,
1520 serde_json::to_vec(&body).map_err(SubcError::Json)?,
1521 )
1522 .map_err(SubcError::FrameBuild)
1523}
1524
1525fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1526 let Ok(value) = serde_json::from_slice::<Value>(body) else {
1527 return false;
1528 };
1529 flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1530}
1531
1532fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1533 let Some(object) = value.as_object() else {
1534 return false;
1535 };
1536 object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1537}
1538
1539fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1540 let Some(object) = value.as_object() else {
1541 return false;
1542 };
1543 if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1544 return false;
1545 }
1546 let Some(content) = object.get("content").and_then(Value::as_object) else {
1547 return false;
1548 };
1549 content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1550}
1551
1552#[allow(clippy::too_many_arguments)]
1553async fn settle_pending_bash_ask_denied(
1554 tx: &WriterSender,
1555 pending: PendingBashAsk,
1556 routes: &HashMap<RouteChannel, RouteIdentity>,
1557 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1558 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1559 shutdown: &Arc<Notify>,
1560 metrics: &DispatchPathMetrics,
1561) -> Result<(), SubcError> {
1562 let completion = bash::bash_denied_untrusted_completion(
1563 pending.route,
1564 pending.tool_corr,
1565 pending.tool_flags,
1566 pending.tool_ver,
1567 pending.root,
1568 pending.request_id,
1569 pending.format_context,
1570 );
1571 bash::handle_bash_deferred_completion(
1572 tx,
1573 completion,
1574 routes,
1575 live_roots,
1576 route_bash_cancels,
1577 shutdown,
1578 metrics,
1579 )
1580 .await
1581}
1582
1583fn take_pending_bash_asks_for_route(
1584 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1585 route: RouteChannel,
1586) -> Vec<PendingBashAsk> {
1587 let keys = pending_bash_asks
1588 .keys()
1589 .copied()
1590 .filter(|key| key.route == route)
1591 .collect::<Vec<_>>();
1592 keys.into_iter()
1593 .filter_map(|key| pending_bash_asks.remove(&key))
1594 .collect()
1595}
1596
1597#[allow(clippy::too_many_arguments)]
1598async fn settle_pending_bash_asks_for_route(
1599 tx: &WriterSender,
1600 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1601 route: RouteChannel,
1602 routes: &HashMap<RouteChannel, RouteIdentity>,
1603 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1604 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1605 shutdown: &Arc<Notify>,
1606 metrics: &DispatchPathMetrics,
1607) -> Result<(), SubcError> {
1608 for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1609 settle_pending_bash_ask_denied(
1610 tx,
1611 pending,
1612 routes,
1613 live_roots,
1614 route_bash_cancels,
1615 shutdown,
1616 metrics,
1617 )
1618 .await?;
1619 }
1620 Ok(())
1621}
1622
1623#[allow(clippy::too_many_arguments)]
1624async fn settle_all_pending_bash_asks(
1625 tx: &WriterSender,
1626 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1627 routes: &HashMap<RouteChannel, RouteIdentity>,
1628 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1629 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1630 shutdown: &Arc<Notify>,
1631 metrics: &DispatchPathMetrics,
1632) -> Result<(), SubcError> {
1633 let pending = pending_bash_asks
1634 .drain()
1635 .map(|(_, pending)| pending)
1636 .collect::<Vec<_>>();
1637 for pending in pending {
1638 settle_pending_bash_ask_denied(
1639 tx,
1640 pending,
1641 routes,
1642 live_roots,
1643 route_bash_cancels,
1644 shutdown,
1645 metrics,
1646 )
1647 .await?;
1648 }
1649 Ok(())
1650}
1651
1652#[allow(clippy::too_many_arguments)]
1653async fn expire_pending_bash_asks(
1654 tx: &WriterSender,
1655 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1656 routes: &HashMap<RouteChannel, RouteIdentity>,
1657 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1658 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1659 shutdown: &Arc<Notify>,
1660 metrics: &DispatchPathMetrics,
1661) -> Result<(), SubcError> {
1662 let now = Instant::now();
1663 let expired = pending_bash_asks
1664 .iter()
1665 .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1666 .collect::<Vec<_>>();
1667 for key in expired {
1668 if let Some(pending) = pending_bash_asks.remove(&key) {
1669 log::debug!(
1670 "subc attach: bash elicitation request {} on route {} expired fail-closed",
1671 key.corr,
1672 pending.route
1673 );
1674 settle_pending_bash_ask_denied(
1675 tx,
1676 pending,
1677 routes,
1678 live_roots,
1679 route_bash_cancels,
1680 shutdown,
1681 metrics,
1682 )
1683 .await?;
1684 }
1685 }
1686 Ok(())
1687}
1688
1689#[allow(clippy::too_many_arguments)]
1690async fn handle_bash_elicitation_reply(
1691 tx: &WriterSender,
1692 frame: &Frame,
1693 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1694 routes: &HashMap<RouteChannel, RouteIdentity>,
1695 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1696 executor: &Arc<Executor>,
1697 shutdown: &Arc<Notify>,
1698 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1699 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1700 metrics: &Arc<DispatchPathMetrics>,
1701 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1702 dispatch: DispatchFn,
1703) -> Result<(), SubcError> {
1704 let key = ReverseCorrKey {
1705 route: route_key(frame.header.channel, frame.header.epoch),
1706 corr: frame.header.corr,
1707 };
1708 let Some(pending) = pending_bash_asks.remove(&key) else {
1709 return Ok(());
1710 };
1711
1712 if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1713 if routes.contains_key(&key.route) {
1714 bash::submit_deferred_bash(
1715 executor,
1716 bash_deferred_tx,
1717 bash_poll_touch_tx,
1718 metrics,
1719 dispatch,
1720 pending.root,
1721 pending.project_root,
1722 pending.session_id,
1723 pending.request_id,
1724 pending.route,
1725 pending.tool_corr,
1726 pending.tool_flags,
1727 pending.tool_ver,
1728 pending.arguments,
1729 pending.format_context,
1730 pending.cancel,
1731 BindTrust::Untrusted,
1732 pending.spawn_principal,
1733 pending.edit_slot_survives,
1734 Some(pending.grants),
1735 );
1736 return Ok(());
1737 }
1738 log::debug!(
1739 "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1740 key.corr,
1741 pending.route
1742 );
1743 }
1744
1745 settle_pending_bash_ask_denied(
1746 tx,
1747 pending,
1748 routes,
1749 live_roots,
1750 route_bash_cancels,
1751 shutdown,
1752 metrics,
1753 )
1754 .await
1755}
1756
1757#[allow(clippy::too_many_arguments)]
1758async fn cancel_pending_bash_ask_for_tool_call(
1759 tx: &WriterSender,
1760 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1761 route: RouteChannel,
1762 tool_corr: u64,
1763 routes: &HashMap<RouteChannel, RouteIdentity>,
1764 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1765 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1766 shutdown: &Arc<Notify>,
1767 metrics: &DispatchPathMetrics,
1768) -> Result<(), SubcError> {
1769 let keys = pending_bash_asks
1770 .iter()
1771 .filter_map(|(key, pending)| {
1772 (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1773 })
1774 .collect::<Vec<_>>();
1775 for key in keys {
1776 if let Some(pending) = pending_bash_asks.remove(&key) {
1777 settle_pending_bash_ask_denied(
1778 tx,
1779 pending,
1780 routes,
1781 live_roots,
1782 route_bash_cancels,
1783 shutdown,
1784 metrics,
1785 )
1786 .await?;
1787 }
1788 }
1789 Ok(())
1790}
1791
1792fn remove_root_channel(
1793 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1794 root: &ProjectRootId,
1795 channel: RouteChannel,
1796) {
1797 let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1798 channels.remove(&channel);
1799 channels.is_empty()
1800 } else {
1801 false
1802 };
1803 if remove_root {
1804 root_channels.remove(root);
1805 }
1806}
1807
1808fn remove_route_channel(
1809 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1810 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1811 channel: RouteChannel,
1812) -> Option<RouteIdentity> {
1813 let removed = routes.remove(&channel);
1814 if let Some(identity) = &removed {
1815 remove_root_channel(root_channels, &identity.root, channel);
1816 }
1817 removed
1818}
1819
1820fn insert_route_channel(
1821 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1822 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1823 channel: RouteChannel,
1824 identity: RouteIdentity,
1825) {
1826 if let Some(previous) = routes.insert(channel, identity.clone()) {
1827 remove_root_channel(root_channels, &previous.root, channel);
1828 }
1829 root_channels
1830 .entry(identity.root.clone())
1831 .or_default()
1832 .insert(channel);
1833}
1834
1835fn insert_bg_subscription_index(
1836 bg_sub_by_session: &mut BgSubsBySession,
1837 root: ProjectRootId,
1838 session: String,
1839 channel: RouteChannel,
1840) {
1841 bg_sub_by_session
1842 .entry((root, session))
1843 .or_default()
1844 .insert(channel);
1845}
1846
1847fn remove_bg_subscription_index(
1848 bg_sub_by_session: &mut BgSubsBySession,
1849 channel: RouteChannel,
1850 identity: Option<&RouteIdentity>,
1851) {
1852 if let Some(identity) = identity {
1853 let key = (identity.root.clone(), identity.session.clone());
1854 let remove_key = bg_sub_by_session.get_mut(&key).is_some_and(|channels| {
1855 channels.remove(&channel);
1856 channels.is_empty()
1857 });
1858 if remove_key {
1859 bg_sub_by_session.remove(&key);
1860 }
1861 } else {
1862 bg_sub_by_session.retain(|_, channels| {
1863 channels.remove(&channel);
1864 !channels.is_empty()
1865 });
1866 }
1867}
1868
1869fn route_removal_will_quiesce_root(
1870 root: &ProjectRootId,
1871 route: RouteChannel,
1872 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1873 has_pending_bind: bool,
1874 replacement_root: Option<&ProjectRootId>,
1875) -> bool {
1876 let removes_last_route = root_channels
1877 .get(root)
1878 .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1879 removes_last_route && !has_pending_bind && replacement_root != Some(root)
1880}
1881
1882fn should_quiesce_removed_root(
1883 root: &ProjectRootId,
1884 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1885 has_pending_bind: bool,
1886 replacement_root: Option<&ProjectRootId>,
1887) -> bool {
1888 !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1889}
1890
1891async fn end_bg_subscription(
1892 writer_tx: &WriterSender,
1893 metrics: &DispatchPathMetrics,
1894 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1895 bg_sub_by_session: &mut BgSubsBySession,
1896 bg_wake_pending: &mut HashSet<RouteChannel>,
1897 channel: RouteChannel,
1898 identity: Option<&RouteIdentity>,
1899 cause: &str,
1900) -> Result<(), SubcError> {
1901 if let Some(sub) = bg_subs.remove(&channel) {
1902 bg_wake_pending.remove(&channel);
1903 remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1904 metrics.record_bg_subscription_ended(&sub.root, &sub.session, channel, cause);
1905 push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1906 }
1907 Ok(())
1908}
1909
1910#[allow(clippy::too_many_arguments)]
1911async fn teardown_installed_route(
1912 tx: &WriterSender,
1913 metrics: &DispatchPathMetrics,
1914 executor: &Arc<Executor>,
1915 channel: RouteChannel,
1916 cancellation_reason: &str,
1917 replacement_root: Option<&ProjectRootId>,
1918 installed_route_epochs: &mut HashMap<u16, u32>,
1919 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1920 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1921 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1922 bg_sub_by_session: &mut BgSubsBySession,
1923 bg_wake_pending: &mut HashSet<RouteChannel>,
1924 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1925 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1926 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1927 active_tool_calls: &ActiveToolCalls,
1928 pending_inspects: &mut PendingSubcInspects,
1929 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1930 retry_buffer: &mut RetryBuffer,
1931 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1932 shutdown: &Arc<Notify>,
1933 tool_response_body_limit: usize,
1934) -> Result<(), SubcError> {
1935 remove_installed_route(installed_route_epochs, channel);
1936 let bg_end_cause = match cancellation_reason {
1937 "Goodbye" => "goodbye",
1938 "higher-epoch RouteBind" => "higher-epoch",
1939 other => other,
1940 };
1941 end_bg_subscription(
1942 tx,
1943 metrics,
1944 bg_subs,
1945 bg_sub_by_session,
1946 bg_wake_pending,
1947 channel,
1948 routes.get(&channel),
1949 bg_end_cause,
1950 )
1951 .await?;
1952 settle_pending_bash_asks_for_route(
1953 tx,
1954 pending_bash_asks,
1955 channel,
1956 routes,
1957 live_roots,
1958 route_bash_cancels,
1959 shutdown,
1960 metrics,
1961 )
1962 .await?;
1963 if let Some(cancel) = route_bash_cancels.remove(&channel) {
1964 cancel.token.cancel();
1965 }
1966 for resolved in pending_inspects.drain_route(channel, executor) {
1967 deliver_resolved_subc_inspect(
1968 tx,
1969 resolved,
1970 routes,
1971 live_roots,
1972 executor.as_ref(),
1973 active_tool_calls,
1974 shutdown,
1975 metrics,
1976 tool_response_body_limit,
1977 )
1978 .await?;
1979 }
1980 cancel_active_tool_calls_for_route(active_tool_calls, executor, channel, cancellation_reason);
1981 if let Some(pending) = pending_binds.get_mut(&channel) {
1982 pending.cancelled = true;
1983 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1984 log::debug!(
1985 "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1986 channel.channel
1987 );
1988 }
1989 let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1990 if let Some(identity) = routes.get(&channel) {
1991 let has_pending_bind = pending_binds
1992 .values()
1993 .any(|pending| pending.bind_root_id == identity.root);
1994 if route_removal_will_quiesce_root(
1995 &identity.root,
1996 channel,
1997 root_channels,
1998 has_pending_bind,
1999 replacement_root,
2000 ) {
2001 if let Some(ctx) = executor.actor_context(&identity.root) {
2002 ctx.mark_subc_unbound();
2005 }
2006 }
2007 }
2008 if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
2009 let session_still_routed = routes
2010 .values()
2011 .any(|route| route.root == identity.root && route.session == identity.session);
2012 if !session_still_routed {
2013 if let Some(ctx) = executor.actor_context(&identity.root) {
2014 ctx.hashline_bindings()
2015 .teardown(identity.root.as_path(), &identity.session);
2016 }
2017 }
2018 if migrated > 0 {
2019 log::debug!(
2020 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
2021 channel.channel
2022 );
2023 }
2024 if let Some(meta) = live_roots.get_mut(&identity.root) {
2025 let idle_for = meta.last_touched.elapsed();
2026 meta.note_activity();
2027 log::debug!(
2028 "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
2029 channel.channel,
2030 identity.root.as_path().display(),
2031 identity.harness,
2032 identity.session,
2033 idle_for
2034 );
2035 } else {
2036 log::debug!(
2037 "subc attach: route {} torn down for root {} harness {} session {}",
2038 channel.channel,
2039 identity.root.as_path().display(),
2040 identity.harness,
2041 identity.session
2042 );
2043 }
2044 let has_pending_bind = pending_binds
2045 .values()
2046 .any(|pending| pending.bind_root_id == identity.root);
2047 if should_quiesce_removed_root(
2048 &identity.root,
2049 root_channels,
2050 has_pending_bind,
2051 replacement_root,
2052 ) {
2053 quiesce_unbound_root(&identity.root, live_roots, executor);
2054 }
2055 } else {
2056 if migrated > 0 {
2057 log::debug!(
2058 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
2059 channel.channel
2060 );
2061 }
2062 log::debug!("subc attach: unbound route {} torn down", channel.channel);
2063 }
2064 Ok(())
2065}
2066
2067fn remember_session_identity(
2068 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2069 identity: &RouteIdentity,
2070) {
2071 let key = (identity.root.clone(), identity.session.clone());
2072 if matches!(identity.trust, BindTrust::Untrusted)
2073 && session_identity
2074 .get(&key)
2075 .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
2076 {
2077 return;
2078 }
2079
2080 session_identity.insert(
2085 key,
2086 RetainedSessionIdentity {
2087 harness: identity.harness.clone(),
2088 trust: identity.trust,
2089 },
2090 );
2091}
2092
2093fn replay_key_for_session(
2094 session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2095 root: &ProjectRootId,
2096 session: &str,
2097) -> Option<(push::ReplayKey, BindTrust)> {
2098 let retained = session_identity.get(&(root.clone(), session.to_string()))?;
2099 Some((
2100 push::ReplayKey {
2101 root: root.clone(),
2102 harness: retained.harness.clone(),
2103 session: session.to_string(),
2104 },
2105 retained.trust,
2106 ))
2107}
2108pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
2111
2112#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2113enum ModuleLoopExit {
2114 Graceful,
2115 SkipSearchFlush,
2116}
2117
2118pub fn run_subc_mode(
2123 connection_file_path: &Path,
2124 ctx: Arc<AppContext>,
2125 executor: Arc<Executor>,
2126 dispatch: DispatchFn,
2127 user_config_path: Option<PathBuf>,
2128) -> Result<(), SubcError> {
2129 run_subc_mode_inner(
2133 connection_file_path,
2134 ctx,
2135 executor,
2136 dispatch,
2137 user_config_path,
2138 false,
2139 MAX_FRAME_BODY_LEN as usize,
2140 )
2141}
2142
2143fn run_subc_mode_inner(
2144 connection_file_path: &Path,
2145 ctx: Arc<AppContext>,
2146 executor: Arc<Executor>,
2147 dispatch: DispatchFn,
2148 user_config_path: Option<PathBuf>,
2149 allow_native_passthrough: bool,
2150 tool_response_body_limit: usize,
2151) -> Result<(), SubcError> {
2152 let runtime = tokio::runtime::Builder::new_current_thread()
2153 .enable_all()
2154 .build()
2155 .map_err(SubcError::Runtime)?;
2156
2157 let executor_for_loop = Arc::clone(&executor);
2158 let loop_result = runtime.block_on(async move {
2159 let shared_app = ctx.app();
2160 drop(ctx);
2161 let stream = connect_and_authenticate(connection_file_path).await?;
2162 log::info!(
2163 "subc attach: authenticated to daemon via {}",
2164 connection_file_path.display()
2165 );
2166 let (read_half, write_half) = tokio::io::split(stream);
2167 run_module_loop(
2168 read_half,
2169 write_half,
2170 connection_file_path,
2171 shared_app,
2172 executor_for_loop,
2173 dispatch,
2174 user_config_path,
2175 allow_native_passthrough,
2176 tool_response_body_limit,
2177 )
2178 .await
2179 });
2180
2181 let actor_contexts = executor.actor_contexts();
2182 if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
2183 flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
2186 }
2187 for actor_ctx in &actor_contexts {
2188 actor_ctx.lsp().shutdown_all();
2189 actor_ctx.bash_background().detach();
2190 }
2191
2192 loop_result.map(|_| ())
2193}
2194
2195fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
2196 for actor_ctx in actor_contexts {
2197 let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
2198 }
2199 let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
2200}
2201
2202#[doc(hidden)]
2207pub fn run_subc_mode_for_test(
2208 connection_file_path: &Path,
2209 ctx: Arc<AppContext>,
2210 executor: Arc<Executor>,
2211 dispatch: DispatchFn,
2212 user_config_path: Option<PathBuf>,
2213) -> Result<(), SubcError> {
2214 run_subc_mode_inner(
2215 connection_file_path,
2216 ctx,
2217 executor,
2218 dispatch,
2219 user_config_path,
2220 true,
2221 MAX_FRAME_BODY_LEN as usize,
2222 )
2223}
2224
2225#[doc(hidden)]
2228pub fn run_subc_mode_for_test_with_response_body_limit(
2229 connection_file_path: &Path,
2230 ctx: Arc<AppContext>,
2231 executor: Arc<Executor>,
2232 dispatch: DispatchFn,
2233 user_config_path: Option<PathBuf>,
2234 tool_response_body_limit: usize,
2235) -> Result<(), SubcError> {
2236 assert!((4 * 1_024..=MAX_FRAME_BODY_LEN as usize).contains(&tool_response_body_limit));
2237 run_subc_mode_inner(
2238 connection_file_path,
2239 ctx,
2240 executor,
2241 dispatch,
2242 user_config_path,
2243 true,
2244 tool_response_body_limit,
2245 )
2246}
2247
2248#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2249enum AttachErrorClass {
2250 Transient,
2251 Permanent,
2252}
2253
2254impl fmt::Display for AttachErrorClass {
2255 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2256 match self {
2257 Self::Transient => f.write_str("transient"),
2258 Self::Permanent => f.write_str("permanent"),
2259 }
2260 }
2261}
2262
2263#[derive(Clone, Copy)]
2264struct AttachRetryPolicy {
2265 budget: Duration,
2266 initial_backoff: Duration,
2267 max_backoff: Duration,
2268 jitter_percent: u64,
2269}
2270
2271const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
2272 budget: ATTACH_RETRY_BUDGET,
2273 initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
2274 max_backoff: ATTACH_RETRY_MAX_BACKOFF,
2275 jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
2276};
2277
2278fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
2281 let transient = match error {
2282 SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
2283 SubcError::Auth { source, .. } => match source {
2284 subc_transport::AuthError::Timeout { .. }
2285 | subc_transport::AuthError::UnexpectedEof { .. } => true,
2286 subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
2287 _ => false,
2288 },
2289 _ => false,
2290 };
2291 if transient {
2292 AttachErrorClass::Transient
2293 } else {
2294 AttachErrorClass::Permanent
2295 }
2296}
2297
2298fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
2299 matches!(
2300 kind,
2301 io::ErrorKind::ConnectionRefused
2302 | io::ErrorKind::TimedOut
2303 | io::ErrorKind::ConnectionReset
2304 | io::ErrorKind::ConnectionAborted
2305 | io::ErrorKind::BrokenPipe
2306 | io::ErrorKind::UnexpectedEof
2307 )
2308}
2309
2310async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
2314 connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
2315}
2316
2317async fn connect_and_authenticate_with_policy(
2318 connection_file_path: &Path,
2319 policy: AttachRetryPolicy,
2320) -> Result<TcpStream, SubcError> {
2321 let started_at = Instant::now();
2322 let deadline = started_at + policy.budget;
2323 let mut attempt = 0_u32;
2324 let mut backoff = policy.initial_backoff;
2325 let mut history = Vec::new();
2326
2327 loop {
2328 attempt = attempt.saturating_add(1);
2329 let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2330 Ok(stream) => return Ok(stream),
2331 Err(error) => error,
2332 };
2333 let class = classify_attach_error(&error);
2334 let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2335 history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2336
2337 if class == AttachErrorClass::Permanent {
2338 log_attach_final_failure(started_at.elapsed(), &history);
2339 return Err(error);
2340 }
2341
2342 let remaining = deadline.saturating_duration_since(Instant::now());
2343 if remaining.is_zero() {
2344 log_attach_final_failure(started_at.elapsed(), &history);
2345 return Err(error);
2346 }
2347
2348 let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2349 log::info!(
2350 "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2351 );
2352 tokio::time::sleep(delay).await;
2353
2354 if Instant::now() >= deadline {
2355 log_attach_final_failure(started_at.elapsed(), &history);
2356 return Err(error);
2357 }
2358 backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2359 }
2360}
2361
2362fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2363 let jitter_percent = jitter_percent.min(100);
2364 if jitter_percent == 0 {
2365 return base;
2366 }
2367
2368 let mut random_bytes = [0_u8; 8];
2369 let random = if getrandom::fill(&mut random_bytes).is_ok() {
2370 u64::from_le_bytes(random_bytes)
2371 } else {
2372 let timestamp = std::time::SystemTime::now()
2373 .duration_since(std::time::UNIX_EPOCH)
2374 .unwrap_or_default()
2375 .subsec_nanos();
2376 u64::from(timestamp) ^ u64::from(attempt)
2377 };
2378 let span = jitter_percent.saturating_mul(2).saturating_add(1);
2379 let multiplier_percent = 100 - jitter_percent + random % span;
2380 let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2381 Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2382}
2383
2384fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2385 log::error!(
2386 "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2387 history.len(),
2388 history.join(" | ")
2389 );
2390}
2391
2392async fn connect_and_authenticate_once(
2393 connection_file_path: &Path,
2394 deadline: Instant,
2395) -> Result<TcpStream, SubcError> {
2396 let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2399 SubcError::ConnectionFile {
2400 path: connection_file_path.to_path_buf(),
2401 source,
2402 }
2403 })?;
2404
2405 let endpoint = conn
2406 .endpoints
2407 .first()
2408 .ok_or_else(|| SubcError::NoEndpoint {
2409 path: connection_file_path.to_path_buf(),
2410 })?;
2411 let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2412 let ip = endpoint
2413 .host
2414 .parse::<IpAddr>()
2415 .map_err(|_| SubcError::InvalidEndpoint {
2416 path: connection_file_path.to_path_buf(),
2417 endpoint: endpoint_label.clone(),
2418 })?;
2419 let addr = SocketAddr::new(ip, endpoint.port);
2420
2421 let connect_budget = deadline.saturating_duration_since(Instant::now());
2422 let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2423 .await
2424 .map_err(|_| SubcError::Connect {
2425 endpoint: endpoint_label.clone(),
2426 source: io::Error::new(
2427 io::ErrorKind::TimedOut,
2428 "initial subc attach retry budget elapsed during TCP connect",
2429 ),
2430 })?
2431 .map_err(|source| SubcError::Connect {
2432 endpoint: endpoint_label.clone(),
2433 source,
2434 })?;
2435 stream
2436 .set_nodelay(true)
2437 .map_err(|source| SubcError::Connect {
2438 endpoint: endpoint_label.clone(),
2439 source,
2440 })?;
2441
2442 let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2443 authenticate_client(&mut stream, &conn, auth_budget)
2444 .await
2445 .map_err(|source| SubcError::Auth {
2446 endpoint: endpoint_label,
2447 source,
2448 })?;
2449
2450 Ok(stream)
2451}
2452
2453#[allow(clippy::too_many_arguments)]
2454async fn process_route_bind_completion(
2455 writer_tx: &WriterSender,
2456 completion: RouteBindCompletion,
2457 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2458 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2459 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2460 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2461 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2462 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2463 installed_route_epochs: &mut HashMap<u16, u32>,
2464 executor: &Arc<Executor>,
2465 shutdown: &Arc<Notify>,
2466 metrics: &Arc<DispatchPathMetrics>,
2467) -> Result<(), SubcError> {
2468 decrement_counted_channel(&metrics.control_completion_queued);
2469 handle_route_bind_completion(
2470 writer_tx,
2471 completion,
2472 routes,
2473 root_channels,
2474 session_identity,
2475 push_buffer,
2476 live_roots,
2477 pending_binds,
2478 installed_route_epochs,
2479 executor,
2480 shutdown,
2481 metrics,
2482 )
2483 .await
2484}
2485
2486#[allow(clippy::too_many_arguments)]
2487async fn drain_pending_route_bind_completions(
2488 control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2489 writer_tx: &WriterSender,
2490 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2491 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2492 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2493 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2494 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2495 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2496 installed_route_epochs: &mut HashMap<u16, u32>,
2497 executor: &Arc<Executor>,
2498 shutdown: &Arc<Notify>,
2499 metrics: &Arc<DispatchPathMetrics>,
2500) -> Result<usize, SubcError> {
2501 let mut drained = 0;
2502 while let Ok(completion) = control_completion_rx.try_recv() {
2503 process_route_bind_completion(
2504 writer_tx,
2505 completion,
2506 routes,
2507 root_channels,
2508 session_identity,
2509 push_buffer,
2510 live_roots,
2511 pending_binds,
2512 installed_route_epochs,
2513 executor,
2514 shutdown,
2515 metrics,
2516 )
2517 .await?;
2518 drained += 1;
2519 }
2520 Ok(drained)
2521}
2522
2523async fn run_module_loop<R, W>(
2527 mut read: R,
2528 mut write: W,
2529 connection_file_path: &Path,
2530 shared_app: Arc<App>,
2531 executor: Arc<Executor>,
2532 dispatch: DispatchFn,
2533 user_config_path: Option<PathBuf>,
2534 allow_native_passthrough: bool,
2535 tool_response_body_limit: usize,
2536) -> Result<ModuleLoopExit, SubcError>
2537where
2538 R: AsyncRead + Unpin + Send + 'static,
2539 W: AsyncWrite + Unpin + Send + 'static,
2540{
2541 let hello = ModuleHelloBody {
2545 manifest: build_manifest(),
2546 protocol_ver: PROTOCOL_VERSION,
2547 control_ops: control_ops(),
2548 launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2549 };
2550 let hello_frame = Frame::build(
2551 FrameType::Hello,
2552 control_flags(),
2553 0,
2554 0,
2555 HELLO_CORR,
2556 serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2557 )
2558 .map_err(SubcError::FrameBuild)?;
2559 write_frame(&mut write, &hello_frame)
2560 .await
2561 .map_err(SubcError::FrameIo)?;
2562
2563 match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2565 None => return Err(SubcError::ClosedBeforeHelloAck),
2566 Some(frame) => match frame.header.ty {
2567 FrameType::HelloAck => {
2568 log::info!("subc attach: registered (HelloAck received)");
2569 }
2570 FrameType::Error => {
2571 let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2572 return Err(SubcError::HelloRejected { body });
2573 }
2574 other => return Err(SubcError::UnexpectedFrame { ty: other }),
2575 },
2576 }
2577
2578 let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2579 let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2580 let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2581 let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2588 let reader_task = spawn_reader_task(read, reader_tx);
2589 let shutdown = Arc::new(Notify::new());
2590 let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2599 let mut next_maintenance_at = next_drain_at;
2600 #[cfg(any(target_os = "macos", target_os = "linux"))]
2603 let mut last_slack_relief: Option<std::time::Instant> = None;
2604 let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2605 let (bash_deferred_tx, mut bash_deferred_rx) =
2606 mpsc::channel::<bash::BashDeferredCompletion>(256);
2607 let (inspect_deferred_tx, mut inspect_deferred_rx) =
2608 mpsc::unbounded_channel::<PendingSubcInspect>();
2609 let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2610 let (control_completion_tx, mut control_completion_rx) =
2611 mpsc::channel::<RouteBindCompletion>(256);
2612 let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2613 let lossy_overflow = Arc::new(push::LossyOverflow::default());
2614 let lossy_seq = Arc::new(AtomicU64::new(0));
2615 let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2616 let (fleet_status_client, fleet_status_task) =
2617 spawn_fleet_status_dial(connection_file_path, 64);
2618 let push_senders = PushSenders {
2619 lossy_tx,
2620 reliable_tx,
2621 lossy_overflow: Arc::clone(&lossy_overflow),
2622 lossy_seq,
2623 fleet_status_client: fleet_status_client.clone(),
2624 };
2625 let connection_cancel = PersistentCancelSignal::new();
2626 let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2627 let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2628 let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2629 let mut bg_sub_by_session: BgSubsBySession = HashMap::new();
2630 let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2631 let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2632 let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2633 let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2634 HashMap::new();
2635 let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2636 let mut retry_buffer: RetryBuffer = HashMap::new();
2637 let mut reclaimed_routes = ReclaimedRoutes::default();
2638 let mut completed_tasks = push::CompletedTaskIds::default();
2639 let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2640 let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2641 let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2642 let mut next_bash_ask_corr: u64 = 1;
2643 let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2644 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
2645 let pending_inspect_setups = Arc::new(AtomicUsize::new(0));
2646 let mut pending_inspects = PendingSubcInspects::default();
2647 let health_rollup_cache = HealthRollupCache::new();
2648 health_rollup_cache.refresh(&executor, &shared_app);
2649 let mut next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2650
2651 let loop_result: Result<ModuleLoopExit, SubcError> = 'module_loop: loop {
2652 shared_app.set_open_route_count(routes.len());
2653 if tokio::time::Instant::now() >= next_health_rollup_at {
2654 health_rollup_cache.refresh(&executor, &shared_app);
2655 next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2656 }
2657 crate::logging::perf_tick(Some(&executor));
2658 dispatch_path_metrics.mark_frame_loop_tick();
2659 let ready_inspects = pending_inspects.poll_ready(executor.as_ref());
2660 for resolved in ready_inspects {
2661 if let Err(error) = deliver_resolved_subc_inspect(
2662 &writer_tx,
2663 resolved,
2664 &routes,
2665 &mut live_roots,
2666 executor.as_ref(),
2667 &active_tool_calls,
2668 &shutdown,
2669 &dispatch_path_metrics,
2670 tool_response_body_limit,
2671 )
2672 .await
2673 {
2674 break 'module_loop Err(error);
2675 }
2676 }
2677 if let Err(error) = expire_pending_bash_asks(
2678 &writer_tx,
2679 &mut pending_bash_asks,
2680 &routes,
2681 &mut live_roots,
2682 &mut route_bash_cancels,
2683 &shutdown,
2684 &dispatch_path_metrics,
2685 )
2686 .await
2687 {
2688 break Err(error);
2689 }
2690
2691 match drain_pending_route_bind_completions(
2695 &mut control_completion_rx,
2696 &writer_tx,
2697 &mut routes,
2698 &mut root_channels,
2699 &mut session_identity,
2700 &mut push_buffer,
2701 &mut live_roots,
2702 &mut pending_binds,
2703 &mut installed_route_epochs,
2704 &executor,
2705 &shutdown,
2706 &dispatch_path_metrics,
2707 )
2708 .await
2709 {
2710 Ok(drained) => {
2711 if drained > 0 {
2712 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2713 health_rollup_cache.refresh(&executor, &shared_app);
2714 next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2715 }
2716 }
2717 Err(error) => break Err(error),
2718 }
2719
2720 if tokio::time::Instant::now() >= next_drain_at {
2721 push::emit_bg_event_wakes(
2722 &writer_tx,
2723 &dispatch_path_metrics,
2724 &bg_subs,
2725 &mut bg_wake_pending,
2726 );
2727 warn_slow_pending_binds(&mut pending_binds, &executor);
2728 if let Err(error) = expire_overdue_route_binds(
2729 &writer_tx,
2730 &executor,
2731 &mut pending_binds,
2732 &mut installed_route_epochs,
2733 &dispatch_path_metrics,
2734 )
2735 .await
2736 {
2737 break Err(error);
2738 }
2739
2740 let retried = push::drain_retry_buffers_for_bound_routes(
2741 &writer_tx,
2742 &dispatch_path_metrics,
2743 &routes,
2744 &mut retry_buffer,
2745 );
2746 if retried > 0 {
2747 log::debug!(
2748 "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2749 );
2750 }
2751
2752 next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2753 }
2754
2755 let overflow_batch = lossy_overflow.drain();
2761 if !overflow_batch.is_empty() {
2762 let (_, deferred) = push::drain_reliable_push_turn(
2763 &writer_tx,
2764 &dispatch_path_metrics,
2765 &routes,
2766 &root_channels,
2767 &session_identity,
2768 &mut retry_buffer,
2769 &mut push_buffer,
2770 &mut completed_tasks,
2771 &bg_sub_by_session,
2772 &mut bg_wake_pending,
2773 &mut bg_wake_epoch,
2774 &mut reliable_rx,
2775 None,
2776 );
2777 if deferred {
2778 tokio::task::yield_now().await;
2779 }
2780
2781 let mut batch = Vec::new();
2782 while let Ok(item) = lossy_rx.try_recv() {
2783 batch.push(item);
2784 }
2785 batch.extend(overflow_batch);
2786 push::process_lossy_push_envelope_batch(
2787 &writer_tx,
2788 &dispatch_path_metrics,
2789 &routes,
2790 &root_channels,
2791 &completed_tasks,
2792 batch,
2793 );
2794 }
2795
2796 tokio::select! {
2797 biased;
2798 Some(completion) = control_completion_rx.recv() => {
2799 if let Err(error) = process_route_bind_completion(
2800 &writer_tx,
2801 completion,
2802 &mut routes,
2803 &mut root_channels,
2804 &mut session_identity,
2805 &mut push_buffer,
2806 &mut live_roots,
2807 &mut pending_binds,
2808 &mut installed_route_epochs,
2809 &executor,
2810 &shutdown,
2811 &dispatch_path_metrics,
2812 )
2813 .await
2814 {
2815 break Err(error);
2816 }
2817 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2818 next_health_rollup_at = tokio::time::Instant::now();
2819 }
2820 _ = shutdown.notified() => {
2821 log::warn!("subc attach: fatal executor response requested teardown");
2822 break Ok(ModuleLoopExit::SkipSearchFlush);
2823 }
2824 maybe_frame = reader_rx.recv() => {
2825 let frame = match maybe_frame {
2826 None => {
2827 log::info!("subc attach: daemon closed connection");
2828 break Ok(ModuleLoopExit::Graceful);
2829 }
2830 Some(Err(error)) => break Err(error),
2831 Some(Ok(frame)) => frame,
2832 };
2833 let phase_trace = frame.phase_trace;
2834 let frame = frame.frame;
2835
2836 if !ingress_route_should_be_processed(
2837 &installed_route_epochs,
2838 &reclaimed_routes,
2839 &frame,
2840 ) {
2841 log::debug!(
2842 "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2843 frame.header.ty,
2844 frame.header.channel,
2845 frame.header.epoch
2846 );
2847 continue;
2848 }
2849
2850 match frame.header.ty {
2851 FrameType::Ping if frame.header.channel == 0 => {
2852 let pong = match Frame::build_with_version(
2853 frame.header.ver,
2854 FrameType::Pong,
2855 frame.header.flags,
2856 0,
2857 0,
2858 frame.header.corr,
2859 Vec::new(),
2860 ) {
2861 Ok(pong) => pong,
2862 Err(error) => break Err(SubcError::FrameBuild(error)),
2863 };
2864 if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2865 break Err(error);
2866 }
2867 }
2868 FrameType::Goodbye if frame.header.channel == 0 => {
2869 log::info!("subc attach: received channel-0 Goodbye");
2870 break Ok(ModuleLoopExit::Graceful);
2871 }
2872 FrameType::Goodbye => {
2873 let channel = route_key(frame.header.channel, frame.header.epoch);
2874 if let Err(error) = teardown_installed_route(
2875 &writer_tx,
2876 &dispatch_path_metrics,
2877 &executor,
2878 channel,
2879 "Goodbye",
2880 None,
2881 &mut installed_route_epochs,
2882 &mut routes,
2883 &mut root_channels,
2884 &mut bg_subs,
2885 &mut bg_sub_by_session,
2886 &mut bg_wake_pending,
2887 &mut pending_bash_asks,
2888 &mut live_roots,
2889 &mut route_bash_cancels,
2890 &active_tool_calls,
2891 &mut pending_inspects,
2892 &mut pending_binds,
2893 &mut retry_buffer,
2894 &mut push_buffer,
2895 &shutdown,
2896 tool_response_body_limit,
2897 )
2898 .await
2899 {
2900 break Err(error);
2901 }
2902 }
2903 FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2904 if let Err(error) = handle_bash_elicitation_reply(
2905 &writer_tx,
2906 &frame,
2907 &mut pending_bash_asks,
2908 &routes,
2909 &mut live_roots,
2910 &executor,
2911 &shutdown,
2912 &bash_deferred_tx,
2913 &bash_poll_touch_tx,
2914 &dispatch_path_metrics,
2915 &mut route_bash_cancels,
2916 dispatch,
2917 )
2918 .await
2919 {
2920 break Err(error);
2921 }
2922 }
2923 FrameType::Request if frame.header.channel == 0 => {
2924 if let Err(error) = handle_control_request(
2925 &writer_tx,
2926 &frame,
2927 &shared_app,
2928 &executor,
2929 &mut live_roots,
2930 &mut pending_binds,
2931 &mut installed_route_epochs,
2932 &mut routes,
2933 &mut root_channels,
2934 &mut bg_subs,
2935 &mut bg_sub_by_session,
2936 &mut bg_wake_pending,
2937 &mut pending_bash_asks,
2938 &mut route_bash_cancels,
2939 &active_tool_calls,
2940 &mut pending_inspects,
2941 &mut retry_buffer,
2942 &mut push_buffer,
2943 &shutdown,
2944 &control_completion_tx,
2945 &dispatch_path_metrics,
2946 &health_rollup_cache,
2947 &push_senders,
2948 dispatch,
2949 user_config_path.as_deref(),
2950 tool_response_body_limit,
2951 )
2952 .await
2953 {
2954 break Err(error);
2955 }
2956 }
2957 FrameType::Request => {
2958 if let Err(error) = handle_tool_call(
2959 &writer_tx,
2960 &frame,
2961 phase_trace,
2962 &routes,
2963 &pending_binds,
2964 &mut live_roots,
2965 &executor,
2966 &active_tool_calls,
2967 &pending_inspect_setups,
2968 &shutdown,
2969 &connection_cancel,
2970 &bash_deferred_tx,
2971 &bash_poll_touch_tx,
2972 &dispatch_path_metrics,
2973 &mut route_bash_cancels,
2974 &mut pending_bash_asks,
2975 &mut next_bash_ask_corr,
2976 &mut bg_subs,
2977 &mut bg_sub_by_session,
2978 &mut bg_wake_pending,
2979 &mut bg_wake_epoch,
2980 dispatch,
2981 &inspect_deferred_tx,
2982 allow_native_passthrough,
2983 tool_response_body_limit,
2984 )
2985 .await
2986 {
2987 break Err(error);
2988 }
2989 }
2990 FrameType::Cancel => {
2991 let channel = route_key(frame.header.channel, frame.header.epoch);
2992 cancel_active_tool_call(
2993 &active_tool_calls,
2994 executor.as_ref(),
2995 channel,
2996 frame.header.corr,
2997 "Cancel frame",
2998 );
2999 if bg_subs.contains_key(&channel) {
3000 if let Err(error) = end_bg_subscription(
3001 &writer_tx,
3002 &dispatch_path_metrics,
3003 &mut bg_subs,
3004 &mut bg_sub_by_session,
3005 &mut bg_wake_pending,
3006 channel,
3007 routes.get(&channel),
3008 "cancel",
3009 )
3010 .await
3011 {
3012 break Err(error);
3013 }
3014 }
3015 if let Err(error) = cancel_pending_bash_ask_for_tool_call(
3016 &writer_tx,
3017 &mut pending_bash_asks,
3018 channel,
3019 frame.header.corr,
3020 &routes,
3021 &mut live_roots,
3022 &mut route_bash_cancels,
3023 &shutdown,
3024 &dispatch_path_metrics,
3025 )
3026 .await
3027 {
3028 break Err(error);
3029 }
3030 }
3031 _ => {}
3035 }
3036 }
3037 Some(pending) = inspect_deferred_rx.recv() => {
3038 if routes.contains_key(&pending.route) {
3039 pending_inspects.register(pending);
3040 } else {
3041 if let Some(cancellation) = &pending.pending.cancellation {
3042 cancellation.request_cancel();
3043 }
3044 finish_active_tool_call(&active_tool_calls, pending.route, pending.corr);
3045 }
3046 }
3047 Some((root_id, frame)) = reliable_rx.recv() => {
3048 let (_, deferred) = push::drain_reliable_push_turn(
3052 &writer_tx,
3053 &dispatch_path_metrics,
3054 &routes,
3055 &root_channels,
3056 &session_identity,
3057 &mut retry_buffer,
3058 &mut push_buffer,
3059 &mut completed_tasks,
3060 &bg_sub_by_session,
3061 &mut bg_wake_pending,
3062 &mut bg_wake_epoch,
3063 &mut reliable_rx,
3064 Some((root_id, frame)),
3065 );
3066 if deferred {
3067 tokio::task::yield_now().await;
3068 }
3069 }
3070 Some((order, root_id, frame)) = lossy_rx.recv() => {
3071 let (_, deferred) = push::drain_reliable_push_turn(
3075 &writer_tx,
3076 &dispatch_path_metrics,
3077 &routes,
3078 &root_channels,
3079 &session_identity,
3080 &mut retry_buffer,
3081 &mut push_buffer,
3082 &mut completed_tasks,
3083 &bg_sub_by_session,
3084 &mut bg_wake_pending,
3085 &mut bg_wake_epoch,
3086 &mut reliable_rx,
3087 None,
3088 );
3089 if deferred {
3090 tokio::task::yield_now().await;
3091 }
3092
3093 let mut batch = vec![(order, root_id, frame)];
3100 while let Ok(item) = lossy_rx.try_recv() {
3101 batch.push(item);
3102 }
3103 batch.extend(lossy_overflow.drain());
3104 push::process_lossy_push_envelope_batch(
3105 &writer_tx,
3106 &dispatch_path_metrics,
3107 &routes,
3108 &root_channels,
3109 &completed_tasks,
3110 batch,
3111 );
3112 }
3113 Some(done) = bash_deferred_rx.recv() => {
3114 decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
3115 if let Err(error) = bash::handle_bash_deferred_completion(
3116 &writer_tx,
3117 done,
3118 &routes,
3119 &mut live_roots,
3120 &mut route_bash_cancels,
3121 &shutdown,
3122 &dispatch_path_metrics,
3123 )
3124 .await
3125 {
3126 break Err(error);
3127 }
3128 }
3129 Some(root_id) = bash_poll_touch_rx.recv() => {
3130 decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
3131 if let Some(meta) = live_roots.get_mut(&root_id) {
3132 meta.note_activity();
3133 }
3134 }
3135 Some(completion) = maintenance_rx.recv() => {
3136 decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
3137 let root_id = completion.root_id.clone();
3138 let response = completion.response;
3139 let response_is_fatal = response_is_fatal_panic(&response);
3140 let bind_pending = pending_binds
3141 .values()
3142 .any(|pending| pending.bind_root_id == root_id);
3143 let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
3144 let defer_requeue = meta.unbound_quiesced || bind_pending;
3145 note_maintenance_completion(
3146 meta,
3147 completion.requeue_kind,
3148 response_is_fatal,
3149 defer_requeue,
3150 );
3151 should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
3152 } else {
3153 false
3154 };
3155 if requiesce {
3156 quiesce_unbound_root(&root_id, &mut live_roots, &executor);
3157 }
3158 push::clear_stale_bg_wakes_for_empty_sessions(
3159 &root_id,
3160 &completion.empty_bg_sessions,
3161 &bg_sub_by_session,
3162 &mut bg_wake_pending,
3163 &bg_wake_epoch,
3164 );
3165 if response_is_fatal {
3166 if let Some(meta) = live_roots.get_mut(&root_id) {
3167 meta.maintenance_poisoned = true;
3168 }
3169 log::warn!(
3170 "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
3171 );
3172 }
3173 }
3174 _ = tokio::time::sleep(PENDING_POLL_INTERVAL), if !pending_inspects.is_empty() => {
3175 }
3178 _ = tokio::time::sleep_until(next_drain_at) => {
3179 }
3182 _ = tokio::time::sleep_until(next_maintenance_at) => {
3183 crate::logging::maybe_sweep_logs();
3188 let reaped_lsp_children = shared_app
3189 .lsp_child_registry()
3190 .reap_children_with_gone_cwd_or_reclaimed_root();
3191 if reaped_lsp_children > 0 {
3192 log::warn!(
3193 "subc attach: reaped {reaped_lsp_children} LSP child process group(s) with a deleted cwd or reclaimed root"
3194 );
3195 }
3196 let reap = reap_idle_roots(
3197 Instant::now(),
3198 &mut live_roots,
3199 &pending_binds,
3200 &root_channels,
3201 &executor,
3202 &dispatch_path_metrics,
3203 );
3204 for root_id in &reap.forgotten_deleted_roots {
3205 purge_deleted_root_residents(
3206 root_id,
3207 &mut routes,
3208 &mut root_channels,
3209 &mut installed_route_epochs,
3210 &mut route_bash_cancels,
3211 &active_tool_calls,
3212 executor.as_ref(),
3213 &mut retry_buffer,
3214 &mut reclaimed_routes,
3215 &mut session_identity,
3216 &mut push_buffer,
3217 &mut bg_subs,
3218 &mut bg_sub_by_session,
3219 &mut bg_wake_pending,
3220 &mut bg_wake_epoch,
3221 &mut pending_bash_asks,
3222 &dispatch_path_metrics,
3223 );
3224 }
3225 if reap.evicted > 0 {
3226 log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
3227 }
3228 submit_due_maintenance_jobs(
3229 &executor,
3230 &mut live_roots,
3231 &pending_binds,
3232 &bg_sub_by_session,
3233 &bg_wake_pending,
3234 &bg_wake_epoch,
3235 &maintenance_tx,
3236 &dispatch_path_metrics,
3237 );
3238 #[cfg(any(target_os = "macos", target_os = "linux"))]
3245 {
3246 let now_std = std::time::Instant::now();
3247 if crate::memory::spawn_allocator_slack_relief_if_due(
3248 last_slack_relief,
3249 now_std,
3250 ) {
3251 last_slack_relief = Some(now_std);
3252 }
3253 }
3254 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3255 }
3256 }
3257 };
3258
3259 shared_app.set_open_route_count(0);
3260
3261 connection_cancel.cancel();
3262 cancel_all_active_tool_calls(&active_tool_calls, executor.as_ref(), "connection teardown");
3263 let setup_drain_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
3264 while pending_inspect_setups.load(Ordering::SeqCst) != 0
3265 && tokio::time::Instant::now() < setup_drain_deadline
3266 {
3267 tokio::select! {
3268 biased;
3269 Some(pending) = inspect_deferred_rx.recv() => pending_inspects.register(pending),
3270 _ = tokio::time::sleep(Duration::from_millis(5)) => {}
3271 }
3272 }
3273 if pending_inspect_setups.load(Ordering::SeqCst) != 0 {
3274 log::warn!(
3275 "subc attach: timed out waiting for deferred inspect setup registration during shutdown"
3276 );
3277 }
3278 while let Ok(pending) = inspect_deferred_rx.try_recv() {
3279 pending_inspects.register(pending);
3280 }
3281 for resolved in pending_inspects.drain_on_shutdown(executor.as_ref()) {
3282 if let Err(error) = deliver_resolved_subc_inspect(
3283 &writer_tx,
3284 resolved,
3285 &routes,
3286 &mut live_roots,
3287 executor.as_ref(),
3288 &active_tool_calls,
3289 &shutdown,
3290 &dispatch_path_metrics,
3291 tool_response_body_limit,
3292 )
3293 .await
3294 {
3295 log::warn!("subc attach: failed to emit inspect shutdown terminal: {error}");
3296 }
3297 }
3298 quiesce_connection_roots(
3301 &mut live_roots,
3302 &mut pending_binds,
3303 &mut routes,
3304 &mut root_channels,
3305 &mut installed_route_epochs,
3306 &mut route_bash_cancels,
3307 &active_tool_calls,
3308 &executor,
3309 );
3310
3311 fleet_status_client.set_route_live(false);
3312 fleet_status_task.abort();
3313 let _ = fleet_status_task.await;
3314
3315 let mut loop_result = loop_result;
3316 if !pending_bash_asks.is_empty() {
3317 let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3318 if let Err(error) = settle_all_pending_bash_asks(
3319 &writer_tx,
3320 &mut pending_bash_asks,
3321 &no_routes,
3322 &mut live_roots,
3323 &mut route_bash_cancels,
3324 &shutdown,
3325 &dispatch_path_metrics,
3326 )
3327 .await
3328 {
3329 loop_result = loop_result.and(Err(error));
3330 }
3331 }
3332
3333 reader_task.abort();
3336 drop(writer_tx);
3337 let writer_result = finish_writer_task(writer_task).await;
3338 loop_result.and_then(|exit| writer_result.map(|_| exit))
3339}
3340
3341fn spawn_writer_task<W>(
3342 mut write: W,
3343 mut rx: mpsc::Receiver<WriterFrame>,
3344 metrics: Arc<DispatchPathMetrics>,
3345) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
3346where
3347 W: AsyncWrite + Unpin + Send + 'static,
3348{
3349 tokio::spawn(async move {
3350 let mut write_buffer = Vec::new();
3351 while let Some(mut queued) = rx.recv().await {
3352 let measure = queued.tool_response_trace.is_some();
3353 let dequeued = measure.then(Instant::now);
3354 metrics.writer_active.store(true, Ordering::Relaxed);
3355 decrement_counted_channel(&metrics.writer_queued);
3356 let write_timing = write_frame_contiguous(
3357 &mut write,
3358 queued.frame(),
3359 queued.body(),
3360 &mut write_buffer,
3361 measure,
3362 )
3363 .await;
3364 metrics.writer_active.store(false, Ordering::Relaxed);
3365 let write_timing = write_timing?;
3366
3367 if let (Some(trace), Some(dequeued), Some(write_timing)) =
3368 (queued.tool_response_trace.take(), dequeued, write_timing)
3369 {
3370 if let Some(completed) = trace.finish(
3371 dequeued,
3372 write_timing.write_started,
3373 write_timing.write_finished,
3374 write_timing.frame_bytes,
3375 ) {
3376 log_ctx::with_session(Some(completed.session), || {
3377 crate::logging::note_tool_call_trace(
3378 &completed.name,
3379 &completed.root,
3380 completed.channel,
3381 completed.corr,
3382 completed.phases,
3383 );
3384 });
3385 }
3386 }
3387 }
3388 Ok(())
3389 })
3390}
3391
3392struct FrameWriteTiming {
3393 write_started: Instant,
3394 write_finished: Instant,
3395 frame_bytes: usize,
3396}
3397
3398async fn write_frame_contiguous<W>(
3402 writer: &mut W,
3403 frame: &Frame,
3404 body: &[u8],
3405 buffer: &mut Vec<u8>,
3406 measure: bool,
3407) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
3408where
3409 W: AsyncWrite + Unpin,
3410{
3411 if frame.header.len as usize != body.len() {
3412 return Err(subc_transport::FrameIoError::BodyLengthMismatch {
3413 header_len: frame.header.len,
3414 body_len: body.len(),
3415 });
3416 }
3417
3418 let header = frame.header.encode();
3419 buffer.clear();
3420 buffer.reserve(header.len() + body.len());
3421 buffer.extend_from_slice(&header);
3422 buffer.extend_from_slice(body);
3423 let write_started = measure.then(Instant::now);
3424 writer
3425 .write_all(buffer)
3426 .await
3427 .map_err(subc_transport::FrameIoError::Io)?;
3428 Ok(write_started.map(|write_started| FrameWriteTiming {
3429 write_started,
3430 write_finished: Instant::now(),
3431 frame_bytes: buffer.len(),
3432 }))
3433}
3434
3435fn spawn_reader_task<R>(
3436 mut read: R,
3437 tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3438) -> JoinHandle<()>
3439where
3440 R: AsyncRead + Unpin + Send + 'static,
3441{
3442 tokio::spawn(async move {
3443 loop {
3444 match read_frame(&mut read).await {
3445 Ok(Some(frame)) => {
3446 let decoded = DecodedFrame {
3447 frame,
3448 phase_trace: PhaseTrace::new(Instant::now()),
3449 };
3450 if tx.send(Ok(decoded)).await.is_err() {
3451 return;
3452 }
3453 }
3454 Ok(None) => {
3455 return;
3457 }
3458 Err(error) => {
3459 if let subc_transport::FrameIoError::Io(io_error) = &error {
3466 if matches!(
3467 io_error.kind(),
3468 std::io::ErrorKind::ConnectionReset
3469 | std::io::ErrorKind::ConnectionAborted
3470 ) {
3471 log::info!(
3472 "subc attach: connection reset by daemon; treating as close"
3473 );
3474 return;
3475 }
3476 }
3477 let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3478 return;
3479 }
3480 }
3481 }
3482 })
3483}
3484
3485async fn finish_writer_task(
3486 mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3487) -> Result<(), SubcError> {
3488 match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3489 Ok(Ok(Ok(()))) => Ok(()),
3490 Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3491 Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3492 Err(_) => {
3493 writer_task.abort();
3494 Ok(())
3495 }
3496 }
3497}
3498
3499fn register_actor_for_bind(
3500 shared_app: &Arc<App>,
3501 executor: &Arc<Executor>,
3502 push_senders: &PushSenders,
3503 bind_root_id: &ProjectRootId,
3504 route_channel: u16,
3505 root_was_live: bool,
3506) -> bool {
3507 if executor.actor_registered(bind_root_id) {
3508 log::debug!(
3509 "subc attach: reusing actor for route {} root {}",
3510 route_channel,
3511 bind_root_id.as_path().display()
3512 );
3513 return false;
3514 }
3515
3516 if root_was_live {
3517 log::warn!(
3518 "subc attach: recreating missing actor for live root {} on route {}",
3519 bind_root_id.as_path().display(),
3520 route_channel
3521 );
3522 }
3523
3524 let actor_ctx = Arc::new(AppContext::from_app(
3525 Arc::clone(shared_app),
3526 Config::default(),
3527 ));
3528 install_bash_compressor(&actor_ctx);
3529 actor_ctx.install_fleet_status_client(Some(push_senders.fleet_status_client.clone()));
3530 actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
3531 push_senders.clone(),
3532 bind_root_id.clone(),
3533 )));
3534 let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
3535 drop(actor_ctx);
3536 if inserted {
3537 log::debug!(
3541 "subc attach: registered actor for route {} root {}",
3542 route_channel,
3543 bind_root_id.as_path().display()
3544 );
3545 } else {
3546 log::debug!(
3547 "subc attach: actor appeared while binding route {} root {}; reusing it",
3548 route_channel,
3549 bind_root_id.as_path().display()
3550 );
3551 }
3552 inserted
3553}
3554
3555fn rollback_pending_bind_actor(
3556 executor: &Arc<Executor>,
3557 live_roots: &HashMap<ProjectRootId, RootMeta>,
3558 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3559 root_id: &ProjectRootId,
3560 inserted_new_actor: bool,
3561) {
3562 if !inserted_new_actor || live_roots.contains_key(root_id) {
3563 return;
3564 }
3565
3566 if let Some((route, pending)) = pending_binds
3567 .iter_mut()
3568 .find(|(_, pending)| &pending.bind_root_id == root_id)
3569 {
3570 pending.inserted_new_actor = true;
3571 log::debug!(
3572 "subc attach: transferred rollback ownership for root {} to pending route {}",
3573 root_id.as_path().display(),
3574 route
3575 );
3576 return;
3577 }
3578
3579 executor.remove_actor(root_id);
3580}
3581
3582fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
3583 match response.data.get("code").and_then(|code| code.as_str()) {
3584 Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
3589 Some("cache_key_probe_failed") => "cache_key_probe_failed",
3593 Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
3597 _ => "config_divergence",
3598 }
3599}
3600
3601fn queue_post_bind_configure_and_completion_maintenance(
3602 root_id: &ProjectRootId,
3603 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3604) {
3605 let Some(meta) = live_roots.get_mut(root_id) else {
3606 return;
3607 };
3608 if meta.maintenance_poisoned || meta.maintenance_pending {
3609 return;
3610 }
3611
3612 meta.maintenance_pending = true;
3613 meta.maintenance_queued_kinds
3614 .push_back(MaintenanceDrainKind::ConfigureTail);
3615 meta.maintenance_queued_kinds
3616 .push_back(MaintenanceDrainKind::CompletionDrains);
3617}
3618
3619#[allow(clippy::too_many_arguments)]
3620async fn handle_route_bind_completion(
3621 tx: &WriterSender,
3622 completion: RouteBindCompletion,
3623 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3624 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3625 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
3626 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3627 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3628 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3629 installed_route_epochs: &mut HashMap<u16, u32>,
3630 executor: &Arc<Executor>,
3631 shutdown: &Arc<Notify>,
3632 metrics: &Arc<DispatchPathMetrics>,
3633) -> Result<(), SubcError> {
3634 let route_id = completion.route;
3635 let Some(pending) = pending_binds.remove(&route_id) else {
3636 log::warn!(
3637 "subc attach: dropping RouteBind completion for non-pending route {}",
3638 completion.route
3639 );
3640 rollback_pending_bind_actor(
3641 executor,
3642 live_roots,
3643 pending_binds,
3644 &completion.bind_root_id,
3645 completion.inserted_new_actor,
3646 );
3647 let has_pending_bind = pending_binds
3648 .values()
3649 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3650 if !root_channels
3651 .get(&completion.bind_root_id)
3652 .is_some_and(|channels| !channels.is_empty())
3653 && !has_pending_bind
3654 {
3655 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3656 }
3657 remove_installed_route(installed_route_epochs, route_id);
3658 return Ok(());
3659 };
3660
3661 if pending.bind_root_id != completion.bind_root_id {
3662 log::warn!(
3663 "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
3664 completion.route,
3665 pending.bind_root_id.as_path().display(),
3666 completion.bind_root_id.as_path().display()
3667 );
3668 }
3669
3670 let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
3671 if pending.cancelled {
3672 rollback_pending_bind_actor(
3673 executor,
3674 live_roots,
3675 pending_binds,
3676 &completion.bind_root_id,
3677 inserted_new_actor,
3678 );
3679 let has_pending_bind = pending_binds
3680 .values()
3681 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3682 if !root_channels
3683 .get(&completion.bind_root_id)
3684 .is_some_and(|channels| !channels.is_empty())
3685 && !has_pending_bind
3686 {
3687 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3688 }
3689 log::debug!(
3690 "subc attach: discarded completed RouteBind for cancelled route {} root {}",
3691 completion.route,
3692 completion.bind_root_id.as_path().display()
3693 );
3694 remove_installed_route(installed_route_epochs, route_id);
3695 return Ok(());
3696 }
3697
3698 let failure = if !completion.configure_response.success {
3699 Some((
3700 &completion.configure_response,
3701 "configure failed during route bind",
3702 ))
3703 } else {
3704 None
3705 };
3706
3707 if let Some((response, fallback)) = failure {
3708 rollback_pending_bind_actor(
3709 executor,
3710 live_roots,
3711 pending_binds,
3712 &completion.bind_root_id,
3713 inserted_new_actor,
3714 );
3715 let has_pending_bind = pending_binds
3716 .values()
3717 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3718 if !root_channels
3719 .get(&completion.bind_root_id)
3720 .is_some_and(|channels| !channels.is_empty())
3721 && !has_pending_bind
3722 {
3723 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3724 }
3725 let message = response_message(response, fallback);
3726 let fatal = response_is_fatal_panic(response);
3727 let error_code = route_bind_error_code_for_configure_response(response);
3728 send_route_bind_error_parts(
3729 tx,
3730 completion.ver,
3731 completion.corr,
3732 completion.flags,
3733 error_code,
3734 &message,
3735 metrics,
3736 )
3737 .await?;
3738 remove_installed_route(installed_route_epochs, route_id);
3739 if fatal {
3740 signal_fatal_teardown(
3741 tx,
3742 Some(completion.route),
3743 completion.ver,
3744 completion.corr,
3745 shutdown,
3746 metrics,
3747 )
3748 .await;
3749 }
3750 return Ok(());
3751 }
3752
3753 remember_session_identity(session_identity, &completion.identity);
3754 let replay_key = push::ReplayKey::from_identity(&completion.identity);
3755 let bind_trust = completion.identity.trust;
3756 insert_route_channel(routes, root_channels, route_id, completion.identity);
3757 let restore_watcher = live_roots
3758 .get(&completion.bind_root_id)
3759 .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3760 live_roots
3761 .entry(completion.bind_root_id.clone())
3762 .and_modify(|meta| {
3763 meta.reactivate_bound();
3764 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3765 meta.maintenance_poisoned = false;
3766 })
3767 .or_insert_with(|| RootMeta::new(Instant::now()));
3768 if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3769 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3770 meta.maintenance_poisoned = false;
3771 }
3772 if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3773 ctx.mark_subc_bound();
3774 if restore_watcher {
3775 crate::commands::configure::ensure_project_watcher(&ctx);
3776 }
3777 }
3778
3779 let ack =
3780 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3781 let response = Frame::build_with_version(
3782 completion.ver,
3783 FrameType::Response,
3784 control_flags(),
3785 0,
3786 0,
3787 completion.corr,
3788 ack,
3789 )
3790 .map_err(SubcError::FrameBuild)?;
3791 send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3792 queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3793 let replayed = push::replay_buffered_push_frames(
3794 tx,
3795 metrics,
3796 route_id,
3797 push_buffer,
3798 &replay_key,
3799 bind_trust,
3800 );
3801 if replayed > 0 {
3802 log::debug!(
3803 "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3804 replayed,
3805 completion.route,
3806 replay_key.root.as_path().display(),
3807 replay_key.harness,
3808 replay_key.session
3809 );
3810 }
3811 log::info!(
3812 "subc attach: route {} bound to root {}",
3813 completion.route,
3814 completion.bind_root_id.as_path().display()
3815 );
3816 Ok(())
3817}
3818
3819async fn expire_overdue_route_binds(
3820 tx: &WriterSender,
3821 executor: &Arc<Executor>,
3822 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3823 installed_route_epochs: &mut HashMap<u16, u32>,
3824 metrics: &DispatchPathMetrics,
3825) -> Result<(), SubcError> {
3826 let now = Instant::now();
3827 let expired: Vec<_> = pending_binds
3828 .iter()
3829 .filter_map(|(route, pending)| {
3830 let age = now.saturating_duration_since(pending.started_at);
3831 (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3832 (
3833 *route,
3834 pending.corr,
3835 pending.ver,
3836 pending.flags,
3837 pending.bind_root_id.clone(),
3838 pending.configure_request_id.clone(),
3839 age,
3840 )
3841 })
3842 })
3843 .collect();
3844
3845 for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3846 if let Some(pending) = pending_binds.get_mut(&route) {
3847 pending.cancelled = true;
3848 pending.deadline_reported = true;
3849 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3850 log::debug!(
3851 "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3852 );
3853 }
3854 remove_installed_route(installed_route_epochs, route);
3855 let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3856 let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3857 send_route_bind_error_parts(
3858 tx,
3859 ver,
3860 corr,
3861 flags,
3862 "actor_not_ready",
3863 &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3864 metrics,
3865 )
3866 .await?;
3867 log::warn!(
3868 "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3869 route,
3870 root_id.as_path().display(),
3871 deadline_ms,
3872 configure_request_id
3873 );
3874 }
3875
3876 Ok(())
3877}
3878
3879#[allow(clippy::too_many_arguments)]
3884async fn handle_control_request(
3885 tx: &WriterSender,
3886 frame: &Frame,
3887 shared_app: &Arc<App>,
3888 executor: &Arc<Executor>,
3889 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3890 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3891 installed_route_epochs: &mut HashMap<u16, u32>,
3892 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3893 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3894 bg_subs: &mut HashMap<RouteChannel, BgSub>,
3895 bg_sub_by_session: &mut BgSubsBySession,
3896 bg_wake_pending: &mut HashSet<RouteChannel>,
3897 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3898 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3899 active_tool_calls: &ActiveToolCalls,
3900 pending_inspects: &mut PendingSubcInspects,
3901 retry_buffer: &mut RetryBuffer,
3902 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3903 shutdown: &Arc<Notify>,
3904 control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3905 metrics: &Arc<DispatchPathMetrics>,
3906 health_rollup_cache: &HealthRollupCache,
3907 push_senders: &PushSenders,
3908 dispatch: DispatchFn,
3909 user_config_path: Option<&Path>,
3910 tool_response_body_limit: usize,
3911) -> Result<(), SubcError> {
3912 let request =
3913 serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3914 match request {
3915 ModuleControlRequest::RouteBind {
3916 route_channel,
3917 epoch,
3918 target: _,
3919 identity,
3920 principal,
3921 consumer_capabilities,
3922 admission_facts: _,
3923 } => {
3924 let route_id = route_key(route_channel, epoch);
3925 if epoch == 0 {
3926 return send_route_bind_error(
3927 tx,
3928 frame,
3929 "config_divergence",
3930 "route bind uses an invalid channel generation",
3931 metrics,
3932 )
3933 .await;
3934 }
3935 let mut bind_root_id = None;
3936 if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3937 if installed_epoch >= epoch {
3938 return send_route_bind_error(
3939 tx,
3940 frame,
3941 "config_divergence",
3942 "route bind generation is not newer than the installed generation",
3943 metrics,
3944 )
3945 .await;
3946 }
3947
3948 let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3949 Ok(root_id) => root_id,
3950 Err(error) => {
3951 return send_route_bind_error(
3952 tx,
3953 frame,
3954 "config_divergence",
3955 &format!("invalid route project root: {error}"),
3956 metrics,
3957 )
3958 .await;
3959 }
3960 };
3961 teardown_installed_route(
3962 tx,
3963 metrics,
3964 executor,
3965 route_key(route_channel, installed_epoch),
3966 "higher-epoch RouteBind",
3967 Some(&replacement_root),
3968 installed_route_epochs,
3969 routes,
3970 root_channels,
3971 bg_subs,
3972 bg_sub_by_session,
3973 bg_wake_pending,
3974 pending_bash_asks,
3975 live_roots,
3976 route_bash_cancels,
3977 active_tool_calls,
3978 pending_inspects,
3979 pending_binds,
3980 retry_buffer,
3981 push_buffer,
3982 shutdown,
3983 tool_response_body_limit,
3984 )
3985 .await?;
3986 bind_root_id = Some(replacement_root);
3987 }
3988 if pending_binds.contains_key(&route_id) {
3989 return send_route_bind_error(
3990 tx,
3991 frame,
3992 "config_divergence",
3993 "route bind is already pending for channel",
3994 metrics,
3995 )
3996 .await;
3997 }
3998 let bind_root_id = match bind_root_id {
3999 Some(root_id) => root_id,
4000 None => match ProjectRootId::from_path(&identity.project_root) {
4001 Ok(root_id) => root_id,
4002 Err(error) => {
4003 return send_route_bind_error(
4004 tx,
4005 frame,
4006 "config_divergence",
4007 &format!("invalid route project root: {error}"),
4008 metrics,
4009 )
4010 .await;
4011 }
4012 },
4013 };
4014
4015 let request_id = format!("subc-bind-{route_channel}");
4018 let bind_project_root = identity.project_root.clone();
4019 let bind_harness = identity.harness.clone();
4020 let bind_session = identity.session.clone();
4021 let bind_trust = trust_for_bind(&bind_harness, &principal);
4022 let bind_principal_id = principal_id(&principal);
4023 let consumer_elicitation_capable = consumer_capabilities
4028 .as_ref()
4029 .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
4030 log::info!(
4031 "subc attach: route {} harness={} principal={} trust={} elicitation={}",
4032 route_channel,
4033 bind_harness,
4034 principal_label(&principal),
4035 bind_trust.label(),
4036 consumer_elicitation_capable
4037 );
4038
4039 let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
4051 user_config_path,
4052 Path::new(&bind_project_root),
4053 );
4054 let config_tiers: Vec<Value> = local_tiers
4055 .iter()
4056 .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
4057 .collect();
4058 let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
4059 let configure_json = json!({
4060 "id": request_id,
4061 "command": "configure",
4062 "project_root": bind_project_root,
4063 "harness": bind_harness,
4064 "session_id": bind_session.clone(),
4065 "config": config_tiers,
4066 });
4067 let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
4068 Ok(req) => req,
4069 Err(error) => {
4070 return send_route_bind_error(
4071 tx,
4072 frame,
4073 "config_divergence",
4074 &format!("failed to build configure request: {error}"),
4075 metrics,
4076 )
4077 .await;
4078 }
4079 };
4080
4081 let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
4082 root: bind_root_id.clone(),
4083 project_root: PathBuf::from(&bind_project_root),
4084 harness: bind_harness.clone(),
4085 session: bind_session.clone(),
4086 trust: bind_trust,
4087 spawn_principal: AuthenticatedPrincipal::RouteBind {
4088 trust: bind_trust.sandbox_trust(),
4089 route_channel,
4090 route_epoch: epoch,
4091 project_root: PathBuf::from(&bind_project_root),
4092 harness: bind_harness.clone(),
4093 session_id: bind_session.clone(),
4094 principal_id: bind_principal_id,
4095 },
4096 consumer_elicitation_capable,
4097 }));
4098 let configure_session = route_identity.session.clone();
4099 let root_was_live = live_roots.contains_key(&bind_root_id);
4100 let inserted_new_actor = register_actor_for_bind(
4101 shared_app,
4102 executor,
4103 push_senders,
4104 &bind_root_id,
4105 route_channel,
4106 root_was_live,
4107 );
4108
4109 let configure_request_id = configure_req.id.clone();
4110 installed_route_epochs.insert(route_channel, epoch);
4111 if let Some(meta) = live_roots.get_mut(&bind_root_id) {
4112 meta.maintenance_queued_kinds.clear();
4113 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
4114 }
4115 let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
4116 bind_root_id.clone(),
4117 Lane::Mutating,
4118 configure_request_id.clone(),
4119 Box::new(move |ctx| {
4120 log_ctx::with_session(Some(configure_session.clone()), || {
4121 dispatch(configure_req, ctx)
4122 })
4123 }),
4124 );
4125 pending_binds.insert(
4126 route_id,
4127 PendingBind {
4128 bind_root_id: bind_root_id.clone(),
4129 inserted_new_actor,
4130 cancelled: false,
4131 configure_request_id: configure_request_id.clone(),
4132 started_at: Instant::now(),
4133 warned_half_deadline: false,
4134 deadline_reported: false,
4135 corr: frame.header.corr,
4136 ver: frame.header.ver,
4137 flags: frame.header.flags,
4138 cancellation: configure_cancellation,
4139 },
4140 );
4141
4142 let completion_tx = control_completion_tx.clone();
4143 let completion_identity = route_identity;
4144 let completion_root = bind_root_id.clone();
4145 let completion_route_channel = route_channel;
4146 let completion_ver = frame.header.ver;
4147 let completion_corr = frame.header.corr;
4148 let completion_flags = frame.header.flags;
4149 let completion_metrics = Arc::clone(metrics);
4150 tokio::spawn(async move {
4151 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4152 let configure_response =
4153 await_executor_response(configure_rx, configure_request_id.clone()).await;
4154 let completion = RouteBindCompletion {
4159 route: route_key(completion_route_channel, epoch),
4160 identity: completion_identity,
4161 bind_root_id: completion_root,
4162 inserted_new_actor,
4163 configure_response,
4164 diagnostics_on_edit,
4165 ver: completion_ver,
4166 corr: completion_corr,
4167 flags: completion_flags,
4168 };
4169 if send_counted_channel(
4170 &completion_tx,
4171 &completion_metrics.control_completion_queued,
4172 completion,
4173 )
4174 .await
4175 .is_err()
4176 {
4177 log::debug!(
4178 "subc attach: dropped RouteBind completion for route {} after loop exit",
4179 completion_route_channel
4180 );
4181 }
4182 });
4183
4184 health_rollup_cache.refresh(executor, shared_app);
4185 Ok(())
4186 }
4187 ModuleControlRequest::HealthCheck {} => {
4188 metrics.record_bg_runtime(bg_subs.len(), bg_wake_pending.len());
4189 let report = build_health_report(
4190 health_rollup_cache,
4191 executor,
4192 pending_binds,
4193 metrics,
4194 shared_app,
4195 );
4196 let body = serde_json::to_vec(&ModuleControlResponse::from(report))
4197 .map_err(SubcError::Json)?;
4198 let response = Frame::build_with_version(
4199 frame.header.ver,
4200 FrameType::Response,
4201 frame.header.flags,
4202 0,
4203 0,
4204 frame.header.corr,
4205 body,
4206 )
4207 .map_err(SubcError::FrameBuild)?;
4208 send_frame(tx, metrics, response).await
4209 }
4210 }
4211}
4212
4213fn install_bash_compressor(ctx: &AppContext) {
4214 let filter_registry_handle = ctx.shared_filter_registry();
4216 let compress_flag = ctx.bash_compress_flag();
4217 ctx.bash_background().set_compressor_with_exit_code(
4218 move |command: &str, output: String, exit_code: Option<i32>| {
4219 if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
4220 return crate::compress::CompressionResult::new(output);
4221 }
4222 let registry_guard = match filter_registry_handle.read() {
4223 Ok(g) => g,
4224 Err(poisoned) => poisoned.into_inner(),
4225 };
4226 crate::compress::compress_with_registry_exit_code(
4227 command,
4228 &output,
4229 exit_code,
4230 ®istry_guard,
4231 )
4232 },
4233 );
4234}
4235
4236fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
4237 let mut diagnostics_on_edit = false;
4238 for tier in tiers {
4239 if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
4240 diagnostics_on_edit = value;
4241 }
4242 }
4243 diagnostics_on_edit
4244}
4245
4246fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
4247 let stripped = strip_jsonc(doc);
4248 let value = serde_json::from_str::<Value>(&stripped).ok()?;
4249 value
4250 .get("lsp")
4251 .and_then(Value::as_object)?
4252 .get("diagnostics_on_edit")
4253 .and_then(Value::as_bool)
4254}
4255
4256async fn send_route_bind_error(
4257 tx: &WriterSender,
4258 frame: &Frame,
4259 code: &str,
4260 message: &str,
4261 metrics: &DispatchPathMetrics,
4262) -> Result<(), SubcError> {
4263 send_route_bind_error_parts(
4264 tx,
4265 frame.header.ver,
4266 frame.header.corr,
4267 frame.header.flags,
4268 code,
4269 message,
4270 metrics,
4271 )
4272 .await
4273}
4274
4275async fn send_route_bind_error_parts(
4276 tx: &WriterSender,
4277 ver: u8,
4278 corr: u64,
4279 flags: Flags,
4280 code: &str,
4281 message: &str,
4282 metrics: &DispatchPathMetrics,
4283) -> Result<(), SubcError> {
4284 let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
4285 send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
4286 log_route_bind_rejection(code, message);
4287 Ok(())
4288}
4289
4290fn log_route_bind_rejection(code: &str, message: &str) {
4298 const WINDOW: Duration = Duration::from_secs(60);
4299 static SUPPRESSED: OnceLock<StdMutex<HashMap<String, (Instant, u64)>>> = OnceLock::new();
4300 let map = SUPPRESSED.get_or_init(|| StdMutex::new(HashMap::new()));
4301 let mut map = match map.try_lock() {
4302 Ok(map) => map,
4303 Err(_) => {
4305 log::warn!("subc attach: route bind rejected ({code}): {message}");
4306 return;
4307 }
4308 };
4309 let now = Instant::now();
4310 if map.len() > 512 {
4314 map.retain(|_, (start, _)| now.duration_since(*start) < WINDOW);
4315 }
4316 match map.get_mut(message) {
4317 Some((window_start, suppressed)) if now.duration_since(*window_start) < WINDOW => {
4318 *suppressed += 1;
4319 }
4320 Some((window_start, suppressed)) => {
4321 if *suppressed > 0 {
4322 log::warn!(
4323 "subc attach: route bind rejected ({code}): {message} (repeated {}x in last 60s)",
4324 *suppressed
4325 );
4326 } else {
4327 log::warn!("subc attach: route bind rejected ({code}): {message}");
4328 }
4329 *window_start = now;
4330 *suppressed = 0;
4331 }
4332 None => {
4333 log::warn!("subc attach: route bind rejected ({code}): {message}");
4334 map.insert(message.to_string(), (now, 0));
4335 }
4336 }
4337}
4338
4339async fn handle_tool_call(
4344 tx: &WriterSender,
4345 frame: &Frame,
4346 mut phase_trace: PhaseTrace,
4347 routes: &HashMap<RouteChannel, RouteIdentity>,
4348 pending_binds: &HashMap<RouteChannel, PendingBind>,
4349 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4350 executor: &Arc<Executor>,
4351 active_tool_calls: &ActiveToolCalls,
4352 pending_inspect_setups: &Arc<AtomicUsize>,
4353 shutdown: &Arc<Notify>,
4354 connection_cancel: &PersistentCancelSignal,
4355 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
4356 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
4357 metrics: &Arc<DispatchPathMetrics>,
4358 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
4359 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
4360 next_bash_ask_corr: &mut u64,
4361 bg_subs: &mut HashMap<RouteChannel, BgSub>,
4362 bg_sub_by_session: &mut BgSubsBySession,
4363 bg_wake_pending: &mut HashSet<RouteChannel>,
4364 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
4365 dispatch: DispatchFn,
4366 inspect_deferred_tx: &mpsc::UnboundedSender<PendingSubcInspect>,
4367 allow_native_passthrough: bool,
4368 tool_response_body_limit: usize,
4369) -> Result<(), SubcError> {
4370 let route_id = route_key(frame.header.channel, frame.header.epoch);
4371 if pending_binds.contains_key(&route_id) {
4372 let error = build_error_frame(
4373 frame.header.ver,
4374 frame.header.channel,
4375 frame.header.epoch,
4376 frame.header.corr,
4377 frame.header.flags,
4378 "route_not_bound",
4379 "route is not bound before tool call",
4380 )?;
4381 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
4382 }
4383
4384 let Some(identity) = routes.get(&route_id).cloned() else {
4385 let error = build_error_frame(
4386 frame.header.ver,
4387 frame.header.channel,
4388 frame.header.epoch,
4389 frame.header.corr,
4390 frame.header.flags,
4391 "route_not_bound",
4392 "route is not bound before tool call",
4393 )?;
4394 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
4395 };
4396 let restore_watcher = live_roots
4397 .get(&identity.root)
4398 .is_some_and(|meta| meta.idle_artifacts_evicted);
4399 if let Some(meta) = live_roots.get_mut(&identity.root) {
4400 meta.reactivate_bound();
4401 }
4402 if restore_watcher {
4403 if let Some(ctx) = executor.actor_context(&identity.root) {
4404 crate::commands::configure::ensure_project_watcher(&ctx);
4405 }
4406 }
4407
4408 let route_request =
4409 serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
4410 if matches!(
4411 route_request,
4412 RouteRequest::BgEvents(BgEventsRequest {
4413 op: BgEventsOp::BgEvents
4414 })
4415 ) {
4416 if let Some(old_sub) = bg_subs.get(&route_id).cloned() {
4417 metrics.record_bg_subscription_ended(
4418 &old_sub.root,
4419 &old_sub.session,
4420 route_id,
4421 "resubscribe",
4422 );
4423 push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
4424 }
4425 if !identity.trust.allows_bash_observation() {
4426 bg_subs.remove(&route_id);
4427 bg_wake_pending.remove(&route_id);
4428 remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
4429 let denied_sub = BgSub {
4430 corr: frame.header.corr,
4431 ver: frame.header.ver,
4432 flags: frame.header.flags,
4433 root: identity.root.clone(),
4434 session: identity.session.clone(),
4435 };
4436 metrics.record_bg_subscription_ended(
4437 &identity.root,
4438 &identity.session,
4439 route_id,
4440 "subscribe-denied",
4441 );
4442 push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
4443 return Ok(());
4444 }
4445 bg_subs.insert(
4446 route_id,
4447 BgSub {
4448 corr: frame.header.corr,
4449 ver: frame.header.ver,
4450 flags: frame.header.flags,
4451 root: identity.root.clone(),
4452 session: identity.session.clone(),
4453 },
4454 );
4455 insert_bg_subscription_index(
4456 bg_sub_by_session,
4457 identity.root.clone(),
4458 identity.session.clone(),
4459 route_id,
4460 );
4461 metrics.record_bg_subscription_installed(&identity.root, &identity.session, route_id);
4462 push::arm_bg_wake(
4463 identity.root.clone(),
4464 identity.session.clone(),
4465 route_id,
4466 bg_wake_pending,
4467 bg_wake_epoch,
4468 metrics,
4469 );
4470 return Ok(());
4471 }
4472
4473 let RouteRequest::ToolCall(call) = route_request else {
4474 unreachable!("background event subscription returned above")
4475 };
4476 let bare_name = call.name;
4477 let arguments = strip_agent_preview_arg_owned(call.arguments);
4478 let format_context = crate::subc_format::FormatContext::from_tool_call(
4479 &bare_name,
4480 &arguments,
4481 identity.project_root.as_path(),
4482 );
4483
4484 let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
4485 let bind_trust = identity.trust;
4486 let diagnostics_on_edit = live_roots
4487 .get(&identity.root)
4488 .map(|meta| meta.diagnostics_on_edit)
4489 .unwrap_or(false);
4490
4491 let requests_host = bare_name == "bash"
4492 && arguments
4493 .get("sandbox")
4494 .or_else(|| {
4495 arguments
4496 .get("params")
4497 .and_then(|params| params.get("sandbox"))
4498 })
4499 .and_then(Value::as_str)
4500 == Some("host");
4501 if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
4502 let response = Response::error(
4503 request_id.clone(),
4504 "sandbox_escalation_denied",
4505 "sandbox host escalation is unavailable to untrusted principals",
4506 );
4507 let text = crate::subc_format::format_response_with_context(
4508 &bare_name,
4509 &response,
4510 &format_context,
4511 );
4512 let result = ToolCallResult { text, response };
4513 let response_frame = build_tool_response_frame_with_limit(
4514 frame.header.ver,
4515 route_id,
4516 frame.header.corr,
4517 frame.header.flags,
4518 &result,
4519 bind_trust,
4520 tool_response_body_limit,
4521 )?;
4522 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4523 }
4524
4525 if matches!(bind_trust, BindTrust::Untrusted)
4526 && is_bash_family_tool(&bare_name)
4527 && (bare_name != "bash" || !identity.consumer_elicitation_capable)
4528 {
4529 let response = bash::bash_denied_untrusted_response(request_id.clone());
4530 let text = crate::subc_format::format_response_with_context(
4531 &bare_name,
4532 &response,
4533 &format_context,
4534 );
4535 let result = ToolCallResult { text, response };
4536 let response_frame = build_tool_response_frame_with_limit(
4537 frame.header.ver,
4538 route_id,
4539 frame.header.corr,
4540 frame.header.flags,
4541 &result,
4542 bind_trust,
4543 tool_response_body_limit,
4544 )?;
4545 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4546 }
4547
4548 if !is_subc_agent_core_tool(&bare_name)
4556 && !is_subc_native_plumbing_tool(&bare_name)
4557 && !allow_native_passthrough
4558 {
4559 log::warn!(
4560 "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
4561 bare_name,
4562 frame.header.channel
4563 );
4564 let response = Response::error(
4565 request_id.clone(),
4566 "unknown_tool",
4567 format!("tool {:?} is not in the AFT tool manifest", bare_name),
4568 );
4569 let text = crate::subc_format::format_response_with_context(
4570 &bare_name,
4571 &response,
4572 &format_context,
4573 );
4574 let result = ToolCallResult { text, response };
4575 let response_frame = build_tool_response_frame_with_limit(
4576 frame.header.ver,
4577 route_id,
4578 frame.header.corr,
4579 frame.header.flags,
4580 &result,
4581 bind_trust,
4582 tool_response_body_limit,
4583 )?;
4584 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4585 }
4586
4587 if bare_name == "bash" {
4588 if matches!(bind_trust, BindTrust::Untrusted) {
4589 let plan = match bash::prepare_bash_elicitation_plan(
4590 &arguments,
4591 identity.project_root.as_path(),
4592 ) {
4593 Ok(plan) => plan,
4594 Err(error) => {
4595 let response = Response::error(request_id.clone(), error.code, error.message);
4596 let text = crate::subc_format::format_response_with_context(
4597 &bare_name,
4598 &response,
4599 &format_context,
4600 );
4601 let result = ToolCallResult { text, response };
4602 let response_frame = build_tool_response_frame_with_limit(
4603 frame.header.ver,
4604 route_id,
4605 frame.header.corr,
4606 frame.header.flags,
4607 &result,
4608 bind_trust,
4609 tool_response_body_limit,
4610 )?;
4611 return send_reliable_writer_frame(
4612 tx,
4613 metrics,
4614 response_frame,
4615 "tool response",
4616 )
4617 .await;
4618 }
4619 };
4620
4621 let reverse_corr =
4622 allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
4623 let ask_frame = build_bash_elicitation_request_frame(
4624 frame.header.ver,
4625 route_id,
4626 reverse_corr,
4627 frame.header.flags,
4628 &plan.command,
4629 &plan.asks,
4630 )?;
4631
4632 let meta = live_roots
4633 .entry(identity.root.clone())
4634 .or_insert_with(|| RootMeta::new(Instant::now()));
4635 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4636 meta.reactivate_bound();
4637
4638 let route_cancel =
4639 route_bash_cancels
4640 .entry(route_id)
4641 .or_insert_with(|| bash::RouteBashCancel {
4642 token: PersistentCancelSignal::new(),
4643 active_waits: 0,
4644 });
4645 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4646 let cancel = bash::BashWaitCancel {
4647 connection: connection_cancel.clone(),
4648 route: route_cancel.token.clone(),
4649 };
4650 pending_bash_asks.insert(
4651 ReverseCorrKey {
4652 route: route_id,
4653 corr: reverse_corr,
4654 },
4655 PendingBashAsk {
4656 route: route_id,
4657 tool_corr: frame.header.corr,
4658 tool_flags: frame.header.flags,
4659 tool_ver: frame.header.ver,
4660 root: identity.root.clone(),
4661 project_root: identity.project_root.clone(),
4662 session_id: identity.session.clone(),
4663 spawn_principal: identity.spawn_principal.clone(),
4664 edit_slot_survives: call.edit_slot_survives,
4665 request_id,
4666 arguments,
4667 format_context,
4668 cancel,
4669 grants: plan.grants,
4670 expires_at: Instant::now() + bash_elicitation_timeout(),
4671 },
4672 );
4673 return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
4674 .await;
4675 }
4676
4677 let meta = live_roots
4678 .entry(identity.root.clone())
4679 .or_insert_with(|| RootMeta::new(Instant::now()));
4680 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4681 meta.reactivate_bound();
4682
4683 let route_cancel =
4684 route_bash_cancels
4685 .entry(route_id)
4686 .or_insert_with(|| bash::RouteBashCancel {
4687 token: PersistentCancelSignal::new(),
4688 active_waits: 0,
4689 });
4690 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4691 let cancel = bash::BashWaitCancel {
4692 connection: connection_cancel.clone(),
4693 route: route_cancel.token.clone(),
4694 };
4695
4696 bash::submit_deferred_bash(
4697 executor,
4698 bash_deferred_tx,
4699 bash_poll_touch_tx,
4700 metrics,
4701 dispatch,
4702 identity.root.clone(),
4703 identity.project_root.clone(),
4704 identity.session.clone(),
4705 request_id,
4706 route_id,
4707 frame.header.corr,
4708 frame.header.flags,
4709 frame.header.ver,
4710 arguments,
4711 format_context,
4712 cancel,
4713 bind_trust,
4714 identity.spawn_principal.clone(),
4715 call.edit_slot_survives,
4716 None,
4717 );
4718 return Ok(());
4719 }
4720
4721 let lane = command_lane(&bare_name);
4722 let tool_call_context = ToolCallContext {
4723 project_root: identity.project_root.clone(),
4724 session_id: Some(identity.session.clone()),
4725 request_id: request_id.clone(),
4726 diagnostics_on_edit,
4727 preview: call.preview,
4728 edit_slot_survives: call.edit_slot_survives,
4729 report_registration_downgrade: true,
4730 };
4731
4732 if bare_name == "inspect" {
4733 let Some(inspect_ctx) = executor.actor_context(&identity.root) else {
4734 let response = Response::error(
4735 &request_id,
4736 "actor_not_registered",
4737 "executor actor is not registered",
4738 );
4739 let text = crate::subc_format::format_response_with_context(
4740 "inspect",
4741 &response,
4742 &format_context,
4743 );
4744 let result = ToolCallResult { text, response };
4745 let response_frame = build_tool_response_frame_with_limit(
4746 frame.header.ver,
4747 route_id,
4748 frame.header.corr,
4749 frame.header.flags,
4750 &result,
4751 bind_trust,
4752 tool_response_body_limit,
4753 )?;
4754 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4755 };
4756 let identity_for_run = identity.clone();
4757 let request_id_for_force = request_id.clone();
4758 let format_context_for_run = format_context.clone();
4759 let (setup_tx, setup_rx) = oneshot::channel::<InspectSetupOutcome>();
4760 phase_trace.mark_executor_submitted();
4761 let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
4762 phase_trace.mark_job_admitted();
4763 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4764 let run = || match prepare_tool_call(
4765 "inspect",
4766 arguments,
4767 &format_context_for_run,
4768 &tool_call_context,
4769 ctx,
4770 Some(&mut phase_trace),
4771 ) {
4772 Err(result) => {
4773 let response = result.response;
4774 let _ = setup_tx.send(InspectSetupOutcome::Immediate {
4775 text: result.text,
4776 phase_trace,
4777 });
4778 response
4779 }
4780 Ok(prepared) => {
4781 let DispatchOutcome::Deferred(pending) =
4782 crate::commands::inspect::handle_inspect_deferred_with_restriction(
4783 &prepared.request,
4784 Arc::clone(&inspect_ctx),
4785 matches!(bind_trust, BindTrust::Untrusted),
4786 )
4787 else {
4788 unreachable!("inspect deferred setup returned an immediate response")
4789 };
4790 let _ = setup_tx.send(InspectSetupOutcome::Deferred {
4791 pending,
4792 surface_downgraded: prepared.surface_downgraded,
4793 phase_trace,
4794 });
4795 Response::success(
4796 request_id_for_force.clone(),
4797 json!({ "inspect_deferred": true }),
4798 )
4799 }
4800 };
4801 if matches!(bind_trust, BindTrust::Untrusted) {
4802 ctx.with_force_restrict(&request_id_for_force, run)
4803 } else {
4804 run()
4805 }
4806 })
4807 });
4808 let inspect_setup_guard = PendingInspectSetupGuard::new(Arc::clone(pending_inspect_setups));
4809 let (rx, cancellation) =
4810 executor.submit_cancellable_async(identity.root.clone(), lane, request_id.clone(), job);
4811 active_tool_calls
4812 .lock()
4813 .unwrap_or_else(std::sync::PoisonError::into_inner)
4814 .insert(
4815 (route_id, frame.header.corr),
4816 ActiveToolCall {
4817 root_id: identity.root.clone(),
4818 cancellation,
4819 },
4820 );
4821
4822 let completion_tx = tx.clone();
4823 let completion_shutdown = Arc::clone(shutdown);
4824 let completion_metrics = Arc::clone(metrics);
4825 let active_tool_calls = Arc::clone(active_tool_calls);
4826 let inspect_deferred_tx = inspect_deferred_tx.clone();
4827 let route = route_id;
4828 let corr = frame.header.corr;
4829 let flags = frame.header.flags;
4830 let ver = frame.header.ver;
4831 let root = identity.root.clone();
4832 let session_id = identity.session.clone();
4833 tokio::spawn(async move {
4834 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4835 let _inspect_setup = inspect_setup_guard;
4836 let response = await_executor_response(rx, request_id.clone()).await;
4837 match setup_rx.await {
4838 Ok(InspectSetupOutcome::Deferred {
4839 pending,
4840 surface_downgraded,
4841 phase_trace,
4842 }) => {
4843 let pending = PendingSubcInspect {
4844 route,
4845 corr,
4846 flags,
4847 ver,
4848 root,
4849 session_id,
4850 format_context,
4851 bind_trust,
4852 pending,
4853 surface_downgraded,
4854 phase_trace,
4855 };
4856 if let Err(error) = inspect_deferred_tx.send(pending) {
4857 if let Some(cancellation) = &error.0.pending.cancellation {
4858 cancellation.request_cancel();
4859 }
4860 finish_active_tool_call(&active_tool_calls, route, corr);
4861 }
4862 }
4863 Ok(InspectSetupOutcome::Immediate { text, phase_trace }) => {
4864 finish_active_tool_call(&active_tool_calls, route, corr);
4865 let result = ToolCallResult { text, response };
4866 let fatal = response_is_fatal_panic(&result.response);
4867 match build_tool_response_frame_with_limit(
4868 ver,
4869 route,
4870 corr,
4871 flags,
4872 &result,
4873 bind_trust,
4874 tool_response_body_limit,
4875 ) {
4876 Ok(response_frame) => {
4877 let trace = ToolResponseWriteTrace::new(
4878 phase_trace,
4879 "inspect".to_string(),
4880 identity.project_root.clone(),
4881 identity.session.clone(),
4882 route.channel,
4883 corr,
4884 );
4885 if let Err(error) = send_traced_tool_response_frame(
4886 &completion_tx,
4887 &completion_metrics,
4888 response_frame,
4889 trace,
4890 )
4891 .await
4892 {
4893 log::warn!(
4894 "subc attach: failed to queue inspect setup response: {error}"
4895 );
4896 }
4897 }
4898 Err(error) => {
4899 log::error!(
4900 "subc attach: failed to build inspect setup response: {error}"
4901 );
4902 }
4903 }
4904 if fatal {
4905 signal_fatal_teardown(
4906 &completion_tx,
4907 Some(route),
4908 ver,
4909 corr,
4910 &completion_shutdown,
4911 &completion_metrics,
4912 )
4913 .await;
4914 }
4915 }
4916 Err(_) => {
4917 finish_active_tool_call(&active_tool_calls, route, corr);
4918 let text = crate::subc_format::format_response_with_context(
4919 "inspect",
4920 &response,
4921 &format_context,
4922 );
4923 let result = ToolCallResult { text, response };
4924 if let Ok(response_frame) = build_tool_response_frame_with_limit(
4925 ver,
4926 route,
4927 corr,
4928 flags,
4929 &result,
4930 bind_trust,
4931 tool_response_body_limit,
4932 ) {
4933 let _ = send_reliable_writer_frame(
4934 &completion_tx,
4935 &completion_metrics,
4936 response_frame,
4937 "inspect setup failure",
4938 )
4939 .await;
4940 }
4941 }
4942 }
4943 });
4944 return Ok(());
4945 }
4946
4947 let bare_name_for_frame = bare_name.clone();
4948 let identity_for_run = identity.clone();
4949 let completion_session = identity.session.clone();
4950 let completion_root = identity.project_root.clone();
4951 let request_id_for_force = request_id.clone();
4952 let format_context_for_frame = format_context.clone();
4953 let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
4954 phase_trace.mark_executor_submitted();
4955 let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
4956 phase_trace.mark_job_admitted();
4957 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4958 let run = || {
4959 let finalizer = |response: &mut Response| {
4960 crate::response_finalize::finalize_response_with_bg_completions(
4961 response,
4962 ctx,
4963 &identity_for_run.session,
4964 &bare_name,
4965 bind_trust.allows_bash_observation(),
4966 );
4967 };
4968 match run_tool_call(
4969 &bare_name,
4970 arguments,
4971 &format_context,
4972 &tool_call_context,
4973 ctx,
4974 &dispatch,
4975 Some(&finalizer),
4976 Some(&mut phase_trace),
4977 ) {
4978 ToolCallOutcome::Unary(result) => {
4979 let response = result.response;
4980 let _ = tool_call_tx.send(ToolCallCompletion {
4981 text: result.text,
4982 phase_trace,
4983 });
4984 response
4985 }
4986 }
4987 };
4988 if matches!(bind_trust, BindTrust::Untrusted) {
4989 ctx.with_force_restrict(&request_id_for_force, run)
4990 } else {
4991 run()
4992 }
4993 })
4994 });
4995 let rx = executor.submit_async(identity.root.clone(), lane, request_id.clone(), job);
4996 let completion_tx = tx.clone();
4997 let completion_shutdown = Arc::clone(shutdown);
4998 let route = route_id;
4999 let corr = frame.header.corr;
5000 let flags = frame.header.flags;
5001 let ver = frame.header.ver;
5002 let completion_metrics = Arc::clone(metrics);
5003 tokio::spawn(async move {
5004 let _response_task = ResponseTaskGuard::new(&completion_metrics);
5005 let response = await_executor_response(rx, request_id.clone()).await;
5006 let (text, phase_trace) = match tool_call_rx.await {
5007 Ok(completion) => (completion.text, Some(completion.phase_trace)),
5008 Err(_) => (
5009 crate::subc_format::format_response_with_context(
5010 &bare_name_for_frame,
5011 &response,
5012 &format_context_for_frame,
5013 ),
5014 None,
5015 ),
5016 };
5017 let result = ToolCallResult { text, response };
5018 let fatal = response_is_fatal_panic(&result.response);
5019 match build_tool_response_frame_with_limit(
5020 ver,
5021 route,
5022 corr,
5023 flags,
5024 &result,
5025 bind_trust,
5026 tool_response_body_limit,
5027 ) {
5028 Ok(response_frame) => {
5029 let send_result = if let Some(phase_trace) = phase_trace {
5030 let trace = ToolResponseWriteTrace::new(
5031 phase_trace,
5032 bare_name_for_frame,
5033 completion_root,
5034 completion_session,
5035 route.channel,
5036 corr,
5037 );
5038 send_traced_tool_response_frame(
5039 &completion_tx,
5040 &completion_metrics,
5041 response_frame,
5042 trace,
5043 )
5044 .await
5045 } else {
5046 send_reliable_writer_frame(
5047 &completion_tx,
5048 &completion_metrics,
5049 response_frame,
5050 "tool response",
5051 )
5052 .await
5053 };
5054 if let Err(error) = send_result {
5055 log::warn!("subc attach: failed to queue tool response frame: {error}");
5056 }
5057 }
5058 Err(error) => {
5059 log::error!("subc attach: failed to build tool response frame: {error}");
5060 }
5061 }
5062 if fatal {
5063 signal_fatal_teardown(
5064 &completion_tx,
5065 Some(route),
5066 ver,
5067 corr,
5068 &completion_shutdown,
5069 &completion_metrics,
5070 )
5071 .await;
5072 }
5073 });
5074 Ok(())
5075}
5076
5077fn submit_maintenance_job(
5078 executor: &Arc<Executor>,
5079 root_id: ProjectRootId,
5080 kind: MaintenanceDrainKind,
5081 bg_sessions_to_check: Vec<(String, u64)>,
5082 completion_tx: &mpsc::Sender<MaintenanceCompletion>,
5083 metrics: &Arc<DispatchPathMetrics>,
5084) {
5085 let request_id = format!(
5086 "subc-maintenance-drain-{}-{}",
5087 kind.label(),
5088 root_id.as_path().to_string_lossy()
5089 );
5090 let response_id = request_id.clone();
5091 let completion_root_id = root_id.clone();
5092 let maintenance_generation = executor
5093 .actor_context(&root_id)
5094 .map(|ctx| ctx.configure_generation())
5095 .unwrap_or(0);
5096 let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
5097 let lane = match kind {
5102 MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
5103 MaintenanceDrainKind::Watcher
5104 | MaintenanceDrainKind::Lsp
5105 | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
5106 };
5107 let job: crate::executor::ExecutorJob = Box::new(move |ctx: &AppContext| {
5108 let outcome = match kind {
5109 MaintenanceDrainKind::Watcher => {
5110 let drained = runtime_drain::drain_watcher_events_bounded(
5111 ctx,
5112 runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
5113 );
5114 MaintenanceJobOutcome {
5115 empty_bg_sessions: Vec::new(),
5116 requeue_kind: drained.has_more.then_some(kind),
5117 }
5118 }
5119 MaintenanceDrainKind::Lsp => {
5120 let drained = runtime_drain::drain_lsp_events_bounded(
5121 ctx,
5122 runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
5123 );
5124 MaintenanceJobOutcome {
5125 empty_bg_sessions: Vec::new(),
5126 requeue_kind: drained.has_more.then_some(kind),
5127 }
5128 }
5129 MaintenanceDrainKind::ConfigureTail => {
5130 runtime_drain::drain_deferred_configure_maintenance(ctx);
5131 runtime_drain::drain_configure_warning_events(ctx);
5132 MaintenanceJobOutcome::default()
5133 }
5134 MaintenanceDrainKind::CompletionDrains => {
5135 runtime_drain::drain_search_index_events(ctx);
5136 runtime_drain::drain_callgraph_store_events(ctx);
5137 runtime_drain::drain_semantic_index_events(ctx);
5138 runtime_drain::drain_semantic_refresh_events(ctx);
5139 runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
5140 let empty_bg_sessions = bg_sessions_to_check
5141 .into_iter()
5142 .filter(|(session, _)| {
5143 !ctx.bash_background()
5144 .has_completions_for_session(Some(session.as_str()))
5145 })
5146 .collect();
5147 MaintenanceJobOutcome {
5148 empty_bg_sessions,
5149 requeue_kind: None,
5150 }
5151 }
5152 };
5153 let requeued = outcome.requeue_kind.is_some();
5154 let _ = outcome_tx.send(outcome);
5155 Response::success(
5156 response_id,
5157 json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
5158 )
5159 });
5160 let rx = match kind {
5161 MaintenanceDrainKind::Watcher => executor.submit_coalescable_maintenance_async(
5162 root_id,
5163 lane,
5164 request_id.clone(),
5165 crate::executor::MaintenanceCoalesceKey::WatcherDrain,
5166 job,
5167 ),
5168 MaintenanceDrainKind::Lsp => executor.submit_coalescable_maintenance_async(
5169 root_id,
5170 lane,
5171 request_id.clone(),
5172 crate::executor::MaintenanceCoalesceKey::LspDrain,
5173 job,
5174 ),
5175 MaintenanceDrainKind::ConfigureTail | MaintenanceDrainKind::CompletionDrains => {
5176 executor.submit_maintenance_async(root_id, lane, request_id.clone(), job)
5177 }
5178 };
5179 let completion_tx = completion_tx.clone();
5180 let completion_metrics = Arc::clone(metrics);
5181 tokio::spawn(async move {
5182 let _response_task = ResponseTaskGuard::new(&completion_metrics);
5183 let response = await_executor_response(rx, request_id).await;
5184 let outcome = outcome_rx.await.unwrap_or_default();
5185 let _ = send_counted_channel(
5186 &completion_tx,
5187 &completion_metrics.maintenance_queued,
5188 MaintenanceCompletion {
5189 root_id: completion_root_id,
5190 kind,
5191 response,
5192 empty_bg_sessions: outcome.empty_bg_sessions,
5193 requeue_kind: outcome.requeue_kind,
5194 },
5195 )
5196 .await;
5197 });
5198}
5199
5200async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
5201 rx.await
5202 .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
5203}
5204
5205async fn deliver_resolved_subc_inspect(
5206 tx: &WriterSender,
5207 mut resolved: ResolvedSubcInspect,
5208 routes: &HashMap<RouteChannel, RouteIdentity>,
5209 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
5210 executor: &Executor,
5211 active_tool_calls: &ActiveToolCalls,
5212 shutdown: &Arc<Notify>,
5213 metrics: &DispatchPathMetrics,
5214 tool_response_body_limit: usize,
5215) -> Result<(), SubcError> {
5216 let entry = &mut resolved.entry;
5217 finish_active_tool_call(active_tool_calls, entry.route, entry.corr);
5218 if let Some(meta) = live_roots.get_mut(&entry.root) {
5219 meta.note_activity();
5220 }
5221
5222 let Some(identity) = routes.get(&entry.route) else {
5223 log::debug!(
5224 "subc attach: dropping deferred inspect response {} for unbound route {}",
5225 entry.pending.request_id,
5226 entry.route
5227 );
5228 return Ok(());
5229 };
5230 let Some(ctx) = executor.actor_context(&entry.root) else {
5231 return Ok(());
5232 };
5233 entry.phase_trace.mark_execute_done();
5234 let finalizer = |response: &mut Response| {
5235 crate::response_finalize::finalize_response_with_bg_completions(
5236 response,
5237 &ctx,
5238 &entry.session_id,
5239 "inspect",
5240 entry.bind_trust.allows_bash_observation(),
5241 );
5242 };
5243 let result = finish_tool_call_response(
5244 "inspect",
5245 &entry.format_context,
5246 resolved.response,
5247 entry.surface_downgraded,
5248 Some(&finalizer),
5249 Some(&mut entry.phase_trace),
5250 );
5251 let fatal = response_is_fatal_panic(&result.response);
5252 let response_frame = build_tool_response_frame_with_limit(
5253 entry.ver,
5254 entry.route,
5255 entry.corr,
5256 entry.flags,
5257 &result,
5258 identity.trust,
5259 tool_response_body_limit,
5260 )?;
5261 let trace = ToolResponseWriteTrace::new(
5262 std::mem::replace(&mut entry.phase_trace, PhaseTrace::new(Instant::now())),
5263 "inspect".to_string(),
5264 identity.project_root.clone(),
5265 entry.session_id.clone(),
5266 entry.route.channel,
5267 entry.corr,
5268 );
5269 send_traced_tool_response_frame(tx, metrics, response_frame, trace).await?;
5270 if fatal {
5271 signal_fatal_teardown(
5272 tx,
5273 Some(entry.route),
5274 entry.ver,
5275 entry.corr,
5276 shutdown,
5277 metrics,
5278 )
5279 .await;
5280 }
5281 Ok(())
5282}
5283
5284async fn signal_fatal_teardown(
5285 tx: &WriterSender,
5286 route: Option<RouteChannel>,
5287 ver: u8,
5288 corr: u64,
5289 shutdown: &Arc<Notify>,
5290 metrics: &DispatchPathMetrics,
5291) {
5292 if let Some(route) = route {
5293 if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
5294 if let Err(error) = send_frame(tx, metrics, frame).await {
5295 log::warn!(
5296 "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
5297 );
5298 }
5299 }
5300 }
5301 if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
5302 if let Err(error) = send_frame(tx, metrics, frame).await {
5303 log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
5304 }
5305 }
5306 shutdown.notify_one();
5307}
5308#[derive(Debug, Deserialize)]
5309#[serde(untagged)]
5310enum RouteRequest {
5311 BgEvents(BgEventsRequest),
5312 ToolCall(ToolCallRequest),
5313}
5314
5315#[derive(Debug, Deserialize)]
5316struct BgEventsRequest {
5317 op: BgEventsOp,
5318}
5319
5320#[derive(Debug, Deserialize)]
5321#[serde(rename_all = "snake_case")]
5322enum BgEventsOp {
5323 BgEvents,
5324}
5325
5326#[derive(Debug, Deserialize)]
5327struct ToolCallRequest {
5328 name: String,
5329 #[serde(default)]
5330 arguments: Value,
5331 #[serde(default)]
5333 edit_slot_survives: Option<bool>,
5334 #[serde(default)]
5339 preview: bool,
5340}
5341
5342#[cfg(test)]
5343pub(crate) mod test_support {
5344 use super::*;
5345 use crate::bash_background::BgTaskStatus;
5346 use crate::protocol::{
5347 BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
5348 ProgressFrame, StatusChangedFrame,
5349 };
5350 use serde_json::json;
5351
5352 pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
5353 let dir = tempfile::Builder::new()
5354 .prefix(name)
5355 .tempdir()
5356 .expect("temp root");
5357 let root = ProjectRootId::from_path(dir.path()).expect("project root id");
5358 (dir, root)
5359 }
5360
5361 pub(super) fn test_ctx() -> Arc<AppContext> {
5362 Arc::new(AppContext::new(
5363 Box::new(crate::parser::TreeSitterProvider::new()),
5364 crate::config::Config::default(),
5365 ))
5366 }
5367
5368 fn inspect_context(root: &Path) -> Arc<AppContext> {
5369 let mut config = crate::config::Config::default();
5370 config.project_root = Some(root.to_path_buf());
5371 let ctx = Arc::new(AppContext::new(
5372 Box::new(crate::parser::TreeSitterProvider::new()),
5373 config,
5374 ));
5375 ctx.set_harness(crate::harness::Harness::Opencode);
5376 ctx
5377 }
5378
5379 fn inspect_request(id: &str) -> RawRequest {
5380 serde_json::from_value(json!({ "id": id, "command": "inspect" })).expect("inspect request")
5381 }
5382
5383 fn submit_deferred_inspect_setup(
5384 executor: &Arc<Executor>,
5385 root: &ProjectRootId,
5386 ctx: &Arc<AppContext>,
5387 request_id: &str,
5388 ) -> (PendingResponse, JobCancellation) {
5389 let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
5390 let request = inspect_request(request_id);
5391 let inspect_ctx = Arc::clone(ctx);
5392 let (_rx, cancellation) = executor.submit_cancellable_async(
5393 root.clone(),
5394 Lane::SerialLspStatus,
5395 request_id.to_string(),
5396 Box::new(move |_| {
5397 let DispatchOutcome::Deferred(pending) =
5398 crate::commands::inspect::handle_inspect_deferred_with_restriction(
5399 &request,
5400 inspect_ctx,
5401 true,
5402 )
5403 else {
5404 panic!("inspect setup must defer")
5405 };
5406 pending_tx.send(pending).expect("send pending inspect");
5407 Response::success("inspect-setup", json!({}))
5408 }),
5409 );
5410 let pending = pending_rx
5411 .recv_timeout(Duration::from_secs(1))
5412 .expect("inspect setup leaves the executor lane");
5413 let deadline = Instant::now() + Duration::from_secs(1);
5414 while !executor.actor_is_idle(root) {
5415 assert!(
5416 Instant::now() < deadline,
5417 "inspect setup kept lane counters live"
5418 );
5419 std::thread::sleep(Duration::from_millis(5));
5420 }
5421 (pending, cancellation)
5422 }
5423
5424 fn wait_for_inspect_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
5425 let deadline = Instant::now() + Duration::from_secs(60);
5426 loop {
5427 if let Some(response) = (pending.poll)(ctx) {
5428 return response;
5429 }
5430 assert!(Instant::now() < deadline, "inspect terminal timed out");
5431 std::thread::sleep(Duration::from_millis(5));
5432 }
5433 }
5434
5435 #[test]
5436 fn deferred_inspect_releases_lane_for_bind_and_mutation() {
5437 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
5438 let executor = Arc::new(Executor::new());
5439 let (dir, root) = test_root("deferred-inspect-storm");
5440 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
5441 let ctx = inspect_context(dir.path());
5442 executor.register_actor(root.clone(), Arc::clone(&ctx));
5443 let (started_rx, release_tx) =
5444 crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
5445 let (mut pending, _cancellation) =
5446 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-storm");
5447 started_rx
5448 .recv_timeout(Duration::from_secs(1))
5449 .expect("deferred inspect body starts");
5450
5451 for request_id in ["subc-bind-other-session", "subc-edit-other-session"] {
5452 let response = executor.submit(
5453 root.clone(),
5454 Lane::Mutating,
5455 request_id.to_string(),
5456 Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
5457 );
5458 assert!(
5459 response
5460 .recv_timeout(Duration::from_secs(1))
5461 .expect("writer admits while inspect remains deferred")
5462 .success
5463 );
5464 }
5465 assert_eq!(
5466 crate::commands::inspect::deferred_inspect_root_count_for_test(),
5467 1,
5468 "writer admissions must not finish the detached inspect"
5469 );
5470
5471 release_tx.send(()).expect("release inspect body");
5472 let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
5473 assert!(
5474 terminal.data.get("inspect_terminal").is_some(),
5475 "inspect must still produce its terminal: {:?}",
5476 terminal.data
5477 );
5478 }
5479
5480 #[test]
5481 fn same_root_deferred_inspects_are_single_flight() {
5482 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
5483 let executor = Arc::new(Executor::new());
5484 let (dir, root) = test_root("single-flight-deferred-inspect");
5485 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
5486 let ctx = inspect_context(dir.path());
5487 executor.register_actor(root.clone(), Arc::clone(&ctx));
5488 let (started_rx, release_tx) =
5489 crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
5490 let (mut first, _first_cancellation) =
5491 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-first");
5492 started_rx
5493 .recv_timeout(Duration::from_secs(1))
5494 .expect("first inspect owns the root flight");
5495 let (mut second, second_cancellation) =
5496 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-second");
5497
5498 assert_eq!(
5499 crate::commands::inspect::deferred_inspect_root_count_for_test(),
5500 1,
5501 "only one detached body may run for a root"
5502 );
5503 assert!((second.poll)(&ctx).is_none(), "second inspect must queue");
5504 second_cancellation.request_cancel();
5505 let second_terminal = wait_for_inspect_terminal(&mut second, &ctx);
5506 assert_eq!(second_terminal.data["inspect_terminal"], "interrupted");
5507 assert_eq!(
5508 crate::commands::inspect::deferred_inspect_root_count_for_test(),
5509 1,
5510 "cancelling the queued request must not release the active flight"
5511 );
5512
5513 release_tx.send(()).expect("release first inspect");
5514 let first_terminal = wait_for_inspect_terminal(&mut first, &ctx);
5515 assert_eq!(first_terminal.data["inspect_terminal"], "fresh");
5516 assert_eq!(
5517 crate::commands::inspect::deferred_inspect_root_count_for_test(),
5518 0
5519 );
5520 }
5521
5522 #[test]
5523 fn route_abandonment_cancels_detached_inspect_thread() {
5524 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
5525 let executor = Arc::new(Executor::new());
5526 let (dir, root) = test_root("cancelled-deferred-inspect");
5527 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
5528 let ctx = inspect_context(dir.path());
5529 executor.register_actor(root.clone(), Arc::clone(&ctx));
5530 let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
5531 let route = RouteChannel {
5532 channel: 7,
5533 epoch: 1,
5534 };
5535 let (started_rx, _release_tx) =
5536 crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
5537 let (mut pending, cancellation) =
5538 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-abandoned");
5539 active.lock().expect("active tool call map").insert(
5540 (route, 41),
5541 ActiveToolCall {
5542 root_id: root.clone(),
5543 cancellation,
5544 },
5545 );
5546 started_rx
5547 .recv_timeout(Duration::from_secs(1))
5548 .expect("detached inspect reaches cancellation gate");
5549 assert!(ctx.request_force_restrict("subc-inspect-abandoned"));
5550
5551 assert!(cancel_active_tool_call(
5552 &active,
5553 executor.as_ref(),
5554 route,
5555 41,
5556 "test route abandonment"
5557 ));
5558 let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
5559 assert_eq!(terminal.data["inspect_terminal"], "interrupted");
5560 assert_eq!(
5561 crate::commands::inspect::deferred_inspect_root_count_for_test(),
5562 0
5563 );
5564 assert!(executor.actor_is_idle(&root));
5565 assert!(active.lock().expect("active tool call map").is_empty());
5566 let restriction_deadline = Instant::now() + Duration::from_secs(1);
5567 while ctx.request_force_restrict("subc-inspect-abandoned") {
5568 assert!(
5569 Instant::now() < restriction_deadline,
5570 "detached force-restrict guard leaked"
5571 );
5572 std::thread::sleep(Duration::from_millis(5));
5573 }
5574 }
5575
5576 #[test]
5577 fn shutdown_drain_emits_terminal_and_clears_pending_inspect() {
5578 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
5579 let executor = Arc::new(Executor::new());
5580 let (dir, root) = test_root("shutdown-deferred-inspect");
5581 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
5582 let ctx = inspect_context(dir.path());
5583 executor.register_actor(root.clone(), Arc::clone(&ctx));
5584 let (started_rx, _release_tx) =
5585 crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
5586 let (pending, cancellation) =
5587 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-shutdown");
5588 let route = RouteChannel {
5589 channel: 8,
5590 epoch: 1,
5591 };
5592 started_rx
5593 .recv_timeout(Duration::from_secs(1))
5594 .expect("detached inspect reaches shutdown gate");
5595 let mut registry = PendingSubcInspects::default();
5596 registry.register(PendingSubcInspect {
5597 route,
5598 corr: 42,
5599 flags: Flags::new(false, Priority::Passive, false),
5600 ver: PROTOCOL_VERSION,
5601 root: root.clone(),
5602 session_id: "shutdown-session".to_string(),
5603 format_context: crate::subc_format::FormatContext::from_tool_call(
5604 "inspect",
5605 &json!({}),
5606 dir.path(),
5607 ),
5608 bind_trust: BindTrust::FirstParty,
5609 pending,
5610 surface_downgraded: false,
5611 phase_trace: PhaseTrace::new(Instant::now()),
5612 });
5613 let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::from([(
5614 (route, 42),
5615 ActiveToolCall {
5616 root_id: root.clone(),
5617 cancellation,
5618 },
5619 )])));
5620
5621 let resolved = registry.drain_on_shutdown(executor.as_ref());
5622 assert!(registry.is_empty());
5623 assert_eq!(resolved.len(), 1);
5624 assert_eq!(
5625 resolved[0].response.data["failure_reason"],
5626 "daemon_shutdown"
5627 );
5628 finish_active_tool_call(&active, route, 42);
5629 let deadline = Instant::now() + Duration::from_secs(1);
5630 while crate::commands::inspect::deferred_inspect_root_count_for_test() != 0 {
5631 assert!(Instant::now() < deadline, "shutdown cancellation was inert");
5632 std::thread::sleep(Duration::from_millis(5));
5633 }
5634 assert!(active.lock().expect("active calls").is_empty());
5635 assert!(executor.actor_is_idle(&root));
5636 }
5637
5638 pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
5639 let deadline = Instant::now() + Duration::from_secs(30);
5640 loop {
5641 let observed = ctx.watcher_registry_count();
5642 if observed == expected {
5643 return;
5644 }
5645 assert!(
5646 Instant::now() < deadline,
5647 "watcher count did not settle before deadline: expected={expected}, observed={observed}"
5648 );
5649 std::thread::sleep(Duration::from_millis(50));
5650 }
5651 }
5652
5653 pub(super) fn reap_until_forgotten(
5663 root: &ProjectRootId,
5664 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
5665 pending_binds: &HashMap<RouteChannel, PendingBind>,
5666 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
5667 executor: &Arc<Executor>,
5668 metrics: &DispatchPathMetrics,
5669 ) -> IdleReapOutcome {
5670 let deadline = Instant::now() + Duration::from_secs(30);
5671 loop {
5672 let outcome = reap_idle_roots(
5673 Instant::now(),
5674 live_roots,
5675 pending_binds,
5676 root_channels,
5677 executor,
5678 metrics,
5679 );
5680 if outcome.forgotten_deleted_roots.contains(root) {
5681 return outcome;
5682 }
5683 assert!(
5684 Instant::now() < deadline,
5685 "deleted root was never forgotten: {root:?}"
5686 );
5687 std::thread::sleep(Duration::from_millis(10));
5688 }
5689 }
5690
5691 pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
5692 let deadline = Instant::now() + Duration::from_secs(30);
5693 loop {
5694 let observed = app.actor_root_count();
5695 if observed == expected {
5696 return;
5697 }
5698 assert!(
5699 Instant::now() < deadline,
5700 "actor root count did not settle before deadline: expected={expected}, observed={observed}"
5701 );
5702 std::thread::sleep(Duration::from_millis(50));
5703 }
5704 }
5705
5706 pub(super) fn status_frame(seq: u64) -> PushFrame {
5707 status_frame_with_session(seq, None)
5708 }
5709
5710 pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
5711 PushFrame::StatusChanged(StatusChangedFrame {
5712 frame_type: "status_changed",
5713 session_id: session_id.map(str::to_string),
5714 snapshot: json!({ "seq": seq }),
5715 })
5716 }
5717
5718 pub(super) fn completion_frame(task_id: &str) -> PushFrame {
5719 completion_frame_with_session(task_id, "session-1")
5720 }
5721
5722 pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
5723 PushFrame::BashCompleted(BashCompletedFrame {
5724 frame_type: "bash_completed",
5725 task_id: task_id.to_string(),
5726 session_id: session_id.to_string(),
5727 status: BgTaskStatus::Completed,
5728 exit_code: Some(0),
5729 command: format!("echo {task_id}"),
5730 output_preview: String::new(),
5731 output_truncated: false,
5732 original_tokens: None,
5733 compressed_tokens: None,
5734 tokens_skipped: false,
5735 status_reason: None,
5736 })
5737 }
5738
5739 pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
5740 long_running_frame_with_session(task_id, "session-1", elapsed_ms)
5741 }
5742
5743 pub(super) fn long_running_frame_with_session(
5744 task_id: &str,
5745 session_id: &str,
5746 elapsed_ms: u64,
5747 ) -> PushFrame {
5748 PushFrame::BashLongRunning(BashLongRunningFrame {
5749 frame_type: "bash_long_running",
5750 task_id: task_id.to_string(),
5751 session_id: session_id.to_string(),
5752 command: format!("sleep {elapsed_ms}"),
5753 elapsed_ms,
5754 })
5755 }
5756
5757 pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
5758 PushFrame::BashPatternMatch(BashPatternMatchFrame {
5759 frame_type: "bash_pattern_match",
5760 task_id: "task-pattern".to_string(),
5761 session_id: session_id.to_string(),
5762 watch_id: "watch-1".to_string(),
5763 match_text: "needle".to_string(),
5764 match_offset: 7,
5765 context: "haystack needle".to_string(),
5766 once: true,
5767 reason: "pattern_match",
5768 })
5769 }
5770
5771 pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
5772 PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
5773 frame_type: "configure_warnings",
5774 session_id: session_id.map(str::to_string),
5775 project_root: "/tmp/subc-test".to_string(),
5776 warnings: Vec::new(),
5777 })
5778 }
5779
5780 pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
5781 route_identity_with_trust(root, session_id, BindTrust::FirstParty)
5782 }
5783
5784 pub(super) fn route_identity_with_trust(
5785 root: &ProjectRootId,
5786 session_id: &str,
5787 trust: BindTrust,
5788 ) -> RouteIdentity {
5789 RouteIdentity(Arc::new(RouteIdentityData {
5790 root: root.clone(),
5791 project_root: root.as_path().to_path_buf(),
5792 harness: "opencode".to_string(),
5793 session: session_id.to_string(),
5794 trust,
5795 spawn_principal: AuthenticatedPrincipal::RouteBind {
5796 trust: trust.sandbox_trust(),
5797 route_channel: 0,
5798 route_epoch: 0,
5799 project_root: root.as_path().to_path_buf(),
5800 harness: "opencode".to_string(),
5801 session_id: session_id.to_string(),
5802 principal_id: Some(match trust {
5803 BindTrust::FirstParty => "direct".to_string(),
5804 BindTrust::Untrusted => "unverified".to_string(),
5805 }),
5806 },
5807 consumer_elicitation_capable: false,
5808 }))
5809 }
5810
5811 pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
5812 PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
5813 }
5814
5815 pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
5816 match frame {
5817 PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
5818 _ => None,
5819 }
5820 }
5821
5822 pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
5823 match frame {
5824 PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
5825 _ => None,
5826 }
5827 }
5828
5829 pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
5830 let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
5831 body.get("task_id")
5832 .and_then(serde_json::Value::as_str)
5833 .map(str::to_string)
5834 }
5835}
5836
5837#[cfg(test)]
5838mod tests {
5839 use super::test_support::{
5840 completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
5841 wait_for_actor_root_count, wait_for_watcher_count,
5842 };
5843 use super::*;
5844 use crate::bash_background::BgTaskStatus;
5845
5846 fn attach_error(kind: io::ErrorKind) -> SubcError {
5847 SubcError::Connect {
5848 endpoint: "127.0.0.1:1".to_string(),
5849 source: io::Error::new(kind, "constructed attach failure"),
5850 }
5851 }
5852
5853 fn auth_io_error(kind: io::ErrorKind) -> SubcError {
5854 SubcError::Auth {
5855 endpoint: "127.0.0.1:1".to_string(),
5856 source: subc_transport::AuthError::Io {
5857 stage: subc_transport::AuthStage::ServerProof,
5858 source: io::Error::new(kind, "constructed auth failure"),
5859 },
5860 }
5861 }
5862
5863 #[test]
5864 fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
5865 let transient_errors = vec![
5866 attach_error(io::ErrorKind::ConnectionRefused),
5867 attach_error(io::ErrorKind::TimedOut),
5868 attach_error(io::ErrorKind::ConnectionReset),
5869 auth_io_error(io::ErrorKind::ConnectionAborted),
5870 auth_io_error(io::ErrorKind::BrokenPipe),
5871 SubcError::Auth {
5872 endpoint: "127.0.0.1:1".to_string(),
5873 source: subc_transport::AuthError::UnexpectedEof {
5874 stage: subc_transport::AuthStage::ServerProof,
5875 expected: 4,
5876 actual: 0,
5877 },
5878 },
5879 SubcError::Auth {
5880 endpoint: "127.0.0.1:1".to_string(),
5881 source: subc_transport::AuthError::Timeout {
5882 stage: subc_transport::AuthStage::ServerProof,
5883 deadline: AUTH_DEADLINE,
5884 },
5885 },
5886 ];
5887 for error in &transient_errors {
5888 assert_eq!(
5889 classify_attach_error(error),
5890 AttachErrorClass::Transient,
5891 "expected transient: {error}"
5892 );
5893 }
5894
5895 let permanent_errors = vec![
5896 attach_error(io::ErrorKind::PermissionDenied),
5897 auth_io_error(io::ErrorKind::InvalidData),
5898 SubcError::Auth {
5899 endpoint: "127.0.0.1:1".to_string(),
5900 source: subc_transport::AuthError::InvalidServerProof,
5901 },
5902 SubcError::Auth {
5903 endpoint: "127.0.0.1:1".to_string(),
5904 source: subc_transport::AuthError::DaemonIdMismatch,
5905 },
5906 SubcError::ConnectionFile {
5907 path: PathBuf::from("subc-connection.json"),
5908 source: subc_transport::ConnectionFileError::Invalid {
5909 reason: "constructed invalid file".to_string(),
5910 },
5911 },
5912 SubcError::NoEndpoint {
5913 path: PathBuf::from("subc-connection.json"),
5914 },
5915 SubcError::InvalidEndpoint {
5916 path: PathBuf::from("subc-connection.json"),
5917 endpoint: "not-an-ip:1234".to_string(),
5918 },
5919 ];
5920 for error in &permanent_errors {
5921 assert_eq!(
5922 classify_attach_error(error),
5923 AttachErrorClass::Permanent,
5924 "expected permanent: {error}"
5925 );
5926 }
5927 }
5928
5929 #[test]
5930 fn incompatible_wire_version_is_rejected_before_tcp_connect() {
5931 let conn_dir = tempfile::tempdir().expect("connection tempdir");
5932 let conn_path = conn_dir.path().join("subc-connection.json");
5933 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
5934 listener
5935 .set_nonblocking(true)
5936 .expect("set listener nonblocking");
5937 let port = listener.local_addr().expect("listener addr").port();
5938 connection_file::write_atomic(
5939 &conn_path,
5940 &connection_file::ConnectionInfo {
5941 schema: connection_file::SCHEMA_VERSION,
5942 wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
5943 endpoints: vec![connection_file::Endpoint {
5944 host: "127.0.0.1".to_string(),
5945 port,
5946 }],
5947 key: vec![0x42; subc_transport::KEY_LEN],
5948 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
5949 pid: std::process::id(),
5950 daemon_ver: "subc-test".to_string(),
5951 },
5952 )
5953 .expect("write connection file");
5954
5955 let runtime = tokio::runtime::Builder::new_current_thread()
5956 .enable_all()
5957 .build()
5958 .expect("test runtime");
5959 let result = runtime.block_on(connect_and_authenticate_with_policy(
5960 &conn_path,
5961 AttachRetryPolicy {
5962 budget: Duration::from_secs(1),
5963 initial_backoff: Duration::from_millis(5),
5964 max_backoff: Duration::from_millis(10),
5965 jitter_percent: 0,
5966 },
5967 ));
5968 assert!(matches!(
5969 result,
5970 Err(SubcError::ConnectionFile {
5971 source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
5972 ..
5973 })
5974 ));
5975 assert!(matches!(
5976 listener.accept(),
5977 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
5978 ));
5979 }
5980
5981 #[test]
5982 fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
5983 let conn_dir = tempfile::tempdir().expect("connection tempdir");
5984 let conn_path = conn_dir.path().join("subc-connection.json");
5985 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
5986 let port = listener.local_addr().expect("reserved addr").port();
5987 drop(listener);
5988 connection_file::write_atomic(
5989 &conn_path,
5990 &connection_file::ConnectionInfo {
5991 schema: connection_file::SCHEMA_VERSION,
5992 wire_version: Some(PROTOCOL_VERSION),
5993 endpoints: vec![connection_file::Endpoint {
5994 host: "127.0.0.1".to_string(),
5995 port,
5996 }],
5997 key: vec![0x42; subc_transport::KEY_LEN],
5998 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
5999 pid: std::process::id(),
6000 daemon_ver: "subc-test".to_string(),
6001 },
6002 )
6003 .expect("write connection file");
6004
6005 let policy = AttachRetryPolicy {
6006 budget: Duration::from_millis(40),
6007 initial_backoff: Duration::from_millis(5),
6008 max_backoff: Duration::from_millis(10),
6009 jitter_percent: 0,
6010 };
6011 let runtime = tokio::runtime::Builder::new_current_thread()
6012 .enable_all()
6013 .build()
6014 .expect("test runtime");
6015 let started_at = Instant::now();
6016 let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
6017 let elapsed = started_at.elapsed();
6018 let error = match result {
6019 Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
6020 Err(error) => error,
6021 };
6022
6023 assert!(matches!(error, SubcError::Connect { .. }), "{error}");
6024 assert!(
6025 elapsed >= Duration::from_millis(35),
6026 "retry budget ended too early: {elapsed:?}"
6027 );
6028 assert!(
6029 elapsed < Duration::from_secs(1),
6030 "retry budget was not bounded: {elapsed:?}"
6031 );
6032 }
6033
6034 fn due_maintenance_jobs_without_actor_context(
6035 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6036 budget: usize,
6037 pending_bind_roots: &HashSet<ProjectRootId>,
6038 ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
6039 due_maintenance_jobs(
6040 live_roots,
6041 None,
6042 &HashMap::new(),
6043 &HashSet::new(),
6044 budget,
6045 pending_bind_roots,
6046 )
6047 }
6048
6049 fn actor_ctx_with_dirty_search_index(
6050 root: &Path,
6051 storage: &Path,
6052 file_name: &str,
6053 old_contents: &str,
6054 new_contents: &str,
6055 ) -> (Arc<AppContext>, PathBuf, PathBuf) {
6056 let file = root.join(file_name);
6057 std::fs::write(&file, old_contents).expect("write source");
6058 let canonical_root = std::fs::canonicalize(root).expect("canonical root");
6059 let ctx = Arc::new(AppContext::new(
6060 Box::new(crate::parser::TreeSitterProvider::new()),
6061 Config {
6062 project_root: Some(root.to_path_buf()),
6063 storage_dir: Some(storage.to_path_buf()),
6064 ..Config::default()
6065 },
6066 ));
6067 ctx.set_canonical_cache_root(canonical_root.clone());
6068
6069 let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
6070 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
6071 let git_head = index.stored_git_head().map(str::to_owned);
6072 index.write_to_disk(&cache_dir, git_head.as_deref());
6073
6074 std::fs::write(&file, new_contents).expect("edit source");
6075 index.update_file(&file);
6076 *ctx.search_index()
6077 .write()
6078 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
6079 (ctx, canonical_root, cache_dir)
6080 }
6081
6082 #[test]
6083 fn graceful_shutdown_flushes_every_actor_search_index() {
6084 let storage = tempfile::tempdir().expect("storage tempdir");
6085 let (root1_dir, root1) = test_root("shutdown-flush-root-1");
6086 let (root2_dir, root2) = test_root("shutdown-flush-root-2");
6087 let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
6088 root1_dir.path(),
6089 storage.path(),
6090 "alpha.txt",
6091 "old actor one token\n",
6092 "new actor one token\n",
6093 );
6094 let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
6095 root2_dir.path(),
6096 storage.path(),
6097 "beta.txt",
6098 "old actor two token\n",
6099 "new actor two token\n",
6100 );
6101
6102 let executor = Executor::new();
6103 assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
6104 assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
6105
6106 flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
6107
6108 let mut restored1 =
6109 crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
6110 .expect("load flushed root one index");
6111 restored1.ready = true;
6112 assert_eq!(
6113 restored1
6114 .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
6115 .matches
6116 .len(),
6117 1,
6118 "graceful subc shutdown should flush the first root's trigram delta"
6119 );
6120
6121 let mut restored2 =
6122 crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
6123 .expect("load flushed root two index");
6124 restored2.ready = true;
6125 assert_eq!(
6126 restored2
6127 .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
6128 .matches
6129 .len(),
6130 1,
6131 "graceful subc shutdown should flush every registered root"
6132 );
6133 }
6134
6135 #[test]
6136 fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
6137 let _ = env_logger::builder().is_test(true).try_init();
6138 let (root_dir, root) = test_root("idle-root-reaper");
6139 let storage = tempfile::tempdir().expect("storage tempdir");
6140 std::fs::write(
6141 root_dir.path().join("main.rs"),
6142 "fn entry() { leaf(); }\nfn leaf() {}\n",
6143 )
6144 .expect("source file");
6145 let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
6146 let app = App::default_shared();
6147 let ctx = Arc::new(AppContext::from_app(
6148 Arc::clone(&app),
6149 Config {
6150 project_root: Some(canonical_root.clone()),
6151 storage_dir: Some(storage.path().to_path_buf()),
6152 callgraph_store: true,
6153 search_index: true,
6154 ..Config::default()
6155 },
6156 ));
6157 ctx.set_canonical_cache_root(canonical_root.clone());
6158 let project_key = crate::search_index::artifact_cache_key(&canonical_root);
6159 crate::root_cache::configure_artifact_access(&canonical_root, &project_key, false);
6160 assert!(ctx
6161 .ensure_callgraph_store()
6162 .expect("build callgraph store")
6163 .is_some());
6164
6165 let cache_dir =
6166 crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
6167 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
6168 let git_head = index.stored_git_head().map(str::to_owned);
6169 index.write_to_disk(&cache_dir, git_head.as_deref());
6170 *ctx.search_index()
6171 .write()
6172 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
6173 let seeded_generation =
6176 crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
6177 .expect("seeded artifact generation");
6178 crate::cache_freshness::record_verify_completed(
6179 &canonical_root,
6180 crate::cache_freshness::VerifyArtifact::Search,
6181 Some(seeded_generation),
6182 );
6183 assert!(
6184 matches!(
6185 crate::cache_freshness::warm_verify_plan(
6186 canonical_root.as_path(),
6187 crate::cache_freshness::VerifyArtifact::Search,
6188 Some(seeded_generation),
6189 ),
6190 crate::cache_freshness::WarmVerifyPlan::Skip
6191 ),
6192 "memo must be warm before eviction for the downgrade assertion to bite"
6193 );
6194
6195 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
6196 let _dispatch_tx = dispatch_tx;
6197 let shutdown = Arc::new(AtomicBool::new(false));
6198 let thread_shutdown = Arc::clone(&shutdown);
6199 let join = std::thread::spawn(move || {
6200 while !thread_shutdown.load(Ordering::SeqCst) {
6201 std::thread::yield_now();
6202 }
6203 });
6204 ctx.install_watcher_runtime(
6205 dispatch_rx,
6206 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
6207 );
6208 wait_for_watcher_count(&ctx, 1);
6209
6210 let executor = Arc::new(Executor::new());
6211 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6212 ctx.mark_subc_unbound();
6213 let mut live_roots = HashMap::new();
6214 let mut meta = RootMeta::new(Instant::now());
6215 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
6216 meta.unbound_quiesced = true;
6217 live_roots.insert(root.clone(), meta);
6218
6219 let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
6220 assert!(message.contains("evicted idle root"));
6221 assert!(message.contains("freed ~"));
6222 assert!(message.contains("semantic"));
6223 assert!(!message.contains("semantic not estimated retained"));
6224 assert!(message.contains("trigram"));
6225 assert!(message.contains("retained: bash"));
6226 assert!(message.contains("parser_pool"));
6227
6228 assert_eq!(
6229 reap_idle_roots(
6230 Instant::now(),
6231 &mut live_roots,
6232 &HashMap::new(),
6233 &HashMap::new(),
6234 &executor,
6235 &DispatchPathMetrics::new(),
6236 )
6237 .evicted,
6238 1
6239 );
6240 assert!(ctx.search_index().read().unwrap().is_none());
6241 wait_for_watcher_count(&ctx, 0);
6242 assert!(
6247 matches!(
6248 crate::cache_freshness::warm_verify_plan(
6249 canonical_root.as_path(),
6250 crate::cache_freshness::VerifyArtifact::Search,
6251 Some(seeded_generation),
6252 ),
6253 crate::cache_freshness::WarmVerifyPlan::Strict
6254 ),
6255 "idle eviction must force strict re-verification"
6256 );
6257 assert!(
6258 crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
6259 );
6260 ctx.mark_subc_bound();
6261 assert!(ctx
6262 .ensure_callgraph_store()
6263 .expect("reopen callgraph store")
6264 .is_some());
6265 assert!(live_roots[&root].idle_artifacts_evicted);
6266 }
6267
6268 #[test]
6269 fn idle_root_reaper_applies_ttl_to_unbound_roots() {
6270 let (_root_dir, root) = test_root("idle-root-ttl-gate");
6271 let ctx = test_ctx();
6272 let executor = Arc::new(Executor::new());
6273 assert!(executor.register_actor(root.clone(), ctx));
6274 let ctx = executor.actor_context(&root).expect("actor context");
6275 ctx.mark_subc_unbound();
6276 let now = Instant::now();
6277 let mut meta = RootMeta::new(now);
6278 meta.unbound_quiesced = true;
6279 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6280
6281 assert_eq!(
6285 reap_idle_roots(
6286 now,
6287 &mut live_roots,
6288 &HashMap::new(),
6289 &HashMap::new(),
6290 &executor,
6291 &DispatchPathMetrics::new(),
6292 )
6293 .evicted,
6294 0
6295 );
6296 assert!(!live_roots[&root].idle_artifacts_evicted);
6297
6298 ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
6304 assert_eq!(
6305 reap_idle_roots(
6306 now + IDLE_ROOT_TTL,
6307 &mut live_roots,
6308 &HashMap::new(),
6309 &HashMap::new(),
6310 &executor,
6311 &DispatchPathMetrics::new(),
6312 )
6313 .evicted,
6314 1
6315 );
6316 assert!(live_roots[&root].idle_artifacts_evicted);
6317 assert!(
6318 ctx.take_pending_search_index_paths().is_empty(),
6319 "TTL eviction must dispose retained pending reconciliation paths"
6320 );
6321 }
6322
6323 #[test]
6324 fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
6325 let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
6326 let ctx = test_ctx();
6327 let executor = Arc::new(Executor::new());
6328 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6329 ctx.mark_subc_unbound();
6330
6331 let pending = root.as_path().join("edited-while-unbound.rs");
6337 ctx.add_pending_search_index_paths([pending.clone()]);
6338 let dirty_source = root.as_path().join("dirty.rs");
6339 std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
6340 let mut dirty = crate::search_index::SearchIndex::new();
6341 dirty.ready = true;
6342 dirty.update_file(&dirty_source);
6343 *ctx.search_index()
6344 .write()
6345 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
6346 assert!(ctx.artifact_eviction_blocked());
6347
6348 let mut live_roots = HashMap::new();
6349 let mut meta = RootMeta::new(Instant::now());
6350 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
6351 meta.unbound_quiesced = true;
6352 live_roots.insert(root.clone(), meta);
6353
6354 assert_eq!(
6355 reap_idle_roots(
6356 Instant::now(),
6357 &mut live_roots,
6358 &HashMap::new(),
6359 &HashMap::new(),
6360 &executor,
6361 &DispatchPathMetrics::new(),
6362 )
6363 .evicted,
6364 0,
6365 "the dirty index must still block this eviction"
6366 );
6367 assert_eq!(
6368 ctx.take_pending_search_index_paths(),
6369 vec![pending],
6370 "a blocked eviction must restore the taken pending paths"
6371 );
6372 }
6373
6374 #[test]
6375 fn idle_reap_with_bound_route_keeps_watcher_running() {
6376 let (_root_dir, root) = test_root("bound-root-reap-gate");
6377 let ctx = test_ctx();
6378 let executor = Arc::new(Executor::new());
6379 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6380
6381 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
6382 let _dispatch_tx = dispatch_tx;
6383 let shutdown = Arc::new(AtomicBool::new(false));
6384 let thread_shutdown = Arc::clone(&shutdown);
6385 let join = std::thread::spawn(move || {
6386 while !thread_shutdown.load(Ordering::SeqCst) {
6387 std::thread::yield_now();
6388 }
6389 });
6390 ctx.install_watcher_runtime(
6391 dispatch_rx,
6392 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
6393 );
6394
6395 let mut meta = RootMeta::new(Instant::now());
6396 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
6397 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6398 let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
6399 assert_eq!(
6400 reap_idle_roots(
6401 Instant::now(),
6402 &mut live_roots,
6403 &HashMap::new(),
6404 &bound,
6405 &executor,
6406 &DispatchPathMetrics::new(),
6407 )
6408 .evicted,
6409 0
6410 );
6411 wait_for_watcher_count(&ctx, 1);
6412 ctx.stop_watcher_runtime_in_background();
6413 wait_for_watcher_count(&ctx, 0);
6414 }
6415
6416 #[test]
6417 fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
6418 let (root_dir, root) = test_root("deleted-bound-root-reap");
6419 let executor = Arc::new(Executor::new());
6420 assert!(executor.register_actor(root.clone(), test_ctx()));
6421 root_dir.close().expect("delete project root");
6422
6423 let route = route_key(19, 3);
6424 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6425 let cancel_signal = PersistentCancelSignal::new();
6426 let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
6427 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
6428 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6429 let mut route_bash_cancels = HashMap::from([(
6430 route,
6431 bash::RouteBashCancel {
6432 token: cancel_signal.clone(),
6433 active_waits: 0,
6434 },
6435 )]);
6436 let metrics = DispatchPathMetrics::new();
6437
6438 let first = reap_idle_roots(
6439 Instant::now(),
6440 &mut live_roots,
6441 &HashMap::new(),
6442 &root_channels,
6443 &executor,
6444 &metrics,
6445 );
6446 assert!(first.forgotten_deleted_roots.is_empty());
6447 assert!(executor.actor_registered(&root));
6448
6449 let mut forgotten = Vec::new();
6450 for _ in 0..100 {
6451 let outcome = reap_idle_roots(
6452 Instant::now(),
6453 &mut live_roots,
6454 &HashMap::new(),
6455 &root_channels,
6456 &executor,
6457 &metrics,
6458 );
6459 if !outcome.forgotten_deleted_roots.is_empty() {
6460 forgotten = outcome.forgotten_deleted_roots;
6461 break;
6462 }
6463 std::thread::sleep(Duration::from_millis(10));
6464 }
6465 assert_eq!(forgotten, vec![root.clone()]);
6466 assert!(!executor.actor_registered(&root));
6467
6468 let mut retry_buffer = HashMap::new();
6469 let mut reclaimed_routes = ReclaimedRoutes::default();
6470 let mut session_identity = HashMap::new();
6471 let mut push_buffer = HashMap::new();
6472 let mut bg_subs = HashMap::from([(
6473 route,
6474 BgSub {
6475 corr: 77,
6476 ver: PROTOCOL_VERSION,
6477 flags: control_flags(),
6478 root: root.clone(),
6479 session: "deleted-route".to_string(),
6480 },
6481 )]);
6482 let mut bg_sub_by_session = HashMap::from([(
6483 (root.clone(), "deleted-route".to_string()),
6484 HashSet::from([route]),
6485 )]);
6486 let mut bg_wake_pending = HashSet::from([route]);
6487 let mut bg_wake_epoch = HashMap::new();
6488 let mut pending_bash_asks = HashMap::new();
6489 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6490 health::take_bg_observability_logs_for_test();
6491 purge_deleted_root_residents(
6492 &root,
6493 &mut routes,
6494 &mut root_channels,
6495 &mut installed_route_epochs,
6496 &mut route_bash_cancels,
6497 &active_tool_calls,
6498 executor.as_ref(),
6499 &mut retry_buffer,
6500 &mut reclaimed_routes,
6501 &mut session_identity,
6502 &mut push_buffer,
6503 &mut bg_subs,
6504 &mut bg_sub_by_session,
6505 &mut bg_wake_pending,
6506 &mut bg_wake_epoch,
6507 &mut pending_bash_asks,
6508 &metrics,
6509 );
6510
6511 assert!(routes.is_empty());
6512 assert!(root_channels.is_empty());
6513 assert!(installed_route_epochs.is_empty());
6514 assert!(route_bash_cancels.is_empty());
6515 assert!(reclaimed_routes.contains(route));
6516 assert!(cancel_signal.is_cancelled());
6517 assert_eq!(
6518 health::take_bg_observability_logs_for_test(),
6519 vec![format!(
6520 "subc bg subscription: ended root={} session=deleted-route channel=19@3 cause=root-reclaim suppressed=0",
6521 root.as_path().display()
6522 )]
6523 );
6524 }
6525
6526 #[test]
6533 fn live_root_with_bound_route_is_never_reclaimed() {
6534 let (_root_dir, root) = test_root("live-bound-root-retained");
6535 let ctx = test_ctx();
6536 ctx.mark_subc_unbound();
6537 let executor = Arc::new(Executor::new());
6538 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6539
6540 let route = RouteChannel {
6541 channel: 7,
6542 epoch: 1,
6543 };
6544 let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
6545 meta.unbound_quiesced = true;
6546 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6547 let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
6548
6549 for _ in 0..3 {
6554 let outcome = reap_idle_roots(
6555 Instant::now(),
6556 &mut live_roots,
6557 &HashMap::new(),
6558 &root_channels,
6559 &executor,
6560 &DispatchPathMetrics::new(),
6561 );
6562 assert!(
6563 outcome.forgotten_deleted_roots.is_empty(),
6564 "a root whose directory exists must never be forgotten"
6565 );
6566 }
6567
6568 assert!(live_roots.contains_key(&root), "live root must be retained");
6569 assert!(
6570 executor.actor_registered(&root),
6571 "live root's actor must survive"
6572 );
6573 assert!(
6574 root.as_path().exists(),
6575 "test vehicle must keep the directory alive; otherwise this control proves nothing"
6576 );
6577 }
6578
6579 #[test]
6580 fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
6581 let (root_dir, root) = test_root("deleted-root-first-observation");
6582 let ctx = test_ctx();
6583 ctx.mark_subc_unbound();
6584 let executor = Arc::new(Executor::new());
6585 assert!(executor.register_actor(root.clone(), ctx));
6586 root_dir.close().expect("delete project root");
6587
6588 let mut meta = RootMeta::new(Instant::now());
6589 meta.unbound_quiesced = true;
6590 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6591 let outcome = reap_idle_roots(
6592 Instant::now(),
6593 &mut live_roots,
6594 &HashMap::new(),
6595 &HashMap::new(),
6596 &executor,
6597 &DispatchPathMetrics::new(),
6598 );
6599
6600 assert!(outcome.forgotten_deleted_roots.is_empty());
6601 assert!(live_roots.contains_key(&root));
6602 assert!(executor.actor_registered(&root));
6603 }
6604
6605 fn spawn_background_for_root(
6606 ctx: &AppContext,
6607 root: &ProjectRootId,
6608 storage: &tempfile::TempDir,
6609 session_id: &str,
6610 ) -> (String, u32) {
6611 let command = if cfg!(windows) {
6617 "ping -n 31 127.0.0.1 > nul"
6619 } else {
6620 "sleep 30"
6621 };
6622 let task_id = ctx
6623 .bash_background()
6624 .spawn(
6625 crate::sandbox_spawn::SpawnPlan::Unsandboxed,
6626 command,
6627 session_id.to_string(),
6628 storage.path().to_path_buf(),
6629 HashMap::new(),
6630 Some(Duration::from_secs(60)),
6631 storage.path().to_path_buf(),
6632 8,
6633 true,
6634 false,
6635 Some(root.as_path().to_path_buf()),
6636 )
6637 .expect("spawn background task");
6638 let snapshot = ctx
6639 .bash_background()
6640 .status(
6641 &task_id,
6642 session_id,
6643 Some(root.as_path()),
6644 Some(storage.path()),
6645 0,
6646 )
6647 .expect("background task status");
6648 (task_id, snapshot.child_pid.expect("background child pid"))
6649 }
6650
6651 fn wait_for_background_exit(pid: u32) {
6652 let deadline = Instant::now() + Duration::from_secs(5);
6653 while crate::bash_background::process::is_process_alive(pid) {
6654 assert!(
6655 Instant::now() < deadline,
6656 "background task process survived kill"
6657 );
6658 std::thread::sleep(Duration::from_millis(20));
6659 }
6660 }
6661
6662 #[test]
6663 fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
6664 let (root_dir, root) = test_root("deleted-root-background-task");
6665 let storage = tempfile::tempdir().expect("task storage");
6666 let ctx = test_ctx();
6667 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
6668 assert!(crate::bash_background::process::is_process_alive(pid));
6669
6670 let executor = Arc::new(Executor::new());
6671 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6672 assert!(executor.actor_is_idle(&root));
6673 root_dir.close().expect("delete project root");
6674 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6675 let pending_binds = HashMap::new();
6676 let root_channels = HashMap::new();
6677 let metrics = DispatchPathMetrics::new();
6678
6679 let first = reap_idle_roots(
6680 Instant::now(),
6681 &mut live_roots,
6682 &pending_binds,
6683 &root_channels,
6684 &executor,
6685 &metrics,
6686 );
6687 assert!(first.forgotten_deleted_roots.is_empty());
6688 assert!(crate::bash_background::process::is_process_alive(pid));
6689
6690 let outcome = reap_until_forgotten(
6691 &root,
6692 &mut live_roots,
6693 &pending_binds,
6694 &root_channels,
6695 &executor,
6696 &metrics,
6697 );
6698 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
6699 wait_for_background_exit(pid);
6700
6701 let snapshot = ctx
6702 .bash_background()
6703 .status(
6704 &task_id,
6705 "reclaim-session",
6706 Some(root.as_path()),
6707 Some(storage.path()),
6708 0,
6709 )
6710 .expect("reclaimed task status");
6711 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
6712 assert_eq!(
6713 snapshot.info.status_reason.as_deref(),
6714 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
6715 );
6716 assert_eq!(
6717 serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
6718 crate::bash_background::registry::ROOT_RECLAIMED_REASON
6719 );
6720 let completion = ctx
6721 .bash_background()
6722 .drain_completions_for_session(Some("reclaim-session"))
6723 .pop()
6724 .expect("reclaimed task completion");
6725 assert_eq!(
6726 completion.status_reason.as_deref(),
6727 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
6728 );
6729 }
6730
6731 #[test]
6732 fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
6733 let (root_dir, root) = test_root("existing-root-background-task");
6734 let storage = tempfile::tempdir().expect("task storage");
6735 let ctx = test_ctx();
6736 ctx.mark_subc_unbound();
6737 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
6738
6739 let executor = Arc::new(Executor::new());
6740 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6741 let mut meta = RootMeta::new(
6742 Instant::now()
6743 .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
6744 .expect("old root timestamp"),
6745 );
6746 meta.unbound_quiesced = true;
6747 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6748 let pending_binds = HashMap::new();
6749 let root_channels = HashMap::new();
6750 let metrics = DispatchPathMetrics::new();
6751
6752 for _ in 0..8 {
6753 reap_idle_roots(
6754 Instant::now(),
6755 &mut live_roots,
6756 &pending_binds,
6757 &root_channels,
6758 &executor,
6759 &metrics,
6760 );
6761 std::thread::sleep(Duration::from_millis(10));
6762 }
6763 assert!(root_dir.path().exists());
6764 assert!(crate::bash_background::process::is_process_alive(pid));
6765 let snapshot = ctx
6766 .bash_background()
6767 .status(
6768 &task_id,
6769 "existing-session",
6770 Some(root.as_path()),
6771 Some(storage.path()),
6772 0,
6773 )
6774 .expect("existing task status");
6775 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
6776 let _ = ctx.bash_background().kill(&task_id, "existing-session");
6777 wait_for_background_exit(pid);
6778 }
6779
6780 #[test]
6781 fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
6782 let (root_dir, root) = test_root("restored-root-background-task");
6783 let storage = tempfile::tempdir().expect("task storage");
6784 let ctx = test_ctx();
6785 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
6786
6787 let executor = Arc::new(Executor::new());
6788 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6789 root_dir.close().expect("delete project root");
6790 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6791 let pending_binds = HashMap::new();
6792 let root_channels = HashMap::new();
6793 let metrics = DispatchPathMetrics::new();
6794
6795 let first = reap_idle_roots(
6796 Instant::now(),
6797 &mut live_roots,
6798 &pending_binds,
6799 &root_channels,
6800 &executor,
6801 &metrics,
6802 );
6803 assert!(first.forgotten_deleted_roots.is_empty());
6804 std::fs::create_dir_all(root.as_path()).expect("restore project root");
6805 let second = reap_idle_roots(
6806 Instant::now(),
6807 &mut live_roots,
6808 &pending_binds,
6809 &root_channels,
6810 &executor,
6811 &metrics,
6812 );
6813 assert!(second.forgotten_deleted_roots.is_empty());
6814 assert!(crate::bash_background::process::is_process_alive(pid));
6815 let _ = ctx.bash_background().kill(&task_id, "restored-session");
6816 wait_for_background_exit(pid);
6817 }
6818
6819 #[test]
6820 fn observing_root_again_resets_deleted_sweep_confirmation() {
6821 let (root_dir, root) = test_root("deleted-root-observation-reset");
6822 let ctx = test_ctx();
6823 ctx.mark_subc_unbound();
6824 let executor = Arc::new(Executor::new());
6825 assert!(executor.register_actor(root.clone(), ctx));
6826 root_dir.close().expect("delete project root");
6827
6828 let mut meta = RootMeta::new(Instant::now());
6829 meta.unbound_quiesced = true;
6830 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6831 let pending_binds = HashMap::new();
6832 let root_channels = HashMap::new();
6833 let metrics = DispatchPathMetrics::new();
6834
6835 let first = reap_idle_roots(
6836 Instant::now(),
6837 &mut live_roots,
6838 &pending_binds,
6839 &root_channels,
6840 &executor,
6841 &metrics,
6842 );
6843 assert!(first.forgotten_deleted_roots.is_empty());
6844
6845 std::fs::create_dir_all(root.as_path()).expect("restore project root");
6846 reap_idle_roots(
6847 Instant::now(),
6848 &mut live_roots,
6849 &pending_binds,
6850 &root_channels,
6851 &executor,
6852 &metrics,
6853 );
6854 std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
6855
6856 let after_reset = reap_idle_roots(
6857 Instant::now(),
6858 &mut live_roots,
6859 &pending_binds,
6860 &root_channels,
6861 &executor,
6862 &metrics,
6863 );
6864 assert!(after_reset.forgotten_deleted_roots.is_empty());
6865 assert!(live_roots.contains_key(&root));
6866 assert!(executor.actor_registered(&root));
6867 }
6868
6869 #[test]
6870 fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
6871 let (root_dir, root) = test_root("deleted-root-reap");
6872 let app = App::default_shared();
6873 let ctx = Arc::new(AppContext::from_app(
6874 Arc::clone(&app),
6875 Config {
6876 project_root: Some(root.as_path().to_path_buf()),
6877 ..Config::default()
6878 },
6879 ));
6880 ctx.set_canonical_cache_root(root.as_path().to_path_buf());
6881 ctx.mark_subc_unbound();
6882 let executor = Arc::new(Executor::new());
6883 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6884 assert_eq!(app.actor_root_count(), 1);
6885 drop(ctx);
6886 root_dir.close().expect("delete project root");
6887
6888 let mut meta = RootMeta::new(Instant::now());
6889 meta.unbound_quiesced = true;
6890 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6891 let outcome = reap_until_forgotten(
6892 &root,
6893 &mut live_roots,
6894 &HashMap::new(),
6895 &HashMap::new(),
6896 &executor,
6897 &DispatchPathMetrics::new(),
6898 );
6899 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
6900 assert!(!executor.actor_registered(&root));
6901 assert!(!live_roots.contains_key(&root));
6902 wait_for_actor_root_count(&app, 0);
6903
6904 let status_ctx = AppContext::from_app(app, Config::default());
6905 let status = status_ctx.build_status_snapshot();
6906 assert_eq!(status["runtime"]["live_actor_roots"], 0);
6907 assert_eq!(status["runtime"]["open_routes"], 0);
6908 }
6909
6910 #[test]
6911 fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
6912 let (root_dir, root) = test_root("deleted-root-reap-census");
6913 let executor = Arc::new(Executor::new());
6914 assert!(executor.register_actor(root.clone(), test_ctx()));
6915 root_dir.close().expect("delete project root");
6916
6917 let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
6918 let metrics = DispatchPathMetrics::new();
6919 let outcome = reap_idle_roots(
6920 Instant::now(),
6921 &mut live_roots,
6922 &HashMap::new(),
6923 &HashMap::new(),
6924 &executor,
6925 &metrics,
6926 );
6927 assert_eq!(outcome.evicted, 0);
6928
6929 let app = crate::context::App::default_shared();
6930 let health_rollup_cache = HealthRollupCache::new();
6931 health_rollup_cache.refresh(&executor, &app);
6932 let report = build_health_report(
6933 &health_rollup_cache,
6934 &executor,
6935 &HashMap::new(),
6936 &metrics,
6937 &app,
6938 );
6939 let reap = report
6940 .metrics
6941 .as_ref()
6942 .and_then(|metrics| metrics.get("reap"))
6943 .expect("reap health metrics");
6944 assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
6945 assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
6946 assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
6947 assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
6948 }
6949
6950 #[test]
6951 fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
6952 let (root_dir, root) = test_root("connection-exit-deleted-root");
6953 let executor = Arc::new(Executor::new());
6954 assert!(executor.register_actor(root.clone(), test_ctx()));
6955
6956 let route = route_key(11, 1);
6957 let mut meta = RootMeta::new(Instant::now());
6958 meta.maintenance_pending = true;
6959 meta.maintenance_queued_kinds
6960 .push_back(MaintenanceDrainKind::CompletionDrains);
6961 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6962 let mut pending_binds = HashMap::new();
6963 let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
6964 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
6965 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6966 let mut route_bash_cancels = HashMap::new();
6967 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6968
6969 quiesce_connection_roots(
6970 &mut live_roots,
6971 &mut pending_binds,
6972 &mut routes,
6973 &mut root_channels,
6974 &mut installed_route_epochs,
6975 &mut route_bash_cancels,
6976 &active_tool_calls,
6977 &executor,
6978 );
6979 assert!(live_roots[&root].unbound_quiesced);
6980 assert!(!live_roots[&root].maintenance_pending);
6981 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
6982 assert!(routes.is_empty());
6983 assert!(root_channels.is_empty());
6984
6985 root_dir.close().expect("delete project root");
6986 let metrics = DispatchPathMetrics::new();
6987 let outcome = reap_until_forgotten(
6988 &root,
6989 &mut live_roots,
6990 &pending_binds,
6991 &root_channels,
6992 &executor,
6993 &metrics,
6994 );
6995 let mut session_identity = HashMap::new();
6996 let mut push_buffer = HashMap::new();
6997 let mut bg_subs = HashMap::new();
6998 let mut bg_sub_by_session = HashMap::new();
6999 let mut bg_wake_pending = HashSet::new();
7000 let mut bg_wake_epoch = HashMap::new();
7001 let mut pending_bash_asks = HashMap::new();
7002 let mut retry_buffer = HashMap::new();
7003 let mut reclaimed_routes = ReclaimedRoutes::default();
7004 for forgotten in &outcome.forgotten_deleted_roots {
7005 purge_deleted_root_residents(
7006 forgotten,
7007 &mut routes,
7008 &mut root_channels,
7009 &mut installed_route_epochs,
7010 &mut route_bash_cancels,
7011 &active_tool_calls,
7012 executor.as_ref(),
7013 &mut retry_buffer,
7014 &mut reclaimed_routes,
7015 &mut session_identity,
7016 &mut push_buffer,
7017 &mut bg_subs,
7018 &mut bg_sub_by_session,
7019 &mut bg_wake_pending,
7020 &mut bg_wake_epoch,
7021 &mut pending_bash_asks,
7022 &metrics,
7023 );
7024 }
7025
7026 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
7027 assert!(!executor.actor_registered(&root));
7028 assert!(!live_roots.contains_key(&root));
7029 }
7030
7031 #[test]
7032 fn unbound_root_quiesces_maintenance_without_removing_actor() {
7033 let (_root_dir, root) = test_root("unbound-root-quiesce");
7034 let ctx = test_ctx();
7035 let executor = Arc::new(Executor::new());
7036 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7037 let mut meta = RootMeta::new(Instant::now());
7038 meta.maintenance_pending = true;
7039 meta.maintenance_jobs_in_flight = 1;
7040 meta.maintenance_queued_kinds
7041 .push_back(MaintenanceDrainKind::ConfigureTail);
7042 let mut live_roots = HashMap::from([(root.clone(), meta)]);
7043 *ctx.search_index()
7045 .write()
7046 .unwrap_or_else(std::sync::PoisonError::into_inner) =
7047 Some(crate::search_index::SearchIndex::new());
7048 ctx.set_cache_writer_capabilities(true, true);
7049 let pending = root.as_path().join("pending.rs");
7050 ctx.add_pending_search_index_paths([pending.clone()]);
7051 let canonical_root = root.as_path().to_path_buf();
7055 let artifact = canonical_root.join("cache.bin");
7056 std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
7057 let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
7058 crate::cache_freshness::record_verify_completed(
7059 &canonical_root,
7060 crate::cache_freshness::VerifyArtifact::Search,
7061 seeded_generation,
7062 );
7063 assert!(matches!(
7064 crate::cache_freshness::warm_verify_plan(
7065 &canonical_root,
7066 crate::cache_freshness::VerifyArtifact::Search,
7067 seeded_generation,
7068 ),
7069 crate::cache_freshness::WarmVerifyPlan::Skip
7070 ));
7071 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
7074 let _dispatch_tx = dispatch_tx;
7075 let shutdown = Arc::new(AtomicBool::new(false));
7076 let thread_shutdown = Arc::clone(&shutdown);
7077 let join = std::thread::spawn(move || {
7078 while !thread_shutdown.load(Ordering::SeqCst) {
7079 std::thread::yield_now();
7080 }
7081 });
7082 ctx.install_watcher_runtime(
7083 dispatch_rx,
7084 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
7085 );
7086 assert!(ctx.watcher_runtime_active());
7087
7088 quiesce_unbound_root(&root, &mut live_roots, &executor);
7089 let meta = &live_roots[&root];
7090 assert!(meta.unbound_quiesced);
7091 assert!(ctx.subc_unbound_quiesced());
7092 assert!(meta.maintenance_pending);
7093 assert!(meta.maintenance_queued_kinds.is_empty());
7094 assert!(executor.actor_registered(&root));
7095 assert!(
7099 ctx.search_index()
7100 .read()
7101 .unwrap_or_else(std::sync::PoisonError::into_inner)
7102 .is_some(),
7103 "quiesce must not evict resident artifacts"
7104 );
7105 assert_eq!(
7106 ctx.pending_callgraph_store_force_token(),
7107 None,
7108 "quiesce must not force a callgraph rebuild"
7109 );
7110 assert_eq!(
7111 ctx.take_pending_search_index_paths(),
7112 vec![pending],
7113 "quiesce must retain pending watcher-derived paths"
7114 );
7115 assert!(
7116 matches!(
7117 crate::cache_freshness::warm_verify_plan(
7118 &canonical_root,
7119 crate::cache_freshness::VerifyArtifact::Search,
7120 seeded_generation,
7121 ),
7122 crate::cache_freshness::WarmVerifyPlan::Skip
7123 ),
7124 "quiesce must not invalidate the warm verify memo"
7125 );
7126 assert!(
7127 ctx.watcher_runtime_active(),
7128 "quiesce must not stop a running watcher"
7129 );
7130 ctx.stop_watcher_runtime();
7131
7132 let meta = live_roots.get_mut(&root).expect("root metadata");
7133 note_maintenance_completion(
7134 meta,
7135 Some(MaintenanceDrainKind::ConfigureTail),
7136 false,
7137 meta.unbound_quiesced,
7138 );
7139 assert!(!meta.maintenance_pending);
7140 assert!(meta.maintenance_queued_kinds.is_empty());
7141 }
7142
7143 #[test]
7144 fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
7145 let (_dir, root) = test_root("same-root-replacement");
7146 let route = route_key(7, 1);
7147 let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
7148 let root_channels = HashMap::new();
7149
7150 assert!(!route_removal_will_quiesce_root(
7151 &root,
7152 route,
7153 &installed_channels,
7154 false,
7155 Some(&root),
7156 ));
7157 assert!(route_removal_will_quiesce_root(
7158 &root,
7159 route,
7160 &installed_channels,
7161 false,
7162 None,
7163 ));
7164 assert!(!should_quiesce_removed_root(
7165 &root,
7166 &root_channels,
7167 false,
7168 Some(&root),
7169 ));
7170 assert!(should_quiesce_removed_root(
7171 &root,
7172 &root_channels,
7173 false,
7174 None,
7175 ));
7176 assert!(!should_quiesce_removed_root(
7177 &root,
7178 &root_channels,
7179 true,
7180 None,
7181 ));
7182 }
7183
7184 #[test]
7185 fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
7186 let (_root_dir, root) = test_root("unbound-root-route-count");
7187 let executor = Arc::new(Executor::new());
7188 assert!(executor.register_actor(root.clone(), test_ctx()));
7189 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
7190 let mut root_channels = HashMap::from([(
7191 root.clone(),
7192 HashSet::from([route_key(7, 1), route_key(8, 1)]),
7193 )]);
7194
7195 remove_root_channel(&mut root_channels, &root, route_key(7, 1));
7196 if !root_channels.contains_key(&root) {
7197 quiesce_unbound_root(&root, &mut live_roots, &executor);
7198 }
7199 assert!(!live_roots[&root].unbound_quiesced);
7200
7201 remove_root_channel(&mut root_channels, &root, route_key(8, 1));
7202 if !root_channels.contains_key(&root) {
7203 quiesce_unbound_root(&root, &mut live_roots, &executor);
7204 }
7205 assert!(live_roots[&root].unbound_quiesced);
7206
7207 live_roots
7208 .get_mut(&root)
7209 .expect("root metadata")
7210 .note_activity();
7211 assert!(
7212 live_roots[&root].unbound_quiesced,
7213 "late asynchronous activity must not reactivate an unbound root"
7214 );
7215
7216 live_roots
7217 .get_mut(&root)
7218 .expect("root metadata")
7219 .reactivate_bound();
7220 assert!(!live_roots[&root].unbound_quiesced);
7221 }
7222
7223 #[test]
7224 fn allocator_pressure_relief_requires_every_root_to_be_idle() {
7225 let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
7226 let (_active_dir, active_root) = test_root("allocator-relief-active");
7227 let now = Instant::now();
7228 let mut live_roots = HashMap::new();
7229 let mut idle = RootMeta::new(now);
7230 idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
7231 live_roots.insert(idle_root, idle);
7232 assert!(process_has_been_idle(now, &live_roots));
7233
7234 live_roots.insert(active_root.clone(), RootMeta::new(now));
7235 assert!(!process_has_been_idle(now, &live_roots));
7236
7237 let active = live_roots
7238 .get_mut(&active_root)
7239 .expect("active root metadata");
7240 active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
7241 active.active_bash_waits = 1;
7242 assert!(!process_has_been_idle(now, &live_roots));
7243 }
7244
7245 #[test]
7246 fn pressure_relief_log_reports_before_and_after_measurements() {
7247 let allocator = crate::memory::AllocatorMemorySnapshot {
7248 status: "measured",
7249 bytes_in_use: Some(8 * 1024 * 1024),
7250 size_allocated: Some(12 * 1024 * 1024),
7251 retained_slack_bytes: Some(4 * 1024 * 1024),
7252 not_estimated: None,
7253 };
7254 let relief = crate::memory::AllocatorPressureRelief {
7255 bytes_released: 3 * 1024 * 1024,
7256 rss_before_bytes: Some(20 * 1024 * 1024),
7257 rss_after_bytes: Some(17 * 1024 * 1024),
7258 allocator_before: allocator.clone(),
7259 allocator_after: crate::memory::AllocatorMemorySnapshot {
7260 size_allocated: Some(9 * 1024 * 1024),
7261 retained_slack_bytes: Some(1024 * 1024),
7262 ..allocator
7263 },
7264 };
7265 let message = pressure_relief_label(&relief);
7266 assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
7267 assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
7268 assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
7269 assert!(message.contains("reported 3.0 MB released"));
7270 }
7271
7272 #[test]
7273 fn due_maintenance_jobs_skip_poisoned_roots() {
7274 let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
7275 let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
7276 let mut live_roots = HashMap::new();
7277 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
7278 let mut poisoned_meta = RootMeta::new(Instant::now());
7279 poisoned_meta.maintenance_poisoned = true;
7280 live_roots.insert(poisoned_root.clone(), poisoned_meta);
7281
7282 let (due, deferred) = due_maintenance_jobs_without_actor_context(
7283 &mut live_roots,
7284 MAINTENANCE_SUBMIT_BUDGET,
7285 &HashSet::new(),
7286 );
7287
7288 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
7289 assert!(due.iter().all(|(root, _)| root == &healthy_root));
7290 assert!(!deferred);
7291 assert!(live_roots[&healthy_root].maintenance_pending);
7292 assert_eq!(
7293 live_roots[&healthy_root].maintenance_jobs_in_flight,
7294 INITIAL_MAINTENANCE_JOB_COUNT
7295 );
7296 assert!(!live_roots[&poisoned_root].maintenance_pending);
7297 }
7298
7299 #[test]
7300 fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
7301 let (_dir, root) = test_root("maintenance-unbound");
7302 let mut meta = RootMeta::new(Instant::now());
7303 meta.unbound_quiesced = true;
7304 let mut live_roots = HashMap::from([(root.clone(), meta)]);
7305
7306 let (due, deferred) = due_maintenance_jobs_without_actor_context(
7307 &mut live_roots,
7308 MAINTENANCE_SUBMIT_BUDGET,
7309 &HashSet::new(),
7310 );
7311
7312 assert!(due.is_empty());
7313 assert!(!deferred);
7314 assert!(!live_roots[&root].maintenance_pending);
7315 }
7316
7317 #[test]
7318 fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
7319 let (_dir, root) = test_root("maintenance-idle-bg-subscription");
7320 let ctx = test_ctx();
7321 assert!(!ctx.completion_drains_have_work());
7322
7323 let executor = Executor::new();
7324 assert!(executor.register_actor(root.clone(), ctx));
7325 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
7326 let session = "idle-session".to_string();
7327 let channel = route_key(17, 1);
7328 let metrics = DispatchPathMetrics::new();
7329 let bg_sub_by_session =
7330 HashMap::from([((root.clone(), session.clone()), HashSet::from([channel]))]);
7331 let mut bg_wake_pending = HashSet::new();
7332
7333 let (idle_tick_jobs, deferred) = due_maintenance_jobs(
7334 &mut live_roots,
7335 Some(&executor),
7336 &bg_sub_by_session,
7337 &bg_wake_pending,
7338 MAINTENANCE_SUBMIT_BUDGET,
7339 &HashSet::new(),
7340 );
7341 assert!(idle_tick_jobs.is_empty());
7342 assert!(!deferred);
7343 assert!(!live_roots[&root].maintenance_pending);
7344
7345 let mut bg_wake_epoch = HashMap::new();
7348 push::arm_bg_wake(
7349 root.clone(),
7350 session,
7351 channel,
7352 &mut bg_wake_pending,
7353 &mut bg_wake_epoch,
7354 &metrics,
7355 );
7356 let (next_tick_jobs, deferred) = due_maintenance_jobs(
7357 &mut live_roots,
7358 Some(&executor),
7359 &bg_sub_by_session,
7360 &bg_wake_pending,
7361 MAINTENANCE_SUBMIT_BUDGET,
7362 &HashSet::new(),
7363 );
7364 assert_eq!(
7365 next_tick_jobs,
7366 vec![(root, MaintenanceDrainKind::CompletionDrains)]
7367 );
7368 assert!(!deferred);
7369 }
7370
7371 #[tokio::test]
7372 async fn subc_configure_tail_precedes_completed_search_install() {
7373 let root_dir = tempfile::tempdir().unwrap();
7374 let storage = tempfile::tempdir().unwrap();
7375 let root = ProjectRootId::from_path(root_dir.path()).unwrap();
7376 let (ctx, ignored_path) =
7377 runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
7378 let ctx = Arc::new(ctx);
7379 assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
7380 &ctx,
7381 &ignored_path
7382 ));
7383
7384 let executor = Arc::new(Executor::new());
7385 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7386 let metrics = Arc::new(DispatchPathMetrics::new());
7387 let (completion_tx, mut completion_rx) = mpsc::channel(4);
7388 submit_maintenance_job(
7389 &executor,
7390 root.clone(),
7391 MaintenanceDrainKind::ConfigureTail,
7392 Vec::new(),
7393 &completion_tx,
7394 &metrics,
7395 );
7396 submit_maintenance_job(
7397 &executor,
7398 root,
7399 MaintenanceDrainKind::CompletionDrains,
7400 Vec::new(),
7401 &completion_tx,
7402 &metrics,
7403 );
7404
7405 let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
7406 .await
7407 .expect("configure-tail completion timed out")
7408 .expect("configure-tail completion channel closed");
7409 let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
7410 .await
7411 .expect("completion-drains completion timed out")
7412 .expect("completion-drains completion channel closed");
7413 assert!(first.response.id.contains("configure-tail"));
7414 assert!(second.response.id.contains("completion-drains"));
7415 assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
7416 &ctx,
7417 &ignored_path
7418 ));
7419 assert_eq!(
7420 ctx.search_index()
7421 .read()
7422 .unwrap_or_else(std::sync::PoisonError::into_inner)
7423 .as_ref()
7424 .expect("completed search index installed")
7425 .file_count(),
7426 0,
7427 "configure must install the ignore matcher before pending paths replay"
7428 );
7429 ctx.stop_watcher_runtime();
7430 }
7431
7432 #[test]
7433 fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
7434 let (_dir, root) = test_root("maintenance-post-bind");
7435 let mut live_roots = HashMap::new();
7436 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
7437
7438 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
7439 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
7440
7441 let meta = live_roots.get(&root).expect("root metadata");
7442 assert!(meta.maintenance_pending);
7443 assert_eq!(meta.maintenance_jobs_in_flight, 0);
7444 assert_eq!(
7445 meta.maintenance_queued_kinds
7446 .iter()
7447 .copied()
7448 .collect::<Vec<_>>(),
7449 vec![
7450 MaintenanceDrainKind::ConfigureTail,
7451 MaintenanceDrainKind::CompletionDrains,
7452 ]
7453 );
7454
7455 let (due, deferred) = due_maintenance_jobs_without_actor_context(
7456 &mut live_roots,
7457 MAINTENANCE_SUBMIT_BUDGET,
7458 &HashSet::new(),
7459 );
7460
7461 assert_eq!(
7462 due,
7463 vec![
7464 (root.clone(), MaintenanceDrainKind::ConfigureTail),
7465 (root.clone(), MaintenanceDrainKind::CompletionDrains),
7466 ]
7467 );
7468 assert!(!deferred);
7469 assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
7470 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
7471 }
7472
7473 #[test]
7474 fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
7475 let mut live_roots = HashMap::new();
7476 let mut root_ids = Vec::new();
7477 let mut _dirs = Vec::new();
7478 for index in 0..4 {
7479 let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
7480 live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
7481 root_ids.push(root_id);
7482 _dirs.push(dir);
7483 }
7484
7485 let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
7486 let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
7487 &mut live_roots,
7488 small_budget,
7489 &HashSet::new(),
7490 );
7491
7492 assert_eq!(first_due.len(), small_budget);
7493 assert!(first_deferred);
7494 let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
7495 assert!(first_due_set
7496 .iter()
7497 .all(|root| live_roots[root].maintenance_pending));
7498 assert!(first_due_set
7499 .iter()
7500 .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
7501
7502 let all_roots: HashSet<_> = root_ids.into_iter().collect();
7503 let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
7504 assert!(deferred_roots
7505 .iter()
7506 .all(|root| !live_roots[root].maintenance_pending));
7507 }
7508
7509 #[test]
7510 fn due_maintenance_jobs_defers_pending_bind_roots() {
7511 let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
7512 let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
7513 let mut live_roots = HashMap::new();
7514 live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
7515 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
7516 let pending_bind_roots = HashSet::from([bind_root.clone()]);
7517
7518 let (due, deferred) = due_maintenance_jobs_without_actor_context(
7519 &mut live_roots,
7520 usize::MAX,
7521 &pending_bind_roots,
7522 );
7523
7524 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
7525 assert!(due.iter().all(|(root, _)| root == &healthy_root));
7526 assert!(!deferred);
7527 assert!(!live_roots[&bind_root].maintenance_pending);
7528 assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
7529 }
7530
7531 #[test]
7532 fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
7533 let (_dir, root) = test_root("maintenance-requeue");
7534 let mut live_roots = HashMap::new();
7535 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
7536 let (due, deferred) = due_maintenance_jobs_without_actor_context(
7537 &mut live_roots,
7538 usize::MAX,
7539 &HashSet::new(),
7540 );
7541 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
7542 assert!(due.iter().all(|(due_root, _)| due_root == &root));
7543 assert!(!deferred);
7544
7545 let meta = live_roots.get_mut(&root).unwrap();
7546 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
7547 assert!(meta.maintenance_pending);
7548 assert_eq!(
7549 meta.maintenance_jobs_in_flight,
7550 INITIAL_MAINTENANCE_JOB_COUNT - 1
7551 );
7552 assert_eq!(meta.maintenance_queued_kinds.len(), 1);
7553
7554 let (requeued, deferred) =
7555 due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
7556 assert_eq!(
7557 requeued,
7558 vec![(root.clone(), MaintenanceDrainKind::Watcher)]
7559 );
7560 assert!(!deferred);
7561 let meta = live_roots.get_mut(&root).unwrap();
7562 assert_eq!(
7563 meta.maintenance_jobs_in_flight,
7564 INITIAL_MAINTENANCE_JOB_COUNT
7565 );
7566 assert!(meta.maintenance_queued_kinds.is_empty());
7567
7568 for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
7569 note_maintenance_completion(meta, None, false, false);
7570 }
7571 assert!(!meta.maintenance_pending);
7572 assert_eq!(meta.maintenance_jobs_in_flight, 0);
7573 }
7574
7575 #[test]
7576 fn maintenance_requeue_drops_while_bind_is_pending() {
7577 let (_dir, root) = test_root("maintenance-bind-requeue");
7578 let mut live_roots = HashMap::new();
7579 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
7580 let (due, _) = due_maintenance_jobs_without_actor_context(
7581 &mut live_roots,
7582 usize::MAX,
7583 &HashSet::new(),
7584 );
7585 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
7586
7587 let meta = live_roots.get_mut(&root).unwrap();
7588 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
7589
7590 assert_eq!(
7591 meta.maintenance_jobs_in_flight,
7592 INITIAL_MAINTENANCE_JOB_COUNT - 1
7593 );
7594 assert!(meta.maintenance_queued_kinds.is_empty());
7595 assert!(meta.maintenance_pending);
7596 }
7597
7598 #[test]
7599 fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
7600 let mut meta = RootMeta::new(Instant::now());
7601 meta.unbound_quiesced = true;
7602
7603 assert!(!should_requiesce_after_maintenance(
7604 &meta,
7605 MaintenanceDrainKind::Lsp,
7606 false,
7607 ));
7608 assert!(!should_requiesce_after_maintenance(
7609 &meta,
7610 MaintenanceDrainKind::ConfigureTail,
7611 true,
7612 ));
7613 assert!(should_requiesce_after_maintenance(
7614 &meta,
7615 MaintenanceDrainKind::ConfigureTail,
7616 false,
7617 ));
7618 }
7619
7620 #[test]
7621 fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
7622 let (_dir, root) = test_root("maintenance-fatal");
7623 let mut live_roots = HashMap::new();
7624 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
7625 let (due, _) = due_maintenance_jobs_without_actor_context(
7626 &mut live_roots,
7627 usize::MAX,
7628 &HashSet::new(),
7629 );
7630 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
7631
7632 let meta = live_roots.get_mut(&root).unwrap();
7633 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
7634 assert!(meta.maintenance_poisoned);
7635 assert!(meta.maintenance_queued_kinds.is_empty());
7636
7637 for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
7638 note_maintenance_completion(meta, None, false, false);
7639 }
7640 assert!(!meta.maintenance_pending);
7641 assert_eq!(meta.maintenance_jobs_in_flight, 0);
7642 }
7643
7644 #[test]
7645 fn trust_for_principal_matrix() {
7646 assert_eq!(
7647 trust_for_principal(&Some(Principal::Direct)),
7648 BindTrust::FirstParty
7649 );
7650 for module_id in [
7657 "llm-runner",
7658 "aft",
7659 "broca",
7660 "alfonso-core",
7661 "prefrontal",
7662 "prefrontal-core",
7663 ] {
7664 assert_eq!(
7665 trust_for_principal(&Some(Principal::Reserved {
7666 module_id: module_id.to_string(),
7667 })),
7668 BindTrust::FirstParty,
7669 "reserved module id '{module_id}' must resolve to first-party trust"
7670 );
7671 }
7672 assert_eq!(
7673 trust_for_principal(&Some(Principal::Reserved {
7674 module_id: "subc-mcp".to_string(),
7675 })),
7676 BindTrust::Untrusted
7677 );
7678 assert_eq!(
7679 trust_for_principal(&Some(Principal::Reserved {
7680 module_id: "anything-unknown".to_string(),
7681 })),
7682 BindTrust::Untrusted
7683 );
7684 assert_eq!(
7685 trust_for_principal(&Some(Principal::Unverified)),
7686 BindTrust::Untrusted
7687 );
7688 assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
7689 }
7690
7691 #[test]
7692 fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
7693 let principal = Some(Principal::Direct);
7694 let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
7695 let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
7696
7697 assert_eq!(
7698 trust_for_bind(fingerprint_a, &principal),
7699 BindTrust::Untrusted
7700 );
7701 assert_eq!(
7702 trust_for_bind(fingerprint_b, &principal),
7703 BindTrust::Untrusted
7704 );
7705 }
7706
7707 #[test]
7714 fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
7715 for harness in ["opencode", "pi", "runner", "mcp:claude"] {
7716 assert_eq!(
7717 trust_for_bind(harness, &Some(Principal::Direct)),
7718 BindTrust::FirstParty,
7719 "a direct principal must stay first-party on {harness}"
7720 );
7721 assert_eq!(
7722 trust_for_bind(harness, &Some(Principal::Unverified)),
7723 BindTrust::Untrusted,
7724 "an unverified principal must stay untrusted on {harness}"
7725 );
7726 assert_eq!(
7727 trust_for_bind(harness, &None),
7728 BindTrust::Untrusted,
7729 "an absent principal must fail closed on {harness}"
7730 );
7731 assert_eq!(
7732 trust_for_bind(
7733 harness,
7734 &Some(Principal::Reserved {
7735 module_id: "subc-mcp".to_string(),
7736 })
7737 ),
7738 BindTrust::Untrusted,
7739 "a non-allowlisted reserved module must stay untrusted on {harness}"
7740 );
7741 }
7742 }
7743
7744 #[tokio::test]
7745 async fn persistent_cancel_resolves_when_fired_before_await() {
7746 let signal = PersistentCancelSignal::new();
7750 signal.cancel();
7751 tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
7753 .await
7754 .expect("cancelled() must resolve when cancel fired beforehand");
7755
7756 let racing = PersistentCancelSignal::new();
7758 let racing_for_task = racing.clone();
7759 let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
7760 racing.cancel();
7761 tokio::time::timeout(Duration::from_secs(1), waiter)
7762 .await
7763 .expect("cancelled() must resolve when cancel races the await")
7764 .expect("waiter task panicked");
7765 }
7766
7767 #[test]
7768 fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
7769 let installed = HashMap::from([(7, 9)]);
7770 let mut reclaimed = ReclaimedRoutes::default();
7771 reclaimed.insert(route_key(8, 1));
7772 for ty in [
7773 FrameType::Request,
7774 FrameType::Response,
7775 FrameType::Error,
7776 FrameType::Push,
7777 FrameType::Cancel,
7778 FrameType::Goodbye,
7779 ] {
7780 let body = if ty.is_pure_header() {
7781 Vec::new()
7782 } else {
7783 br#"{}"#.to_vec()
7784 };
7785 let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
7786 assert!(
7787 !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
7788 "{ty:?}"
7789 );
7790 }
7791
7792 let reclaimed_request = Frame::build(
7793 FrameType::Request,
7794 control_flags(),
7795 8,
7796 1,
7797 42,
7798 br#"{}"#.to_vec(),
7799 )
7800 .unwrap();
7801 assert!(ingress_route_should_be_processed(
7802 &installed,
7803 &reclaimed,
7804 &reclaimed_request
7805 ));
7806
7807 let never_installed = Frame::build(
7808 FrameType::Request,
7809 control_flags(),
7810 9,
7811 1,
7812 43,
7813 br#"{}"#.to_vec(),
7814 )
7815 .unwrap();
7816 assert!(!ingress_route_should_be_processed(
7817 &installed,
7818 &reclaimed,
7819 &never_installed
7820 ));
7821
7822 let current = Frame::build(
7823 FrameType::Request,
7824 control_flags(),
7825 7,
7826 9,
7827 43,
7828 br#"{}"#.to_vec(),
7829 )
7830 .unwrap();
7831 let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
7832 assert!(ingress_route_should_be_processed(
7833 &installed, &reclaimed, ¤t
7834 ));
7835 assert!(ingress_route_should_be_processed(
7836 &installed, &reclaimed, &control
7837 ));
7838 assert_eq!(installed, HashMap::from([(7, 9)]));
7839 }
7840
7841 #[tokio::test]
7842 async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
7843 let (_dir, root) = test_root("route-bind-b2-ordering");
7844 let route = route_key(7, 3);
7845 let identity = RouteIdentity(Arc::new(RouteIdentityData {
7846 root: root.clone(),
7847 project_root: root.as_path().to_path_buf(),
7848 harness: "opencode".to_string(),
7849 session: "b2-session".to_string(),
7850 trust: BindTrust::FirstParty,
7851 spawn_principal: AuthenticatedPrincipal::FirstParty,
7852 consumer_elicitation_capable: false,
7853 }));
7854 let replay_key = push::ReplayKey::from_identity(&identity);
7855 let completion = RouteBindCompletion {
7856 route,
7857 identity,
7858 bind_root_id: root.clone(),
7859 inserted_new_actor: false,
7860 configure_response: Response::success("subc-bind-7", json!({})),
7861 diagnostics_on_edit: false,
7862 ver: PROTOCOL_VERSION,
7863 corr: 91,
7864 flags: control_flags(),
7865 };
7866 let mut pending_binds = HashMap::from([(
7867 route,
7868 PendingBind {
7869 bind_root_id: root,
7870 inserted_new_actor: false,
7871 cancelled: false,
7872 configure_request_id: "subc-bind-7".to_string(),
7873 started_at: Instant::now(),
7874 warned_half_deadline: false,
7875 deadline_reported: false,
7876 corr: 91,
7877 ver: PROTOCOL_VERSION,
7878 flags: control_flags(),
7879 cancellation: crate::executor::JobCancellation::new(),
7880 },
7881 )]);
7882 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
7883 let mut push_buffer =
7884 HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
7885 let (writer_tx, mut writer_rx) = mpsc::channel(8);
7886 let metrics = Arc::new(DispatchPathMetrics::new());
7887
7888 handle_route_bind_completion(
7889 &writer_tx,
7890 completion,
7891 &mut HashMap::new(),
7892 &mut HashMap::new(),
7893 &mut HashMap::new(),
7894 &mut push_buffer,
7895 &mut HashMap::new(),
7896 &mut pending_binds,
7897 &mut installed_route_epochs,
7898 &Arc::new(Executor::new()),
7899 &Arc::new(Notify::new()),
7900 &metrics,
7901 )
7902 .await
7903 .unwrap();
7904
7905 let ack = writer_rx.try_recv().expect("RouteBindAck");
7906 assert_eq!(ack.header.ty, FrameType::Response);
7907 assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
7908 let route_frame = writer_rx.try_recv().expect("post-ack route frame");
7909 assert_eq!(route_frame.header.ty, FrameType::Push);
7910 assert_eq!(
7911 (route_frame.header.channel, route_frame.header.epoch),
7912 (route.channel, route.epoch)
7913 );
7914 }
7915}