1use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock, Mutex as StdMutex, OnceLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, JobCancellation, Lane};
31use crate::fleet_status::{spawn_fleet_status_dial, FleetStatusClient};
32use crate::jsonc::strip_jsonc;
33use crate::log_ctx;
34use crate::path_identity::ProjectRootId;
35use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
36use crate::run_tool_call::{
37 run_tool_call, strip_agent_preview_arg_owned, PhaseTrace, ToolCallContext, ToolCallOutcome,
38 ToolCallResult,
39};
40use crate::runtime_drain;
41use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
42
43use subc_protocol::manifest::{
44 Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
45 ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
46};
47use subc_protocol::session::{
48 HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
49 MODULE_CONTROL_OP_HEALTH_CHECK,
50};
51use subc_protocol::{
52 ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, MAX_FRAME_BODY_LEN,
53 PROTOCOL_VERSION,
54};
55use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
56use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
57use tokio::net::TcpStream;
58use tokio::sync::{mpsc, oneshot, Notify};
59use tokio::task::JoinHandle;
60
61const AUTH_DEADLINE: Duration = Duration::from_secs(5);
64const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
65const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
66const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
67const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
68
69const HELLO_CORR: u64 = 1;
71
72const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
75
76const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
80
81const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
85
86const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
90
91const WRITER_QUEUE_CAPACITY: usize = 256;
92
93const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
96
97const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
105const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
106 MaintenanceDrainKind::Watcher,
107 MaintenanceDrainKind::Lsp,
108 MaintenanceDrainKind::ConfigureTail,
109 MaintenanceDrainKind::CompletionDrains,
110];
111#[cfg(test)]
112const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
113
114const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
115const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
116
117const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
118const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
119
120const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
123
124const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
128
129const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
131const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
132
133#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
134struct RouteChannel {
135 channel: u16,
136 epoch: u32,
137}
138
139impl fmt::Display for RouteChannel {
140 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
141 write!(f, "{}@{}", self.channel, self.epoch)
142 }
143}
144
145type PushEnvelope = (ProjectRootId, PushFrame);
146type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
147type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
148mod bash;
149mod health;
150mod manifest;
151mod push;
152mod wire;
153
154use self::health::{
155 build_health_report, warn_slow_pending_binds, DispatchPathMetrics, HealthRollupCache,
156 ReapBlockerCensus, ResponseTaskGuard, HEALTH_ROLLUP_TTL,
157};
158use self::manifest::{
159 build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
160 is_subc_agent_core_tool, is_subc_native_plumbing_tool,
161};
162pub use self::wire::SubcError;
163
164pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
168 manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
169}
170use self::wire::{
171 build_error_frame, build_goodbye_frame, build_tool_response_frame,
172 build_tool_response_frame_with_limit, decrement_counted_channel, response_is_fatal_panic,
173 response_message, send_counted_channel, send_frame, send_reliable_writer_frame,
174 send_traced_tool_response_frame, ToolResponseWriteTrace, WriterFrame, WriterSender,
175};
176
177struct DecodedFrame {
178 frame: Frame,
179 phase_trace: PhaseTrace,
180}
181
182struct ToolCallCompletion {
183 text: String,
184 phase_trace: PhaseTrace,
185}
186
187#[derive(Clone)]
188struct ActiveToolCall {
189 root_id: ProjectRootId,
190 cancellation: JobCancellation,
191}
192
193type ActiveToolCalls = Arc<StdMutex<HashMap<(RouteChannel, u64), ActiveToolCall>>>;
194
195#[derive(Clone)]
196struct PushSenders {
197 lossy_tx: mpsc::Sender<LossyPushEnvelope>,
198 reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
199 lossy_overflow: Arc<push::LossyOverflow>,
200 lossy_seq: Arc<AtomicU64>,
201 fleet_status_client: FleetStatusClient,
202}
203
204#[derive(Clone)]
205struct PersistentCancelSignal {
206 inner: Arc<PersistentCancelInner>,
207}
208
209struct PersistentCancelInner {
210 cancelled: AtomicBool,
211 notify: Notify,
212}
213
214impl PersistentCancelSignal {
215 fn new() -> Self {
216 Self {
217 inner: Arc::new(PersistentCancelInner {
218 cancelled: AtomicBool::new(false),
219 notify: Notify::new(),
220 }),
221 }
222 }
223
224 fn cancel(&self) {
225 if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
226 self.inner.notify.notify_waiters();
227 }
228 }
229
230 fn is_cancelled(&self) -> bool {
231 self.inner.cancelled.load(Ordering::SeqCst)
232 }
233
234 async fn cancelled(&self) {
235 loop {
243 let notified = self.inner.notify.notified();
244 tokio::pin!(notified);
245 notified.as_mut().enable();
246 if self.is_cancelled() {
247 return;
248 }
249 notified.await;
250 }
251 }
252}
253
254fn finish_active_tool_call(active: &ActiveToolCalls, route: RouteChannel, corr: u64) {
255 active
256 .lock()
257 .unwrap_or_else(std::sync::PoisonError::into_inner)
258 .remove(&(route, corr));
259}
260
261fn cancel_active_tool_call(
262 active: &ActiveToolCalls,
263 executor: &Executor,
264 route: RouteChannel,
265 corr: u64,
266 reason: &str,
267) -> bool {
268 let call = active
269 .lock()
270 .unwrap_or_else(std::sync::PoisonError::into_inner)
271 .remove(&(route, corr));
272 let Some(call) = call else {
273 return false;
274 };
275 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
276 log::debug!(
277 "subc attach: cancelled active tool call route={route} corr={corr} reason={reason} outcome={outcome:?}"
278 );
279 true
280}
281
282fn cancel_active_tool_calls_for_route(
283 active: &ActiveToolCalls,
284 executor: &Executor,
285 route: RouteChannel,
286 reason: &str,
287) -> usize {
288 let cancelled = {
289 let mut calls = active
290 .lock()
291 .unwrap_or_else(std::sync::PoisonError::into_inner);
292 let mut cancelled = Vec::new();
293 calls.retain(|(call_route, _), call| {
294 if *call_route == route {
295 cancelled.push(call.clone());
296 false
297 } else {
298 true
299 }
300 });
301 cancelled
302 };
303 for call in &cancelled {
304 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
305 log::debug!(
306 "subc attach: cancelled active tool call route={route} reason={reason} outcome={outcome:?}"
307 );
308 }
309 cancelled.len()
310}
311
312fn cancel_all_active_tool_calls(
313 active: &ActiveToolCalls,
314 executor: &Executor,
315 reason: &str,
316) -> usize {
317 let cancelled = {
318 let mut calls = active
319 .lock()
320 .unwrap_or_else(std::sync::PoisonError::into_inner);
321 calls.drain().map(|(_, call)| call).collect::<Vec<_>>()
322 };
323 for call in &cancelled {
324 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
325 log::debug!("subc attach: cancelled active tool call reason={reason} outcome={outcome:?}");
326 }
327 cancelled.len()
328}
329
330#[derive(Debug, Clone, Copy, PartialEq, Eq)]
331pub(crate) enum BindTrust {
332 FirstParty,
333 Untrusted,
334}
335
336impl BindTrust {
337 fn allows_bash_observation(self) -> bool {
338 matches!(self, Self::FirstParty)
339 }
340
341 fn label(self) -> &'static str {
342 match self {
343 Self::FirstParty => "first_party",
344 Self::Untrusted => "untrusted",
345 }
346 }
347
348 fn sandbox_trust(self) -> PrincipalTrust {
349 match self {
350 Self::FirstParty => PrincipalTrust::FirstParty,
351 Self::Untrusted => PrincipalTrust::Untrusted,
352 }
353 }
354}
355
356pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
357 match principal {
358 Some(Principal::Direct) => BindTrust::FirstParty,
359 Some(Principal::Reserved { module_id })
370 if module_id == "llm-runner"
371 || module_id == "aft"
372 || module_id == "broca"
373 || module_id == "alfonso-core"
374 || module_id == "prefrontal"
375 || module_id == "prefrontal-core" =>
376 {
377 BindTrust::FirstParty
378 }
379 Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
380 BindTrust::Untrusted
381 }
382 }
383}
384
385fn harness_forces_untrusted(harness: &str) -> bool {
386 harness.starts_with("fed:")
387}
388
389pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
390 if harness_forces_untrusted(harness) {
391 BindTrust::Untrusted
392 } else {
393 trust_for_principal(principal)
394 }
395}
396
397fn principal_id(principal: &Option<Principal>) -> Option<String> {
398 match principal {
399 Some(Principal::Direct) => Some("direct".to_string()),
400 Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
401 Some(Principal::Unverified) => Some("unverified".to_string()),
402 None => None,
403 }
404}
405
406fn principal_label(principal: &Option<Principal>) -> String {
407 principal_id(principal).unwrap_or_else(|| "absent".to_string())
408}
409
410#[derive(Debug)]
411struct RootMeta {
417 maintenance_pending: bool,
418 maintenance_jobs_in_flight: usize,
419 maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
420 maintenance_last_submitted: Option<Instant>,
421 maintenance_poisoned: bool,
422 last_touched: Instant,
423 diagnostics_on_edit: bool,
424 active_bash_waits: usize,
425 idle_artifacts_evicted: bool,
426 unbound_quiesced: bool,
427 consecutive_missing_sweeps: u8,
428}
429
430#[derive(Debug)]
431struct PendingBind {
432 bind_root_id: ProjectRootId,
433 inserted_new_actor: bool,
434 cancelled: bool,
435 configure_request_id: String,
436 started_at: Instant,
437 warned_half_deadline: bool,
438 deadline_reported: bool,
439 corr: u64,
440 ver: u8,
441 flags: Flags,
442 cancellation: crate::executor::JobCancellation,
447}
448
449struct RouteBindCompletion {
450 route: RouteChannel,
451 identity: RouteIdentity,
452 bind_root_id: ProjectRootId,
453 inserted_new_actor: bool,
454 configure_response: Response,
455 diagnostics_on_edit: bool,
456 ver: u8,
457 corr: u64,
458 flags: Flags,
459}
460
461#[derive(Debug, Clone)]
462struct RouteIdentity(Arc<RouteIdentityData>);
463
464#[derive(Debug)]
465struct RouteIdentityData {
466 root: ProjectRootId,
467 project_root: PathBuf,
468 harness: String,
469 session: String,
470 trust: BindTrust,
471 spawn_principal: AuthenticatedPrincipal,
472 consumer_elicitation_capable: bool,
473}
474
475impl Deref for RouteIdentity {
476 type Target = RouteIdentityData;
477
478 fn deref(&self) -> &Self::Target {
479 &self.0
480 }
481}
482
483#[derive(Debug, Clone)]
484struct RetainedSessionIdentity {
485 harness: String,
486 trust: BindTrust,
487}
488
489#[derive(Clone)]
490struct BgSub {
491 corr: u64,
492 ver: u8,
493 flags: Flags,
494 root: ProjectRootId,
495 session: String,
496}
497
498type BgSubsBySession = HashMap<(ProjectRootId, String), HashSet<RouteChannel>>;
501
502struct MaintenanceCompletion {
503 root_id: ProjectRootId,
504 kind: MaintenanceDrainKind,
505 response: Response,
506 empty_bg_sessions: Vec<(String, u64)>,
507 requeue_kind: Option<MaintenanceDrainKind>,
508}
509
510#[derive(Clone, Copy, Debug, PartialEq, Eq)]
511enum MaintenanceDrainKind {
512 Watcher,
513 Lsp,
514 ConfigureTail,
515 CompletionDrains,
516}
517
518impl MaintenanceDrainKind {
519 fn label(self) -> &'static str {
520 match self {
521 Self::Watcher => "watcher",
522 Self::Lsp => "lsp",
523 Self::ConfigureTail => "configure-tail",
524 Self::CompletionDrains => "completion-drains",
525 }
526 }
527}
528
529#[derive(Debug, Default)]
530struct MaintenanceJobOutcome {
531 empty_bg_sessions: Vec<(String, u64)>,
532 requeue_kind: Option<MaintenanceDrainKind>,
533}
534
535#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
536struct ReverseCorrKey {
537 route: RouteChannel,
538 corr: u64,
539}
540
541struct PendingBashAsk {
542 route: RouteChannel,
543 tool_corr: u64,
544 tool_flags: Flags,
545 tool_ver: u8,
546 root: ProjectRootId,
547 project_root: PathBuf,
548 session_id: String,
549 spawn_principal: AuthenticatedPrincipal,
550 edit_slot_survives: Option<bool>,
551 request_id: String,
552 arguments: Value,
553 format_context: crate::subc_format::FormatContext,
554 cancel: bash::BashWaitCancel,
555 grants: Vec<String>,
556 expires_at: Instant,
557}
558
559impl RootMeta {
560 fn new(now: Instant) -> Self {
561 Self {
562 maintenance_pending: false,
563 maintenance_jobs_in_flight: 0,
564 maintenance_queued_kinds: VecDeque::new(),
565 maintenance_last_submitted: None,
566 maintenance_poisoned: false,
567 last_touched: now,
568 diagnostics_on_edit: false,
569 active_bash_waits: 0,
570 idle_artifacts_evicted: false,
571 unbound_quiesced: false,
572 consecutive_missing_sweeps: 0,
573 }
574 }
575
576 fn note_activity(&mut self) {
577 self.last_touched = Instant::now();
578 }
579
580 fn reactivate_bound(&mut self) {
581 self.note_activity();
582 self.idle_artifacts_evicted = false;
583 self.unbound_quiesced = false;
584 }
585}
586
587fn due_maintenance_jobs(
588 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
589 executor: Option<&Executor>,
590 bg_sub_by_session: &BgSubsBySession,
591 bg_wake_pending: &HashSet<RouteChannel>,
592 budget: usize,
593 pending_bind_roots: &HashSet<ProjectRootId>,
594) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
595 let mut jobs = Vec::new();
596 let mut deferred = false;
597 let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
598 roots.sort_by(|left, right| {
599 let left_last = live_roots
600 .get(left)
601 .and_then(|meta| meta.maintenance_last_submitted);
602 let right_last = live_roots
603 .get(right)
604 .and_then(|meta| meta.maintenance_last_submitted);
605 left_last
606 .cmp(&right_last)
607 .then_with(|| left.as_path().cmp(right.as_path()))
608 });
609
610 for root_id in roots {
611 let Some(meta) = live_roots.get_mut(&root_id) else {
612 continue;
613 };
614 if meta.maintenance_poisoned {
615 continue;
616 }
617
618 if pending_bind_roots.contains(&root_id) {
619 if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
620 deferred = true;
621 }
622 continue;
623 }
624
625 if !meta.maintenance_pending {
626 if jobs.len() >= budget {
627 deferred = true;
628 continue;
629 }
630 let executor_actor_context =
634 executor.and_then(|executor| executor.actor_context(&root_id));
635 let root_has_pending_bg_wake =
636 bg_sub_by_session.iter().any(|((sub_root, _), channels)| {
637 sub_root == &root_id
638 && channels
639 .iter()
640 .any(|channel| bg_wake_pending.contains(channel))
641 });
642 let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
643 Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
644 .into_iter()
645 .filter(|kind| {
646 if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
647 return false;
648 }
649 match kind {
650 MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
651 MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
652 MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
653 MaintenanceDrainKind::CompletionDrains => {
659 root_has_pending_bg_wake || ctx.completion_drains_have_work()
660 }
661 }
662 })
663 .collect(),
664 None if meta.unbound_quiesced => Vec::new(),
665 None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
667 };
668 if kinds_with_work.is_empty() {
669 continue;
670 }
671 meta.maintenance_pending = true;
672 meta.maintenance_queued_kinds.extend(kinds_with_work);
673 }
674
675 while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
676 if jobs.len() >= budget {
677 meta.maintenance_queued_kinds.push_front(kind);
678 deferred = true;
679 break;
680 }
681 meta.maintenance_jobs_in_flight += 1;
682 meta.maintenance_last_submitted = Some(Instant::now());
683 jobs.push((root_id.clone(), kind));
684 }
685
686 meta.maintenance_pending =
687 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
688 }
689
690 (jobs, deferred)
691}
692
693fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
694 match estimate.estimated_bytes {
695 Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
696 None if estimate.status == "busy" => "busy".to_string(),
697 None => "not estimated".to_string(),
698 }
699}
700
701fn optional_memory_label(bytes: Option<u64>) -> String {
702 bytes.map_or_else(
703 || "not estimated".to_string(),
704 |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
705 )
706}
707
708fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
709 format!(
710 "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
711 optional_memory_label(relief.rss_before_bytes),
712 optional_memory_label(relief.rss_after_bytes),
713 optional_memory_label(relief.allocator_before.bytes_in_use),
714 optional_memory_label(relief.allocator_after.bytes_in_use),
715 optional_memory_label(relief.allocator_before.size_allocated),
716 optional_memory_label(relief.allocator_after.size_allocated),
717 optional_memory_label(relief.allocator_before.retained_slack_bytes),
718 optional_memory_label(relief.allocator_after.retained_slack_bytes),
719 relief.bytes_released as f64 / (1024.0 * 1024.0),
720 )
721}
722
723fn idle_root_eviction_message(
724 root_id: &ProjectRootId,
725 memory: &crate::memory::RootMemorySnapshot,
726 pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
727) -> String {
728 let freed_bytes = [
731 &memory.semantic,
732 &memory.trigram,
733 &memory.symbols,
734 &memory.callgraph,
735 &memory.inspect,
736 ]
737 .iter()
738 .filter_map(|estimate| estimate.estimated_bytes)
739 .fold(0u64, u64::saturating_add);
740 let mut message = format!(
741 "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
742 root_id.as_path().display(),
743 freed_bytes as f64 / (1024.0 * 1024.0),
744 eviction_estimate_label(&memory.semantic),
745 eviction_estimate_label(&memory.trigram),
746 eviction_estimate_label(&memory.symbols),
747 eviction_estimate_label(&memory.callgraph),
748 eviction_estimate_label(&memory.inspect),
749 eviction_estimate_label(&memory.bash),
750 eviction_estimate_label(&memory.lsp),
751 eviction_estimate_label(&memory.parser_pool),
752 );
753 if let Some(pressure_relief) = pressure_relief {
754 message.push_str(&pressure_relief_label(pressure_relief));
755 }
756 message
757}
758
759fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
760 !live_roots.is_empty()
761 && live_roots.values().all(|meta| {
762 now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
763 && meta.active_bash_waits == 0
764 && !meta.maintenance_pending
765 && meta.maintenance_queued_kinds.is_empty()
766 })
767}
768
769fn allocator_pressure_relief_after_idle_sweep(
770 now: Instant,
771 live_roots: &HashMap<ProjectRootId, RootMeta>,
772 executor: &Executor,
773) -> Option<crate::memory::AllocatorPressureRelief> {
774 if !process_has_been_idle(now, live_roots)
775 || live_roots.keys().any(|root_id| {
776 executor
777 .actor_context(root_id)
778 .is_some_and(|ctx| ctx.artifact_eviction_blocked())
779 })
780 {
781 return None;
782 }
783
784 #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))]
785 {
786 Some(crate::memory::relieve_allocator_pressure())
787 }
788 #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))]
789 {
790 None
791 }
792}
793
794fn quiesce_unbound_root(
795 root_id: &ProjectRootId,
796 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
797 executor: &Arc<Executor>,
798) {
799 let Some(meta) = live_roots.get_mut(root_id) else {
800 return;
801 };
802
803 let ctx = executor.actor_context(root_id);
804 if let Some(ctx) = ctx.as_ref() {
805 ctx.mark_subc_unbound();
809 }
810 let cancelled = executor.cancel_queued_maintenance(root_id);
811 let discarded = ctx
819 .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
820 .unwrap_or(0);
821 meta.unbound_quiesced = true;
822 meta.maintenance_queued_kinds.clear();
823 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
824 log::info!(
825 "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s)); cause=goodbye_unbound",
826 root_id.as_path().display(),
827 cancelled,
828 discarded
829 );
830}
831
832#[allow(clippy::too_many_arguments)]
833fn quiesce_connection_roots(
834 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
835 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
836 routes: &mut HashMap<RouteChannel, RouteIdentity>,
837 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
838 installed_route_epochs: &mut HashMap<u16, u32>,
839 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
840 active_tool_calls: &ActiveToolCalls,
841 executor: &Arc<Executor>,
842) {
843 cancel_all_active_tool_calls(active_tool_calls, executor, "connection teardown");
844 for cancel in route_bash_cancels.values() {
845 cancel.token.cancel();
846 }
847 route_bash_cancels.clear();
848
849 let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
850 for pending in pending_binds.values_mut() {
851 pending.cancelled = true;
852 roots.insert(pending.bind_root_id.clone());
853 let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
854 }
855
856 for root_id in roots {
860 if live_roots.contains_key(&root_id) {
861 quiesce_unbound_root(&root_id, live_roots, executor);
862 } else if let Some(ctx) = executor.actor_context(&root_id) {
863 ctx.mark_subc_unbound();
864 executor.cancel_queued_maintenance(&root_id);
865 crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
866 }
867 }
868
869 routes.clear();
870 root_channels.clear();
871 installed_route_epochs.clear();
872}
873
874#[derive(Debug, Default)]
878struct ReclaimedRoutes {
879 highest_epoch_by_channel: HashMap<u16, u32>,
880}
881
882impl ReclaimedRoutes {
883 fn insert(&mut self, route: RouteChannel) {
884 self.highest_epoch_by_channel
885 .entry(route.channel)
886 .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
887 .or_insert(route.epoch);
888 }
889
890 fn contains(&self, route: RouteChannel) -> bool {
891 self.highest_epoch_by_channel
892 .get(&route.channel)
893 .is_some_and(|epoch| route.epoch <= *epoch)
894 }
895}
896
897#[derive(Debug, Default)]
898struct IdleReapOutcome {
899 evicted: usize,
900 forgotten_deleted_roots: Vec<ProjectRootId>,
901}
902
903fn reap_idle_roots(
904 now: Instant,
905 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
906 pending_binds: &HashMap<RouteChannel, PendingBind>,
907 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
908 executor: &Arc<Executor>,
909 metrics: &DispatchPathMetrics,
910) -> IdleReapOutcome {
911 let pending_bind_roots = pending_binds
912 .values()
913 .map(|pending| pending.bind_root_id.clone())
914 .collect::<HashSet<_>>();
915 let mut census = ReapBlockerCensus::default();
916 let mut candidates = Vec::new();
917
918 for (root_id, meta) in live_roots.iter_mut() {
919 let deleted = !root_id.as_path().exists();
920 if deleted {
921 meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
931 } else {
932 meta.consecutive_missing_sweeps = 0;
933 }
934 let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
935 let has_bound_route = root_channels
936 .get(root_id)
937 .is_some_and(|channels| !channels.is_empty());
938 let has_pending_bind = pending_bind_roots.contains(root_id);
939
940 if deleted {
941 let mut retained = false;
942 if !deletion_confirmed {
943 census.absence_unconfirmed += 1;
944 retained = true;
945 }
946 if meta.active_bash_waits > 0 {
950 census.bash_waits += 1;
951 retained = true;
952 }
953 if meta.maintenance_pending {
954 census.maintenance_pending += 1;
955 retained = true;
956 }
957 if !meta.maintenance_queued_kinds.is_empty() {
958 census.maintenance_queued += 1;
959 retained = true;
960 }
961 if has_pending_bind {
962 census.pending_binds += 1;
963 retained = true;
964 }
965 match executor.try_actor_is_idle(root_id) {
966 Some(true) => {}
967 Some(false) => {
968 census.actor_busy += 1;
969 retained = true;
970 }
971 None => {
972 census.actor_state_busy += 1;
973 retained = true;
974 }
975 }
976 if retained {
977 census.deleted_retained += 1;
978 continue;
979 }
980 } else {
981 if has_bound_route
985 || !meta.unbound_quiesced
986 || meta.idle_artifacts_evicted
987 || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
988 || meta.active_bash_waits > 0
989 || meta.maintenance_pending
990 || !meta.maintenance_queued_kinds.is_empty()
991 || has_pending_bind
992 || !executor.actor_is_idle(root_id)
993 {
994 continue;
995 }
996 }
997 candidates.push((root_id.clone(), deleted));
998 }
999
1000 let mut reaped = Vec::new();
1001 let mut forgotten_deleted_roots = Vec::new();
1002 for (root_id, deleted) in candidates {
1003 let Some(ctx) = executor.actor_context(&root_id) else {
1004 if deleted {
1005 census.deleted_retained += 1;
1006 census.actor_busy += 1;
1007 }
1008 continue;
1009 };
1010 if deleted {
1021 ctx.bash_background()
1022 .kill_running_tasks_for_root(root_id.as_path());
1023 }
1024 let taken_pending = Some(ctx.take_pending_reconciliation_state());
1025 if ctx.artifact_eviction_blocked() {
1026 if let Some(pending) = taken_pending {
1027 ctx.restore_pending_reconciliation_state(pending);
1028 }
1029 if deleted {
1030 census.deleted_retained += 1;
1031 census.artifact_eviction_blocked += 1;
1032 }
1033 continue;
1034 }
1035 let memory_before = ctx.memory_root_snapshot();
1036 if !ctx.evict_idle_artifacts() {
1037 if let Some(pending) = taken_pending {
1038 ctx.restore_pending_reconciliation_state(pending);
1039 }
1040 if deleted {
1041 census.deleted_retained += 1;
1042 census.artifact_eviction_failed += 1;
1043 }
1044 continue;
1045 }
1046 drop(taken_pending);
1047 ctx.stop_watcher_runtime_in_background();
1048 ctx.invalidate_artifacts_after_watcher_gap();
1051
1052 if deleted {
1053 if executor.retire_idle_actor_in_background(&root_id) {
1054 live_roots.remove(&root_id);
1055 forgotten_deleted_roots.push(root_id.clone());
1056 } else {
1057 census.deleted_retained += 1;
1058 census.actor_busy += 1;
1059 }
1060 } else {
1061 if let Some(meta) = live_roots.get_mut(&root_id) {
1062 meta.idle_artifacts_evicted = true;
1063 }
1064 ctx.release_idle_reopenable_resources_in_background();
1065 }
1066 reaped.push((root_id, memory_before));
1067 }
1068
1069 metrics.record_reap(census);
1070 if census.deleted_retained > 0 {
1071 log::info!(
1072 "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
1073 census.deleted_retained,
1074 census.blocker_histogram()
1075 );
1076 }
1077
1078 let pressure_relief = (!reaped.is_empty())
1079 .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
1080 .flatten();
1081 for (root_id, memory_before) in &reaped {
1082 log::info!(
1083 "{}",
1084 idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
1085 );
1086 }
1087 IdleReapOutcome {
1088 evicted: reaped.len(),
1089 forgotten_deleted_roots,
1090 }
1091}
1092
1093#[allow(clippy::too_many_arguments)]
1094fn purge_deleted_root_residents(
1095 root_id: &ProjectRootId,
1096 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1097 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1098 installed_route_epochs: &mut HashMap<u16, u32>,
1099 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1100 active_tool_calls: &ActiveToolCalls,
1101 executor: &Executor,
1102 retry_buffer: &mut RetryBuffer,
1103 reclaimed_routes: &mut ReclaimedRoutes,
1104 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1105 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1106 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1107 bg_sub_by_session: &mut BgSubsBySession,
1108 bg_wake_pending: &mut HashSet<RouteChannel>,
1109 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1110 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1111 metrics: &DispatchPathMetrics,
1112) {
1113 let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1114 stale_routes.extend(
1115 routes
1116 .iter()
1117 .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1118 );
1119 stale_routes.extend(
1120 bg_sub_by_session
1121 .iter()
1122 .filter(|((root, _), _)| root == root_id)
1123 .flat_map(|(_, routes)| routes.iter().copied()),
1124 );
1125 stale_routes.extend(
1126 pending_bash_asks
1127 .values()
1128 .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1129 );
1130
1131 for route in stale_routes {
1132 reclaimed_routes.insert(route);
1133 remove_installed_route(installed_route_epochs, route);
1134 remove_route_channel(routes, root_channels, route);
1135 if let Some(cancel) = route_bash_cancels.remove(&route) {
1136 cancel.token.cancel();
1137 }
1138 cancel_active_tool_calls_for_route(active_tool_calls, executor, route, "root reclaim");
1139 retry_buffer.remove(&route);
1140 if let Some(sub) = bg_subs.remove(&route) {
1141 metrics.record_bg_subscription_ended(&sub.root, &sub.session, route, "root-reclaim");
1142 }
1143 bg_wake_pending.remove(&route);
1144 }
1145 root_channels.remove(root_id);
1146 session_identity.retain(|(root, _), _| root != root_id);
1147 push_buffer.retain(|key, _| &key.root != root_id);
1148 bg_wake_epoch.retain(|(root, _), _| root != root_id);
1149 pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1150 bg_sub_by_session.retain(|(root, _), _| root != root_id);
1151
1152 log::info!(
1153 "subc attach: fully forgot deleted root {}; cause=absence_reclaim",
1154 root_id.as_path().display()
1155 );
1156}
1157
1158#[allow(clippy::too_many_arguments)]
1159fn submit_due_maintenance_jobs(
1160 executor: &Arc<Executor>,
1161 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1162 pending_binds: &HashMap<RouteChannel, PendingBind>,
1163 bg_sub_by_session: &BgSubsBySession,
1164 bg_wake_pending: &HashSet<RouteChannel>,
1165 bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1166 maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1167 metrics: &Arc<DispatchPathMetrics>,
1168) {
1169 let pending_bind_roots = pending_binds
1170 .values()
1171 .map(|pending| pending.bind_root_id.clone())
1172 .collect::<HashSet<_>>();
1173 let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1174 live_roots,
1175 Some(executor),
1176 bg_sub_by_session,
1177 bg_wake_pending,
1178 MAINTENANCE_SUBMIT_BUDGET,
1179 &pending_bind_roots,
1180 );
1181 if deferred_jobs {
1182 metrics
1183 .maintenance_budget_deferrals
1184 .fetch_add(1, Ordering::Relaxed);
1185 }
1186 for (root_id, kind) in due_jobs {
1187 let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1188 bg_sub_by_session
1189 .iter()
1190 .filter_map(|((root, session), _)| {
1191 if root == &root_id {
1192 Some((
1193 session.clone(),
1194 bg_wake_epoch
1195 .get(&(root_id.clone(), session.clone()))
1196 .copied()
1197 .unwrap_or(0),
1198 ))
1199 } else {
1200 None
1201 }
1202 })
1203 .collect()
1204 } else {
1205 Vec::new()
1206 };
1207 submit_maintenance_job(
1208 executor,
1209 root_id,
1210 kind,
1211 bg_sessions_to_check,
1212 maintenance_tx,
1213 metrics,
1214 );
1215 }
1216}
1217
1218fn should_requiesce_after_maintenance(
1219 meta: &RootMeta,
1220 completed_kind: MaintenanceDrainKind,
1221 bind_pending: bool,
1222) -> bool {
1223 meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1224}
1225
1226fn note_maintenance_completion(
1227 meta: &mut RootMeta,
1228 requeue_kind: Option<MaintenanceDrainKind>,
1229 fatal: bool,
1230 defer_requeue: bool,
1231) {
1232 if fatal {
1233 meta.maintenance_poisoned = true;
1234 }
1235
1236 if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1237 meta.maintenance_queued_kinds.push_back(kind);
1238 }
1239
1240 meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1241 meta.maintenance_pending =
1242 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1243}
1244
1245fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1246 RouteChannel { channel, epoch }
1247}
1248
1249fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1250 if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1251 installed_epochs.remove(&route.channel);
1252 }
1253}
1254
1255fn ingress_route_should_be_processed(
1256 installed_epochs: &HashMap<u16, u32>,
1257 reclaimed_routes: &ReclaimedRoutes,
1258 frame: &Frame,
1259) -> bool {
1260 if frame.header.channel == 0
1261 || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1262 {
1263 return true;
1264 }
1265
1266 frame.header.ty == FrameType::Request
1271 && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1272}
1273
1274fn bash_elicitation_timeout() -> Duration {
1275 if cfg!(debug_assertions) {
1276 if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1277 if let Ok(ms) = raw.parse::<u64>() {
1278 if ms > 0 {
1279 return Duration::from_millis(ms);
1280 }
1281 }
1282 }
1283 }
1284 BASH_ELICITATION_TIMEOUT
1285}
1286
1287fn allocate_reverse_corr(
1288 pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1289 route: RouteChannel,
1290 next_corr: &mut u64,
1291) -> u64 {
1292 loop {
1293 let corr = *next_corr;
1294 *next_corr = (*next_corr).wrapping_add(1).max(1);
1295 if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1296 return corr;
1297 }
1298 }
1299}
1300
1301fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1302 match kind {
1303 crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1304 crate::bash_permissions::PermissionKind::Bash => "bash",
1305 }
1306}
1307
1308fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1309 let mut patterns = Vec::new();
1310 let mut seen = HashSet::new();
1311 for ask in asks {
1312 for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1313 if seen.insert(pattern.clone()) {
1314 patterns.push(pattern.clone());
1315 }
1316 }
1317 }
1318 patterns
1319}
1320
1321fn bash_elicitation_message(
1322 command: &str,
1323 asks: &[crate::bash_permissions::PermissionAsk],
1324) -> String {
1325 let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1326 let patterns = bash_elicitation_patterns(asks);
1327 let pattern_text = if patterns.is_empty() {
1328 "no matched permission patterns".to_string()
1329 } else {
1330 patterns.join(", ")
1331 };
1332 let ask_kinds = asks
1333 .iter()
1334 .map(|ask| bash_permission_kind_label(&ask.kind))
1335 .collect::<HashSet<_>>()
1336 .into_iter()
1337 .collect::<Vec<_>>()
1338 .join(", ");
1339 if ask_kinds.is_empty() {
1340 format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1341 } else {
1342 format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1343 }
1344}
1345
1346fn bash_elicitation_request_body(
1347 command: &str,
1348 asks: &[crate::bash_permissions::PermissionAsk],
1349) -> Value {
1350 json!({
1351 "method": BASH_ELICITATION_CREATE_METHOD,
1352 "params": {
1353 "mode": "form",
1354 "message": bash_elicitation_message(command, asks),
1355 "requestedSchema": {
1356 "type": "object",
1357 "properties": {
1358 "decision": {
1359 "type": "string",
1360 "enum": ["allow", "deny"],
1361 "description": "Choose allow to run this bash command once, or deny to block it."
1362 }
1363 },
1364 "required": ["decision"],
1365 "additionalProperties": false
1366 },
1367 "_meta": {
1368 "aft": {
1369 "tool": "bash",
1370 "command": command,
1371 "asks": asks
1372 }
1373 }
1374 }
1375 })
1376}
1377
1378fn build_bash_elicitation_request_frame(
1379 ver: u8,
1380 route: RouteChannel,
1381 corr: u64,
1382 flags: Flags,
1383 command: &str,
1384 asks: &[crate::bash_permissions::PermissionAsk],
1385) -> Result<Frame, SubcError> {
1386 let body = bash_elicitation_request_body(command, asks);
1387 Frame::build_with_version(
1388 ver,
1389 FrameType::Request,
1390 flags,
1391 route.channel,
1392 route.epoch,
1393 corr,
1394 serde_json::to_vec(&body).map_err(SubcError::Json)?,
1395 )
1396 .map_err(SubcError::FrameBuild)
1397}
1398
1399fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1400 let Ok(value) = serde_json::from_slice::<Value>(body) else {
1401 return false;
1402 };
1403 flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1404}
1405
1406fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1407 let Some(object) = value.as_object() else {
1408 return false;
1409 };
1410 object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1411}
1412
1413fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1414 let Some(object) = value.as_object() else {
1415 return false;
1416 };
1417 if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1418 return false;
1419 }
1420 let Some(content) = object.get("content").and_then(Value::as_object) else {
1421 return false;
1422 };
1423 content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1424}
1425
1426#[allow(clippy::too_many_arguments)]
1427async fn settle_pending_bash_ask_denied(
1428 tx: &WriterSender,
1429 pending: PendingBashAsk,
1430 routes: &HashMap<RouteChannel, RouteIdentity>,
1431 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1432 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1433 shutdown: &Arc<Notify>,
1434 metrics: &DispatchPathMetrics,
1435) -> Result<(), SubcError> {
1436 let completion = bash::bash_denied_untrusted_completion(
1437 pending.route,
1438 pending.tool_corr,
1439 pending.tool_flags,
1440 pending.tool_ver,
1441 pending.root,
1442 pending.request_id,
1443 pending.format_context,
1444 );
1445 bash::handle_bash_deferred_completion(
1446 tx,
1447 completion,
1448 routes,
1449 live_roots,
1450 route_bash_cancels,
1451 shutdown,
1452 metrics,
1453 )
1454 .await
1455}
1456
1457fn take_pending_bash_asks_for_route(
1458 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1459 route: RouteChannel,
1460) -> Vec<PendingBashAsk> {
1461 let keys = pending_bash_asks
1462 .keys()
1463 .copied()
1464 .filter(|key| key.route == route)
1465 .collect::<Vec<_>>();
1466 keys.into_iter()
1467 .filter_map(|key| pending_bash_asks.remove(&key))
1468 .collect()
1469}
1470
1471#[allow(clippy::too_many_arguments)]
1472async fn settle_pending_bash_asks_for_route(
1473 tx: &WriterSender,
1474 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1475 route: RouteChannel,
1476 routes: &HashMap<RouteChannel, RouteIdentity>,
1477 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1478 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1479 shutdown: &Arc<Notify>,
1480 metrics: &DispatchPathMetrics,
1481) -> Result<(), SubcError> {
1482 for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1483 settle_pending_bash_ask_denied(
1484 tx,
1485 pending,
1486 routes,
1487 live_roots,
1488 route_bash_cancels,
1489 shutdown,
1490 metrics,
1491 )
1492 .await?;
1493 }
1494 Ok(())
1495}
1496
1497#[allow(clippy::too_many_arguments)]
1498async fn settle_all_pending_bash_asks(
1499 tx: &WriterSender,
1500 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1501 routes: &HashMap<RouteChannel, RouteIdentity>,
1502 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1503 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1504 shutdown: &Arc<Notify>,
1505 metrics: &DispatchPathMetrics,
1506) -> Result<(), SubcError> {
1507 let pending = pending_bash_asks
1508 .drain()
1509 .map(|(_, pending)| pending)
1510 .collect::<Vec<_>>();
1511 for pending in pending {
1512 settle_pending_bash_ask_denied(
1513 tx,
1514 pending,
1515 routes,
1516 live_roots,
1517 route_bash_cancels,
1518 shutdown,
1519 metrics,
1520 )
1521 .await?;
1522 }
1523 Ok(())
1524}
1525
1526#[allow(clippy::too_many_arguments)]
1527async fn expire_pending_bash_asks(
1528 tx: &WriterSender,
1529 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1530 routes: &HashMap<RouteChannel, RouteIdentity>,
1531 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1532 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1533 shutdown: &Arc<Notify>,
1534 metrics: &DispatchPathMetrics,
1535) -> Result<(), SubcError> {
1536 let now = Instant::now();
1537 let expired = pending_bash_asks
1538 .iter()
1539 .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1540 .collect::<Vec<_>>();
1541 for key in expired {
1542 if let Some(pending) = pending_bash_asks.remove(&key) {
1543 log::debug!(
1544 "subc attach: bash elicitation request {} on route {} expired fail-closed",
1545 key.corr,
1546 pending.route
1547 );
1548 settle_pending_bash_ask_denied(
1549 tx,
1550 pending,
1551 routes,
1552 live_roots,
1553 route_bash_cancels,
1554 shutdown,
1555 metrics,
1556 )
1557 .await?;
1558 }
1559 }
1560 Ok(())
1561}
1562
1563#[allow(clippy::too_many_arguments)]
1564async fn handle_bash_elicitation_reply(
1565 tx: &WriterSender,
1566 frame: &Frame,
1567 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1568 routes: &HashMap<RouteChannel, RouteIdentity>,
1569 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1570 executor: &Arc<Executor>,
1571 shutdown: &Arc<Notify>,
1572 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1573 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1574 metrics: &Arc<DispatchPathMetrics>,
1575 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1576 dispatch: DispatchFn,
1577) -> Result<(), SubcError> {
1578 let key = ReverseCorrKey {
1579 route: route_key(frame.header.channel, frame.header.epoch),
1580 corr: frame.header.corr,
1581 };
1582 let Some(pending) = pending_bash_asks.remove(&key) else {
1583 return Ok(());
1584 };
1585
1586 if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1587 if routes.contains_key(&key.route) {
1588 bash::submit_deferred_bash(
1589 executor,
1590 bash_deferred_tx,
1591 bash_poll_touch_tx,
1592 metrics,
1593 dispatch,
1594 pending.root,
1595 pending.project_root,
1596 pending.session_id,
1597 pending.request_id,
1598 pending.route,
1599 pending.tool_corr,
1600 pending.tool_flags,
1601 pending.tool_ver,
1602 pending.arguments,
1603 pending.format_context,
1604 pending.cancel,
1605 BindTrust::Untrusted,
1606 pending.spawn_principal,
1607 pending.edit_slot_survives,
1608 Some(pending.grants),
1609 );
1610 return Ok(());
1611 }
1612 log::debug!(
1613 "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1614 key.corr,
1615 pending.route
1616 );
1617 }
1618
1619 settle_pending_bash_ask_denied(
1620 tx,
1621 pending,
1622 routes,
1623 live_roots,
1624 route_bash_cancels,
1625 shutdown,
1626 metrics,
1627 )
1628 .await
1629}
1630
1631#[allow(clippy::too_many_arguments)]
1632async fn cancel_pending_bash_ask_for_tool_call(
1633 tx: &WriterSender,
1634 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1635 route: RouteChannel,
1636 tool_corr: u64,
1637 routes: &HashMap<RouteChannel, RouteIdentity>,
1638 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1639 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1640 shutdown: &Arc<Notify>,
1641 metrics: &DispatchPathMetrics,
1642) -> Result<(), SubcError> {
1643 let keys = pending_bash_asks
1644 .iter()
1645 .filter_map(|(key, pending)| {
1646 (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1647 })
1648 .collect::<Vec<_>>();
1649 for key in keys {
1650 if let Some(pending) = pending_bash_asks.remove(&key) {
1651 settle_pending_bash_ask_denied(
1652 tx,
1653 pending,
1654 routes,
1655 live_roots,
1656 route_bash_cancels,
1657 shutdown,
1658 metrics,
1659 )
1660 .await?;
1661 }
1662 }
1663 Ok(())
1664}
1665
1666fn remove_root_channel(
1667 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1668 root: &ProjectRootId,
1669 channel: RouteChannel,
1670) {
1671 let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1672 channels.remove(&channel);
1673 channels.is_empty()
1674 } else {
1675 false
1676 };
1677 if remove_root {
1678 root_channels.remove(root);
1679 }
1680}
1681
1682fn remove_route_channel(
1683 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1684 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1685 channel: RouteChannel,
1686) -> Option<RouteIdentity> {
1687 let removed = routes.remove(&channel);
1688 if let Some(identity) = &removed {
1689 remove_root_channel(root_channels, &identity.root, channel);
1690 }
1691 removed
1692}
1693
1694fn insert_route_channel(
1695 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1696 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1697 channel: RouteChannel,
1698 identity: RouteIdentity,
1699) {
1700 if let Some(previous) = routes.insert(channel, identity.clone()) {
1701 remove_root_channel(root_channels, &previous.root, channel);
1702 }
1703 root_channels
1704 .entry(identity.root.clone())
1705 .or_default()
1706 .insert(channel);
1707}
1708
1709fn insert_bg_subscription_index(
1710 bg_sub_by_session: &mut BgSubsBySession,
1711 root: ProjectRootId,
1712 session: String,
1713 channel: RouteChannel,
1714) {
1715 bg_sub_by_session
1716 .entry((root, session))
1717 .or_default()
1718 .insert(channel);
1719}
1720
1721fn remove_bg_subscription_index(
1722 bg_sub_by_session: &mut BgSubsBySession,
1723 channel: RouteChannel,
1724 identity: Option<&RouteIdentity>,
1725) {
1726 if let Some(identity) = identity {
1727 let key = (identity.root.clone(), identity.session.clone());
1728 let remove_key = bg_sub_by_session.get_mut(&key).is_some_and(|channels| {
1729 channels.remove(&channel);
1730 channels.is_empty()
1731 });
1732 if remove_key {
1733 bg_sub_by_session.remove(&key);
1734 }
1735 } else {
1736 bg_sub_by_session.retain(|_, channels| {
1737 channels.remove(&channel);
1738 !channels.is_empty()
1739 });
1740 }
1741}
1742
1743fn route_removal_will_quiesce_root(
1744 root: &ProjectRootId,
1745 route: RouteChannel,
1746 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1747 has_pending_bind: bool,
1748 replacement_root: Option<&ProjectRootId>,
1749) -> bool {
1750 let removes_last_route = root_channels
1751 .get(root)
1752 .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1753 removes_last_route && !has_pending_bind && replacement_root != Some(root)
1754}
1755
1756fn should_quiesce_removed_root(
1757 root: &ProjectRootId,
1758 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1759 has_pending_bind: bool,
1760 replacement_root: Option<&ProjectRootId>,
1761) -> bool {
1762 !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1763}
1764
1765async fn end_bg_subscription(
1766 writer_tx: &WriterSender,
1767 metrics: &DispatchPathMetrics,
1768 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1769 bg_sub_by_session: &mut BgSubsBySession,
1770 bg_wake_pending: &mut HashSet<RouteChannel>,
1771 channel: RouteChannel,
1772 identity: Option<&RouteIdentity>,
1773 cause: &str,
1774) -> Result<(), SubcError> {
1775 if let Some(sub) = bg_subs.remove(&channel) {
1776 bg_wake_pending.remove(&channel);
1777 remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1778 metrics.record_bg_subscription_ended(&sub.root, &sub.session, channel, cause);
1779 push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1780 }
1781 Ok(())
1782}
1783
1784#[allow(clippy::too_many_arguments)]
1785async fn teardown_installed_route(
1786 tx: &WriterSender,
1787 metrics: &DispatchPathMetrics,
1788 executor: &Arc<Executor>,
1789 channel: RouteChannel,
1790 cancellation_reason: &str,
1791 replacement_root: Option<&ProjectRootId>,
1792 installed_route_epochs: &mut HashMap<u16, u32>,
1793 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1794 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1795 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1796 bg_sub_by_session: &mut BgSubsBySession,
1797 bg_wake_pending: &mut HashSet<RouteChannel>,
1798 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1799 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1800 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1801 active_tool_calls: &ActiveToolCalls,
1802 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1803 retry_buffer: &mut RetryBuffer,
1804 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1805 shutdown: &Arc<Notify>,
1806) -> Result<(), SubcError> {
1807 remove_installed_route(installed_route_epochs, channel);
1808 let bg_end_cause = match cancellation_reason {
1809 "Goodbye" => "goodbye",
1810 "higher-epoch RouteBind" => "higher-epoch",
1811 other => other,
1812 };
1813 end_bg_subscription(
1814 tx,
1815 metrics,
1816 bg_subs,
1817 bg_sub_by_session,
1818 bg_wake_pending,
1819 channel,
1820 routes.get(&channel),
1821 bg_end_cause,
1822 )
1823 .await?;
1824 settle_pending_bash_asks_for_route(
1825 tx,
1826 pending_bash_asks,
1827 channel,
1828 routes,
1829 live_roots,
1830 route_bash_cancels,
1831 shutdown,
1832 metrics,
1833 )
1834 .await?;
1835 if let Some(cancel) = route_bash_cancels.remove(&channel) {
1836 cancel.token.cancel();
1837 }
1838 cancel_active_tool_calls_for_route(active_tool_calls, executor, channel, cancellation_reason);
1839 if let Some(pending) = pending_binds.get_mut(&channel) {
1840 pending.cancelled = true;
1841 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1842 log::debug!(
1843 "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1844 channel.channel
1845 );
1846 }
1847 let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1848 if let Some(identity) = routes.get(&channel) {
1849 let has_pending_bind = pending_binds
1850 .values()
1851 .any(|pending| pending.bind_root_id == identity.root);
1852 if route_removal_will_quiesce_root(
1853 &identity.root,
1854 channel,
1855 root_channels,
1856 has_pending_bind,
1857 replacement_root,
1858 ) {
1859 if let Some(ctx) = executor.actor_context(&identity.root) {
1860 ctx.mark_subc_unbound();
1863 }
1864 }
1865 }
1866 if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
1867 let session_still_routed = routes
1868 .values()
1869 .any(|route| route.root == identity.root && route.session == identity.session);
1870 if !session_still_routed {
1871 if let Some(ctx) = executor.actor_context(&identity.root) {
1872 ctx.hashline_bindings()
1873 .teardown(identity.root.as_path(), &identity.session);
1874 }
1875 }
1876 if migrated > 0 {
1877 log::debug!(
1878 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
1879 channel.channel
1880 );
1881 }
1882 if let Some(meta) = live_roots.get_mut(&identity.root) {
1883 let idle_for = meta.last_touched.elapsed();
1884 meta.note_activity();
1885 log::debug!(
1886 "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
1887 channel.channel,
1888 identity.root.as_path().display(),
1889 identity.harness,
1890 identity.session,
1891 idle_for
1892 );
1893 } else {
1894 log::debug!(
1895 "subc attach: route {} torn down for root {} harness {} session {}",
1896 channel.channel,
1897 identity.root.as_path().display(),
1898 identity.harness,
1899 identity.session
1900 );
1901 }
1902 let has_pending_bind = pending_binds
1903 .values()
1904 .any(|pending| pending.bind_root_id == identity.root);
1905 if should_quiesce_removed_root(
1906 &identity.root,
1907 root_channels,
1908 has_pending_bind,
1909 replacement_root,
1910 ) {
1911 quiesce_unbound_root(&identity.root, live_roots, executor);
1912 }
1913 } else {
1914 if migrated > 0 {
1915 log::debug!(
1916 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
1917 channel.channel
1918 );
1919 }
1920 log::debug!("subc attach: unbound route {} torn down", channel.channel);
1921 }
1922 Ok(())
1923}
1924
1925fn remember_session_identity(
1926 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1927 identity: &RouteIdentity,
1928) {
1929 let key = (identity.root.clone(), identity.session.clone());
1930 if matches!(identity.trust, BindTrust::Untrusted)
1931 && session_identity
1932 .get(&key)
1933 .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
1934 {
1935 return;
1936 }
1937
1938 session_identity.insert(
1943 key,
1944 RetainedSessionIdentity {
1945 harness: identity.harness.clone(),
1946 trust: identity.trust,
1947 },
1948 );
1949}
1950
1951fn replay_key_for_session(
1952 session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1953 root: &ProjectRootId,
1954 session: &str,
1955) -> Option<(push::ReplayKey, BindTrust)> {
1956 let retained = session_identity.get(&(root.clone(), session.to_string()))?;
1957 Some((
1958 push::ReplayKey {
1959 root: root.clone(),
1960 harness: retained.harness.clone(),
1961 session: session.to_string(),
1962 },
1963 retained.trust,
1964 ))
1965}
1966pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
1969
1970#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1971enum ModuleLoopExit {
1972 Graceful,
1973 SkipSearchFlush,
1974}
1975
1976pub fn run_subc_mode(
1981 connection_file_path: &Path,
1982 ctx: Arc<AppContext>,
1983 executor: Arc<Executor>,
1984 dispatch: DispatchFn,
1985 user_config_path: Option<PathBuf>,
1986) -> Result<(), SubcError> {
1987 run_subc_mode_inner(
1991 connection_file_path,
1992 ctx,
1993 executor,
1994 dispatch,
1995 user_config_path,
1996 false,
1997 MAX_FRAME_BODY_LEN as usize,
1998 )
1999}
2000
2001fn run_subc_mode_inner(
2002 connection_file_path: &Path,
2003 ctx: Arc<AppContext>,
2004 executor: Arc<Executor>,
2005 dispatch: DispatchFn,
2006 user_config_path: Option<PathBuf>,
2007 allow_native_passthrough: bool,
2008 tool_response_body_limit: usize,
2009) -> Result<(), SubcError> {
2010 let runtime = tokio::runtime::Builder::new_current_thread()
2011 .enable_all()
2012 .build()
2013 .map_err(SubcError::Runtime)?;
2014
2015 let executor_for_loop = Arc::clone(&executor);
2016 let loop_result = runtime.block_on(async move {
2017 let shared_app = ctx.app();
2018 drop(ctx);
2019 let stream = connect_and_authenticate(connection_file_path).await?;
2020 log::info!(
2021 "subc attach: authenticated to daemon via {}",
2022 connection_file_path.display()
2023 );
2024 let (read_half, write_half) = tokio::io::split(stream);
2025 run_module_loop(
2026 read_half,
2027 write_half,
2028 connection_file_path,
2029 shared_app,
2030 executor_for_loop,
2031 dispatch,
2032 user_config_path,
2033 allow_native_passthrough,
2034 tool_response_body_limit,
2035 )
2036 .await
2037 });
2038
2039 let actor_contexts = executor.actor_contexts();
2040 if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
2041 flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
2044 }
2045 for actor_ctx in &actor_contexts {
2046 actor_ctx.lsp().shutdown_all();
2047 actor_ctx.bash_background().detach();
2048 }
2049
2050 loop_result.map(|_| ())
2051}
2052
2053fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
2054 for actor_ctx in actor_contexts {
2055 let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
2056 }
2057 let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
2058}
2059
2060#[doc(hidden)]
2065pub fn run_subc_mode_for_test(
2066 connection_file_path: &Path,
2067 ctx: Arc<AppContext>,
2068 executor: Arc<Executor>,
2069 dispatch: DispatchFn,
2070 user_config_path: Option<PathBuf>,
2071) -> Result<(), SubcError> {
2072 run_subc_mode_inner(
2073 connection_file_path,
2074 ctx,
2075 executor,
2076 dispatch,
2077 user_config_path,
2078 true,
2079 MAX_FRAME_BODY_LEN as usize,
2080 )
2081}
2082
2083#[doc(hidden)]
2086pub fn run_subc_mode_for_test_with_response_body_limit(
2087 connection_file_path: &Path,
2088 ctx: Arc<AppContext>,
2089 executor: Arc<Executor>,
2090 dispatch: DispatchFn,
2091 user_config_path: Option<PathBuf>,
2092 tool_response_body_limit: usize,
2093) -> Result<(), SubcError> {
2094 assert!((4 * 1_024..=MAX_FRAME_BODY_LEN as usize).contains(&tool_response_body_limit));
2095 run_subc_mode_inner(
2096 connection_file_path,
2097 ctx,
2098 executor,
2099 dispatch,
2100 user_config_path,
2101 true,
2102 tool_response_body_limit,
2103 )
2104}
2105
2106#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2107enum AttachErrorClass {
2108 Transient,
2109 Permanent,
2110}
2111
2112impl fmt::Display for AttachErrorClass {
2113 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2114 match self {
2115 Self::Transient => f.write_str("transient"),
2116 Self::Permanent => f.write_str("permanent"),
2117 }
2118 }
2119}
2120
2121#[derive(Clone, Copy)]
2122struct AttachRetryPolicy {
2123 budget: Duration,
2124 initial_backoff: Duration,
2125 max_backoff: Duration,
2126 jitter_percent: u64,
2127}
2128
2129const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
2130 budget: ATTACH_RETRY_BUDGET,
2131 initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
2132 max_backoff: ATTACH_RETRY_MAX_BACKOFF,
2133 jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
2134};
2135
2136fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
2139 let transient = match error {
2140 SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
2141 SubcError::Auth { source, .. } => match source {
2142 subc_transport::AuthError::Timeout { .. }
2143 | subc_transport::AuthError::UnexpectedEof { .. } => true,
2144 subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
2145 _ => false,
2146 },
2147 _ => false,
2148 };
2149 if transient {
2150 AttachErrorClass::Transient
2151 } else {
2152 AttachErrorClass::Permanent
2153 }
2154}
2155
2156fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
2157 matches!(
2158 kind,
2159 io::ErrorKind::ConnectionRefused
2160 | io::ErrorKind::TimedOut
2161 | io::ErrorKind::ConnectionReset
2162 | io::ErrorKind::ConnectionAborted
2163 | io::ErrorKind::BrokenPipe
2164 | io::ErrorKind::UnexpectedEof
2165 )
2166}
2167
2168async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
2172 connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
2173}
2174
2175async fn connect_and_authenticate_with_policy(
2176 connection_file_path: &Path,
2177 policy: AttachRetryPolicy,
2178) -> Result<TcpStream, SubcError> {
2179 let started_at = Instant::now();
2180 let deadline = started_at + policy.budget;
2181 let mut attempt = 0_u32;
2182 let mut backoff = policy.initial_backoff;
2183 let mut history = Vec::new();
2184
2185 loop {
2186 attempt = attempt.saturating_add(1);
2187 let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2188 Ok(stream) => return Ok(stream),
2189 Err(error) => error,
2190 };
2191 let class = classify_attach_error(&error);
2192 let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2193 history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2194
2195 if class == AttachErrorClass::Permanent {
2196 log_attach_final_failure(started_at.elapsed(), &history);
2197 return Err(error);
2198 }
2199
2200 let remaining = deadline.saturating_duration_since(Instant::now());
2201 if remaining.is_zero() {
2202 log_attach_final_failure(started_at.elapsed(), &history);
2203 return Err(error);
2204 }
2205
2206 let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2207 log::info!(
2208 "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2209 );
2210 tokio::time::sleep(delay).await;
2211
2212 if Instant::now() >= deadline {
2213 log_attach_final_failure(started_at.elapsed(), &history);
2214 return Err(error);
2215 }
2216 backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2217 }
2218}
2219
2220fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2221 let jitter_percent = jitter_percent.min(100);
2222 if jitter_percent == 0 {
2223 return base;
2224 }
2225
2226 let mut random_bytes = [0_u8; 8];
2227 let random = if getrandom::fill(&mut random_bytes).is_ok() {
2228 u64::from_le_bytes(random_bytes)
2229 } else {
2230 let timestamp = std::time::SystemTime::now()
2231 .duration_since(std::time::UNIX_EPOCH)
2232 .unwrap_or_default()
2233 .subsec_nanos();
2234 u64::from(timestamp) ^ u64::from(attempt)
2235 };
2236 let span = jitter_percent.saturating_mul(2).saturating_add(1);
2237 let multiplier_percent = 100 - jitter_percent + random % span;
2238 let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2239 Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2240}
2241
2242fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2243 log::error!(
2244 "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2245 history.len(),
2246 history.join(" | ")
2247 );
2248}
2249
2250async fn connect_and_authenticate_once(
2251 connection_file_path: &Path,
2252 deadline: Instant,
2253) -> Result<TcpStream, SubcError> {
2254 let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2257 SubcError::ConnectionFile {
2258 path: connection_file_path.to_path_buf(),
2259 source,
2260 }
2261 })?;
2262
2263 let endpoint = conn
2264 .endpoints
2265 .first()
2266 .ok_or_else(|| SubcError::NoEndpoint {
2267 path: connection_file_path.to_path_buf(),
2268 })?;
2269 let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2270 let ip = endpoint
2271 .host
2272 .parse::<IpAddr>()
2273 .map_err(|_| SubcError::InvalidEndpoint {
2274 path: connection_file_path.to_path_buf(),
2275 endpoint: endpoint_label.clone(),
2276 })?;
2277 let addr = SocketAddr::new(ip, endpoint.port);
2278
2279 let connect_budget = deadline.saturating_duration_since(Instant::now());
2280 let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2281 .await
2282 .map_err(|_| SubcError::Connect {
2283 endpoint: endpoint_label.clone(),
2284 source: io::Error::new(
2285 io::ErrorKind::TimedOut,
2286 "initial subc attach retry budget elapsed during TCP connect",
2287 ),
2288 })?
2289 .map_err(|source| SubcError::Connect {
2290 endpoint: endpoint_label.clone(),
2291 source,
2292 })?;
2293 stream
2294 .set_nodelay(true)
2295 .map_err(|source| SubcError::Connect {
2296 endpoint: endpoint_label.clone(),
2297 source,
2298 })?;
2299
2300 let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2301 authenticate_client(&mut stream, &conn, auth_budget)
2302 .await
2303 .map_err(|source| SubcError::Auth {
2304 endpoint: endpoint_label,
2305 source,
2306 })?;
2307
2308 Ok(stream)
2309}
2310
2311#[allow(clippy::too_many_arguments)]
2312async fn process_route_bind_completion(
2313 writer_tx: &WriterSender,
2314 completion: RouteBindCompletion,
2315 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2316 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2317 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2318 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2319 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2320 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2321 installed_route_epochs: &mut HashMap<u16, u32>,
2322 executor: &Arc<Executor>,
2323 shutdown: &Arc<Notify>,
2324 metrics: &Arc<DispatchPathMetrics>,
2325) -> Result<(), SubcError> {
2326 decrement_counted_channel(&metrics.control_completion_queued);
2327 handle_route_bind_completion(
2328 writer_tx,
2329 completion,
2330 routes,
2331 root_channels,
2332 session_identity,
2333 push_buffer,
2334 live_roots,
2335 pending_binds,
2336 installed_route_epochs,
2337 executor,
2338 shutdown,
2339 metrics,
2340 )
2341 .await
2342}
2343
2344#[allow(clippy::too_many_arguments)]
2345async fn drain_pending_route_bind_completions(
2346 control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2347 writer_tx: &WriterSender,
2348 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2349 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2350 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2351 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2352 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2353 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2354 installed_route_epochs: &mut HashMap<u16, u32>,
2355 executor: &Arc<Executor>,
2356 shutdown: &Arc<Notify>,
2357 metrics: &Arc<DispatchPathMetrics>,
2358) -> Result<usize, SubcError> {
2359 let mut drained = 0;
2360 while let Ok(completion) = control_completion_rx.try_recv() {
2361 process_route_bind_completion(
2362 writer_tx,
2363 completion,
2364 routes,
2365 root_channels,
2366 session_identity,
2367 push_buffer,
2368 live_roots,
2369 pending_binds,
2370 installed_route_epochs,
2371 executor,
2372 shutdown,
2373 metrics,
2374 )
2375 .await?;
2376 drained += 1;
2377 }
2378 Ok(drained)
2379}
2380
2381async fn run_module_loop<R, W>(
2385 mut read: R,
2386 mut write: W,
2387 connection_file_path: &Path,
2388 shared_app: Arc<App>,
2389 executor: Arc<Executor>,
2390 dispatch: DispatchFn,
2391 user_config_path: Option<PathBuf>,
2392 allow_native_passthrough: bool,
2393 tool_response_body_limit: usize,
2394) -> Result<ModuleLoopExit, SubcError>
2395where
2396 R: AsyncRead + Unpin + Send + 'static,
2397 W: AsyncWrite + Unpin + Send + 'static,
2398{
2399 let hello = ModuleHelloBody {
2403 manifest: build_manifest(),
2404 protocol_ver: PROTOCOL_VERSION,
2405 control_ops: control_ops(),
2406 launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2407 };
2408 let hello_frame = Frame::build(
2409 FrameType::Hello,
2410 control_flags(),
2411 0,
2412 0,
2413 HELLO_CORR,
2414 serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2415 )
2416 .map_err(SubcError::FrameBuild)?;
2417 write_frame(&mut write, &hello_frame)
2418 .await
2419 .map_err(SubcError::FrameIo)?;
2420
2421 match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2423 None => return Err(SubcError::ClosedBeforeHelloAck),
2424 Some(frame) => match frame.header.ty {
2425 FrameType::HelloAck => {
2426 log::info!("subc attach: registered (HelloAck received)");
2427 }
2428 FrameType::Error => {
2429 let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2430 return Err(SubcError::HelloRejected { body });
2431 }
2432 other => return Err(SubcError::UnexpectedFrame { ty: other }),
2433 },
2434 }
2435
2436 let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2437 let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2438 let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2439 let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2446 let reader_task = spawn_reader_task(read, reader_tx);
2447 let shutdown = Arc::new(Notify::new());
2448 let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2457 let mut next_maintenance_at = next_drain_at;
2458 #[cfg(any(target_os = "macos", target_os = "linux"))]
2461 let mut last_slack_relief: Option<std::time::Instant> = None;
2462 let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2463 let (bash_deferred_tx, mut bash_deferred_rx) =
2464 mpsc::channel::<bash::BashDeferredCompletion>(256);
2465 let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2466 let (control_completion_tx, mut control_completion_rx) =
2467 mpsc::channel::<RouteBindCompletion>(256);
2468 let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2469 let lossy_overflow = Arc::new(push::LossyOverflow::default());
2470 let lossy_seq = Arc::new(AtomicU64::new(0));
2471 let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2472 let (fleet_status_client, fleet_status_task) =
2473 spawn_fleet_status_dial(connection_file_path, 64);
2474 let push_senders = PushSenders {
2475 lossy_tx,
2476 reliable_tx,
2477 lossy_overflow: Arc::clone(&lossy_overflow),
2478 lossy_seq,
2479 fleet_status_client: fleet_status_client.clone(),
2480 };
2481 let connection_cancel = PersistentCancelSignal::new();
2482 let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2483 let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2484 let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2485 let mut bg_sub_by_session: BgSubsBySession = HashMap::new();
2486 let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2487 let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2488 let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2489 let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2490 HashMap::new();
2491 let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2492 let mut retry_buffer: RetryBuffer = HashMap::new();
2493 let mut reclaimed_routes = ReclaimedRoutes::default();
2494 let mut completed_tasks = push::CompletedTaskIds::default();
2495 let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2496 let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2497 let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2498 let mut next_bash_ask_corr: u64 = 1;
2499 let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2500 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
2501 let health_rollup_cache = HealthRollupCache::new();
2502 health_rollup_cache.refresh(&executor, &shared_app);
2503 let mut next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2504
2505 let loop_result: Result<ModuleLoopExit, SubcError> = loop {
2506 shared_app.set_open_route_count(routes.len());
2507 if tokio::time::Instant::now() >= next_health_rollup_at {
2508 health_rollup_cache.refresh(&executor, &shared_app);
2509 next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2510 }
2511 crate::logging::perf_tick(Some(&executor));
2512 dispatch_path_metrics.mark_frame_loop_tick();
2513 if let Err(error) = expire_pending_bash_asks(
2514 &writer_tx,
2515 &mut pending_bash_asks,
2516 &routes,
2517 &mut live_roots,
2518 &mut route_bash_cancels,
2519 &shutdown,
2520 &dispatch_path_metrics,
2521 )
2522 .await
2523 {
2524 break Err(error);
2525 }
2526
2527 match drain_pending_route_bind_completions(
2531 &mut control_completion_rx,
2532 &writer_tx,
2533 &mut routes,
2534 &mut root_channels,
2535 &mut session_identity,
2536 &mut push_buffer,
2537 &mut live_roots,
2538 &mut pending_binds,
2539 &mut installed_route_epochs,
2540 &executor,
2541 &shutdown,
2542 &dispatch_path_metrics,
2543 )
2544 .await
2545 {
2546 Ok(drained) => {
2547 if drained > 0 {
2548 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2549 health_rollup_cache.refresh(&executor, &shared_app);
2550 next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2551 }
2552 }
2553 Err(error) => break Err(error),
2554 }
2555
2556 if tokio::time::Instant::now() >= next_drain_at {
2557 push::emit_bg_event_wakes(
2558 &writer_tx,
2559 &dispatch_path_metrics,
2560 &bg_subs,
2561 &mut bg_wake_pending,
2562 );
2563 warn_slow_pending_binds(&mut pending_binds, &executor);
2564 if let Err(error) = expire_overdue_route_binds(
2565 &writer_tx,
2566 &executor,
2567 &mut pending_binds,
2568 &mut installed_route_epochs,
2569 &dispatch_path_metrics,
2570 )
2571 .await
2572 {
2573 break Err(error);
2574 }
2575
2576 let retried = push::drain_retry_buffers_for_bound_routes(
2577 &writer_tx,
2578 &dispatch_path_metrics,
2579 &routes,
2580 &mut retry_buffer,
2581 );
2582 if retried > 0 {
2583 log::debug!(
2584 "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2585 );
2586 }
2587
2588 next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2589 }
2590
2591 let overflow_batch = lossy_overflow.drain();
2597 if !overflow_batch.is_empty() {
2598 let (_, deferred) = push::drain_reliable_push_turn(
2599 &writer_tx,
2600 &dispatch_path_metrics,
2601 &routes,
2602 &root_channels,
2603 &session_identity,
2604 &mut retry_buffer,
2605 &mut push_buffer,
2606 &mut completed_tasks,
2607 &bg_sub_by_session,
2608 &mut bg_wake_pending,
2609 &mut bg_wake_epoch,
2610 &mut reliable_rx,
2611 None,
2612 );
2613 if deferred {
2614 tokio::task::yield_now().await;
2615 }
2616
2617 let mut batch = Vec::new();
2618 while let Ok(item) = lossy_rx.try_recv() {
2619 batch.push(item);
2620 }
2621 batch.extend(overflow_batch);
2622 push::process_lossy_push_envelope_batch(
2623 &writer_tx,
2624 &dispatch_path_metrics,
2625 &routes,
2626 &root_channels,
2627 &completed_tasks,
2628 batch,
2629 );
2630 }
2631
2632 tokio::select! {
2633 biased;
2634 Some(completion) = control_completion_rx.recv() => {
2635 if let Err(error) = process_route_bind_completion(
2636 &writer_tx,
2637 completion,
2638 &mut routes,
2639 &mut root_channels,
2640 &mut session_identity,
2641 &mut push_buffer,
2642 &mut live_roots,
2643 &mut pending_binds,
2644 &mut installed_route_epochs,
2645 &executor,
2646 &shutdown,
2647 &dispatch_path_metrics,
2648 )
2649 .await
2650 {
2651 break Err(error);
2652 }
2653 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2654 next_health_rollup_at = tokio::time::Instant::now();
2655 }
2656 _ = shutdown.notified() => {
2657 log::warn!("subc attach: fatal executor response requested teardown");
2658 break Ok(ModuleLoopExit::SkipSearchFlush);
2659 }
2660 maybe_frame = reader_rx.recv() => {
2661 let frame = match maybe_frame {
2662 None => {
2663 log::info!("subc attach: daemon closed connection");
2664 break Ok(ModuleLoopExit::Graceful);
2665 }
2666 Some(Err(error)) => break Err(error),
2667 Some(Ok(frame)) => frame,
2668 };
2669 let phase_trace = frame.phase_trace;
2670 let frame = frame.frame;
2671
2672 if !ingress_route_should_be_processed(
2673 &installed_route_epochs,
2674 &reclaimed_routes,
2675 &frame,
2676 ) {
2677 log::debug!(
2678 "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2679 frame.header.ty,
2680 frame.header.channel,
2681 frame.header.epoch
2682 );
2683 continue;
2684 }
2685
2686 match frame.header.ty {
2687 FrameType::Ping if frame.header.channel == 0 => {
2688 let pong = match Frame::build_with_version(
2689 frame.header.ver,
2690 FrameType::Pong,
2691 frame.header.flags,
2692 0,
2693 0,
2694 frame.header.corr,
2695 Vec::new(),
2696 ) {
2697 Ok(pong) => pong,
2698 Err(error) => break Err(SubcError::FrameBuild(error)),
2699 };
2700 if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2701 break Err(error);
2702 }
2703 }
2704 FrameType::Goodbye if frame.header.channel == 0 => {
2705 log::info!("subc attach: received channel-0 Goodbye");
2706 break Ok(ModuleLoopExit::Graceful);
2707 }
2708 FrameType::Goodbye => {
2709 let channel = route_key(frame.header.channel, frame.header.epoch);
2710 if let Err(error) = teardown_installed_route(
2711 &writer_tx,
2712 &dispatch_path_metrics,
2713 &executor,
2714 channel,
2715 "Goodbye",
2716 None,
2717 &mut installed_route_epochs,
2718 &mut routes,
2719 &mut root_channels,
2720 &mut bg_subs,
2721 &mut bg_sub_by_session,
2722 &mut bg_wake_pending,
2723 &mut pending_bash_asks,
2724 &mut live_roots,
2725 &mut route_bash_cancels,
2726 &active_tool_calls,
2727 &mut pending_binds,
2728 &mut retry_buffer,
2729 &mut push_buffer,
2730 &shutdown,
2731 )
2732 .await
2733 {
2734 break Err(error);
2735 }
2736 }
2737 FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2738 if let Err(error) = handle_bash_elicitation_reply(
2739 &writer_tx,
2740 &frame,
2741 &mut pending_bash_asks,
2742 &routes,
2743 &mut live_roots,
2744 &executor,
2745 &shutdown,
2746 &bash_deferred_tx,
2747 &bash_poll_touch_tx,
2748 &dispatch_path_metrics,
2749 &mut route_bash_cancels,
2750 dispatch,
2751 )
2752 .await
2753 {
2754 break Err(error);
2755 }
2756 }
2757 FrameType::Request if frame.header.channel == 0 => {
2758 if let Err(error) = handle_control_request(
2759 &writer_tx,
2760 &frame,
2761 &shared_app,
2762 &executor,
2763 &mut live_roots,
2764 &mut pending_binds,
2765 &mut installed_route_epochs,
2766 &mut routes,
2767 &mut root_channels,
2768 &mut bg_subs,
2769 &mut bg_sub_by_session,
2770 &mut bg_wake_pending,
2771 &mut pending_bash_asks,
2772 &mut route_bash_cancels,
2773 &active_tool_calls,
2774 &mut retry_buffer,
2775 &mut push_buffer,
2776 &shutdown,
2777 &control_completion_tx,
2778 &dispatch_path_metrics,
2779 &health_rollup_cache,
2780 &push_senders,
2781 dispatch,
2782 user_config_path.as_deref(),
2783 )
2784 .await
2785 {
2786 break Err(error);
2787 }
2788 }
2789 FrameType::Request => {
2790 if let Err(error) = handle_tool_call(
2791 &writer_tx,
2792 &frame,
2793 phase_trace,
2794 &routes,
2795 &pending_binds,
2796 &mut live_roots,
2797 &executor,
2798 &active_tool_calls,
2799 &shutdown,
2800 &connection_cancel,
2801 &bash_deferred_tx,
2802 &bash_poll_touch_tx,
2803 &dispatch_path_metrics,
2804 &mut route_bash_cancels,
2805 &mut pending_bash_asks,
2806 &mut next_bash_ask_corr,
2807 &mut bg_subs,
2808 &mut bg_sub_by_session,
2809 &mut bg_wake_pending,
2810 &mut bg_wake_epoch,
2811 dispatch,
2812 allow_native_passthrough,
2813 tool_response_body_limit,
2814 )
2815 .await
2816 {
2817 break Err(error);
2818 }
2819 }
2820 FrameType::Cancel => {
2821 let channel = route_key(frame.header.channel, frame.header.epoch);
2822 cancel_active_tool_call(
2823 &active_tool_calls,
2824 executor.as_ref(),
2825 channel,
2826 frame.header.corr,
2827 "Cancel frame",
2828 );
2829 if bg_subs.contains_key(&channel) {
2830 if let Err(error) = end_bg_subscription(
2831 &writer_tx,
2832 &dispatch_path_metrics,
2833 &mut bg_subs,
2834 &mut bg_sub_by_session,
2835 &mut bg_wake_pending,
2836 channel,
2837 routes.get(&channel),
2838 "cancel",
2839 )
2840 .await
2841 {
2842 break Err(error);
2843 }
2844 }
2845 if let Err(error) = cancel_pending_bash_ask_for_tool_call(
2846 &writer_tx,
2847 &mut pending_bash_asks,
2848 channel,
2849 frame.header.corr,
2850 &routes,
2851 &mut live_roots,
2852 &mut route_bash_cancels,
2853 &shutdown,
2854 &dispatch_path_metrics,
2855 )
2856 .await
2857 {
2858 break Err(error);
2859 }
2860 }
2861 _ => {}
2865 }
2866 }
2867 Some((root_id, frame)) = reliable_rx.recv() => {
2868 let (_, deferred) = push::drain_reliable_push_turn(
2872 &writer_tx,
2873 &dispatch_path_metrics,
2874 &routes,
2875 &root_channels,
2876 &session_identity,
2877 &mut retry_buffer,
2878 &mut push_buffer,
2879 &mut completed_tasks,
2880 &bg_sub_by_session,
2881 &mut bg_wake_pending,
2882 &mut bg_wake_epoch,
2883 &mut reliable_rx,
2884 Some((root_id, frame)),
2885 );
2886 if deferred {
2887 tokio::task::yield_now().await;
2888 }
2889 }
2890 Some((order, root_id, frame)) = lossy_rx.recv() => {
2891 let (_, deferred) = push::drain_reliable_push_turn(
2895 &writer_tx,
2896 &dispatch_path_metrics,
2897 &routes,
2898 &root_channels,
2899 &session_identity,
2900 &mut retry_buffer,
2901 &mut push_buffer,
2902 &mut completed_tasks,
2903 &bg_sub_by_session,
2904 &mut bg_wake_pending,
2905 &mut bg_wake_epoch,
2906 &mut reliable_rx,
2907 None,
2908 );
2909 if deferred {
2910 tokio::task::yield_now().await;
2911 }
2912
2913 let mut batch = vec![(order, root_id, frame)];
2920 while let Ok(item) = lossy_rx.try_recv() {
2921 batch.push(item);
2922 }
2923 batch.extend(lossy_overflow.drain());
2924 push::process_lossy_push_envelope_batch(
2925 &writer_tx,
2926 &dispatch_path_metrics,
2927 &routes,
2928 &root_channels,
2929 &completed_tasks,
2930 batch,
2931 );
2932 }
2933 Some(done) = bash_deferred_rx.recv() => {
2934 decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
2935 if let Err(error) = bash::handle_bash_deferred_completion(
2936 &writer_tx,
2937 done,
2938 &routes,
2939 &mut live_roots,
2940 &mut route_bash_cancels,
2941 &shutdown,
2942 &dispatch_path_metrics,
2943 )
2944 .await
2945 {
2946 break Err(error);
2947 }
2948 }
2949 Some(root_id) = bash_poll_touch_rx.recv() => {
2950 decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
2951 if let Some(meta) = live_roots.get_mut(&root_id) {
2952 meta.note_activity();
2953 }
2954 }
2955 Some(completion) = maintenance_rx.recv() => {
2956 decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
2957 let root_id = completion.root_id.clone();
2958 let response = completion.response;
2959 let response_is_fatal = response_is_fatal_panic(&response);
2960 let bind_pending = pending_binds
2961 .values()
2962 .any(|pending| pending.bind_root_id == root_id);
2963 let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
2964 let defer_requeue = meta.unbound_quiesced || bind_pending;
2965 note_maintenance_completion(
2966 meta,
2967 completion.requeue_kind,
2968 response_is_fatal,
2969 defer_requeue,
2970 );
2971 should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
2972 } else {
2973 false
2974 };
2975 if requiesce {
2976 quiesce_unbound_root(&root_id, &mut live_roots, &executor);
2977 }
2978 push::clear_stale_bg_wakes_for_empty_sessions(
2979 &root_id,
2980 &completion.empty_bg_sessions,
2981 &bg_sub_by_session,
2982 &mut bg_wake_pending,
2983 &bg_wake_epoch,
2984 );
2985 if response_is_fatal {
2986 if let Some(meta) = live_roots.get_mut(&root_id) {
2987 meta.maintenance_poisoned = true;
2988 }
2989 log::warn!(
2990 "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
2991 );
2992 }
2993 }
2994 _ = tokio::time::sleep_until(next_drain_at) => {
2995 }
2998 _ = tokio::time::sleep_until(next_maintenance_at) => {
2999 crate::logging::maybe_sweep_logs();
3004 let reaped_lsp_children = shared_app
3005 .lsp_child_registry()
3006 .reap_children_with_gone_cwd_or_reclaimed_root();
3007 if reaped_lsp_children > 0 {
3008 log::warn!(
3009 "subc attach: reaped {reaped_lsp_children} LSP child process group(s) with a deleted cwd or reclaimed root"
3010 );
3011 }
3012 let reap = reap_idle_roots(
3013 Instant::now(),
3014 &mut live_roots,
3015 &pending_binds,
3016 &root_channels,
3017 &executor,
3018 &dispatch_path_metrics,
3019 );
3020 for root_id in &reap.forgotten_deleted_roots {
3021 purge_deleted_root_residents(
3022 root_id,
3023 &mut routes,
3024 &mut root_channels,
3025 &mut installed_route_epochs,
3026 &mut route_bash_cancels,
3027 &active_tool_calls,
3028 executor.as_ref(),
3029 &mut retry_buffer,
3030 &mut reclaimed_routes,
3031 &mut session_identity,
3032 &mut push_buffer,
3033 &mut bg_subs,
3034 &mut bg_sub_by_session,
3035 &mut bg_wake_pending,
3036 &mut bg_wake_epoch,
3037 &mut pending_bash_asks,
3038 &dispatch_path_metrics,
3039 );
3040 }
3041 if reap.evicted > 0 {
3042 log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
3043 }
3044 submit_due_maintenance_jobs(
3045 &executor,
3046 &mut live_roots,
3047 &pending_binds,
3048 &bg_sub_by_session,
3049 &bg_wake_pending,
3050 &bg_wake_epoch,
3051 &maintenance_tx,
3052 &dispatch_path_metrics,
3053 );
3054 #[cfg(any(target_os = "macos", target_os = "linux"))]
3061 {
3062 let now_std = std::time::Instant::now();
3063 if crate::memory::spawn_allocator_slack_relief_if_due(
3064 last_slack_relief,
3065 now_std,
3066 ) {
3067 last_slack_relief = Some(now_std);
3068 }
3069 }
3070 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3071 }
3072 }
3073 };
3074
3075 shared_app.set_open_route_count(0);
3076
3077 connection_cancel.cancel();
3078 quiesce_connection_roots(
3081 &mut live_roots,
3082 &mut pending_binds,
3083 &mut routes,
3084 &mut root_channels,
3085 &mut installed_route_epochs,
3086 &mut route_bash_cancels,
3087 &active_tool_calls,
3088 &executor,
3089 );
3090
3091 fleet_status_client.set_route_live(false);
3092 fleet_status_task.abort();
3093 let _ = fleet_status_task.await;
3094
3095 let mut loop_result = loop_result;
3096 if !pending_bash_asks.is_empty() {
3097 let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3098 if let Err(error) = settle_all_pending_bash_asks(
3099 &writer_tx,
3100 &mut pending_bash_asks,
3101 &no_routes,
3102 &mut live_roots,
3103 &mut route_bash_cancels,
3104 &shutdown,
3105 &dispatch_path_metrics,
3106 )
3107 .await
3108 {
3109 loop_result = loop_result.and(Err(error));
3110 }
3111 }
3112
3113 reader_task.abort();
3116 drop(writer_tx);
3117 let writer_result = finish_writer_task(writer_task).await;
3118 loop_result.and_then(|exit| writer_result.map(|_| exit))
3119}
3120
3121fn spawn_writer_task<W>(
3122 mut write: W,
3123 mut rx: mpsc::Receiver<WriterFrame>,
3124 metrics: Arc<DispatchPathMetrics>,
3125) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
3126where
3127 W: AsyncWrite + Unpin + Send + 'static,
3128{
3129 tokio::spawn(async move {
3130 let mut write_buffer = Vec::new();
3131 while let Some(mut queued) = rx.recv().await {
3132 let measure = queued.tool_response_trace.is_some();
3133 let dequeued = measure.then(Instant::now);
3134 metrics.writer_active.store(true, Ordering::Relaxed);
3135 decrement_counted_channel(&metrics.writer_queued);
3136 let write_timing = write_frame_contiguous(
3137 &mut write,
3138 queued.frame(),
3139 queued.body(),
3140 &mut write_buffer,
3141 measure,
3142 )
3143 .await;
3144 metrics.writer_active.store(false, Ordering::Relaxed);
3145 let write_timing = write_timing?;
3146
3147 if let (Some(trace), Some(dequeued), Some(write_timing)) =
3148 (queued.tool_response_trace.take(), dequeued, write_timing)
3149 {
3150 if let Some(completed) = trace.finish(
3151 dequeued,
3152 write_timing.write_started,
3153 write_timing.write_finished,
3154 write_timing.frame_bytes,
3155 ) {
3156 log_ctx::with_session(Some(completed.session), || {
3157 crate::logging::note_tool_call_trace(
3158 &completed.name,
3159 &completed.root,
3160 completed.channel,
3161 completed.corr,
3162 completed.phases,
3163 );
3164 });
3165 }
3166 }
3167 }
3168 Ok(())
3169 })
3170}
3171
3172struct FrameWriteTiming {
3173 write_started: Instant,
3174 write_finished: Instant,
3175 frame_bytes: usize,
3176}
3177
3178async fn write_frame_contiguous<W>(
3182 writer: &mut W,
3183 frame: &Frame,
3184 body: &[u8],
3185 buffer: &mut Vec<u8>,
3186 measure: bool,
3187) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
3188where
3189 W: AsyncWrite + Unpin,
3190{
3191 if frame.header.len as usize != body.len() {
3192 return Err(subc_transport::FrameIoError::BodyLengthMismatch {
3193 header_len: frame.header.len,
3194 body_len: body.len(),
3195 });
3196 }
3197
3198 let header = frame.header.encode();
3199 buffer.clear();
3200 buffer.reserve(header.len() + body.len());
3201 buffer.extend_from_slice(&header);
3202 buffer.extend_from_slice(body);
3203 let write_started = measure.then(Instant::now);
3204 writer
3205 .write_all(buffer)
3206 .await
3207 .map_err(subc_transport::FrameIoError::Io)?;
3208 Ok(write_started.map(|write_started| FrameWriteTiming {
3209 write_started,
3210 write_finished: Instant::now(),
3211 frame_bytes: buffer.len(),
3212 }))
3213}
3214
3215fn spawn_reader_task<R>(
3216 mut read: R,
3217 tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3218) -> JoinHandle<()>
3219where
3220 R: AsyncRead + Unpin + Send + 'static,
3221{
3222 tokio::spawn(async move {
3223 loop {
3224 match read_frame(&mut read).await {
3225 Ok(Some(frame)) => {
3226 let decoded = DecodedFrame {
3227 frame,
3228 phase_trace: PhaseTrace::new(Instant::now()),
3229 };
3230 if tx.send(Ok(decoded)).await.is_err() {
3231 return;
3232 }
3233 }
3234 Ok(None) => {
3235 return;
3237 }
3238 Err(error) => {
3239 if let subc_transport::FrameIoError::Io(io_error) = &error {
3246 if matches!(
3247 io_error.kind(),
3248 std::io::ErrorKind::ConnectionReset
3249 | std::io::ErrorKind::ConnectionAborted
3250 ) {
3251 log::info!(
3252 "subc attach: connection reset by daemon; treating as close"
3253 );
3254 return;
3255 }
3256 }
3257 let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3258 return;
3259 }
3260 }
3261 }
3262 })
3263}
3264
3265async fn finish_writer_task(
3266 mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3267) -> Result<(), SubcError> {
3268 match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3269 Ok(Ok(Ok(()))) => Ok(()),
3270 Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3271 Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3272 Err(_) => {
3273 writer_task.abort();
3274 Ok(())
3275 }
3276 }
3277}
3278
3279fn register_actor_for_bind(
3280 shared_app: &Arc<App>,
3281 executor: &Arc<Executor>,
3282 push_senders: &PushSenders,
3283 bind_root_id: &ProjectRootId,
3284 route_channel: u16,
3285 root_was_live: bool,
3286) -> bool {
3287 if executor.actor_registered(bind_root_id) {
3288 log::debug!(
3289 "subc attach: reusing actor for route {} root {}",
3290 route_channel,
3291 bind_root_id.as_path().display()
3292 );
3293 return false;
3294 }
3295
3296 if root_was_live {
3297 log::warn!(
3298 "subc attach: recreating missing actor for live root {} on route {}",
3299 bind_root_id.as_path().display(),
3300 route_channel
3301 );
3302 }
3303
3304 let actor_ctx = Arc::new(AppContext::from_app(
3305 Arc::clone(shared_app),
3306 Config::default(),
3307 ));
3308 install_bash_compressor(&actor_ctx);
3309 actor_ctx.install_fleet_status_client(Some(push_senders.fleet_status_client.clone()));
3310 actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
3311 push_senders.clone(),
3312 bind_root_id.clone(),
3313 )));
3314 let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
3315 drop(actor_ctx);
3316 if inserted {
3317 log::debug!(
3321 "subc attach: registered actor for route {} root {}",
3322 route_channel,
3323 bind_root_id.as_path().display()
3324 );
3325 } else {
3326 log::debug!(
3327 "subc attach: actor appeared while binding route {} root {}; reusing it",
3328 route_channel,
3329 bind_root_id.as_path().display()
3330 );
3331 }
3332 inserted
3333}
3334
3335fn rollback_pending_bind_actor(
3336 executor: &Arc<Executor>,
3337 live_roots: &HashMap<ProjectRootId, RootMeta>,
3338 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3339 root_id: &ProjectRootId,
3340 inserted_new_actor: bool,
3341) {
3342 if !inserted_new_actor || live_roots.contains_key(root_id) {
3343 return;
3344 }
3345
3346 if let Some((route, pending)) = pending_binds
3347 .iter_mut()
3348 .find(|(_, pending)| &pending.bind_root_id == root_id)
3349 {
3350 pending.inserted_new_actor = true;
3351 log::debug!(
3352 "subc attach: transferred rollback ownership for root {} to pending route {}",
3353 root_id.as_path().display(),
3354 route
3355 );
3356 return;
3357 }
3358
3359 executor.remove_actor(root_id);
3360}
3361
3362fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
3363 match response.data.get("code").and_then(|code| code.as_str()) {
3364 Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
3369 Some("cache_key_probe_failed") => "cache_key_probe_failed",
3373 Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
3377 _ => "config_divergence",
3378 }
3379}
3380
3381fn queue_post_bind_configure_and_completion_maintenance(
3382 root_id: &ProjectRootId,
3383 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3384) {
3385 let Some(meta) = live_roots.get_mut(root_id) else {
3386 return;
3387 };
3388 if meta.maintenance_poisoned || meta.maintenance_pending {
3389 return;
3390 }
3391
3392 meta.maintenance_pending = true;
3393 meta.maintenance_queued_kinds
3394 .push_back(MaintenanceDrainKind::ConfigureTail);
3395 meta.maintenance_queued_kinds
3396 .push_back(MaintenanceDrainKind::CompletionDrains);
3397}
3398
3399#[allow(clippy::too_many_arguments)]
3400async fn handle_route_bind_completion(
3401 tx: &WriterSender,
3402 completion: RouteBindCompletion,
3403 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3404 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3405 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
3406 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3407 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3408 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3409 installed_route_epochs: &mut HashMap<u16, u32>,
3410 executor: &Arc<Executor>,
3411 shutdown: &Arc<Notify>,
3412 metrics: &Arc<DispatchPathMetrics>,
3413) -> Result<(), SubcError> {
3414 let route_id = completion.route;
3415 let Some(pending) = pending_binds.remove(&route_id) else {
3416 log::warn!(
3417 "subc attach: dropping RouteBind completion for non-pending route {}",
3418 completion.route
3419 );
3420 rollback_pending_bind_actor(
3421 executor,
3422 live_roots,
3423 pending_binds,
3424 &completion.bind_root_id,
3425 completion.inserted_new_actor,
3426 );
3427 let has_pending_bind = pending_binds
3428 .values()
3429 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3430 if !root_channels
3431 .get(&completion.bind_root_id)
3432 .is_some_and(|channels| !channels.is_empty())
3433 && !has_pending_bind
3434 {
3435 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3436 }
3437 remove_installed_route(installed_route_epochs, route_id);
3438 return Ok(());
3439 };
3440
3441 if pending.bind_root_id != completion.bind_root_id {
3442 log::warn!(
3443 "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
3444 completion.route,
3445 pending.bind_root_id.as_path().display(),
3446 completion.bind_root_id.as_path().display()
3447 );
3448 }
3449
3450 let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
3451 if pending.cancelled {
3452 rollback_pending_bind_actor(
3453 executor,
3454 live_roots,
3455 pending_binds,
3456 &completion.bind_root_id,
3457 inserted_new_actor,
3458 );
3459 let has_pending_bind = pending_binds
3460 .values()
3461 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3462 if !root_channels
3463 .get(&completion.bind_root_id)
3464 .is_some_and(|channels| !channels.is_empty())
3465 && !has_pending_bind
3466 {
3467 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3468 }
3469 log::debug!(
3470 "subc attach: discarded completed RouteBind for cancelled route {} root {}",
3471 completion.route,
3472 completion.bind_root_id.as_path().display()
3473 );
3474 remove_installed_route(installed_route_epochs, route_id);
3475 return Ok(());
3476 }
3477
3478 let failure = if !completion.configure_response.success {
3479 Some((
3480 &completion.configure_response,
3481 "configure failed during route bind",
3482 ))
3483 } else {
3484 None
3485 };
3486
3487 if let Some((response, fallback)) = failure {
3488 rollback_pending_bind_actor(
3489 executor,
3490 live_roots,
3491 pending_binds,
3492 &completion.bind_root_id,
3493 inserted_new_actor,
3494 );
3495 let has_pending_bind = pending_binds
3496 .values()
3497 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3498 if !root_channels
3499 .get(&completion.bind_root_id)
3500 .is_some_and(|channels| !channels.is_empty())
3501 && !has_pending_bind
3502 {
3503 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3504 }
3505 let message = response_message(response, fallback);
3506 let fatal = response_is_fatal_panic(response);
3507 let error_code = route_bind_error_code_for_configure_response(response);
3508 send_route_bind_error_parts(
3509 tx,
3510 completion.ver,
3511 completion.corr,
3512 completion.flags,
3513 error_code,
3514 &message,
3515 metrics,
3516 )
3517 .await?;
3518 remove_installed_route(installed_route_epochs, route_id);
3519 if fatal {
3520 signal_fatal_teardown(
3521 tx,
3522 Some(completion.route),
3523 completion.ver,
3524 completion.corr,
3525 shutdown,
3526 metrics,
3527 )
3528 .await;
3529 }
3530 return Ok(());
3531 }
3532
3533 remember_session_identity(session_identity, &completion.identity);
3534 let replay_key = push::ReplayKey::from_identity(&completion.identity);
3535 let bind_trust = completion.identity.trust;
3536 insert_route_channel(routes, root_channels, route_id, completion.identity);
3537 let restore_watcher = live_roots
3538 .get(&completion.bind_root_id)
3539 .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3540 live_roots
3541 .entry(completion.bind_root_id.clone())
3542 .and_modify(|meta| {
3543 meta.reactivate_bound();
3544 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3545 meta.maintenance_poisoned = false;
3546 })
3547 .or_insert_with(|| RootMeta::new(Instant::now()));
3548 if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3549 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3550 meta.maintenance_poisoned = false;
3551 }
3552 if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3553 ctx.mark_subc_bound();
3554 if restore_watcher {
3555 crate::commands::configure::ensure_project_watcher(&ctx);
3556 }
3557 }
3558
3559 let ack =
3560 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3561 let response = Frame::build_with_version(
3562 completion.ver,
3563 FrameType::Response,
3564 control_flags(),
3565 0,
3566 0,
3567 completion.corr,
3568 ack,
3569 )
3570 .map_err(SubcError::FrameBuild)?;
3571 send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3572 queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3573 let replayed = push::replay_buffered_push_frames(
3574 tx,
3575 metrics,
3576 route_id,
3577 push_buffer,
3578 &replay_key,
3579 bind_trust,
3580 );
3581 if replayed > 0 {
3582 log::debug!(
3583 "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3584 replayed,
3585 completion.route,
3586 replay_key.root.as_path().display(),
3587 replay_key.harness,
3588 replay_key.session
3589 );
3590 }
3591 log::info!(
3592 "subc attach: route {} bound to root {}",
3593 completion.route,
3594 completion.bind_root_id.as_path().display()
3595 );
3596 Ok(())
3597}
3598
3599async fn expire_overdue_route_binds(
3600 tx: &WriterSender,
3601 executor: &Arc<Executor>,
3602 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3603 installed_route_epochs: &mut HashMap<u16, u32>,
3604 metrics: &DispatchPathMetrics,
3605) -> Result<(), SubcError> {
3606 let now = Instant::now();
3607 let expired: Vec<_> = pending_binds
3608 .iter()
3609 .filter_map(|(route, pending)| {
3610 let age = now.saturating_duration_since(pending.started_at);
3611 (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3612 (
3613 *route,
3614 pending.corr,
3615 pending.ver,
3616 pending.flags,
3617 pending.bind_root_id.clone(),
3618 pending.configure_request_id.clone(),
3619 age,
3620 )
3621 })
3622 })
3623 .collect();
3624
3625 for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3626 if let Some(pending) = pending_binds.get_mut(&route) {
3627 pending.cancelled = true;
3628 pending.deadline_reported = true;
3629 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3630 log::debug!(
3631 "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3632 );
3633 }
3634 remove_installed_route(installed_route_epochs, route);
3635 let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3636 let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3637 send_route_bind_error_parts(
3638 tx,
3639 ver,
3640 corr,
3641 flags,
3642 "actor_not_ready",
3643 &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3644 metrics,
3645 )
3646 .await?;
3647 log::warn!(
3648 "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3649 route,
3650 root_id.as_path().display(),
3651 deadline_ms,
3652 configure_request_id
3653 );
3654 }
3655
3656 Ok(())
3657}
3658
3659#[allow(clippy::too_many_arguments)]
3664async fn handle_control_request(
3665 tx: &WriterSender,
3666 frame: &Frame,
3667 shared_app: &Arc<App>,
3668 executor: &Arc<Executor>,
3669 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3670 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3671 installed_route_epochs: &mut HashMap<u16, u32>,
3672 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3673 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3674 bg_subs: &mut HashMap<RouteChannel, BgSub>,
3675 bg_sub_by_session: &mut BgSubsBySession,
3676 bg_wake_pending: &mut HashSet<RouteChannel>,
3677 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3678 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3679 active_tool_calls: &ActiveToolCalls,
3680 retry_buffer: &mut RetryBuffer,
3681 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3682 shutdown: &Arc<Notify>,
3683 control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3684 metrics: &Arc<DispatchPathMetrics>,
3685 health_rollup_cache: &HealthRollupCache,
3686 push_senders: &PushSenders,
3687 dispatch: DispatchFn,
3688 user_config_path: Option<&Path>,
3689) -> Result<(), SubcError> {
3690 let request =
3691 serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3692 match request {
3693 ModuleControlRequest::RouteBind {
3694 route_channel,
3695 epoch,
3696 target: _,
3697 identity,
3698 principal,
3699 consumer_capabilities,
3700 admission_facts: _,
3701 } => {
3702 let route_id = route_key(route_channel, epoch);
3703 if epoch == 0 {
3704 return send_route_bind_error(
3705 tx,
3706 frame,
3707 "config_divergence",
3708 "route bind uses an invalid channel generation",
3709 metrics,
3710 )
3711 .await;
3712 }
3713 let mut bind_root_id = None;
3714 if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3715 if installed_epoch >= epoch {
3716 return send_route_bind_error(
3717 tx,
3718 frame,
3719 "config_divergence",
3720 "route bind generation is not newer than the installed generation",
3721 metrics,
3722 )
3723 .await;
3724 }
3725
3726 let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3727 Ok(root_id) => root_id,
3728 Err(error) => {
3729 return send_route_bind_error(
3730 tx,
3731 frame,
3732 "config_divergence",
3733 &format!("invalid route project root: {error}"),
3734 metrics,
3735 )
3736 .await;
3737 }
3738 };
3739 teardown_installed_route(
3740 tx,
3741 metrics,
3742 executor,
3743 route_key(route_channel, installed_epoch),
3744 "higher-epoch RouteBind",
3745 Some(&replacement_root),
3746 installed_route_epochs,
3747 routes,
3748 root_channels,
3749 bg_subs,
3750 bg_sub_by_session,
3751 bg_wake_pending,
3752 pending_bash_asks,
3753 live_roots,
3754 route_bash_cancels,
3755 active_tool_calls,
3756 pending_binds,
3757 retry_buffer,
3758 push_buffer,
3759 shutdown,
3760 )
3761 .await?;
3762 bind_root_id = Some(replacement_root);
3763 }
3764 if pending_binds.contains_key(&route_id) {
3765 return send_route_bind_error(
3766 tx,
3767 frame,
3768 "config_divergence",
3769 "route bind is already pending for channel",
3770 metrics,
3771 )
3772 .await;
3773 }
3774 let bind_root_id = match bind_root_id {
3775 Some(root_id) => root_id,
3776 None => match ProjectRootId::from_path(&identity.project_root) {
3777 Ok(root_id) => root_id,
3778 Err(error) => {
3779 return send_route_bind_error(
3780 tx,
3781 frame,
3782 "config_divergence",
3783 &format!("invalid route project root: {error}"),
3784 metrics,
3785 )
3786 .await;
3787 }
3788 },
3789 };
3790
3791 let request_id = format!("subc-bind-{route_channel}");
3794 let bind_project_root = identity.project_root.clone();
3795 let bind_harness = identity.harness.clone();
3796 let bind_session = identity.session.clone();
3797 let bind_trust = trust_for_bind(&bind_harness, &principal);
3798 let bind_principal_id = principal_id(&principal);
3799 let consumer_elicitation_capable = consumer_capabilities
3804 .as_ref()
3805 .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
3806 log::info!(
3807 "subc attach: route {} harness={} principal={} trust={} elicitation={}",
3808 route_channel,
3809 bind_harness,
3810 principal_label(&principal),
3811 bind_trust.label(),
3812 consumer_elicitation_capable
3813 );
3814
3815 let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
3827 user_config_path,
3828 Path::new(&bind_project_root),
3829 );
3830 let config_tiers: Vec<Value> = local_tiers
3831 .iter()
3832 .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
3833 .collect();
3834 let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
3835 let configure_json = json!({
3836 "id": request_id,
3837 "command": "configure",
3838 "project_root": bind_project_root,
3839 "harness": bind_harness,
3840 "session_id": bind_session.clone(),
3841 "config": config_tiers,
3842 });
3843 let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
3844 Ok(req) => req,
3845 Err(error) => {
3846 return send_route_bind_error(
3847 tx,
3848 frame,
3849 "config_divergence",
3850 &format!("failed to build configure request: {error}"),
3851 metrics,
3852 )
3853 .await;
3854 }
3855 };
3856
3857 let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
3858 root: bind_root_id.clone(),
3859 project_root: PathBuf::from(&bind_project_root),
3860 harness: bind_harness.clone(),
3861 session: bind_session.clone(),
3862 trust: bind_trust,
3863 spawn_principal: AuthenticatedPrincipal::RouteBind {
3864 trust: bind_trust.sandbox_trust(),
3865 route_channel,
3866 route_epoch: epoch,
3867 project_root: PathBuf::from(&bind_project_root),
3868 harness: bind_harness.clone(),
3869 session_id: bind_session.clone(),
3870 principal_id: bind_principal_id,
3871 },
3872 consumer_elicitation_capable,
3873 }));
3874 let configure_session = route_identity.session.clone();
3875 let root_was_live = live_roots.contains_key(&bind_root_id);
3876 let inserted_new_actor = register_actor_for_bind(
3877 shared_app,
3878 executor,
3879 push_senders,
3880 &bind_root_id,
3881 route_channel,
3882 root_was_live,
3883 );
3884
3885 let configure_request_id = configure_req.id.clone();
3886 installed_route_epochs.insert(route_channel, epoch);
3887 if let Some(meta) = live_roots.get_mut(&bind_root_id) {
3888 meta.maintenance_queued_kinds.clear();
3889 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
3890 }
3891 let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
3892 bind_root_id.clone(),
3893 Lane::Mutating,
3894 configure_request_id.clone(),
3895 Box::new(move |ctx| {
3896 log_ctx::with_session(Some(configure_session.clone()), || {
3897 dispatch(configure_req, ctx)
3898 })
3899 }),
3900 );
3901 pending_binds.insert(
3902 route_id,
3903 PendingBind {
3904 bind_root_id: bind_root_id.clone(),
3905 inserted_new_actor,
3906 cancelled: false,
3907 configure_request_id: configure_request_id.clone(),
3908 started_at: Instant::now(),
3909 warned_half_deadline: false,
3910 deadline_reported: false,
3911 corr: frame.header.corr,
3912 ver: frame.header.ver,
3913 flags: frame.header.flags,
3914 cancellation: configure_cancellation,
3915 },
3916 );
3917
3918 let completion_tx = control_completion_tx.clone();
3919 let completion_identity = route_identity;
3920 let completion_root = bind_root_id.clone();
3921 let completion_route_channel = route_channel;
3922 let completion_ver = frame.header.ver;
3923 let completion_corr = frame.header.corr;
3924 let completion_flags = frame.header.flags;
3925 let completion_metrics = Arc::clone(metrics);
3926 tokio::spawn(async move {
3927 let _response_task = ResponseTaskGuard::new(&completion_metrics);
3928 let configure_response =
3929 await_executor_response(configure_rx, configure_request_id.clone()).await;
3930 let completion = RouteBindCompletion {
3935 route: route_key(completion_route_channel, epoch),
3936 identity: completion_identity,
3937 bind_root_id: completion_root,
3938 inserted_new_actor,
3939 configure_response,
3940 diagnostics_on_edit,
3941 ver: completion_ver,
3942 corr: completion_corr,
3943 flags: completion_flags,
3944 };
3945 if send_counted_channel(
3946 &completion_tx,
3947 &completion_metrics.control_completion_queued,
3948 completion,
3949 )
3950 .await
3951 .is_err()
3952 {
3953 log::debug!(
3954 "subc attach: dropped RouteBind completion for route {} after loop exit",
3955 completion_route_channel
3956 );
3957 }
3958 });
3959
3960 health_rollup_cache.refresh(executor, shared_app);
3961 Ok(())
3962 }
3963 ModuleControlRequest::HealthCheck {} => {
3964 metrics.record_bg_runtime(bg_subs.len(), bg_wake_pending.len());
3965 let report = build_health_report(
3966 health_rollup_cache,
3967 executor,
3968 pending_binds,
3969 metrics,
3970 shared_app,
3971 );
3972 let body = serde_json::to_vec(&ModuleControlResponse::from(report))
3973 .map_err(SubcError::Json)?;
3974 let response = Frame::build_with_version(
3975 frame.header.ver,
3976 FrameType::Response,
3977 frame.header.flags,
3978 0,
3979 0,
3980 frame.header.corr,
3981 body,
3982 )
3983 .map_err(SubcError::FrameBuild)?;
3984 send_frame(tx, metrics, response).await
3985 }
3986 }
3987}
3988
3989fn install_bash_compressor(ctx: &AppContext) {
3990 let filter_registry_handle = ctx.shared_filter_registry();
3992 let compress_flag = ctx.bash_compress_flag();
3993 ctx.bash_background().set_compressor_with_exit_code(
3994 move |command: &str, output: String, exit_code: Option<i32>| {
3995 if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
3996 return crate::compress::CompressionResult::new(output);
3997 }
3998 let registry_guard = match filter_registry_handle.read() {
3999 Ok(g) => g,
4000 Err(poisoned) => poisoned.into_inner(),
4001 };
4002 crate::compress::compress_with_registry_exit_code(
4003 command,
4004 &output,
4005 exit_code,
4006 ®istry_guard,
4007 )
4008 },
4009 );
4010}
4011
4012fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
4013 let mut diagnostics_on_edit = false;
4014 for tier in tiers {
4015 if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
4016 diagnostics_on_edit = value;
4017 }
4018 }
4019 diagnostics_on_edit
4020}
4021
4022fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
4023 let stripped = strip_jsonc(doc);
4024 let value = serde_json::from_str::<Value>(&stripped).ok()?;
4025 value
4026 .get("lsp")
4027 .and_then(Value::as_object)?
4028 .get("diagnostics_on_edit")
4029 .and_then(Value::as_bool)
4030}
4031
4032async fn send_route_bind_error(
4033 tx: &WriterSender,
4034 frame: &Frame,
4035 code: &str,
4036 message: &str,
4037 metrics: &DispatchPathMetrics,
4038) -> Result<(), SubcError> {
4039 send_route_bind_error_parts(
4040 tx,
4041 frame.header.ver,
4042 frame.header.corr,
4043 frame.header.flags,
4044 code,
4045 message,
4046 metrics,
4047 )
4048 .await
4049}
4050
4051async fn send_route_bind_error_parts(
4052 tx: &WriterSender,
4053 ver: u8,
4054 corr: u64,
4055 flags: Flags,
4056 code: &str,
4057 message: &str,
4058 metrics: &DispatchPathMetrics,
4059) -> Result<(), SubcError> {
4060 let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
4061 send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
4062 log_route_bind_rejection(code, message);
4063 Ok(())
4064}
4065
4066fn log_route_bind_rejection(code: &str, message: &str) {
4074 const WINDOW: Duration = Duration::from_secs(60);
4075 static SUPPRESSED: OnceLock<StdMutex<HashMap<String, (Instant, u64)>>> = OnceLock::new();
4076 let map = SUPPRESSED.get_or_init(|| StdMutex::new(HashMap::new()));
4077 let mut map = match map.try_lock() {
4078 Ok(map) => map,
4079 Err(_) => {
4081 log::warn!("subc attach: route bind rejected ({code}): {message}");
4082 return;
4083 }
4084 };
4085 let now = Instant::now();
4086 if map.len() > 512 {
4090 map.retain(|_, (start, _)| now.duration_since(*start) < WINDOW);
4091 }
4092 match map.get_mut(message) {
4093 Some((window_start, suppressed)) if now.duration_since(*window_start) < WINDOW => {
4094 *suppressed += 1;
4095 }
4096 Some((window_start, suppressed)) => {
4097 if *suppressed > 0 {
4098 log::warn!(
4099 "subc attach: route bind rejected ({code}): {message} (repeated {}x in last 60s)",
4100 *suppressed
4101 );
4102 } else {
4103 log::warn!("subc attach: route bind rejected ({code}): {message}");
4104 }
4105 *window_start = now;
4106 *suppressed = 0;
4107 }
4108 None => {
4109 log::warn!("subc attach: route bind rejected ({code}): {message}");
4110 map.insert(message.to_string(), (now, 0));
4111 }
4112 }
4113}
4114
4115async fn handle_tool_call(
4120 tx: &WriterSender,
4121 frame: &Frame,
4122 mut phase_trace: PhaseTrace,
4123 routes: &HashMap<RouteChannel, RouteIdentity>,
4124 pending_binds: &HashMap<RouteChannel, PendingBind>,
4125 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4126 executor: &Arc<Executor>,
4127 active_tool_calls: &ActiveToolCalls,
4128 shutdown: &Arc<Notify>,
4129 connection_cancel: &PersistentCancelSignal,
4130 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
4131 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
4132 metrics: &Arc<DispatchPathMetrics>,
4133 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
4134 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
4135 next_bash_ask_corr: &mut u64,
4136 bg_subs: &mut HashMap<RouteChannel, BgSub>,
4137 bg_sub_by_session: &mut BgSubsBySession,
4138 bg_wake_pending: &mut HashSet<RouteChannel>,
4139 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
4140 dispatch: DispatchFn,
4141 allow_native_passthrough: bool,
4142 tool_response_body_limit: usize,
4143) -> Result<(), SubcError> {
4144 let route_id = route_key(frame.header.channel, frame.header.epoch);
4145 if pending_binds.contains_key(&route_id) {
4146 let error = build_error_frame(
4147 frame.header.ver,
4148 frame.header.channel,
4149 frame.header.epoch,
4150 frame.header.corr,
4151 frame.header.flags,
4152 "route_not_bound",
4153 "route is not bound before tool call",
4154 )?;
4155 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
4156 }
4157
4158 let Some(identity) = routes.get(&route_id).cloned() else {
4159 let error = build_error_frame(
4160 frame.header.ver,
4161 frame.header.channel,
4162 frame.header.epoch,
4163 frame.header.corr,
4164 frame.header.flags,
4165 "route_not_bound",
4166 "route is not bound before tool call",
4167 )?;
4168 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
4169 };
4170 let restore_watcher = live_roots
4171 .get(&identity.root)
4172 .is_some_and(|meta| meta.idle_artifacts_evicted);
4173 if let Some(meta) = live_roots.get_mut(&identity.root) {
4174 meta.reactivate_bound();
4175 }
4176 if restore_watcher {
4177 if let Some(ctx) = executor.actor_context(&identity.root) {
4178 crate::commands::configure::ensure_project_watcher(&ctx);
4179 }
4180 }
4181
4182 let route_request =
4183 serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
4184 if matches!(
4185 route_request,
4186 RouteRequest::BgEvents(BgEventsRequest {
4187 op: BgEventsOp::BgEvents
4188 })
4189 ) {
4190 if let Some(old_sub) = bg_subs.get(&route_id).cloned() {
4191 metrics.record_bg_subscription_ended(
4192 &old_sub.root,
4193 &old_sub.session,
4194 route_id,
4195 "resubscribe",
4196 );
4197 push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
4198 }
4199 if !identity.trust.allows_bash_observation() {
4200 bg_subs.remove(&route_id);
4201 bg_wake_pending.remove(&route_id);
4202 remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
4203 let denied_sub = BgSub {
4204 corr: frame.header.corr,
4205 ver: frame.header.ver,
4206 flags: frame.header.flags,
4207 root: identity.root.clone(),
4208 session: identity.session.clone(),
4209 };
4210 metrics.record_bg_subscription_ended(
4211 &identity.root,
4212 &identity.session,
4213 route_id,
4214 "subscribe-denied",
4215 );
4216 push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
4217 return Ok(());
4218 }
4219 bg_subs.insert(
4220 route_id,
4221 BgSub {
4222 corr: frame.header.corr,
4223 ver: frame.header.ver,
4224 flags: frame.header.flags,
4225 root: identity.root.clone(),
4226 session: identity.session.clone(),
4227 },
4228 );
4229 insert_bg_subscription_index(
4230 bg_sub_by_session,
4231 identity.root.clone(),
4232 identity.session.clone(),
4233 route_id,
4234 );
4235 metrics.record_bg_subscription_installed(&identity.root, &identity.session, route_id);
4236 push::arm_bg_wake(
4237 identity.root.clone(),
4238 identity.session.clone(),
4239 route_id,
4240 bg_wake_pending,
4241 bg_wake_epoch,
4242 metrics,
4243 );
4244 return Ok(());
4245 }
4246
4247 let RouteRequest::ToolCall(call) = route_request else {
4248 unreachable!("background event subscription returned above")
4249 };
4250 let bare_name = call.name;
4251 let arguments = strip_agent_preview_arg_owned(call.arguments);
4252 let format_context = crate::subc_format::FormatContext::from_tool_call(
4253 &bare_name,
4254 &arguments,
4255 identity.project_root.as_path(),
4256 );
4257
4258 let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
4259 let bind_trust = identity.trust;
4260 let diagnostics_on_edit = live_roots
4261 .get(&identity.root)
4262 .map(|meta| meta.diagnostics_on_edit)
4263 .unwrap_or(false);
4264
4265 let requests_host = bare_name == "bash"
4266 && arguments
4267 .get("sandbox")
4268 .or_else(|| {
4269 arguments
4270 .get("params")
4271 .and_then(|params| params.get("sandbox"))
4272 })
4273 .and_then(Value::as_str)
4274 == Some("host");
4275 if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
4276 let response = Response::error(
4277 request_id.clone(),
4278 "sandbox_escalation_denied",
4279 "sandbox host escalation is unavailable to untrusted principals",
4280 );
4281 let text = crate::subc_format::format_response_with_context(
4282 &bare_name,
4283 &response,
4284 &format_context,
4285 );
4286 let result = ToolCallResult { text, response };
4287 let response_frame = build_tool_response_frame_with_limit(
4288 frame.header.ver,
4289 route_id,
4290 frame.header.corr,
4291 frame.header.flags,
4292 &result,
4293 bind_trust,
4294 tool_response_body_limit,
4295 )?;
4296 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4297 }
4298
4299 if matches!(bind_trust, BindTrust::Untrusted)
4300 && is_bash_family_tool(&bare_name)
4301 && (bare_name != "bash" || !identity.consumer_elicitation_capable)
4302 {
4303 let response = bash::bash_denied_untrusted_response(request_id.clone());
4304 let text = crate::subc_format::format_response_with_context(
4305 &bare_name,
4306 &response,
4307 &format_context,
4308 );
4309 let result = ToolCallResult { text, response };
4310 let response_frame = build_tool_response_frame_with_limit(
4311 frame.header.ver,
4312 route_id,
4313 frame.header.corr,
4314 frame.header.flags,
4315 &result,
4316 bind_trust,
4317 tool_response_body_limit,
4318 )?;
4319 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4320 }
4321
4322 if !is_subc_agent_core_tool(&bare_name)
4330 && !is_subc_native_plumbing_tool(&bare_name)
4331 && !allow_native_passthrough
4332 {
4333 log::warn!(
4334 "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
4335 bare_name,
4336 frame.header.channel
4337 );
4338 let response = Response::error(
4339 request_id.clone(),
4340 "unknown_tool",
4341 format!("tool {:?} is not in the AFT tool manifest", bare_name),
4342 );
4343 let text = crate::subc_format::format_response_with_context(
4344 &bare_name,
4345 &response,
4346 &format_context,
4347 );
4348 let result = ToolCallResult { text, response };
4349 let response_frame = build_tool_response_frame_with_limit(
4350 frame.header.ver,
4351 route_id,
4352 frame.header.corr,
4353 frame.header.flags,
4354 &result,
4355 bind_trust,
4356 tool_response_body_limit,
4357 )?;
4358 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4359 }
4360
4361 if bare_name == "bash" {
4362 if matches!(bind_trust, BindTrust::Untrusted) {
4363 let plan = match bash::prepare_bash_elicitation_plan(
4364 &arguments,
4365 identity.project_root.as_path(),
4366 ) {
4367 Ok(plan) => plan,
4368 Err(error) => {
4369 let response = Response::error(request_id.clone(), error.code, error.message);
4370 let text = crate::subc_format::format_response_with_context(
4371 &bare_name,
4372 &response,
4373 &format_context,
4374 );
4375 let result = ToolCallResult { text, response };
4376 let response_frame = build_tool_response_frame_with_limit(
4377 frame.header.ver,
4378 route_id,
4379 frame.header.corr,
4380 frame.header.flags,
4381 &result,
4382 bind_trust,
4383 tool_response_body_limit,
4384 )?;
4385 return send_reliable_writer_frame(
4386 tx,
4387 metrics,
4388 response_frame,
4389 "tool response",
4390 )
4391 .await;
4392 }
4393 };
4394
4395 let reverse_corr =
4396 allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
4397 let ask_frame = build_bash_elicitation_request_frame(
4398 frame.header.ver,
4399 route_id,
4400 reverse_corr,
4401 frame.header.flags,
4402 &plan.command,
4403 &plan.asks,
4404 )?;
4405
4406 let meta = live_roots
4407 .entry(identity.root.clone())
4408 .or_insert_with(|| RootMeta::new(Instant::now()));
4409 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4410 meta.reactivate_bound();
4411
4412 let route_cancel =
4413 route_bash_cancels
4414 .entry(route_id)
4415 .or_insert_with(|| bash::RouteBashCancel {
4416 token: PersistentCancelSignal::new(),
4417 active_waits: 0,
4418 });
4419 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4420 let cancel = bash::BashWaitCancel {
4421 connection: connection_cancel.clone(),
4422 route: route_cancel.token.clone(),
4423 };
4424 pending_bash_asks.insert(
4425 ReverseCorrKey {
4426 route: route_id,
4427 corr: reverse_corr,
4428 },
4429 PendingBashAsk {
4430 route: route_id,
4431 tool_corr: frame.header.corr,
4432 tool_flags: frame.header.flags,
4433 tool_ver: frame.header.ver,
4434 root: identity.root.clone(),
4435 project_root: identity.project_root.clone(),
4436 session_id: identity.session.clone(),
4437 spawn_principal: identity.spawn_principal.clone(),
4438 edit_slot_survives: call.edit_slot_survives,
4439 request_id,
4440 arguments,
4441 format_context,
4442 cancel,
4443 grants: plan.grants,
4444 expires_at: Instant::now() + bash_elicitation_timeout(),
4445 },
4446 );
4447 return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
4448 .await;
4449 }
4450
4451 let meta = live_roots
4452 .entry(identity.root.clone())
4453 .or_insert_with(|| RootMeta::new(Instant::now()));
4454 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4455 meta.reactivate_bound();
4456
4457 let route_cancel =
4458 route_bash_cancels
4459 .entry(route_id)
4460 .or_insert_with(|| bash::RouteBashCancel {
4461 token: PersistentCancelSignal::new(),
4462 active_waits: 0,
4463 });
4464 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4465 let cancel = bash::BashWaitCancel {
4466 connection: connection_cancel.clone(),
4467 route: route_cancel.token.clone(),
4468 };
4469
4470 bash::submit_deferred_bash(
4471 executor,
4472 bash_deferred_tx,
4473 bash_poll_touch_tx,
4474 metrics,
4475 dispatch,
4476 identity.root.clone(),
4477 identity.project_root.clone(),
4478 identity.session.clone(),
4479 request_id,
4480 route_id,
4481 frame.header.corr,
4482 frame.header.flags,
4483 frame.header.ver,
4484 arguments,
4485 format_context,
4486 cancel,
4487 bind_trust,
4488 identity.spawn_principal.clone(),
4489 call.edit_slot_survives,
4490 None,
4491 );
4492 return Ok(());
4493 }
4494
4495 let lane = command_lane(&bare_name);
4496 let tool_call_context = ToolCallContext {
4497 project_root: identity.project_root.clone(),
4498 session_id: Some(identity.session.clone()),
4499 request_id: request_id.clone(),
4500 diagnostics_on_edit,
4501 preview: call.preview,
4502 edit_slot_survives: call.edit_slot_survives,
4503 report_registration_downgrade: true,
4504 };
4505 let bare_name_for_frame = bare_name.clone();
4506 let identity_for_run = identity.clone();
4507 let completion_session = identity.session.clone();
4508 let completion_root = identity.project_root.clone();
4509 let request_id_for_force = request_id.clone();
4510 let format_context_for_frame = format_context.clone();
4511 let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
4512 let cancellable_inspect = bare_name == "inspect";
4513 phase_trace.mark_executor_submitted();
4514 let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
4515 phase_trace.mark_job_admitted();
4516 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4517 let run = || {
4518 let finalizer = |response: &mut Response| {
4519 crate::response_finalize::finalize_response_with_bg_completions(
4520 response,
4521 ctx,
4522 &identity_for_run.session,
4523 &bare_name,
4524 bind_trust.allows_bash_observation(),
4525 );
4526 };
4527 match run_tool_call(
4528 &bare_name,
4529 arguments,
4530 &format_context,
4531 &tool_call_context,
4532 ctx,
4533 &dispatch,
4534 Some(&finalizer),
4535 Some(&mut phase_trace),
4536 ) {
4537 ToolCallOutcome::Unary(result) => {
4538 let response = result.response;
4539 let _ = tool_call_tx.send(ToolCallCompletion {
4540 text: result.text,
4541 phase_trace,
4542 });
4543 response
4544 }
4545 }
4546 };
4547 if matches!(bind_trust, BindTrust::Untrusted) {
4548 ctx.with_force_restrict(&request_id_for_force, run)
4549 } else {
4550 run()
4551 }
4552 })
4553 });
4554 let (rx, cancellation) = if cancellable_inspect {
4555 let (rx, cancellation) =
4556 executor.submit_cancellable_async(identity.root.clone(), lane, request_id.clone(), job);
4557 (rx, Some(cancellation))
4558 } else {
4559 (
4560 executor.submit_async(identity.root.clone(), lane, request_id.clone(), job),
4561 None,
4562 )
4563 };
4564 if let Some(cancellation) = cancellation {
4565 active_tool_calls
4566 .lock()
4567 .unwrap_or_else(std::sync::PoisonError::into_inner)
4568 .insert(
4569 (route_id, frame.header.corr),
4570 ActiveToolCall {
4571 root_id: identity.root.clone(),
4572 cancellation,
4573 },
4574 );
4575 }
4576 let completion_tx = tx.clone();
4577 let completion_shutdown = Arc::clone(shutdown);
4578 let route = route_id;
4579 let corr = frame.header.corr;
4580 let flags = frame.header.flags;
4581 let ver = frame.header.ver;
4582 let completion_metrics = Arc::clone(metrics);
4583 let active_tool_calls = Arc::clone(active_tool_calls);
4584 tokio::spawn(async move {
4585 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4586 let response = await_executor_response(rx, request_id.clone()).await;
4587 if cancellable_inspect {
4588 finish_active_tool_call(&active_tool_calls, route, corr);
4589 }
4590 let (text, phase_trace) = match tool_call_rx.await {
4591 Ok(completion) => (completion.text, Some(completion.phase_trace)),
4592 Err(_) => (
4593 crate::subc_format::format_response_with_context(
4594 &bare_name_for_frame,
4595 &response,
4596 &format_context_for_frame,
4597 ),
4598 None,
4599 ),
4600 };
4601 let result = ToolCallResult { text, response };
4602 let fatal = response_is_fatal_panic(&result.response);
4603 match build_tool_response_frame_with_limit(
4604 ver,
4605 route,
4606 corr,
4607 flags,
4608 &result,
4609 bind_trust,
4610 tool_response_body_limit,
4611 ) {
4612 Ok(response_frame) => {
4613 let send_result = if let Some(phase_trace) = phase_trace {
4614 let trace = ToolResponseWriteTrace::new(
4615 phase_trace,
4616 bare_name_for_frame,
4617 completion_root,
4618 completion_session,
4619 route.channel,
4620 corr,
4621 );
4622 send_traced_tool_response_frame(
4623 &completion_tx,
4624 &completion_metrics,
4625 response_frame,
4626 trace,
4627 )
4628 .await
4629 } else {
4630 send_reliable_writer_frame(
4631 &completion_tx,
4632 &completion_metrics,
4633 response_frame,
4634 "tool response",
4635 )
4636 .await
4637 };
4638 if let Err(error) = send_result {
4639 log::warn!("subc attach: failed to queue tool response frame: {error}");
4640 }
4641 }
4642 Err(error) => {
4643 log::error!("subc attach: failed to build tool response frame: {error}");
4644 }
4645 }
4646 if fatal {
4647 signal_fatal_teardown(
4648 &completion_tx,
4649 Some(route),
4650 ver,
4651 corr,
4652 &completion_shutdown,
4653 &completion_metrics,
4654 )
4655 .await;
4656 }
4657 });
4658 Ok(())
4659}
4660
4661fn submit_maintenance_job(
4662 executor: &Arc<Executor>,
4663 root_id: ProjectRootId,
4664 kind: MaintenanceDrainKind,
4665 bg_sessions_to_check: Vec<(String, u64)>,
4666 completion_tx: &mpsc::Sender<MaintenanceCompletion>,
4667 metrics: &Arc<DispatchPathMetrics>,
4668) {
4669 let request_id = format!(
4670 "subc-maintenance-drain-{}-{}",
4671 kind.label(),
4672 root_id.as_path().to_string_lossy()
4673 );
4674 let response_id = request_id.clone();
4675 let completion_root_id = root_id.clone();
4676 let maintenance_generation = executor
4677 .actor_context(&root_id)
4678 .map(|ctx| ctx.configure_generation())
4679 .unwrap_or(0);
4680 let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
4681 let lane = match kind {
4686 MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
4687 MaintenanceDrainKind::Watcher
4688 | MaintenanceDrainKind::Lsp
4689 | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
4690 };
4691 let job: crate::executor::ExecutorJob = Box::new(move |ctx: &AppContext| {
4692 let outcome = match kind {
4693 MaintenanceDrainKind::Watcher => {
4694 let drained = runtime_drain::drain_watcher_events_bounded(
4695 ctx,
4696 runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
4697 );
4698 MaintenanceJobOutcome {
4699 empty_bg_sessions: Vec::new(),
4700 requeue_kind: drained.has_more.then_some(kind),
4701 }
4702 }
4703 MaintenanceDrainKind::Lsp => {
4704 let drained = runtime_drain::drain_lsp_events_bounded(
4705 ctx,
4706 runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
4707 );
4708 MaintenanceJobOutcome {
4709 empty_bg_sessions: Vec::new(),
4710 requeue_kind: drained.has_more.then_some(kind),
4711 }
4712 }
4713 MaintenanceDrainKind::ConfigureTail => {
4714 runtime_drain::drain_deferred_configure_maintenance(ctx);
4715 runtime_drain::drain_configure_warning_events(ctx);
4716 MaintenanceJobOutcome::default()
4717 }
4718 MaintenanceDrainKind::CompletionDrains => {
4719 runtime_drain::drain_search_index_events(ctx);
4720 runtime_drain::drain_callgraph_store_events(ctx);
4721 runtime_drain::drain_semantic_index_events(ctx);
4722 runtime_drain::drain_semantic_refresh_events(ctx);
4723 runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
4724 let empty_bg_sessions = bg_sessions_to_check
4725 .into_iter()
4726 .filter(|(session, _)| {
4727 !ctx.bash_background()
4728 .has_completions_for_session(Some(session.as_str()))
4729 })
4730 .collect();
4731 MaintenanceJobOutcome {
4732 empty_bg_sessions,
4733 requeue_kind: None,
4734 }
4735 }
4736 };
4737 let requeued = outcome.requeue_kind.is_some();
4738 let _ = outcome_tx.send(outcome);
4739 Response::success(
4740 response_id,
4741 json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
4742 )
4743 });
4744 let rx = match kind {
4745 MaintenanceDrainKind::Watcher => executor.submit_coalescable_maintenance_async(
4746 root_id,
4747 lane,
4748 request_id.clone(),
4749 crate::executor::MaintenanceCoalesceKey::WatcherDrain,
4750 job,
4751 ),
4752 MaintenanceDrainKind::Lsp => executor.submit_coalescable_maintenance_async(
4753 root_id,
4754 lane,
4755 request_id.clone(),
4756 crate::executor::MaintenanceCoalesceKey::LspDrain,
4757 job,
4758 ),
4759 MaintenanceDrainKind::ConfigureTail | MaintenanceDrainKind::CompletionDrains => {
4760 executor.submit_maintenance_async(root_id, lane, request_id.clone(), job)
4761 }
4762 };
4763 let completion_tx = completion_tx.clone();
4764 let completion_metrics = Arc::clone(metrics);
4765 tokio::spawn(async move {
4766 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4767 let response = await_executor_response(rx, request_id).await;
4768 let outcome = outcome_rx.await.unwrap_or_default();
4769 let _ = send_counted_channel(
4770 &completion_tx,
4771 &completion_metrics.maintenance_queued,
4772 MaintenanceCompletion {
4773 root_id: completion_root_id,
4774 kind,
4775 response,
4776 empty_bg_sessions: outcome.empty_bg_sessions,
4777 requeue_kind: outcome.requeue_kind,
4778 },
4779 )
4780 .await;
4781 });
4782}
4783
4784async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
4785 rx.await
4786 .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
4787}
4788async fn signal_fatal_teardown(
4789 tx: &WriterSender,
4790 route: Option<RouteChannel>,
4791 ver: u8,
4792 corr: u64,
4793 shutdown: &Arc<Notify>,
4794 metrics: &DispatchPathMetrics,
4795) {
4796 if let Some(route) = route {
4797 if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
4798 if let Err(error) = send_frame(tx, metrics, frame).await {
4799 log::warn!(
4800 "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
4801 );
4802 }
4803 }
4804 }
4805 if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
4806 if let Err(error) = send_frame(tx, metrics, frame).await {
4807 log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
4808 }
4809 }
4810 shutdown.notify_one();
4811}
4812#[derive(Debug, Deserialize)]
4813#[serde(untagged)]
4814enum RouteRequest {
4815 BgEvents(BgEventsRequest),
4816 ToolCall(ToolCallRequest),
4817}
4818
4819#[derive(Debug, Deserialize)]
4820struct BgEventsRequest {
4821 op: BgEventsOp,
4822}
4823
4824#[derive(Debug, Deserialize)]
4825#[serde(rename_all = "snake_case")]
4826enum BgEventsOp {
4827 BgEvents,
4828}
4829
4830#[derive(Debug, Deserialize)]
4831struct ToolCallRequest {
4832 name: String,
4833 #[serde(default)]
4834 arguments: Value,
4835 #[serde(default)]
4837 edit_slot_survives: Option<bool>,
4838 #[serde(default)]
4843 preview: bool,
4844}
4845
4846#[cfg(test)]
4847pub(crate) mod test_support {
4848 use super::*;
4849 use crate::bash_background::BgTaskStatus;
4850 use crate::protocol::{
4851 BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
4852 ProgressFrame, StatusChangedFrame,
4853 };
4854 use serde_json::json;
4855
4856 pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
4857 let dir = tempfile::Builder::new()
4858 .prefix(name)
4859 .tempdir()
4860 .expect("temp root");
4861 let root = ProjectRootId::from_path(dir.path()).expect("project root id");
4862 (dir, root)
4863 }
4864
4865 pub(super) fn test_ctx() -> Arc<AppContext> {
4866 Arc::new(AppContext::new(
4867 Box::new(crate::parser::TreeSitterProvider::new()),
4868 crate::config::Config::default(),
4869 ))
4870 }
4871
4872 #[test]
4873 fn cancelled_inspect_releases_lsp_lane_for_queued_bind() {
4874 let executor = Arc::new(Executor::new());
4875 let (_dir, root) = test_root("cancelled-inspect-lane");
4876 executor.register_actor(root.clone(), test_ctx());
4877 let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
4878 let route = RouteChannel {
4879 channel: 7,
4880 epoch: 1,
4881 };
4882 let (inspect_started_tx, inspect_started_rx) = std::sync::mpsc::sync_channel(1);
4883 let (_inspect_rx, cancellation) = executor.submit_cancellable_async(
4884 root.clone(),
4885 Lane::SerialLspStatus,
4886 "abandoned-inspect".to_string(),
4887 Box::new(move |_| {
4888 inspect_started_tx.send(()).expect("inspect starts");
4889 while !crate::executor::current_job_cancellation()
4890 .is_some_and(|token| token.cancel_requested_before_commit())
4891 {
4892 std::thread::sleep(Duration::from_millis(5));
4893 }
4894 Response::error(
4895 "abandoned-inspect",
4896 "inspect_interrupted",
4897 "inspect request abandoned",
4898 )
4899 }),
4900 );
4901 active.lock().expect("active tool call map").insert(
4902 (route, 41),
4903 ActiveToolCall {
4904 root_id: root.clone(),
4905 cancellation,
4906 },
4907 );
4908 inspect_started_rx
4909 .recv_timeout(Duration::from_secs(1))
4910 .expect("inspect occupies the LSP lane");
4911
4912 let (bind_started_tx, bind_started_rx) = std::sync::mpsc::sync_channel(1);
4913 let bind = executor.submit(
4914 root,
4915 Lane::Mutating,
4916 "subc-bind-after-abandonment".to_string(),
4917 Box::new(move |_| {
4918 bind_started_tx.send(()).expect("bind starts");
4919 Response::success("bind", json!({}))
4920 }),
4921 );
4922 assert!(
4923 bind_started_rx
4924 .recv_timeout(Duration::from_millis(50))
4925 .is_err(),
4926 "bind must remain behind the live LSP reader"
4927 );
4928
4929 assert!(cancel_active_tool_call(
4930 &active,
4931 executor.as_ref(),
4932 route,
4933 41,
4934 "test abandonment"
4935 ));
4936 bind_started_rx
4937 .recv_timeout(Duration::from_secs(1))
4938 .expect("bind starts after inspect cancellation boundary");
4939 bind.recv_timeout(Duration::from_secs(1))
4940 .expect("bind completes after inspect cancellation");
4941 assert!(active.lock().expect("active tool call map").is_empty());
4942 }
4943
4944 pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
4945 let deadline = Instant::now() + Duration::from_secs(30);
4946 loop {
4947 let observed = ctx.watcher_registry_count();
4948 if observed == expected {
4949 return;
4950 }
4951 assert!(
4952 Instant::now() < deadline,
4953 "watcher count did not settle before deadline: expected={expected}, observed={observed}"
4954 );
4955 std::thread::sleep(Duration::from_millis(50));
4956 }
4957 }
4958
4959 pub(super) fn reap_until_forgotten(
4969 root: &ProjectRootId,
4970 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4971 pending_binds: &HashMap<RouteChannel, PendingBind>,
4972 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4973 executor: &Arc<Executor>,
4974 metrics: &DispatchPathMetrics,
4975 ) -> IdleReapOutcome {
4976 let deadline = Instant::now() + Duration::from_secs(30);
4977 loop {
4978 let outcome = reap_idle_roots(
4979 Instant::now(),
4980 live_roots,
4981 pending_binds,
4982 root_channels,
4983 executor,
4984 metrics,
4985 );
4986 if outcome.forgotten_deleted_roots.contains(root) {
4987 return outcome;
4988 }
4989 assert!(
4990 Instant::now() < deadline,
4991 "deleted root was never forgotten: {root:?}"
4992 );
4993 std::thread::sleep(Duration::from_millis(10));
4994 }
4995 }
4996
4997 pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
4998 let deadline = Instant::now() + Duration::from_secs(30);
4999 loop {
5000 let observed = app.actor_root_count();
5001 if observed == expected {
5002 return;
5003 }
5004 assert!(
5005 Instant::now() < deadline,
5006 "actor root count did not settle before deadline: expected={expected}, observed={observed}"
5007 );
5008 std::thread::sleep(Duration::from_millis(50));
5009 }
5010 }
5011
5012 pub(super) fn status_frame(seq: u64) -> PushFrame {
5013 status_frame_with_session(seq, None)
5014 }
5015
5016 pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
5017 PushFrame::StatusChanged(StatusChangedFrame {
5018 frame_type: "status_changed",
5019 session_id: session_id.map(str::to_string),
5020 snapshot: json!({ "seq": seq }),
5021 })
5022 }
5023
5024 pub(super) fn completion_frame(task_id: &str) -> PushFrame {
5025 completion_frame_with_session(task_id, "session-1")
5026 }
5027
5028 pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
5029 PushFrame::BashCompleted(BashCompletedFrame {
5030 frame_type: "bash_completed",
5031 task_id: task_id.to_string(),
5032 session_id: session_id.to_string(),
5033 status: BgTaskStatus::Completed,
5034 exit_code: Some(0),
5035 command: format!("echo {task_id}"),
5036 output_preview: String::new(),
5037 output_truncated: false,
5038 original_tokens: None,
5039 compressed_tokens: None,
5040 tokens_skipped: false,
5041 status_reason: None,
5042 })
5043 }
5044
5045 pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
5046 long_running_frame_with_session(task_id, "session-1", elapsed_ms)
5047 }
5048
5049 pub(super) fn long_running_frame_with_session(
5050 task_id: &str,
5051 session_id: &str,
5052 elapsed_ms: u64,
5053 ) -> PushFrame {
5054 PushFrame::BashLongRunning(BashLongRunningFrame {
5055 frame_type: "bash_long_running",
5056 task_id: task_id.to_string(),
5057 session_id: session_id.to_string(),
5058 command: format!("sleep {elapsed_ms}"),
5059 elapsed_ms,
5060 })
5061 }
5062
5063 pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
5064 PushFrame::BashPatternMatch(BashPatternMatchFrame {
5065 frame_type: "bash_pattern_match",
5066 task_id: "task-pattern".to_string(),
5067 session_id: session_id.to_string(),
5068 watch_id: "watch-1".to_string(),
5069 match_text: "needle".to_string(),
5070 match_offset: 7,
5071 context: "haystack needle".to_string(),
5072 once: true,
5073 reason: "pattern_match",
5074 })
5075 }
5076
5077 pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
5078 PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
5079 frame_type: "configure_warnings",
5080 session_id: session_id.map(str::to_string),
5081 project_root: "/tmp/subc-test".to_string(),
5082 warnings: Vec::new(),
5083 })
5084 }
5085
5086 pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
5087 route_identity_with_trust(root, session_id, BindTrust::FirstParty)
5088 }
5089
5090 pub(super) fn route_identity_with_trust(
5091 root: &ProjectRootId,
5092 session_id: &str,
5093 trust: BindTrust,
5094 ) -> RouteIdentity {
5095 RouteIdentity(Arc::new(RouteIdentityData {
5096 root: root.clone(),
5097 project_root: root.as_path().to_path_buf(),
5098 harness: "opencode".to_string(),
5099 session: session_id.to_string(),
5100 trust,
5101 spawn_principal: AuthenticatedPrincipal::RouteBind {
5102 trust: trust.sandbox_trust(),
5103 route_channel: 0,
5104 route_epoch: 0,
5105 project_root: root.as_path().to_path_buf(),
5106 harness: "opencode".to_string(),
5107 session_id: session_id.to_string(),
5108 principal_id: Some(match trust {
5109 BindTrust::FirstParty => "direct".to_string(),
5110 BindTrust::Untrusted => "unverified".to_string(),
5111 }),
5112 },
5113 consumer_elicitation_capable: false,
5114 }))
5115 }
5116
5117 pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
5118 PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
5119 }
5120
5121 pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
5122 match frame {
5123 PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
5124 _ => None,
5125 }
5126 }
5127
5128 pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
5129 match frame {
5130 PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
5131 _ => None,
5132 }
5133 }
5134
5135 pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
5136 let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
5137 body.get("task_id")
5138 .and_then(serde_json::Value::as_str)
5139 .map(str::to_string)
5140 }
5141}
5142
5143#[cfg(test)]
5144mod tests {
5145 use super::test_support::{
5146 completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
5147 wait_for_actor_root_count, wait_for_watcher_count,
5148 };
5149 use super::*;
5150 use crate::bash_background::BgTaskStatus;
5151
5152 fn attach_error(kind: io::ErrorKind) -> SubcError {
5153 SubcError::Connect {
5154 endpoint: "127.0.0.1:1".to_string(),
5155 source: io::Error::new(kind, "constructed attach failure"),
5156 }
5157 }
5158
5159 fn auth_io_error(kind: io::ErrorKind) -> SubcError {
5160 SubcError::Auth {
5161 endpoint: "127.0.0.1:1".to_string(),
5162 source: subc_transport::AuthError::Io {
5163 stage: subc_transport::AuthStage::ServerProof,
5164 source: io::Error::new(kind, "constructed auth failure"),
5165 },
5166 }
5167 }
5168
5169 #[test]
5170 fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
5171 let transient_errors = vec![
5172 attach_error(io::ErrorKind::ConnectionRefused),
5173 attach_error(io::ErrorKind::TimedOut),
5174 attach_error(io::ErrorKind::ConnectionReset),
5175 auth_io_error(io::ErrorKind::ConnectionAborted),
5176 auth_io_error(io::ErrorKind::BrokenPipe),
5177 SubcError::Auth {
5178 endpoint: "127.0.0.1:1".to_string(),
5179 source: subc_transport::AuthError::UnexpectedEof {
5180 stage: subc_transport::AuthStage::ServerProof,
5181 expected: 4,
5182 actual: 0,
5183 },
5184 },
5185 SubcError::Auth {
5186 endpoint: "127.0.0.1:1".to_string(),
5187 source: subc_transport::AuthError::Timeout {
5188 stage: subc_transport::AuthStage::ServerProof,
5189 deadline: AUTH_DEADLINE,
5190 },
5191 },
5192 ];
5193 for error in &transient_errors {
5194 assert_eq!(
5195 classify_attach_error(error),
5196 AttachErrorClass::Transient,
5197 "expected transient: {error}"
5198 );
5199 }
5200
5201 let permanent_errors = vec![
5202 attach_error(io::ErrorKind::PermissionDenied),
5203 auth_io_error(io::ErrorKind::InvalidData),
5204 SubcError::Auth {
5205 endpoint: "127.0.0.1:1".to_string(),
5206 source: subc_transport::AuthError::InvalidServerProof,
5207 },
5208 SubcError::Auth {
5209 endpoint: "127.0.0.1:1".to_string(),
5210 source: subc_transport::AuthError::DaemonIdMismatch,
5211 },
5212 SubcError::ConnectionFile {
5213 path: PathBuf::from("subc-connection.json"),
5214 source: subc_transport::ConnectionFileError::Invalid {
5215 reason: "constructed invalid file".to_string(),
5216 },
5217 },
5218 SubcError::NoEndpoint {
5219 path: PathBuf::from("subc-connection.json"),
5220 },
5221 SubcError::InvalidEndpoint {
5222 path: PathBuf::from("subc-connection.json"),
5223 endpoint: "not-an-ip:1234".to_string(),
5224 },
5225 ];
5226 for error in &permanent_errors {
5227 assert_eq!(
5228 classify_attach_error(error),
5229 AttachErrorClass::Permanent,
5230 "expected permanent: {error}"
5231 );
5232 }
5233 }
5234
5235 #[test]
5236 fn incompatible_wire_version_is_rejected_before_tcp_connect() {
5237 let conn_dir = tempfile::tempdir().expect("connection tempdir");
5238 let conn_path = conn_dir.path().join("subc-connection.json");
5239 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
5240 listener
5241 .set_nonblocking(true)
5242 .expect("set listener nonblocking");
5243 let port = listener.local_addr().expect("listener addr").port();
5244 connection_file::write_atomic(
5245 &conn_path,
5246 &connection_file::ConnectionInfo {
5247 schema: connection_file::SCHEMA_VERSION,
5248 wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
5249 endpoints: vec![connection_file::Endpoint {
5250 host: "127.0.0.1".to_string(),
5251 port,
5252 }],
5253 key: vec![0x42; subc_transport::KEY_LEN],
5254 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
5255 pid: std::process::id(),
5256 daemon_ver: "subc-test".to_string(),
5257 },
5258 )
5259 .expect("write connection file");
5260
5261 let runtime = tokio::runtime::Builder::new_current_thread()
5262 .enable_all()
5263 .build()
5264 .expect("test runtime");
5265 let result = runtime.block_on(connect_and_authenticate_with_policy(
5266 &conn_path,
5267 AttachRetryPolicy {
5268 budget: Duration::from_secs(1),
5269 initial_backoff: Duration::from_millis(5),
5270 max_backoff: Duration::from_millis(10),
5271 jitter_percent: 0,
5272 },
5273 ));
5274 assert!(matches!(
5275 result,
5276 Err(SubcError::ConnectionFile {
5277 source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
5278 ..
5279 })
5280 ));
5281 assert!(matches!(
5282 listener.accept(),
5283 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
5284 ));
5285 }
5286
5287 #[test]
5288 fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
5289 let conn_dir = tempfile::tempdir().expect("connection tempdir");
5290 let conn_path = conn_dir.path().join("subc-connection.json");
5291 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
5292 let port = listener.local_addr().expect("reserved addr").port();
5293 drop(listener);
5294 connection_file::write_atomic(
5295 &conn_path,
5296 &connection_file::ConnectionInfo {
5297 schema: connection_file::SCHEMA_VERSION,
5298 wire_version: Some(PROTOCOL_VERSION),
5299 endpoints: vec![connection_file::Endpoint {
5300 host: "127.0.0.1".to_string(),
5301 port,
5302 }],
5303 key: vec![0x42; subc_transport::KEY_LEN],
5304 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
5305 pid: std::process::id(),
5306 daemon_ver: "subc-test".to_string(),
5307 },
5308 )
5309 .expect("write connection file");
5310
5311 let policy = AttachRetryPolicy {
5312 budget: Duration::from_millis(40),
5313 initial_backoff: Duration::from_millis(5),
5314 max_backoff: Duration::from_millis(10),
5315 jitter_percent: 0,
5316 };
5317 let runtime = tokio::runtime::Builder::new_current_thread()
5318 .enable_all()
5319 .build()
5320 .expect("test runtime");
5321 let started_at = Instant::now();
5322 let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
5323 let elapsed = started_at.elapsed();
5324 let error = match result {
5325 Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
5326 Err(error) => error,
5327 };
5328
5329 assert!(matches!(error, SubcError::Connect { .. }), "{error}");
5330 assert!(
5331 elapsed >= Duration::from_millis(35),
5332 "retry budget ended too early: {elapsed:?}"
5333 );
5334 assert!(
5335 elapsed < Duration::from_secs(1),
5336 "retry budget was not bounded: {elapsed:?}"
5337 );
5338 }
5339
5340 fn due_maintenance_jobs_without_actor_context(
5341 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
5342 budget: usize,
5343 pending_bind_roots: &HashSet<ProjectRootId>,
5344 ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
5345 due_maintenance_jobs(
5346 live_roots,
5347 None,
5348 &HashMap::new(),
5349 &HashSet::new(),
5350 budget,
5351 pending_bind_roots,
5352 )
5353 }
5354
5355 fn actor_ctx_with_dirty_search_index(
5356 root: &Path,
5357 storage: &Path,
5358 file_name: &str,
5359 old_contents: &str,
5360 new_contents: &str,
5361 ) -> (Arc<AppContext>, PathBuf, PathBuf) {
5362 let file = root.join(file_name);
5363 std::fs::write(&file, old_contents).expect("write source");
5364 let canonical_root = std::fs::canonicalize(root).expect("canonical root");
5365 let ctx = Arc::new(AppContext::new(
5366 Box::new(crate::parser::TreeSitterProvider::new()),
5367 Config {
5368 project_root: Some(root.to_path_buf()),
5369 storage_dir: Some(storage.to_path_buf()),
5370 ..Config::default()
5371 },
5372 ));
5373 ctx.set_canonical_cache_root(canonical_root.clone());
5374
5375 let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
5376 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
5377 let git_head = index.stored_git_head().map(str::to_owned);
5378 index.write_to_disk(&cache_dir, git_head.as_deref());
5379
5380 std::fs::write(&file, new_contents).expect("edit source");
5381 index.update_file(&file);
5382 *ctx.search_index()
5383 .write()
5384 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
5385 (ctx, canonical_root, cache_dir)
5386 }
5387
5388 #[test]
5389 fn graceful_shutdown_flushes_every_actor_search_index() {
5390 let storage = tempfile::tempdir().expect("storage tempdir");
5391 let (root1_dir, root1) = test_root("shutdown-flush-root-1");
5392 let (root2_dir, root2) = test_root("shutdown-flush-root-2");
5393 let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
5394 root1_dir.path(),
5395 storage.path(),
5396 "alpha.txt",
5397 "old actor one token\n",
5398 "new actor one token\n",
5399 );
5400 let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
5401 root2_dir.path(),
5402 storage.path(),
5403 "beta.txt",
5404 "old actor two token\n",
5405 "new actor two token\n",
5406 );
5407
5408 let executor = Executor::new();
5409 assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
5410 assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
5411
5412 flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
5413
5414 let mut restored1 =
5415 crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
5416 .expect("load flushed root one index");
5417 restored1.ready = true;
5418 assert_eq!(
5419 restored1
5420 .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
5421 .matches
5422 .len(),
5423 1,
5424 "graceful subc shutdown should flush the first root's trigram delta"
5425 );
5426
5427 let mut restored2 =
5428 crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
5429 .expect("load flushed root two index");
5430 restored2.ready = true;
5431 assert_eq!(
5432 restored2
5433 .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
5434 .matches
5435 .len(),
5436 1,
5437 "graceful subc shutdown should flush every registered root"
5438 );
5439 }
5440
5441 #[test]
5442 fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
5443 let _ = env_logger::builder().is_test(true).try_init();
5444 let (root_dir, root) = test_root("idle-root-reaper");
5445 let storage = tempfile::tempdir().expect("storage tempdir");
5446 std::fs::write(
5447 root_dir.path().join("main.rs"),
5448 "fn entry() { leaf(); }\nfn leaf() {}\n",
5449 )
5450 .expect("source file");
5451 let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
5452 let app = App::default_shared();
5453 let ctx = Arc::new(AppContext::from_app(
5454 Arc::clone(&app),
5455 Config {
5456 project_root: Some(canonical_root.clone()),
5457 storage_dir: Some(storage.path().to_path_buf()),
5458 callgraph_store: true,
5459 search_index: true,
5460 ..Config::default()
5461 },
5462 ));
5463 ctx.set_canonical_cache_root(canonical_root.clone());
5464 let project_key = crate::search_index::artifact_cache_key(&canonical_root);
5465 crate::root_cache::configure_artifact_access(&canonical_root, &project_key, false);
5466 assert!(ctx
5467 .ensure_callgraph_store()
5468 .expect("build callgraph store")
5469 .is_some());
5470
5471 let cache_dir =
5472 crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
5473 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
5474 let git_head = index.stored_git_head().map(str::to_owned);
5475 index.write_to_disk(&cache_dir, git_head.as_deref());
5476 *ctx.search_index()
5477 .write()
5478 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
5479 let seeded_generation =
5482 crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
5483 .expect("seeded artifact generation");
5484 crate::cache_freshness::record_verify_completed(
5485 &canonical_root,
5486 crate::cache_freshness::VerifyArtifact::Search,
5487 Some(seeded_generation),
5488 );
5489 assert!(
5490 matches!(
5491 crate::cache_freshness::warm_verify_plan(
5492 canonical_root.as_path(),
5493 crate::cache_freshness::VerifyArtifact::Search,
5494 Some(seeded_generation),
5495 ),
5496 crate::cache_freshness::WarmVerifyPlan::Skip
5497 ),
5498 "memo must be warm before eviction for the downgrade assertion to bite"
5499 );
5500
5501 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5502 let _dispatch_tx = dispatch_tx;
5503 let shutdown = Arc::new(AtomicBool::new(false));
5504 let thread_shutdown = Arc::clone(&shutdown);
5505 let join = std::thread::spawn(move || {
5506 while !thread_shutdown.load(Ordering::SeqCst) {
5507 std::thread::yield_now();
5508 }
5509 });
5510 ctx.install_watcher_runtime(
5511 dispatch_rx,
5512 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5513 );
5514 wait_for_watcher_count(&ctx, 1);
5515
5516 let executor = Arc::new(Executor::new());
5517 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5518 ctx.mark_subc_unbound();
5519 let mut live_roots = HashMap::new();
5520 let mut meta = RootMeta::new(Instant::now());
5521 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5522 meta.unbound_quiesced = true;
5523 live_roots.insert(root.clone(), meta);
5524
5525 let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
5526 assert!(message.contains("evicted idle root"));
5527 assert!(message.contains("freed ~"));
5528 assert!(message.contains("semantic"));
5529 assert!(!message.contains("semantic not estimated retained"));
5530 assert!(message.contains("trigram"));
5531 assert!(message.contains("retained: bash"));
5532 assert!(message.contains("parser_pool"));
5533
5534 assert_eq!(
5535 reap_idle_roots(
5536 Instant::now(),
5537 &mut live_roots,
5538 &HashMap::new(),
5539 &HashMap::new(),
5540 &executor,
5541 &DispatchPathMetrics::new(),
5542 )
5543 .evicted,
5544 1
5545 );
5546 assert!(ctx.search_index().read().unwrap().is_none());
5547 wait_for_watcher_count(&ctx, 0);
5548 assert!(
5553 matches!(
5554 crate::cache_freshness::warm_verify_plan(
5555 canonical_root.as_path(),
5556 crate::cache_freshness::VerifyArtifact::Search,
5557 Some(seeded_generation),
5558 ),
5559 crate::cache_freshness::WarmVerifyPlan::Strict
5560 ),
5561 "idle eviction must force strict re-verification"
5562 );
5563 assert!(
5564 crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
5565 );
5566 ctx.mark_subc_bound();
5567 assert!(ctx
5568 .ensure_callgraph_store()
5569 .expect("reopen callgraph store")
5570 .is_some());
5571 assert!(live_roots[&root].idle_artifacts_evicted);
5572 }
5573
5574 #[test]
5575 fn idle_root_reaper_applies_ttl_to_unbound_roots() {
5576 let (_root_dir, root) = test_root("idle-root-ttl-gate");
5577 let ctx = test_ctx();
5578 let executor = Arc::new(Executor::new());
5579 assert!(executor.register_actor(root.clone(), ctx));
5580 let ctx = executor.actor_context(&root).expect("actor context");
5581 ctx.mark_subc_unbound();
5582 let now = Instant::now();
5583 let mut meta = RootMeta::new(now);
5584 meta.unbound_quiesced = true;
5585 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5586
5587 assert_eq!(
5591 reap_idle_roots(
5592 now,
5593 &mut live_roots,
5594 &HashMap::new(),
5595 &HashMap::new(),
5596 &executor,
5597 &DispatchPathMetrics::new(),
5598 )
5599 .evicted,
5600 0
5601 );
5602 assert!(!live_roots[&root].idle_artifacts_evicted);
5603
5604 ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
5610 assert_eq!(
5611 reap_idle_roots(
5612 now + IDLE_ROOT_TTL,
5613 &mut live_roots,
5614 &HashMap::new(),
5615 &HashMap::new(),
5616 &executor,
5617 &DispatchPathMetrics::new(),
5618 )
5619 .evicted,
5620 1
5621 );
5622 assert!(live_roots[&root].idle_artifacts_evicted);
5623 assert!(
5624 ctx.take_pending_search_index_paths().is_empty(),
5625 "TTL eviction must dispose retained pending reconciliation paths"
5626 );
5627 }
5628
5629 #[test]
5630 fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
5631 let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
5632 let ctx = test_ctx();
5633 let executor = Arc::new(Executor::new());
5634 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5635 ctx.mark_subc_unbound();
5636
5637 let pending = root.as_path().join("edited-while-unbound.rs");
5643 ctx.add_pending_search_index_paths([pending.clone()]);
5644 let dirty_source = root.as_path().join("dirty.rs");
5645 std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
5646 let mut dirty = crate::search_index::SearchIndex::new();
5647 dirty.ready = true;
5648 dirty.update_file(&dirty_source);
5649 *ctx.search_index()
5650 .write()
5651 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
5652 assert!(ctx.artifact_eviction_blocked());
5653
5654 let mut live_roots = HashMap::new();
5655 let mut meta = RootMeta::new(Instant::now());
5656 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5657 meta.unbound_quiesced = true;
5658 live_roots.insert(root.clone(), meta);
5659
5660 assert_eq!(
5661 reap_idle_roots(
5662 Instant::now(),
5663 &mut live_roots,
5664 &HashMap::new(),
5665 &HashMap::new(),
5666 &executor,
5667 &DispatchPathMetrics::new(),
5668 )
5669 .evicted,
5670 0,
5671 "the dirty index must still block this eviction"
5672 );
5673 assert_eq!(
5674 ctx.take_pending_search_index_paths(),
5675 vec![pending],
5676 "a blocked eviction must restore the taken pending paths"
5677 );
5678 }
5679
5680 #[test]
5681 fn idle_reap_with_bound_route_keeps_watcher_running() {
5682 let (_root_dir, root) = test_root("bound-root-reap-gate");
5683 let ctx = test_ctx();
5684 let executor = Arc::new(Executor::new());
5685 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5686
5687 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5688 let _dispatch_tx = dispatch_tx;
5689 let shutdown = Arc::new(AtomicBool::new(false));
5690 let thread_shutdown = Arc::clone(&shutdown);
5691 let join = std::thread::spawn(move || {
5692 while !thread_shutdown.load(Ordering::SeqCst) {
5693 std::thread::yield_now();
5694 }
5695 });
5696 ctx.install_watcher_runtime(
5697 dispatch_rx,
5698 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5699 );
5700
5701 let mut meta = RootMeta::new(Instant::now());
5702 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5703 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5704 let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
5705 assert_eq!(
5706 reap_idle_roots(
5707 Instant::now(),
5708 &mut live_roots,
5709 &HashMap::new(),
5710 &bound,
5711 &executor,
5712 &DispatchPathMetrics::new(),
5713 )
5714 .evicted,
5715 0
5716 );
5717 wait_for_watcher_count(&ctx, 1);
5718 ctx.stop_watcher_runtime_in_background();
5719 wait_for_watcher_count(&ctx, 0);
5720 }
5721
5722 #[test]
5723 fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
5724 let (root_dir, root) = test_root("deleted-bound-root-reap");
5725 let executor = Arc::new(Executor::new());
5726 assert!(executor.register_actor(root.clone(), test_ctx()));
5727 root_dir.close().expect("delete project root");
5728
5729 let route = route_key(19, 3);
5730 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5731 let cancel_signal = PersistentCancelSignal::new();
5732 let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
5733 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5734 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5735 let mut route_bash_cancels = HashMap::from([(
5736 route,
5737 bash::RouteBashCancel {
5738 token: cancel_signal.clone(),
5739 active_waits: 0,
5740 },
5741 )]);
5742 let metrics = DispatchPathMetrics::new();
5743
5744 let first = reap_idle_roots(
5745 Instant::now(),
5746 &mut live_roots,
5747 &HashMap::new(),
5748 &root_channels,
5749 &executor,
5750 &metrics,
5751 );
5752 assert!(first.forgotten_deleted_roots.is_empty());
5753 assert!(executor.actor_registered(&root));
5754
5755 let mut forgotten = Vec::new();
5756 for _ in 0..100 {
5757 let outcome = reap_idle_roots(
5758 Instant::now(),
5759 &mut live_roots,
5760 &HashMap::new(),
5761 &root_channels,
5762 &executor,
5763 &metrics,
5764 );
5765 if !outcome.forgotten_deleted_roots.is_empty() {
5766 forgotten = outcome.forgotten_deleted_roots;
5767 break;
5768 }
5769 std::thread::sleep(Duration::from_millis(10));
5770 }
5771 assert_eq!(forgotten, vec![root.clone()]);
5772 assert!(!executor.actor_registered(&root));
5773
5774 let mut retry_buffer = HashMap::new();
5775 let mut reclaimed_routes = ReclaimedRoutes::default();
5776 let mut session_identity = HashMap::new();
5777 let mut push_buffer = HashMap::new();
5778 let mut bg_subs = HashMap::from([(
5779 route,
5780 BgSub {
5781 corr: 77,
5782 ver: PROTOCOL_VERSION,
5783 flags: control_flags(),
5784 root: root.clone(),
5785 session: "deleted-route".to_string(),
5786 },
5787 )]);
5788 let mut bg_sub_by_session = HashMap::from([(
5789 (root.clone(), "deleted-route".to_string()),
5790 HashSet::from([route]),
5791 )]);
5792 let mut bg_wake_pending = HashSet::from([route]);
5793 let mut bg_wake_epoch = HashMap::new();
5794 let mut pending_bash_asks = HashMap::new();
5795 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
5796 health::take_bg_observability_logs_for_test();
5797 purge_deleted_root_residents(
5798 &root,
5799 &mut routes,
5800 &mut root_channels,
5801 &mut installed_route_epochs,
5802 &mut route_bash_cancels,
5803 &active_tool_calls,
5804 executor.as_ref(),
5805 &mut retry_buffer,
5806 &mut reclaimed_routes,
5807 &mut session_identity,
5808 &mut push_buffer,
5809 &mut bg_subs,
5810 &mut bg_sub_by_session,
5811 &mut bg_wake_pending,
5812 &mut bg_wake_epoch,
5813 &mut pending_bash_asks,
5814 &metrics,
5815 );
5816
5817 assert!(routes.is_empty());
5818 assert!(root_channels.is_empty());
5819 assert!(installed_route_epochs.is_empty());
5820 assert!(route_bash_cancels.is_empty());
5821 assert!(reclaimed_routes.contains(route));
5822 assert!(cancel_signal.is_cancelled());
5823 assert_eq!(
5824 health::take_bg_observability_logs_for_test(),
5825 vec![format!(
5826 "subc bg subscription: ended root={} session=deleted-route channel=19@3 cause=root-reclaim suppressed=0",
5827 root.as_path().display()
5828 )]
5829 );
5830 }
5831
5832 #[test]
5839 fn live_root_with_bound_route_is_never_reclaimed() {
5840 let (_root_dir, root) = test_root("live-bound-root-retained");
5841 let ctx = test_ctx();
5842 ctx.mark_subc_unbound();
5843 let executor = Arc::new(Executor::new());
5844 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5845
5846 let route = RouteChannel {
5847 channel: 7,
5848 epoch: 1,
5849 };
5850 let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
5851 meta.unbound_quiesced = true;
5852 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5853 let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5854
5855 for _ in 0..3 {
5860 let outcome = reap_idle_roots(
5861 Instant::now(),
5862 &mut live_roots,
5863 &HashMap::new(),
5864 &root_channels,
5865 &executor,
5866 &DispatchPathMetrics::new(),
5867 );
5868 assert!(
5869 outcome.forgotten_deleted_roots.is_empty(),
5870 "a root whose directory exists must never be forgotten"
5871 );
5872 }
5873
5874 assert!(live_roots.contains_key(&root), "live root must be retained");
5875 assert!(
5876 executor.actor_registered(&root),
5877 "live root's actor must survive"
5878 );
5879 assert!(
5880 root.as_path().exists(),
5881 "test vehicle must keep the directory alive; otherwise this control proves nothing"
5882 );
5883 }
5884
5885 #[test]
5886 fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
5887 let (root_dir, root) = test_root("deleted-root-first-observation");
5888 let ctx = test_ctx();
5889 ctx.mark_subc_unbound();
5890 let executor = Arc::new(Executor::new());
5891 assert!(executor.register_actor(root.clone(), ctx));
5892 root_dir.close().expect("delete project root");
5893
5894 let mut meta = RootMeta::new(Instant::now());
5895 meta.unbound_quiesced = true;
5896 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5897 let outcome = reap_idle_roots(
5898 Instant::now(),
5899 &mut live_roots,
5900 &HashMap::new(),
5901 &HashMap::new(),
5902 &executor,
5903 &DispatchPathMetrics::new(),
5904 );
5905
5906 assert!(outcome.forgotten_deleted_roots.is_empty());
5907 assert!(live_roots.contains_key(&root));
5908 assert!(executor.actor_registered(&root));
5909 }
5910
5911 fn spawn_background_for_root(
5912 ctx: &AppContext,
5913 root: &ProjectRootId,
5914 storage: &tempfile::TempDir,
5915 session_id: &str,
5916 ) -> (String, u32) {
5917 let command = if cfg!(windows) {
5923 "ping -n 31 127.0.0.1 > nul"
5925 } else {
5926 "sleep 30"
5927 };
5928 let task_id = ctx
5929 .bash_background()
5930 .spawn(
5931 crate::sandbox_spawn::SpawnPlan::Unsandboxed,
5932 command,
5933 session_id.to_string(),
5934 storage.path().to_path_buf(),
5935 HashMap::new(),
5936 Some(Duration::from_secs(60)),
5937 storage.path().to_path_buf(),
5938 8,
5939 true,
5940 false,
5941 Some(root.as_path().to_path_buf()),
5942 )
5943 .expect("spawn background task");
5944 let snapshot = ctx
5945 .bash_background()
5946 .status(
5947 &task_id,
5948 session_id,
5949 Some(root.as_path()),
5950 Some(storage.path()),
5951 0,
5952 )
5953 .expect("background task status");
5954 (task_id, snapshot.child_pid.expect("background child pid"))
5955 }
5956
5957 fn wait_for_background_exit(pid: u32) {
5958 let deadline = Instant::now() + Duration::from_secs(5);
5959 while crate::bash_background::process::is_process_alive(pid) {
5960 assert!(
5961 Instant::now() < deadline,
5962 "background task process survived kill"
5963 );
5964 std::thread::sleep(Duration::from_millis(20));
5965 }
5966 }
5967
5968 #[test]
5969 fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
5970 let (root_dir, root) = test_root("deleted-root-background-task");
5971 let storage = tempfile::tempdir().expect("task storage");
5972 let ctx = test_ctx();
5973 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
5974 assert!(crate::bash_background::process::is_process_alive(pid));
5975
5976 let executor = Arc::new(Executor::new());
5977 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5978 assert!(executor.actor_is_idle(&root));
5979 root_dir.close().expect("delete project root");
5980 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5981 let pending_binds = HashMap::new();
5982 let root_channels = HashMap::new();
5983 let metrics = DispatchPathMetrics::new();
5984
5985 let first = reap_idle_roots(
5986 Instant::now(),
5987 &mut live_roots,
5988 &pending_binds,
5989 &root_channels,
5990 &executor,
5991 &metrics,
5992 );
5993 assert!(first.forgotten_deleted_roots.is_empty());
5994 assert!(crate::bash_background::process::is_process_alive(pid));
5995
5996 let outcome = reap_until_forgotten(
5997 &root,
5998 &mut live_roots,
5999 &pending_binds,
6000 &root_channels,
6001 &executor,
6002 &metrics,
6003 );
6004 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
6005 wait_for_background_exit(pid);
6006
6007 let snapshot = ctx
6008 .bash_background()
6009 .status(
6010 &task_id,
6011 "reclaim-session",
6012 Some(root.as_path()),
6013 Some(storage.path()),
6014 0,
6015 )
6016 .expect("reclaimed task status");
6017 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
6018 assert_eq!(
6019 snapshot.info.status_reason.as_deref(),
6020 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
6021 );
6022 assert_eq!(
6023 serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
6024 crate::bash_background::registry::ROOT_RECLAIMED_REASON
6025 );
6026 let completion = ctx
6027 .bash_background()
6028 .drain_completions_for_session(Some("reclaim-session"))
6029 .pop()
6030 .expect("reclaimed task completion");
6031 assert_eq!(
6032 completion.status_reason.as_deref(),
6033 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
6034 );
6035 }
6036
6037 #[test]
6038 fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
6039 let (root_dir, root) = test_root("existing-root-background-task");
6040 let storage = tempfile::tempdir().expect("task storage");
6041 let ctx = test_ctx();
6042 ctx.mark_subc_unbound();
6043 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
6044
6045 let executor = Arc::new(Executor::new());
6046 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6047 let mut meta = RootMeta::new(
6048 Instant::now()
6049 .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
6050 .expect("old root timestamp"),
6051 );
6052 meta.unbound_quiesced = true;
6053 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6054 let pending_binds = HashMap::new();
6055 let root_channels = HashMap::new();
6056 let metrics = DispatchPathMetrics::new();
6057
6058 for _ in 0..8 {
6059 reap_idle_roots(
6060 Instant::now(),
6061 &mut live_roots,
6062 &pending_binds,
6063 &root_channels,
6064 &executor,
6065 &metrics,
6066 );
6067 std::thread::sleep(Duration::from_millis(10));
6068 }
6069 assert!(root_dir.path().exists());
6070 assert!(crate::bash_background::process::is_process_alive(pid));
6071 let snapshot = ctx
6072 .bash_background()
6073 .status(
6074 &task_id,
6075 "existing-session",
6076 Some(root.as_path()),
6077 Some(storage.path()),
6078 0,
6079 )
6080 .expect("existing task status");
6081 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
6082 let _ = ctx.bash_background().kill(&task_id, "existing-session");
6083 wait_for_background_exit(pid);
6084 }
6085
6086 #[test]
6087 fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
6088 let (root_dir, root) = test_root("restored-root-background-task");
6089 let storage = tempfile::tempdir().expect("task storage");
6090 let ctx = test_ctx();
6091 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
6092
6093 let executor = Arc::new(Executor::new());
6094 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6095 root_dir.close().expect("delete project root");
6096 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6097 let pending_binds = HashMap::new();
6098 let root_channels = HashMap::new();
6099 let metrics = DispatchPathMetrics::new();
6100
6101 let first = reap_idle_roots(
6102 Instant::now(),
6103 &mut live_roots,
6104 &pending_binds,
6105 &root_channels,
6106 &executor,
6107 &metrics,
6108 );
6109 assert!(first.forgotten_deleted_roots.is_empty());
6110 std::fs::create_dir_all(root.as_path()).expect("restore project root");
6111 let second = reap_idle_roots(
6112 Instant::now(),
6113 &mut live_roots,
6114 &pending_binds,
6115 &root_channels,
6116 &executor,
6117 &metrics,
6118 );
6119 assert!(second.forgotten_deleted_roots.is_empty());
6120 assert!(crate::bash_background::process::is_process_alive(pid));
6121 let _ = ctx.bash_background().kill(&task_id, "restored-session");
6122 wait_for_background_exit(pid);
6123 }
6124
6125 #[test]
6126 fn observing_root_again_resets_deleted_sweep_confirmation() {
6127 let (root_dir, root) = test_root("deleted-root-observation-reset");
6128 let ctx = test_ctx();
6129 ctx.mark_subc_unbound();
6130 let executor = Arc::new(Executor::new());
6131 assert!(executor.register_actor(root.clone(), ctx));
6132 root_dir.close().expect("delete project root");
6133
6134 let mut meta = RootMeta::new(Instant::now());
6135 meta.unbound_quiesced = true;
6136 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6137 let pending_binds = HashMap::new();
6138 let root_channels = HashMap::new();
6139 let metrics = DispatchPathMetrics::new();
6140
6141 let first = reap_idle_roots(
6142 Instant::now(),
6143 &mut live_roots,
6144 &pending_binds,
6145 &root_channels,
6146 &executor,
6147 &metrics,
6148 );
6149 assert!(first.forgotten_deleted_roots.is_empty());
6150
6151 std::fs::create_dir_all(root.as_path()).expect("restore project root");
6152 reap_idle_roots(
6153 Instant::now(),
6154 &mut live_roots,
6155 &pending_binds,
6156 &root_channels,
6157 &executor,
6158 &metrics,
6159 );
6160 std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
6161
6162 let after_reset = reap_idle_roots(
6163 Instant::now(),
6164 &mut live_roots,
6165 &pending_binds,
6166 &root_channels,
6167 &executor,
6168 &metrics,
6169 );
6170 assert!(after_reset.forgotten_deleted_roots.is_empty());
6171 assert!(live_roots.contains_key(&root));
6172 assert!(executor.actor_registered(&root));
6173 }
6174
6175 #[test]
6176 fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
6177 let (root_dir, root) = test_root("deleted-root-reap");
6178 let app = App::default_shared();
6179 let ctx = Arc::new(AppContext::from_app(
6180 Arc::clone(&app),
6181 Config {
6182 project_root: Some(root.as_path().to_path_buf()),
6183 ..Config::default()
6184 },
6185 ));
6186 ctx.set_canonical_cache_root(root.as_path().to_path_buf());
6187 ctx.mark_subc_unbound();
6188 let executor = Arc::new(Executor::new());
6189 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6190 assert_eq!(app.actor_root_count(), 1);
6191 drop(ctx);
6192 root_dir.close().expect("delete project root");
6193
6194 let mut meta = RootMeta::new(Instant::now());
6195 meta.unbound_quiesced = true;
6196 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6197 let outcome = reap_until_forgotten(
6198 &root,
6199 &mut live_roots,
6200 &HashMap::new(),
6201 &HashMap::new(),
6202 &executor,
6203 &DispatchPathMetrics::new(),
6204 );
6205 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
6206 assert!(!executor.actor_registered(&root));
6207 assert!(!live_roots.contains_key(&root));
6208 wait_for_actor_root_count(&app, 0);
6209
6210 let status_ctx = AppContext::from_app(app, Config::default());
6211 let status = status_ctx.build_status_snapshot();
6212 assert_eq!(status["runtime"]["live_actor_roots"], 0);
6213 assert_eq!(status["runtime"]["open_routes"], 0);
6214 }
6215
6216 #[test]
6217 fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
6218 let (root_dir, root) = test_root("deleted-root-reap-census");
6219 let executor = Arc::new(Executor::new());
6220 assert!(executor.register_actor(root.clone(), test_ctx()));
6221 root_dir.close().expect("delete project root");
6222
6223 let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
6224 let metrics = DispatchPathMetrics::new();
6225 let outcome = reap_idle_roots(
6226 Instant::now(),
6227 &mut live_roots,
6228 &HashMap::new(),
6229 &HashMap::new(),
6230 &executor,
6231 &metrics,
6232 );
6233 assert_eq!(outcome.evicted, 0);
6234
6235 let app = crate::context::App::default_shared();
6236 let health_rollup_cache = HealthRollupCache::new();
6237 health_rollup_cache.refresh(&executor, &app);
6238 let report = build_health_report(
6239 &health_rollup_cache,
6240 &executor,
6241 &HashMap::new(),
6242 &metrics,
6243 &app,
6244 );
6245 let reap = report
6246 .metrics
6247 .as_ref()
6248 .and_then(|metrics| metrics.get("reap"))
6249 .expect("reap health metrics");
6250 assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
6251 assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
6252 assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
6253 assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
6254 }
6255
6256 #[test]
6257 fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
6258 let (root_dir, root) = test_root("connection-exit-deleted-root");
6259 let executor = Arc::new(Executor::new());
6260 assert!(executor.register_actor(root.clone(), test_ctx()));
6261
6262 let route = route_key(11, 1);
6263 let mut meta = RootMeta::new(Instant::now());
6264 meta.maintenance_pending = true;
6265 meta.maintenance_queued_kinds
6266 .push_back(MaintenanceDrainKind::CompletionDrains);
6267 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6268 let mut pending_binds = HashMap::new();
6269 let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
6270 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
6271 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6272 let mut route_bash_cancels = HashMap::new();
6273 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6274
6275 quiesce_connection_roots(
6276 &mut live_roots,
6277 &mut pending_binds,
6278 &mut routes,
6279 &mut root_channels,
6280 &mut installed_route_epochs,
6281 &mut route_bash_cancels,
6282 &active_tool_calls,
6283 &executor,
6284 );
6285 assert!(live_roots[&root].unbound_quiesced);
6286 assert!(!live_roots[&root].maintenance_pending);
6287 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
6288 assert!(routes.is_empty());
6289 assert!(root_channels.is_empty());
6290
6291 root_dir.close().expect("delete project root");
6292 let metrics = DispatchPathMetrics::new();
6293 let outcome = reap_until_forgotten(
6294 &root,
6295 &mut live_roots,
6296 &pending_binds,
6297 &root_channels,
6298 &executor,
6299 &metrics,
6300 );
6301 let mut session_identity = HashMap::new();
6302 let mut push_buffer = HashMap::new();
6303 let mut bg_subs = HashMap::new();
6304 let mut bg_sub_by_session = HashMap::new();
6305 let mut bg_wake_pending = HashSet::new();
6306 let mut bg_wake_epoch = HashMap::new();
6307 let mut pending_bash_asks = HashMap::new();
6308 let mut retry_buffer = HashMap::new();
6309 let mut reclaimed_routes = ReclaimedRoutes::default();
6310 for forgotten in &outcome.forgotten_deleted_roots {
6311 purge_deleted_root_residents(
6312 forgotten,
6313 &mut routes,
6314 &mut root_channels,
6315 &mut installed_route_epochs,
6316 &mut route_bash_cancels,
6317 &active_tool_calls,
6318 executor.as_ref(),
6319 &mut retry_buffer,
6320 &mut reclaimed_routes,
6321 &mut session_identity,
6322 &mut push_buffer,
6323 &mut bg_subs,
6324 &mut bg_sub_by_session,
6325 &mut bg_wake_pending,
6326 &mut bg_wake_epoch,
6327 &mut pending_bash_asks,
6328 &metrics,
6329 );
6330 }
6331
6332 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
6333 assert!(!executor.actor_registered(&root));
6334 assert!(!live_roots.contains_key(&root));
6335 }
6336
6337 #[test]
6338 fn unbound_root_quiesces_maintenance_without_removing_actor() {
6339 let (_root_dir, root) = test_root("unbound-root-quiesce");
6340 let ctx = test_ctx();
6341 let executor = Arc::new(Executor::new());
6342 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6343 let mut meta = RootMeta::new(Instant::now());
6344 meta.maintenance_pending = true;
6345 meta.maintenance_jobs_in_flight = 1;
6346 meta.maintenance_queued_kinds
6347 .push_back(MaintenanceDrainKind::ConfigureTail);
6348 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6349 *ctx.search_index()
6351 .write()
6352 .unwrap_or_else(std::sync::PoisonError::into_inner) =
6353 Some(crate::search_index::SearchIndex::new());
6354 ctx.set_cache_writer_capabilities(true, true);
6355 let pending = root.as_path().join("pending.rs");
6356 ctx.add_pending_search_index_paths([pending.clone()]);
6357 let canonical_root = root.as_path().to_path_buf();
6361 let artifact = canonical_root.join("cache.bin");
6362 std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
6363 let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
6364 crate::cache_freshness::record_verify_completed(
6365 &canonical_root,
6366 crate::cache_freshness::VerifyArtifact::Search,
6367 seeded_generation,
6368 );
6369 assert!(matches!(
6370 crate::cache_freshness::warm_verify_plan(
6371 &canonical_root,
6372 crate::cache_freshness::VerifyArtifact::Search,
6373 seeded_generation,
6374 ),
6375 crate::cache_freshness::WarmVerifyPlan::Skip
6376 ));
6377 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
6380 let _dispatch_tx = dispatch_tx;
6381 let shutdown = Arc::new(AtomicBool::new(false));
6382 let thread_shutdown = Arc::clone(&shutdown);
6383 let join = std::thread::spawn(move || {
6384 while !thread_shutdown.load(Ordering::SeqCst) {
6385 std::thread::yield_now();
6386 }
6387 });
6388 ctx.install_watcher_runtime(
6389 dispatch_rx,
6390 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
6391 );
6392 assert!(ctx.watcher_runtime_active());
6393
6394 quiesce_unbound_root(&root, &mut live_roots, &executor);
6395 let meta = &live_roots[&root];
6396 assert!(meta.unbound_quiesced);
6397 assert!(ctx.subc_unbound_quiesced());
6398 assert!(meta.maintenance_pending);
6399 assert!(meta.maintenance_queued_kinds.is_empty());
6400 assert!(executor.actor_registered(&root));
6401 assert!(
6405 ctx.search_index()
6406 .read()
6407 .unwrap_or_else(std::sync::PoisonError::into_inner)
6408 .is_some(),
6409 "quiesce must not evict resident artifacts"
6410 );
6411 assert_eq!(
6412 ctx.pending_callgraph_store_force_token(),
6413 None,
6414 "quiesce must not force a callgraph rebuild"
6415 );
6416 assert_eq!(
6417 ctx.take_pending_search_index_paths(),
6418 vec![pending],
6419 "quiesce must retain pending watcher-derived paths"
6420 );
6421 assert!(
6422 matches!(
6423 crate::cache_freshness::warm_verify_plan(
6424 &canonical_root,
6425 crate::cache_freshness::VerifyArtifact::Search,
6426 seeded_generation,
6427 ),
6428 crate::cache_freshness::WarmVerifyPlan::Skip
6429 ),
6430 "quiesce must not invalidate the warm verify memo"
6431 );
6432 assert!(
6433 ctx.watcher_runtime_active(),
6434 "quiesce must not stop a running watcher"
6435 );
6436 ctx.stop_watcher_runtime();
6437
6438 let meta = live_roots.get_mut(&root).expect("root metadata");
6439 note_maintenance_completion(
6440 meta,
6441 Some(MaintenanceDrainKind::ConfigureTail),
6442 false,
6443 meta.unbound_quiesced,
6444 );
6445 assert!(!meta.maintenance_pending);
6446 assert!(meta.maintenance_queued_kinds.is_empty());
6447 }
6448
6449 #[test]
6450 fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
6451 let (_dir, root) = test_root("same-root-replacement");
6452 let route = route_key(7, 1);
6453 let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
6454 let root_channels = HashMap::new();
6455
6456 assert!(!route_removal_will_quiesce_root(
6457 &root,
6458 route,
6459 &installed_channels,
6460 false,
6461 Some(&root),
6462 ));
6463 assert!(route_removal_will_quiesce_root(
6464 &root,
6465 route,
6466 &installed_channels,
6467 false,
6468 None,
6469 ));
6470 assert!(!should_quiesce_removed_root(
6471 &root,
6472 &root_channels,
6473 false,
6474 Some(&root),
6475 ));
6476 assert!(should_quiesce_removed_root(
6477 &root,
6478 &root_channels,
6479 false,
6480 None,
6481 ));
6482 assert!(!should_quiesce_removed_root(
6483 &root,
6484 &root_channels,
6485 true,
6486 None,
6487 ));
6488 }
6489
6490 #[test]
6491 fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
6492 let (_root_dir, root) = test_root("unbound-root-route-count");
6493 let executor = Arc::new(Executor::new());
6494 assert!(executor.register_actor(root.clone(), test_ctx()));
6495 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6496 let mut root_channels = HashMap::from([(
6497 root.clone(),
6498 HashSet::from([route_key(7, 1), route_key(8, 1)]),
6499 )]);
6500
6501 remove_root_channel(&mut root_channels, &root, route_key(7, 1));
6502 if !root_channels.contains_key(&root) {
6503 quiesce_unbound_root(&root, &mut live_roots, &executor);
6504 }
6505 assert!(!live_roots[&root].unbound_quiesced);
6506
6507 remove_root_channel(&mut root_channels, &root, route_key(8, 1));
6508 if !root_channels.contains_key(&root) {
6509 quiesce_unbound_root(&root, &mut live_roots, &executor);
6510 }
6511 assert!(live_roots[&root].unbound_quiesced);
6512
6513 live_roots
6514 .get_mut(&root)
6515 .expect("root metadata")
6516 .note_activity();
6517 assert!(
6518 live_roots[&root].unbound_quiesced,
6519 "late asynchronous activity must not reactivate an unbound root"
6520 );
6521
6522 live_roots
6523 .get_mut(&root)
6524 .expect("root metadata")
6525 .reactivate_bound();
6526 assert!(!live_roots[&root].unbound_quiesced);
6527 }
6528
6529 #[test]
6530 fn allocator_pressure_relief_requires_every_root_to_be_idle() {
6531 let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
6532 let (_active_dir, active_root) = test_root("allocator-relief-active");
6533 let now = Instant::now();
6534 let mut live_roots = HashMap::new();
6535 let mut idle = RootMeta::new(now);
6536 idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
6537 live_roots.insert(idle_root, idle);
6538 assert!(process_has_been_idle(now, &live_roots));
6539
6540 live_roots.insert(active_root.clone(), RootMeta::new(now));
6541 assert!(!process_has_been_idle(now, &live_roots));
6542
6543 let active = live_roots
6544 .get_mut(&active_root)
6545 .expect("active root metadata");
6546 active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
6547 active.active_bash_waits = 1;
6548 assert!(!process_has_been_idle(now, &live_roots));
6549 }
6550
6551 #[test]
6552 fn pressure_relief_log_reports_before_and_after_measurements() {
6553 let allocator = crate::memory::AllocatorMemorySnapshot {
6554 status: "measured",
6555 bytes_in_use: Some(8 * 1024 * 1024),
6556 size_allocated: Some(12 * 1024 * 1024),
6557 retained_slack_bytes: Some(4 * 1024 * 1024),
6558 not_estimated: None,
6559 };
6560 let relief = crate::memory::AllocatorPressureRelief {
6561 bytes_released: 3 * 1024 * 1024,
6562 rss_before_bytes: Some(20 * 1024 * 1024),
6563 rss_after_bytes: Some(17 * 1024 * 1024),
6564 allocator_before: allocator.clone(),
6565 allocator_after: crate::memory::AllocatorMemorySnapshot {
6566 size_allocated: Some(9 * 1024 * 1024),
6567 retained_slack_bytes: Some(1024 * 1024),
6568 ..allocator
6569 },
6570 };
6571 let message = pressure_relief_label(&relief);
6572 assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
6573 assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
6574 assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
6575 assert!(message.contains("reported 3.0 MB released"));
6576 }
6577
6578 #[test]
6579 fn due_maintenance_jobs_skip_poisoned_roots() {
6580 let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
6581 let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
6582 let mut live_roots = HashMap::new();
6583 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6584 let mut poisoned_meta = RootMeta::new(Instant::now());
6585 poisoned_meta.maintenance_poisoned = true;
6586 live_roots.insert(poisoned_root.clone(), poisoned_meta);
6587
6588 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6589 &mut live_roots,
6590 MAINTENANCE_SUBMIT_BUDGET,
6591 &HashSet::new(),
6592 );
6593
6594 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6595 assert!(due.iter().all(|(root, _)| root == &healthy_root));
6596 assert!(!deferred);
6597 assert!(live_roots[&healthy_root].maintenance_pending);
6598 assert_eq!(
6599 live_roots[&healthy_root].maintenance_jobs_in_flight,
6600 INITIAL_MAINTENANCE_JOB_COUNT
6601 );
6602 assert!(!live_roots[&poisoned_root].maintenance_pending);
6603 }
6604
6605 #[test]
6606 fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
6607 let (_dir, root) = test_root("maintenance-unbound");
6608 let mut meta = RootMeta::new(Instant::now());
6609 meta.unbound_quiesced = true;
6610 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6611
6612 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6613 &mut live_roots,
6614 MAINTENANCE_SUBMIT_BUDGET,
6615 &HashSet::new(),
6616 );
6617
6618 assert!(due.is_empty());
6619 assert!(!deferred);
6620 assert!(!live_roots[&root].maintenance_pending);
6621 }
6622
6623 #[test]
6624 fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
6625 let (_dir, root) = test_root("maintenance-idle-bg-subscription");
6626 let ctx = test_ctx();
6627 assert!(!ctx.completion_drains_have_work());
6628
6629 let executor = Executor::new();
6630 assert!(executor.register_actor(root.clone(), ctx));
6631 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6632 let session = "idle-session".to_string();
6633 let channel = route_key(17, 1);
6634 let metrics = DispatchPathMetrics::new();
6635 let bg_sub_by_session =
6636 HashMap::from([((root.clone(), session.clone()), HashSet::from([channel]))]);
6637 let mut bg_wake_pending = HashSet::new();
6638
6639 let (idle_tick_jobs, deferred) = due_maintenance_jobs(
6640 &mut live_roots,
6641 Some(&executor),
6642 &bg_sub_by_session,
6643 &bg_wake_pending,
6644 MAINTENANCE_SUBMIT_BUDGET,
6645 &HashSet::new(),
6646 );
6647 assert!(idle_tick_jobs.is_empty());
6648 assert!(!deferred);
6649 assert!(!live_roots[&root].maintenance_pending);
6650
6651 let mut bg_wake_epoch = HashMap::new();
6654 push::arm_bg_wake(
6655 root.clone(),
6656 session,
6657 channel,
6658 &mut bg_wake_pending,
6659 &mut bg_wake_epoch,
6660 &metrics,
6661 );
6662 let (next_tick_jobs, deferred) = due_maintenance_jobs(
6663 &mut live_roots,
6664 Some(&executor),
6665 &bg_sub_by_session,
6666 &bg_wake_pending,
6667 MAINTENANCE_SUBMIT_BUDGET,
6668 &HashSet::new(),
6669 );
6670 assert_eq!(
6671 next_tick_jobs,
6672 vec![(root, MaintenanceDrainKind::CompletionDrains)]
6673 );
6674 assert!(!deferred);
6675 }
6676
6677 #[tokio::test]
6678 async fn subc_configure_tail_precedes_completed_search_install() {
6679 let root_dir = tempfile::tempdir().unwrap();
6680 let storage = tempfile::tempdir().unwrap();
6681 let root = ProjectRootId::from_path(root_dir.path()).unwrap();
6682 let (ctx, ignored_path) =
6683 runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
6684 let ctx = Arc::new(ctx);
6685 assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
6686 &ctx,
6687 &ignored_path
6688 ));
6689
6690 let executor = Arc::new(Executor::new());
6691 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6692 let metrics = Arc::new(DispatchPathMetrics::new());
6693 let (completion_tx, mut completion_rx) = mpsc::channel(4);
6694 submit_maintenance_job(
6695 &executor,
6696 root.clone(),
6697 MaintenanceDrainKind::ConfigureTail,
6698 Vec::new(),
6699 &completion_tx,
6700 &metrics,
6701 );
6702 submit_maintenance_job(
6703 &executor,
6704 root,
6705 MaintenanceDrainKind::CompletionDrains,
6706 Vec::new(),
6707 &completion_tx,
6708 &metrics,
6709 );
6710
6711 let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6712 .await
6713 .expect("configure-tail completion timed out")
6714 .expect("configure-tail completion channel closed");
6715 let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6716 .await
6717 .expect("completion-drains completion timed out")
6718 .expect("completion-drains completion channel closed");
6719 assert!(first.response.id.contains("configure-tail"));
6720 assert!(second.response.id.contains("completion-drains"));
6721 assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
6722 &ctx,
6723 &ignored_path
6724 ));
6725 assert_eq!(
6726 ctx.search_index()
6727 .read()
6728 .unwrap_or_else(std::sync::PoisonError::into_inner)
6729 .as_ref()
6730 .expect("completed search index installed")
6731 .file_count(),
6732 0,
6733 "configure must install the ignore matcher before pending paths replay"
6734 );
6735 ctx.stop_watcher_runtime();
6736 }
6737
6738 #[test]
6739 fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
6740 let (_dir, root) = test_root("maintenance-post-bind");
6741 let mut live_roots = HashMap::new();
6742 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6743
6744 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6745 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6746
6747 let meta = live_roots.get(&root).expect("root metadata");
6748 assert!(meta.maintenance_pending);
6749 assert_eq!(meta.maintenance_jobs_in_flight, 0);
6750 assert_eq!(
6751 meta.maintenance_queued_kinds
6752 .iter()
6753 .copied()
6754 .collect::<Vec<_>>(),
6755 vec![
6756 MaintenanceDrainKind::ConfigureTail,
6757 MaintenanceDrainKind::CompletionDrains,
6758 ]
6759 );
6760
6761 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6762 &mut live_roots,
6763 MAINTENANCE_SUBMIT_BUDGET,
6764 &HashSet::new(),
6765 );
6766
6767 assert_eq!(
6768 due,
6769 vec![
6770 (root.clone(), MaintenanceDrainKind::ConfigureTail),
6771 (root.clone(), MaintenanceDrainKind::CompletionDrains),
6772 ]
6773 );
6774 assert!(!deferred);
6775 assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
6776 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
6777 }
6778
6779 #[test]
6780 fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
6781 let mut live_roots = HashMap::new();
6782 let mut root_ids = Vec::new();
6783 let mut _dirs = Vec::new();
6784 for index in 0..4 {
6785 let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
6786 live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
6787 root_ids.push(root_id);
6788 _dirs.push(dir);
6789 }
6790
6791 let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
6792 let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
6793 &mut live_roots,
6794 small_budget,
6795 &HashSet::new(),
6796 );
6797
6798 assert_eq!(first_due.len(), small_budget);
6799 assert!(first_deferred);
6800 let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
6801 assert!(first_due_set
6802 .iter()
6803 .all(|root| live_roots[root].maintenance_pending));
6804 assert!(first_due_set
6805 .iter()
6806 .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
6807
6808 let all_roots: HashSet<_> = root_ids.into_iter().collect();
6809 let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
6810 assert!(deferred_roots
6811 .iter()
6812 .all(|root| !live_roots[root].maintenance_pending));
6813 }
6814
6815 #[test]
6816 fn due_maintenance_jobs_defers_pending_bind_roots() {
6817 let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
6818 let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
6819 let mut live_roots = HashMap::new();
6820 live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
6821 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6822 let pending_bind_roots = HashSet::from([bind_root.clone()]);
6823
6824 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6825 &mut live_roots,
6826 usize::MAX,
6827 &pending_bind_roots,
6828 );
6829
6830 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6831 assert!(due.iter().all(|(root, _)| root == &healthy_root));
6832 assert!(!deferred);
6833 assert!(!live_roots[&bind_root].maintenance_pending);
6834 assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
6835 }
6836
6837 #[test]
6838 fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
6839 let (_dir, root) = test_root("maintenance-requeue");
6840 let mut live_roots = HashMap::new();
6841 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6842 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6843 &mut live_roots,
6844 usize::MAX,
6845 &HashSet::new(),
6846 );
6847 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6848 assert!(due.iter().all(|(due_root, _)| due_root == &root));
6849 assert!(!deferred);
6850
6851 let meta = live_roots.get_mut(&root).unwrap();
6852 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
6853 assert!(meta.maintenance_pending);
6854 assert_eq!(
6855 meta.maintenance_jobs_in_flight,
6856 INITIAL_MAINTENANCE_JOB_COUNT - 1
6857 );
6858 assert_eq!(meta.maintenance_queued_kinds.len(), 1);
6859
6860 let (requeued, deferred) =
6861 due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
6862 assert_eq!(
6863 requeued,
6864 vec![(root.clone(), MaintenanceDrainKind::Watcher)]
6865 );
6866 assert!(!deferred);
6867 let meta = live_roots.get_mut(&root).unwrap();
6868 assert_eq!(
6869 meta.maintenance_jobs_in_flight,
6870 INITIAL_MAINTENANCE_JOB_COUNT
6871 );
6872 assert!(meta.maintenance_queued_kinds.is_empty());
6873
6874 for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
6875 note_maintenance_completion(meta, None, false, false);
6876 }
6877 assert!(!meta.maintenance_pending);
6878 assert_eq!(meta.maintenance_jobs_in_flight, 0);
6879 }
6880
6881 #[test]
6882 fn maintenance_requeue_drops_while_bind_is_pending() {
6883 let (_dir, root) = test_root("maintenance-bind-requeue");
6884 let mut live_roots = HashMap::new();
6885 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6886 let (due, _) = due_maintenance_jobs_without_actor_context(
6887 &mut live_roots,
6888 usize::MAX,
6889 &HashSet::new(),
6890 );
6891 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6892
6893 let meta = live_roots.get_mut(&root).unwrap();
6894 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
6895
6896 assert_eq!(
6897 meta.maintenance_jobs_in_flight,
6898 INITIAL_MAINTENANCE_JOB_COUNT - 1
6899 );
6900 assert!(meta.maintenance_queued_kinds.is_empty());
6901 assert!(meta.maintenance_pending);
6902 }
6903
6904 #[test]
6905 fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
6906 let mut meta = RootMeta::new(Instant::now());
6907 meta.unbound_quiesced = true;
6908
6909 assert!(!should_requiesce_after_maintenance(
6910 &meta,
6911 MaintenanceDrainKind::Lsp,
6912 false,
6913 ));
6914 assert!(!should_requiesce_after_maintenance(
6915 &meta,
6916 MaintenanceDrainKind::ConfigureTail,
6917 true,
6918 ));
6919 assert!(should_requiesce_after_maintenance(
6920 &meta,
6921 MaintenanceDrainKind::ConfigureTail,
6922 false,
6923 ));
6924 }
6925
6926 #[test]
6927 fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
6928 let (_dir, root) = test_root("maintenance-fatal");
6929 let mut live_roots = HashMap::new();
6930 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6931 let (due, _) = due_maintenance_jobs_without_actor_context(
6932 &mut live_roots,
6933 usize::MAX,
6934 &HashSet::new(),
6935 );
6936 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6937
6938 let meta = live_roots.get_mut(&root).unwrap();
6939 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
6940 assert!(meta.maintenance_poisoned);
6941 assert!(meta.maintenance_queued_kinds.is_empty());
6942
6943 for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
6944 note_maintenance_completion(meta, None, false, false);
6945 }
6946 assert!(!meta.maintenance_pending);
6947 assert_eq!(meta.maintenance_jobs_in_flight, 0);
6948 }
6949
6950 #[test]
6951 fn trust_for_principal_matrix() {
6952 assert_eq!(
6953 trust_for_principal(&Some(Principal::Direct)),
6954 BindTrust::FirstParty
6955 );
6956 for module_id in [
6963 "llm-runner",
6964 "aft",
6965 "broca",
6966 "alfonso-core",
6967 "prefrontal",
6968 "prefrontal-core",
6969 ] {
6970 assert_eq!(
6971 trust_for_principal(&Some(Principal::Reserved {
6972 module_id: module_id.to_string(),
6973 })),
6974 BindTrust::FirstParty,
6975 "reserved module id '{module_id}' must resolve to first-party trust"
6976 );
6977 }
6978 assert_eq!(
6979 trust_for_principal(&Some(Principal::Reserved {
6980 module_id: "subc-mcp".to_string(),
6981 })),
6982 BindTrust::Untrusted
6983 );
6984 assert_eq!(
6985 trust_for_principal(&Some(Principal::Reserved {
6986 module_id: "anything-unknown".to_string(),
6987 })),
6988 BindTrust::Untrusted
6989 );
6990 assert_eq!(
6991 trust_for_principal(&Some(Principal::Unverified)),
6992 BindTrust::Untrusted
6993 );
6994 assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
6995 }
6996
6997 #[test]
6998 fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
6999 let principal = Some(Principal::Direct);
7000 let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
7001 let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
7002
7003 assert_eq!(
7004 trust_for_bind(fingerprint_a, &principal),
7005 BindTrust::Untrusted
7006 );
7007 assert_eq!(
7008 trust_for_bind(fingerprint_b, &principal),
7009 BindTrust::Untrusted
7010 );
7011 }
7012
7013 #[test]
7020 fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
7021 for harness in ["opencode", "pi", "runner", "mcp:claude"] {
7022 assert_eq!(
7023 trust_for_bind(harness, &Some(Principal::Direct)),
7024 BindTrust::FirstParty,
7025 "a direct principal must stay first-party on {harness}"
7026 );
7027 assert_eq!(
7028 trust_for_bind(harness, &Some(Principal::Unverified)),
7029 BindTrust::Untrusted,
7030 "an unverified principal must stay untrusted on {harness}"
7031 );
7032 assert_eq!(
7033 trust_for_bind(harness, &None),
7034 BindTrust::Untrusted,
7035 "an absent principal must fail closed on {harness}"
7036 );
7037 assert_eq!(
7038 trust_for_bind(
7039 harness,
7040 &Some(Principal::Reserved {
7041 module_id: "subc-mcp".to_string(),
7042 })
7043 ),
7044 BindTrust::Untrusted,
7045 "a non-allowlisted reserved module must stay untrusted on {harness}"
7046 );
7047 }
7048 }
7049
7050 #[tokio::test]
7051 async fn persistent_cancel_resolves_when_fired_before_await() {
7052 let signal = PersistentCancelSignal::new();
7056 signal.cancel();
7057 tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
7059 .await
7060 .expect("cancelled() must resolve when cancel fired beforehand");
7061
7062 let racing = PersistentCancelSignal::new();
7064 let racing_for_task = racing.clone();
7065 let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
7066 racing.cancel();
7067 tokio::time::timeout(Duration::from_secs(1), waiter)
7068 .await
7069 .expect("cancelled() must resolve when cancel races the await")
7070 .expect("waiter task panicked");
7071 }
7072
7073 #[test]
7074 fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
7075 let installed = HashMap::from([(7, 9)]);
7076 let mut reclaimed = ReclaimedRoutes::default();
7077 reclaimed.insert(route_key(8, 1));
7078 for ty in [
7079 FrameType::Request,
7080 FrameType::Response,
7081 FrameType::Error,
7082 FrameType::Push,
7083 FrameType::Cancel,
7084 FrameType::Goodbye,
7085 ] {
7086 let body = if ty.is_pure_header() {
7087 Vec::new()
7088 } else {
7089 br#"{}"#.to_vec()
7090 };
7091 let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
7092 assert!(
7093 !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
7094 "{ty:?}"
7095 );
7096 }
7097
7098 let reclaimed_request = Frame::build(
7099 FrameType::Request,
7100 control_flags(),
7101 8,
7102 1,
7103 42,
7104 br#"{}"#.to_vec(),
7105 )
7106 .unwrap();
7107 assert!(ingress_route_should_be_processed(
7108 &installed,
7109 &reclaimed,
7110 &reclaimed_request
7111 ));
7112
7113 let never_installed = Frame::build(
7114 FrameType::Request,
7115 control_flags(),
7116 9,
7117 1,
7118 43,
7119 br#"{}"#.to_vec(),
7120 )
7121 .unwrap();
7122 assert!(!ingress_route_should_be_processed(
7123 &installed,
7124 &reclaimed,
7125 &never_installed
7126 ));
7127
7128 let current = Frame::build(
7129 FrameType::Request,
7130 control_flags(),
7131 7,
7132 9,
7133 43,
7134 br#"{}"#.to_vec(),
7135 )
7136 .unwrap();
7137 let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
7138 assert!(ingress_route_should_be_processed(
7139 &installed, &reclaimed, ¤t
7140 ));
7141 assert!(ingress_route_should_be_processed(
7142 &installed, &reclaimed, &control
7143 ));
7144 assert_eq!(installed, HashMap::from([(7, 9)]));
7145 }
7146
7147 #[tokio::test]
7148 async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
7149 let (_dir, root) = test_root("route-bind-b2-ordering");
7150 let route = route_key(7, 3);
7151 let identity = RouteIdentity(Arc::new(RouteIdentityData {
7152 root: root.clone(),
7153 project_root: root.as_path().to_path_buf(),
7154 harness: "opencode".to_string(),
7155 session: "b2-session".to_string(),
7156 trust: BindTrust::FirstParty,
7157 spawn_principal: AuthenticatedPrincipal::FirstParty,
7158 consumer_elicitation_capable: false,
7159 }));
7160 let replay_key = push::ReplayKey::from_identity(&identity);
7161 let completion = RouteBindCompletion {
7162 route,
7163 identity,
7164 bind_root_id: root.clone(),
7165 inserted_new_actor: false,
7166 configure_response: Response::success("subc-bind-7", json!({})),
7167 diagnostics_on_edit: false,
7168 ver: PROTOCOL_VERSION,
7169 corr: 91,
7170 flags: control_flags(),
7171 };
7172 let mut pending_binds = HashMap::from([(
7173 route,
7174 PendingBind {
7175 bind_root_id: root,
7176 inserted_new_actor: false,
7177 cancelled: false,
7178 configure_request_id: "subc-bind-7".to_string(),
7179 started_at: Instant::now(),
7180 warned_half_deadline: false,
7181 deadline_reported: false,
7182 corr: 91,
7183 ver: PROTOCOL_VERSION,
7184 flags: control_flags(),
7185 cancellation: crate::executor::JobCancellation::new(),
7186 },
7187 )]);
7188 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
7189 let mut push_buffer =
7190 HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
7191 let (writer_tx, mut writer_rx) = mpsc::channel(8);
7192 let metrics = Arc::new(DispatchPathMetrics::new());
7193
7194 handle_route_bind_completion(
7195 &writer_tx,
7196 completion,
7197 &mut HashMap::new(),
7198 &mut HashMap::new(),
7199 &mut HashMap::new(),
7200 &mut push_buffer,
7201 &mut HashMap::new(),
7202 &mut pending_binds,
7203 &mut installed_route_epochs,
7204 &Arc::new(Executor::new()),
7205 &Arc::new(Notify::new()),
7206 &metrics,
7207 )
7208 .await
7209 .unwrap();
7210
7211 let ack = writer_rx.try_recv().expect("RouteBindAck");
7212 assert_eq!(ack.header.ty, FrameType::Response);
7213 assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
7214 let route_frame = writer_rx.try_recv().expect("post-ack route frame");
7215 assert_eq!(route_frame.header.ty, FrameType::Push);
7216 assert_eq!(
7217 (route_frame.header.channel, route_frame.header.epoch),
7218 (route.channel, route.epoch)
7219 );
7220 }
7221}