1use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock, Mutex as StdMutex, OnceLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, Lane};
31use crate::jsonc::strip_jsonc;
32use crate::log_ctx;
33use crate::path_identity::ProjectRootId;
34use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
35use crate::run_tool_call::{
36 run_tool_call, strip_agent_preview_arg_owned, PhaseTrace, ToolCallContext, ToolCallOutcome,
37 ToolCallResult,
38};
39use crate::runtime_drain;
40use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
41
42use subc_protocol::manifest::{
43 Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
44 ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
45};
46use subc_protocol::session::{
47 HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
48 MODULE_CONTROL_OP_HEALTH_CHECK,
49};
50use subc_protocol::{
51 ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, PROTOCOL_VERSION,
52};
53use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
54use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
55use tokio::net::TcpStream;
56use tokio::sync::{mpsc, oneshot, Notify};
57use tokio::task::JoinHandle;
58
59const AUTH_DEADLINE: Duration = Duration::from_secs(5);
62const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
63const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
64const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
65const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
66
67const HELLO_CORR: u64 = 1;
69
70const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
73
74const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
78
79const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
83
84const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
88
89const WRITER_QUEUE_CAPACITY: usize = 256;
90
91const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
94
95const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
103const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
104 MaintenanceDrainKind::Watcher,
105 MaintenanceDrainKind::Lsp,
106 MaintenanceDrainKind::ConfigureTail,
107 MaintenanceDrainKind::CompletionDrains,
108];
109#[cfg(test)]
110const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
111
112const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
113const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
114
115const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
116const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
117
118const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
121
122const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
126
127const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
129const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
130
131#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
132struct RouteChannel {
133 channel: u16,
134 epoch: u32,
135}
136
137impl fmt::Display for RouteChannel {
138 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
139 write!(f, "{}@{}", self.channel, self.epoch)
140 }
141}
142
143type PushEnvelope = (ProjectRootId, PushFrame);
144type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
145type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
146mod bash;
147mod health;
148mod manifest;
149mod push;
150mod wire;
151
152use self::health::{
153 build_health_report, warn_slow_pending_binds, DispatchPathMetrics, ReapBlockerCensus,
154 ResponseTaskGuard,
155};
156use self::manifest::{
157 build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
158 is_subc_agent_core_tool, is_subc_native_plumbing_tool,
159};
160pub use self::wire::SubcError;
161
162pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
166 manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
167}
168use self::wire::{
169 build_error_frame, build_goodbye_frame, build_tool_response_frame, decrement_counted_channel,
170 response_is_fatal_panic, response_message, send_counted_channel, send_frame,
171 send_reliable_writer_frame, send_traced_tool_response_frame, ToolResponseWriteTrace,
172 WriterFrame, WriterSender,
173};
174
175struct DecodedFrame {
176 frame: Frame,
177 phase_trace: PhaseTrace,
178}
179
180struct ToolCallCompletion {
181 text: String,
182 phase_trace: PhaseTrace,
183}
184
185#[derive(Clone)]
186struct PushSenders {
187 lossy_tx: mpsc::Sender<LossyPushEnvelope>,
188 reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
189 lossy_overflow: Arc<push::LossyOverflow>,
190 lossy_seq: Arc<AtomicU64>,
191}
192
193#[derive(Clone)]
194struct PersistentCancelSignal {
195 inner: Arc<PersistentCancelInner>,
196}
197
198struct PersistentCancelInner {
199 cancelled: AtomicBool,
200 notify: Notify,
201}
202
203impl PersistentCancelSignal {
204 fn new() -> Self {
205 Self {
206 inner: Arc::new(PersistentCancelInner {
207 cancelled: AtomicBool::new(false),
208 notify: Notify::new(),
209 }),
210 }
211 }
212
213 fn cancel(&self) {
214 if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
215 self.inner.notify.notify_waiters();
216 }
217 }
218
219 fn is_cancelled(&self) -> bool {
220 self.inner.cancelled.load(Ordering::SeqCst)
221 }
222
223 async fn cancelled(&self) {
224 loop {
232 let notified = self.inner.notify.notified();
233 tokio::pin!(notified);
234 notified.as_mut().enable();
235 if self.is_cancelled() {
236 return;
237 }
238 notified.await;
239 }
240 }
241}
242
243#[derive(Debug, Clone, Copy, PartialEq, Eq)]
244pub(crate) enum BindTrust {
245 FirstParty,
246 Untrusted,
247}
248
249impl BindTrust {
250 fn allows_bash_observation(self) -> bool {
251 matches!(self, Self::FirstParty)
252 }
253
254 fn label(self) -> &'static str {
255 match self {
256 Self::FirstParty => "first_party",
257 Self::Untrusted => "untrusted",
258 }
259 }
260
261 fn sandbox_trust(self) -> PrincipalTrust {
262 match self {
263 Self::FirstParty => PrincipalTrust::FirstParty,
264 Self::Untrusted => PrincipalTrust::Untrusted,
265 }
266 }
267}
268
269pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
270 match principal {
271 Some(Principal::Direct) => BindTrust::FirstParty,
272 Some(Principal::Reserved { module_id })
283 if module_id == "llm-runner"
284 || module_id == "aft"
285 || module_id == "broca"
286 || module_id == "alfonso-core"
287 || module_id == "prefrontal"
288 || module_id == "prefrontal-core" =>
289 {
290 BindTrust::FirstParty
291 }
292 Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
293 BindTrust::Untrusted
294 }
295 }
296}
297
298fn harness_forces_untrusted(harness: &str) -> bool {
299 harness.starts_with("fed:")
300}
301
302pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
303 if harness_forces_untrusted(harness) {
304 BindTrust::Untrusted
305 } else {
306 trust_for_principal(principal)
307 }
308}
309
310fn principal_id(principal: &Option<Principal>) -> Option<String> {
311 match principal {
312 Some(Principal::Direct) => Some("direct".to_string()),
313 Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
314 Some(Principal::Unverified) => Some("unverified".to_string()),
315 None => None,
316 }
317}
318
319fn principal_label(principal: &Option<Principal>) -> String {
320 principal_id(principal).unwrap_or_else(|| "absent".to_string())
321}
322
323#[derive(Debug)]
324struct RootMeta {
330 maintenance_pending: bool,
331 maintenance_jobs_in_flight: usize,
332 maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
333 maintenance_last_submitted: Option<Instant>,
334 maintenance_poisoned: bool,
335 last_touched: Instant,
336 diagnostics_on_edit: bool,
337 active_bash_waits: usize,
338 idle_artifacts_evicted: bool,
339 unbound_quiesced: bool,
340 consecutive_missing_sweeps: u8,
341}
342
343#[derive(Debug)]
344struct PendingBind {
345 bind_root_id: ProjectRootId,
346 inserted_new_actor: bool,
347 cancelled: bool,
348 configure_request_id: String,
349 started_at: Instant,
350 warned_half_deadline: bool,
351 deadline_reported: bool,
352 corr: u64,
353 ver: u8,
354 flags: Flags,
355 cancellation: crate::executor::JobCancellation,
360}
361
362struct RouteBindCompletion {
363 route: RouteChannel,
364 identity: RouteIdentity,
365 bind_root_id: ProjectRootId,
366 inserted_new_actor: bool,
367 configure_response: Response,
368 diagnostics_on_edit: bool,
369 ver: u8,
370 corr: u64,
371 flags: Flags,
372}
373
374#[derive(Debug, Clone)]
375struct RouteIdentity(Arc<RouteIdentityData>);
376
377#[derive(Debug)]
378struct RouteIdentityData {
379 root: ProjectRootId,
380 project_root: PathBuf,
381 harness: String,
382 session: String,
383 trust: BindTrust,
384 spawn_principal: AuthenticatedPrincipal,
385 consumer_elicitation_capable: bool,
386}
387
388impl Deref for RouteIdentity {
389 type Target = RouteIdentityData;
390
391 fn deref(&self) -> &Self::Target {
392 &self.0
393 }
394}
395
396#[derive(Debug, Clone)]
397struct RetainedSessionIdentity {
398 harness: String,
399 trust: BindTrust,
400}
401
402#[derive(Clone, Copy)]
403struct BgSub {
404 corr: u64,
405 ver: u8,
406 flags: Flags,
407}
408
409struct MaintenanceCompletion {
410 root_id: ProjectRootId,
411 kind: MaintenanceDrainKind,
412 response: Response,
413 empty_bg_sessions: Vec<(String, u64)>,
414 requeue_kind: Option<MaintenanceDrainKind>,
415}
416
417#[derive(Clone, Copy, Debug, PartialEq, Eq)]
418enum MaintenanceDrainKind {
419 Watcher,
420 Lsp,
421 ConfigureTail,
422 CompletionDrains,
423}
424
425impl MaintenanceDrainKind {
426 fn label(self) -> &'static str {
427 match self {
428 Self::Watcher => "watcher",
429 Self::Lsp => "lsp",
430 Self::ConfigureTail => "configure-tail",
431 Self::CompletionDrains => "completion-drains",
432 }
433 }
434}
435
436#[derive(Debug, Default)]
437struct MaintenanceJobOutcome {
438 empty_bg_sessions: Vec<(String, u64)>,
439 requeue_kind: Option<MaintenanceDrainKind>,
440}
441
442#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
443struct ReverseCorrKey {
444 route: RouteChannel,
445 corr: u64,
446}
447
448struct PendingBashAsk {
449 route: RouteChannel,
450 tool_corr: u64,
451 tool_flags: Flags,
452 tool_ver: u8,
453 root: ProjectRootId,
454 project_root: PathBuf,
455 session_id: String,
456 spawn_principal: AuthenticatedPrincipal,
457 request_id: String,
458 arguments: Value,
459 format_context: crate::subc_format::FormatContext,
460 cancel: bash::BashWaitCancel,
461 grants: Vec<String>,
462 expires_at: Instant,
463}
464
465impl RootMeta {
466 fn new(now: Instant) -> Self {
467 Self {
468 maintenance_pending: false,
469 maintenance_jobs_in_flight: 0,
470 maintenance_queued_kinds: VecDeque::new(),
471 maintenance_last_submitted: None,
472 maintenance_poisoned: false,
473 last_touched: now,
474 diagnostics_on_edit: false,
475 active_bash_waits: 0,
476 idle_artifacts_evicted: false,
477 unbound_quiesced: false,
478 consecutive_missing_sweeps: 0,
479 }
480 }
481
482 fn note_activity(&mut self) {
483 self.last_touched = Instant::now();
484 }
485
486 fn reactivate_bound(&mut self) {
487 self.note_activity();
488 self.idle_artifacts_evicted = false;
489 self.unbound_quiesced = false;
490 }
491}
492
493fn due_maintenance_jobs(
494 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
495 executor: Option<&Executor>,
496 bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
497 bg_wake_pending: &HashSet<RouteChannel>,
498 budget: usize,
499 pending_bind_roots: &HashSet<ProjectRootId>,
500) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
501 let mut jobs = Vec::new();
502 let mut deferred = false;
503 let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
504 roots.sort_by(|left, right| {
505 let left_last = live_roots
506 .get(left)
507 .and_then(|meta| meta.maintenance_last_submitted);
508 let right_last = live_roots
509 .get(right)
510 .and_then(|meta| meta.maintenance_last_submitted);
511 left_last
512 .cmp(&right_last)
513 .then_with(|| left.as_path().cmp(right.as_path()))
514 });
515
516 for root_id in roots {
517 let Some(meta) = live_roots.get_mut(&root_id) else {
518 continue;
519 };
520 if meta.maintenance_poisoned {
521 continue;
522 }
523
524 if pending_bind_roots.contains(&root_id) {
525 if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
526 deferred = true;
527 }
528 continue;
529 }
530
531 if !meta.maintenance_pending {
532 if jobs.len() >= budget {
533 deferred = true;
534 continue;
535 }
536 let executor_actor_context =
540 executor.and_then(|executor| executor.actor_context(&root_id));
541 let root_has_pending_bg_wake =
542 bg_sub_by_session.iter().any(|((sub_root, _), channel)| {
543 sub_root == &root_id && bg_wake_pending.contains(channel)
544 });
545 let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
546 Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
547 .into_iter()
548 .filter(|kind| {
549 if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
550 return false;
551 }
552 match kind {
553 MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
554 MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
555 MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
556 MaintenanceDrainKind::CompletionDrains => {
562 root_has_pending_bg_wake || ctx.completion_drains_have_work()
563 }
564 }
565 })
566 .collect(),
567 None if meta.unbound_quiesced => Vec::new(),
568 None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
570 };
571 if kinds_with_work.is_empty() {
572 continue;
573 }
574 meta.maintenance_pending = true;
575 meta.maintenance_queued_kinds.extend(kinds_with_work);
576 }
577
578 while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
579 if jobs.len() >= budget {
580 meta.maintenance_queued_kinds.push_front(kind);
581 deferred = true;
582 break;
583 }
584 meta.maintenance_jobs_in_flight += 1;
585 meta.maintenance_last_submitted = Some(Instant::now());
586 jobs.push((root_id.clone(), kind));
587 }
588
589 meta.maintenance_pending =
590 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
591 }
592
593 (jobs, deferred)
594}
595
596fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
597 match estimate.estimated_bytes {
598 Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
599 None if estimate.status == "busy" => "busy".to_string(),
600 None => "not estimated".to_string(),
601 }
602}
603
604fn optional_memory_label(bytes: Option<u64>) -> String {
605 bytes.map_or_else(
606 || "not estimated".to_string(),
607 |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
608 )
609}
610
611fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
612 format!(
613 "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
614 optional_memory_label(relief.rss_before_bytes),
615 optional_memory_label(relief.rss_after_bytes),
616 optional_memory_label(relief.allocator_before.bytes_in_use),
617 optional_memory_label(relief.allocator_after.bytes_in_use),
618 optional_memory_label(relief.allocator_before.size_allocated),
619 optional_memory_label(relief.allocator_after.size_allocated),
620 optional_memory_label(relief.allocator_before.retained_slack_bytes),
621 optional_memory_label(relief.allocator_after.retained_slack_bytes),
622 relief.bytes_released as f64 / (1024.0 * 1024.0),
623 )
624}
625
626fn idle_root_eviction_message(
627 root_id: &ProjectRootId,
628 memory: &crate::memory::RootMemorySnapshot,
629 pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
630) -> String {
631 let freed_bytes = [
634 &memory.semantic,
635 &memory.trigram,
636 &memory.symbols,
637 &memory.callgraph,
638 &memory.inspect,
639 ]
640 .iter()
641 .filter_map(|estimate| estimate.estimated_bytes)
642 .fold(0u64, u64::saturating_add);
643 let mut message = format!(
644 "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
645 root_id.as_path().display(),
646 freed_bytes as f64 / (1024.0 * 1024.0),
647 eviction_estimate_label(&memory.semantic),
648 eviction_estimate_label(&memory.trigram),
649 eviction_estimate_label(&memory.symbols),
650 eviction_estimate_label(&memory.callgraph),
651 eviction_estimate_label(&memory.inspect),
652 eviction_estimate_label(&memory.bash),
653 eviction_estimate_label(&memory.lsp),
654 eviction_estimate_label(&memory.parser_pool),
655 );
656 if let Some(pressure_relief) = pressure_relief {
657 message.push_str(&pressure_relief_label(pressure_relief));
658 }
659 message
660}
661
662fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
663 !live_roots.is_empty()
664 && live_roots.values().all(|meta| {
665 now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
666 && meta.active_bash_waits == 0
667 && !meta.maintenance_pending
668 && meta.maintenance_queued_kinds.is_empty()
669 })
670}
671
672fn allocator_pressure_relief_after_idle_sweep(
673 now: Instant,
674 live_roots: &HashMap<ProjectRootId, RootMeta>,
675 executor: &Executor,
676) -> Option<crate::memory::AllocatorPressureRelief> {
677 if !process_has_been_idle(now, live_roots)
678 || live_roots.keys().any(|root_id| {
679 executor
680 .actor_context(root_id)
681 .is_some_and(|ctx| ctx.artifact_eviction_blocked())
682 })
683 {
684 return None;
685 }
686
687 #[cfg(target_os = "macos")]
688 {
689 Some(crate::memory::relieve_allocator_pressure())
690 }
691 #[cfg(not(target_os = "macos"))]
692 {
693 None
694 }
695}
696
697fn quiesce_unbound_root(
698 root_id: &ProjectRootId,
699 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
700 executor: &Arc<Executor>,
701) {
702 let Some(meta) = live_roots.get_mut(root_id) else {
703 return;
704 };
705
706 let ctx = executor.actor_context(root_id);
707 if let Some(ctx) = ctx.as_ref() {
708 ctx.mark_subc_unbound();
712 }
713 let cancelled = executor.cancel_queued_maintenance(root_id);
714 let discarded = ctx
722 .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
723 .unwrap_or(0);
724 meta.unbound_quiesced = true;
725 meta.maintenance_queued_kinds.clear();
726 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
727 log::info!(
728 "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s)); cause=goodbye_unbound",
729 root_id.as_path().display(),
730 cancelled,
731 discarded
732 );
733}
734
735#[allow(clippy::too_many_arguments)]
736fn quiesce_connection_roots(
737 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
738 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
739 routes: &mut HashMap<RouteChannel, RouteIdentity>,
740 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
741 installed_route_epochs: &mut HashMap<u16, u32>,
742 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
743 executor: &Arc<Executor>,
744) {
745 for cancel in route_bash_cancels.values() {
746 cancel.token.cancel();
747 }
748 route_bash_cancels.clear();
749
750 let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
751 for pending in pending_binds.values_mut() {
752 pending.cancelled = true;
753 roots.insert(pending.bind_root_id.clone());
754 let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
755 }
756
757 for root_id in roots {
761 if live_roots.contains_key(&root_id) {
762 quiesce_unbound_root(&root_id, live_roots, executor);
763 } else if let Some(ctx) = executor.actor_context(&root_id) {
764 ctx.mark_subc_unbound();
765 executor.cancel_queued_maintenance(&root_id);
766 crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
767 }
768 }
769
770 routes.clear();
771 root_channels.clear();
772 installed_route_epochs.clear();
773}
774
775#[derive(Debug, Default)]
779struct ReclaimedRoutes {
780 highest_epoch_by_channel: HashMap<u16, u32>,
781}
782
783impl ReclaimedRoutes {
784 fn insert(&mut self, route: RouteChannel) {
785 self.highest_epoch_by_channel
786 .entry(route.channel)
787 .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
788 .or_insert(route.epoch);
789 }
790
791 fn contains(&self, route: RouteChannel) -> bool {
792 self.highest_epoch_by_channel
793 .get(&route.channel)
794 .is_some_and(|epoch| route.epoch <= *epoch)
795 }
796}
797
798#[derive(Debug, Default)]
799struct IdleReapOutcome {
800 evicted: usize,
801 forgotten_deleted_roots: Vec<ProjectRootId>,
802}
803
804fn reap_idle_roots(
805 now: Instant,
806 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
807 pending_binds: &HashMap<RouteChannel, PendingBind>,
808 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
809 executor: &Arc<Executor>,
810 metrics: &DispatchPathMetrics,
811) -> IdleReapOutcome {
812 let pending_bind_roots = pending_binds
813 .values()
814 .map(|pending| pending.bind_root_id.clone())
815 .collect::<HashSet<_>>();
816 let mut census = ReapBlockerCensus::default();
817 let mut candidates = Vec::new();
818
819 for (root_id, meta) in live_roots.iter_mut() {
820 let deleted = !root_id.as_path().exists();
821 if deleted {
822 meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
832 } else {
833 meta.consecutive_missing_sweeps = 0;
834 }
835 let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
836 let has_bound_route = root_channels
837 .get(root_id)
838 .is_some_and(|channels| !channels.is_empty());
839 let has_pending_bind = pending_bind_roots.contains(root_id);
840
841 if deleted {
842 let mut retained = false;
843 if !deletion_confirmed {
844 census.absence_unconfirmed += 1;
845 retained = true;
846 }
847 if meta.active_bash_waits > 0 {
851 census.bash_waits += 1;
852 retained = true;
853 }
854 if meta.maintenance_pending {
855 census.maintenance_pending += 1;
856 retained = true;
857 }
858 if !meta.maintenance_queued_kinds.is_empty() {
859 census.maintenance_queued += 1;
860 retained = true;
861 }
862 if has_pending_bind {
863 census.pending_binds += 1;
864 retained = true;
865 }
866 match executor.try_actor_is_idle(root_id) {
867 Some(true) => {}
868 Some(false) => {
869 census.actor_busy += 1;
870 retained = true;
871 }
872 None => {
873 census.actor_state_busy += 1;
874 retained = true;
875 }
876 }
877 if retained {
878 census.deleted_retained += 1;
879 continue;
880 }
881 } else {
882 if has_bound_route
886 || !meta.unbound_quiesced
887 || meta.idle_artifacts_evicted
888 || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
889 || meta.active_bash_waits > 0
890 || meta.maintenance_pending
891 || !meta.maintenance_queued_kinds.is_empty()
892 || has_pending_bind
893 || !executor.actor_is_idle(root_id)
894 {
895 continue;
896 }
897 }
898 candidates.push((root_id.clone(), deleted));
899 }
900
901 let mut reaped = Vec::new();
902 let mut forgotten_deleted_roots = Vec::new();
903 for (root_id, deleted) in candidates {
904 let Some(ctx) = executor.actor_context(&root_id) else {
905 if deleted {
906 census.deleted_retained += 1;
907 census.actor_busy += 1;
908 }
909 continue;
910 };
911 if deleted {
922 ctx.bash_background()
923 .kill_running_tasks_for_root(root_id.as_path());
924 }
925 let taken_pending = Some(ctx.take_pending_reconciliation_state());
926 if ctx.artifact_eviction_blocked() {
927 if let Some(pending) = taken_pending {
928 ctx.restore_pending_reconciliation_state(pending);
929 }
930 if deleted {
931 census.deleted_retained += 1;
932 census.artifact_eviction_blocked += 1;
933 }
934 continue;
935 }
936 let memory_before = ctx.memory_root_snapshot();
937 if !ctx.evict_idle_artifacts() {
938 if let Some(pending) = taken_pending {
939 ctx.restore_pending_reconciliation_state(pending);
940 }
941 if deleted {
942 census.deleted_retained += 1;
943 census.artifact_eviction_failed += 1;
944 }
945 continue;
946 }
947 drop(taken_pending);
948 ctx.stop_watcher_runtime_in_background();
949 ctx.invalidate_artifacts_after_watcher_gap();
952
953 if deleted {
954 if executor.retire_idle_actor_in_background(&root_id) {
955 live_roots.remove(&root_id);
956 forgotten_deleted_roots.push(root_id.clone());
957 } else {
958 census.deleted_retained += 1;
959 census.actor_busy += 1;
960 }
961 } else {
962 if let Some(meta) = live_roots.get_mut(&root_id) {
963 meta.idle_artifacts_evicted = true;
964 }
965 ctx.release_idle_reopenable_resources_in_background();
966 }
967 reaped.push((root_id, memory_before));
968 }
969
970 metrics.record_reap(census);
971 if census.deleted_retained > 0 {
972 log::info!(
973 "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
974 census.deleted_retained,
975 census.blocker_histogram()
976 );
977 }
978
979 let pressure_relief = (!reaped.is_empty())
980 .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
981 .flatten();
982 for (root_id, memory_before) in &reaped {
983 log::info!(
984 "{}",
985 idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
986 );
987 }
988 IdleReapOutcome {
989 evicted: reaped.len(),
990 forgotten_deleted_roots,
991 }
992}
993
994#[allow(clippy::too_many_arguments)]
995fn purge_deleted_root_residents(
996 root_id: &ProjectRootId,
997 routes: &mut HashMap<RouteChannel, RouteIdentity>,
998 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
999 installed_route_epochs: &mut HashMap<u16, u32>,
1000 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1001 retry_buffer: &mut RetryBuffer,
1002 reclaimed_routes: &mut ReclaimedRoutes,
1003 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1004 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1005 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1006 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1007 bg_wake_pending: &mut HashSet<RouteChannel>,
1008 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1009 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1010) {
1011 let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1012 stale_routes.extend(
1013 routes
1014 .iter()
1015 .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1016 );
1017 stale_routes.extend(
1018 bg_sub_by_session
1019 .iter()
1020 .filter_map(|((root, _), route)| (root == root_id).then_some(*route)),
1021 );
1022 stale_routes.extend(
1023 pending_bash_asks
1024 .values()
1025 .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1026 );
1027
1028 for route in stale_routes {
1029 reclaimed_routes.insert(route);
1030 remove_installed_route(installed_route_epochs, route);
1031 remove_route_channel(routes, root_channels, route);
1032 if let Some(cancel) = route_bash_cancels.remove(&route) {
1033 cancel.token.cancel();
1034 }
1035 retry_buffer.remove(&route);
1036 bg_subs.remove(&route);
1037 bg_wake_pending.remove(&route);
1038 }
1039 root_channels.remove(root_id);
1040 session_identity.retain(|(root, _), _| root != root_id);
1041 push_buffer.retain(|key, _| &key.root != root_id);
1042 bg_wake_epoch.retain(|(root, _), _| root != root_id);
1043 pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1044 bg_sub_by_session.retain(|(root, _), _| root != root_id);
1045
1046 log::info!(
1047 "subc attach: fully forgot deleted root {}; cause=absence_reclaim",
1048 root_id.as_path().display()
1049 );
1050}
1051
1052#[allow(clippy::too_many_arguments)]
1053fn submit_due_maintenance_jobs(
1054 executor: &Arc<Executor>,
1055 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1056 pending_binds: &HashMap<RouteChannel, PendingBind>,
1057 bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
1058 bg_wake_pending: &HashSet<RouteChannel>,
1059 bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1060 maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1061 metrics: &Arc<DispatchPathMetrics>,
1062) {
1063 let pending_bind_roots = pending_binds
1064 .values()
1065 .map(|pending| pending.bind_root_id.clone())
1066 .collect::<HashSet<_>>();
1067 let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1068 live_roots,
1069 Some(executor),
1070 bg_sub_by_session,
1071 bg_wake_pending,
1072 MAINTENANCE_SUBMIT_BUDGET,
1073 &pending_bind_roots,
1074 );
1075 if deferred_jobs {
1076 metrics
1077 .maintenance_budget_deferrals
1078 .fetch_add(1, Ordering::Relaxed);
1079 }
1080 for (root_id, kind) in due_jobs {
1081 let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1082 bg_sub_by_session
1083 .iter()
1084 .filter_map(|((root, session), _)| {
1085 if root == &root_id {
1086 Some((
1087 session.clone(),
1088 bg_wake_epoch
1089 .get(&(root_id.clone(), session.clone()))
1090 .copied()
1091 .unwrap_or(0),
1092 ))
1093 } else {
1094 None
1095 }
1096 })
1097 .collect()
1098 } else {
1099 Vec::new()
1100 };
1101 submit_maintenance_job(
1102 executor,
1103 root_id,
1104 kind,
1105 bg_sessions_to_check,
1106 maintenance_tx,
1107 metrics,
1108 );
1109 }
1110}
1111
1112fn should_requiesce_after_maintenance(
1113 meta: &RootMeta,
1114 completed_kind: MaintenanceDrainKind,
1115 bind_pending: bool,
1116) -> bool {
1117 meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1118}
1119
1120fn note_maintenance_completion(
1121 meta: &mut RootMeta,
1122 requeue_kind: Option<MaintenanceDrainKind>,
1123 fatal: bool,
1124 defer_requeue: bool,
1125) {
1126 if fatal {
1127 meta.maintenance_poisoned = true;
1128 }
1129
1130 if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1131 meta.maintenance_queued_kinds.push_back(kind);
1132 }
1133
1134 meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1135 meta.maintenance_pending =
1136 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1137}
1138
1139fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1140 RouteChannel { channel, epoch }
1141}
1142
1143fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1144 if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1145 installed_epochs.remove(&route.channel);
1146 }
1147}
1148
1149fn ingress_route_should_be_processed(
1150 installed_epochs: &HashMap<u16, u32>,
1151 reclaimed_routes: &ReclaimedRoutes,
1152 frame: &Frame,
1153) -> bool {
1154 if frame.header.channel == 0
1155 || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1156 {
1157 return true;
1158 }
1159
1160 frame.header.ty == FrameType::Request
1165 && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1166}
1167
1168fn bash_elicitation_timeout() -> Duration {
1169 if cfg!(debug_assertions) {
1170 if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1171 if let Ok(ms) = raw.parse::<u64>() {
1172 if ms > 0 {
1173 return Duration::from_millis(ms);
1174 }
1175 }
1176 }
1177 }
1178 BASH_ELICITATION_TIMEOUT
1179}
1180
1181fn allocate_reverse_corr(
1182 pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1183 route: RouteChannel,
1184 next_corr: &mut u64,
1185) -> u64 {
1186 loop {
1187 let corr = *next_corr;
1188 *next_corr = (*next_corr).wrapping_add(1).max(1);
1189 if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1190 return corr;
1191 }
1192 }
1193}
1194
1195fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1196 match kind {
1197 crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1198 crate::bash_permissions::PermissionKind::Bash => "bash",
1199 }
1200}
1201
1202fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1203 let mut patterns = Vec::new();
1204 let mut seen = HashSet::new();
1205 for ask in asks {
1206 for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1207 if seen.insert(pattern.clone()) {
1208 patterns.push(pattern.clone());
1209 }
1210 }
1211 }
1212 patterns
1213}
1214
1215fn bash_elicitation_message(
1216 command: &str,
1217 asks: &[crate::bash_permissions::PermissionAsk],
1218) -> String {
1219 let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1220 let patterns = bash_elicitation_patterns(asks);
1221 let pattern_text = if patterns.is_empty() {
1222 "no matched permission patterns".to_string()
1223 } else {
1224 patterns.join(", ")
1225 };
1226 let ask_kinds = asks
1227 .iter()
1228 .map(|ask| bash_permission_kind_label(&ask.kind))
1229 .collect::<HashSet<_>>()
1230 .into_iter()
1231 .collect::<Vec<_>>()
1232 .join(", ");
1233 if ask_kinds.is_empty() {
1234 format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1235 } else {
1236 format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1237 }
1238}
1239
1240fn bash_elicitation_request_body(
1241 command: &str,
1242 asks: &[crate::bash_permissions::PermissionAsk],
1243) -> Value {
1244 json!({
1245 "method": BASH_ELICITATION_CREATE_METHOD,
1246 "params": {
1247 "mode": "form",
1248 "message": bash_elicitation_message(command, asks),
1249 "requestedSchema": {
1250 "type": "object",
1251 "properties": {
1252 "decision": {
1253 "type": "string",
1254 "enum": ["allow", "deny"],
1255 "description": "Choose allow to run this bash command once, or deny to block it."
1256 }
1257 },
1258 "required": ["decision"],
1259 "additionalProperties": false
1260 },
1261 "_meta": {
1262 "aft": {
1263 "tool": "bash",
1264 "command": command,
1265 "asks": asks
1266 }
1267 }
1268 }
1269 })
1270}
1271
1272fn build_bash_elicitation_request_frame(
1273 ver: u8,
1274 route: RouteChannel,
1275 corr: u64,
1276 flags: Flags,
1277 command: &str,
1278 asks: &[crate::bash_permissions::PermissionAsk],
1279) -> Result<Frame, SubcError> {
1280 let body = bash_elicitation_request_body(command, asks);
1281 Frame::build_with_version(
1282 ver,
1283 FrameType::Request,
1284 flags,
1285 route.channel,
1286 route.epoch,
1287 corr,
1288 serde_json::to_vec(&body).map_err(SubcError::Json)?,
1289 )
1290 .map_err(SubcError::FrameBuild)
1291}
1292
1293fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1294 let Ok(value) = serde_json::from_slice::<Value>(body) else {
1295 return false;
1296 };
1297 flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1298}
1299
1300fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1301 let Some(object) = value.as_object() else {
1302 return false;
1303 };
1304 object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1305}
1306
1307fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1308 let Some(object) = value.as_object() else {
1309 return false;
1310 };
1311 if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1312 return false;
1313 }
1314 let Some(content) = object.get("content").and_then(Value::as_object) else {
1315 return false;
1316 };
1317 content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1318}
1319
1320#[allow(clippy::too_many_arguments)]
1321async fn settle_pending_bash_ask_denied(
1322 tx: &WriterSender,
1323 pending: PendingBashAsk,
1324 routes: &HashMap<RouteChannel, RouteIdentity>,
1325 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1326 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1327 shutdown: &Arc<Notify>,
1328 metrics: &DispatchPathMetrics,
1329) -> Result<(), SubcError> {
1330 let completion = bash::bash_denied_untrusted_completion(
1331 pending.route,
1332 pending.tool_corr,
1333 pending.tool_flags,
1334 pending.tool_ver,
1335 pending.root,
1336 pending.request_id,
1337 pending.format_context,
1338 );
1339 bash::handle_bash_deferred_completion(
1340 tx,
1341 completion,
1342 routes,
1343 live_roots,
1344 route_bash_cancels,
1345 shutdown,
1346 metrics,
1347 )
1348 .await
1349}
1350
1351fn take_pending_bash_asks_for_route(
1352 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1353 route: RouteChannel,
1354) -> Vec<PendingBashAsk> {
1355 let keys = pending_bash_asks
1356 .keys()
1357 .copied()
1358 .filter(|key| key.route == route)
1359 .collect::<Vec<_>>();
1360 keys.into_iter()
1361 .filter_map(|key| pending_bash_asks.remove(&key))
1362 .collect()
1363}
1364
1365#[allow(clippy::too_many_arguments)]
1366async fn settle_pending_bash_asks_for_route(
1367 tx: &WriterSender,
1368 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1369 route: RouteChannel,
1370 routes: &HashMap<RouteChannel, RouteIdentity>,
1371 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1372 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1373 shutdown: &Arc<Notify>,
1374 metrics: &DispatchPathMetrics,
1375) -> Result<(), SubcError> {
1376 for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1377 settle_pending_bash_ask_denied(
1378 tx,
1379 pending,
1380 routes,
1381 live_roots,
1382 route_bash_cancels,
1383 shutdown,
1384 metrics,
1385 )
1386 .await?;
1387 }
1388 Ok(())
1389}
1390
1391#[allow(clippy::too_many_arguments)]
1392async fn settle_all_pending_bash_asks(
1393 tx: &WriterSender,
1394 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1395 routes: &HashMap<RouteChannel, RouteIdentity>,
1396 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1397 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1398 shutdown: &Arc<Notify>,
1399 metrics: &DispatchPathMetrics,
1400) -> Result<(), SubcError> {
1401 let pending = pending_bash_asks
1402 .drain()
1403 .map(|(_, pending)| pending)
1404 .collect::<Vec<_>>();
1405 for pending in pending {
1406 settle_pending_bash_ask_denied(
1407 tx,
1408 pending,
1409 routes,
1410 live_roots,
1411 route_bash_cancels,
1412 shutdown,
1413 metrics,
1414 )
1415 .await?;
1416 }
1417 Ok(())
1418}
1419
1420#[allow(clippy::too_many_arguments)]
1421async fn expire_pending_bash_asks(
1422 tx: &WriterSender,
1423 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1424 routes: &HashMap<RouteChannel, RouteIdentity>,
1425 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1426 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1427 shutdown: &Arc<Notify>,
1428 metrics: &DispatchPathMetrics,
1429) -> Result<(), SubcError> {
1430 let now = Instant::now();
1431 let expired = pending_bash_asks
1432 .iter()
1433 .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1434 .collect::<Vec<_>>();
1435 for key in expired {
1436 if let Some(pending) = pending_bash_asks.remove(&key) {
1437 log::debug!(
1438 "subc attach: bash elicitation request {} on route {} expired fail-closed",
1439 key.corr,
1440 pending.route
1441 );
1442 settle_pending_bash_ask_denied(
1443 tx,
1444 pending,
1445 routes,
1446 live_roots,
1447 route_bash_cancels,
1448 shutdown,
1449 metrics,
1450 )
1451 .await?;
1452 }
1453 }
1454 Ok(())
1455}
1456
1457#[allow(clippy::too_many_arguments)]
1458async fn handle_bash_elicitation_reply(
1459 tx: &WriterSender,
1460 frame: &Frame,
1461 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1462 routes: &HashMap<RouteChannel, RouteIdentity>,
1463 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1464 executor: &Arc<Executor>,
1465 shutdown: &Arc<Notify>,
1466 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1467 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1468 metrics: &Arc<DispatchPathMetrics>,
1469 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1470 dispatch: DispatchFn,
1471) -> Result<(), SubcError> {
1472 let key = ReverseCorrKey {
1473 route: route_key(frame.header.channel, frame.header.epoch),
1474 corr: frame.header.corr,
1475 };
1476 let Some(pending) = pending_bash_asks.remove(&key) else {
1477 return Ok(());
1478 };
1479
1480 if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1481 if routes.contains_key(&key.route) {
1482 bash::submit_deferred_bash(
1483 executor,
1484 bash_deferred_tx,
1485 bash_poll_touch_tx,
1486 metrics,
1487 dispatch,
1488 pending.root,
1489 pending.project_root,
1490 pending.session_id,
1491 pending.request_id,
1492 pending.route,
1493 pending.tool_corr,
1494 pending.tool_flags,
1495 pending.tool_ver,
1496 pending.arguments,
1497 pending.format_context,
1498 pending.cancel,
1499 BindTrust::Untrusted,
1500 pending.spawn_principal,
1501 Some(pending.grants),
1502 );
1503 return Ok(());
1504 }
1505 log::debug!(
1506 "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1507 key.corr,
1508 pending.route
1509 );
1510 }
1511
1512 settle_pending_bash_ask_denied(
1513 tx,
1514 pending,
1515 routes,
1516 live_roots,
1517 route_bash_cancels,
1518 shutdown,
1519 metrics,
1520 )
1521 .await
1522}
1523
1524#[allow(clippy::too_many_arguments)]
1525async fn cancel_pending_bash_ask_for_tool_call(
1526 tx: &WriterSender,
1527 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1528 route: RouteChannel,
1529 tool_corr: u64,
1530 routes: &HashMap<RouteChannel, RouteIdentity>,
1531 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1532 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1533 shutdown: &Arc<Notify>,
1534 metrics: &DispatchPathMetrics,
1535) -> Result<(), SubcError> {
1536 let keys = pending_bash_asks
1537 .iter()
1538 .filter_map(|(key, pending)| {
1539 (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1540 })
1541 .collect::<Vec<_>>();
1542 for key in keys {
1543 if let Some(pending) = pending_bash_asks.remove(&key) {
1544 settle_pending_bash_ask_denied(
1545 tx,
1546 pending,
1547 routes,
1548 live_roots,
1549 route_bash_cancels,
1550 shutdown,
1551 metrics,
1552 )
1553 .await?;
1554 }
1555 }
1556 Ok(())
1557}
1558
1559fn remove_root_channel(
1560 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1561 root: &ProjectRootId,
1562 channel: RouteChannel,
1563) {
1564 let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1565 channels.remove(&channel);
1566 channels.is_empty()
1567 } else {
1568 false
1569 };
1570 if remove_root {
1571 root_channels.remove(root);
1572 }
1573}
1574
1575fn remove_route_channel(
1576 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1577 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1578 channel: RouteChannel,
1579) -> Option<RouteIdentity> {
1580 let removed = routes.remove(&channel);
1581 if let Some(identity) = &removed {
1582 remove_root_channel(root_channels, &identity.root, channel);
1583 }
1584 removed
1585}
1586
1587fn insert_route_channel(
1588 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1589 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1590 channel: RouteChannel,
1591 identity: RouteIdentity,
1592) {
1593 if let Some(previous) = routes.insert(channel, identity.clone()) {
1594 remove_root_channel(root_channels, &previous.root, channel);
1595 }
1596 root_channels
1597 .entry(identity.root.clone())
1598 .or_default()
1599 .insert(channel);
1600}
1601
1602fn remove_bg_subscription_index(
1603 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1604 channel: RouteChannel,
1605 identity: Option<&RouteIdentity>,
1606) {
1607 if let Some(identity) = identity {
1608 let key = (identity.root.clone(), identity.session.clone());
1609 if bg_sub_by_session.get(&key).copied() == Some(channel) {
1610 bg_sub_by_session.remove(&key);
1611 }
1612 } else {
1613 bg_sub_by_session.retain(|_, mapped_channel| *mapped_channel != channel);
1614 }
1615}
1616
1617fn route_removal_will_quiesce_root(
1618 root: &ProjectRootId,
1619 route: RouteChannel,
1620 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1621 has_pending_bind: bool,
1622 replacement_root: Option<&ProjectRootId>,
1623) -> bool {
1624 let removes_last_route = root_channels
1625 .get(root)
1626 .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1627 removes_last_route && !has_pending_bind && replacement_root != Some(root)
1628}
1629
1630fn should_quiesce_removed_root(
1631 root: &ProjectRootId,
1632 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1633 has_pending_bind: bool,
1634 replacement_root: Option<&ProjectRootId>,
1635) -> bool {
1636 !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1637}
1638
1639async fn end_bg_subscription(
1640 writer_tx: &WriterSender,
1641 metrics: &DispatchPathMetrics,
1642 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1643 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1644 bg_wake_pending: &mut HashSet<RouteChannel>,
1645 channel: RouteChannel,
1646 identity: Option<&RouteIdentity>,
1647) -> Result<(), SubcError> {
1648 if let Some(sub) = bg_subs.remove(&channel) {
1649 bg_wake_pending.remove(&channel);
1650 remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1651 push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1652 }
1653 Ok(())
1654}
1655
1656#[allow(clippy::too_many_arguments)]
1657async fn teardown_installed_route(
1658 tx: &WriterSender,
1659 metrics: &DispatchPathMetrics,
1660 executor: &Arc<Executor>,
1661 channel: RouteChannel,
1662 cancellation_reason: &str,
1663 replacement_root: Option<&ProjectRootId>,
1664 installed_route_epochs: &mut HashMap<u16, u32>,
1665 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1666 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1667 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1668 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1669 bg_wake_pending: &mut HashSet<RouteChannel>,
1670 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1671 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1672 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1673 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1674 retry_buffer: &mut RetryBuffer,
1675 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1676 shutdown: &Arc<Notify>,
1677) -> Result<(), SubcError> {
1678 remove_installed_route(installed_route_epochs, channel);
1679 end_bg_subscription(
1680 tx,
1681 metrics,
1682 bg_subs,
1683 bg_sub_by_session,
1684 bg_wake_pending,
1685 channel,
1686 routes.get(&channel),
1687 )
1688 .await?;
1689 settle_pending_bash_asks_for_route(
1690 tx,
1691 pending_bash_asks,
1692 channel,
1693 routes,
1694 live_roots,
1695 route_bash_cancels,
1696 shutdown,
1697 metrics,
1698 )
1699 .await?;
1700 if let Some(cancel) = route_bash_cancels.remove(&channel) {
1701 cancel.token.cancel();
1702 }
1703 if let Some(pending) = pending_binds.get_mut(&channel) {
1704 pending.cancelled = true;
1705 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1706 log::debug!(
1707 "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1708 channel.channel
1709 );
1710 }
1711 let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1712 if let Some(identity) = routes.get(&channel) {
1713 let has_pending_bind = pending_binds
1714 .values()
1715 .any(|pending| pending.bind_root_id == identity.root);
1716 if route_removal_will_quiesce_root(
1717 &identity.root,
1718 channel,
1719 root_channels,
1720 has_pending_bind,
1721 replacement_root,
1722 ) {
1723 if let Some(ctx) = executor.actor_context(&identity.root) {
1724 ctx.mark_subc_unbound();
1727 }
1728 }
1729 }
1730 if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
1731 if migrated > 0 {
1732 log::debug!(
1733 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
1734 channel.channel
1735 );
1736 }
1737 if let Some(meta) = live_roots.get_mut(&identity.root) {
1738 let idle_for = meta.last_touched.elapsed();
1739 meta.note_activity();
1740 log::debug!(
1741 "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
1742 channel.channel,
1743 identity.root.as_path().display(),
1744 identity.harness,
1745 identity.session,
1746 idle_for
1747 );
1748 } else {
1749 log::debug!(
1750 "subc attach: route {} torn down for root {} harness {} session {}",
1751 channel.channel,
1752 identity.root.as_path().display(),
1753 identity.harness,
1754 identity.session
1755 );
1756 }
1757 let has_pending_bind = pending_binds
1758 .values()
1759 .any(|pending| pending.bind_root_id == identity.root);
1760 if should_quiesce_removed_root(
1761 &identity.root,
1762 root_channels,
1763 has_pending_bind,
1764 replacement_root,
1765 ) {
1766 quiesce_unbound_root(&identity.root, live_roots, executor);
1767 }
1768 } else {
1769 if migrated > 0 {
1770 log::debug!(
1771 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
1772 channel.channel
1773 );
1774 }
1775 log::debug!("subc attach: unbound route {} torn down", channel.channel);
1776 }
1777 Ok(())
1778}
1779
1780fn remember_session_identity(
1781 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1782 identity: &RouteIdentity,
1783) {
1784 let key = (identity.root.clone(), identity.session.clone());
1785 if matches!(identity.trust, BindTrust::Untrusted)
1786 && session_identity
1787 .get(&key)
1788 .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
1789 {
1790 return;
1791 }
1792
1793 session_identity.insert(
1798 key,
1799 RetainedSessionIdentity {
1800 harness: identity.harness.clone(),
1801 trust: identity.trust,
1802 },
1803 );
1804}
1805
1806fn replay_key_for_session(
1807 session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1808 root: &ProjectRootId,
1809 session: &str,
1810) -> Option<(push::ReplayKey, BindTrust)> {
1811 let retained = session_identity.get(&(root.clone(), session.to_string()))?;
1812 Some((
1813 push::ReplayKey {
1814 root: root.clone(),
1815 harness: retained.harness.clone(),
1816 session: session.to_string(),
1817 },
1818 retained.trust,
1819 ))
1820}
1821pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
1824
1825#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1826enum ModuleLoopExit {
1827 Graceful,
1828 SkipSearchFlush,
1829}
1830
1831pub fn run_subc_mode(
1836 connection_file_path: &Path,
1837 ctx: Arc<AppContext>,
1838 executor: Arc<Executor>,
1839 dispatch: DispatchFn,
1840 user_config_path: Option<PathBuf>,
1841) -> Result<(), SubcError> {
1842 run_subc_mode_inner(
1846 connection_file_path,
1847 ctx,
1848 executor,
1849 dispatch,
1850 user_config_path,
1851 false,
1852 )
1853}
1854
1855fn run_subc_mode_inner(
1856 connection_file_path: &Path,
1857 ctx: Arc<AppContext>,
1858 executor: Arc<Executor>,
1859 dispatch: DispatchFn,
1860 user_config_path: Option<PathBuf>,
1861 allow_native_passthrough: bool,
1862) -> Result<(), SubcError> {
1863 let runtime = tokio::runtime::Builder::new_current_thread()
1864 .enable_all()
1865 .build()
1866 .map_err(SubcError::Runtime)?;
1867
1868 let executor_for_loop = Arc::clone(&executor);
1869 let loop_result = runtime.block_on(async move {
1870 let shared_app = ctx.app();
1871 drop(ctx);
1872 let stream = connect_and_authenticate(connection_file_path).await?;
1873 log::info!(
1874 "subc attach: authenticated to daemon via {}",
1875 connection_file_path.display()
1876 );
1877 let (read_half, write_half) = tokio::io::split(stream);
1878 run_module_loop(
1879 read_half,
1880 write_half,
1881 shared_app,
1882 executor_for_loop,
1883 dispatch,
1884 user_config_path,
1885 allow_native_passthrough,
1886 )
1887 .await
1888 });
1889
1890 let actor_contexts = executor.actor_contexts();
1891 if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
1892 flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
1895 }
1896 for actor_ctx in &actor_contexts {
1897 actor_ctx.lsp().shutdown_all();
1898 actor_ctx.bash_background().detach();
1899 }
1900
1901 loop_result.map(|_| ())
1902}
1903
1904fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
1905 for actor_ctx in actor_contexts {
1906 let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
1907 }
1908 let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
1909}
1910
1911#[doc(hidden)]
1916pub fn run_subc_mode_for_test(
1917 connection_file_path: &Path,
1918 ctx: Arc<AppContext>,
1919 executor: Arc<Executor>,
1920 dispatch: DispatchFn,
1921 user_config_path: Option<PathBuf>,
1922) -> Result<(), SubcError> {
1923 run_subc_mode_inner(
1924 connection_file_path,
1925 ctx,
1926 executor,
1927 dispatch,
1928 user_config_path,
1929 true,
1930 )
1931}
1932
1933#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1934enum AttachErrorClass {
1935 Transient,
1936 Permanent,
1937}
1938
1939impl fmt::Display for AttachErrorClass {
1940 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1941 match self {
1942 Self::Transient => f.write_str("transient"),
1943 Self::Permanent => f.write_str("permanent"),
1944 }
1945 }
1946}
1947
1948#[derive(Clone, Copy)]
1949struct AttachRetryPolicy {
1950 budget: Duration,
1951 initial_backoff: Duration,
1952 max_backoff: Duration,
1953 jitter_percent: u64,
1954}
1955
1956const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
1957 budget: ATTACH_RETRY_BUDGET,
1958 initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
1959 max_backoff: ATTACH_RETRY_MAX_BACKOFF,
1960 jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
1961};
1962
1963fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
1966 let transient = match error {
1967 SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
1968 SubcError::Auth { source, .. } => match source {
1969 subc_transport::AuthError::Timeout { .. }
1970 | subc_transport::AuthError::UnexpectedEof { .. } => true,
1971 subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
1972 _ => false,
1973 },
1974 _ => false,
1975 };
1976 if transient {
1977 AttachErrorClass::Transient
1978 } else {
1979 AttachErrorClass::Permanent
1980 }
1981}
1982
1983fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
1984 matches!(
1985 kind,
1986 io::ErrorKind::ConnectionRefused
1987 | io::ErrorKind::TimedOut
1988 | io::ErrorKind::ConnectionReset
1989 | io::ErrorKind::ConnectionAborted
1990 | io::ErrorKind::BrokenPipe
1991 | io::ErrorKind::UnexpectedEof
1992 )
1993}
1994
1995async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
1999 connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
2000}
2001
2002async fn connect_and_authenticate_with_policy(
2003 connection_file_path: &Path,
2004 policy: AttachRetryPolicy,
2005) -> Result<TcpStream, SubcError> {
2006 let started_at = Instant::now();
2007 let deadline = started_at + policy.budget;
2008 let mut attempt = 0_u32;
2009 let mut backoff = policy.initial_backoff;
2010 let mut history = Vec::new();
2011
2012 loop {
2013 attempt = attempt.saturating_add(1);
2014 let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2015 Ok(stream) => return Ok(stream),
2016 Err(error) => error,
2017 };
2018 let class = classify_attach_error(&error);
2019 let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2020 history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2021
2022 if class == AttachErrorClass::Permanent {
2023 log_attach_final_failure(started_at.elapsed(), &history);
2024 return Err(error);
2025 }
2026
2027 let remaining = deadline.saturating_duration_since(Instant::now());
2028 if remaining.is_zero() {
2029 log_attach_final_failure(started_at.elapsed(), &history);
2030 return Err(error);
2031 }
2032
2033 let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2034 log::info!(
2035 "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2036 );
2037 tokio::time::sleep(delay).await;
2038
2039 if Instant::now() >= deadline {
2040 log_attach_final_failure(started_at.elapsed(), &history);
2041 return Err(error);
2042 }
2043 backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2044 }
2045}
2046
2047fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2048 let jitter_percent = jitter_percent.min(100);
2049 if jitter_percent == 0 {
2050 return base;
2051 }
2052
2053 let mut random_bytes = [0_u8; 8];
2054 let random = if getrandom::fill(&mut random_bytes).is_ok() {
2055 u64::from_le_bytes(random_bytes)
2056 } else {
2057 let timestamp = std::time::SystemTime::now()
2058 .duration_since(std::time::UNIX_EPOCH)
2059 .unwrap_or_default()
2060 .subsec_nanos();
2061 u64::from(timestamp) ^ u64::from(attempt)
2062 };
2063 let span = jitter_percent.saturating_mul(2).saturating_add(1);
2064 let multiplier_percent = 100 - jitter_percent + random % span;
2065 let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2066 Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2067}
2068
2069fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2070 log::error!(
2071 "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2072 history.len(),
2073 history.join(" | ")
2074 );
2075}
2076
2077async fn connect_and_authenticate_once(
2078 connection_file_path: &Path,
2079 deadline: Instant,
2080) -> Result<TcpStream, SubcError> {
2081 let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2084 SubcError::ConnectionFile {
2085 path: connection_file_path.to_path_buf(),
2086 source,
2087 }
2088 })?;
2089
2090 let endpoint = conn
2091 .endpoints
2092 .first()
2093 .ok_or_else(|| SubcError::NoEndpoint {
2094 path: connection_file_path.to_path_buf(),
2095 })?;
2096 let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2097 let ip = endpoint
2098 .host
2099 .parse::<IpAddr>()
2100 .map_err(|_| SubcError::InvalidEndpoint {
2101 path: connection_file_path.to_path_buf(),
2102 endpoint: endpoint_label.clone(),
2103 })?;
2104 let addr = SocketAddr::new(ip, endpoint.port);
2105
2106 let connect_budget = deadline.saturating_duration_since(Instant::now());
2107 let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2108 .await
2109 .map_err(|_| SubcError::Connect {
2110 endpoint: endpoint_label.clone(),
2111 source: io::Error::new(
2112 io::ErrorKind::TimedOut,
2113 "initial subc attach retry budget elapsed during TCP connect",
2114 ),
2115 })?
2116 .map_err(|source| SubcError::Connect {
2117 endpoint: endpoint_label.clone(),
2118 source,
2119 })?;
2120 stream
2121 .set_nodelay(true)
2122 .map_err(|source| SubcError::Connect {
2123 endpoint: endpoint_label.clone(),
2124 source,
2125 })?;
2126
2127 let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2128 authenticate_client(&mut stream, &conn, auth_budget)
2129 .await
2130 .map_err(|source| SubcError::Auth {
2131 endpoint: endpoint_label,
2132 source,
2133 })?;
2134
2135 Ok(stream)
2136}
2137
2138#[allow(clippy::too_many_arguments)]
2139async fn process_route_bind_completion(
2140 writer_tx: &WriterSender,
2141 completion: RouteBindCompletion,
2142 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2143 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2144 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2145 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2146 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2147 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2148 installed_route_epochs: &mut HashMap<u16, u32>,
2149 executor: &Arc<Executor>,
2150 shutdown: &Arc<Notify>,
2151 metrics: &Arc<DispatchPathMetrics>,
2152) -> Result<(), SubcError> {
2153 decrement_counted_channel(&metrics.control_completion_queued);
2154 handle_route_bind_completion(
2155 writer_tx,
2156 completion,
2157 routes,
2158 root_channels,
2159 session_identity,
2160 push_buffer,
2161 live_roots,
2162 pending_binds,
2163 installed_route_epochs,
2164 executor,
2165 shutdown,
2166 metrics,
2167 )
2168 .await
2169}
2170
2171#[allow(clippy::too_many_arguments)]
2172async fn drain_pending_route_bind_completions(
2173 control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2174 writer_tx: &WriterSender,
2175 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2176 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2177 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2178 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2179 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2180 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2181 installed_route_epochs: &mut HashMap<u16, u32>,
2182 executor: &Arc<Executor>,
2183 shutdown: &Arc<Notify>,
2184 metrics: &Arc<DispatchPathMetrics>,
2185) -> Result<usize, SubcError> {
2186 let mut drained = 0;
2187 while let Ok(completion) = control_completion_rx.try_recv() {
2188 process_route_bind_completion(
2189 writer_tx,
2190 completion,
2191 routes,
2192 root_channels,
2193 session_identity,
2194 push_buffer,
2195 live_roots,
2196 pending_binds,
2197 installed_route_epochs,
2198 executor,
2199 shutdown,
2200 metrics,
2201 )
2202 .await?;
2203 drained += 1;
2204 }
2205 Ok(drained)
2206}
2207
2208async fn run_module_loop<R, W>(
2212 mut read: R,
2213 mut write: W,
2214 shared_app: Arc<App>,
2215 executor: Arc<Executor>,
2216 dispatch: DispatchFn,
2217 user_config_path: Option<PathBuf>,
2218 allow_native_passthrough: bool,
2219) -> Result<ModuleLoopExit, SubcError>
2220where
2221 R: AsyncRead + Unpin + Send + 'static,
2222 W: AsyncWrite + Unpin + Send + 'static,
2223{
2224 let hello = ModuleHelloBody {
2228 manifest: build_manifest(),
2229 protocol_ver: PROTOCOL_VERSION,
2230 control_ops: control_ops(),
2231 launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2232 };
2233 let hello_frame = Frame::build(
2234 FrameType::Hello,
2235 control_flags(),
2236 0,
2237 0,
2238 HELLO_CORR,
2239 serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2240 )
2241 .map_err(SubcError::FrameBuild)?;
2242 write_frame(&mut write, &hello_frame)
2243 .await
2244 .map_err(SubcError::FrameIo)?;
2245
2246 match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2248 None => return Err(SubcError::ClosedBeforeHelloAck),
2249 Some(frame) => match frame.header.ty {
2250 FrameType::HelloAck => {
2251 log::info!("subc attach: registered (HelloAck received)");
2252 }
2253 FrameType::Error => {
2254 let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2255 return Err(SubcError::HelloRejected { body });
2256 }
2257 other => return Err(SubcError::UnexpectedFrame { ty: other }),
2258 },
2259 }
2260
2261 let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2262 let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2263 let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2264 let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2271 let reader_task = spawn_reader_task(read, reader_tx);
2272 let shutdown = Arc::new(Notify::new());
2273 let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2282 let mut next_maintenance_at = next_drain_at;
2283 #[cfg(target_os = "macos")]
2286 let mut last_slack_relief: Option<std::time::Instant> = None;
2287 let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2288 let (bash_deferred_tx, mut bash_deferred_rx) =
2289 mpsc::channel::<bash::BashDeferredCompletion>(256);
2290 let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2291 let (control_completion_tx, mut control_completion_rx) =
2292 mpsc::channel::<RouteBindCompletion>(256);
2293 let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2294 let lossy_overflow = Arc::new(push::LossyOverflow::default());
2295 let lossy_seq = Arc::new(AtomicU64::new(0));
2296 let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2297 let push_senders = PushSenders {
2298 lossy_tx,
2299 reliable_tx,
2300 lossy_overflow: Arc::clone(&lossy_overflow),
2301 lossy_seq,
2302 };
2303 let connection_cancel = PersistentCancelSignal::new();
2304 let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2305 let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2306 let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2307 let mut bg_sub_by_session: HashMap<(ProjectRootId, String), RouteChannel> = HashMap::new();
2308 let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2309 let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2310 let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2311 let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2312 HashMap::new();
2313 let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2314 let mut retry_buffer: RetryBuffer = HashMap::new();
2315 let mut reclaimed_routes = ReclaimedRoutes::default();
2316 let mut completed_tasks = push::CompletedTaskIds::default();
2317 let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2318 let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2319 let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2320 let mut next_bash_ask_corr: u64 = 1;
2321 let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2322
2323 let loop_result: Result<ModuleLoopExit, SubcError> = loop {
2324 shared_app.set_open_route_count(routes.len());
2325 crate::logging::perf_tick(Some(&executor));
2326 dispatch_path_metrics.mark_frame_loop_tick();
2327 if let Err(error) = expire_pending_bash_asks(
2328 &writer_tx,
2329 &mut pending_bash_asks,
2330 &routes,
2331 &mut live_roots,
2332 &mut route_bash_cancels,
2333 &shutdown,
2334 &dispatch_path_metrics,
2335 )
2336 .await
2337 {
2338 break Err(error);
2339 }
2340
2341 match drain_pending_route_bind_completions(
2345 &mut control_completion_rx,
2346 &writer_tx,
2347 &mut routes,
2348 &mut root_channels,
2349 &mut session_identity,
2350 &mut push_buffer,
2351 &mut live_roots,
2352 &mut pending_binds,
2353 &mut installed_route_epochs,
2354 &executor,
2355 &shutdown,
2356 &dispatch_path_metrics,
2357 )
2358 .await
2359 {
2360 Ok(drained) => {
2361 if drained > 0 {
2362 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2363 }
2364 }
2365 Err(error) => break Err(error),
2366 }
2367
2368 if tokio::time::Instant::now() >= next_drain_at {
2369 push::emit_bg_event_wakes(
2370 &writer_tx,
2371 &dispatch_path_metrics,
2372 &bg_subs,
2373 &mut bg_wake_pending,
2374 );
2375 warn_slow_pending_binds(&mut pending_binds, &executor);
2376 if let Err(error) = expire_overdue_route_binds(
2377 &writer_tx,
2378 &executor,
2379 &mut pending_binds,
2380 &mut installed_route_epochs,
2381 &dispatch_path_metrics,
2382 )
2383 .await
2384 {
2385 break Err(error);
2386 }
2387
2388 let retried = push::drain_retry_buffers_for_bound_routes(
2389 &writer_tx,
2390 &dispatch_path_metrics,
2391 &routes,
2392 &mut retry_buffer,
2393 );
2394 if retried > 0 {
2395 log::debug!(
2396 "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2397 );
2398 }
2399
2400 next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2401 }
2402
2403 let overflow_batch = lossy_overflow.drain();
2409 if !overflow_batch.is_empty() {
2410 let (_, deferred) = push::drain_reliable_push_turn(
2411 &writer_tx,
2412 &dispatch_path_metrics,
2413 &routes,
2414 &root_channels,
2415 &session_identity,
2416 &mut retry_buffer,
2417 &mut push_buffer,
2418 &mut completed_tasks,
2419 &bg_sub_by_session,
2420 &mut bg_wake_pending,
2421 &mut bg_wake_epoch,
2422 &mut reliable_rx,
2423 None,
2424 );
2425 if deferred {
2426 tokio::task::yield_now().await;
2427 }
2428
2429 let mut batch = Vec::new();
2430 while let Ok(item) = lossy_rx.try_recv() {
2431 batch.push(item);
2432 }
2433 batch.extend(overflow_batch);
2434 push::process_lossy_push_envelope_batch(
2435 &writer_tx,
2436 &dispatch_path_metrics,
2437 &routes,
2438 &root_channels,
2439 &completed_tasks,
2440 batch,
2441 );
2442 }
2443
2444 tokio::select! {
2445 biased;
2446 Some(completion) = control_completion_rx.recv() => {
2447 if let Err(error) = process_route_bind_completion(
2448 &writer_tx,
2449 completion,
2450 &mut routes,
2451 &mut root_channels,
2452 &mut session_identity,
2453 &mut push_buffer,
2454 &mut live_roots,
2455 &mut pending_binds,
2456 &mut installed_route_epochs,
2457 &executor,
2458 &shutdown,
2459 &dispatch_path_metrics,
2460 )
2461 .await
2462 {
2463 break Err(error);
2464 }
2465 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2466 }
2467 _ = shutdown.notified() => {
2468 log::warn!("subc attach: fatal executor response requested teardown");
2469 break Ok(ModuleLoopExit::SkipSearchFlush);
2470 }
2471 maybe_frame = reader_rx.recv() => {
2472 let frame = match maybe_frame {
2473 None => {
2474 log::info!("subc attach: daemon closed connection");
2475 break Ok(ModuleLoopExit::Graceful);
2476 }
2477 Some(Err(error)) => break Err(error),
2478 Some(Ok(frame)) => frame,
2479 };
2480 let phase_trace = frame.phase_trace;
2481 let frame = frame.frame;
2482
2483 if !ingress_route_should_be_processed(
2484 &installed_route_epochs,
2485 &reclaimed_routes,
2486 &frame,
2487 ) {
2488 log::debug!(
2489 "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2490 frame.header.ty,
2491 frame.header.channel,
2492 frame.header.epoch
2493 );
2494 continue;
2495 }
2496
2497 match frame.header.ty {
2498 FrameType::Ping if frame.header.channel == 0 => {
2499 let pong = match Frame::build_with_version(
2500 frame.header.ver,
2501 FrameType::Pong,
2502 frame.header.flags,
2503 0,
2504 0,
2505 frame.header.corr,
2506 Vec::new(),
2507 ) {
2508 Ok(pong) => pong,
2509 Err(error) => break Err(SubcError::FrameBuild(error)),
2510 };
2511 if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2512 break Err(error);
2513 }
2514 }
2515 FrameType::Goodbye if frame.header.channel == 0 => {
2516 log::info!("subc attach: received channel-0 Goodbye");
2517 break Ok(ModuleLoopExit::Graceful);
2518 }
2519 FrameType::Goodbye => {
2520 let channel = route_key(frame.header.channel, frame.header.epoch);
2521 if let Err(error) = teardown_installed_route(
2522 &writer_tx,
2523 &dispatch_path_metrics,
2524 &executor,
2525 channel,
2526 "Goodbye",
2527 None,
2528 &mut installed_route_epochs,
2529 &mut routes,
2530 &mut root_channels,
2531 &mut bg_subs,
2532 &mut bg_sub_by_session,
2533 &mut bg_wake_pending,
2534 &mut pending_bash_asks,
2535 &mut live_roots,
2536 &mut route_bash_cancels,
2537 &mut pending_binds,
2538 &mut retry_buffer,
2539 &mut push_buffer,
2540 &shutdown,
2541 )
2542 .await
2543 {
2544 break Err(error);
2545 }
2546 }
2547 FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2548 if let Err(error) = handle_bash_elicitation_reply(
2549 &writer_tx,
2550 &frame,
2551 &mut pending_bash_asks,
2552 &routes,
2553 &mut live_roots,
2554 &executor,
2555 &shutdown,
2556 &bash_deferred_tx,
2557 &bash_poll_touch_tx,
2558 &dispatch_path_metrics,
2559 &mut route_bash_cancels,
2560 dispatch,
2561 )
2562 .await
2563 {
2564 break Err(error);
2565 }
2566 }
2567 FrameType::Request if frame.header.channel == 0 => {
2568 if let Err(error) = handle_control_request(
2569 &writer_tx,
2570 &frame,
2571 &shared_app,
2572 &executor,
2573 &mut live_roots,
2574 &mut pending_binds,
2575 &mut installed_route_epochs,
2576 &mut routes,
2577 &mut root_channels,
2578 &mut bg_subs,
2579 &mut bg_sub_by_session,
2580 &mut bg_wake_pending,
2581 &mut pending_bash_asks,
2582 &mut route_bash_cancels,
2583 &mut retry_buffer,
2584 &mut push_buffer,
2585 &shutdown,
2586 &control_completion_tx,
2587 &dispatch_path_metrics,
2588 &push_senders,
2589 dispatch,
2590 user_config_path.as_deref(),
2591 )
2592 .await
2593 {
2594 break Err(error);
2595 }
2596 }
2597 FrameType::Request => {
2598 if let Err(error) = handle_tool_call(
2599 &writer_tx,
2600 &frame,
2601 phase_trace,
2602 &routes,
2603 &pending_binds,
2604 &mut live_roots,
2605 &executor,
2606 &shutdown,
2607 &connection_cancel,
2608 &bash_deferred_tx,
2609 &bash_poll_touch_tx,
2610 &dispatch_path_metrics,
2611 &mut route_bash_cancels,
2612 &mut pending_bash_asks,
2613 &mut next_bash_ask_corr,
2614 &mut bg_subs,
2615 &mut bg_sub_by_session,
2616 &mut bg_wake_pending,
2617 &mut bg_wake_epoch,
2618 dispatch,
2619 allow_native_passthrough,
2620 )
2621 .await
2622 {
2623 break Err(error);
2624 }
2625 }
2626 FrameType::Cancel => {
2627 let channel = route_key(frame.header.channel, frame.header.epoch);
2628 if bg_subs.contains_key(&channel) {
2629 if let Err(error) = end_bg_subscription(
2630 &writer_tx,
2631 &dispatch_path_metrics,
2632 &mut bg_subs,
2633 &mut bg_sub_by_session,
2634 &mut bg_wake_pending,
2635 channel,
2636 routes.get(&channel),
2637 )
2638 .await
2639 {
2640 break Err(error);
2641 }
2642 }
2643 if let Err(error) = cancel_pending_bash_ask_for_tool_call(
2644 &writer_tx,
2645 &mut pending_bash_asks,
2646 channel,
2647 frame.header.corr,
2648 &routes,
2649 &mut live_roots,
2650 &mut route_bash_cancels,
2651 &shutdown,
2652 &dispatch_path_metrics,
2653 )
2654 .await
2655 {
2656 break Err(error);
2657 }
2658 }
2659 _ => {}
2664 }
2665 }
2666 Some((root_id, frame)) = reliable_rx.recv() => {
2667 let (_, deferred) = push::drain_reliable_push_turn(
2671 &writer_tx,
2672 &dispatch_path_metrics,
2673 &routes,
2674 &root_channels,
2675 &session_identity,
2676 &mut retry_buffer,
2677 &mut push_buffer,
2678 &mut completed_tasks,
2679 &bg_sub_by_session,
2680 &mut bg_wake_pending,
2681 &mut bg_wake_epoch,
2682 &mut reliable_rx,
2683 Some((root_id, frame)),
2684 );
2685 if deferred {
2686 tokio::task::yield_now().await;
2687 }
2688 }
2689 Some((order, root_id, frame)) = lossy_rx.recv() => {
2690 let (_, deferred) = push::drain_reliable_push_turn(
2694 &writer_tx,
2695 &dispatch_path_metrics,
2696 &routes,
2697 &root_channels,
2698 &session_identity,
2699 &mut retry_buffer,
2700 &mut push_buffer,
2701 &mut completed_tasks,
2702 &bg_sub_by_session,
2703 &mut bg_wake_pending,
2704 &mut bg_wake_epoch,
2705 &mut reliable_rx,
2706 None,
2707 );
2708 if deferred {
2709 tokio::task::yield_now().await;
2710 }
2711
2712 let mut batch = vec![(order, root_id, frame)];
2719 while let Ok(item) = lossy_rx.try_recv() {
2720 batch.push(item);
2721 }
2722 batch.extend(lossy_overflow.drain());
2723 push::process_lossy_push_envelope_batch(
2724 &writer_tx,
2725 &dispatch_path_metrics,
2726 &routes,
2727 &root_channels,
2728 &completed_tasks,
2729 batch,
2730 );
2731 }
2732 Some(done) = bash_deferred_rx.recv() => {
2733 decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
2734 if let Err(error) = bash::handle_bash_deferred_completion(
2735 &writer_tx,
2736 done,
2737 &routes,
2738 &mut live_roots,
2739 &mut route_bash_cancels,
2740 &shutdown,
2741 &dispatch_path_metrics,
2742 )
2743 .await
2744 {
2745 break Err(error);
2746 }
2747 }
2748 Some(root_id) = bash_poll_touch_rx.recv() => {
2749 decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
2750 if let Some(meta) = live_roots.get_mut(&root_id) {
2751 meta.note_activity();
2752 }
2753 }
2754 Some(completion) = maintenance_rx.recv() => {
2755 decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
2756 let root_id = completion.root_id.clone();
2757 let response = completion.response;
2758 let response_is_fatal = response_is_fatal_panic(&response);
2759 let bind_pending = pending_binds
2760 .values()
2761 .any(|pending| pending.bind_root_id == root_id);
2762 let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
2763 let defer_requeue = meta.unbound_quiesced || bind_pending;
2764 note_maintenance_completion(
2765 meta,
2766 completion.requeue_kind,
2767 response_is_fatal,
2768 defer_requeue,
2769 );
2770 should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
2771 } else {
2772 false
2773 };
2774 if requiesce {
2775 quiesce_unbound_root(&root_id, &mut live_roots, &executor);
2776 }
2777 push::clear_stale_bg_wakes_for_empty_sessions(
2778 &root_id,
2779 &completion.empty_bg_sessions,
2780 &bg_sub_by_session,
2781 &mut bg_wake_pending,
2782 &bg_wake_epoch,
2783 );
2784 if response_is_fatal {
2785 if let Some(meta) = live_roots.get_mut(&root_id) {
2786 meta.maintenance_poisoned = true;
2787 }
2788 log::warn!(
2789 "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
2790 );
2791 }
2792 }
2793 _ = tokio::time::sleep_until(next_drain_at) => {
2794 }
2797 _ = tokio::time::sleep_until(next_maintenance_at) => {
2798 crate::logging::maybe_sweep_logs();
2803 let reaped_lsp_children = shared_app
2804 .lsp_child_registry()
2805 .reap_children_with_gone_cwd();
2806 if reaped_lsp_children > 0 {
2807 log::warn!(
2808 "subc attach: reaped {reaped_lsp_children} LSP child process group(s) whose cwd no longer exists"
2809 );
2810 }
2811 let reap = reap_idle_roots(
2812 Instant::now(),
2813 &mut live_roots,
2814 &pending_binds,
2815 &root_channels,
2816 &executor,
2817 &dispatch_path_metrics,
2818 );
2819 for root_id in &reap.forgotten_deleted_roots {
2820 purge_deleted_root_residents(
2821 root_id,
2822 &mut routes,
2823 &mut root_channels,
2824 &mut installed_route_epochs,
2825 &mut route_bash_cancels,
2826 &mut retry_buffer,
2827 &mut reclaimed_routes,
2828 &mut session_identity,
2829 &mut push_buffer,
2830 &mut bg_subs,
2831 &mut bg_sub_by_session,
2832 &mut bg_wake_pending,
2833 &mut bg_wake_epoch,
2834 &mut pending_bash_asks,
2835 );
2836 }
2837 if reap.evicted > 0 {
2838 log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
2839 }
2840 submit_due_maintenance_jobs(
2841 &executor,
2842 &mut live_roots,
2843 &pending_binds,
2844 &bg_sub_by_session,
2845 &bg_wake_pending,
2846 &bg_wake_epoch,
2847 &maintenance_tx,
2848 &dispatch_path_metrics,
2849 );
2850 #[cfg(target_os = "macos")]
2857 {
2858 let now_std = std::time::Instant::now();
2859 if crate::memory::spawn_allocator_slack_relief_if_due(
2860 last_slack_relief,
2861 now_std,
2862 ) {
2863 last_slack_relief = Some(now_std);
2864 }
2865 }
2866 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2867 }
2868 }
2869 };
2870
2871 shared_app.set_open_route_count(0);
2872
2873 connection_cancel.cancel();
2874 quiesce_connection_roots(
2877 &mut live_roots,
2878 &mut pending_binds,
2879 &mut routes,
2880 &mut root_channels,
2881 &mut installed_route_epochs,
2882 &mut route_bash_cancels,
2883 &executor,
2884 );
2885
2886 let mut loop_result = loop_result;
2887 if !pending_bash_asks.is_empty() {
2888 let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2889 if let Err(error) = settle_all_pending_bash_asks(
2890 &writer_tx,
2891 &mut pending_bash_asks,
2892 &no_routes,
2893 &mut live_roots,
2894 &mut route_bash_cancels,
2895 &shutdown,
2896 &dispatch_path_metrics,
2897 )
2898 .await
2899 {
2900 loop_result = loop_result.and(Err(error));
2901 }
2902 }
2903
2904 reader_task.abort();
2907 drop(writer_tx);
2908 let writer_result = finish_writer_task(writer_task).await;
2909 loop_result.and_then(|exit| writer_result.map(|_| exit))
2910}
2911
2912fn spawn_writer_task<W>(
2913 mut write: W,
2914 mut rx: mpsc::Receiver<WriterFrame>,
2915 metrics: Arc<DispatchPathMetrics>,
2916) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
2917where
2918 W: AsyncWrite + Unpin + Send + 'static,
2919{
2920 tokio::spawn(async move {
2921 let mut write_buffer = Vec::new();
2922 while let Some(mut queued) = rx.recv().await {
2923 let measure = queued.tool_response_trace.is_some();
2924 let dequeued = measure.then(Instant::now);
2925 metrics.writer_active.store(true, Ordering::Relaxed);
2926 decrement_counted_channel(&metrics.writer_queued);
2927 let write_timing = write_frame_contiguous(
2928 &mut write,
2929 queued.frame(),
2930 queued.body(),
2931 &mut write_buffer,
2932 measure,
2933 )
2934 .await;
2935 metrics.writer_active.store(false, Ordering::Relaxed);
2936 let write_timing = write_timing?;
2937
2938 if let (Some(trace), Some(dequeued), Some(write_timing)) =
2939 (queued.tool_response_trace.take(), dequeued, write_timing)
2940 {
2941 if let Some(completed) = trace.finish(
2942 dequeued,
2943 write_timing.write_started,
2944 write_timing.write_finished,
2945 write_timing.frame_bytes,
2946 ) {
2947 log_ctx::with_session(Some(completed.session), || {
2948 crate::logging::note_tool_call_trace(
2949 &completed.name,
2950 &completed.root,
2951 completed.channel,
2952 completed.corr,
2953 completed.phases,
2954 );
2955 });
2956 }
2957 }
2958 }
2959 Ok(())
2960 })
2961}
2962
2963struct FrameWriteTiming {
2964 write_started: Instant,
2965 write_finished: Instant,
2966 frame_bytes: usize,
2967}
2968
2969async fn write_frame_contiguous<W>(
2973 writer: &mut W,
2974 frame: &Frame,
2975 body: &[u8],
2976 buffer: &mut Vec<u8>,
2977 measure: bool,
2978) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
2979where
2980 W: AsyncWrite + Unpin,
2981{
2982 if frame.header.len as usize != body.len() {
2983 return Err(subc_transport::FrameIoError::BodyLengthMismatch {
2984 header_len: frame.header.len,
2985 body_len: body.len(),
2986 });
2987 }
2988
2989 let header = frame.header.encode();
2990 buffer.clear();
2991 buffer.reserve(header.len() + body.len());
2992 buffer.extend_from_slice(&header);
2993 buffer.extend_from_slice(body);
2994 let write_started = measure.then(Instant::now);
2995 writer
2996 .write_all(buffer)
2997 .await
2998 .map_err(subc_transport::FrameIoError::Io)?;
2999 Ok(write_started.map(|write_started| FrameWriteTiming {
3000 write_started,
3001 write_finished: Instant::now(),
3002 frame_bytes: buffer.len(),
3003 }))
3004}
3005
3006fn spawn_reader_task<R>(
3007 mut read: R,
3008 tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3009) -> JoinHandle<()>
3010where
3011 R: AsyncRead + Unpin + Send + 'static,
3012{
3013 tokio::spawn(async move {
3014 loop {
3015 match read_frame(&mut read).await {
3016 Ok(Some(frame)) => {
3017 let decoded = DecodedFrame {
3018 frame,
3019 phase_trace: PhaseTrace::new(Instant::now()),
3020 };
3021 if tx.send(Ok(decoded)).await.is_err() {
3022 return;
3023 }
3024 }
3025 Ok(None) => {
3026 return;
3028 }
3029 Err(error) => {
3030 if let subc_transport::FrameIoError::Io(io_error) = &error {
3037 if matches!(
3038 io_error.kind(),
3039 std::io::ErrorKind::ConnectionReset
3040 | std::io::ErrorKind::ConnectionAborted
3041 ) {
3042 log::info!(
3043 "subc attach: connection reset by daemon; treating as close"
3044 );
3045 return;
3046 }
3047 }
3048 let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3049 return;
3050 }
3051 }
3052 }
3053 })
3054}
3055
3056async fn finish_writer_task(
3057 mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3058) -> Result<(), SubcError> {
3059 match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3060 Ok(Ok(Ok(()))) => Ok(()),
3061 Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3062 Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3063 Err(_) => {
3064 writer_task.abort();
3065 Ok(())
3066 }
3067 }
3068}
3069
3070fn register_actor_for_bind(
3071 shared_app: &Arc<App>,
3072 executor: &Arc<Executor>,
3073 push_senders: &PushSenders,
3074 bind_root_id: &ProjectRootId,
3075 route_channel: u16,
3076 root_was_live: bool,
3077) -> bool {
3078 if executor.actor_registered(bind_root_id) {
3079 log::debug!(
3080 "subc attach: reusing actor for route {} root {}",
3081 route_channel,
3082 bind_root_id.as_path().display()
3083 );
3084 return false;
3085 }
3086
3087 if root_was_live {
3088 log::warn!(
3089 "subc attach: recreating missing actor for live root {} on route {}",
3090 bind_root_id.as_path().display(),
3091 route_channel
3092 );
3093 }
3094
3095 let actor_ctx = Arc::new(AppContext::from_app(
3096 Arc::clone(shared_app),
3097 Config::default(),
3098 ));
3099 install_bash_compressor(&actor_ctx);
3100 actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
3101 push_senders.clone(),
3102 bind_root_id.clone(),
3103 )));
3104 let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
3105 drop(actor_ctx);
3106 if inserted {
3107 log::debug!(
3111 "subc attach: registered actor for route {} root {}",
3112 route_channel,
3113 bind_root_id.as_path().display()
3114 );
3115 } else {
3116 log::debug!(
3117 "subc attach: actor appeared while binding route {} root {}; reusing it",
3118 route_channel,
3119 bind_root_id.as_path().display()
3120 );
3121 }
3122 inserted
3123}
3124
3125fn rollback_pending_bind_actor(
3126 executor: &Arc<Executor>,
3127 live_roots: &HashMap<ProjectRootId, RootMeta>,
3128 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3129 root_id: &ProjectRootId,
3130 inserted_new_actor: bool,
3131) {
3132 if !inserted_new_actor || live_roots.contains_key(root_id) {
3133 return;
3134 }
3135
3136 if let Some((route, pending)) = pending_binds
3137 .iter_mut()
3138 .find(|(_, pending)| &pending.bind_root_id == root_id)
3139 {
3140 pending.inserted_new_actor = true;
3141 log::debug!(
3142 "subc attach: transferred rollback ownership for root {} to pending route {}",
3143 root_id.as_path().display(),
3144 route
3145 );
3146 return;
3147 }
3148
3149 executor.remove_actor(root_id);
3150}
3151
3152fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
3153 match response.data.get("code").and_then(|code| code.as_str()) {
3154 Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
3159 Some("cache_key_probe_failed") => "cache_key_probe_failed",
3163 Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
3167 _ => "config_divergence",
3168 }
3169}
3170
3171fn queue_post_bind_configure_and_completion_maintenance(
3172 root_id: &ProjectRootId,
3173 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3174) {
3175 let Some(meta) = live_roots.get_mut(root_id) else {
3176 return;
3177 };
3178 if meta.maintenance_poisoned || meta.maintenance_pending {
3179 return;
3180 }
3181
3182 meta.maintenance_pending = true;
3183 meta.maintenance_queued_kinds
3184 .push_back(MaintenanceDrainKind::ConfigureTail);
3185 meta.maintenance_queued_kinds
3186 .push_back(MaintenanceDrainKind::CompletionDrains);
3187}
3188
3189#[allow(clippy::too_many_arguments)]
3190async fn handle_route_bind_completion(
3191 tx: &WriterSender,
3192 completion: RouteBindCompletion,
3193 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3194 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3195 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
3196 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3197 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3198 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3199 installed_route_epochs: &mut HashMap<u16, u32>,
3200 executor: &Arc<Executor>,
3201 shutdown: &Arc<Notify>,
3202 metrics: &Arc<DispatchPathMetrics>,
3203) -> Result<(), SubcError> {
3204 let route_id = completion.route;
3205 let Some(pending) = pending_binds.remove(&route_id) else {
3206 log::warn!(
3207 "subc attach: dropping RouteBind completion for non-pending route {}",
3208 completion.route
3209 );
3210 rollback_pending_bind_actor(
3211 executor,
3212 live_roots,
3213 pending_binds,
3214 &completion.bind_root_id,
3215 completion.inserted_new_actor,
3216 );
3217 let has_pending_bind = pending_binds
3218 .values()
3219 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3220 if !root_channels
3221 .get(&completion.bind_root_id)
3222 .is_some_and(|channels| !channels.is_empty())
3223 && !has_pending_bind
3224 {
3225 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3226 }
3227 remove_installed_route(installed_route_epochs, route_id);
3228 return Ok(());
3229 };
3230
3231 if pending.bind_root_id != completion.bind_root_id {
3232 log::warn!(
3233 "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
3234 completion.route,
3235 pending.bind_root_id.as_path().display(),
3236 completion.bind_root_id.as_path().display()
3237 );
3238 }
3239
3240 let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
3241 if pending.cancelled {
3242 rollback_pending_bind_actor(
3243 executor,
3244 live_roots,
3245 pending_binds,
3246 &completion.bind_root_id,
3247 inserted_new_actor,
3248 );
3249 let has_pending_bind = pending_binds
3250 .values()
3251 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3252 if !root_channels
3253 .get(&completion.bind_root_id)
3254 .is_some_and(|channels| !channels.is_empty())
3255 && !has_pending_bind
3256 {
3257 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3258 }
3259 log::debug!(
3260 "subc attach: discarded completed RouteBind for cancelled route {} root {}",
3261 completion.route,
3262 completion.bind_root_id.as_path().display()
3263 );
3264 remove_installed_route(installed_route_epochs, route_id);
3265 return Ok(());
3266 }
3267
3268 let failure = if !completion.configure_response.success {
3269 Some((
3270 &completion.configure_response,
3271 "configure failed during route bind",
3272 ))
3273 } else {
3274 None
3275 };
3276
3277 if let Some((response, fallback)) = failure {
3278 rollback_pending_bind_actor(
3279 executor,
3280 live_roots,
3281 pending_binds,
3282 &completion.bind_root_id,
3283 inserted_new_actor,
3284 );
3285 let has_pending_bind = pending_binds
3286 .values()
3287 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3288 if !root_channels
3289 .get(&completion.bind_root_id)
3290 .is_some_and(|channels| !channels.is_empty())
3291 && !has_pending_bind
3292 {
3293 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3294 }
3295 let message = response_message(response, fallback);
3296 let fatal = response_is_fatal_panic(response);
3297 let error_code = route_bind_error_code_for_configure_response(response);
3298 send_route_bind_error_parts(
3299 tx,
3300 completion.ver,
3301 completion.corr,
3302 completion.flags,
3303 error_code,
3304 &message,
3305 metrics,
3306 )
3307 .await?;
3308 remove_installed_route(installed_route_epochs, route_id);
3309 if fatal {
3310 signal_fatal_teardown(
3311 tx,
3312 Some(completion.route),
3313 completion.ver,
3314 completion.corr,
3315 shutdown,
3316 metrics,
3317 )
3318 .await;
3319 }
3320 return Ok(());
3321 }
3322
3323 remember_session_identity(session_identity, &completion.identity);
3324 let replay_key = push::ReplayKey::from_identity(&completion.identity);
3325 let bind_trust = completion.identity.trust;
3326 insert_route_channel(routes, root_channels, route_id, completion.identity);
3327 let restore_watcher = live_roots
3328 .get(&completion.bind_root_id)
3329 .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3330 live_roots
3331 .entry(completion.bind_root_id.clone())
3332 .and_modify(|meta| {
3333 meta.reactivate_bound();
3334 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3335 meta.maintenance_poisoned = false;
3336 })
3337 .or_insert_with(|| RootMeta::new(Instant::now()));
3338 if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3339 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3340 meta.maintenance_poisoned = false;
3341 }
3342 if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3343 ctx.mark_subc_bound();
3344 if restore_watcher {
3345 crate::commands::configure::ensure_project_watcher(&ctx);
3346 }
3347 }
3348
3349 let ack =
3350 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3351 let response = Frame::build_with_version(
3352 completion.ver,
3353 FrameType::Response,
3354 control_flags(),
3355 0,
3356 0,
3357 completion.corr,
3358 ack,
3359 )
3360 .map_err(SubcError::FrameBuild)?;
3361 send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3362 queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3363 let replayed = push::replay_buffered_push_frames(
3364 tx,
3365 metrics,
3366 route_id,
3367 push_buffer,
3368 &replay_key,
3369 bind_trust,
3370 );
3371 if replayed > 0 {
3372 log::debug!(
3373 "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3374 replayed,
3375 completion.route,
3376 replay_key.root.as_path().display(),
3377 replay_key.harness,
3378 replay_key.session
3379 );
3380 }
3381 log::info!(
3382 "subc attach: route {} bound to root {}",
3383 completion.route,
3384 completion.bind_root_id.as_path().display()
3385 );
3386 Ok(())
3387}
3388
3389async fn expire_overdue_route_binds(
3390 tx: &WriterSender,
3391 executor: &Arc<Executor>,
3392 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3393 installed_route_epochs: &mut HashMap<u16, u32>,
3394 metrics: &DispatchPathMetrics,
3395) -> Result<(), SubcError> {
3396 let now = Instant::now();
3397 let expired: Vec<_> = pending_binds
3398 .iter()
3399 .filter_map(|(route, pending)| {
3400 let age = now.saturating_duration_since(pending.started_at);
3401 (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3402 (
3403 *route,
3404 pending.corr,
3405 pending.ver,
3406 pending.flags,
3407 pending.bind_root_id.clone(),
3408 pending.configure_request_id.clone(),
3409 age,
3410 )
3411 })
3412 })
3413 .collect();
3414
3415 for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3416 if let Some(pending) = pending_binds.get_mut(&route) {
3417 pending.cancelled = true;
3418 pending.deadline_reported = true;
3419 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3420 log::debug!(
3421 "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3422 );
3423 }
3424 remove_installed_route(installed_route_epochs, route);
3425 let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3426 let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3427 send_route_bind_error_parts(
3428 tx,
3429 ver,
3430 corr,
3431 flags,
3432 "actor_not_ready",
3433 &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3434 metrics,
3435 )
3436 .await?;
3437 log::warn!(
3438 "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3439 route,
3440 root_id.as_path().display(),
3441 deadline_ms,
3442 configure_request_id
3443 );
3444 }
3445
3446 Ok(())
3447}
3448
3449#[allow(clippy::too_many_arguments)]
3454async fn handle_control_request(
3455 tx: &WriterSender,
3456 frame: &Frame,
3457 shared_app: &Arc<App>,
3458 executor: &Arc<Executor>,
3459 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3460 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3461 installed_route_epochs: &mut HashMap<u16, u32>,
3462 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3463 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3464 bg_subs: &mut HashMap<RouteChannel, BgSub>,
3465 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3466 bg_wake_pending: &mut HashSet<RouteChannel>,
3467 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3468 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3469 retry_buffer: &mut RetryBuffer,
3470 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3471 shutdown: &Arc<Notify>,
3472 control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3473 metrics: &Arc<DispatchPathMetrics>,
3474 push_senders: &PushSenders,
3475 dispatch: DispatchFn,
3476 user_config_path: Option<&Path>,
3477) -> Result<(), SubcError> {
3478 let request =
3479 serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3480 match request {
3481 ModuleControlRequest::RouteBind {
3482 route_channel,
3483 epoch,
3484 target: _,
3485 identity,
3486 principal,
3487 consumer_capabilities,
3488 } => {
3489 let route_id = route_key(route_channel, epoch);
3490 if epoch == 0 {
3491 return send_route_bind_error(
3492 tx,
3493 frame,
3494 "config_divergence",
3495 "route bind uses an invalid channel generation",
3496 metrics,
3497 )
3498 .await;
3499 }
3500 let mut bind_root_id = None;
3501 if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3502 if installed_epoch >= epoch {
3503 return send_route_bind_error(
3504 tx,
3505 frame,
3506 "config_divergence",
3507 "route bind generation is not newer than the installed generation",
3508 metrics,
3509 )
3510 .await;
3511 }
3512
3513 let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3514 Ok(root_id) => root_id,
3515 Err(error) => {
3516 return send_route_bind_error(
3517 tx,
3518 frame,
3519 "config_divergence",
3520 &format!("invalid route project root: {error}"),
3521 metrics,
3522 )
3523 .await;
3524 }
3525 };
3526 teardown_installed_route(
3527 tx,
3528 metrics,
3529 executor,
3530 route_key(route_channel, installed_epoch),
3531 "higher-epoch RouteBind",
3532 Some(&replacement_root),
3533 installed_route_epochs,
3534 routes,
3535 root_channels,
3536 bg_subs,
3537 bg_sub_by_session,
3538 bg_wake_pending,
3539 pending_bash_asks,
3540 live_roots,
3541 route_bash_cancels,
3542 pending_binds,
3543 retry_buffer,
3544 push_buffer,
3545 shutdown,
3546 )
3547 .await?;
3548 bind_root_id = Some(replacement_root);
3549 }
3550 if pending_binds.contains_key(&route_id) {
3551 return send_route_bind_error(
3552 tx,
3553 frame,
3554 "config_divergence",
3555 "route bind is already pending for channel",
3556 metrics,
3557 )
3558 .await;
3559 }
3560 let bind_root_id = match bind_root_id {
3561 Some(root_id) => root_id,
3562 None => match ProjectRootId::from_path(&identity.project_root) {
3563 Ok(root_id) => root_id,
3564 Err(error) => {
3565 return send_route_bind_error(
3566 tx,
3567 frame,
3568 "config_divergence",
3569 &format!("invalid route project root: {error}"),
3570 metrics,
3571 )
3572 .await;
3573 }
3574 },
3575 };
3576
3577 let request_id = format!("subc-bind-{route_channel}");
3580 let bind_project_root = identity.project_root.clone();
3581 let bind_harness = identity.harness.clone();
3582 let bind_session = identity.session.clone();
3583 let bind_trust = trust_for_bind(&bind_harness, &principal);
3584 let bind_principal_id = principal_id(&principal);
3585 let consumer_elicitation_capable = consumer_capabilities
3590 .as_ref()
3591 .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
3592 log::info!(
3593 "subc attach: route {} harness={} principal={} trust={} elicitation={}",
3594 route_channel,
3595 bind_harness,
3596 principal_label(&principal),
3597 bind_trust.label(),
3598 consumer_elicitation_capable
3599 );
3600
3601 let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
3613 user_config_path,
3614 Path::new(&bind_project_root),
3615 );
3616 let config_tiers: Vec<Value> = local_tiers
3617 .iter()
3618 .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
3619 .collect();
3620 let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
3621 let configure_json = json!({
3622 "id": request_id,
3623 "command": "configure",
3624 "project_root": bind_project_root,
3625 "harness": bind_harness,
3626 "session_id": bind_session.clone(),
3627 "config": config_tiers,
3628 });
3629 let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
3630 Ok(req) => req,
3631 Err(error) => {
3632 return send_route_bind_error(
3633 tx,
3634 frame,
3635 "config_divergence",
3636 &format!("failed to build configure request: {error}"),
3637 metrics,
3638 )
3639 .await;
3640 }
3641 };
3642
3643 let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
3644 root: bind_root_id.clone(),
3645 project_root: PathBuf::from(&bind_project_root),
3646 harness: bind_harness.clone(),
3647 session: bind_session.clone(),
3648 trust: bind_trust,
3649 spawn_principal: AuthenticatedPrincipal::RouteBind {
3650 trust: bind_trust.sandbox_trust(),
3651 route_channel,
3652 route_epoch: epoch,
3653 project_root: PathBuf::from(&bind_project_root),
3654 harness: bind_harness.clone(),
3655 session_id: bind_session.clone(),
3656 principal_id: bind_principal_id,
3657 },
3658 consumer_elicitation_capable,
3659 }));
3660 let configure_session = route_identity.session.clone();
3661 let root_was_live = live_roots.contains_key(&bind_root_id);
3662 let inserted_new_actor = register_actor_for_bind(
3663 shared_app,
3664 executor,
3665 push_senders,
3666 &bind_root_id,
3667 route_channel,
3668 root_was_live,
3669 );
3670
3671 let configure_request_id = configure_req.id.clone();
3672 installed_route_epochs.insert(route_channel, epoch);
3673 if let Some(meta) = live_roots.get_mut(&bind_root_id) {
3674 meta.maintenance_queued_kinds.clear();
3675 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
3676 }
3677 let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
3678 bind_root_id.clone(),
3679 Lane::Mutating,
3680 configure_request_id.clone(),
3681 Box::new(move |ctx| {
3682 log_ctx::with_session(Some(configure_session.clone()), || {
3683 dispatch(configure_req, ctx)
3684 })
3685 }),
3686 );
3687 pending_binds.insert(
3688 route_id,
3689 PendingBind {
3690 bind_root_id: bind_root_id.clone(),
3691 inserted_new_actor,
3692 cancelled: false,
3693 configure_request_id: configure_request_id.clone(),
3694 started_at: Instant::now(),
3695 warned_half_deadline: false,
3696 deadline_reported: false,
3697 corr: frame.header.corr,
3698 ver: frame.header.ver,
3699 flags: frame.header.flags,
3700 cancellation: configure_cancellation,
3701 },
3702 );
3703
3704 let completion_tx = control_completion_tx.clone();
3705 let completion_identity = route_identity;
3706 let completion_root = bind_root_id.clone();
3707 let completion_route_channel = route_channel;
3708 let completion_ver = frame.header.ver;
3709 let completion_corr = frame.header.corr;
3710 let completion_flags = frame.header.flags;
3711 let completion_metrics = Arc::clone(metrics);
3712 tokio::spawn(async move {
3713 let _response_task = ResponseTaskGuard::new(&completion_metrics);
3714 let configure_response =
3715 await_executor_response(configure_rx, configure_request_id.clone()).await;
3716 let completion = RouteBindCompletion {
3721 route: route_key(completion_route_channel, epoch),
3722 identity: completion_identity,
3723 bind_root_id: completion_root,
3724 inserted_new_actor,
3725 configure_response,
3726 diagnostics_on_edit,
3727 ver: completion_ver,
3728 corr: completion_corr,
3729 flags: completion_flags,
3730 };
3731 if send_counted_channel(
3732 &completion_tx,
3733 &completion_metrics.control_completion_queued,
3734 completion,
3735 )
3736 .await
3737 .is_err()
3738 {
3739 log::debug!(
3740 "subc attach: dropped RouteBind completion for route {} after loop exit",
3741 completion_route_channel
3742 );
3743 }
3744 });
3745
3746 Ok(())
3747 }
3748 ModuleControlRequest::HealthCheck {} => {
3749 let report = build_health_report(executor, pending_binds, metrics, shared_app);
3750 let body = serde_json::to_vec(&ModuleControlResponse::from(report))
3751 .map_err(SubcError::Json)?;
3752 let response = Frame::build_with_version(
3753 frame.header.ver,
3754 FrameType::Response,
3755 frame.header.flags,
3756 0,
3757 0,
3758 frame.header.corr,
3759 body,
3760 )
3761 .map_err(SubcError::FrameBuild)?;
3762 send_frame(tx, metrics, response).await
3763 }
3764 }
3765}
3766
3767fn install_bash_compressor(ctx: &AppContext) {
3768 let filter_registry_handle = ctx.shared_filter_registry();
3770 let compress_flag = ctx.bash_compress_flag();
3771 ctx.bash_background().set_compressor_with_exit_code(
3772 move |command: &str, output: String, exit_code: Option<i32>| {
3773 if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
3774 return crate::compress::CompressionResult::new(output);
3775 }
3776 let registry_guard = match filter_registry_handle.read() {
3777 Ok(g) => g,
3778 Err(poisoned) => poisoned.into_inner(),
3779 };
3780 crate::compress::compress_with_registry_exit_code(
3781 command,
3782 &output,
3783 exit_code,
3784 ®istry_guard,
3785 )
3786 },
3787 );
3788}
3789
3790fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
3791 let mut diagnostics_on_edit = false;
3792 for tier in tiers {
3793 if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
3794 diagnostics_on_edit = value;
3795 }
3796 }
3797 diagnostics_on_edit
3798}
3799
3800fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
3801 let stripped = strip_jsonc(doc);
3802 let value = serde_json::from_str::<Value>(&stripped).ok()?;
3803 value
3804 .get("lsp")
3805 .and_then(Value::as_object)?
3806 .get("diagnostics_on_edit")
3807 .and_then(Value::as_bool)
3808}
3809
3810async fn send_route_bind_error(
3811 tx: &WriterSender,
3812 frame: &Frame,
3813 code: &str,
3814 message: &str,
3815 metrics: &DispatchPathMetrics,
3816) -> Result<(), SubcError> {
3817 send_route_bind_error_parts(
3818 tx,
3819 frame.header.ver,
3820 frame.header.corr,
3821 frame.header.flags,
3822 code,
3823 message,
3824 metrics,
3825 )
3826 .await
3827}
3828
3829async fn send_route_bind_error_parts(
3830 tx: &WriterSender,
3831 ver: u8,
3832 corr: u64,
3833 flags: Flags,
3834 code: &str,
3835 message: &str,
3836 metrics: &DispatchPathMetrics,
3837) -> Result<(), SubcError> {
3838 let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
3839 send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
3840 log_route_bind_rejection(code, message);
3841 Ok(())
3842}
3843
3844fn log_route_bind_rejection(code: &str, message: &str) {
3852 const WINDOW: Duration = Duration::from_secs(60);
3853 static SUPPRESSED: OnceLock<StdMutex<HashMap<String, (Instant, u64)>>> = OnceLock::new();
3854 let map = SUPPRESSED.get_or_init(|| StdMutex::new(HashMap::new()));
3855 let mut map = match map.try_lock() {
3856 Ok(map) => map,
3857 Err(_) => {
3859 log::warn!("subc attach: route bind rejected ({code}): {message}");
3860 return;
3861 }
3862 };
3863 let now = Instant::now();
3864 if map.len() > 512 {
3868 map.retain(|_, (start, _)| now.duration_since(*start) < WINDOW);
3869 }
3870 match map.get_mut(message) {
3871 Some((window_start, suppressed)) if now.duration_since(*window_start) < WINDOW => {
3872 *suppressed += 1;
3873 }
3874 Some((window_start, suppressed)) => {
3875 if *suppressed > 0 {
3876 log::warn!(
3877 "subc attach: route bind rejected ({code}): {message} (repeated {}x in last 60s)",
3878 *suppressed
3879 );
3880 } else {
3881 log::warn!("subc attach: route bind rejected ({code}): {message}");
3882 }
3883 *window_start = now;
3884 *suppressed = 0;
3885 }
3886 None => {
3887 log::warn!("subc attach: route bind rejected ({code}): {message}");
3888 map.insert(message.to_string(), (now, 0));
3889 }
3890 }
3891}
3892
3893async fn handle_tool_call(
3898 tx: &WriterSender,
3899 frame: &Frame,
3900 mut phase_trace: PhaseTrace,
3901 routes: &HashMap<RouteChannel, RouteIdentity>,
3902 pending_binds: &HashMap<RouteChannel, PendingBind>,
3903 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3904 executor: &Arc<Executor>,
3905 shutdown: &Arc<Notify>,
3906 connection_cancel: &PersistentCancelSignal,
3907 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
3908 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
3909 metrics: &Arc<DispatchPathMetrics>,
3910 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3911 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3912 next_bash_ask_corr: &mut u64,
3913 bg_subs: &mut HashMap<RouteChannel, BgSub>,
3914 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3915 bg_wake_pending: &mut HashSet<RouteChannel>,
3916 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
3917 dispatch: DispatchFn,
3918 allow_native_passthrough: bool,
3919) -> Result<(), SubcError> {
3920 let route_id = route_key(frame.header.channel, frame.header.epoch);
3921 if pending_binds.contains_key(&route_id) {
3922 let error = build_error_frame(
3923 frame.header.ver,
3924 frame.header.channel,
3925 frame.header.epoch,
3926 frame.header.corr,
3927 frame.header.flags,
3928 "route_not_bound",
3929 "route is not bound before tool call",
3930 )?;
3931 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3932 }
3933
3934 let Some(identity) = routes.get(&route_id).cloned() else {
3935 let error = build_error_frame(
3936 frame.header.ver,
3937 frame.header.channel,
3938 frame.header.epoch,
3939 frame.header.corr,
3940 frame.header.flags,
3941 "route_not_bound",
3942 "route is not bound before tool call",
3943 )?;
3944 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3945 };
3946 let restore_watcher = live_roots
3947 .get(&identity.root)
3948 .is_some_and(|meta| meta.idle_artifacts_evicted);
3949 if let Some(meta) = live_roots.get_mut(&identity.root) {
3950 meta.reactivate_bound();
3951 }
3952 if restore_watcher {
3953 if let Some(ctx) = executor.actor_context(&identity.root) {
3954 crate::commands::configure::ensure_project_watcher(&ctx);
3955 }
3956 }
3957
3958 let route_request =
3959 serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
3960 if matches!(
3961 route_request,
3962 RouteRequest::BgEvents(BgEventsRequest {
3963 op: BgEventsOp::BgEvents
3964 })
3965 ) {
3966 if let Some(old_sub) = bg_subs.get(&route_id).copied() {
3967 push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
3968 }
3969 if !identity.trust.allows_bash_observation() {
3970 bg_subs.remove(&route_id);
3971 bg_wake_pending.remove(&route_id);
3972 remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
3973 let denied_sub = BgSub {
3974 corr: frame.header.corr,
3975 ver: frame.header.ver,
3976 flags: frame.header.flags,
3977 };
3978 push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
3979 return Ok(());
3980 }
3981 bg_subs.insert(
3982 route_id,
3983 BgSub {
3984 corr: frame.header.corr,
3985 ver: frame.header.ver,
3986 flags: frame.header.flags,
3987 },
3988 );
3989 bg_sub_by_session.insert((identity.root.clone(), identity.session.clone()), route_id);
3990 push::arm_bg_wake(
3991 identity.root.clone(),
3992 identity.session.clone(),
3993 route_id,
3994 bg_wake_pending,
3995 bg_wake_epoch,
3996 );
3997 return Ok(());
3998 }
3999
4000 let RouteRequest::ToolCall(call) = route_request else {
4001 unreachable!("background event subscription returned above")
4002 };
4003 let bare_name = call.name;
4004 let arguments = strip_agent_preview_arg_owned(call.arguments);
4005 let format_context = crate::subc_format::FormatContext::from_tool_call(
4006 &bare_name,
4007 &arguments,
4008 identity.project_root.as_path(),
4009 );
4010
4011 let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
4012 let bind_trust = identity.trust;
4013 let diagnostics_on_edit = live_roots
4014 .get(&identity.root)
4015 .map(|meta| meta.diagnostics_on_edit)
4016 .unwrap_or(false);
4017
4018 let requests_host = bare_name == "bash"
4019 && arguments
4020 .get("sandbox")
4021 .or_else(|| {
4022 arguments
4023 .get("params")
4024 .and_then(|params| params.get("sandbox"))
4025 })
4026 .and_then(Value::as_str)
4027 == Some("host");
4028 if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
4029 let response = Response::error(
4030 request_id.clone(),
4031 "sandbox_escalation_denied",
4032 "sandbox host escalation is unavailable to untrusted principals",
4033 );
4034 let text = crate::subc_format::format_response_with_context(
4035 &bare_name,
4036 &response,
4037 &format_context,
4038 );
4039 let result = ToolCallResult { text, response };
4040 let response_frame = build_tool_response_frame(
4041 frame.header.ver,
4042 route_id,
4043 frame.header.corr,
4044 frame.header.flags,
4045 &result,
4046 bind_trust,
4047 )?;
4048 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4049 }
4050
4051 if matches!(bind_trust, BindTrust::Untrusted)
4052 && is_bash_family_tool(&bare_name)
4053 && (bare_name != "bash" || !identity.consumer_elicitation_capable)
4054 {
4055 let response = bash::bash_denied_untrusted_response(request_id.clone());
4056 let text = crate::subc_format::format_response_with_context(
4057 &bare_name,
4058 &response,
4059 &format_context,
4060 );
4061 let result = ToolCallResult { text, response };
4062 let response_frame = build_tool_response_frame(
4063 frame.header.ver,
4064 route_id,
4065 frame.header.corr,
4066 frame.header.flags,
4067 &result,
4068 bind_trust,
4069 )?;
4070 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4071 }
4072
4073 if !is_subc_agent_core_tool(&bare_name)
4081 && !is_subc_native_plumbing_tool(&bare_name)
4082 && !allow_native_passthrough
4083 {
4084 log::warn!(
4085 "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
4086 bare_name,
4087 frame.header.channel
4088 );
4089 let response = Response::error(
4090 request_id.clone(),
4091 "unknown_tool",
4092 format!("tool {:?} is not in the AFT tool manifest", bare_name),
4093 );
4094 let text = crate::subc_format::format_response_with_context(
4095 &bare_name,
4096 &response,
4097 &format_context,
4098 );
4099 let result = ToolCallResult { text, response };
4100 let response_frame = build_tool_response_frame(
4101 frame.header.ver,
4102 route_id,
4103 frame.header.corr,
4104 frame.header.flags,
4105 &result,
4106 bind_trust,
4107 )?;
4108 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4109 }
4110
4111 if bare_name == "bash" {
4112 if matches!(bind_trust, BindTrust::Untrusted) {
4113 let plan = match bash::prepare_bash_elicitation_plan(
4114 &arguments,
4115 identity.project_root.as_path(),
4116 ) {
4117 Ok(plan) => plan,
4118 Err(error) => {
4119 let response = Response::error(request_id.clone(), error.code, error.message);
4120 let text = crate::subc_format::format_response_with_context(
4121 &bare_name,
4122 &response,
4123 &format_context,
4124 );
4125 let result = ToolCallResult { text, response };
4126 let response_frame = build_tool_response_frame(
4127 frame.header.ver,
4128 route_id,
4129 frame.header.corr,
4130 frame.header.flags,
4131 &result,
4132 bind_trust,
4133 )?;
4134 return send_reliable_writer_frame(
4135 tx,
4136 metrics,
4137 response_frame,
4138 "tool response",
4139 )
4140 .await;
4141 }
4142 };
4143
4144 let reverse_corr =
4145 allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
4146 let ask_frame = build_bash_elicitation_request_frame(
4147 frame.header.ver,
4148 route_id,
4149 reverse_corr,
4150 frame.header.flags,
4151 &plan.command,
4152 &plan.asks,
4153 )?;
4154
4155 let meta = live_roots
4156 .entry(identity.root.clone())
4157 .or_insert_with(|| RootMeta::new(Instant::now()));
4158 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4159 meta.reactivate_bound();
4160
4161 let route_cancel =
4162 route_bash_cancels
4163 .entry(route_id)
4164 .or_insert_with(|| bash::RouteBashCancel {
4165 token: PersistentCancelSignal::new(),
4166 active_waits: 0,
4167 });
4168 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4169 let cancel = bash::BashWaitCancel {
4170 connection: connection_cancel.clone(),
4171 route: route_cancel.token.clone(),
4172 };
4173 pending_bash_asks.insert(
4174 ReverseCorrKey {
4175 route: route_id,
4176 corr: reverse_corr,
4177 },
4178 PendingBashAsk {
4179 route: route_id,
4180 tool_corr: frame.header.corr,
4181 tool_flags: frame.header.flags,
4182 tool_ver: frame.header.ver,
4183 root: identity.root.clone(),
4184 project_root: identity.project_root.clone(),
4185 session_id: identity.session.clone(),
4186 spawn_principal: identity.spawn_principal.clone(),
4187 request_id,
4188 arguments,
4189 format_context,
4190 cancel,
4191 grants: plan.grants,
4192 expires_at: Instant::now() + bash_elicitation_timeout(),
4193 },
4194 );
4195 return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
4196 .await;
4197 }
4198
4199 let meta = live_roots
4200 .entry(identity.root.clone())
4201 .or_insert_with(|| RootMeta::new(Instant::now()));
4202 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4203 meta.reactivate_bound();
4204
4205 let route_cancel =
4206 route_bash_cancels
4207 .entry(route_id)
4208 .or_insert_with(|| bash::RouteBashCancel {
4209 token: PersistentCancelSignal::new(),
4210 active_waits: 0,
4211 });
4212 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4213 let cancel = bash::BashWaitCancel {
4214 connection: connection_cancel.clone(),
4215 route: route_cancel.token.clone(),
4216 };
4217
4218 bash::submit_deferred_bash(
4219 executor,
4220 bash_deferred_tx,
4221 bash_poll_touch_tx,
4222 metrics,
4223 dispatch,
4224 identity.root.clone(),
4225 identity.project_root.clone(),
4226 identity.session.clone(),
4227 request_id,
4228 route_id,
4229 frame.header.corr,
4230 frame.header.flags,
4231 frame.header.ver,
4232 arguments,
4233 format_context,
4234 cancel,
4235 bind_trust,
4236 identity.spawn_principal.clone(),
4237 None,
4238 );
4239 return Ok(());
4240 }
4241
4242 let lane = command_lane(&bare_name);
4243 let tool_call_context = ToolCallContext {
4244 project_root: identity.project_root.clone(),
4245 session_id: Some(identity.session.clone()),
4246 request_id: request_id.clone(),
4247 diagnostics_on_edit,
4248 preview: call.preview,
4249 };
4250 let bare_name_for_frame = bare_name.clone();
4251 let identity_for_run = identity.clone();
4252 let completion_session = identity.session.clone();
4253 let completion_root = identity.project_root.clone();
4254 let request_id_for_force = request_id.clone();
4255 let format_context_for_frame = format_context.clone();
4256 let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
4257 phase_trace.mark_executor_submitted();
4258 let rx = executor.submit_async(
4259 identity.root.clone(),
4260 lane,
4261 request_id.clone(),
4262 Box::new(move |ctx| {
4263 phase_trace.mark_job_admitted();
4264 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4265 let run = || {
4266 let finalizer = |response: &mut Response| {
4267 crate::response_finalize::finalize_response_with_bg_completions(
4268 response,
4269 ctx,
4270 &identity_for_run.session,
4271 &bare_name,
4272 bind_trust.allows_bash_observation(),
4273 );
4274 };
4275 match run_tool_call(
4276 &bare_name,
4277 arguments,
4278 &format_context,
4279 &tool_call_context,
4280 ctx,
4281 &dispatch,
4282 Some(&finalizer),
4283 Some(&mut phase_trace),
4284 ) {
4285 ToolCallOutcome::Unary(result) => {
4286 let response = result.response;
4287 let _ = tool_call_tx.send(ToolCallCompletion {
4288 text: result.text,
4289 phase_trace,
4290 });
4291 response
4292 }
4293 }
4294 };
4295 if matches!(bind_trust, BindTrust::Untrusted) {
4296 ctx.with_force_restrict(&request_id_for_force, run)
4297 } else {
4298 run()
4299 }
4300 })
4301 }),
4302 );
4303 let completion_tx = tx.clone();
4304 let completion_shutdown = Arc::clone(shutdown);
4305 let route = route_id;
4306 let corr = frame.header.corr;
4307 let flags = frame.header.flags;
4308 let ver = frame.header.ver;
4309 let completion_metrics = Arc::clone(metrics);
4310 tokio::spawn(async move {
4311 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4312 let response = await_executor_response(rx, request_id.clone()).await;
4313 let (text, phase_trace) = match tool_call_rx.await {
4314 Ok(completion) => (completion.text, Some(completion.phase_trace)),
4315 Err(_) => (
4316 crate::subc_format::format_response_with_context(
4317 &bare_name_for_frame,
4318 &response,
4319 &format_context_for_frame,
4320 ),
4321 None,
4322 ),
4323 };
4324 let result = ToolCallResult { text, response };
4325 let fatal = response_is_fatal_panic(&result.response);
4326 match build_tool_response_frame(ver, route, corr, flags, &result, bind_trust) {
4327 Ok(response_frame) => {
4328 let send_result = if let Some(phase_trace) = phase_trace {
4329 let trace = ToolResponseWriteTrace::new(
4330 phase_trace,
4331 bare_name_for_frame,
4332 completion_root,
4333 completion_session,
4334 route.channel,
4335 corr,
4336 );
4337 send_traced_tool_response_frame(
4338 &completion_tx,
4339 &completion_metrics,
4340 response_frame,
4341 trace,
4342 )
4343 .await
4344 } else {
4345 send_reliable_writer_frame(
4346 &completion_tx,
4347 &completion_metrics,
4348 response_frame,
4349 "tool response",
4350 )
4351 .await
4352 };
4353 if let Err(error) = send_result {
4354 log::warn!("subc attach: failed to queue tool response frame: {error}");
4355 }
4356 }
4357 Err(error) => {
4358 log::error!("subc attach: failed to build tool response frame: {error}");
4359 }
4360 }
4361 if fatal {
4362 signal_fatal_teardown(
4363 &completion_tx,
4364 Some(route),
4365 ver,
4366 corr,
4367 &completion_shutdown,
4368 &completion_metrics,
4369 )
4370 .await;
4371 }
4372 });
4373 Ok(())
4374}
4375
4376fn submit_maintenance_job(
4377 executor: &Arc<Executor>,
4378 root_id: ProjectRootId,
4379 kind: MaintenanceDrainKind,
4380 bg_sessions_to_check: Vec<(String, u64)>,
4381 completion_tx: &mpsc::Sender<MaintenanceCompletion>,
4382 metrics: &Arc<DispatchPathMetrics>,
4383) {
4384 let request_id = format!(
4385 "subc-maintenance-drain-{}-{}",
4386 kind.label(),
4387 root_id.as_path().to_string_lossy()
4388 );
4389 let response_id = request_id.clone();
4390 let completion_root_id = root_id.clone();
4391 let maintenance_generation = executor
4392 .actor_context(&root_id)
4393 .map(|ctx| ctx.configure_generation())
4394 .unwrap_or(0);
4395 let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
4396 let lane = match kind {
4401 MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
4402 MaintenanceDrainKind::Watcher
4403 | MaintenanceDrainKind::Lsp
4404 | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
4405 };
4406 let rx = executor.submit_maintenance_async(
4407 root_id,
4408 lane,
4409 request_id.clone(),
4410 Box::new(move |ctx| {
4411 let outcome = match kind {
4412 MaintenanceDrainKind::Watcher => {
4413 let drained = runtime_drain::drain_watcher_events_bounded(
4414 ctx,
4415 runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
4416 );
4417 MaintenanceJobOutcome {
4418 empty_bg_sessions: Vec::new(),
4419 requeue_kind: drained.has_more.then_some(kind),
4420 }
4421 }
4422 MaintenanceDrainKind::Lsp => {
4423 let drained = runtime_drain::drain_lsp_events_bounded(
4424 ctx,
4425 runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
4426 );
4427 MaintenanceJobOutcome {
4428 empty_bg_sessions: Vec::new(),
4429 requeue_kind: drained.has_more.then_some(kind),
4430 }
4431 }
4432 MaintenanceDrainKind::ConfigureTail => {
4433 runtime_drain::drain_deferred_configure_maintenance(ctx);
4434 runtime_drain::drain_configure_warning_events(ctx);
4435 MaintenanceJobOutcome::default()
4436 }
4437 MaintenanceDrainKind::CompletionDrains => {
4438 runtime_drain::drain_search_index_events(ctx);
4439 runtime_drain::drain_callgraph_store_events(ctx);
4440 runtime_drain::drain_semantic_index_events(ctx);
4441 runtime_drain::drain_semantic_refresh_events(ctx);
4442 runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
4443 let empty_bg_sessions = bg_sessions_to_check
4444 .into_iter()
4445 .filter(|(session, _)| {
4446 !ctx.bash_background()
4447 .has_completions_for_session(Some(session.as_str()))
4448 })
4449 .collect();
4450 MaintenanceJobOutcome {
4451 empty_bg_sessions,
4452 requeue_kind: None,
4453 }
4454 }
4455 };
4456 let requeued = outcome.requeue_kind.is_some();
4457 let _ = outcome_tx.send(outcome);
4458 Response::success(
4459 response_id,
4460 json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
4461 )
4462 }),
4463 );
4464 let completion_tx = completion_tx.clone();
4465 let completion_metrics = Arc::clone(metrics);
4466 tokio::spawn(async move {
4467 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4468 let response = await_executor_response(rx, request_id).await;
4469 let outcome = outcome_rx.await.unwrap_or_default();
4470 let _ = send_counted_channel(
4471 &completion_tx,
4472 &completion_metrics.maintenance_queued,
4473 MaintenanceCompletion {
4474 root_id: completion_root_id,
4475 kind,
4476 response,
4477 empty_bg_sessions: outcome.empty_bg_sessions,
4478 requeue_kind: outcome.requeue_kind,
4479 },
4480 )
4481 .await;
4482 });
4483}
4484
4485async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
4486 rx.await
4487 .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
4488}
4489async fn signal_fatal_teardown(
4490 tx: &WriterSender,
4491 route: Option<RouteChannel>,
4492 ver: u8,
4493 corr: u64,
4494 shutdown: &Arc<Notify>,
4495 metrics: &DispatchPathMetrics,
4496) {
4497 if let Some(route) = route {
4498 if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
4499 if let Err(error) = send_frame(tx, metrics, frame).await {
4500 log::warn!(
4501 "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
4502 );
4503 }
4504 }
4505 }
4506 if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
4507 if let Err(error) = send_frame(tx, metrics, frame).await {
4508 log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
4509 }
4510 }
4511 shutdown.notify_one();
4512}
4513#[derive(Debug, Deserialize)]
4514#[serde(untagged)]
4515enum RouteRequest {
4516 BgEvents(BgEventsRequest),
4517 ToolCall(ToolCallRequest),
4518}
4519
4520#[derive(Debug, Deserialize)]
4521struct BgEventsRequest {
4522 op: BgEventsOp,
4523}
4524
4525#[derive(Debug, Deserialize)]
4526#[serde(rename_all = "snake_case")]
4527enum BgEventsOp {
4528 BgEvents,
4529}
4530
4531#[derive(Debug, Deserialize)]
4532struct ToolCallRequest {
4533 name: String,
4534 #[serde(default)]
4535 arguments: Value,
4536 #[serde(default)]
4541 preview: bool,
4542}
4543
4544#[cfg(test)]
4545pub(crate) mod test_support {
4546 use super::*;
4547 use crate::bash_background::BgTaskStatus;
4548 use crate::protocol::{
4549 BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
4550 ProgressFrame, StatusChangedFrame,
4551 };
4552 use serde_json::json;
4553
4554 pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
4555 let dir = tempfile::Builder::new()
4556 .prefix(name)
4557 .tempdir()
4558 .expect("temp root");
4559 let root = ProjectRootId::from_path(dir.path()).expect("project root id");
4560 (dir, root)
4561 }
4562
4563 pub(super) fn test_ctx() -> Arc<AppContext> {
4564 Arc::new(AppContext::new(
4565 Box::new(crate::parser::TreeSitterProvider::new()),
4566 crate::config::Config::default(),
4567 ))
4568 }
4569
4570 pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
4571 let deadline = Instant::now() + Duration::from_secs(30);
4572 loop {
4573 let observed = ctx.watcher_registry_count();
4574 if observed == expected {
4575 return;
4576 }
4577 assert!(
4578 Instant::now() < deadline,
4579 "watcher count did not settle before deadline: expected={expected}, observed={observed}"
4580 );
4581 std::thread::sleep(Duration::from_millis(50));
4582 }
4583 }
4584
4585 pub(super) fn reap_until_forgotten(
4595 root: &ProjectRootId,
4596 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4597 pending_binds: &HashMap<RouteChannel, PendingBind>,
4598 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4599 executor: &Arc<Executor>,
4600 metrics: &DispatchPathMetrics,
4601 ) -> IdleReapOutcome {
4602 let deadline = Instant::now() + Duration::from_secs(30);
4603 loop {
4604 let outcome = reap_idle_roots(
4605 Instant::now(),
4606 live_roots,
4607 pending_binds,
4608 root_channels,
4609 executor,
4610 metrics,
4611 );
4612 if outcome.forgotten_deleted_roots.contains(root) {
4613 return outcome;
4614 }
4615 assert!(
4616 Instant::now() < deadline,
4617 "deleted root was never forgotten: {root:?}"
4618 );
4619 std::thread::sleep(Duration::from_millis(10));
4620 }
4621 }
4622
4623 pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
4624 let deadline = Instant::now() + Duration::from_secs(30);
4625 loop {
4626 let observed = app.actor_root_count();
4627 if observed == expected {
4628 return;
4629 }
4630 assert!(
4631 Instant::now() < deadline,
4632 "actor root count did not settle before deadline: expected={expected}, observed={observed}"
4633 );
4634 std::thread::sleep(Duration::from_millis(50));
4635 }
4636 }
4637
4638 pub(super) fn status_frame(seq: u64) -> PushFrame {
4639 status_frame_with_session(seq, None)
4640 }
4641
4642 pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
4643 PushFrame::StatusChanged(StatusChangedFrame {
4644 frame_type: "status_changed",
4645 session_id: session_id.map(str::to_string),
4646 snapshot: json!({ "seq": seq }),
4647 })
4648 }
4649
4650 pub(super) fn completion_frame(task_id: &str) -> PushFrame {
4651 completion_frame_with_session(task_id, "session-1")
4652 }
4653
4654 pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
4655 PushFrame::BashCompleted(BashCompletedFrame {
4656 frame_type: "bash_completed",
4657 task_id: task_id.to_string(),
4658 session_id: session_id.to_string(),
4659 status: BgTaskStatus::Completed,
4660 exit_code: Some(0),
4661 command: format!("echo {task_id}"),
4662 output_preview: String::new(),
4663 output_truncated: false,
4664 original_tokens: None,
4665 compressed_tokens: None,
4666 tokens_skipped: false,
4667 status_reason: None,
4668 })
4669 }
4670
4671 pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
4672 long_running_frame_with_session(task_id, "session-1", elapsed_ms)
4673 }
4674
4675 pub(super) fn long_running_frame_with_session(
4676 task_id: &str,
4677 session_id: &str,
4678 elapsed_ms: u64,
4679 ) -> PushFrame {
4680 PushFrame::BashLongRunning(BashLongRunningFrame {
4681 frame_type: "bash_long_running",
4682 task_id: task_id.to_string(),
4683 session_id: session_id.to_string(),
4684 command: format!("sleep {elapsed_ms}"),
4685 elapsed_ms,
4686 })
4687 }
4688
4689 pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
4690 PushFrame::BashPatternMatch(BashPatternMatchFrame {
4691 frame_type: "bash_pattern_match",
4692 task_id: "task-pattern".to_string(),
4693 session_id: session_id.to_string(),
4694 watch_id: "watch-1".to_string(),
4695 match_text: "needle".to_string(),
4696 match_offset: 7,
4697 context: "haystack needle".to_string(),
4698 once: true,
4699 reason: "pattern_match",
4700 })
4701 }
4702
4703 pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
4704 PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
4705 frame_type: "configure_warnings",
4706 session_id: session_id.map(str::to_string),
4707 project_root: "/tmp/subc-test".to_string(),
4708 warnings: Vec::new(),
4709 })
4710 }
4711
4712 pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
4713 route_identity_with_trust(root, session_id, BindTrust::FirstParty)
4714 }
4715
4716 pub(super) fn route_identity_with_trust(
4717 root: &ProjectRootId,
4718 session_id: &str,
4719 trust: BindTrust,
4720 ) -> RouteIdentity {
4721 RouteIdentity(Arc::new(RouteIdentityData {
4722 root: root.clone(),
4723 project_root: root.as_path().to_path_buf(),
4724 harness: "opencode".to_string(),
4725 session: session_id.to_string(),
4726 trust,
4727 spawn_principal: AuthenticatedPrincipal::RouteBind {
4728 trust: trust.sandbox_trust(),
4729 route_channel: 0,
4730 route_epoch: 0,
4731 project_root: root.as_path().to_path_buf(),
4732 harness: "opencode".to_string(),
4733 session_id: session_id.to_string(),
4734 principal_id: Some(match trust {
4735 BindTrust::FirstParty => "direct".to_string(),
4736 BindTrust::Untrusted => "unverified".to_string(),
4737 }),
4738 },
4739 consumer_elicitation_capable: false,
4740 }))
4741 }
4742
4743 pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
4744 PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
4745 }
4746
4747 pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
4748 match frame {
4749 PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
4750 _ => None,
4751 }
4752 }
4753
4754 pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
4755 match frame {
4756 PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
4757 _ => None,
4758 }
4759 }
4760
4761 pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
4762 let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
4763 body.get("task_id")
4764 .and_then(serde_json::Value::as_str)
4765 .map(str::to_string)
4766 }
4767}
4768
4769#[cfg(test)]
4770mod tests {
4771 use super::test_support::{
4772 completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
4773 wait_for_actor_root_count, wait_for_watcher_count,
4774 };
4775 use super::*;
4776 use crate::bash_background::BgTaskStatus;
4777
4778 fn attach_error(kind: io::ErrorKind) -> SubcError {
4779 SubcError::Connect {
4780 endpoint: "127.0.0.1:1".to_string(),
4781 source: io::Error::new(kind, "constructed attach failure"),
4782 }
4783 }
4784
4785 fn auth_io_error(kind: io::ErrorKind) -> SubcError {
4786 SubcError::Auth {
4787 endpoint: "127.0.0.1:1".to_string(),
4788 source: subc_transport::AuthError::Io {
4789 stage: subc_transport::AuthStage::ServerProof,
4790 source: io::Error::new(kind, "constructed auth failure"),
4791 },
4792 }
4793 }
4794
4795 #[test]
4796 fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
4797 let transient_errors = vec![
4798 attach_error(io::ErrorKind::ConnectionRefused),
4799 attach_error(io::ErrorKind::TimedOut),
4800 attach_error(io::ErrorKind::ConnectionReset),
4801 auth_io_error(io::ErrorKind::ConnectionAborted),
4802 auth_io_error(io::ErrorKind::BrokenPipe),
4803 SubcError::Auth {
4804 endpoint: "127.0.0.1:1".to_string(),
4805 source: subc_transport::AuthError::UnexpectedEof {
4806 stage: subc_transport::AuthStage::ServerProof,
4807 expected: 4,
4808 actual: 0,
4809 },
4810 },
4811 SubcError::Auth {
4812 endpoint: "127.0.0.1:1".to_string(),
4813 source: subc_transport::AuthError::Timeout {
4814 stage: subc_transport::AuthStage::ServerProof,
4815 deadline: AUTH_DEADLINE,
4816 },
4817 },
4818 ];
4819 for error in &transient_errors {
4820 assert_eq!(
4821 classify_attach_error(error),
4822 AttachErrorClass::Transient,
4823 "expected transient: {error}"
4824 );
4825 }
4826
4827 let permanent_errors = vec![
4828 attach_error(io::ErrorKind::PermissionDenied),
4829 auth_io_error(io::ErrorKind::InvalidData),
4830 SubcError::Auth {
4831 endpoint: "127.0.0.1:1".to_string(),
4832 source: subc_transport::AuthError::InvalidServerProof,
4833 },
4834 SubcError::Auth {
4835 endpoint: "127.0.0.1:1".to_string(),
4836 source: subc_transport::AuthError::DaemonIdMismatch,
4837 },
4838 SubcError::ConnectionFile {
4839 path: PathBuf::from("subc-connection.json"),
4840 source: subc_transport::ConnectionFileError::Invalid {
4841 reason: "constructed invalid file".to_string(),
4842 },
4843 },
4844 SubcError::NoEndpoint {
4845 path: PathBuf::from("subc-connection.json"),
4846 },
4847 SubcError::InvalidEndpoint {
4848 path: PathBuf::from("subc-connection.json"),
4849 endpoint: "not-an-ip:1234".to_string(),
4850 },
4851 ];
4852 for error in &permanent_errors {
4853 assert_eq!(
4854 classify_attach_error(error),
4855 AttachErrorClass::Permanent,
4856 "expected permanent: {error}"
4857 );
4858 }
4859 }
4860
4861 #[test]
4862 fn incompatible_wire_version_is_rejected_before_tcp_connect() {
4863 let conn_dir = tempfile::tempdir().expect("connection tempdir");
4864 let conn_path = conn_dir.path().join("subc-connection.json");
4865 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
4866 listener
4867 .set_nonblocking(true)
4868 .expect("set listener nonblocking");
4869 let port = listener.local_addr().expect("listener addr").port();
4870 connection_file::write_atomic(
4871 &conn_path,
4872 &connection_file::ConnectionInfo {
4873 schema: connection_file::SCHEMA_VERSION,
4874 wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
4875 endpoints: vec![connection_file::Endpoint {
4876 host: "127.0.0.1".to_string(),
4877 port,
4878 }],
4879 key: vec![0x42; subc_transport::KEY_LEN],
4880 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4881 pid: std::process::id(),
4882 daemon_ver: "subc-test".to_string(),
4883 },
4884 )
4885 .expect("write connection file");
4886
4887 let runtime = tokio::runtime::Builder::new_current_thread()
4888 .enable_all()
4889 .build()
4890 .expect("test runtime");
4891 let result = runtime.block_on(connect_and_authenticate_with_policy(
4892 &conn_path,
4893 AttachRetryPolicy {
4894 budget: Duration::from_secs(1),
4895 initial_backoff: Duration::from_millis(5),
4896 max_backoff: Duration::from_millis(10),
4897 jitter_percent: 0,
4898 },
4899 ));
4900 assert!(matches!(
4901 result,
4902 Err(SubcError::ConnectionFile {
4903 source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
4904 ..
4905 })
4906 ));
4907 assert!(matches!(
4908 listener.accept(),
4909 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
4910 ));
4911 }
4912
4913 #[test]
4914 fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
4915 let conn_dir = tempfile::tempdir().expect("connection tempdir");
4916 let conn_path = conn_dir.path().join("subc-connection.json");
4917 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
4918 let port = listener.local_addr().expect("reserved addr").port();
4919 drop(listener);
4920 connection_file::write_atomic(
4921 &conn_path,
4922 &connection_file::ConnectionInfo {
4923 schema: connection_file::SCHEMA_VERSION,
4924 wire_version: Some(PROTOCOL_VERSION),
4925 endpoints: vec![connection_file::Endpoint {
4926 host: "127.0.0.1".to_string(),
4927 port,
4928 }],
4929 key: vec![0x42; subc_transport::KEY_LEN],
4930 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4931 pid: std::process::id(),
4932 daemon_ver: "subc-test".to_string(),
4933 },
4934 )
4935 .expect("write connection file");
4936
4937 let policy = AttachRetryPolicy {
4938 budget: Duration::from_millis(40),
4939 initial_backoff: Duration::from_millis(5),
4940 max_backoff: Duration::from_millis(10),
4941 jitter_percent: 0,
4942 };
4943 let runtime = tokio::runtime::Builder::new_current_thread()
4944 .enable_all()
4945 .build()
4946 .expect("test runtime");
4947 let started_at = Instant::now();
4948 let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
4949 let elapsed = started_at.elapsed();
4950 let error = match result {
4951 Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
4952 Err(error) => error,
4953 };
4954
4955 assert!(matches!(error, SubcError::Connect { .. }), "{error}");
4956 assert!(
4957 elapsed >= Duration::from_millis(35),
4958 "retry budget ended too early: {elapsed:?}"
4959 );
4960 assert!(
4961 elapsed < Duration::from_secs(1),
4962 "retry budget was not bounded: {elapsed:?}"
4963 );
4964 }
4965
4966 fn due_maintenance_jobs_without_actor_context(
4967 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4968 budget: usize,
4969 pending_bind_roots: &HashSet<ProjectRootId>,
4970 ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
4971 due_maintenance_jobs(
4972 live_roots,
4973 None,
4974 &HashMap::new(),
4975 &HashSet::new(),
4976 budget,
4977 pending_bind_roots,
4978 )
4979 }
4980
4981 fn actor_ctx_with_dirty_search_index(
4982 root: &Path,
4983 storage: &Path,
4984 file_name: &str,
4985 old_contents: &str,
4986 new_contents: &str,
4987 ) -> (Arc<AppContext>, PathBuf, PathBuf) {
4988 let file = root.join(file_name);
4989 std::fs::write(&file, old_contents).expect("write source");
4990 let canonical_root = std::fs::canonicalize(root).expect("canonical root");
4991 let ctx = Arc::new(AppContext::new(
4992 Box::new(crate::parser::TreeSitterProvider::new()),
4993 Config {
4994 project_root: Some(root.to_path_buf()),
4995 storage_dir: Some(storage.to_path_buf()),
4996 ..Config::default()
4997 },
4998 ));
4999 ctx.set_canonical_cache_root(canonical_root.clone());
5000
5001 let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
5002 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
5003 let git_head = index.stored_git_head().map(str::to_owned);
5004 index.write_to_disk(&cache_dir, git_head.as_deref());
5005
5006 std::fs::write(&file, new_contents).expect("edit source");
5007 index.update_file(&file);
5008 *ctx.search_index()
5009 .write()
5010 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
5011 (ctx, canonical_root, cache_dir)
5012 }
5013
5014 #[test]
5015 fn graceful_shutdown_flushes_every_actor_search_index() {
5016 let storage = tempfile::tempdir().expect("storage tempdir");
5017 let (root1_dir, root1) = test_root("shutdown-flush-root-1");
5018 let (root2_dir, root2) = test_root("shutdown-flush-root-2");
5019 let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
5020 root1_dir.path(),
5021 storage.path(),
5022 "alpha.txt",
5023 "old actor one token\n",
5024 "new actor one token\n",
5025 );
5026 let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
5027 root2_dir.path(),
5028 storage.path(),
5029 "beta.txt",
5030 "old actor two token\n",
5031 "new actor two token\n",
5032 );
5033
5034 let executor = Executor::new();
5035 assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
5036 assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
5037
5038 flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
5039
5040 let mut restored1 =
5041 crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
5042 .expect("load flushed root one index");
5043 restored1.ready = true;
5044 assert_eq!(
5045 restored1
5046 .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
5047 .matches
5048 .len(),
5049 1,
5050 "graceful subc shutdown should flush the first root's trigram delta"
5051 );
5052
5053 let mut restored2 =
5054 crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
5055 .expect("load flushed root two index");
5056 restored2.ready = true;
5057 assert_eq!(
5058 restored2
5059 .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
5060 .matches
5061 .len(),
5062 1,
5063 "graceful subc shutdown should flush every registered root"
5064 );
5065 }
5066
5067 #[test]
5068 fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
5069 let _ = env_logger::builder().is_test(true).try_init();
5070 let (root_dir, root) = test_root("idle-root-reaper");
5071 let storage = tempfile::tempdir().expect("storage tempdir");
5072 std::fs::write(
5073 root_dir.path().join("main.rs"),
5074 "fn entry() { leaf(); }\nfn leaf() {}\n",
5075 )
5076 .expect("source file");
5077 let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
5078 let app = App::default_shared();
5079 let ctx = Arc::new(AppContext::from_app(
5080 Arc::clone(&app),
5081 Config {
5082 project_root: Some(canonical_root.clone()),
5083 storage_dir: Some(storage.path().to_path_buf()),
5084 callgraph_store: true,
5085 search_index: true,
5086 ..Config::default()
5087 },
5088 ));
5089 ctx.set_canonical_cache_root(canonical_root.clone());
5090 let project_key = crate::search_index::artifact_cache_key(&canonical_root);
5091 crate::root_cache::configure_artifact_access(&canonical_root, &project_key, false);
5092 assert!(ctx
5093 .ensure_callgraph_store()
5094 .expect("build callgraph store")
5095 .is_some());
5096
5097 let cache_dir =
5098 crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
5099 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
5100 let git_head = index.stored_git_head().map(str::to_owned);
5101 index.write_to_disk(&cache_dir, git_head.as_deref());
5102 *ctx.search_index()
5103 .write()
5104 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
5105 let seeded_generation =
5108 crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
5109 .expect("seeded artifact generation");
5110 crate::cache_freshness::record_verify_completed(
5111 &canonical_root,
5112 crate::cache_freshness::VerifyArtifact::Search,
5113 Some(seeded_generation),
5114 );
5115 assert!(
5116 matches!(
5117 crate::cache_freshness::warm_verify_plan(
5118 canonical_root.as_path(),
5119 crate::cache_freshness::VerifyArtifact::Search,
5120 Some(seeded_generation),
5121 ),
5122 crate::cache_freshness::WarmVerifyPlan::Skip
5123 ),
5124 "memo must be warm before eviction for the downgrade assertion to bite"
5125 );
5126
5127 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5128 let _dispatch_tx = dispatch_tx;
5129 let shutdown = Arc::new(AtomicBool::new(false));
5130 let thread_shutdown = Arc::clone(&shutdown);
5131 let join = std::thread::spawn(move || {
5132 while !thread_shutdown.load(Ordering::SeqCst) {
5133 std::thread::yield_now();
5134 }
5135 });
5136 ctx.install_watcher_runtime(
5137 dispatch_rx,
5138 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5139 );
5140 wait_for_watcher_count(&ctx, 1);
5141
5142 let executor = Arc::new(Executor::new());
5143 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5144 ctx.mark_subc_unbound();
5145 let mut live_roots = HashMap::new();
5146 let mut meta = RootMeta::new(Instant::now());
5147 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5148 meta.unbound_quiesced = true;
5149 live_roots.insert(root.clone(), meta);
5150
5151 let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
5152 assert!(message.contains("evicted idle root"));
5153 assert!(message.contains("freed ~"));
5154 assert!(message.contains("semantic"));
5155 assert!(!message.contains("semantic not estimated retained"));
5156 assert!(message.contains("trigram"));
5157 assert!(message.contains("retained: bash"));
5158 assert!(message.contains("parser_pool"));
5159
5160 assert_eq!(
5161 reap_idle_roots(
5162 Instant::now(),
5163 &mut live_roots,
5164 &HashMap::new(),
5165 &HashMap::new(),
5166 &executor,
5167 &DispatchPathMetrics::new(),
5168 )
5169 .evicted,
5170 1
5171 );
5172 assert!(ctx.search_index().read().unwrap().is_none());
5173 wait_for_watcher_count(&ctx, 0);
5174 assert!(
5179 matches!(
5180 crate::cache_freshness::warm_verify_plan(
5181 canonical_root.as_path(),
5182 crate::cache_freshness::VerifyArtifact::Search,
5183 Some(seeded_generation),
5184 ),
5185 crate::cache_freshness::WarmVerifyPlan::Strict
5186 ),
5187 "idle eviction must force strict re-verification"
5188 );
5189 assert!(
5190 crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
5191 );
5192 ctx.mark_subc_bound();
5193 assert!(ctx
5194 .ensure_callgraph_store()
5195 .expect("reopen callgraph store")
5196 .is_some());
5197 assert!(live_roots[&root].idle_artifacts_evicted);
5198 }
5199
5200 #[test]
5201 fn idle_root_reaper_applies_ttl_to_unbound_roots() {
5202 let (_root_dir, root) = test_root("idle-root-ttl-gate");
5203 let ctx = test_ctx();
5204 let executor = Arc::new(Executor::new());
5205 assert!(executor.register_actor(root.clone(), ctx));
5206 let ctx = executor.actor_context(&root).expect("actor context");
5207 ctx.mark_subc_unbound();
5208 let now = Instant::now();
5209 let mut meta = RootMeta::new(now);
5210 meta.unbound_quiesced = true;
5211 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5212
5213 assert_eq!(
5217 reap_idle_roots(
5218 now,
5219 &mut live_roots,
5220 &HashMap::new(),
5221 &HashMap::new(),
5222 &executor,
5223 &DispatchPathMetrics::new(),
5224 )
5225 .evicted,
5226 0
5227 );
5228 assert!(!live_roots[&root].idle_artifacts_evicted);
5229
5230 ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
5236 assert_eq!(
5237 reap_idle_roots(
5238 now + IDLE_ROOT_TTL,
5239 &mut live_roots,
5240 &HashMap::new(),
5241 &HashMap::new(),
5242 &executor,
5243 &DispatchPathMetrics::new(),
5244 )
5245 .evicted,
5246 1
5247 );
5248 assert!(live_roots[&root].idle_artifacts_evicted);
5249 assert!(
5250 ctx.take_pending_search_index_paths().is_empty(),
5251 "TTL eviction must dispose retained pending reconciliation paths"
5252 );
5253 }
5254
5255 #[test]
5256 fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
5257 let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
5258 let ctx = test_ctx();
5259 let executor = Arc::new(Executor::new());
5260 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5261 ctx.mark_subc_unbound();
5262
5263 let pending = root.as_path().join("edited-while-unbound.rs");
5269 ctx.add_pending_search_index_paths([pending.clone()]);
5270 let dirty_source = root.as_path().join("dirty.rs");
5271 std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
5272 let mut dirty = crate::search_index::SearchIndex::new();
5273 dirty.ready = true;
5274 dirty.update_file(&dirty_source);
5275 *ctx.search_index()
5276 .write()
5277 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
5278 assert!(ctx.artifact_eviction_blocked());
5279
5280 let mut live_roots = HashMap::new();
5281 let mut meta = RootMeta::new(Instant::now());
5282 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5283 meta.unbound_quiesced = true;
5284 live_roots.insert(root.clone(), meta);
5285
5286 assert_eq!(
5287 reap_idle_roots(
5288 Instant::now(),
5289 &mut live_roots,
5290 &HashMap::new(),
5291 &HashMap::new(),
5292 &executor,
5293 &DispatchPathMetrics::new(),
5294 )
5295 .evicted,
5296 0,
5297 "the dirty index must still block this eviction"
5298 );
5299 assert_eq!(
5300 ctx.take_pending_search_index_paths(),
5301 vec![pending],
5302 "a blocked eviction must restore the taken pending paths"
5303 );
5304 }
5305
5306 #[test]
5307 fn idle_reap_with_bound_route_keeps_watcher_running() {
5308 let (_root_dir, root) = test_root("bound-root-reap-gate");
5309 let ctx = test_ctx();
5310 let executor = Arc::new(Executor::new());
5311 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5312
5313 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5314 let _dispatch_tx = dispatch_tx;
5315 let shutdown = Arc::new(AtomicBool::new(false));
5316 let thread_shutdown = Arc::clone(&shutdown);
5317 let join = std::thread::spawn(move || {
5318 while !thread_shutdown.load(Ordering::SeqCst) {
5319 std::thread::yield_now();
5320 }
5321 });
5322 ctx.install_watcher_runtime(
5323 dispatch_rx,
5324 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5325 );
5326
5327 let mut meta = RootMeta::new(Instant::now());
5328 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5329 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5330 let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
5331 assert_eq!(
5332 reap_idle_roots(
5333 Instant::now(),
5334 &mut live_roots,
5335 &HashMap::new(),
5336 &bound,
5337 &executor,
5338 &DispatchPathMetrics::new(),
5339 )
5340 .evicted,
5341 0
5342 );
5343 wait_for_watcher_count(&ctx, 1);
5344 ctx.stop_watcher_runtime_in_background();
5345 wait_for_watcher_count(&ctx, 0);
5346 }
5347
5348 #[test]
5349 fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
5350 let (root_dir, root) = test_root("deleted-bound-root-reap");
5351 let executor = Arc::new(Executor::new());
5352 assert!(executor.register_actor(root.clone(), test_ctx()));
5353 root_dir.close().expect("delete project root");
5354
5355 let route = route_key(19, 3);
5356 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5357 let cancel_signal = PersistentCancelSignal::new();
5358 let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
5359 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5360 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5361 let mut route_bash_cancels = HashMap::from([(
5362 route,
5363 bash::RouteBashCancel {
5364 token: cancel_signal.clone(),
5365 active_waits: 0,
5366 },
5367 )]);
5368 let metrics = DispatchPathMetrics::new();
5369
5370 let first = reap_idle_roots(
5371 Instant::now(),
5372 &mut live_roots,
5373 &HashMap::new(),
5374 &root_channels,
5375 &executor,
5376 &metrics,
5377 );
5378 assert!(first.forgotten_deleted_roots.is_empty());
5379 assert!(executor.actor_registered(&root));
5380
5381 let mut forgotten = Vec::new();
5382 for _ in 0..100 {
5383 let outcome = reap_idle_roots(
5384 Instant::now(),
5385 &mut live_roots,
5386 &HashMap::new(),
5387 &root_channels,
5388 &executor,
5389 &metrics,
5390 );
5391 if !outcome.forgotten_deleted_roots.is_empty() {
5392 forgotten = outcome.forgotten_deleted_roots;
5393 break;
5394 }
5395 std::thread::sleep(Duration::from_millis(10));
5396 }
5397 assert_eq!(forgotten, vec![root.clone()]);
5398 assert!(!executor.actor_registered(&root));
5399
5400 let mut retry_buffer = HashMap::new();
5401 let mut reclaimed_routes = ReclaimedRoutes::default();
5402 let mut session_identity = HashMap::new();
5403 let mut push_buffer = HashMap::new();
5404 let mut bg_subs = HashMap::new();
5405 let mut bg_sub_by_session = HashMap::new();
5406 let mut bg_wake_pending = HashSet::new();
5407 let mut bg_wake_epoch = HashMap::new();
5408 let mut pending_bash_asks = HashMap::new();
5409 purge_deleted_root_residents(
5410 &root,
5411 &mut routes,
5412 &mut root_channels,
5413 &mut installed_route_epochs,
5414 &mut route_bash_cancels,
5415 &mut retry_buffer,
5416 &mut reclaimed_routes,
5417 &mut session_identity,
5418 &mut push_buffer,
5419 &mut bg_subs,
5420 &mut bg_sub_by_session,
5421 &mut bg_wake_pending,
5422 &mut bg_wake_epoch,
5423 &mut pending_bash_asks,
5424 );
5425
5426 assert!(routes.is_empty());
5427 assert!(root_channels.is_empty());
5428 assert!(installed_route_epochs.is_empty());
5429 assert!(route_bash_cancels.is_empty());
5430 assert!(reclaimed_routes.contains(route));
5431 assert!(cancel_signal.is_cancelled());
5432 }
5433
5434 #[test]
5441 fn live_root_with_bound_route_is_never_reclaimed() {
5442 let (_root_dir, root) = test_root("live-bound-root-retained");
5443 let ctx = test_ctx();
5444 ctx.mark_subc_unbound();
5445 let executor = Arc::new(Executor::new());
5446 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5447
5448 let route = RouteChannel {
5449 channel: 7,
5450 epoch: 1,
5451 };
5452 let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
5453 meta.unbound_quiesced = true;
5454 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5455 let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5456
5457 for _ in 0..3 {
5462 let outcome = reap_idle_roots(
5463 Instant::now(),
5464 &mut live_roots,
5465 &HashMap::new(),
5466 &root_channels,
5467 &executor,
5468 &DispatchPathMetrics::new(),
5469 );
5470 assert!(
5471 outcome.forgotten_deleted_roots.is_empty(),
5472 "a root whose directory exists must never be forgotten"
5473 );
5474 }
5475
5476 assert!(live_roots.contains_key(&root), "live root must be retained");
5477 assert!(
5478 executor.actor_registered(&root),
5479 "live root's actor must survive"
5480 );
5481 assert!(
5482 root.as_path().exists(),
5483 "test vehicle must keep the directory alive; otherwise this control proves nothing"
5484 );
5485 }
5486
5487 #[test]
5488 fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
5489 let (root_dir, root) = test_root("deleted-root-first-observation");
5490 let ctx = test_ctx();
5491 ctx.mark_subc_unbound();
5492 let executor = Arc::new(Executor::new());
5493 assert!(executor.register_actor(root.clone(), ctx));
5494 root_dir.close().expect("delete project root");
5495
5496 let mut meta = RootMeta::new(Instant::now());
5497 meta.unbound_quiesced = true;
5498 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5499 let outcome = reap_idle_roots(
5500 Instant::now(),
5501 &mut live_roots,
5502 &HashMap::new(),
5503 &HashMap::new(),
5504 &executor,
5505 &DispatchPathMetrics::new(),
5506 );
5507
5508 assert!(outcome.forgotten_deleted_roots.is_empty());
5509 assert!(live_roots.contains_key(&root));
5510 assert!(executor.actor_registered(&root));
5511 }
5512
5513 fn spawn_background_for_root(
5514 ctx: &AppContext,
5515 root: &ProjectRootId,
5516 storage: &tempfile::TempDir,
5517 session_id: &str,
5518 ) -> (String, u32) {
5519 let command = if cfg!(windows) {
5525 "ping -n 31 127.0.0.1 > nul"
5527 } else {
5528 "sleep 30"
5529 };
5530 let task_id = ctx
5531 .bash_background()
5532 .spawn(
5533 crate::sandbox_spawn::SpawnPlan::Unsandboxed,
5534 command,
5535 session_id.to_string(),
5536 storage.path().to_path_buf(),
5537 HashMap::new(),
5538 Some(Duration::from_secs(60)),
5539 storage.path().to_path_buf(),
5540 8,
5541 true,
5542 false,
5543 Some(root.as_path().to_path_buf()),
5544 )
5545 .expect("spawn background task");
5546 let snapshot = ctx
5547 .bash_background()
5548 .status(
5549 &task_id,
5550 session_id,
5551 Some(root.as_path()),
5552 Some(storage.path()),
5553 0,
5554 )
5555 .expect("background task status");
5556 (task_id, snapshot.child_pid.expect("background child pid"))
5557 }
5558
5559 fn wait_for_background_exit(pid: u32) {
5560 let deadline = Instant::now() + Duration::from_secs(5);
5561 while crate::bash_background::process::is_process_alive(pid) {
5562 assert!(
5563 Instant::now() < deadline,
5564 "background task process survived kill"
5565 );
5566 std::thread::sleep(Duration::from_millis(20));
5567 }
5568 }
5569
5570 #[test]
5571 fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
5572 let (root_dir, root) = test_root("deleted-root-background-task");
5573 let storage = tempfile::tempdir().expect("task storage");
5574 let ctx = test_ctx();
5575 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
5576 assert!(crate::bash_background::process::is_process_alive(pid));
5577
5578 let executor = Arc::new(Executor::new());
5579 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5580 assert!(executor.actor_is_idle(&root));
5581 root_dir.close().expect("delete project root");
5582 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5583 let pending_binds = HashMap::new();
5584 let root_channels = HashMap::new();
5585 let metrics = DispatchPathMetrics::new();
5586
5587 let first = reap_idle_roots(
5588 Instant::now(),
5589 &mut live_roots,
5590 &pending_binds,
5591 &root_channels,
5592 &executor,
5593 &metrics,
5594 );
5595 assert!(first.forgotten_deleted_roots.is_empty());
5596 assert!(crate::bash_background::process::is_process_alive(pid));
5597
5598 let outcome = reap_until_forgotten(
5599 &root,
5600 &mut live_roots,
5601 &pending_binds,
5602 &root_channels,
5603 &executor,
5604 &metrics,
5605 );
5606 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5607 wait_for_background_exit(pid);
5608
5609 let snapshot = ctx
5610 .bash_background()
5611 .status(
5612 &task_id,
5613 "reclaim-session",
5614 Some(root.as_path()),
5615 Some(storage.path()),
5616 0,
5617 )
5618 .expect("reclaimed task status");
5619 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
5620 assert_eq!(
5621 snapshot.info.status_reason.as_deref(),
5622 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
5623 );
5624 assert_eq!(
5625 serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
5626 crate::bash_background::registry::ROOT_RECLAIMED_REASON
5627 );
5628 let completion = ctx
5629 .bash_background()
5630 .drain_completions_for_session(Some("reclaim-session"))
5631 .pop()
5632 .expect("reclaimed task completion");
5633 assert_eq!(
5634 completion.status_reason.as_deref(),
5635 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
5636 );
5637 }
5638
5639 #[test]
5640 fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
5641 let (root_dir, root) = test_root("existing-root-background-task");
5642 let storage = tempfile::tempdir().expect("task storage");
5643 let ctx = test_ctx();
5644 ctx.mark_subc_unbound();
5645 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
5646
5647 let executor = Arc::new(Executor::new());
5648 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5649 let mut meta = RootMeta::new(
5650 Instant::now()
5651 .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
5652 .expect("old root timestamp"),
5653 );
5654 meta.unbound_quiesced = true;
5655 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5656 let pending_binds = HashMap::new();
5657 let root_channels = HashMap::new();
5658 let metrics = DispatchPathMetrics::new();
5659
5660 for _ in 0..8 {
5661 reap_idle_roots(
5662 Instant::now(),
5663 &mut live_roots,
5664 &pending_binds,
5665 &root_channels,
5666 &executor,
5667 &metrics,
5668 );
5669 std::thread::sleep(Duration::from_millis(10));
5670 }
5671 assert!(root_dir.path().exists());
5672 assert!(crate::bash_background::process::is_process_alive(pid));
5673 let snapshot = ctx
5674 .bash_background()
5675 .status(
5676 &task_id,
5677 "existing-session",
5678 Some(root.as_path()),
5679 Some(storage.path()),
5680 0,
5681 )
5682 .expect("existing task status");
5683 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5684 let _ = ctx.bash_background().kill(&task_id, "existing-session");
5685 wait_for_background_exit(pid);
5686 }
5687
5688 #[test]
5689 fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
5690 let (root_dir, root) = test_root("restored-root-background-task");
5691 let storage = tempfile::tempdir().expect("task storage");
5692 let ctx = test_ctx();
5693 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
5694
5695 let executor = Arc::new(Executor::new());
5696 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5697 root_dir.close().expect("delete project root");
5698 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5699 let pending_binds = HashMap::new();
5700 let root_channels = HashMap::new();
5701 let metrics = DispatchPathMetrics::new();
5702
5703 let first = reap_idle_roots(
5704 Instant::now(),
5705 &mut live_roots,
5706 &pending_binds,
5707 &root_channels,
5708 &executor,
5709 &metrics,
5710 );
5711 assert!(first.forgotten_deleted_roots.is_empty());
5712 std::fs::create_dir_all(root.as_path()).expect("restore project root");
5713 let second = reap_idle_roots(
5714 Instant::now(),
5715 &mut live_roots,
5716 &pending_binds,
5717 &root_channels,
5718 &executor,
5719 &metrics,
5720 );
5721 assert!(second.forgotten_deleted_roots.is_empty());
5722 assert!(crate::bash_background::process::is_process_alive(pid));
5723 let _ = ctx.bash_background().kill(&task_id, "restored-session");
5724 wait_for_background_exit(pid);
5725 }
5726
5727 #[test]
5728 fn observing_root_again_resets_deleted_sweep_confirmation() {
5729 let (root_dir, root) = test_root("deleted-root-observation-reset");
5730 let ctx = test_ctx();
5731 ctx.mark_subc_unbound();
5732 let executor = Arc::new(Executor::new());
5733 assert!(executor.register_actor(root.clone(), ctx));
5734 root_dir.close().expect("delete project root");
5735
5736 let mut meta = RootMeta::new(Instant::now());
5737 meta.unbound_quiesced = true;
5738 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5739 let pending_binds = HashMap::new();
5740 let root_channels = HashMap::new();
5741 let metrics = DispatchPathMetrics::new();
5742
5743 let first = reap_idle_roots(
5744 Instant::now(),
5745 &mut live_roots,
5746 &pending_binds,
5747 &root_channels,
5748 &executor,
5749 &metrics,
5750 );
5751 assert!(first.forgotten_deleted_roots.is_empty());
5752
5753 std::fs::create_dir_all(root.as_path()).expect("restore project root");
5754 reap_idle_roots(
5755 Instant::now(),
5756 &mut live_roots,
5757 &pending_binds,
5758 &root_channels,
5759 &executor,
5760 &metrics,
5761 );
5762 std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
5763
5764 let after_reset = reap_idle_roots(
5765 Instant::now(),
5766 &mut live_roots,
5767 &pending_binds,
5768 &root_channels,
5769 &executor,
5770 &metrics,
5771 );
5772 assert!(after_reset.forgotten_deleted_roots.is_empty());
5773 assert!(live_roots.contains_key(&root));
5774 assert!(executor.actor_registered(&root));
5775 }
5776
5777 #[test]
5778 fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
5779 let (root_dir, root) = test_root("deleted-root-reap");
5780 let app = App::default_shared();
5781 let ctx = Arc::new(AppContext::from_app(
5782 Arc::clone(&app),
5783 Config {
5784 project_root: Some(root.as_path().to_path_buf()),
5785 ..Config::default()
5786 },
5787 ));
5788 ctx.set_canonical_cache_root(root.as_path().to_path_buf());
5789 ctx.mark_subc_unbound();
5790 let executor = Arc::new(Executor::new());
5791 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5792 assert_eq!(app.actor_root_count(), 1);
5793 drop(ctx);
5794 root_dir.close().expect("delete project root");
5795
5796 let mut meta = RootMeta::new(Instant::now());
5797 meta.unbound_quiesced = true;
5798 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5799 let outcome = reap_until_forgotten(
5800 &root,
5801 &mut live_roots,
5802 &HashMap::new(),
5803 &HashMap::new(),
5804 &executor,
5805 &DispatchPathMetrics::new(),
5806 );
5807 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5808 assert!(!executor.actor_registered(&root));
5809 assert!(!live_roots.contains_key(&root));
5810 wait_for_actor_root_count(&app, 0);
5811
5812 let status_ctx = AppContext::from_app(app, Config::default());
5813 let status = status_ctx.build_status_snapshot();
5814 assert_eq!(status["runtime"]["live_actor_roots"], 0);
5815 assert_eq!(status["runtime"]["open_routes"], 0);
5816 }
5817
5818 #[test]
5819 fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
5820 let (root_dir, root) = test_root("deleted-root-reap-census");
5821 let executor = Arc::new(Executor::new());
5822 assert!(executor.register_actor(root.clone(), test_ctx()));
5823 root_dir.close().expect("delete project root");
5824
5825 let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
5826 let metrics = DispatchPathMetrics::new();
5827 let outcome = reap_idle_roots(
5828 Instant::now(),
5829 &mut live_roots,
5830 &HashMap::new(),
5831 &HashMap::new(),
5832 &executor,
5833 &metrics,
5834 );
5835 assert_eq!(outcome.evicted, 0);
5836
5837 let report = build_health_report(
5838 &executor,
5839 &HashMap::new(),
5840 &metrics,
5841 &crate::context::App::default_shared(),
5842 );
5843 let reap = report
5844 .metrics
5845 .as_ref()
5846 .and_then(|metrics| metrics.get("reap"))
5847 .expect("reap health metrics");
5848 assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
5849 assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
5850 assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
5851 assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
5852 }
5853
5854 #[test]
5855 fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
5856 let (root_dir, root) = test_root("connection-exit-deleted-root");
5857 let executor = Arc::new(Executor::new());
5858 assert!(executor.register_actor(root.clone(), test_ctx()));
5859
5860 let route = route_key(11, 1);
5861 let mut meta = RootMeta::new(Instant::now());
5862 meta.maintenance_pending = true;
5863 meta.maintenance_queued_kinds
5864 .push_back(MaintenanceDrainKind::CompletionDrains);
5865 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5866 let mut pending_binds = HashMap::new();
5867 let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
5868 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5869 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5870 let mut route_bash_cancels = HashMap::new();
5871
5872 quiesce_connection_roots(
5873 &mut live_roots,
5874 &mut pending_binds,
5875 &mut routes,
5876 &mut root_channels,
5877 &mut installed_route_epochs,
5878 &mut route_bash_cancels,
5879 &executor,
5880 );
5881 assert!(live_roots[&root].unbound_quiesced);
5882 assert!(!live_roots[&root].maintenance_pending);
5883 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
5884 assert!(routes.is_empty());
5885 assert!(root_channels.is_empty());
5886
5887 root_dir.close().expect("delete project root");
5888 let metrics = DispatchPathMetrics::new();
5889 let outcome = reap_until_forgotten(
5890 &root,
5891 &mut live_roots,
5892 &pending_binds,
5893 &root_channels,
5894 &executor,
5895 &metrics,
5896 );
5897 let mut session_identity = HashMap::new();
5898 let mut push_buffer = HashMap::new();
5899 let mut bg_subs = HashMap::new();
5900 let mut bg_sub_by_session = HashMap::new();
5901 let mut bg_wake_pending = HashSet::new();
5902 let mut bg_wake_epoch = HashMap::new();
5903 let mut pending_bash_asks = HashMap::new();
5904 let mut retry_buffer = HashMap::new();
5905 let mut reclaimed_routes = ReclaimedRoutes::default();
5906 for forgotten in &outcome.forgotten_deleted_roots {
5907 purge_deleted_root_residents(
5908 forgotten,
5909 &mut routes,
5910 &mut root_channels,
5911 &mut installed_route_epochs,
5912 &mut route_bash_cancels,
5913 &mut retry_buffer,
5914 &mut reclaimed_routes,
5915 &mut session_identity,
5916 &mut push_buffer,
5917 &mut bg_subs,
5918 &mut bg_sub_by_session,
5919 &mut bg_wake_pending,
5920 &mut bg_wake_epoch,
5921 &mut pending_bash_asks,
5922 );
5923 }
5924
5925 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5926 assert!(!executor.actor_registered(&root));
5927 assert!(!live_roots.contains_key(&root));
5928 }
5929
5930 #[test]
5931 fn unbound_root_quiesces_maintenance_without_removing_actor() {
5932 let (_root_dir, root) = test_root("unbound-root-quiesce");
5933 let ctx = test_ctx();
5934 let executor = Arc::new(Executor::new());
5935 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5936 let mut meta = RootMeta::new(Instant::now());
5937 meta.maintenance_pending = true;
5938 meta.maintenance_jobs_in_flight = 1;
5939 meta.maintenance_queued_kinds
5940 .push_back(MaintenanceDrainKind::ConfigureTail);
5941 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5942 *ctx.search_index()
5944 .write()
5945 .unwrap_or_else(std::sync::PoisonError::into_inner) =
5946 Some(crate::search_index::SearchIndex::new());
5947 ctx.set_cache_writer_capabilities(true, true);
5948 let pending = root.as_path().join("pending.rs");
5949 ctx.add_pending_search_index_paths([pending.clone()]);
5950 let canonical_root = root.as_path().to_path_buf();
5954 let artifact = canonical_root.join("cache.bin");
5955 std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
5956 let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
5957 crate::cache_freshness::record_verify_completed(
5958 &canonical_root,
5959 crate::cache_freshness::VerifyArtifact::Search,
5960 seeded_generation,
5961 );
5962 assert!(matches!(
5963 crate::cache_freshness::warm_verify_plan(
5964 &canonical_root,
5965 crate::cache_freshness::VerifyArtifact::Search,
5966 seeded_generation,
5967 ),
5968 crate::cache_freshness::WarmVerifyPlan::Skip
5969 ));
5970 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5973 let _dispatch_tx = dispatch_tx;
5974 let shutdown = Arc::new(AtomicBool::new(false));
5975 let thread_shutdown = Arc::clone(&shutdown);
5976 let join = std::thread::spawn(move || {
5977 while !thread_shutdown.load(Ordering::SeqCst) {
5978 std::thread::yield_now();
5979 }
5980 });
5981 ctx.install_watcher_runtime(
5982 dispatch_rx,
5983 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5984 );
5985 assert!(ctx.watcher_runtime_active());
5986
5987 quiesce_unbound_root(&root, &mut live_roots, &executor);
5988 let meta = &live_roots[&root];
5989 assert!(meta.unbound_quiesced);
5990 assert!(ctx.subc_unbound_quiesced());
5991 assert!(meta.maintenance_pending);
5992 assert!(meta.maintenance_queued_kinds.is_empty());
5993 assert!(executor.actor_registered(&root));
5994 assert!(
5998 ctx.search_index()
5999 .read()
6000 .unwrap_or_else(std::sync::PoisonError::into_inner)
6001 .is_some(),
6002 "quiesce must not evict resident artifacts"
6003 );
6004 assert_eq!(
6005 ctx.pending_callgraph_store_force_token(),
6006 None,
6007 "quiesce must not force a callgraph rebuild"
6008 );
6009 assert_eq!(
6010 ctx.take_pending_search_index_paths(),
6011 vec![pending],
6012 "quiesce must retain pending watcher-derived paths"
6013 );
6014 assert!(
6015 matches!(
6016 crate::cache_freshness::warm_verify_plan(
6017 &canonical_root,
6018 crate::cache_freshness::VerifyArtifact::Search,
6019 seeded_generation,
6020 ),
6021 crate::cache_freshness::WarmVerifyPlan::Skip
6022 ),
6023 "quiesce must not invalidate the warm verify memo"
6024 );
6025 assert!(
6026 ctx.watcher_runtime_active(),
6027 "quiesce must not stop a running watcher"
6028 );
6029 ctx.stop_watcher_runtime();
6030
6031 let meta = live_roots.get_mut(&root).expect("root metadata");
6032 note_maintenance_completion(
6033 meta,
6034 Some(MaintenanceDrainKind::ConfigureTail),
6035 false,
6036 meta.unbound_quiesced,
6037 );
6038 assert!(!meta.maintenance_pending);
6039 assert!(meta.maintenance_queued_kinds.is_empty());
6040 }
6041
6042 #[test]
6043 fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
6044 let (_dir, root) = test_root("same-root-replacement");
6045 let route = route_key(7, 1);
6046 let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
6047 let root_channels = HashMap::new();
6048
6049 assert!(!route_removal_will_quiesce_root(
6050 &root,
6051 route,
6052 &installed_channels,
6053 false,
6054 Some(&root),
6055 ));
6056 assert!(route_removal_will_quiesce_root(
6057 &root,
6058 route,
6059 &installed_channels,
6060 false,
6061 None,
6062 ));
6063 assert!(!should_quiesce_removed_root(
6064 &root,
6065 &root_channels,
6066 false,
6067 Some(&root),
6068 ));
6069 assert!(should_quiesce_removed_root(
6070 &root,
6071 &root_channels,
6072 false,
6073 None,
6074 ));
6075 assert!(!should_quiesce_removed_root(
6076 &root,
6077 &root_channels,
6078 true,
6079 None,
6080 ));
6081 }
6082
6083 #[test]
6084 fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
6085 let (_root_dir, root) = test_root("unbound-root-route-count");
6086 let executor = Arc::new(Executor::new());
6087 assert!(executor.register_actor(root.clone(), test_ctx()));
6088 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6089 let mut root_channels = HashMap::from([(
6090 root.clone(),
6091 HashSet::from([route_key(7, 1), route_key(8, 1)]),
6092 )]);
6093
6094 remove_root_channel(&mut root_channels, &root, route_key(7, 1));
6095 if !root_channels.contains_key(&root) {
6096 quiesce_unbound_root(&root, &mut live_roots, &executor);
6097 }
6098 assert!(!live_roots[&root].unbound_quiesced);
6099
6100 remove_root_channel(&mut root_channels, &root, route_key(8, 1));
6101 if !root_channels.contains_key(&root) {
6102 quiesce_unbound_root(&root, &mut live_roots, &executor);
6103 }
6104 assert!(live_roots[&root].unbound_quiesced);
6105
6106 live_roots
6107 .get_mut(&root)
6108 .expect("root metadata")
6109 .note_activity();
6110 assert!(
6111 live_roots[&root].unbound_quiesced,
6112 "late asynchronous activity must not reactivate an unbound root"
6113 );
6114
6115 live_roots
6116 .get_mut(&root)
6117 .expect("root metadata")
6118 .reactivate_bound();
6119 assert!(!live_roots[&root].unbound_quiesced);
6120 }
6121
6122 #[test]
6123 fn allocator_pressure_relief_requires_every_root_to_be_idle() {
6124 let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
6125 let (_active_dir, active_root) = test_root("allocator-relief-active");
6126 let now = Instant::now();
6127 let mut live_roots = HashMap::new();
6128 let mut idle = RootMeta::new(now);
6129 idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
6130 live_roots.insert(idle_root, idle);
6131 assert!(process_has_been_idle(now, &live_roots));
6132
6133 live_roots.insert(active_root.clone(), RootMeta::new(now));
6134 assert!(!process_has_been_idle(now, &live_roots));
6135
6136 let active = live_roots
6137 .get_mut(&active_root)
6138 .expect("active root metadata");
6139 active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
6140 active.active_bash_waits = 1;
6141 assert!(!process_has_been_idle(now, &live_roots));
6142 }
6143
6144 #[test]
6145 fn pressure_relief_log_reports_before_and_after_measurements() {
6146 let allocator = crate::memory::AllocatorMemorySnapshot {
6147 status: "measured",
6148 bytes_in_use: Some(8 * 1024 * 1024),
6149 size_allocated: Some(12 * 1024 * 1024),
6150 retained_slack_bytes: Some(4 * 1024 * 1024),
6151 not_estimated: None,
6152 };
6153 let relief = crate::memory::AllocatorPressureRelief {
6154 bytes_released: 3 * 1024 * 1024,
6155 rss_before_bytes: Some(20 * 1024 * 1024),
6156 rss_after_bytes: Some(17 * 1024 * 1024),
6157 allocator_before: allocator.clone(),
6158 allocator_after: crate::memory::AllocatorMemorySnapshot {
6159 size_allocated: Some(9 * 1024 * 1024),
6160 retained_slack_bytes: Some(1024 * 1024),
6161 ..allocator
6162 },
6163 };
6164 let message = pressure_relief_label(&relief);
6165 assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
6166 assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
6167 assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
6168 assert!(message.contains("reported 3.0 MB released"));
6169 }
6170
6171 #[test]
6172 fn due_maintenance_jobs_skip_poisoned_roots() {
6173 let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
6174 let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
6175 let mut live_roots = HashMap::new();
6176 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6177 let mut poisoned_meta = RootMeta::new(Instant::now());
6178 poisoned_meta.maintenance_poisoned = true;
6179 live_roots.insert(poisoned_root.clone(), poisoned_meta);
6180
6181 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6182 &mut live_roots,
6183 MAINTENANCE_SUBMIT_BUDGET,
6184 &HashSet::new(),
6185 );
6186
6187 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6188 assert!(due.iter().all(|(root, _)| root == &healthy_root));
6189 assert!(!deferred);
6190 assert!(live_roots[&healthy_root].maintenance_pending);
6191 assert_eq!(
6192 live_roots[&healthy_root].maintenance_jobs_in_flight,
6193 INITIAL_MAINTENANCE_JOB_COUNT
6194 );
6195 assert!(!live_roots[&poisoned_root].maintenance_pending);
6196 }
6197
6198 #[test]
6199 fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
6200 let (_dir, root) = test_root("maintenance-unbound");
6201 let mut meta = RootMeta::new(Instant::now());
6202 meta.unbound_quiesced = true;
6203 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6204
6205 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6206 &mut live_roots,
6207 MAINTENANCE_SUBMIT_BUDGET,
6208 &HashSet::new(),
6209 );
6210
6211 assert!(due.is_empty());
6212 assert!(!deferred);
6213 assert!(!live_roots[&root].maintenance_pending);
6214 }
6215
6216 #[test]
6217 fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
6218 let (_dir, root) = test_root("maintenance-idle-bg-subscription");
6219 let ctx = test_ctx();
6220 assert!(!ctx.completion_drains_have_work());
6221
6222 let executor = Executor::new();
6223 assert!(executor.register_actor(root.clone(), ctx));
6224 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6225 let session = "idle-session".to_string();
6226 let channel = route_key(17, 1);
6227 let bg_sub_by_session = HashMap::from([((root.clone(), session.clone()), channel)]);
6228 let mut bg_wake_pending = HashSet::new();
6229
6230 let (idle_tick_jobs, deferred) = due_maintenance_jobs(
6231 &mut live_roots,
6232 Some(&executor),
6233 &bg_sub_by_session,
6234 &bg_wake_pending,
6235 MAINTENANCE_SUBMIT_BUDGET,
6236 &HashSet::new(),
6237 );
6238 assert!(idle_tick_jobs.is_empty());
6239 assert!(!deferred);
6240 assert!(!live_roots[&root].maintenance_pending);
6241
6242 let mut bg_wake_epoch = HashMap::new();
6245 push::arm_bg_wake(
6246 root.clone(),
6247 session,
6248 channel,
6249 &mut bg_wake_pending,
6250 &mut bg_wake_epoch,
6251 );
6252 let (next_tick_jobs, deferred) = due_maintenance_jobs(
6253 &mut live_roots,
6254 Some(&executor),
6255 &bg_sub_by_session,
6256 &bg_wake_pending,
6257 MAINTENANCE_SUBMIT_BUDGET,
6258 &HashSet::new(),
6259 );
6260 assert_eq!(
6261 next_tick_jobs,
6262 vec![(root, MaintenanceDrainKind::CompletionDrains)]
6263 );
6264 assert!(!deferred);
6265 }
6266
6267 #[tokio::test]
6268 async fn subc_configure_tail_precedes_completed_search_install() {
6269 let root_dir = tempfile::tempdir().unwrap();
6270 let storage = tempfile::tempdir().unwrap();
6271 let root = ProjectRootId::from_path(root_dir.path()).unwrap();
6272 let (ctx, ignored_path) =
6273 runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
6274 let ctx = Arc::new(ctx);
6275 assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
6276 &ctx,
6277 &ignored_path
6278 ));
6279
6280 let executor = Arc::new(Executor::new());
6281 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6282 let metrics = Arc::new(DispatchPathMetrics::new());
6283 let (completion_tx, mut completion_rx) = mpsc::channel(4);
6284 submit_maintenance_job(
6285 &executor,
6286 root.clone(),
6287 MaintenanceDrainKind::ConfigureTail,
6288 Vec::new(),
6289 &completion_tx,
6290 &metrics,
6291 );
6292 submit_maintenance_job(
6293 &executor,
6294 root,
6295 MaintenanceDrainKind::CompletionDrains,
6296 Vec::new(),
6297 &completion_tx,
6298 &metrics,
6299 );
6300
6301 let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6302 .await
6303 .expect("configure-tail completion timed out")
6304 .expect("configure-tail completion channel closed");
6305 let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6306 .await
6307 .expect("completion-drains completion timed out")
6308 .expect("completion-drains completion channel closed");
6309 assert!(first.response.id.contains("configure-tail"));
6310 assert!(second.response.id.contains("completion-drains"));
6311 assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
6312 &ctx,
6313 &ignored_path
6314 ));
6315 assert_eq!(
6316 ctx.search_index()
6317 .read()
6318 .unwrap_or_else(std::sync::PoisonError::into_inner)
6319 .as_ref()
6320 .expect("completed search index installed")
6321 .file_count(),
6322 0,
6323 "configure must install the ignore matcher before pending paths replay"
6324 );
6325 ctx.stop_watcher_runtime();
6326 }
6327
6328 #[test]
6329 fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
6330 let (_dir, root) = test_root("maintenance-post-bind");
6331 let mut live_roots = HashMap::new();
6332 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6333
6334 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6335 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6336
6337 let meta = live_roots.get(&root).expect("root metadata");
6338 assert!(meta.maintenance_pending);
6339 assert_eq!(meta.maintenance_jobs_in_flight, 0);
6340 assert_eq!(
6341 meta.maintenance_queued_kinds
6342 .iter()
6343 .copied()
6344 .collect::<Vec<_>>(),
6345 vec![
6346 MaintenanceDrainKind::ConfigureTail,
6347 MaintenanceDrainKind::CompletionDrains,
6348 ]
6349 );
6350
6351 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6352 &mut live_roots,
6353 MAINTENANCE_SUBMIT_BUDGET,
6354 &HashSet::new(),
6355 );
6356
6357 assert_eq!(
6358 due,
6359 vec![
6360 (root.clone(), MaintenanceDrainKind::ConfigureTail),
6361 (root.clone(), MaintenanceDrainKind::CompletionDrains),
6362 ]
6363 );
6364 assert!(!deferred);
6365 assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
6366 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
6367 }
6368
6369 #[test]
6370 fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
6371 let mut live_roots = HashMap::new();
6372 let mut root_ids = Vec::new();
6373 let mut _dirs = Vec::new();
6374 for index in 0..4 {
6375 let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
6376 live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
6377 root_ids.push(root_id);
6378 _dirs.push(dir);
6379 }
6380
6381 let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
6382 let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
6383 &mut live_roots,
6384 small_budget,
6385 &HashSet::new(),
6386 );
6387
6388 assert_eq!(first_due.len(), small_budget);
6389 assert!(first_deferred);
6390 let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
6391 assert!(first_due_set
6392 .iter()
6393 .all(|root| live_roots[root].maintenance_pending));
6394 assert!(first_due_set
6395 .iter()
6396 .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
6397
6398 let all_roots: HashSet<_> = root_ids.into_iter().collect();
6399 let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
6400 assert!(deferred_roots
6401 .iter()
6402 .all(|root| !live_roots[root].maintenance_pending));
6403 }
6404
6405 #[test]
6406 fn due_maintenance_jobs_defers_pending_bind_roots() {
6407 let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
6408 let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
6409 let mut live_roots = HashMap::new();
6410 live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
6411 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6412 let pending_bind_roots = HashSet::from([bind_root.clone()]);
6413
6414 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6415 &mut live_roots,
6416 usize::MAX,
6417 &pending_bind_roots,
6418 );
6419
6420 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6421 assert!(due.iter().all(|(root, _)| root == &healthy_root));
6422 assert!(!deferred);
6423 assert!(!live_roots[&bind_root].maintenance_pending);
6424 assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
6425 }
6426
6427 #[test]
6428 fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
6429 let (_dir, root) = test_root("maintenance-requeue");
6430 let mut live_roots = HashMap::new();
6431 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6432 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6433 &mut live_roots,
6434 usize::MAX,
6435 &HashSet::new(),
6436 );
6437 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6438 assert!(due.iter().all(|(due_root, _)| due_root == &root));
6439 assert!(!deferred);
6440
6441 let meta = live_roots.get_mut(&root).unwrap();
6442 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
6443 assert!(meta.maintenance_pending);
6444 assert_eq!(
6445 meta.maintenance_jobs_in_flight,
6446 INITIAL_MAINTENANCE_JOB_COUNT - 1
6447 );
6448 assert_eq!(meta.maintenance_queued_kinds.len(), 1);
6449
6450 let (requeued, deferred) =
6451 due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
6452 assert_eq!(
6453 requeued,
6454 vec![(root.clone(), MaintenanceDrainKind::Watcher)]
6455 );
6456 assert!(!deferred);
6457 let meta = live_roots.get_mut(&root).unwrap();
6458 assert_eq!(
6459 meta.maintenance_jobs_in_flight,
6460 INITIAL_MAINTENANCE_JOB_COUNT
6461 );
6462 assert!(meta.maintenance_queued_kinds.is_empty());
6463
6464 for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
6465 note_maintenance_completion(meta, None, false, false);
6466 }
6467 assert!(!meta.maintenance_pending);
6468 assert_eq!(meta.maintenance_jobs_in_flight, 0);
6469 }
6470
6471 #[test]
6472 fn maintenance_requeue_drops_while_bind_is_pending() {
6473 let (_dir, root) = test_root("maintenance-bind-requeue");
6474 let mut live_roots = HashMap::new();
6475 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6476 let (due, _) = due_maintenance_jobs_without_actor_context(
6477 &mut live_roots,
6478 usize::MAX,
6479 &HashSet::new(),
6480 );
6481 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6482
6483 let meta = live_roots.get_mut(&root).unwrap();
6484 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
6485
6486 assert_eq!(
6487 meta.maintenance_jobs_in_flight,
6488 INITIAL_MAINTENANCE_JOB_COUNT - 1
6489 );
6490 assert!(meta.maintenance_queued_kinds.is_empty());
6491 assert!(meta.maintenance_pending);
6492 }
6493
6494 #[test]
6495 fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
6496 let mut meta = RootMeta::new(Instant::now());
6497 meta.unbound_quiesced = true;
6498
6499 assert!(!should_requiesce_after_maintenance(
6500 &meta,
6501 MaintenanceDrainKind::Lsp,
6502 false,
6503 ));
6504 assert!(!should_requiesce_after_maintenance(
6505 &meta,
6506 MaintenanceDrainKind::ConfigureTail,
6507 true,
6508 ));
6509 assert!(should_requiesce_after_maintenance(
6510 &meta,
6511 MaintenanceDrainKind::ConfigureTail,
6512 false,
6513 ));
6514 }
6515
6516 #[test]
6517 fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
6518 let (_dir, root) = test_root("maintenance-fatal");
6519 let mut live_roots = HashMap::new();
6520 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6521 let (due, _) = due_maintenance_jobs_without_actor_context(
6522 &mut live_roots,
6523 usize::MAX,
6524 &HashSet::new(),
6525 );
6526 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6527
6528 let meta = live_roots.get_mut(&root).unwrap();
6529 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
6530 assert!(meta.maintenance_poisoned);
6531 assert!(meta.maintenance_queued_kinds.is_empty());
6532
6533 for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
6534 note_maintenance_completion(meta, None, false, false);
6535 }
6536 assert!(!meta.maintenance_pending);
6537 assert_eq!(meta.maintenance_jobs_in_flight, 0);
6538 }
6539
6540 #[test]
6541 fn trust_for_principal_matrix() {
6542 assert_eq!(
6543 trust_for_principal(&Some(Principal::Direct)),
6544 BindTrust::FirstParty
6545 );
6546 for module_id in [
6553 "llm-runner",
6554 "aft",
6555 "broca",
6556 "alfonso-core",
6557 "prefrontal",
6558 "prefrontal-core",
6559 ] {
6560 assert_eq!(
6561 trust_for_principal(&Some(Principal::Reserved {
6562 module_id: module_id.to_string(),
6563 })),
6564 BindTrust::FirstParty,
6565 "reserved module id '{module_id}' must resolve to first-party trust"
6566 );
6567 }
6568 assert_eq!(
6569 trust_for_principal(&Some(Principal::Reserved {
6570 module_id: "subc-mcp".to_string(),
6571 })),
6572 BindTrust::Untrusted
6573 );
6574 assert_eq!(
6575 trust_for_principal(&Some(Principal::Reserved {
6576 module_id: "anything-unknown".to_string(),
6577 })),
6578 BindTrust::Untrusted
6579 );
6580 assert_eq!(
6581 trust_for_principal(&Some(Principal::Unverified)),
6582 BindTrust::Untrusted
6583 );
6584 assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
6585 }
6586
6587 #[test]
6588 fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
6589 let principal = Some(Principal::Direct);
6590 let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
6591 let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
6592
6593 assert_eq!(
6594 trust_for_bind(fingerprint_a, &principal),
6595 BindTrust::Untrusted
6596 );
6597 assert_eq!(
6598 trust_for_bind(fingerprint_b, &principal),
6599 BindTrust::Untrusted
6600 );
6601 }
6602
6603 #[test]
6610 fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
6611 for harness in ["opencode", "pi", "runner", "mcp:claude"] {
6612 assert_eq!(
6613 trust_for_bind(harness, &Some(Principal::Direct)),
6614 BindTrust::FirstParty,
6615 "a direct principal must stay first-party on {harness}"
6616 );
6617 assert_eq!(
6618 trust_for_bind(harness, &Some(Principal::Unverified)),
6619 BindTrust::Untrusted,
6620 "an unverified principal must stay untrusted on {harness}"
6621 );
6622 assert_eq!(
6623 trust_for_bind(harness, &None),
6624 BindTrust::Untrusted,
6625 "an absent principal must fail closed on {harness}"
6626 );
6627 assert_eq!(
6628 trust_for_bind(
6629 harness,
6630 &Some(Principal::Reserved {
6631 module_id: "subc-mcp".to_string(),
6632 })
6633 ),
6634 BindTrust::Untrusted,
6635 "a non-allowlisted reserved module must stay untrusted on {harness}"
6636 );
6637 }
6638 }
6639
6640 #[tokio::test]
6641 async fn persistent_cancel_resolves_when_fired_before_await() {
6642 let signal = PersistentCancelSignal::new();
6646 signal.cancel();
6647 tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
6649 .await
6650 .expect("cancelled() must resolve when cancel fired beforehand");
6651
6652 let racing = PersistentCancelSignal::new();
6654 let racing_for_task = racing.clone();
6655 let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
6656 racing.cancel();
6657 tokio::time::timeout(Duration::from_secs(1), waiter)
6658 .await
6659 .expect("cancelled() must resolve when cancel races the await")
6660 .expect("waiter task panicked");
6661 }
6662
6663 #[test]
6664 fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
6665 let installed = HashMap::from([(7, 9)]);
6666 let mut reclaimed = ReclaimedRoutes::default();
6667 reclaimed.insert(route_key(8, 1));
6668 for ty in [
6669 FrameType::Request,
6670 FrameType::Response,
6671 FrameType::Error,
6672 FrameType::Push,
6673 FrameType::Cancel,
6674 FrameType::Goodbye,
6675 ] {
6676 let body = if ty.is_pure_header() {
6677 Vec::new()
6678 } else {
6679 br#"{}"#.to_vec()
6680 };
6681 let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
6682 assert!(
6683 !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
6684 "{ty:?}"
6685 );
6686 }
6687
6688 let reclaimed_request = Frame::build(
6689 FrameType::Request,
6690 control_flags(),
6691 8,
6692 1,
6693 42,
6694 br#"{}"#.to_vec(),
6695 )
6696 .unwrap();
6697 assert!(ingress_route_should_be_processed(
6698 &installed,
6699 &reclaimed,
6700 &reclaimed_request
6701 ));
6702
6703 let never_installed = Frame::build(
6704 FrameType::Request,
6705 control_flags(),
6706 9,
6707 1,
6708 43,
6709 br#"{}"#.to_vec(),
6710 )
6711 .unwrap();
6712 assert!(!ingress_route_should_be_processed(
6713 &installed,
6714 &reclaimed,
6715 &never_installed
6716 ));
6717
6718 let current = Frame::build(
6719 FrameType::Request,
6720 control_flags(),
6721 7,
6722 9,
6723 43,
6724 br#"{}"#.to_vec(),
6725 )
6726 .unwrap();
6727 let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
6728 assert!(ingress_route_should_be_processed(
6729 &installed, &reclaimed, ¤t
6730 ));
6731 assert!(ingress_route_should_be_processed(
6732 &installed, &reclaimed, &control
6733 ));
6734 assert_eq!(installed, HashMap::from([(7, 9)]));
6735 }
6736
6737 #[tokio::test]
6738 async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
6739 let (_dir, root) = test_root("route-bind-b2-ordering");
6740 let route = route_key(7, 3);
6741 let identity = RouteIdentity(Arc::new(RouteIdentityData {
6742 root: root.clone(),
6743 project_root: root.as_path().to_path_buf(),
6744 harness: "opencode".to_string(),
6745 session: "b2-session".to_string(),
6746 trust: BindTrust::FirstParty,
6747 spawn_principal: AuthenticatedPrincipal::FirstParty,
6748 consumer_elicitation_capable: false,
6749 }));
6750 let replay_key = push::ReplayKey::from_identity(&identity);
6751 let completion = RouteBindCompletion {
6752 route,
6753 identity,
6754 bind_root_id: root.clone(),
6755 inserted_new_actor: false,
6756 configure_response: Response::success("subc-bind-7", json!({})),
6757 diagnostics_on_edit: false,
6758 ver: PROTOCOL_VERSION,
6759 corr: 91,
6760 flags: control_flags(),
6761 };
6762 let mut pending_binds = HashMap::from([(
6763 route,
6764 PendingBind {
6765 bind_root_id: root,
6766 inserted_new_actor: false,
6767 cancelled: false,
6768 configure_request_id: "subc-bind-7".to_string(),
6769 started_at: Instant::now(),
6770 warned_half_deadline: false,
6771 deadline_reported: false,
6772 corr: 91,
6773 ver: PROTOCOL_VERSION,
6774 flags: control_flags(),
6775 cancellation: crate::executor::JobCancellation::new(),
6776 },
6777 )]);
6778 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6779 let mut push_buffer =
6780 HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
6781 let (writer_tx, mut writer_rx) = mpsc::channel(8);
6782 let metrics = Arc::new(DispatchPathMetrics::new());
6783
6784 handle_route_bind_completion(
6785 &writer_tx,
6786 completion,
6787 &mut HashMap::new(),
6788 &mut HashMap::new(),
6789 &mut HashMap::new(),
6790 &mut push_buffer,
6791 &mut HashMap::new(),
6792 &mut pending_binds,
6793 &mut installed_route_epochs,
6794 &Arc::new(Executor::new()),
6795 &Arc::new(Notify::new()),
6796 &metrics,
6797 )
6798 .await
6799 .unwrap();
6800
6801 let ack = writer_rx.try_recv().expect("RouteBindAck");
6802 assert_eq!(ack.header.ty, FrameType::Response);
6803 assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
6804 let route_frame = writer_rx.try_recv().expect("post-ack route frame");
6805 assert_eq!(route_frame.header.ty, FrameType::Push);
6806 assert_eq!(
6807 (route_frame.header.channel, route_frame.header.epoch),
6808 (route.channel, route.epoch)
6809 );
6810 }
6811}