1use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, Lane};
31use crate::jsonc::strip_jsonc;
32use crate::log_ctx;
33use crate::path_identity::ProjectRootId;
34use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
35use crate::run_tool_call::{
36 run_tool_call, strip_agent_preview_arg_owned, PhaseTrace, ToolCallContext, ToolCallOutcome,
37 ToolCallResult,
38};
39use crate::runtime_drain;
40use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
41
42use subc_protocol::manifest::{
43 Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
44 ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
45};
46use subc_protocol::session::{
47 HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
48 MODULE_CONTROL_OP_HEALTH_CHECK,
49};
50use subc_protocol::{
51 ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, PROTOCOL_VERSION,
52};
53use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
54use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
55use tokio::net::TcpStream;
56use tokio::sync::{mpsc, oneshot, Notify};
57use tokio::task::JoinHandle;
58
59const AUTH_DEADLINE: Duration = Duration::from_secs(5);
62const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
63const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
64const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
65const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
66
67const HELLO_CORR: u64 = 1;
69
70const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
73
74const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
78
79const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
83
84const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
88
89const WRITER_QUEUE_CAPACITY: usize = 256;
90
91const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
94
95const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
103const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
104 MaintenanceDrainKind::Watcher,
105 MaintenanceDrainKind::Lsp,
106 MaintenanceDrainKind::ConfigureTail,
107 MaintenanceDrainKind::CompletionDrains,
108];
109#[cfg(test)]
110const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
111
112const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
113const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
114
115const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
116const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
117
118const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
121
122const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
126
127const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
129const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
130
131#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
132struct RouteChannel {
133 channel: u16,
134 epoch: u32,
135}
136
137impl fmt::Display for RouteChannel {
138 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
139 write!(f, "{}@{}", self.channel, self.epoch)
140 }
141}
142
143type PushEnvelope = (ProjectRootId, PushFrame);
144type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
145type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
146mod bash;
147mod health;
148mod manifest;
149mod push;
150mod wire;
151
152use self::health::{
153 build_health_report, warn_slow_pending_binds, DispatchPathMetrics, ReapBlockerCensus,
154 ResponseTaskGuard,
155};
156use self::manifest::{
157 build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
158 is_subc_agent_core_tool, is_subc_native_plumbing_tool,
159};
160pub use self::wire::SubcError;
161
162pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
166 manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
167}
168use self::wire::{
169 build_error_frame, build_goodbye_frame, build_tool_response_frame, decrement_counted_channel,
170 response_is_fatal_panic, response_message, send_counted_channel, send_frame,
171 send_reliable_writer_frame, send_traced_tool_response_frame, ToolResponseWriteTrace,
172 WriterFrame, WriterSender,
173};
174
175struct DecodedFrame {
176 frame: Frame,
177 phase_trace: PhaseTrace,
178}
179
180struct ToolCallCompletion {
181 text: String,
182 phase_trace: PhaseTrace,
183}
184
185#[derive(Clone)]
186struct PushSenders {
187 lossy_tx: mpsc::Sender<LossyPushEnvelope>,
188 reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
189 lossy_overflow: Arc<push::LossyOverflow>,
190 lossy_seq: Arc<AtomicU64>,
191}
192
193#[derive(Clone)]
194struct PersistentCancelSignal {
195 inner: Arc<PersistentCancelInner>,
196}
197
198struct PersistentCancelInner {
199 cancelled: AtomicBool,
200 notify: Notify,
201}
202
203impl PersistentCancelSignal {
204 fn new() -> Self {
205 Self {
206 inner: Arc::new(PersistentCancelInner {
207 cancelled: AtomicBool::new(false),
208 notify: Notify::new(),
209 }),
210 }
211 }
212
213 fn cancel(&self) {
214 if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
215 self.inner.notify.notify_waiters();
216 }
217 }
218
219 fn is_cancelled(&self) -> bool {
220 self.inner.cancelled.load(Ordering::SeqCst)
221 }
222
223 async fn cancelled(&self) {
224 loop {
232 let notified = self.inner.notify.notified();
233 tokio::pin!(notified);
234 notified.as_mut().enable();
235 if self.is_cancelled() {
236 return;
237 }
238 notified.await;
239 }
240 }
241}
242
243#[derive(Debug, Clone, Copy, PartialEq, Eq)]
244pub(crate) enum BindTrust {
245 FirstParty,
246 Untrusted,
247}
248
249impl BindTrust {
250 fn allows_bash_observation(self) -> bool {
251 matches!(self, Self::FirstParty)
252 }
253
254 fn label(self) -> &'static str {
255 match self {
256 Self::FirstParty => "first_party",
257 Self::Untrusted => "untrusted",
258 }
259 }
260
261 fn sandbox_trust(self) -> PrincipalTrust {
262 match self {
263 Self::FirstParty => PrincipalTrust::FirstParty,
264 Self::Untrusted => PrincipalTrust::Untrusted,
265 }
266 }
267}
268
269pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
270 match principal {
271 Some(Principal::Direct) => BindTrust::FirstParty,
272 Some(Principal::Reserved { module_id })
273 if module_id == "llm-runner"
274 || module_id == "aft"
275 || module_id == "broca"
276 || module_id == "alfonso-core" =>
277 {
278 BindTrust::FirstParty
279 }
280 Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
281 BindTrust::Untrusted
282 }
283 }
284}
285
286fn harness_forces_untrusted(harness: &str) -> bool {
287 harness.starts_with("fed:")
288}
289
290pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
291 if harness_forces_untrusted(harness) {
292 BindTrust::Untrusted
293 } else {
294 trust_for_principal(principal)
295 }
296}
297
298fn principal_id(principal: &Option<Principal>) -> Option<String> {
299 match principal {
300 Some(Principal::Direct) => Some("direct".to_string()),
301 Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
302 Some(Principal::Unverified) => Some("unverified".to_string()),
303 None => None,
304 }
305}
306
307fn principal_label(principal: &Option<Principal>) -> String {
308 principal_id(principal).unwrap_or_else(|| "absent".to_string())
309}
310
311#[derive(Debug)]
312struct RootMeta {
318 maintenance_pending: bool,
319 maintenance_jobs_in_flight: usize,
320 maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
321 maintenance_last_submitted: Option<Instant>,
322 maintenance_poisoned: bool,
323 last_touched: Instant,
324 diagnostics_on_edit: bool,
325 active_bash_waits: usize,
326 idle_artifacts_evicted: bool,
327 unbound_quiesced: bool,
328}
329
330#[derive(Debug)]
331struct PendingBind {
332 bind_root_id: ProjectRootId,
333 inserted_new_actor: bool,
334 cancelled: bool,
335 configure_request_id: String,
336 started_at: Instant,
337 warned_half_deadline: bool,
338 deadline_reported: bool,
339 corr: u64,
340 ver: u8,
341 flags: Flags,
342 cancellation: crate::executor::JobCancellation,
347}
348
349struct RouteBindCompletion {
350 route: RouteChannel,
351 identity: RouteIdentity,
352 bind_root_id: ProjectRootId,
353 inserted_new_actor: bool,
354 configure_response: Response,
355 diagnostics_on_edit: bool,
356 ver: u8,
357 corr: u64,
358 flags: Flags,
359}
360
361#[derive(Debug, Clone)]
362struct RouteIdentity(Arc<RouteIdentityData>);
363
364#[derive(Debug)]
365struct RouteIdentityData {
366 root: ProjectRootId,
367 project_root: PathBuf,
368 harness: String,
369 session: String,
370 trust: BindTrust,
371 spawn_principal: AuthenticatedPrincipal,
372 consumer_elicitation_capable: bool,
373}
374
375impl Deref for RouteIdentity {
376 type Target = RouteIdentityData;
377
378 fn deref(&self) -> &Self::Target {
379 &self.0
380 }
381}
382
383#[derive(Debug, Clone)]
384struct RetainedSessionIdentity {
385 harness: String,
386 trust: BindTrust,
387}
388
389#[derive(Clone, Copy)]
390struct BgSub {
391 corr: u64,
392 ver: u8,
393 flags: Flags,
394}
395
396struct MaintenanceCompletion {
397 root_id: ProjectRootId,
398 kind: MaintenanceDrainKind,
399 response: Response,
400 empty_bg_sessions: Vec<(String, u64)>,
401 requeue_kind: Option<MaintenanceDrainKind>,
402}
403
404#[derive(Clone, Copy, Debug, PartialEq, Eq)]
405enum MaintenanceDrainKind {
406 Watcher,
407 Lsp,
408 ConfigureTail,
409 CompletionDrains,
410}
411
412impl MaintenanceDrainKind {
413 fn label(self) -> &'static str {
414 match self {
415 Self::Watcher => "watcher",
416 Self::Lsp => "lsp",
417 Self::ConfigureTail => "configure-tail",
418 Self::CompletionDrains => "completion-drains",
419 }
420 }
421}
422
423#[derive(Debug, Default)]
424struct MaintenanceJobOutcome {
425 empty_bg_sessions: Vec<(String, u64)>,
426 requeue_kind: Option<MaintenanceDrainKind>,
427}
428
429#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
430struct ReverseCorrKey {
431 route: RouteChannel,
432 corr: u64,
433}
434
435struct PendingBashAsk {
436 route: RouteChannel,
437 tool_corr: u64,
438 tool_flags: Flags,
439 tool_ver: u8,
440 root: ProjectRootId,
441 project_root: PathBuf,
442 session_id: String,
443 spawn_principal: AuthenticatedPrincipal,
444 request_id: String,
445 arguments: Value,
446 format_context: crate::subc_format::FormatContext,
447 cancel: bash::BashWaitCancel,
448 grants: Vec<String>,
449 expires_at: Instant,
450}
451
452impl RootMeta {
453 fn new(now: Instant) -> Self {
454 Self {
455 maintenance_pending: false,
456 maintenance_jobs_in_flight: 0,
457 maintenance_queued_kinds: VecDeque::new(),
458 maintenance_last_submitted: None,
459 maintenance_poisoned: false,
460 last_touched: now,
461 diagnostics_on_edit: false,
462 active_bash_waits: 0,
463 idle_artifacts_evicted: false,
464 unbound_quiesced: false,
465 }
466 }
467
468 fn note_activity(&mut self) {
469 self.last_touched = Instant::now();
470 }
471
472 fn reactivate_bound(&mut self) {
473 self.note_activity();
474 self.idle_artifacts_evicted = false;
475 self.unbound_quiesced = false;
476 }
477}
478
479fn due_maintenance_jobs(
480 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
481 executor: Option<&Executor>,
482 bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
483 bg_wake_pending: &HashSet<RouteChannel>,
484 budget: usize,
485 pending_bind_roots: &HashSet<ProjectRootId>,
486) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
487 let mut jobs = Vec::new();
488 let mut deferred = false;
489 let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
490 roots.sort_by(|left, right| {
491 let left_last = live_roots
492 .get(left)
493 .and_then(|meta| meta.maintenance_last_submitted);
494 let right_last = live_roots
495 .get(right)
496 .and_then(|meta| meta.maintenance_last_submitted);
497 left_last
498 .cmp(&right_last)
499 .then_with(|| left.as_path().cmp(right.as_path()))
500 });
501
502 for root_id in roots {
503 let Some(meta) = live_roots.get_mut(&root_id) else {
504 continue;
505 };
506 if meta.maintenance_poisoned {
507 continue;
508 }
509
510 if pending_bind_roots.contains(&root_id) {
511 if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
512 deferred = true;
513 }
514 continue;
515 }
516
517 if !meta.maintenance_pending {
518 if jobs.len() >= budget {
519 deferred = true;
520 continue;
521 }
522 let executor_actor_context =
526 executor.and_then(|executor| executor.actor_context(&root_id));
527 let root_has_pending_bg_wake =
528 bg_sub_by_session.iter().any(|((sub_root, _), channel)| {
529 sub_root == &root_id && bg_wake_pending.contains(channel)
530 });
531 let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
532 Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
533 .into_iter()
534 .filter(|kind| {
535 if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
536 return false;
537 }
538 match kind {
539 MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
540 MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
541 MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
542 MaintenanceDrainKind::CompletionDrains => {
548 root_has_pending_bg_wake || ctx.completion_drains_have_work()
549 }
550 }
551 })
552 .collect(),
553 None if meta.unbound_quiesced => Vec::new(),
554 None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
556 };
557 if kinds_with_work.is_empty() {
558 continue;
559 }
560 meta.maintenance_pending = true;
561 meta.maintenance_queued_kinds.extend(kinds_with_work);
562 }
563
564 while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
565 if jobs.len() >= budget {
566 meta.maintenance_queued_kinds.push_front(kind);
567 deferred = true;
568 break;
569 }
570 meta.maintenance_jobs_in_flight += 1;
571 meta.maintenance_last_submitted = Some(Instant::now());
572 jobs.push((root_id.clone(), kind));
573 }
574
575 meta.maintenance_pending =
576 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
577 }
578
579 (jobs, deferred)
580}
581
582fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
583 match estimate.estimated_bytes {
584 Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
585 None if estimate.status == "busy" => "busy".to_string(),
586 None => "not estimated".to_string(),
587 }
588}
589
590fn optional_memory_label(bytes: Option<u64>) -> String {
591 bytes.map_or_else(
592 || "not estimated".to_string(),
593 |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
594 )
595}
596
597fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
598 format!(
599 "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
600 optional_memory_label(relief.rss_before_bytes),
601 optional_memory_label(relief.rss_after_bytes),
602 optional_memory_label(relief.allocator_before.bytes_in_use),
603 optional_memory_label(relief.allocator_after.bytes_in_use),
604 optional_memory_label(relief.allocator_before.size_allocated),
605 optional_memory_label(relief.allocator_after.size_allocated),
606 optional_memory_label(relief.allocator_before.retained_slack_bytes),
607 optional_memory_label(relief.allocator_after.retained_slack_bytes),
608 relief.bytes_released as f64 / (1024.0 * 1024.0),
609 )
610}
611
612fn idle_root_eviction_message(
613 root_id: &ProjectRootId,
614 memory: &crate::memory::RootMemorySnapshot,
615 pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
616) -> String {
617 let freed_bytes = [
620 &memory.semantic,
621 &memory.trigram,
622 &memory.symbols,
623 &memory.callgraph,
624 &memory.inspect,
625 ]
626 .iter()
627 .filter_map(|estimate| estimate.estimated_bytes)
628 .fold(0u64, u64::saturating_add);
629 let mut message = format!(
630 "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
631 root_id.as_path().display(),
632 freed_bytes as f64 / (1024.0 * 1024.0),
633 eviction_estimate_label(&memory.semantic),
634 eviction_estimate_label(&memory.trigram),
635 eviction_estimate_label(&memory.symbols),
636 eviction_estimate_label(&memory.callgraph),
637 eviction_estimate_label(&memory.inspect),
638 eviction_estimate_label(&memory.bash),
639 eviction_estimate_label(&memory.lsp),
640 eviction_estimate_label(&memory.parser_pool),
641 );
642 if let Some(pressure_relief) = pressure_relief {
643 message.push_str(&pressure_relief_label(pressure_relief));
644 }
645 message
646}
647
648fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
649 !live_roots.is_empty()
650 && live_roots.values().all(|meta| {
651 now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
652 && meta.active_bash_waits == 0
653 && !meta.maintenance_pending
654 && meta.maintenance_queued_kinds.is_empty()
655 })
656}
657
658fn allocator_pressure_relief_after_idle_sweep(
659 now: Instant,
660 live_roots: &HashMap<ProjectRootId, RootMeta>,
661 executor: &Executor,
662) -> Option<crate::memory::AllocatorPressureRelief> {
663 if !process_has_been_idle(now, live_roots)
664 || live_roots.keys().any(|root_id| {
665 executor
666 .actor_context(root_id)
667 .is_some_and(|ctx| ctx.artifact_eviction_blocked())
668 })
669 {
670 return None;
671 }
672
673 #[cfg(target_os = "macos")]
674 {
675 Some(crate::memory::relieve_allocator_pressure())
676 }
677 #[cfg(not(target_os = "macos"))]
678 {
679 None
680 }
681}
682
683fn quiesce_unbound_root(
684 root_id: &ProjectRootId,
685 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
686 executor: &Arc<Executor>,
687) {
688 let Some(meta) = live_roots.get_mut(root_id) else {
689 return;
690 };
691
692 let ctx = executor.actor_context(root_id);
693 if let Some(ctx) = ctx.as_ref() {
694 ctx.mark_subc_unbound();
698 }
699 let cancelled = executor.cancel_queued_maintenance(root_id);
700 let discarded = ctx
708 .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
709 .unwrap_or(0);
710 meta.unbound_quiesced = true;
711 meta.maintenance_queued_kinds.clear();
712 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
713 log::debug!(
714 "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s))",
715 root_id.as_path().display(),
716 cancelled,
717 discarded
718 );
719}
720
721#[allow(clippy::too_many_arguments)]
722fn quiesce_connection_roots(
723 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
724 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
725 routes: &mut HashMap<RouteChannel, RouteIdentity>,
726 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
727 installed_route_epochs: &mut HashMap<u16, u32>,
728 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
729 executor: &Arc<Executor>,
730) {
731 for cancel in route_bash_cancels.values() {
732 cancel.token.cancel();
733 }
734 route_bash_cancels.clear();
735
736 let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
737 for pending in pending_binds.values_mut() {
738 pending.cancelled = true;
739 roots.insert(pending.bind_root_id.clone());
740 let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
741 }
742
743 for root_id in roots {
747 if live_roots.contains_key(&root_id) {
748 quiesce_unbound_root(&root_id, live_roots, executor);
749 } else if let Some(ctx) = executor.actor_context(&root_id) {
750 ctx.mark_subc_unbound();
751 executor.cancel_queued_maintenance(&root_id);
752 crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
753 }
754 }
755
756 routes.clear();
757 root_channels.clear();
758 installed_route_epochs.clear();
759}
760
761#[derive(Debug, Default)]
762struct IdleReapOutcome {
763 evicted: usize,
764 forgotten_deleted_roots: Vec<ProjectRootId>,
765}
766
767fn reap_idle_roots(
768 now: Instant,
769 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
770 pending_binds: &HashMap<RouteChannel, PendingBind>,
771 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
772 executor: &Arc<Executor>,
773 metrics: &DispatchPathMetrics,
774) -> IdleReapOutcome {
775 let pending_bind_roots = pending_binds
776 .values()
777 .map(|pending| pending.bind_root_id.clone())
778 .collect::<HashSet<_>>();
779 let mut census = ReapBlockerCensus::default();
780 let mut candidates = Vec::new();
781
782 for (root_id, meta) in live_roots.iter() {
783 let deleted = !root_id.as_path().exists();
784 let has_bound_route = root_channels
785 .get(root_id)
786 .is_some_and(|channels| !channels.is_empty());
787 let has_pending_bind = pending_bind_roots.contains(root_id);
788
789 if deleted {
790 let mut retained = false;
791 if has_bound_route {
792 census.bound_routes += 1;
793 retained = true;
794 }
795 if !meta.unbound_quiesced {
797 census.unbound_quiesced += 1;
798 retained = true;
799 }
800 if meta.active_bash_waits > 0 {
801 census.bash_waits += 1;
802 retained = true;
803 }
804 if meta.maintenance_pending {
805 census.maintenance_pending += 1;
806 retained = true;
807 }
808 if !meta.maintenance_queued_kinds.is_empty() {
809 census.maintenance_queued += 1;
810 retained = true;
811 }
812 if has_pending_bind {
813 census.pending_binds += 1;
814 retained = true;
815 }
816 match executor.try_actor_is_idle(root_id) {
817 Some(true) => {}
818 Some(false) => {
819 census.actor_busy += 1;
820 retained = true;
821 }
822 None => {
823 census.actor_state_busy += 1;
824 retained = true;
825 }
826 }
827 if retained {
828 census.deleted_retained += 1;
829 continue;
830 }
831 } else {
832 if has_bound_route
836 || !meta.unbound_quiesced
837 || meta.idle_artifacts_evicted
838 || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
839 || meta.active_bash_waits > 0
840 || meta.maintenance_pending
841 || !meta.maintenance_queued_kinds.is_empty()
842 || has_pending_bind
843 || !executor.actor_is_idle(root_id)
844 {
845 continue;
846 }
847 }
848 candidates.push((root_id.clone(), deleted));
849 }
850
851 let mut reaped = Vec::new();
852 let mut forgotten_deleted_roots = Vec::new();
853 for (root_id, deleted) in candidates {
854 let Some(ctx) = executor.actor_context(&root_id) else {
855 if deleted {
856 census.deleted_retained += 1;
857 census.actor_busy += 1;
858 }
859 continue;
860 };
861 let taken_pending = Some(ctx.take_pending_reconciliation_state());
866 if ctx.artifact_eviction_blocked() {
867 if let Some(pending) = taken_pending {
868 ctx.restore_pending_reconciliation_state(pending);
869 }
870 if deleted {
871 census.deleted_retained += 1;
872 census.artifact_eviction_blocked += 1;
873 }
874 continue;
875 }
876 let memory_before = ctx.memory_root_snapshot();
877 if !ctx.evict_idle_artifacts() {
878 if let Some(pending) = taken_pending {
879 ctx.restore_pending_reconciliation_state(pending);
880 }
881 if deleted {
882 census.deleted_retained += 1;
883 census.artifact_eviction_failed += 1;
884 }
885 continue;
886 }
887 drop(taken_pending);
888 ctx.stop_watcher_runtime_in_background();
889 ctx.invalidate_artifacts_after_watcher_gap();
892
893 if deleted {
894 if executor.retire_idle_actor_in_background(&root_id) {
895 live_roots.remove(&root_id);
896 forgotten_deleted_roots.push(root_id.clone());
897 } else {
898 census.deleted_retained += 1;
899 census.actor_busy += 1;
900 }
901 } else {
902 if let Some(meta) = live_roots.get_mut(&root_id) {
903 meta.idle_artifacts_evicted = true;
904 }
905 ctx.release_idle_reopenable_resources_in_background();
906 }
907 reaped.push((root_id, memory_before));
908 }
909
910 metrics.record_reap(census);
911 if census.deleted_retained > 0 {
912 log::info!(
913 "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
914 census.deleted_retained,
915 census.blocker_histogram()
916 );
917 }
918
919 let pressure_relief = (!reaped.is_empty())
920 .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
921 .flatten();
922 for (root_id, memory_before) in &reaped {
923 log::info!(
924 "{}",
925 idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
926 );
927 }
928 IdleReapOutcome {
929 evicted: reaped.len(),
930 forgotten_deleted_roots,
931 }
932}
933
934#[allow(clippy::too_many_arguments)]
935fn purge_deleted_root_residents(
936 root_id: &ProjectRootId,
937 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
938 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
939 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
940 bg_subs: &mut HashMap<RouteChannel, BgSub>,
941 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
942 bg_wake_pending: &mut HashSet<RouteChannel>,
943 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
944 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
945) {
946 root_channels.remove(root_id);
947 session_identity.retain(|(root, _), _| root != root_id);
948 push_buffer.retain(|key, _| &key.root != root_id);
949 bg_wake_epoch.retain(|(root, _), _| root != root_id);
950 pending_bash_asks.retain(|_, ask| &ask.root != root_id);
951
952 let stale_subscriptions = bg_sub_by_session
953 .iter()
954 .filter_map(|((root, _), channel)| (root == root_id).then_some(*channel))
955 .collect::<Vec<_>>();
956 bg_sub_by_session.retain(|(root, _), _| root != root_id);
957 for channel in stale_subscriptions {
958 bg_subs.remove(&channel);
959 bg_wake_pending.remove(&channel);
960 }
961 log::info!(
962 "subc attach: fully forgot deleted root {}",
963 root_id.as_path().display()
964 );
965}
966
967#[allow(clippy::too_many_arguments)]
968fn submit_due_maintenance_jobs(
969 executor: &Arc<Executor>,
970 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
971 pending_binds: &HashMap<RouteChannel, PendingBind>,
972 bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
973 bg_wake_pending: &HashSet<RouteChannel>,
974 bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
975 maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
976 metrics: &Arc<DispatchPathMetrics>,
977) {
978 let pending_bind_roots = pending_binds
979 .values()
980 .map(|pending| pending.bind_root_id.clone())
981 .collect::<HashSet<_>>();
982 let (due_jobs, deferred_jobs) = due_maintenance_jobs(
983 live_roots,
984 Some(executor),
985 bg_sub_by_session,
986 bg_wake_pending,
987 MAINTENANCE_SUBMIT_BUDGET,
988 &pending_bind_roots,
989 );
990 if deferred_jobs {
991 metrics
992 .maintenance_budget_deferrals
993 .fetch_add(1, Ordering::Relaxed);
994 }
995 for (root_id, kind) in due_jobs {
996 let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
997 bg_sub_by_session
998 .iter()
999 .filter_map(|((root, session), _)| {
1000 if root == &root_id {
1001 Some((
1002 session.clone(),
1003 bg_wake_epoch
1004 .get(&(root_id.clone(), session.clone()))
1005 .copied()
1006 .unwrap_or(0),
1007 ))
1008 } else {
1009 None
1010 }
1011 })
1012 .collect()
1013 } else {
1014 Vec::new()
1015 };
1016 submit_maintenance_job(
1017 executor,
1018 root_id,
1019 kind,
1020 bg_sessions_to_check,
1021 maintenance_tx,
1022 metrics,
1023 );
1024 }
1025}
1026
1027fn should_requiesce_after_maintenance(
1028 meta: &RootMeta,
1029 completed_kind: MaintenanceDrainKind,
1030 bind_pending: bool,
1031) -> bool {
1032 meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1033}
1034
1035fn note_maintenance_completion(
1036 meta: &mut RootMeta,
1037 requeue_kind: Option<MaintenanceDrainKind>,
1038 fatal: bool,
1039 defer_requeue: bool,
1040) {
1041 if fatal {
1042 meta.maintenance_poisoned = true;
1043 }
1044
1045 if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1046 meta.maintenance_queued_kinds.push_back(kind);
1047 }
1048
1049 meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1050 meta.maintenance_pending =
1051 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1052}
1053
1054fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1055 RouteChannel { channel, epoch }
1056}
1057
1058fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1059 if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1060 installed_epochs.remove(&route.channel);
1061 }
1062}
1063
1064fn ingress_route_is_current(installed_epochs: &HashMap<u16, u32>, frame: &Frame) -> bool {
1065 frame.header.channel == 0
1066 || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1067}
1068
1069fn bash_elicitation_timeout() -> Duration {
1070 if cfg!(debug_assertions) {
1071 if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1072 if let Ok(ms) = raw.parse::<u64>() {
1073 if ms > 0 {
1074 return Duration::from_millis(ms);
1075 }
1076 }
1077 }
1078 }
1079 BASH_ELICITATION_TIMEOUT
1080}
1081
1082fn allocate_reverse_corr(
1083 pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1084 route: RouteChannel,
1085 next_corr: &mut u64,
1086) -> u64 {
1087 loop {
1088 let corr = *next_corr;
1089 *next_corr = (*next_corr).wrapping_add(1).max(1);
1090 if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1091 return corr;
1092 }
1093 }
1094}
1095
1096fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1097 match kind {
1098 crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1099 crate::bash_permissions::PermissionKind::Bash => "bash",
1100 }
1101}
1102
1103fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1104 let mut patterns = Vec::new();
1105 let mut seen = HashSet::new();
1106 for ask in asks {
1107 for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1108 if seen.insert(pattern.clone()) {
1109 patterns.push(pattern.clone());
1110 }
1111 }
1112 }
1113 patterns
1114}
1115
1116fn bash_elicitation_message(
1117 command: &str,
1118 asks: &[crate::bash_permissions::PermissionAsk],
1119) -> String {
1120 let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1121 let patterns = bash_elicitation_patterns(asks);
1122 let pattern_text = if patterns.is_empty() {
1123 "no matched permission patterns".to_string()
1124 } else {
1125 patterns.join(", ")
1126 };
1127 let ask_kinds = asks
1128 .iter()
1129 .map(|ask| bash_permission_kind_label(&ask.kind))
1130 .collect::<HashSet<_>>()
1131 .into_iter()
1132 .collect::<Vec<_>>()
1133 .join(", ");
1134 if ask_kinds.is_empty() {
1135 format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1136 } else {
1137 format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1138 }
1139}
1140
1141fn bash_elicitation_request_body(
1142 command: &str,
1143 asks: &[crate::bash_permissions::PermissionAsk],
1144) -> Value {
1145 json!({
1146 "method": BASH_ELICITATION_CREATE_METHOD,
1147 "params": {
1148 "mode": "form",
1149 "message": bash_elicitation_message(command, asks),
1150 "requestedSchema": {
1151 "type": "object",
1152 "properties": {
1153 "decision": {
1154 "type": "string",
1155 "enum": ["allow", "deny"],
1156 "description": "Choose allow to run this bash command once, or deny to block it."
1157 }
1158 },
1159 "required": ["decision"],
1160 "additionalProperties": false
1161 },
1162 "_meta": {
1163 "aft": {
1164 "tool": "bash",
1165 "command": command,
1166 "asks": asks
1167 }
1168 }
1169 }
1170 })
1171}
1172
1173fn build_bash_elicitation_request_frame(
1174 ver: u8,
1175 route: RouteChannel,
1176 corr: u64,
1177 flags: Flags,
1178 command: &str,
1179 asks: &[crate::bash_permissions::PermissionAsk],
1180) -> Result<Frame, SubcError> {
1181 let body = bash_elicitation_request_body(command, asks);
1182 Frame::build_with_version(
1183 ver,
1184 FrameType::Request,
1185 flags,
1186 route.channel,
1187 route.epoch,
1188 corr,
1189 serde_json::to_vec(&body).map_err(SubcError::Json)?,
1190 )
1191 .map_err(SubcError::FrameBuild)
1192}
1193
1194fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1195 let Ok(value) = serde_json::from_slice::<Value>(body) else {
1196 return false;
1197 };
1198 flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1199}
1200
1201fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1202 let Some(object) = value.as_object() else {
1203 return false;
1204 };
1205 object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1206}
1207
1208fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1209 let Some(object) = value.as_object() else {
1210 return false;
1211 };
1212 if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1213 return false;
1214 }
1215 let Some(content) = object.get("content").and_then(Value::as_object) else {
1216 return false;
1217 };
1218 content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1219}
1220
1221#[allow(clippy::too_many_arguments)]
1222async fn settle_pending_bash_ask_denied(
1223 tx: &WriterSender,
1224 pending: PendingBashAsk,
1225 routes: &HashMap<RouteChannel, RouteIdentity>,
1226 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1227 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1228 shutdown: &Arc<Notify>,
1229 metrics: &DispatchPathMetrics,
1230) -> Result<(), SubcError> {
1231 let completion = bash::bash_denied_untrusted_completion(
1232 pending.route,
1233 pending.tool_corr,
1234 pending.tool_flags,
1235 pending.tool_ver,
1236 pending.root,
1237 pending.request_id,
1238 pending.format_context,
1239 );
1240 bash::handle_bash_deferred_completion(
1241 tx,
1242 completion,
1243 routes,
1244 live_roots,
1245 route_bash_cancels,
1246 shutdown,
1247 metrics,
1248 )
1249 .await
1250}
1251
1252fn take_pending_bash_asks_for_route(
1253 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1254 route: RouteChannel,
1255) -> Vec<PendingBashAsk> {
1256 let keys = pending_bash_asks
1257 .keys()
1258 .copied()
1259 .filter(|key| key.route == route)
1260 .collect::<Vec<_>>();
1261 keys.into_iter()
1262 .filter_map(|key| pending_bash_asks.remove(&key))
1263 .collect()
1264}
1265
1266#[allow(clippy::too_many_arguments)]
1267async fn settle_pending_bash_asks_for_route(
1268 tx: &WriterSender,
1269 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1270 route: RouteChannel,
1271 routes: &HashMap<RouteChannel, RouteIdentity>,
1272 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1273 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1274 shutdown: &Arc<Notify>,
1275 metrics: &DispatchPathMetrics,
1276) -> Result<(), SubcError> {
1277 for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1278 settle_pending_bash_ask_denied(
1279 tx,
1280 pending,
1281 routes,
1282 live_roots,
1283 route_bash_cancels,
1284 shutdown,
1285 metrics,
1286 )
1287 .await?;
1288 }
1289 Ok(())
1290}
1291
1292#[allow(clippy::too_many_arguments)]
1293async fn settle_all_pending_bash_asks(
1294 tx: &WriterSender,
1295 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1296 routes: &HashMap<RouteChannel, RouteIdentity>,
1297 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1298 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1299 shutdown: &Arc<Notify>,
1300 metrics: &DispatchPathMetrics,
1301) -> Result<(), SubcError> {
1302 let pending = pending_bash_asks
1303 .drain()
1304 .map(|(_, pending)| pending)
1305 .collect::<Vec<_>>();
1306 for pending in pending {
1307 settle_pending_bash_ask_denied(
1308 tx,
1309 pending,
1310 routes,
1311 live_roots,
1312 route_bash_cancels,
1313 shutdown,
1314 metrics,
1315 )
1316 .await?;
1317 }
1318 Ok(())
1319}
1320
1321#[allow(clippy::too_many_arguments)]
1322async fn expire_pending_bash_asks(
1323 tx: &WriterSender,
1324 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1325 routes: &HashMap<RouteChannel, RouteIdentity>,
1326 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1327 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1328 shutdown: &Arc<Notify>,
1329 metrics: &DispatchPathMetrics,
1330) -> Result<(), SubcError> {
1331 let now = Instant::now();
1332 let expired = pending_bash_asks
1333 .iter()
1334 .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1335 .collect::<Vec<_>>();
1336 for key in expired {
1337 if let Some(pending) = pending_bash_asks.remove(&key) {
1338 log::debug!(
1339 "subc attach: bash elicitation request {} on route {} expired fail-closed",
1340 key.corr,
1341 pending.route
1342 );
1343 settle_pending_bash_ask_denied(
1344 tx,
1345 pending,
1346 routes,
1347 live_roots,
1348 route_bash_cancels,
1349 shutdown,
1350 metrics,
1351 )
1352 .await?;
1353 }
1354 }
1355 Ok(())
1356}
1357
1358#[allow(clippy::too_many_arguments)]
1359async fn handle_bash_elicitation_reply(
1360 tx: &WriterSender,
1361 frame: &Frame,
1362 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1363 routes: &HashMap<RouteChannel, RouteIdentity>,
1364 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1365 executor: &Arc<Executor>,
1366 shutdown: &Arc<Notify>,
1367 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1368 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1369 metrics: &Arc<DispatchPathMetrics>,
1370 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1371 dispatch: DispatchFn,
1372) -> Result<(), SubcError> {
1373 let key = ReverseCorrKey {
1374 route: route_key(frame.header.channel, frame.header.epoch),
1375 corr: frame.header.corr,
1376 };
1377 let Some(pending) = pending_bash_asks.remove(&key) else {
1378 return Ok(());
1379 };
1380
1381 if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1382 if routes.contains_key(&key.route) {
1383 bash::submit_deferred_bash(
1384 executor,
1385 bash_deferred_tx,
1386 bash_poll_touch_tx,
1387 metrics,
1388 dispatch,
1389 pending.root,
1390 pending.project_root,
1391 pending.session_id,
1392 pending.request_id,
1393 pending.route,
1394 pending.tool_corr,
1395 pending.tool_flags,
1396 pending.tool_ver,
1397 pending.arguments,
1398 pending.format_context,
1399 pending.cancel,
1400 BindTrust::Untrusted,
1401 pending.spawn_principal,
1402 Some(pending.grants),
1403 );
1404 return Ok(());
1405 }
1406 log::debug!(
1407 "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1408 key.corr,
1409 pending.route
1410 );
1411 }
1412
1413 settle_pending_bash_ask_denied(
1414 tx,
1415 pending,
1416 routes,
1417 live_roots,
1418 route_bash_cancels,
1419 shutdown,
1420 metrics,
1421 )
1422 .await
1423}
1424
1425#[allow(clippy::too_many_arguments)]
1426async fn cancel_pending_bash_ask_for_tool_call(
1427 tx: &WriterSender,
1428 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1429 route: RouteChannel,
1430 tool_corr: u64,
1431 routes: &HashMap<RouteChannel, RouteIdentity>,
1432 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1433 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1434 shutdown: &Arc<Notify>,
1435 metrics: &DispatchPathMetrics,
1436) -> Result<(), SubcError> {
1437 let keys = pending_bash_asks
1438 .iter()
1439 .filter_map(|(key, pending)| {
1440 (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1441 })
1442 .collect::<Vec<_>>();
1443 for key in keys {
1444 if let Some(pending) = pending_bash_asks.remove(&key) {
1445 settle_pending_bash_ask_denied(
1446 tx,
1447 pending,
1448 routes,
1449 live_roots,
1450 route_bash_cancels,
1451 shutdown,
1452 metrics,
1453 )
1454 .await?;
1455 }
1456 }
1457 Ok(())
1458}
1459
1460fn remove_root_channel(
1461 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1462 root: &ProjectRootId,
1463 channel: RouteChannel,
1464) {
1465 let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1466 channels.remove(&channel);
1467 channels.is_empty()
1468 } else {
1469 false
1470 };
1471 if remove_root {
1472 root_channels.remove(root);
1473 }
1474}
1475
1476fn remove_route_channel(
1477 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1478 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1479 channel: RouteChannel,
1480) -> Option<RouteIdentity> {
1481 let removed = routes.remove(&channel);
1482 if let Some(identity) = &removed {
1483 remove_root_channel(root_channels, &identity.root, channel);
1484 }
1485 removed
1486}
1487
1488fn insert_route_channel(
1489 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1490 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1491 channel: RouteChannel,
1492 identity: RouteIdentity,
1493) {
1494 if let Some(previous) = routes.insert(channel, identity.clone()) {
1495 remove_root_channel(root_channels, &previous.root, channel);
1496 }
1497 root_channels
1498 .entry(identity.root.clone())
1499 .or_default()
1500 .insert(channel);
1501}
1502
1503fn remove_bg_subscription_index(
1504 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1505 channel: RouteChannel,
1506 identity: Option<&RouteIdentity>,
1507) {
1508 if let Some(identity) = identity {
1509 let key = (identity.root.clone(), identity.session.clone());
1510 if bg_sub_by_session.get(&key).copied() == Some(channel) {
1511 bg_sub_by_session.remove(&key);
1512 }
1513 } else {
1514 bg_sub_by_session.retain(|_, mapped_channel| *mapped_channel != channel);
1515 }
1516}
1517
1518fn route_removal_will_quiesce_root(
1519 root: &ProjectRootId,
1520 route: RouteChannel,
1521 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1522 has_pending_bind: bool,
1523 replacement_root: Option<&ProjectRootId>,
1524) -> bool {
1525 let removes_last_route = root_channels
1526 .get(root)
1527 .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1528 removes_last_route && !has_pending_bind && replacement_root != Some(root)
1529}
1530
1531fn should_quiesce_removed_root(
1532 root: &ProjectRootId,
1533 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1534 has_pending_bind: bool,
1535 replacement_root: Option<&ProjectRootId>,
1536) -> bool {
1537 !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1538}
1539
1540async fn end_bg_subscription(
1541 writer_tx: &WriterSender,
1542 metrics: &DispatchPathMetrics,
1543 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1544 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1545 bg_wake_pending: &mut HashSet<RouteChannel>,
1546 channel: RouteChannel,
1547 identity: Option<&RouteIdentity>,
1548) -> Result<(), SubcError> {
1549 if let Some(sub) = bg_subs.remove(&channel) {
1550 bg_wake_pending.remove(&channel);
1551 remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1552 push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1553 }
1554 Ok(())
1555}
1556
1557#[allow(clippy::too_many_arguments)]
1558async fn teardown_installed_route(
1559 tx: &WriterSender,
1560 metrics: &DispatchPathMetrics,
1561 executor: &Arc<Executor>,
1562 channel: RouteChannel,
1563 cancellation_reason: &str,
1564 replacement_root: Option<&ProjectRootId>,
1565 installed_route_epochs: &mut HashMap<u16, u32>,
1566 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1567 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1568 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1569 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1570 bg_wake_pending: &mut HashSet<RouteChannel>,
1571 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1572 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1573 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1574 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1575 retry_buffer: &mut RetryBuffer,
1576 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1577 shutdown: &Arc<Notify>,
1578) -> Result<(), SubcError> {
1579 remove_installed_route(installed_route_epochs, channel);
1580 end_bg_subscription(
1581 tx,
1582 metrics,
1583 bg_subs,
1584 bg_sub_by_session,
1585 bg_wake_pending,
1586 channel,
1587 routes.get(&channel),
1588 )
1589 .await?;
1590 settle_pending_bash_asks_for_route(
1591 tx,
1592 pending_bash_asks,
1593 channel,
1594 routes,
1595 live_roots,
1596 route_bash_cancels,
1597 shutdown,
1598 metrics,
1599 )
1600 .await?;
1601 if let Some(cancel) = route_bash_cancels.remove(&channel) {
1602 cancel.token.cancel();
1603 }
1604 if let Some(pending) = pending_binds.get_mut(&channel) {
1605 pending.cancelled = true;
1606 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1607 log::debug!(
1608 "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1609 channel.channel
1610 );
1611 }
1612 let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1613 if let Some(identity) = routes.get(&channel) {
1614 let has_pending_bind = pending_binds
1615 .values()
1616 .any(|pending| pending.bind_root_id == identity.root);
1617 if route_removal_will_quiesce_root(
1618 &identity.root,
1619 channel,
1620 root_channels,
1621 has_pending_bind,
1622 replacement_root,
1623 ) {
1624 if let Some(ctx) = executor.actor_context(&identity.root) {
1625 ctx.mark_subc_unbound();
1628 }
1629 }
1630 }
1631 if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
1632 if migrated > 0 {
1633 log::debug!(
1634 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
1635 channel.channel
1636 );
1637 }
1638 if let Some(meta) = live_roots.get_mut(&identity.root) {
1639 let idle_for = meta.last_touched.elapsed();
1640 meta.note_activity();
1641 log::debug!(
1642 "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
1643 channel.channel,
1644 identity.root.as_path().display(),
1645 identity.harness,
1646 identity.session,
1647 idle_for
1648 );
1649 } else {
1650 log::debug!(
1651 "subc attach: route {} torn down for root {} harness {} session {}",
1652 channel.channel,
1653 identity.root.as_path().display(),
1654 identity.harness,
1655 identity.session
1656 );
1657 }
1658 let has_pending_bind = pending_binds
1659 .values()
1660 .any(|pending| pending.bind_root_id == identity.root);
1661 if should_quiesce_removed_root(
1662 &identity.root,
1663 root_channels,
1664 has_pending_bind,
1665 replacement_root,
1666 ) {
1667 quiesce_unbound_root(&identity.root, live_roots, executor);
1668 }
1669 } else {
1670 if migrated > 0 {
1671 log::debug!(
1672 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
1673 channel.channel
1674 );
1675 }
1676 log::debug!("subc attach: unbound route {} torn down", channel.channel);
1677 }
1678 Ok(())
1679}
1680
1681fn remember_session_identity(
1682 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1683 identity: &RouteIdentity,
1684) {
1685 let key = (identity.root.clone(), identity.session.clone());
1686 if matches!(identity.trust, BindTrust::Untrusted)
1687 && session_identity
1688 .get(&key)
1689 .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
1690 {
1691 return;
1692 }
1693
1694 session_identity.insert(
1699 key,
1700 RetainedSessionIdentity {
1701 harness: identity.harness.clone(),
1702 trust: identity.trust,
1703 },
1704 );
1705}
1706
1707fn replay_key_for_session(
1708 session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1709 root: &ProjectRootId,
1710 session: &str,
1711) -> Option<(push::ReplayKey, BindTrust)> {
1712 let retained = session_identity.get(&(root.clone(), session.to_string()))?;
1713 Some((
1714 push::ReplayKey {
1715 root: root.clone(),
1716 harness: retained.harness.clone(),
1717 session: session.to_string(),
1718 },
1719 retained.trust,
1720 ))
1721}
1722pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
1725
1726#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1727enum ModuleLoopExit {
1728 Graceful,
1729 SkipSearchFlush,
1730}
1731
1732pub fn run_subc_mode(
1737 connection_file_path: &Path,
1738 ctx: Arc<AppContext>,
1739 executor: Arc<Executor>,
1740 dispatch: DispatchFn,
1741 user_config_path: Option<PathBuf>,
1742) -> Result<(), SubcError> {
1743 run_subc_mode_inner(
1747 connection_file_path,
1748 ctx,
1749 executor,
1750 dispatch,
1751 user_config_path,
1752 false,
1753 )
1754}
1755
1756fn run_subc_mode_inner(
1757 connection_file_path: &Path,
1758 ctx: Arc<AppContext>,
1759 executor: Arc<Executor>,
1760 dispatch: DispatchFn,
1761 user_config_path: Option<PathBuf>,
1762 allow_native_passthrough: bool,
1763) -> Result<(), SubcError> {
1764 let runtime = tokio::runtime::Builder::new_current_thread()
1765 .enable_all()
1766 .build()
1767 .map_err(SubcError::Runtime)?;
1768
1769 let executor_for_loop = Arc::clone(&executor);
1770 let loop_result = runtime.block_on(async move {
1771 let shared_app = ctx.app();
1772 drop(ctx);
1773 let stream = connect_and_authenticate(connection_file_path).await?;
1774 log::info!(
1775 "subc attach: authenticated to daemon via {}",
1776 connection_file_path.display()
1777 );
1778 let (read_half, write_half) = tokio::io::split(stream);
1779 run_module_loop(
1780 read_half,
1781 write_half,
1782 shared_app,
1783 executor_for_loop,
1784 dispatch,
1785 user_config_path,
1786 allow_native_passthrough,
1787 )
1788 .await
1789 });
1790
1791 let actor_contexts = executor.actor_contexts();
1792 if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
1793 flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
1796 }
1797 for actor_ctx in &actor_contexts {
1798 actor_ctx.lsp().shutdown_all();
1799 actor_ctx.bash_background().detach();
1800 }
1801
1802 loop_result.map(|_| ())
1803}
1804
1805fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
1806 for actor_ctx in actor_contexts {
1807 let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
1808 }
1809 let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
1810}
1811
1812#[doc(hidden)]
1817pub fn run_subc_mode_for_test(
1818 connection_file_path: &Path,
1819 ctx: Arc<AppContext>,
1820 executor: Arc<Executor>,
1821 dispatch: DispatchFn,
1822 user_config_path: Option<PathBuf>,
1823) -> Result<(), SubcError> {
1824 run_subc_mode_inner(
1825 connection_file_path,
1826 ctx,
1827 executor,
1828 dispatch,
1829 user_config_path,
1830 true,
1831 )
1832}
1833
1834#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1835enum AttachErrorClass {
1836 Transient,
1837 Permanent,
1838}
1839
1840impl fmt::Display for AttachErrorClass {
1841 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1842 match self {
1843 Self::Transient => f.write_str("transient"),
1844 Self::Permanent => f.write_str("permanent"),
1845 }
1846 }
1847}
1848
1849#[derive(Clone, Copy)]
1850struct AttachRetryPolicy {
1851 budget: Duration,
1852 initial_backoff: Duration,
1853 max_backoff: Duration,
1854 jitter_percent: u64,
1855}
1856
1857const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
1858 budget: ATTACH_RETRY_BUDGET,
1859 initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
1860 max_backoff: ATTACH_RETRY_MAX_BACKOFF,
1861 jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
1862};
1863
1864fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
1867 let transient = match error {
1868 SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
1869 SubcError::Auth { source, .. } => match source {
1870 subc_transport::AuthError::Timeout { .. }
1871 | subc_transport::AuthError::UnexpectedEof { .. } => true,
1872 subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
1873 _ => false,
1874 },
1875 _ => false,
1876 };
1877 if transient {
1878 AttachErrorClass::Transient
1879 } else {
1880 AttachErrorClass::Permanent
1881 }
1882}
1883
1884fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
1885 matches!(
1886 kind,
1887 io::ErrorKind::ConnectionRefused
1888 | io::ErrorKind::TimedOut
1889 | io::ErrorKind::ConnectionReset
1890 | io::ErrorKind::ConnectionAborted
1891 | io::ErrorKind::BrokenPipe
1892 | io::ErrorKind::UnexpectedEof
1893 )
1894}
1895
1896async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
1900 connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
1901}
1902
1903async fn connect_and_authenticate_with_policy(
1904 connection_file_path: &Path,
1905 policy: AttachRetryPolicy,
1906) -> Result<TcpStream, SubcError> {
1907 let started_at = Instant::now();
1908 let deadline = started_at + policy.budget;
1909 let mut attempt = 0_u32;
1910 let mut backoff = policy.initial_backoff;
1911 let mut history = Vec::new();
1912
1913 loop {
1914 attempt = attempt.saturating_add(1);
1915 let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
1916 Ok(stream) => return Ok(stream),
1917 Err(error) => error,
1918 };
1919 let class = classify_attach_error(&error);
1920 let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
1921 history.push(format!("attempt {attempt} [{class}]: {error_text}"));
1922
1923 if class == AttachErrorClass::Permanent {
1924 log_attach_final_failure(started_at.elapsed(), &history);
1925 return Err(error);
1926 }
1927
1928 let remaining = deadline.saturating_duration_since(Instant::now());
1929 if remaining.is_zero() {
1930 log_attach_final_failure(started_at.elapsed(), &history);
1931 return Err(error);
1932 }
1933
1934 let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
1935 log::info!(
1936 "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
1937 );
1938 tokio::time::sleep(delay).await;
1939
1940 if Instant::now() >= deadline {
1941 log_attach_final_failure(started_at.elapsed(), &history);
1942 return Err(error);
1943 }
1944 backoff = backoff.saturating_mul(2).min(policy.max_backoff);
1945 }
1946}
1947
1948fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
1949 let jitter_percent = jitter_percent.min(100);
1950 if jitter_percent == 0 {
1951 return base;
1952 }
1953
1954 let mut random_bytes = [0_u8; 8];
1955 let random = if getrandom::fill(&mut random_bytes).is_ok() {
1956 u64::from_le_bytes(random_bytes)
1957 } else {
1958 let timestamp = std::time::SystemTime::now()
1959 .duration_since(std::time::UNIX_EPOCH)
1960 .unwrap_or_default()
1961 .subsec_nanos();
1962 u64::from(timestamp) ^ u64::from(attempt)
1963 };
1964 let span = jitter_percent.saturating_mul(2).saturating_add(1);
1965 let multiplier_percent = 100 - jitter_percent + random % span;
1966 let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
1967 Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
1968}
1969
1970fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
1971 log::error!(
1972 "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
1973 history.len(),
1974 history.join(" | ")
1975 );
1976}
1977
1978async fn connect_and_authenticate_once(
1979 connection_file_path: &Path,
1980 deadline: Instant,
1981) -> Result<TcpStream, SubcError> {
1982 let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
1985 SubcError::ConnectionFile {
1986 path: connection_file_path.to_path_buf(),
1987 source,
1988 }
1989 })?;
1990
1991 let endpoint = conn
1992 .endpoints
1993 .first()
1994 .ok_or_else(|| SubcError::NoEndpoint {
1995 path: connection_file_path.to_path_buf(),
1996 })?;
1997 let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
1998 let ip = endpoint
1999 .host
2000 .parse::<IpAddr>()
2001 .map_err(|_| SubcError::InvalidEndpoint {
2002 path: connection_file_path.to_path_buf(),
2003 endpoint: endpoint_label.clone(),
2004 })?;
2005 let addr = SocketAddr::new(ip, endpoint.port);
2006
2007 let connect_budget = deadline.saturating_duration_since(Instant::now());
2008 let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2009 .await
2010 .map_err(|_| SubcError::Connect {
2011 endpoint: endpoint_label.clone(),
2012 source: io::Error::new(
2013 io::ErrorKind::TimedOut,
2014 "initial subc attach retry budget elapsed during TCP connect",
2015 ),
2016 })?
2017 .map_err(|source| SubcError::Connect {
2018 endpoint: endpoint_label.clone(),
2019 source,
2020 })?;
2021 stream
2022 .set_nodelay(true)
2023 .map_err(|source| SubcError::Connect {
2024 endpoint: endpoint_label.clone(),
2025 source,
2026 })?;
2027
2028 let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2029 authenticate_client(&mut stream, &conn, auth_budget)
2030 .await
2031 .map_err(|source| SubcError::Auth {
2032 endpoint: endpoint_label,
2033 source,
2034 })?;
2035
2036 Ok(stream)
2037}
2038
2039#[allow(clippy::too_many_arguments)]
2040async fn process_route_bind_completion(
2041 writer_tx: &WriterSender,
2042 completion: RouteBindCompletion,
2043 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2044 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2045 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2046 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2047 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2048 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2049 installed_route_epochs: &mut HashMap<u16, u32>,
2050 executor: &Arc<Executor>,
2051 shutdown: &Arc<Notify>,
2052 metrics: &Arc<DispatchPathMetrics>,
2053) -> Result<(), SubcError> {
2054 decrement_counted_channel(&metrics.control_completion_queued);
2055 handle_route_bind_completion(
2056 writer_tx,
2057 completion,
2058 routes,
2059 root_channels,
2060 session_identity,
2061 push_buffer,
2062 live_roots,
2063 pending_binds,
2064 installed_route_epochs,
2065 executor,
2066 shutdown,
2067 metrics,
2068 )
2069 .await
2070}
2071
2072#[allow(clippy::too_many_arguments)]
2073async fn drain_pending_route_bind_completions(
2074 control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2075 writer_tx: &WriterSender,
2076 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2077 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2078 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2079 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2080 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2081 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2082 installed_route_epochs: &mut HashMap<u16, u32>,
2083 executor: &Arc<Executor>,
2084 shutdown: &Arc<Notify>,
2085 metrics: &Arc<DispatchPathMetrics>,
2086) -> Result<usize, SubcError> {
2087 let mut drained = 0;
2088 while let Ok(completion) = control_completion_rx.try_recv() {
2089 process_route_bind_completion(
2090 writer_tx,
2091 completion,
2092 routes,
2093 root_channels,
2094 session_identity,
2095 push_buffer,
2096 live_roots,
2097 pending_binds,
2098 installed_route_epochs,
2099 executor,
2100 shutdown,
2101 metrics,
2102 )
2103 .await?;
2104 drained += 1;
2105 }
2106 Ok(drained)
2107}
2108
2109async fn run_module_loop<R, W>(
2113 mut read: R,
2114 mut write: W,
2115 shared_app: Arc<App>,
2116 executor: Arc<Executor>,
2117 dispatch: DispatchFn,
2118 user_config_path: Option<PathBuf>,
2119 allow_native_passthrough: bool,
2120) -> Result<ModuleLoopExit, SubcError>
2121where
2122 R: AsyncRead + Unpin + Send + 'static,
2123 W: AsyncWrite + Unpin + Send + 'static,
2124{
2125 let hello = ModuleHelloBody {
2129 manifest: build_manifest(),
2130 protocol_ver: PROTOCOL_VERSION,
2131 control_ops: control_ops(),
2132 launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2133 };
2134 let hello_frame = Frame::build(
2135 FrameType::Hello,
2136 control_flags(),
2137 0,
2138 0,
2139 HELLO_CORR,
2140 serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2141 )
2142 .map_err(SubcError::FrameBuild)?;
2143 write_frame(&mut write, &hello_frame)
2144 .await
2145 .map_err(SubcError::FrameIo)?;
2146
2147 match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2149 None => return Err(SubcError::ClosedBeforeHelloAck),
2150 Some(frame) => match frame.header.ty {
2151 FrameType::HelloAck => {
2152 log::info!("subc attach: registered (HelloAck received)");
2153 }
2154 FrameType::Error => {
2155 let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2156 return Err(SubcError::HelloRejected { body });
2157 }
2158 other => return Err(SubcError::UnexpectedFrame { ty: other }),
2159 },
2160 }
2161
2162 let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2163 let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2164 let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2165 let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2172 let reader_task = spawn_reader_task(read, reader_tx);
2173 let shutdown = Arc::new(Notify::new());
2174 let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2183 let mut next_maintenance_at = next_drain_at;
2184 let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2185 let (bash_deferred_tx, mut bash_deferred_rx) =
2186 mpsc::channel::<bash::BashDeferredCompletion>(256);
2187 let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2188 let (control_completion_tx, mut control_completion_rx) =
2189 mpsc::channel::<RouteBindCompletion>(256);
2190 let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2191 let lossy_overflow = Arc::new(push::LossyOverflow::default());
2192 let lossy_seq = Arc::new(AtomicU64::new(0));
2193 let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2194 let push_senders = PushSenders {
2195 lossy_tx,
2196 reliable_tx,
2197 lossy_overflow: Arc::clone(&lossy_overflow),
2198 lossy_seq,
2199 };
2200 let connection_cancel = PersistentCancelSignal::new();
2201 let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2202 let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2203 let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2204 let mut bg_sub_by_session: HashMap<(ProjectRootId, String), RouteChannel> = HashMap::new();
2205 let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2206 let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2207 let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2208 let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2209 HashMap::new();
2210 let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2211 let mut retry_buffer: RetryBuffer = HashMap::new();
2212 let mut completed_tasks = push::CompletedTaskIds::default();
2213 let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2214 let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2215 let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2216 let mut next_bash_ask_corr: u64 = 1;
2217 let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2218
2219 let loop_result: Result<ModuleLoopExit, SubcError> = loop {
2220 shared_app.set_open_route_count(routes.len());
2221 crate::logging::perf_tick(Some(&executor));
2222 dispatch_path_metrics.mark_frame_loop_tick();
2223 if let Err(error) = expire_pending_bash_asks(
2224 &writer_tx,
2225 &mut pending_bash_asks,
2226 &routes,
2227 &mut live_roots,
2228 &mut route_bash_cancels,
2229 &shutdown,
2230 &dispatch_path_metrics,
2231 )
2232 .await
2233 {
2234 break Err(error);
2235 }
2236
2237 match drain_pending_route_bind_completions(
2241 &mut control_completion_rx,
2242 &writer_tx,
2243 &mut routes,
2244 &mut root_channels,
2245 &mut session_identity,
2246 &mut push_buffer,
2247 &mut live_roots,
2248 &mut pending_binds,
2249 &mut installed_route_epochs,
2250 &executor,
2251 &shutdown,
2252 &dispatch_path_metrics,
2253 )
2254 .await
2255 {
2256 Ok(drained) => {
2257 if drained > 0 {
2258 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2259 }
2260 }
2261 Err(error) => break Err(error),
2262 }
2263
2264 if tokio::time::Instant::now() >= next_drain_at {
2265 push::emit_bg_event_wakes(
2266 &writer_tx,
2267 &dispatch_path_metrics,
2268 &bg_subs,
2269 &mut bg_wake_pending,
2270 );
2271 warn_slow_pending_binds(&mut pending_binds, &executor);
2272 if let Err(error) = expire_overdue_route_binds(
2273 &writer_tx,
2274 &executor,
2275 &mut pending_binds,
2276 &mut installed_route_epochs,
2277 &dispatch_path_metrics,
2278 )
2279 .await
2280 {
2281 break Err(error);
2282 }
2283
2284 let retried = push::drain_retry_buffers_for_bound_routes(
2285 &writer_tx,
2286 &dispatch_path_metrics,
2287 &routes,
2288 &mut retry_buffer,
2289 );
2290 if retried > 0 {
2291 log::debug!(
2292 "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2293 );
2294 }
2295
2296 next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2297 }
2298
2299 let overflow_batch = lossy_overflow.drain();
2305 if !overflow_batch.is_empty() {
2306 let (_, deferred) = push::drain_reliable_push_turn(
2307 &writer_tx,
2308 &dispatch_path_metrics,
2309 &routes,
2310 &root_channels,
2311 &session_identity,
2312 &mut retry_buffer,
2313 &mut push_buffer,
2314 &mut completed_tasks,
2315 &bg_sub_by_session,
2316 &mut bg_wake_pending,
2317 &mut bg_wake_epoch,
2318 &mut reliable_rx,
2319 None,
2320 );
2321 if deferred {
2322 tokio::task::yield_now().await;
2323 }
2324
2325 let mut batch = Vec::new();
2326 while let Ok(item) = lossy_rx.try_recv() {
2327 batch.push(item);
2328 }
2329 batch.extend(overflow_batch);
2330 push::process_lossy_push_envelope_batch(
2331 &writer_tx,
2332 &dispatch_path_metrics,
2333 &routes,
2334 &root_channels,
2335 &completed_tasks,
2336 batch,
2337 );
2338 }
2339
2340 tokio::select! {
2341 biased;
2342 Some(completion) = control_completion_rx.recv() => {
2343 if let Err(error) = process_route_bind_completion(
2344 &writer_tx,
2345 completion,
2346 &mut routes,
2347 &mut root_channels,
2348 &mut session_identity,
2349 &mut push_buffer,
2350 &mut live_roots,
2351 &mut pending_binds,
2352 &mut installed_route_epochs,
2353 &executor,
2354 &shutdown,
2355 &dispatch_path_metrics,
2356 )
2357 .await
2358 {
2359 break Err(error);
2360 }
2361 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2362 }
2363 _ = shutdown.notified() => {
2364 log::warn!("subc attach: fatal executor response requested teardown");
2365 break Ok(ModuleLoopExit::SkipSearchFlush);
2366 }
2367 maybe_frame = reader_rx.recv() => {
2368 let frame = match maybe_frame {
2369 None => {
2370 log::info!("subc attach: daemon closed connection");
2371 break Ok(ModuleLoopExit::Graceful);
2372 }
2373 Some(Err(error)) => break Err(error),
2374 Some(Ok(frame)) => frame,
2375 };
2376 let phase_trace = frame.phase_trace;
2377 let frame = frame.frame;
2378
2379 if !ingress_route_is_current(&installed_route_epochs, &frame) {
2380 log::debug!(
2381 "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2382 frame.header.ty,
2383 frame.header.channel,
2384 frame.header.epoch
2385 );
2386 continue;
2387 }
2388
2389 match frame.header.ty {
2390 FrameType::Ping if frame.header.channel == 0 => {
2391 let pong = match Frame::build_with_version(
2392 frame.header.ver,
2393 FrameType::Pong,
2394 frame.header.flags,
2395 0,
2396 0,
2397 frame.header.corr,
2398 Vec::new(),
2399 ) {
2400 Ok(pong) => pong,
2401 Err(error) => break Err(SubcError::FrameBuild(error)),
2402 };
2403 if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2404 break Err(error);
2405 }
2406 }
2407 FrameType::Goodbye if frame.header.channel == 0 => {
2408 log::info!("subc attach: received channel-0 Goodbye");
2409 break Ok(ModuleLoopExit::Graceful);
2410 }
2411 FrameType::Goodbye => {
2412 let channel = route_key(frame.header.channel, frame.header.epoch);
2413 if let Err(error) = teardown_installed_route(
2414 &writer_tx,
2415 &dispatch_path_metrics,
2416 &executor,
2417 channel,
2418 "Goodbye",
2419 None,
2420 &mut installed_route_epochs,
2421 &mut routes,
2422 &mut root_channels,
2423 &mut bg_subs,
2424 &mut bg_sub_by_session,
2425 &mut bg_wake_pending,
2426 &mut pending_bash_asks,
2427 &mut live_roots,
2428 &mut route_bash_cancels,
2429 &mut pending_binds,
2430 &mut retry_buffer,
2431 &mut push_buffer,
2432 &shutdown,
2433 )
2434 .await
2435 {
2436 break Err(error);
2437 }
2438 }
2439 FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2440 if let Err(error) = handle_bash_elicitation_reply(
2441 &writer_tx,
2442 &frame,
2443 &mut pending_bash_asks,
2444 &routes,
2445 &mut live_roots,
2446 &executor,
2447 &shutdown,
2448 &bash_deferred_tx,
2449 &bash_poll_touch_tx,
2450 &dispatch_path_metrics,
2451 &mut route_bash_cancels,
2452 dispatch,
2453 )
2454 .await
2455 {
2456 break Err(error);
2457 }
2458 }
2459 FrameType::Request if frame.header.channel == 0 => {
2460 if let Err(error) = handle_control_request(
2461 &writer_tx,
2462 &frame,
2463 &shared_app,
2464 &executor,
2465 &mut live_roots,
2466 &mut pending_binds,
2467 &mut installed_route_epochs,
2468 &mut routes,
2469 &mut root_channels,
2470 &mut bg_subs,
2471 &mut bg_sub_by_session,
2472 &mut bg_wake_pending,
2473 &mut pending_bash_asks,
2474 &mut route_bash_cancels,
2475 &mut retry_buffer,
2476 &mut push_buffer,
2477 &shutdown,
2478 &control_completion_tx,
2479 &dispatch_path_metrics,
2480 &push_senders,
2481 dispatch,
2482 user_config_path.as_deref(),
2483 )
2484 .await
2485 {
2486 break Err(error);
2487 }
2488 }
2489 FrameType::Request => {
2490 if let Err(error) = handle_tool_call(
2491 &writer_tx,
2492 &frame,
2493 phase_trace,
2494 &routes,
2495 &pending_binds,
2496 &mut live_roots,
2497 &executor,
2498 &shutdown,
2499 &connection_cancel,
2500 &bash_deferred_tx,
2501 &bash_poll_touch_tx,
2502 &dispatch_path_metrics,
2503 &mut route_bash_cancels,
2504 &mut pending_bash_asks,
2505 &mut next_bash_ask_corr,
2506 &mut bg_subs,
2507 &mut bg_sub_by_session,
2508 &mut bg_wake_pending,
2509 &mut bg_wake_epoch,
2510 dispatch,
2511 allow_native_passthrough,
2512 )
2513 .await
2514 {
2515 break Err(error);
2516 }
2517 }
2518 FrameType::Cancel => {
2519 let channel = route_key(frame.header.channel, frame.header.epoch);
2520 if bg_subs.contains_key(&channel) {
2521 if let Err(error) = end_bg_subscription(
2522 &writer_tx,
2523 &dispatch_path_metrics,
2524 &mut bg_subs,
2525 &mut bg_sub_by_session,
2526 &mut bg_wake_pending,
2527 channel,
2528 routes.get(&channel),
2529 )
2530 .await
2531 {
2532 break Err(error);
2533 }
2534 }
2535 if let Err(error) = cancel_pending_bash_ask_for_tool_call(
2536 &writer_tx,
2537 &mut pending_bash_asks,
2538 channel,
2539 frame.header.corr,
2540 &routes,
2541 &mut live_roots,
2542 &mut route_bash_cancels,
2543 &shutdown,
2544 &dispatch_path_metrics,
2545 )
2546 .await
2547 {
2548 break Err(error);
2549 }
2550 }
2551 _ => {}
2556 }
2557 }
2558 Some((root_id, frame)) = reliable_rx.recv() => {
2559 let (_, deferred) = push::drain_reliable_push_turn(
2563 &writer_tx,
2564 &dispatch_path_metrics,
2565 &routes,
2566 &root_channels,
2567 &session_identity,
2568 &mut retry_buffer,
2569 &mut push_buffer,
2570 &mut completed_tasks,
2571 &bg_sub_by_session,
2572 &mut bg_wake_pending,
2573 &mut bg_wake_epoch,
2574 &mut reliable_rx,
2575 Some((root_id, frame)),
2576 );
2577 if deferred {
2578 tokio::task::yield_now().await;
2579 }
2580 }
2581 Some((order, root_id, frame)) = lossy_rx.recv() => {
2582 let (_, deferred) = push::drain_reliable_push_turn(
2586 &writer_tx,
2587 &dispatch_path_metrics,
2588 &routes,
2589 &root_channels,
2590 &session_identity,
2591 &mut retry_buffer,
2592 &mut push_buffer,
2593 &mut completed_tasks,
2594 &bg_sub_by_session,
2595 &mut bg_wake_pending,
2596 &mut bg_wake_epoch,
2597 &mut reliable_rx,
2598 None,
2599 );
2600 if deferred {
2601 tokio::task::yield_now().await;
2602 }
2603
2604 let mut batch = vec![(order, root_id, frame)];
2611 while let Ok(item) = lossy_rx.try_recv() {
2612 batch.push(item);
2613 }
2614 batch.extend(lossy_overflow.drain());
2615 push::process_lossy_push_envelope_batch(
2616 &writer_tx,
2617 &dispatch_path_metrics,
2618 &routes,
2619 &root_channels,
2620 &completed_tasks,
2621 batch,
2622 );
2623 }
2624 Some(done) = bash_deferred_rx.recv() => {
2625 decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
2626 if let Err(error) = bash::handle_bash_deferred_completion(
2627 &writer_tx,
2628 done,
2629 &routes,
2630 &mut live_roots,
2631 &mut route_bash_cancels,
2632 &shutdown,
2633 &dispatch_path_metrics,
2634 )
2635 .await
2636 {
2637 break Err(error);
2638 }
2639 }
2640 Some(root_id) = bash_poll_touch_rx.recv() => {
2641 decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
2642 if let Some(meta) = live_roots.get_mut(&root_id) {
2643 meta.note_activity();
2644 }
2645 }
2646 Some(completion) = maintenance_rx.recv() => {
2647 decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
2648 let root_id = completion.root_id.clone();
2649 let response = completion.response;
2650 let response_is_fatal = response_is_fatal_panic(&response);
2651 let bind_pending = pending_binds
2652 .values()
2653 .any(|pending| pending.bind_root_id == root_id);
2654 let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
2655 let defer_requeue = meta.unbound_quiesced || bind_pending;
2656 note_maintenance_completion(
2657 meta,
2658 completion.requeue_kind,
2659 response_is_fatal,
2660 defer_requeue,
2661 );
2662 should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
2663 } else {
2664 false
2665 };
2666 if requiesce {
2667 quiesce_unbound_root(&root_id, &mut live_roots, &executor);
2668 }
2669 push::clear_stale_bg_wakes_for_empty_sessions(
2670 &root_id,
2671 &completion.empty_bg_sessions,
2672 &bg_sub_by_session,
2673 &mut bg_wake_pending,
2674 &bg_wake_epoch,
2675 );
2676 if response_is_fatal {
2677 if let Some(meta) = live_roots.get_mut(&root_id) {
2678 meta.maintenance_poisoned = true;
2679 }
2680 log::warn!(
2681 "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
2682 );
2683 }
2684 }
2685 _ = tokio::time::sleep_until(next_drain_at) => {
2686 }
2689 _ = tokio::time::sleep_until(next_maintenance_at) => {
2690 let reap = reap_idle_roots(
2695 Instant::now(),
2696 &mut live_roots,
2697 &pending_binds,
2698 &root_channels,
2699 &executor,
2700 &dispatch_path_metrics,
2701 );
2702 for root_id in &reap.forgotten_deleted_roots {
2703 purge_deleted_root_residents(
2704 root_id,
2705 &mut root_channels,
2706 &mut session_identity,
2707 &mut push_buffer,
2708 &mut bg_subs,
2709 &mut bg_sub_by_session,
2710 &mut bg_wake_pending,
2711 &mut bg_wake_epoch,
2712 &mut pending_bash_asks,
2713 );
2714 }
2715 if reap.evicted > 0 {
2716 log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
2717 }
2718 submit_due_maintenance_jobs(
2719 &executor,
2720 &mut live_roots,
2721 &pending_binds,
2722 &bg_sub_by_session,
2723 &bg_wake_pending,
2724 &bg_wake_epoch,
2725 &maintenance_tx,
2726 &dispatch_path_metrics,
2727 );
2728 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2729 }
2730 }
2731 };
2732
2733 shared_app.set_open_route_count(0);
2734
2735 connection_cancel.cancel();
2736 quiesce_connection_roots(
2739 &mut live_roots,
2740 &mut pending_binds,
2741 &mut routes,
2742 &mut root_channels,
2743 &mut installed_route_epochs,
2744 &mut route_bash_cancels,
2745 &executor,
2746 );
2747
2748 let mut loop_result = loop_result;
2749 if !pending_bash_asks.is_empty() {
2750 let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2751 if let Err(error) = settle_all_pending_bash_asks(
2752 &writer_tx,
2753 &mut pending_bash_asks,
2754 &no_routes,
2755 &mut live_roots,
2756 &mut route_bash_cancels,
2757 &shutdown,
2758 &dispatch_path_metrics,
2759 )
2760 .await
2761 {
2762 loop_result = loop_result.and(Err(error));
2763 }
2764 }
2765
2766 reader_task.abort();
2769 drop(writer_tx);
2770 let writer_result = finish_writer_task(writer_task).await;
2771 loop_result.and_then(|exit| writer_result.map(|_| exit))
2772}
2773
2774fn spawn_writer_task<W>(
2775 mut write: W,
2776 mut rx: mpsc::Receiver<WriterFrame>,
2777 metrics: Arc<DispatchPathMetrics>,
2778) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
2779where
2780 W: AsyncWrite + Unpin + Send + 'static,
2781{
2782 tokio::spawn(async move {
2783 let mut write_buffer = Vec::new();
2784 while let Some(mut queued) = rx.recv().await {
2785 let measure = queued.tool_response_trace.is_some();
2786 let dequeued = measure.then(Instant::now);
2787 metrics.writer_active.store(true, Ordering::Relaxed);
2788 decrement_counted_channel(&metrics.writer_queued);
2789 let write_timing = write_frame_contiguous(
2790 &mut write,
2791 queued.frame(),
2792 queued.body(),
2793 &mut write_buffer,
2794 measure,
2795 )
2796 .await;
2797 metrics.writer_active.store(false, Ordering::Relaxed);
2798 let write_timing = write_timing?;
2799
2800 if let (Some(trace), Some(dequeued), Some(write_timing)) =
2801 (queued.tool_response_trace.take(), dequeued, write_timing)
2802 {
2803 if let Some(completed) = trace.finish(
2804 dequeued,
2805 write_timing.write_started,
2806 write_timing.write_finished,
2807 write_timing.frame_bytes,
2808 ) {
2809 log_ctx::with_session(Some(completed.session), || {
2810 crate::logging::note_tool_call_trace(
2811 &completed.name,
2812 &completed.root,
2813 completed.channel,
2814 completed.corr,
2815 completed.phases,
2816 );
2817 });
2818 }
2819 }
2820 }
2821 Ok(())
2822 })
2823}
2824
2825struct FrameWriteTiming {
2826 write_started: Instant,
2827 write_finished: Instant,
2828 frame_bytes: usize,
2829}
2830
2831async fn write_frame_contiguous<W>(
2835 writer: &mut W,
2836 frame: &Frame,
2837 body: &[u8],
2838 buffer: &mut Vec<u8>,
2839 measure: bool,
2840) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
2841where
2842 W: AsyncWrite + Unpin,
2843{
2844 if frame.header.len as usize != body.len() {
2845 return Err(subc_transport::FrameIoError::BodyLengthMismatch {
2846 header_len: frame.header.len,
2847 body_len: body.len(),
2848 });
2849 }
2850
2851 let header = frame.header.encode();
2852 buffer.clear();
2853 buffer.reserve(header.len() + body.len());
2854 buffer.extend_from_slice(&header);
2855 buffer.extend_from_slice(body);
2856 let write_started = measure.then(Instant::now);
2857 writer
2858 .write_all(buffer)
2859 .await
2860 .map_err(subc_transport::FrameIoError::Io)?;
2861 Ok(write_started.map(|write_started| FrameWriteTiming {
2862 write_started,
2863 write_finished: Instant::now(),
2864 frame_bytes: buffer.len(),
2865 }))
2866}
2867
2868fn spawn_reader_task<R>(
2869 mut read: R,
2870 tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
2871) -> JoinHandle<()>
2872where
2873 R: AsyncRead + Unpin + Send + 'static,
2874{
2875 tokio::spawn(async move {
2876 loop {
2877 match read_frame(&mut read).await {
2878 Ok(Some(frame)) => {
2879 let decoded = DecodedFrame {
2880 frame,
2881 phase_trace: PhaseTrace::new(Instant::now()),
2882 };
2883 if tx.send(Ok(decoded)).await.is_err() {
2884 return;
2885 }
2886 }
2887 Ok(None) => {
2888 return;
2890 }
2891 Err(error) => {
2892 if let subc_transport::FrameIoError::Io(io_error) = &error {
2899 if matches!(
2900 io_error.kind(),
2901 std::io::ErrorKind::ConnectionReset
2902 | std::io::ErrorKind::ConnectionAborted
2903 ) {
2904 log::info!(
2905 "subc attach: connection reset by daemon; treating as close"
2906 );
2907 return;
2908 }
2909 }
2910 let _ = tx.send(Err(SubcError::FrameIo(error))).await;
2911 return;
2912 }
2913 }
2914 }
2915 })
2916}
2917
2918async fn finish_writer_task(
2919 mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
2920) -> Result<(), SubcError> {
2921 match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
2922 Ok(Ok(Ok(()))) => Ok(()),
2923 Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
2924 Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
2925 Err(_) => {
2926 writer_task.abort();
2927 Ok(())
2928 }
2929 }
2930}
2931
2932fn register_actor_for_bind(
2933 shared_app: &Arc<App>,
2934 executor: &Arc<Executor>,
2935 push_senders: &PushSenders,
2936 bind_root_id: &ProjectRootId,
2937 route_channel: u16,
2938 root_was_live: bool,
2939) -> bool {
2940 if executor.actor_registered(bind_root_id) {
2941 log::debug!(
2942 "subc attach: reusing actor for route {} root {}",
2943 route_channel,
2944 bind_root_id.as_path().display()
2945 );
2946 return false;
2947 }
2948
2949 if root_was_live {
2950 log::warn!(
2951 "subc attach: recreating missing actor for live root {} on route {}",
2952 bind_root_id.as_path().display(),
2953 route_channel
2954 );
2955 }
2956
2957 let actor_ctx = Arc::new(AppContext::from_app(
2958 Arc::clone(shared_app),
2959 Config::default(),
2960 ));
2961 install_bash_compressor(&actor_ctx);
2962 actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
2963 push_senders.clone(),
2964 bind_root_id.clone(),
2965 )));
2966 let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
2967 drop(actor_ctx);
2968 if inserted {
2969 log::debug!(
2973 "subc attach: registered actor for route {} root {}",
2974 route_channel,
2975 bind_root_id.as_path().display()
2976 );
2977 } else {
2978 log::debug!(
2979 "subc attach: actor appeared while binding route {} root {}; reusing it",
2980 route_channel,
2981 bind_root_id.as_path().display()
2982 );
2983 }
2984 inserted
2985}
2986
2987fn rollback_pending_bind_actor(
2988 executor: &Arc<Executor>,
2989 live_roots: &HashMap<ProjectRootId, RootMeta>,
2990 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2991 root_id: &ProjectRootId,
2992 inserted_new_actor: bool,
2993) {
2994 if !inserted_new_actor || live_roots.contains_key(root_id) {
2995 return;
2996 }
2997
2998 if let Some((route, pending)) = pending_binds
2999 .iter_mut()
3000 .find(|(_, pending)| &pending.bind_root_id == root_id)
3001 {
3002 pending.inserted_new_actor = true;
3003 log::debug!(
3004 "subc attach: transferred rollback ownership for root {} to pending route {}",
3005 root_id.as_path().display(),
3006 route
3007 );
3008 return;
3009 }
3010
3011 executor.remove_actor(root_id);
3012}
3013
3014fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
3015 match response.data.get("code").and_then(|code| code.as_str()) {
3016 Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
3021 Some("cache_key_probe_failed") => "cache_key_probe_failed",
3025 Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
3029 _ => "config_divergence",
3030 }
3031}
3032
3033fn queue_post_bind_configure_and_completion_maintenance(
3034 root_id: &ProjectRootId,
3035 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3036) {
3037 let Some(meta) = live_roots.get_mut(root_id) else {
3038 return;
3039 };
3040 if meta.maintenance_poisoned || meta.maintenance_pending {
3041 return;
3042 }
3043
3044 meta.maintenance_pending = true;
3045 meta.maintenance_queued_kinds
3046 .push_back(MaintenanceDrainKind::ConfigureTail);
3047 meta.maintenance_queued_kinds
3048 .push_back(MaintenanceDrainKind::CompletionDrains);
3049}
3050
3051#[allow(clippy::too_many_arguments)]
3052async fn handle_route_bind_completion(
3053 tx: &WriterSender,
3054 completion: RouteBindCompletion,
3055 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3056 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3057 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
3058 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3059 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3060 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3061 installed_route_epochs: &mut HashMap<u16, u32>,
3062 executor: &Arc<Executor>,
3063 shutdown: &Arc<Notify>,
3064 metrics: &Arc<DispatchPathMetrics>,
3065) -> Result<(), SubcError> {
3066 let route_id = completion.route;
3067 let Some(pending) = pending_binds.remove(&route_id) else {
3068 log::warn!(
3069 "subc attach: dropping RouteBind completion for non-pending route {}",
3070 completion.route
3071 );
3072 rollback_pending_bind_actor(
3073 executor,
3074 live_roots,
3075 pending_binds,
3076 &completion.bind_root_id,
3077 completion.inserted_new_actor,
3078 );
3079 let has_pending_bind = pending_binds
3080 .values()
3081 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3082 if !root_channels
3083 .get(&completion.bind_root_id)
3084 .is_some_and(|channels| !channels.is_empty())
3085 && !has_pending_bind
3086 {
3087 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3088 }
3089 remove_installed_route(installed_route_epochs, route_id);
3090 return Ok(());
3091 };
3092
3093 if pending.bind_root_id != completion.bind_root_id {
3094 log::warn!(
3095 "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
3096 completion.route,
3097 pending.bind_root_id.as_path().display(),
3098 completion.bind_root_id.as_path().display()
3099 );
3100 }
3101
3102 let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
3103 if pending.cancelled {
3104 rollback_pending_bind_actor(
3105 executor,
3106 live_roots,
3107 pending_binds,
3108 &completion.bind_root_id,
3109 inserted_new_actor,
3110 );
3111 let has_pending_bind = pending_binds
3112 .values()
3113 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3114 if !root_channels
3115 .get(&completion.bind_root_id)
3116 .is_some_and(|channels| !channels.is_empty())
3117 && !has_pending_bind
3118 {
3119 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3120 }
3121 log::debug!(
3122 "subc attach: discarded completed RouteBind for cancelled route {} root {}",
3123 completion.route,
3124 completion.bind_root_id.as_path().display()
3125 );
3126 remove_installed_route(installed_route_epochs, route_id);
3127 return Ok(());
3128 }
3129
3130 let failure = if !completion.configure_response.success {
3131 Some((
3132 &completion.configure_response,
3133 "configure failed during route bind",
3134 ))
3135 } else {
3136 None
3137 };
3138
3139 if let Some((response, fallback)) = failure {
3140 rollback_pending_bind_actor(
3141 executor,
3142 live_roots,
3143 pending_binds,
3144 &completion.bind_root_id,
3145 inserted_new_actor,
3146 );
3147 let has_pending_bind = pending_binds
3148 .values()
3149 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3150 if !root_channels
3151 .get(&completion.bind_root_id)
3152 .is_some_and(|channels| !channels.is_empty())
3153 && !has_pending_bind
3154 {
3155 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3156 }
3157 let message = response_message(response, fallback);
3158 let fatal = response_is_fatal_panic(response);
3159 let error_code = route_bind_error_code_for_configure_response(response);
3160 send_route_bind_error_parts(
3161 tx,
3162 completion.ver,
3163 completion.corr,
3164 completion.flags,
3165 error_code,
3166 &message,
3167 metrics,
3168 )
3169 .await?;
3170 remove_installed_route(installed_route_epochs, route_id);
3171 if fatal {
3172 signal_fatal_teardown(
3173 tx,
3174 Some(completion.route),
3175 completion.ver,
3176 completion.corr,
3177 shutdown,
3178 metrics,
3179 )
3180 .await;
3181 }
3182 return Ok(());
3183 }
3184
3185 remember_session_identity(session_identity, &completion.identity);
3186 let replay_key = push::ReplayKey::from_identity(&completion.identity);
3187 let bind_trust = completion.identity.trust;
3188 insert_route_channel(routes, root_channels, route_id, completion.identity);
3189 let restore_watcher = live_roots
3190 .get(&completion.bind_root_id)
3191 .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3192 live_roots
3193 .entry(completion.bind_root_id.clone())
3194 .and_modify(|meta| {
3195 meta.reactivate_bound();
3196 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3197 meta.maintenance_poisoned = false;
3198 })
3199 .or_insert_with(|| RootMeta::new(Instant::now()));
3200 if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3201 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3202 meta.maintenance_poisoned = false;
3203 }
3204 if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3205 ctx.mark_subc_bound();
3206 if restore_watcher {
3207 crate::commands::configure::ensure_project_watcher(&ctx);
3208 }
3209 }
3210
3211 let ack =
3212 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3213 let response = Frame::build_with_version(
3214 completion.ver,
3215 FrameType::Response,
3216 control_flags(),
3217 0,
3218 0,
3219 completion.corr,
3220 ack,
3221 )
3222 .map_err(SubcError::FrameBuild)?;
3223 send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3224 queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3225 let replayed = push::replay_buffered_push_frames(
3226 tx,
3227 metrics,
3228 route_id,
3229 push_buffer,
3230 &replay_key,
3231 bind_trust,
3232 );
3233 if replayed > 0 {
3234 log::debug!(
3235 "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3236 replayed,
3237 completion.route,
3238 replay_key.root.as_path().display(),
3239 replay_key.harness,
3240 replay_key.session
3241 );
3242 }
3243 log::info!(
3244 "subc attach: route {} bound to root {}",
3245 completion.route,
3246 completion.bind_root_id.as_path().display()
3247 );
3248 Ok(())
3249}
3250
3251async fn expire_overdue_route_binds(
3252 tx: &WriterSender,
3253 executor: &Arc<Executor>,
3254 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3255 installed_route_epochs: &mut HashMap<u16, u32>,
3256 metrics: &DispatchPathMetrics,
3257) -> Result<(), SubcError> {
3258 let now = Instant::now();
3259 let expired: Vec<_> = pending_binds
3260 .iter()
3261 .filter_map(|(route, pending)| {
3262 let age = now.saturating_duration_since(pending.started_at);
3263 (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3264 (
3265 *route,
3266 pending.corr,
3267 pending.ver,
3268 pending.flags,
3269 pending.bind_root_id.clone(),
3270 pending.configure_request_id.clone(),
3271 age,
3272 )
3273 })
3274 })
3275 .collect();
3276
3277 for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3278 if let Some(pending) = pending_binds.get_mut(&route) {
3279 pending.cancelled = true;
3280 pending.deadline_reported = true;
3281 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3282 log::debug!(
3283 "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3284 );
3285 }
3286 remove_installed_route(installed_route_epochs, route);
3287 let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3288 let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3289 send_route_bind_error_parts(
3290 tx,
3291 ver,
3292 corr,
3293 flags,
3294 "actor_not_ready",
3295 &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3296 metrics,
3297 )
3298 .await?;
3299 log::warn!(
3300 "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3301 route,
3302 root_id.as_path().display(),
3303 deadline_ms,
3304 configure_request_id
3305 );
3306 }
3307
3308 Ok(())
3309}
3310
3311#[allow(clippy::too_many_arguments)]
3316async fn handle_control_request(
3317 tx: &WriterSender,
3318 frame: &Frame,
3319 shared_app: &Arc<App>,
3320 executor: &Arc<Executor>,
3321 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3322 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3323 installed_route_epochs: &mut HashMap<u16, u32>,
3324 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3325 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3326 bg_subs: &mut HashMap<RouteChannel, BgSub>,
3327 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3328 bg_wake_pending: &mut HashSet<RouteChannel>,
3329 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3330 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3331 retry_buffer: &mut RetryBuffer,
3332 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3333 shutdown: &Arc<Notify>,
3334 control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3335 metrics: &Arc<DispatchPathMetrics>,
3336 push_senders: &PushSenders,
3337 dispatch: DispatchFn,
3338 user_config_path: Option<&Path>,
3339) -> Result<(), SubcError> {
3340 let request =
3341 serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3342 match request {
3343 ModuleControlRequest::RouteBind {
3344 route_channel,
3345 epoch,
3346 target: _,
3347 identity,
3348 principal,
3349 consumer_capabilities,
3350 } => {
3351 let route_id = route_key(route_channel, epoch);
3352 if epoch == 0 {
3353 return send_route_bind_error(
3354 tx,
3355 frame,
3356 "config_divergence",
3357 "route bind uses an invalid channel generation",
3358 metrics,
3359 )
3360 .await;
3361 }
3362 let mut bind_root_id = None;
3363 if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3364 if installed_epoch >= epoch {
3365 return send_route_bind_error(
3366 tx,
3367 frame,
3368 "config_divergence",
3369 "route bind generation is not newer than the installed generation",
3370 metrics,
3371 )
3372 .await;
3373 }
3374
3375 let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3376 Ok(root_id) => root_id,
3377 Err(error) => {
3378 return send_route_bind_error(
3379 tx,
3380 frame,
3381 "config_divergence",
3382 &format!("invalid route project root: {error}"),
3383 metrics,
3384 )
3385 .await;
3386 }
3387 };
3388 teardown_installed_route(
3389 tx,
3390 metrics,
3391 executor,
3392 route_key(route_channel, installed_epoch),
3393 "higher-epoch RouteBind",
3394 Some(&replacement_root),
3395 installed_route_epochs,
3396 routes,
3397 root_channels,
3398 bg_subs,
3399 bg_sub_by_session,
3400 bg_wake_pending,
3401 pending_bash_asks,
3402 live_roots,
3403 route_bash_cancels,
3404 pending_binds,
3405 retry_buffer,
3406 push_buffer,
3407 shutdown,
3408 )
3409 .await?;
3410 bind_root_id = Some(replacement_root);
3411 }
3412 if pending_binds.contains_key(&route_id) {
3413 return send_route_bind_error(
3414 tx,
3415 frame,
3416 "config_divergence",
3417 "route bind is already pending for channel",
3418 metrics,
3419 )
3420 .await;
3421 }
3422 let bind_root_id = match bind_root_id {
3423 Some(root_id) => root_id,
3424 None => match ProjectRootId::from_path(&identity.project_root) {
3425 Ok(root_id) => root_id,
3426 Err(error) => {
3427 return send_route_bind_error(
3428 tx,
3429 frame,
3430 "config_divergence",
3431 &format!("invalid route project root: {error}"),
3432 metrics,
3433 )
3434 .await;
3435 }
3436 },
3437 };
3438
3439 let request_id = format!("subc-bind-{route_channel}");
3442 let bind_project_root = identity.project_root.clone();
3443 let bind_harness = identity.harness.clone();
3444 let bind_session = identity.session.clone();
3445 let bind_trust = trust_for_bind(&bind_harness, &principal);
3446 let bind_principal_id = principal_id(&principal);
3447 let consumer_elicitation_capable = consumer_capabilities
3452 .as_ref()
3453 .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
3454 log::info!(
3455 "subc attach: route {} harness={} principal={} trust={} elicitation={}",
3456 route_channel,
3457 bind_harness,
3458 principal_label(&principal),
3459 bind_trust.label(),
3460 consumer_elicitation_capable
3461 );
3462
3463 let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
3475 user_config_path,
3476 Path::new(&bind_project_root),
3477 );
3478 let config_tiers: Vec<Value> = local_tiers
3479 .iter()
3480 .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
3481 .collect();
3482 let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
3483 let configure_json = json!({
3484 "id": request_id,
3485 "command": "configure",
3486 "project_root": bind_project_root,
3487 "harness": bind_harness,
3488 "session_id": bind_session.clone(),
3489 "config": config_tiers,
3490 });
3491 let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
3492 Ok(req) => req,
3493 Err(error) => {
3494 return send_route_bind_error(
3495 tx,
3496 frame,
3497 "config_divergence",
3498 &format!("failed to build configure request: {error}"),
3499 metrics,
3500 )
3501 .await;
3502 }
3503 };
3504
3505 let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
3506 root: bind_root_id.clone(),
3507 project_root: PathBuf::from(&bind_project_root),
3508 harness: bind_harness.clone(),
3509 session: bind_session.clone(),
3510 trust: bind_trust,
3511 spawn_principal: AuthenticatedPrincipal::RouteBind {
3512 trust: bind_trust.sandbox_trust(),
3513 route_channel,
3514 route_epoch: epoch,
3515 project_root: PathBuf::from(&bind_project_root),
3516 harness: bind_harness.clone(),
3517 session_id: bind_session.clone(),
3518 principal_id: bind_principal_id,
3519 },
3520 consumer_elicitation_capable,
3521 }));
3522 let configure_session = route_identity.session.clone();
3523 let root_was_live = live_roots.contains_key(&bind_root_id);
3524 let inserted_new_actor = register_actor_for_bind(
3525 shared_app,
3526 executor,
3527 push_senders,
3528 &bind_root_id,
3529 route_channel,
3530 root_was_live,
3531 );
3532
3533 let configure_request_id = configure_req.id.clone();
3534 installed_route_epochs.insert(route_channel, epoch);
3535 if let Some(meta) = live_roots.get_mut(&bind_root_id) {
3536 meta.maintenance_queued_kinds.clear();
3537 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
3538 }
3539 let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
3540 bind_root_id.clone(),
3541 Lane::Mutating,
3542 configure_request_id.clone(),
3543 Box::new(move |ctx| {
3544 log_ctx::with_session(Some(configure_session.clone()), || {
3545 dispatch(configure_req, ctx)
3546 })
3547 }),
3548 );
3549 pending_binds.insert(
3550 route_id,
3551 PendingBind {
3552 bind_root_id: bind_root_id.clone(),
3553 inserted_new_actor,
3554 cancelled: false,
3555 configure_request_id: configure_request_id.clone(),
3556 started_at: Instant::now(),
3557 warned_half_deadline: false,
3558 deadline_reported: false,
3559 corr: frame.header.corr,
3560 ver: frame.header.ver,
3561 flags: frame.header.flags,
3562 cancellation: configure_cancellation,
3563 },
3564 );
3565
3566 let completion_tx = control_completion_tx.clone();
3567 let completion_identity = route_identity;
3568 let completion_root = bind_root_id.clone();
3569 let completion_route_channel = route_channel;
3570 let completion_ver = frame.header.ver;
3571 let completion_corr = frame.header.corr;
3572 let completion_flags = frame.header.flags;
3573 let completion_metrics = Arc::clone(metrics);
3574 tokio::spawn(async move {
3575 let _response_task = ResponseTaskGuard::new(&completion_metrics);
3576 let configure_response =
3577 await_executor_response(configure_rx, configure_request_id.clone()).await;
3578 let completion = RouteBindCompletion {
3583 route: route_key(completion_route_channel, epoch),
3584 identity: completion_identity,
3585 bind_root_id: completion_root,
3586 inserted_new_actor,
3587 configure_response,
3588 diagnostics_on_edit,
3589 ver: completion_ver,
3590 corr: completion_corr,
3591 flags: completion_flags,
3592 };
3593 if send_counted_channel(
3594 &completion_tx,
3595 &completion_metrics.control_completion_queued,
3596 completion,
3597 )
3598 .await
3599 .is_err()
3600 {
3601 log::debug!(
3602 "subc attach: dropped RouteBind completion for route {} after loop exit",
3603 completion_route_channel
3604 );
3605 }
3606 });
3607
3608 Ok(())
3609 }
3610 ModuleControlRequest::HealthCheck {} => {
3611 let report = build_health_report(executor, pending_binds, metrics, shared_app);
3612 let body = serde_json::to_vec(&ModuleControlResponse::from(report))
3613 .map_err(SubcError::Json)?;
3614 let response = Frame::build_with_version(
3615 frame.header.ver,
3616 FrameType::Response,
3617 frame.header.flags,
3618 0,
3619 0,
3620 frame.header.corr,
3621 body,
3622 )
3623 .map_err(SubcError::FrameBuild)?;
3624 send_frame(tx, metrics, response).await
3625 }
3626 }
3627}
3628
3629fn install_bash_compressor(ctx: &AppContext) {
3630 let filter_registry_handle = ctx.shared_filter_registry();
3632 let compress_flag = ctx.bash_compress_flag();
3633 ctx.bash_background().set_compressor_with_exit_code(
3634 move |command: &str, output: String, exit_code: Option<i32>| {
3635 if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
3636 return crate::compress::CompressionResult::new(output);
3637 }
3638 let registry_guard = match filter_registry_handle.read() {
3639 Ok(g) => g,
3640 Err(poisoned) => poisoned.into_inner(),
3641 };
3642 crate::compress::compress_with_registry_exit_code(
3643 command,
3644 &output,
3645 exit_code,
3646 ®istry_guard,
3647 )
3648 },
3649 );
3650}
3651
3652fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
3653 let mut diagnostics_on_edit = false;
3654 for tier in tiers {
3655 if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
3656 diagnostics_on_edit = value;
3657 }
3658 }
3659 diagnostics_on_edit
3660}
3661
3662fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
3663 let stripped = strip_jsonc(doc);
3664 let value = serde_json::from_str::<Value>(&stripped).ok()?;
3665 value
3666 .get("lsp")
3667 .and_then(Value::as_object)?
3668 .get("diagnostics_on_edit")
3669 .and_then(Value::as_bool)
3670}
3671
3672async fn send_route_bind_error(
3673 tx: &WriterSender,
3674 frame: &Frame,
3675 code: &str,
3676 message: &str,
3677 metrics: &DispatchPathMetrics,
3678) -> Result<(), SubcError> {
3679 send_route_bind_error_parts(
3680 tx,
3681 frame.header.ver,
3682 frame.header.corr,
3683 frame.header.flags,
3684 code,
3685 message,
3686 metrics,
3687 )
3688 .await
3689}
3690
3691async fn send_route_bind_error_parts(
3692 tx: &WriterSender,
3693 ver: u8,
3694 corr: u64,
3695 flags: Flags,
3696 code: &str,
3697 message: &str,
3698 metrics: &DispatchPathMetrics,
3699) -> Result<(), SubcError> {
3700 let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
3701 send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
3702 log::warn!("subc attach: route bind rejected ({code}): {message}");
3703 Ok(())
3704}
3705
3706async fn handle_tool_call(
3711 tx: &WriterSender,
3712 frame: &Frame,
3713 mut phase_trace: PhaseTrace,
3714 routes: &HashMap<RouteChannel, RouteIdentity>,
3715 pending_binds: &HashMap<RouteChannel, PendingBind>,
3716 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3717 executor: &Arc<Executor>,
3718 shutdown: &Arc<Notify>,
3719 connection_cancel: &PersistentCancelSignal,
3720 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
3721 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
3722 metrics: &Arc<DispatchPathMetrics>,
3723 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3724 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3725 next_bash_ask_corr: &mut u64,
3726 bg_subs: &mut HashMap<RouteChannel, BgSub>,
3727 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3728 bg_wake_pending: &mut HashSet<RouteChannel>,
3729 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
3730 dispatch: DispatchFn,
3731 allow_native_passthrough: bool,
3732) -> Result<(), SubcError> {
3733 let route_id = route_key(frame.header.channel, frame.header.epoch);
3734 if pending_binds.contains_key(&route_id) {
3735 let error = build_error_frame(
3736 frame.header.ver,
3737 frame.header.channel,
3738 frame.header.epoch,
3739 frame.header.corr,
3740 frame.header.flags,
3741 "route_not_bound",
3742 "route is not bound before tool call",
3743 )?;
3744 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3745 }
3746
3747 let Some(identity) = routes.get(&route_id).cloned() else {
3748 let error = build_error_frame(
3749 frame.header.ver,
3750 frame.header.channel,
3751 frame.header.epoch,
3752 frame.header.corr,
3753 frame.header.flags,
3754 "route_not_bound",
3755 "route is not bound before tool call",
3756 )?;
3757 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3758 };
3759 let restore_watcher = live_roots
3760 .get(&identity.root)
3761 .is_some_and(|meta| meta.idle_artifacts_evicted);
3762 if let Some(meta) = live_roots.get_mut(&identity.root) {
3763 meta.reactivate_bound();
3764 }
3765 if restore_watcher {
3766 if let Some(ctx) = executor.actor_context(&identity.root) {
3767 crate::commands::configure::ensure_project_watcher(&ctx);
3768 }
3769 }
3770
3771 let route_request =
3772 serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
3773 if matches!(
3774 route_request,
3775 RouteRequest::BgEvents(BgEventsRequest {
3776 op: BgEventsOp::BgEvents
3777 })
3778 ) {
3779 if let Some(old_sub) = bg_subs.get(&route_id).copied() {
3780 push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
3781 }
3782 if !identity.trust.allows_bash_observation() {
3783 bg_subs.remove(&route_id);
3784 bg_wake_pending.remove(&route_id);
3785 remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
3786 let denied_sub = BgSub {
3787 corr: frame.header.corr,
3788 ver: frame.header.ver,
3789 flags: frame.header.flags,
3790 };
3791 push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
3792 return Ok(());
3793 }
3794 bg_subs.insert(
3795 route_id,
3796 BgSub {
3797 corr: frame.header.corr,
3798 ver: frame.header.ver,
3799 flags: frame.header.flags,
3800 },
3801 );
3802 bg_sub_by_session.insert((identity.root.clone(), identity.session.clone()), route_id);
3803 push::arm_bg_wake(
3804 identity.root.clone(),
3805 identity.session.clone(),
3806 route_id,
3807 bg_wake_pending,
3808 bg_wake_epoch,
3809 );
3810 return Ok(());
3811 }
3812
3813 let RouteRequest::ToolCall(call) = route_request else {
3814 unreachable!("background event subscription returned above")
3815 };
3816 let bare_name = call.name;
3817 let arguments = strip_agent_preview_arg_owned(call.arguments);
3818 let format_context = crate::subc_format::FormatContext::from_tool_call(
3819 &bare_name,
3820 &arguments,
3821 identity.project_root.as_path(),
3822 );
3823
3824 let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
3825 let bind_trust = identity.trust;
3826 let diagnostics_on_edit = live_roots
3827 .get(&identity.root)
3828 .map(|meta| meta.diagnostics_on_edit)
3829 .unwrap_or(false);
3830
3831 let requests_host = bare_name == "bash"
3832 && arguments
3833 .get("sandbox")
3834 .or_else(|| {
3835 arguments
3836 .get("params")
3837 .and_then(|params| params.get("sandbox"))
3838 })
3839 .and_then(Value::as_str)
3840 == Some("host");
3841 if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
3842 let response = Response::error(
3843 request_id.clone(),
3844 "sandbox_escalation_denied",
3845 "sandbox host escalation is unavailable to untrusted principals",
3846 );
3847 let text = crate::subc_format::format_response_with_context(
3848 &bare_name,
3849 &response,
3850 &format_context,
3851 );
3852 let result = ToolCallResult { text, response };
3853 let response_frame = build_tool_response_frame(
3854 frame.header.ver,
3855 route_id,
3856 frame.header.corr,
3857 frame.header.flags,
3858 &result,
3859 bind_trust,
3860 )?;
3861 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3862 }
3863
3864 if matches!(bind_trust, BindTrust::Untrusted)
3865 && is_bash_family_tool(&bare_name)
3866 && (bare_name != "bash" || !identity.consumer_elicitation_capable)
3867 {
3868 let response = bash::bash_denied_untrusted_response(request_id.clone());
3869 let text = crate::subc_format::format_response_with_context(
3870 &bare_name,
3871 &response,
3872 &format_context,
3873 );
3874 let result = ToolCallResult { text, response };
3875 let response_frame = build_tool_response_frame(
3876 frame.header.ver,
3877 route_id,
3878 frame.header.corr,
3879 frame.header.flags,
3880 &result,
3881 bind_trust,
3882 )?;
3883 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3884 }
3885
3886 if !is_subc_agent_core_tool(&bare_name)
3894 && !is_subc_native_plumbing_tool(&bare_name)
3895 && !allow_native_passthrough
3896 {
3897 log::warn!(
3898 "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
3899 bare_name,
3900 frame.header.channel
3901 );
3902 let response = Response::error(
3903 request_id.clone(),
3904 "unknown_tool",
3905 format!("tool {:?} is not in the AFT tool manifest", bare_name),
3906 );
3907 let text = crate::subc_format::format_response_with_context(
3908 &bare_name,
3909 &response,
3910 &format_context,
3911 );
3912 let result = ToolCallResult { text, response };
3913 let response_frame = build_tool_response_frame(
3914 frame.header.ver,
3915 route_id,
3916 frame.header.corr,
3917 frame.header.flags,
3918 &result,
3919 bind_trust,
3920 )?;
3921 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3922 }
3923
3924 if bare_name == "bash" {
3925 if matches!(bind_trust, BindTrust::Untrusted) {
3926 let plan = match bash::prepare_bash_elicitation_plan(
3927 &arguments,
3928 identity.project_root.as_path(),
3929 ) {
3930 Ok(plan) => plan,
3931 Err(error) => {
3932 let response = Response::error(request_id.clone(), error.code, error.message);
3933 let text = crate::subc_format::format_response_with_context(
3934 &bare_name,
3935 &response,
3936 &format_context,
3937 );
3938 let result = ToolCallResult { text, response };
3939 let response_frame = build_tool_response_frame(
3940 frame.header.ver,
3941 route_id,
3942 frame.header.corr,
3943 frame.header.flags,
3944 &result,
3945 bind_trust,
3946 )?;
3947 return send_reliable_writer_frame(
3948 tx,
3949 metrics,
3950 response_frame,
3951 "tool response",
3952 )
3953 .await;
3954 }
3955 };
3956
3957 let reverse_corr =
3958 allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
3959 let ask_frame = build_bash_elicitation_request_frame(
3960 frame.header.ver,
3961 route_id,
3962 reverse_corr,
3963 frame.header.flags,
3964 &plan.command,
3965 &plan.asks,
3966 )?;
3967
3968 let meta = live_roots
3969 .entry(identity.root.clone())
3970 .or_insert_with(|| RootMeta::new(Instant::now()));
3971 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
3972 meta.reactivate_bound();
3973
3974 let route_cancel =
3975 route_bash_cancels
3976 .entry(route_id)
3977 .or_insert_with(|| bash::RouteBashCancel {
3978 token: PersistentCancelSignal::new(),
3979 active_waits: 0,
3980 });
3981 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
3982 let cancel = bash::BashWaitCancel {
3983 connection: connection_cancel.clone(),
3984 route: route_cancel.token.clone(),
3985 };
3986 pending_bash_asks.insert(
3987 ReverseCorrKey {
3988 route: route_id,
3989 corr: reverse_corr,
3990 },
3991 PendingBashAsk {
3992 route: route_id,
3993 tool_corr: frame.header.corr,
3994 tool_flags: frame.header.flags,
3995 tool_ver: frame.header.ver,
3996 root: identity.root.clone(),
3997 project_root: identity.project_root.clone(),
3998 session_id: identity.session.clone(),
3999 spawn_principal: identity.spawn_principal.clone(),
4000 request_id,
4001 arguments,
4002 format_context,
4003 cancel,
4004 grants: plan.grants,
4005 expires_at: Instant::now() + bash_elicitation_timeout(),
4006 },
4007 );
4008 return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
4009 .await;
4010 }
4011
4012 let meta = live_roots
4013 .entry(identity.root.clone())
4014 .or_insert_with(|| RootMeta::new(Instant::now()));
4015 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4016 meta.reactivate_bound();
4017
4018 let route_cancel =
4019 route_bash_cancels
4020 .entry(route_id)
4021 .or_insert_with(|| bash::RouteBashCancel {
4022 token: PersistentCancelSignal::new(),
4023 active_waits: 0,
4024 });
4025 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4026 let cancel = bash::BashWaitCancel {
4027 connection: connection_cancel.clone(),
4028 route: route_cancel.token.clone(),
4029 };
4030
4031 bash::submit_deferred_bash(
4032 executor,
4033 bash_deferred_tx,
4034 bash_poll_touch_tx,
4035 metrics,
4036 dispatch,
4037 identity.root.clone(),
4038 identity.project_root.clone(),
4039 identity.session.clone(),
4040 request_id,
4041 route_id,
4042 frame.header.corr,
4043 frame.header.flags,
4044 frame.header.ver,
4045 arguments,
4046 format_context,
4047 cancel,
4048 bind_trust,
4049 identity.spawn_principal.clone(),
4050 None,
4051 );
4052 return Ok(());
4053 }
4054
4055 let lane = command_lane(&bare_name);
4056 let tool_call_context = ToolCallContext {
4057 project_root: identity.project_root.clone(),
4058 session_id: Some(identity.session.clone()),
4059 request_id: request_id.clone(),
4060 diagnostics_on_edit,
4061 preview: call.preview,
4062 };
4063 let bare_name_for_frame = bare_name.clone();
4064 let identity_for_run = identity.clone();
4065 let completion_session = identity.session.clone();
4066 let completion_root = identity.project_root.clone();
4067 let request_id_for_force = request_id.clone();
4068 let format_context_for_frame = format_context.clone();
4069 let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
4070 phase_trace.mark_executor_submitted();
4071 let rx = executor.submit_async(
4072 identity.root.clone(),
4073 lane,
4074 request_id.clone(),
4075 Box::new(move |ctx| {
4076 phase_trace.mark_job_admitted();
4077 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4078 let run = || {
4079 let finalizer = |response: &mut Response| {
4080 crate::response_finalize::finalize_response_with_bg_completions(
4081 response,
4082 ctx,
4083 &identity_for_run.session,
4084 &bare_name,
4085 bind_trust.allows_bash_observation(),
4086 );
4087 };
4088 match run_tool_call(
4089 &bare_name,
4090 arguments,
4091 &format_context,
4092 &tool_call_context,
4093 ctx,
4094 &dispatch,
4095 Some(&finalizer),
4096 Some(&mut phase_trace),
4097 ) {
4098 ToolCallOutcome::Unary(result) => {
4099 let response = result.response;
4100 let _ = tool_call_tx.send(ToolCallCompletion {
4101 text: result.text,
4102 phase_trace,
4103 });
4104 response
4105 }
4106 }
4107 };
4108 if matches!(bind_trust, BindTrust::Untrusted) {
4109 ctx.with_force_restrict(&request_id_for_force, run)
4110 } else {
4111 run()
4112 }
4113 })
4114 }),
4115 );
4116 let completion_tx = tx.clone();
4117 let completion_shutdown = Arc::clone(shutdown);
4118 let route = route_id;
4119 let corr = frame.header.corr;
4120 let flags = frame.header.flags;
4121 let ver = frame.header.ver;
4122 let completion_metrics = Arc::clone(metrics);
4123 tokio::spawn(async move {
4124 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4125 let response = await_executor_response(rx, request_id.clone()).await;
4126 let (text, phase_trace) = match tool_call_rx.await {
4127 Ok(completion) => (completion.text, Some(completion.phase_trace)),
4128 Err(_) => (
4129 crate::subc_format::format_response_with_context(
4130 &bare_name_for_frame,
4131 &response,
4132 &format_context_for_frame,
4133 ),
4134 None,
4135 ),
4136 };
4137 let result = ToolCallResult { text, response };
4138 let fatal = response_is_fatal_panic(&result.response);
4139 match build_tool_response_frame(ver, route, corr, flags, &result, bind_trust) {
4140 Ok(response_frame) => {
4141 let send_result = if let Some(phase_trace) = phase_trace {
4142 let trace = ToolResponseWriteTrace::new(
4143 phase_trace,
4144 bare_name_for_frame,
4145 completion_root,
4146 completion_session,
4147 route.channel,
4148 corr,
4149 );
4150 send_traced_tool_response_frame(
4151 &completion_tx,
4152 &completion_metrics,
4153 response_frame,
4154 trace,
4155 )
4156 .await
4157 } else {
4158 send_reliable_writer_frame(
4159 &completion_tx,
4160 &completion_metrics,
4161 response_frame,
4162 "tool response",
4163 )
4164 .await
4165 };
4166 if let Err(error) = send_result {
4167 log::warn!("subc attach: failed to queue tool response frame: {error}");
4168 }
4169 }
4170 Err(error) => {
4171 log::error!("subc attach: failed to build tool response frame: {error}");
4172 }
4173 }
4174 if fatal {
4175 signal_fatal_teardown(
4176 &completion_tx,
4177 Some(route),
4178 ver,
4179 corr,
4180 &completion_shutdown,
4181 &completion_metrics,
4182 )
4183 .await;
4184 }
4185 });
4186 Ok(())
4187}
4188
4189fn submit_maintenance_job(
4190 executor: &Arc<Executor>,
4191 root_id: ProjectRootId,
4192 kind: MaintenanceDrainKind,
4193 bg_sessions_to_check: Vec<(String, u64)>,
4194 completion_tx: &mpsc::Sender<MaintenanceCompletion>,
4195 metrics: &Arc<DispatchPathMetrics>,
4196) {
4197 let request_id = format!(
4198 "subc-maintenance-drain-{}-{}",
4199 kind.label(),
4200 root_id.as_path().to_string_lossy()
4201 );
4202 let response_id = request_id.clone();
4203 let completion_root_id = root_id.clone();
4204 let maintenance_generation = executor
4205 .actor_context(&root_id)
4206 .map(|ctx| ctx.configure_generation())
4207 .unwrap_or(0);
4208 let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
4209 let lane = match kind {
4214 MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
4215 MaintenanceDrainKind::Watcher
4216 | MaintenanceDrainKind::Lsp
4217 | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
4218 };
4219 let rx = executor.submit_maintenance_async(
4220 root_id,
4221 lane,
4222 request_id.clone(),
4223 Box::new(move |ctx| {
4224 let outcome = match kind {
4225 MaintenanceDrainKind::Watcher => {
4226 let drained = runtime_drain::drain_watcher_events_bounded(
4227 ctx,
4228 runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
4229 );
4230 MaintenanceJobOutcome {
4231 empty_bg_sessions: Vec::new(),
4232 requeue_kind: drained.has_more.then_some(kind),
4233 }
4234 }
4235 MaintenanceDrainKind::Lsp => {
4236 let drained = runtime_drain::drain_lsp_events_bounded(
4237 ctx,
4238 runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
4239 );
4240 MaintenanceJobOutcome {
4241 empty_bg_sessions: Vec::new(),
4242 requeue_kind: drained.has_more.then_some(kind),
4243 }
4244 }
4245 MaintenanceDrainKind::ConfigureTail => {
4246 runtime_drain::drain_deferred_configure_maintenance(ctx);
4247 runtime_drain::drain_configure_warning_events(ctx);
4248 MaintenanceJobOutcome::default()
4249 }
4250 MaintenanceDrainKind::CompletionDrains => {
4251 runtime_drain::drain_search_index_events(ctx);
4252 runtime_drain::drain_callgraph_store_events(ctx);
4253 runtime_drain::drain_semantic_index_events(ctx);
4254 runtime_drain::drain_semantic_refresh_events(ctx);
4255 runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
4256 let empty_bg_sessions = bg_sessions_to_check
4257 .into_iter()
4258 .filter(|(session, _)| {
4259 !ctx.bash_background()
4260 .has_completions_for_session(Some(session.as_str()))
4261 })
4262 .collect();
4263 MaintenanceJobOutcome {
4264 empty_bg_sessions,
4265 requeue_kind: None,
4266 }
4267 }
4268 };
4269 let requeued = outcome.requeue_kind.is_some();
4270 let _ = outcome_tx.send(outcome);
4271 Response::success(
4272 response_id,
4273 json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
4274 )
4275 }),
4276 );
4277 let completion_tx = completion_tx.clone();
4278 let completion_metrics = Arc::clone(metrics);
4279 tokio::spawn(async move {
4280 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4281 let response = await_executor_response(rx, request_id).await;
4282 let outcome = outcome_rx.await.unwrap_or_default();
4283 let _ = send_counted_channel(
4284 &completion_tx,
4285 &completion_metrics.maintenance_queued,
4286 MaintenanceCompletion {
4287 root_id: completion_root_id,
4288 kind,
4289 response,
4290 empty_bg_sessions: outcome.empty_bg_sessions,
4291 requeue_kind: outcome.requeue_kind,
4292 },
4293 )
4294 .await;
4295 });
4296}
4297
4298async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
4299 rx.await
4300 .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
4301}
4302async fn signal_fatal_teardown(
4303 tx: &WriterSender,
4304 route: Option<RouteChannel>,
4305 ver: u8,
4306 corr: u64,
4307 shutdown: &Arc<Notify>,
4308 metrics: &DispatchPathMetrics,
4309) {
4310 if let Some(route) = route {
4311 if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
4312 if let Err(error) = send_frame(tx, metrics, frame).await {
4313 log::warn!(
4314 "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
4315 );
4316 }
4317 }
4318 }
4319 if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
4320 if let Err(error) = send_frame(tx, metrics, frame).await {
4321 log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
4322 }
4323 }
4324 shutdown.notify_one();
4325}
4326#[derive(Debug, Deserialize)]
4327#[serde(untagged)]
4328enum RouteRequest {
4329 BgEvents(BgEventsRequest),
4330 ToolCall(ToolCallRequest),
4331}
4332
4333#[derive(Debug, Deserialize)]
4334struct BgEventsRequest {
4335 op: BgEventsOp,
4336}
4337
4338#[derive(Debug, Deserialize)]
4339#[serde(rename_all = "snake_case")]
4340enum BgEventsOp {
4341 BgEvents,
4342}
4343
4344#[derive(Debug, Deserialize)]
4345struct ToolCallRequest {
4346 name: String,
4347 #[serde(default)]
4348 arguments: Value,
4349 #[serde(default)]
4354 preview: bool,
4355}
4356
4357#[cfg(test)]
4358pub(crate) mod test_support {
4359 use super::*;
4360 use crate::bash_background::BgTaskStatus;
4361 use crate::protocol::{
4362 BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
4363 ProgressFrame, StatusChangedFrame,
4364 };
4365 use serde_json::json;
4366
4367 pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
4368 let dir = tempfile::Builder::new()
4369 .prefix(name)
4370 .tempdir()
4371 .expect("temp root");
4372 let root = ProjectRootId::from_path(dir.path()).expect("project root id");
4373 (dir, root)
4374 }
4375
4376 pub(super) fn test_ctx() -> Arc<AppContext> {
4377 Arc::new(AppContext::new(
4378 Box::new(crate::parser::TreeSitterProvider::new()),
4379 crate::config::Config::default(),
4380 ))
4381 }
4382
4383 pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
4384 let deadline = Instant::now() + Duration::from_secs(30);
4385 loop {
4386 let observed = ctx.watcher_registry_count();
4387 if observed == expected {
4388 return;
4389 }
4390 assert!(
4391 Instant::now() < deadline,
4392 "watcher count did not settle before deadline: expected={expected}, observed={observed}"
4393 );
4394 std::thread::sleep(Duration::from_millis(50));
4395 }
4396 }
4397
4398 pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
4399 let deadline = Instant::now() + Duration::from_secs(30);
4400 loop {
4401 let observed = app.actor_root_count();
4402 if observed == expected {
4403 return;
4404 }
4405 assert!(
4406 Instant::now() < deadline,
4407 "actor root count did not settle before deadline: expected={expected}, observed={observed}"
4408 );
4409 std::thread::sleep(Duration::from_millis(50));
4410 }
4411 }
4412
4413 pub(super) fn status_frame(seq: u64) -> PushFrame {
4414 status_frame_with_session(seq, None)
4415 }
4416
4417 pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
4418 PushFrame::StatusChanged(StatusChangedFrame {
4419 frame_type: "status_changed",
4420 session_id: session_id.map(str::to_string),
4421 snapshot: json!({ "seq": seq }),
4422 })
4423 }
4424
4425 pub(super) fn completion_frame(task_id: &str) -> PushFrame {
4426 completion_frame_with_session(task_id, "session-1")
4427 }
4428
4429 pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
4430 PushFrame::BashCompleted(BashCompletedFrame {
4431 frame_type: "bash_completed",
4432 task_id: task_id.to_string(),
4433 session_id: session_id.to_string(),
4434 status: BgTaskStatus::Completed,
4435 exit_code: Some(0),
4436 command: format!("echo {task_id}"),
4437 output_preview: String::new(),
4438 output_truncated: false,
4439 original_tokens: None,
4440 compressed_tokens: None,
4441 tokens_skipped: false,
4442 })
4443 }
4444
4445 pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
4446 long_running_frame_with_session(task_id, "session-1", elapsed_ms)
4447 }
4448
4449 pub(super) fn long_running_frame_with_session(
4450 task_id: &str,
4451 session_id: &str,
4452 elapsed_ms: u64,
4453 ) -> PushFrame {
4454 PushFrame::BashLongRunning(BashLongRunningFrame {
4455 frame_type: "bash_long_running",
4456 task_id: task_id.to_string(),
4457 session_id: session_id.to_string(),
4458 command: format!("sleep {elapsed_ms}"),
4459 elapsed_ms,
4460 })
4461 }
4462
4463 pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
4464 PushFrame::BashPatternMatch(BashPatternMatchFrame {
4465 frame_type: "bash_pattern_match",
4466 task_id: "task-pattern".to_string(),
4467 session_id: session_id.to_string(),
4468 watch_id: "watch-1".to_string(),
4469 match_text: "needle".to_string(),
4470 match_offset: 7,
4471 context: "haystack needle".to_string(),
4472 once: true,
4473 reason: "pattern_match",
4474 })
4475 }
4476
4477 pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
4478 PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
4479 frame_type: "configure_warnings",
4480 session_id: session_id.map(str::to_string),
4481 project_root: "/tmp/subc-test".to_string(),
4482 warnings: Vec::new(),
4483 })
4484 }
4485
4486 pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
4487 route_identity_with_trust(root, session_id, BindTrust::FirstParty)
4488 }
4489
4490 pub(super) fn route_identity_with_trust(
4491 root: &ProjectRootId,
4492 session_id: &str,
4493 trust: BindTrust,
4494 ) -> RouteIdentity {
4495 RouteIdentity(Arc::new(RouteIdentityData {
4496 root: root.clone(),
4497 project_root: root.as_path().to_path_buf(),
4498 harness: "opencode".to_string(),
4499 session: session_id.to_string(),
4500 trust,
4501 spawn_principal: AuthenticatedPrincipal::RouteBind {
4502 trust: trust.sandbox_trust(),
4503 route_channel: 0,
4504 route_epoch: 0,
4505 project_root: root.as_path().to_path_buf(),
4506 harness: "opencode".to_string(),
4507 session_id: session_id.to_string(),
4508 principal_id: Some(match trust {
4509 BindTrust::FirstParty => "direct".to_string(),
4510 BindTrust::Untrusted => "unverified".to_string(),
4511 }),
4512 },
4513 consumer_elicitation_capable: false,
4514 }))
4515 }
4516
4517 pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
4518 PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
4519 }
4520
4521 pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
4522 match frame {
4523 PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
4524 _ => None,
4525 }
4526 }
4527
4528 pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
4529 match frame {
4530 PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
4531 _ => None,
4532 }
4533 }
4534
4535 pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
4536 let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
4537 body.get("task_id")
4538 .and_then(serde_json::Value::as_str)
4539 .map(str::to_string)
4540 }
4541}
4542
4543#[cfg(test)]
4544mod tests {
4545 use super::test_support::{
4546 completion_frame, route_identity, test_ctx, test_root, wait_for_actor_root_count,
4547 wait_for_watcher_count,
4548 };
4549 use super::*;
4550
4551 fn attach_error(kind: io::ErrorKind) -> SubcError {
4552 SubcError::Connect {
4553 endpoint: "127.0.0.1:1".to_string(),
4554 source: io::Error::new(kind, "constructed attach failure"),
4555 }
4556 }
4557
4558 fn auth_io_error(kind: io::ErrorKind) -> SubcError {
4559 SubcError::Auth {
4560 endpoint: "127.0.0.1:1".to_string(),
4561 source: subc_transport::AuthError::Io {
4562 stage: subc_transport::AuthStage::ServerProof,
4563 source: io::Error::new(kind, "constructed auth failure"),
4564 },
4565 }
4566 }
4567
4568 #[test]
4569 fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
4570 let transient_errors = vec![
4571 attach_error(io::ErrorKind::ConnectionRefused),
4572 attach_error(io::ErrorKind::TimedOut),
4573 attach_error(io::ErrorKind::ConnectionReset),
4574 auth_io_error(io::ErrorKind::ConnectionAborted),
4575 auth_io_error(io::ErrorKind::BrokenPipe),
4576 SubcError::Auth {
4577 endpoint: "127.0.0.1:1".to_string(),
4578 source: subc_transport::AuthError::UnexpectedEof {
4579 stage: subc_transport::AuthStage::ServerProof,
4580 expected: 4,
4581 actual: 0,
4582 },
4583 },
4584 SubcError::Auth {
4585 endpoint: "127.0.0.1:1".to_string(),
4586 source: subc_transport::AuthError::Timeout {
4587 stage: subc_transport::AuthStage::ServerProof,
4588 deadline: AUTH_DEADLINE,
4589 },
4590 },
4591 ];
4592 for error in &transient_errors {
4593 assert_eq!(
4594 classify_attach_error(error),
4595 AttachErrorClass::Transient,
4596 "expected transient: {error}"
4597 );
4598 }
4599
4600 let permanent_errors = vec![
4601 attach_error(io::ErrorKind::PermissionDenied),
4602 auth_io_error(io::ErrorKind::InvalidData),
4603 SubcError::Auth {
4604 endpoint: "127.0.0.1:1".to_string(),
4605 source: subc_transport::AuthError::InvalidServerProof,
4606 },
4607 SubcError::Auth {
4608 endpoint: "127.0.0.1:1".to_string(),
4609 source: subc_transport::AuthError::DaemonIdMismatch,
4610 },
4611 SubcError::ConnectionFile {
4612 path: PathBuf::from("subc-connection.json"),
4613 source: subc_transport::ConnectionFileError::Invalid {
4614 reason: "constructed invalid file".to_string(),
4615 },
4616 },
4617 SubcError::NoEndpoint {
4618 path: PathBuf::from("subc-connection.json"),
4619 },
4620 SubcError::InvalidEndpoint {
4621 path: PathBuf::from("subc-connection.json"),
4622 endpoint: "not-an-ip:1234".to_string(),
4623 },
4624 ];
4625 for error in &permanent_errors {
4626 assert_eq!(
4627 classify_attach_error(error),
4628 AttachErrorClass::Permanent,
4629 "expected permanent: {error}"
4630 );
4631 }
4632 }
4633
4634 #[test]
4635 fn incompatible_wire_version_is_rejected_before_tcp_connect() {
4636 let conn_dir = tempfile::tempdir().expect("connection tempdir");
4637 let conn_path = conn_dir.path().join("subc-connection.json");
4638 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
4639 listener
4640 .set_nonblocking(true)
4641 .expect("set listener nonblocking");
4642 let port = listener.local_addr().expect("listener addr").port();
4643 connection_file::write_atomic(
4644 &conn_path,
4645 &connection_file::ConnectionInfo {
4646 schema: connection_file::SCHEMA_VERSION,
4647 wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
4648 endpoints: vec![connection_file::Endpoint {
4649 host: "127.0.0.1".to_string(),
4650 port,
4651 }],
4652 key: vec![0x42; subc_transport::KEY_LEN],
4653 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4654 pid: std::process::id(),
4655 daemon_ver: "subc-test".to_string(),
4656 },
4657 )
4658 .expect("write connection file");
4659
4660 let runtime = tokio::runtime::Builder::new_current_thread()
4661 .enable_all()
4662 .build()
4663 .expect("test runtime");
4664 let result = runtime.block_on(connect_and_authenticate_with_policy(
4665 &conn_path,
4666 AttachRetryPolicy {
4667 budget: Duration::from_secs(1),
4668 initial_backoff: Duration::from_millis(5),
4669 max_backoff: Duration::from_millis(10),
4670 jitter_percent: 0,
4671 },
4672 ));
4673 assert!(matches!(
4674 result,
4675 Err(SubcError::ConnectionFile {
4676 source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
4677 ..
4678 })
4679 ));
4680 assert!(matches!(
4681 listener.accept(),
4682 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
4683 ));
4684 }
4685
4686 #[test]
4687 fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
4688 let conn_dir = tempfile::tempdir().expect("connection tempdir");
4689 let conn_path = conn_dir.path().join("subc-connection.json");
4690 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
4691 let port = listener.local_addr().expect("reserved addr").port();
4692 drop(listener);
4693 connection_file::write_atomic(
4694 &conn_path,
4695 &connection_file::ConnectionInfo {
4696 schema: connection_file::SCHEMA_VERSION,
4697 wire_version: Some(PROTOCOL_VERSION),
4698 endpoints: vec![connection_file::Endpoint {
4699 host: "127.0.0.1".to_string(),
4700 port,
4701 }],
4702 key: vec![0x42; subc_transport::KEY_LEN],
4703 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4704 pid: std::process::id(),
4705 daemon_ver: "subc-test".to_string(),
4706 },
4707 )
4708 .expect("write connection file");
4709
4710 let policy = AttachRetryPolicy {
4711 budget: Duration::from_millis(40),
4712 initial_backoff: Duration::from_millis(5),
4713 max_backoff: Duration::from_millis(10),
4714 jitter_percent: 0,
4715 };
4716 let runtime = tokio::runtime::Builder::new_current_thread()
4717 .enable_all()
4718 .build()
4719 .expect("test runtime");
4720 let started_at = Instant::now();
4721 let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
4722 let elapsed = started_at.elapsed();
4723 let error = match result {
4724 Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
4725 Err(error) => error,
4726 };
4727
4728 assert!(matches!(error, SubcError::Connect { .. }), "{error}");
4729 assert!(
4730 elapsed >= Duration::from_millis(35),
4731 "retry budget ended too early: {elapsed:?}"
4732 );
4733 assert!(
4734 elapsed < Duration::from_secs(1),
4735 "retry budget was not bounded: {elapsed:?}"
4736 );
4737 }
4738
4739 fn due_maintenance_jobs_without_actor_context(
4740 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4741 budget: usize,
4742 pending_bind_roots: &HashSet<ProjectRootId>,
4743 ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
4744 due_maintenance_jobs(
4745 live_roots,
4746 None,
4747 &HashMap::new(),
4748 &HashSet::new(),
4749 budget,
4750 pending_bind_roots,
4751 )
4752 }
4753
4754 fn actor_ctx_with_dirty_search_index(
4755 root: &Path,
4756 storage: &Path,
4757 file_name: &str,
4758 old_contents: &str,
4759 new_contents: &str,
4760 ) -> (Arc<AppContext>, PathBuf, PathBuf) {
4761 let file = root.join(file_name);
4762 std::fs::write(&file, old_contents).expect("write source");
4763 let canonical_root = std::fs::canonicalize(root).expect("canonical root");
4764 let ctx = Arc::new(AppContext::new(
4765 Box::new(crate::parser::TreeSitterProvider::new()),
4766 Config {
4767 project_root: Some(root.to_path_buf()),
4768 storage_dir: Some(storage.to_path_buf()),
4769 ..Config::default()
4770 },
4771 ));
4772 ctx.set_canonical_cache_root(canonical_root.clone());
4773
4774 let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
4775 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
4776 let git_head = index.stored_git_head().map(str::to_owned);
4777 index.write_to_disk(&cache_dir, git_head.as_deref());
4778
4779 std::fs::write(&file, new_contents).expect("edit source");
4780 index.update_file(&file);
4781 *ctx.search_index()
4782 .write()
4783 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
4784 (ctx, canonical_root, cache_dir)
4785 }
4786
4787 #[test]
4788 fn graceful_shutdown_flushes_every_actor_search_index() {
4789 let storage = tempfile::tempdir().expect("storage tempdir");
4790 let (root1_dir, root1) = test_root("shutdown-flush-root-1");
4791 let (root2_dir, root2) = test_root("shutdown-flush-root-2");
4792 let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
4793 root1_dir.path(),
4794 storage.path(),
4795 "alpha.txt",
4796 "old actor one token\n",
4797 "new actor one token\n",
4798 );
4799 let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
4800 root2_dir.path(),
4801 storage.path(),
4802 "beta.txt",
4803 "old actor two token\n",
4804 "new actor two token\n",
4805 );
4806
4807 let executor = Executor::new();
4808 assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
4809 assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
4810
4811 flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
4812
4813 let mut restored1 =
4814 crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
4815 .expect("load flushed root one index");
4816 restored1.ready = true;
4817 assert_eq!(
4818 restored1
4819 .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
4820 .matches
4821 .len(),
4822 1,
4823 "graceful subc shutdown should flush the first root's trigram delta"
4824 );
4825
4826 let mut restored2 =
4827 crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
4828 .expect("load flushed root two index");
4829 restored2.ready = true;
4830 assert_eq!(
4831 restored2
4832 .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
4833 .matches
4834 .len(),
4835 1,
4836 "graceful subc shutdown should flush every registered root"
4837 );
4838 }
4839
4840 #[test]
4841 fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
4842 let _ = env_logger::builder().is_test(true).try_init();
4843 let (root_dir, root) = test_root("idle-root-reaper");
4844 let storage = tempfile::tempdir().expect("storage tempdir");
4845 std::fs::write(
4846 root_dir.path().join("main.rs"),
4847 "fn entry() { leaf(); }\nfn leaf() {}\n",
4848 )
4849 .expect("source file");
4850 let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
4851 let app = App::default_shared();
4852 let ctx = Arc::new(AppContext::from_app(
4853 Arc::clone(&app),
4854 Config {
4855 project_root: Some(canonical_root.clone()),
4856 storage_dir: Some(storage.path().to_path_buf()),
4857 callgraph_store: true,
4858 search_index: true,
4859 ..Config::default()
4860 },
4861 ));
4862 ctx.set_canonical_cache_root(canonical_root.clone());
4863 assert!(ctx
4864 .ensure_callgraph_store()
4865 .expect("build callgraph store")
4866 .is_some());
4867
4868 let cache_dir =
4869 crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
4870 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
4871 let git_head = index.stored_git_head().map(str::to_owned);
4872 index.write_to_disk(&cache_dir, git_head.as_deref());
4873 *ctx.search_index()
4874 .write()
4875 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
4876 let seeded_generation =
4879 crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
4880 .expect("seeded artifact generation");
4881 crate::cache_freshness::record_verify_completed(
4882 &canonical_root,
4883 crate::cache_freshness::VerifyArtifact::Search,
4884 Some(seeded_generation),
4885 );
4886 assert!(
4887 matches!(
4888 crate::cache_freshness::warm_verify_plan(
4889 canonical_root.as_path(),
4890 crate::cache_freshness::VerifyArtifact::Search,
4891 Some(seeded_generation),
4892 ),
4893 crate::cache_freshness::WarmVerifyPlan::Skip
4894 ),
4895 "memo must be warm before eviction for the downgrade assertion to bite"
4896 );
4897
4898 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
4899 let _dispatch_tx = dispatch_tx;
4900 let shutdown = Arc::new(AtomicBool::new(false));
4901 let thread_shutdown = Arc::clone(&shutdown);
4902 let join = std::thread::spawn(move || {
4903 while !thread_shutdown.load(Ordering::SeqCst) {
4904 std::thread::yield_now();
4905 }
4906 });
4907 ctx.install_watcher_runtime(
4908 dispatch_rx,
4909 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
4910 );
4911 wait_for_watcher_count(&ctx, 1);
4912
4913 let executor = Arc::new(Executor::new());
4914 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4915 ctx.mark_subc_unbound();
4916 let mut live_roots = HashMap::new();
4917 let mut meta = RootMeta::new(Instant::now());
4918 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
4919 meta.unbound_quiesced = true;
4920 live_roots.insert(root.clone(), meta);
4921
4922 let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
4923 assert!(message.contains("evicted idle root"));
4924 assert!(message.contains("freed ~"));
4925 assert!(message.contains("semantic"));
4926 assert!(!message.contains("semantic not estimated retained"));
4927 assert!(message.contains("trigram"));
4928 assert!(message.contains("retained: bash"));
4929 assert!(message.contains("parser_pool"));
4930
4931 assert_eq!(
4932 reap_idle_roots(
4933 Instant::now(),
4934 &mut live_roots,
4935 &HashMap::new(),
4936 &HashMap::new(),
4937 &executor,
4938 &DispatchPathMetrics::new(),
4939 )
4940 .evicted,
4941 1
4942 );
4943 assert!(ctx.search_index().read().unwrap().is_none());
4944 wait_for_watcher_count(&ctx, 0);
4945 assert!(
4950 matches!(
4951 crate::cache_freshness::warm_verify_plan(
4952 canonical_root.as_path(),
4953 crate::cache_freshness::VerifyArtifact::Search,
4954 Some(seeded_generation),
4955 ),
4956 crate::cache_freshness::WarmVerifyPlan::Strict
4957 ),
4958 "idle eviction must force strict re-verification"
4959 );
4960 assert!(
4961 crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
4962 );
4963 ctx.mark_subc_bound();
4964 assert!(ctx
4965 .ensure_callgraph_store()
4966 .expect("reopen callgraph store")
4967 .is_some());
4968 assert!(live_roots[&root].idle_artifacts_evicted);
4969 }
4970
4971 #[test]
4972 fn idle_root_reaper_applies_ttl_to_unbound_roots() {
4973 let (_root_dir, root) = test_root("idle-root-ttl-gate");
4974 let ctx = test_ctx();
4975 let executor = Arc::new(Executor::new());
4976 assert!(executor.register_actor(root.clone(), ctx));
4977 let ctx = executor.actor_context(&root).expect("actor context");
4978 ctx.mark_subc_unbound();
4979 let now = Instant::now();
4980 let mut meta = RootMeta::new(now);
4981 meta.unbound_quiesced = true;
4982 let mut live_roots = HashMap::from([(root.clone(), meta)]);
4983
4984 assert_eq!(
4988 reap_idle_roots(
4989 now,
4990 &mut live_roots,
4991 &HashMap::new(),
4992 &HashMap::new(),
4993 &executor,
4994 &DispatchPathMetrics::new(),
4995 )
4996 .evicted,
4997 0
4998 );
4999 assert!(!live_roots[&root].idle_artifacts_evicted);
5000
5001 ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
5007 assert_eq!(
5008 reap_idle_roots(
5009 now + IDLE_ROOT_TTL,
5010 &mut live_roots,
5011 &HashMap::new(),
5012 &HashMap::new(),
5013 &executor,
5014 &DispatchPathMetrics::new(),
5015 )
5016 .evicted,
5017 1
5018 );
5019 assert!(live_roots[&root].idle_artifacts_evicted);
5020 assert!(
5021 ctx.take_pending_search_index_paths().is_empty(),
5022 "TTL eviction must dispose retained pending reconciliation paths"
5023 );
5024 }
5025
5026 #[test]
5027 fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
5028 let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
5029 let ctx = test_ctx();
5030 let executor = Arc::new(Executor::new());
5031 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5032 ctx.mark_subc_unbound();
5033
5034 let pending = root.as_path().join("edited-while-unbound.rs");
5040 ctx.add_pending_search_index_paths([pending.clone()]);
5041 let dirty_source = root.as_path().join("dirty.rs");
5042 std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
5043 let mut dirty = crate::search_index::SearchIndex::new();
5044 dirty.ready = true;
5045 dirty.update_file(&dirty_source);
5046 *ctx.search_index()
5047 .write()
5048 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
5049 assert!(ctx.artifact_eviction_blocked());
5050
5051 let mut live_roots = HashMap::new();
5052 let mut meta = RootMeta::new(Instant::now());
5053 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5054 meta.unbound_quiesced = true;
5055 live_roots.insert(root.clone(), meta);
5056
5057 assert_eq!(
5058 reap_idle_roots(
5059 Instant::now(),
5060 &mut live_roots,
5061 &HashMap::new(),
5062 &HashMap::new(),
5063 &executor,
5064 &DispatchPathMetrics::new(),
5065 )
5066 .evicted,
5067 0,
5068 "the dirty index must still block this eviction"
5069 );
5070 assert_eq!(
5071 ctx.take_pending_search_index_paths(),
5072 vec![pending],
5073 "a blocked eviction must restore the taken pending paths"
5074 );
5075 }
5076
5077 #[test]
5078 fn idle_reap_with_bound_route_keeps_watcher_running() {
5079 let (_root_dir, root) = test_root("bound-root-reap-gate");
5080 let ctx = test_ctx();
5081 let executor = Arc::new(Executor::new());
5082 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5083
5084 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5085 let _dispatch_tx = dispatch_tx;
5086 let shutdown = Arc::new(AtomicBool::new(false));
5087 let thread_shutdown = Arc::clone(&shutdown);
5088 let join = std::thread::spawn(move || {
5089 while !thread_shutdown.load(Ordering::SeqCst) {
5090 std::thread::yield_now();
5091 }
5092 });
5093 ctx.install_watcher_runtime(
5094 dispatch_rx,
5095 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5096 );
5097
5098 let mut meta = RootMeta::new(Instant::now());
5099 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5100 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5101 let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
5102 assert_eq!(
5103 reap_idle_roots(
5104 Instant::now(),
5105 &mut live_roots,
5106 &HashMap::new(),
5107 &bound,
5108 &executor,
5109 &DispatchPathMetrics::new(),
5110 )
5111 .evicted,
5112 0
5113 );
5114 wait_for_watcher_count(&ctx, 1);
5115 ctx.stop_watcher_runtime_in_background();
5116 wait_for_watcher_count(&ctx, 0);
5117 }
5118
5119 #[test]
5120 fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
5121 let (root_dir, root) = test_root("deleted-root-reap");
5122 let app = App::default_shared();
5123 let ctx = Arc::new(AppContext::from_app(
5124 Arc::clone(&app),
5125 Config {
5126 project_root: Some(root.as_path().to_path_buf()),
5127 ..Config::default()
5128 },
5129 ));
5130 ctx.set_canonical_cache_root(root.as_path().to_path_buf());
5131 ctx.mark_subc_unbound();
5132 let executor = Arc::new(Executor::new());
5133 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5134 assert_eq!(app.actor_root_count(), 1);
5135 drop(ctx);
5136 root_dir.close().expect("delete project root");
5137
5138 let mut meta = RootMeta::new(Instant::now());
5139 meta.unbound_quiesced = true;
5140 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5141 let outcome = reap_idle_roots(
5142 Instant::now(),
5143 &mut live_roots,
5144 &HashMap::new(),
5145 &HashMap::new(),
5146 &executor,
5147 &DispatchPathMetrics::new(),
5148 );
5149 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5150 assert!(!executor.actor_registered(&root));
5151 assert!(!live_roots.contains_key(&root));
5152 wait_for_actor_root_count(&app, 0);
5153
5154 let status_ctx = AppContext::from_app(app, Config::default());
5155 let status = status_ctx.build_status_snapshot();
5156 assert_eq!(status["runtime"]["live_actor_roots"], 0);
5157 assert_eq!(status["runtime"]["open_routes"], 0);
5158 }
5159
5160 #[test]
5161 fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
5162 let (root_dir, root) = test_root("deleted-root-reap-census");
5163 let executor = Arc::new(Executor::new());
5164 assert!(executor.register_actor(root.clone(), test_ctx()));
5165 root_dir.close().expect("delete project root");
5166
5167 let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
5168 let metrics = DispatchPathMetrics::new();
5169 let outcome = reap_idle_roots(
5170 Instant::now(),
5171 &mut live_roots,
5172 &HashMap::new(),
5173 &HashMap::new(),
5174 &executor,
5175 &metrics,
5176 );
5177 assert_eq!(outcome.evicted, 0);
5178
5179 let report = build_health_report(
5180 &executor,
5181 &HashMap::new(),
5182 &metrics,
5183 &crate::context::App::default_shared(),
5184 );
5185 let reap = report
5186 .metrics
5187 .as_ref()
5188 .and_then(|metrics| metrics.get("reap"))
5189 .expect("reap health metrics");
5190 assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
5191 assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(1));
5192 assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
5193 }
5194
5195 #[test]
5196 fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
5197 let (root_dir, root) = test_root("connection-exit-deleted-root");
5198 let executor = Arc::new(Executor::new());
5199 assert!(executor.register_actor(root.clone(), test_ctx()));
5200
5201 let route = route_key(11, 1);
5202 let mut meta = RootMeta::new(Instant::now());
5203 meta.maintenance_pending = true;
5204 meta.maintenance_queued_kinds
5205 .push_back(MaintenanceDrainKind::CompletionDrains);
5206 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5207 let mut pending_binds = HashMap::new();
5208 let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
5209 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5210 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5211 let mut route_bash_cancels = HashMap::new();
5212
5213 quiesce_connection_roots(
5214 &mut live_roots,
5215 &mut pending_binds,
5216 &mut routes,
5217 &mut root_channels,
5218 &mut installed_route_epochs,
5219 &mut route_bash_cancels,
5220 &executor,
5221 );
5222 assert!(live_roots[&root].unbound_quiesced);
5223 assert!(!live_roots[&root].maintenance_pending);
5224 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
5225 assert!(routes.is_empty());
5226 assert!(root_channels.is_empty());
5227
5228 root_dir.close().expect("delete project root");
5229 let metrics = DispatchPathMetrics::new();
5230 let outcome = reap_idle_roots(
5231 Instant::now(),
5232 &mut live_roots,
5233 &pending_binds,
5234 &root_channels,
5235 &executor,
5236 &metrics,
5237 );
5238 let mut session_identity = HashMap::new();
5239 let mut push_buffer = HashMap::new();
5240 let mut bg_subs = HashMap::new();
5241 let mut bg_sub_by_session = HashMap::new();
5242 let mut bg_wake_pending = HashSet::new();
5243 let mut bg_wake_epoch = HashMap::new();
5244 let mut pending_bash_asks = HashMap::new();
5245 for forgotten in &outcome.forgotten_deleted_roots {
5246 purge_deleted_root_residents(
5247 forgotten,
5248 &mut root_channels,
5249 &mut session_identity,
5250 &mut push_buffer,
5251 &mut bg_subs,
5252 &mut bg_sub_by_session,
5253 &mut bg_wake_pending,
5254 &mut bg_wake_epoch,
5255 &mut pending_bash_asks,
5256 );
5257 }
5258
5259 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5260 assert!(!executor.actor_registered(&root));
5261 assert!(!live_roots.contains_key(&root));
5262 }
5263
5264 #[test]
5265 fn unbound_root_quiesces_maintenance_without_removing_actor() {
5266 let (_root_dir, root) = test_root("unbound-root-quiesce");
5267 let ctx = test_ctx();
5268 let executor = Arc::new(Executor::new());
5269 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5270 let mut meta = RootMeta::new(Instant::now());
5271 meta.maintenance_pending = true;
5272 meta.maintenance_jobs_in_flight = 1;
5273 meta.maintenance_queued_kinds
5274 .push_back(MaintenanceDrainKind::ConfigureTail);
5275 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5276 *ctx.search_index()
5278 .write()
5279 .unwrap_or_else(std::sync::PoisonError::into_inner) =
5280 Some(crate::search_index::SearchIndex::new());
5281 ctx.set_cache_writer_capabilities(true, true);
5282 let pending = root.as_path().join("pending.rs");
5283 ctx.add_pending_search_index_paths([pending.clone()]);
5284 let canonical_root = root.as_path().to_path_buf();
5288 let artifact = canonical_root.join("cache.bin");
5289 std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
5290 let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
5291 crate::cache_freshness::record_verify_completed(
5292 &canonical_root,
5293 crate::cache_freshness::VerifyArtifact::Search,
5294 seeded_generation,
5295 );
5296 assert!(matches!(
5297 crate::cache_freshness::warm_verify_plan(
5298 &canonical_root,
5299 crate::cache_freshness::VerifyArtifact::Search,
5300 seeded_generation,
5301 ),
5302 crate::cache_freshness::WarmVerifyPlan::Skip
5303 ));
5304 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5307 let _dispatch_tx = dispatch_tx;
5308 let shutdown = Arc::new(AtomicBool::new(false));
5309 let thread_shutdown = Arc::clone(&shutdown);
5310 let join = std::thread::spawn(move || {
5311 while !thread_shutdown.load(Ordering::SeqCst) {
5312 std::thread::yield_now();
5313 }
5314 });
5315 ctx.install_watcher_runtime(
5316 dispatch_rx,
5317 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5318 );
5319 assert!(ctx.watcher_runtime_active());
5320
5321 quiesce_unbound_root(&root, &mut live_roots, &executor);
5322 let meta = &live_roots[&root];
5323 assert!(meta.unbound_quiesced);
5324 assert!(ctx.subc_unbound_quiesced());
5325 assert!(meta.maintenance_pending);
5326 assert!(meta.maintenance_queued_kinds.is_empty());
5327 assert!(executor.actor_registered(&root));
5328 assert!(
5332 ctx.search_index()
5333 .read()
5334 .unwrap_or_else(std::sync::PoisonError::into_inner)
5335 .is_some(),
5336 "quiesce must not evict resident artifacts"
5337 );
5338 assert_eq!(
5339 ctx.pending_callgraph_store_force_token(),
5340 None,
5341 "quiesce must not force a callgraph rebuild"
5342 );
5343 assert_eq!(
5344 ctx.take_pending_search_index_paths(),
5345 vec![pending],
5346 "quiesce must retain pending watcher-derived paths"
5347 );
5348 assert!(
5349 matches!(
5350 crate::cache_freshness::warm_verify_plan(
5351 &canonical_root,
5352 crate::cache_freshness::VerifyArtifact::Search,
5353 seeded_generation,
5354 ),
5355 crate::cache_freshness::WarmVerifyPlan::Skip
5356 ),
5357 "quiesce must not invalidate the warm verify memo"
5358 );
5359 assert!(
5360 ctx.watcher_runtime_active(),
5361 "quiesce must not stop a running watcher"
5362 );
5363 ctx.stop_watcher_runtime();
5364
5365 let meta = live_roots.get_mut(&root).expect("root metadata");
5366 note_maintenance_completion(
5367 meta,
5368 Some(MaintenanceDrainKind::ConfigureTail),
5369 false,
5370 meta.unbound_quiesced,
5371 );
5372 assert!(!meta.maintenance_pending);
5373 assert!(meta.maintenance_queued_kinds.is_empty());
5374 }
5375
5376 #[test]
5377 fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
5378 let (_dir, root) = test_root("same-root-replacement");
5379 let route = route_key(7, 1);
5380 let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5381 let root_channels = HashMap::new();
5382
5383 assert!(!route_removal_will_quiesce_root(
5384 &root,
5385 route,
5386 &installed_channels,
5387 false,
5388 Some(&root),
5389 ));
5390 assert!(route_removal_will_quiesce_root(
5391 &root,
5392 route,
5393 &installed_channels,
5394 false,
5395 None,
5396 ));
5397 assert!(!should_quiesce_removed_root(
5398 &root,
5399 &root_channels,
5400 false,
5401 Some(&root),
5402 ));
5403 assert!(should_quiesce_removed_root(
5404 &root,
5405 &root_channels,
5406 false,
5407 None,
5408 ));
5409 assert!(!should_quiesce_removed_root(
5410 &root,
5411 &root_channels,
5412 true,
5413 None,
5414 ));
5415 }
5416
5417 #[test]
5418 fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
5419 let (_root_dir, root) = test_root("unbound-root-route-count");
5420 let executor = Arc::new(Executor::new());
5421 assert!(executor.register_actor(root.clone(), test_ctx()));
5422 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5423 let mut root_channels = HashMap::from([(
5424 root.clone(),
5425 HashSet::from([route_key(7, 1), route_key(8, 1)]),
5426 )]);
5427
5428 remove_root_channel(&mut root_channels, &root, route_key(7, 1));
5429 if !root_channels.contains_key(&root) {
5430 quiesce_unbound_root(&root, &mut live_roots, &executor);
5431 }
5432 assert!(!live_roots[&root].unbound_quiesced);
5433
5434 remove_root_channel(&mut root_channels, &root, route_key(8, 1));
5435 if !root_channels.contains_key(&root) {
5436 quiesce_unbound_root(&root, &mut live_roots, &executor);
5437 }
5438 assert!(live_roots[&root].unbound_quiesced);
5439
5440 live_roots
5441 .get_mut(&root)
5442 .expect("root metadata")
5443 .note_activity();
5444 assert!(
5445 live_roots[&root].unbound_quiesced,
5446 "late asynchronous activity must not reactivate an unbound root"
5447 );
5448
5449 live_roots
5450 .get_mut(&root)
5451 .expect("root metadata")
5452 .reactivate_bound();
5453 assert!(!live_roots[&root].unbound_quiesced);
5454 }
5455
5456 #[test]
5457 fn allocator_pressure_relief_requires_every_root_to_be_idle() {
5458 let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
5459 let (_active_dir, active_root) = test_root("allocator-relief-active");
5460 let now = Instant::now();
5461 let mut live_roots = HashMap::new();
5462 let mut idle = RootMeta::new(now);
5463 idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
5464 live_roots.insert(idle_root, idle);
5465 assert!(process_has_been_idle(now, &live_roots));
5466
5467 live_roots.insert(active_root.clone(), RootMeta::new(now));
5468 assert!(!process_has_been_idle(now, &live_roots));
5469
5470 let active = live_roots
5471 .get_mut(&active_root)
5472 .expect("active root metadata");
5473 active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
5474 active.active_bash_waits = 1;
5475 assert!(!process_has_been_idle(now, &live_roots));
5476 }
5477
5478 #[test]
5479 fn pressure_relief_log_reports_before_and_after_measurements() {
5480 let allocator = crate::memory::AllocatorMemorySnapshot {
5481 status: "measured",
5482 bytes_in_use: Some(8 * 1024 * 1024),
5483 size_allocated: Some(12 * 1024 * 1024),
5484 retained_slack_bytes: Some(4 * 1024 * 1024),
5485 not_estimated: None,
5486 };
5487 let relief = crate::memory::AllocatorPressureRelief {
5488 bytes_released: 3 * 1024 * 1024,
5489 rss_before_bytes: Some(20 * 1024 * 1024),
5490 rss_after_bytes: Some(17 * 1024 * 1024),
5491 allocator_before: allocator.clone(),
5492 allocator_after: crate::memory::AllocatorMemorySnapshot {
5493 size_allocated: Some(9 * 1024 * 1024),
5494 retained_slack_bytes: Some(1024 * 1024),
5495 ..allocator
5496 },
5497 };
5498 let message = pressure_relief_label(&relief);
5499 assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
5500 assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
5501 assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
5502 assert!(message.contains("reported 3.0 MB released"));
5503 }
5504
5505 #[test]
5506 fn due_maintenance_jobs_skip_poisoned_roots() {
5507 let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
5508 let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
5509 let mut live_roots = HashMap::new();
5510 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
5511 let mut poisoned_meta = RootMeta::new(Instant::now());
5512 poisoned_meta.maintenance_poisoned = true;
5513 live_roots.insert(poisoned_root.clone(), poisoned_meta);
5514
5515 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5516 &mut live_roots,
5517 MAINTENANCE_SUBMIT_BUDGET,
5518 &HashSet::new(),
5519 );
5520
5521 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5522 assert!(due.iter().all(|(root, _)| root == &healthy_root));
5523 assert!(!deferred);
5524 assert!(live_roots[&healthy_root].maintenance_pending);
5525 assert_eq!(
5526 live_roots[&healthy_root].maintenance_jobs_in_flight,
5527 INITIAL_MAINTENANCE_JOB_COUNT
5528 );
5529 assert!(!live_roots[&poisoned_root].maintenance_pending);
5530 }
5531
5532 #[test]
5533 fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
5534 let (_dir, root) = test_root("maintenance-unbound");
5535 let mut meta = RootMeta::new(Instant::now());
5536 meta.unbound_quiesced = true;
5537 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5538
5539 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5540 &mut live_roots,
5541 MAINTENANCE_SUBMIT_BUDGET,
5542 &HashSet::new(),
5543 );
5544
5545 assert!(due.is_empty());
5546 assert!(!deferred);
5547 assert!(!live_roots[&root].maintenance_pending);
5548 }
5549
5550 #[test]
5551 fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
5552 let (_dir, root) = test_root("maintenance-idle-bg-subscription");
5553 let ctx = test_ctx();
5554 assert!(!ctx.completion_drains_have_work());
5555
5556 let executor = Executor::new();
5557 assert!(executor.register_actor(root.clone(), ctx));
5558 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5559 let session = "idle-session".to_string();
5560 let channel = route_key(17, 1);
5561 let bg_sub_by_session = HashMap::from([((root.clone(), session.clone()), channel)]);
5562 let mut bg_wake_pending = HashSet::new();
5563
5564 let (idle_tick_jobs, deferred) = due_maintenance_jobs(
5565 &mut live_roots,
5566 Some(&executor),
5567 &bg_sub_by_session,
5568 &bg_wake_pending,
5569 MAINTENANCE_SUBMIT_BUDGET,
5570 &HashSet::new(),
5571 );
5572 assert!(idle_tick_jobs.is_empty());
5573 assert!(!deferred);
5574 assert!(!live_roots[&root].maintenance_pending);
5575
5576 let mut bg_wake_epoch = HashMap::new();
5579 push::arm_bg_wake(
5580 root.clone(),
5581 session,
5582 channel,
5583 &mut bg_wake_pending,
5584 &mut bg_wake_epoch,
5585 );
5586 let (next_tick_jobs, deferred) = due_maintenance_jobs(
5587 &mut live_roots,
5588 Some(&executor),
5589 &bg_sub_by_session,
5590 &bg_wake_pending,
5591 MAINTENANCE_SUBMIT_BUDGET,
5592 &HashSet::new(),
5593 );
5594 assert_eq!(
5595 next_tick_jobs,
5596 vec![(root, MaintenanceDrainKind::CompletionDrains)]
5597 );
5598 assert!(!deferred);
5599 }
5600
5601 #[tokio::test]
5602 async fn subc_configure_tail_precedes_completed_search_install() {
5603 let root_dir = tempfile::tempdir().unwrap();
5604 let storage = tempfile::tempdir().unwrap();
5605 let root = ProjectRootId::from_path(root_dir.path()).unwrap();
5606 let (ctx, ignored_path) =
5607 runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
5608 let ctx = Arc::new(ctx);
5609 assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
5610 &ctx,
5611 &ignored_path
5612 ));
5613
5614 let executor = Arc::new(Executor::new());
5615 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5616 let metrics = Arc::new(DispatchPathMetrics::new());
5617 let (completion_tx, mut completion_rx) = mpsc::channel(4);
5618 submit_maintenance_job(
5619 &executor,
5620 root.clone(),
5621 MaintenanceDrainKind::ConfigureTail,
5622 Vec::new(),
5623 &completion_tx,
5624 &metrics,
5625 );
5626 submit_maintenance_job(
5627 &executor,
5628 root,
5629 MaintenanceDrainKind::CompletionDrains,
5630 Vec::new(),
5631 &completion_tx,
5632 &metrics,
5633 );
5634
5635 let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
5636 .await
5637 .expect("configure-tail completion timed out")
5638 .expect("configure-tail completion channel closed");
5639 let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
5640 .await
5641 .expect("completion-drains completion timed out")
5642 .expect("completion-drains completion channel closed");
5643 assert!(first.response.id.contains("configure-tail"));
5644 assert!(second.response.id.contains("completion-drains"));
5645 assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
5646 &ctx,
5647 &ignored_path
5648 ));
5649 assert_eq!(
5650 ctx.search_index()
5651 .read()
5652 .unwrap_or_else(std::sync::PoisonError::into_inner)
5653 .as_ref()
5654 .expect("completed search index installed")
5655 .file_count(),
5656 0,
5657 "configure must install the ignore matcher before pending paths replay"
5658 );
5659 ctx.stop_watcher_runtime();
5660 }
5661
5662 #[test]
5663 fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
5664 let (_dir, root) = test_root("maintenance-post-bind");
5665 let mut live_roots = HashMap::new();
5666 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5667
5668 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
5669 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
5670
5671 let meta = live_roots.get(&root).expect("root metadata");
5672 assert!(meta.maintenance_pending);
5673 assert_eq!(meta.maintenance_jobs_in_flight, 0);
5674 assert_eq!(
5675 meta.maintenance_queued_kinds
5676 .iter()
5677 .copied()
5678 .collect::<Vec<_>>(),
5679 vec![
5680 MaintenanceDrainKind::ConfigureTail,
5681 MaintenanceDrainKind::CompletionDrains,
5682 ]
5683 );
5684
5685 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5686 &mut live_roots,
5687 MAINTENANCE_SUBMIT_BUDGET,
5688 &HashSet::new(),
5689 );
5690
5691 assert_eq!(
5692 due,
5693 vec![
5694 (root.clone(), MaintenanceDrainKind::ConfigureTail),
5695 (root.clone(), MaintenanceDrainKind::CompletionDrains),
5696 ]
5697 );
5698 assert!(!deferred);
5699 assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
5700 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
5701 }
5702
5703 #[test]
5704 fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
5705 let mut live_roots = HashMap::new();
5706 let mut root_ids = Vec::new();
5707 let mut _dirs = Vec::new();
5708 for index in 0..4 {
5709 let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
5710 live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
5711 root_ids.push(root_id);
5712 _dirs.push(dir);
5713 }
5714
5715 let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
5716 let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
5717 &mut live_roots,
5718 small_budget,
5719 &HashSet::new(),
5720 );
5721
5722 assert_eq!(first_due.len(), small_budget);
5723 assert!(first_deferred);
5724 let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
5725 assert!(first_due_set
5726 .iter()
5727 .all(|root| live_roots[root].maintenance_pending));
5728 assert!(first_due_set
5729 .iter()
5730 .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
5731
5732 let all_roots: HashSet<_> = root_ids.into_iter().collect();
5733 let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
5734 assert!(deferred_roots
5735 .iter()
5736 .all(|root| !live_roots[root].maintenance_pending));
5737 }
5738
5739 #[test]
5740 fn due_maintenance_jobs_defers_pending_bind_roots() {
5741 let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
5742 let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
5743 let mut live_roots = HashMap::new();
5744 live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
5745 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
5746 let pending_bind_roots = HashSet::from([bind_root.clone()]);
5747
5748 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5749 &mut live_roots,
5750 usize::MAX,
5751 &pending_bind_roots,
5752 );
5753
5754 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5755 assert!(due.iter().all(|(root, _)| root == &healthy_root));
5756 assert!(!deferred);
5757 assert!(!live_roots[&bind_root].maintenance_pending);
5758 assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
5759 }
5760
5761 #[test]
5762 fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
5763 let (_dir, root) = test_root("maintenance-requeue");
5764 let mut live_roots = HashMap::new();
5765 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5766 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5767 &mut live_roots,
5768 usize::MAX,
5769 &HashSet::new(),
5770 );
5771 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5772 assert!(due.iter().all(|(due_root, _)| due_root == &root));
5773 assert!(!deferred);
5774
5775 let meta = live_roots.get_mut(&root).unwrap();
5776 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
5777 assert!(meta.maintenance_pending);
5778 assert_eq!(
5779 meta.maintenance_jobs_in_flight,
5780 INITIAL_MAINTENANCE_JOB_COUNT - 1
5781 );
5782 assert_eq!(meta.maintenance_queued_kinds.len(), 1);
5783
5784 let (requeued, deferred) =
5785 due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
5786 assert_eq!(
5787 requeued,
5788 vec![(root.clone(), MaintenanceDrainKind::Watcher)]
5789 );
5790 assert!(!deferred);
5791 let meta = live_roots.get_mut(&root).unwrap();
5792 assert_eq!(
5793 meta.maintenance_jobs_in_flight,
5794 INITIAL_MAINTENANCE_JOB_COUNT
5795 );
5796 assert!(meta.maintenance_queued_kinds.is_empty());
5797
5798 for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
5799 note_maintenance_completion(meta, None, false, false);
5800 }
5801 assert!(!meta.maintenance_pending);
5802 assert_eq!(meta.maintenance_jobs_in_flight, 0);
5803 }
5804
5805 #[test]
5806 fn maintenance_requeue_drops_while_bind_is_pending() {
5807 let (_dir, root) = test_root("maintenance-bind-requeue");
5808 let mut live_roots = HashMap::new();
5809 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5810 let (due, _) = due_maintenance_jobs_without_actor_context(
5811 &mut live_roots,
5812 usize::MAX,
5813 &HashSet::new(),
5814 );
5815 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5816
5817 let meta = live_roots.get_mut(&root).unwrap();
5818 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
5819
5820 assert_eq!(
5821 meta.maintenance_jobs_in_flight,
5822 INITIAL_MAINTENANCE_JOB_COUNT - 1
5823 );
5824 assert!(meta.maintenance_queued_kinds.is_empty());
5825 assert!(meta.maintenance_pending);
5826 }
5827
5828 #[test]
5829 fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
5830 let mut meta = RootMeta::new(Instant::now());
5831 meta.unbound_quiesced = true;
5832
5833 assert!(!should_requiesce_after_maintenance(
5834 &meta,
5835 MaintenanceDrainKind::Lsp,
5836 false,
5837 ));
5838 assert!(!should_requiesce_after_maintenance(
5839 &meta,
5840 MaintenanceDrainKind::ConfigureTail,
5841 true,
5842 ));
5843 assert!(should_requiesce_after_maintenance(
5844 &meta,
5845 MaintenanceDrainKind::ConfigureTail,
5846 false,
5847 ));
5848 }
5849
5850 #[test]
5851 fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
5852 let (_dir, root) = test_root("maintenance-fatal");
5853 let mut live_roots = HashMap::new();
5854 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5855 let (due, _) = due_maintenance_jobs_without_actor_context(
5856 &mut live_roots,
5857 usize::MAX,
5858 &HashSet::new(),
5859 );
5860 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5861
5862 let meta = live_roots.get_mut(&root).unwrap();
5863 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
5864 assert!(meta.maintenance_poisoned);
5865 assert!(meta.maintenance_queued_kinds.is_empty());
5866
5867 for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
5868 note_maintenance_completion(meta, None, false, false);
5869 }
5870 assert!(!meta.maintenance_pending);
5871 assert_eq!(meta.maintenance_jobs_in_flight, 0);
5872 }
5873
5874 #[test]
5875 fn trust_for_principal_matrix() {
5876 assert_eq!(
5877 trust_for_principal(&Some(Principal::Direct)),
5878 BindTrust::FirstParty
5879 );
5880 assert_eq!(
5881 trust_for_principal(&Some(Principal::Reserved {
5882 module_id: "llm-runner".to_string(),
5883 })),
5884 BindTrust::FirstParty
5885 );
5886 assert_eq!(
5887 trust_for_principal(&Some(Principal::Reserved {
5888 module_id: "aft".to_string(),
5889 })),
5890 BindTrust::FirstParty
5891 );
5892 assert_eq!(
5893 trust_for_principal(&Some(Principal::Reserved {
5894 module_id: "subc-mcp".to_string(),
5895 })),
5896 BindTrust::Untrusted
5897 );
5898 assert_eq!(
5899 trust_for_principal(&Some(Principal::Reserved {
5900 module_id: "anything-unknown".to_string(),
5901 })),
5902 BindTrust::Untrusted
5903 );
5904 assert_eq!(
5905 trust_for_principal(&Some(Principal::Unverified)),
5906 BindTrust::Untrusted
5907 );
5908 assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
5909 }
5910
5911 #[test]
5912 fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
5913 let principal = Some(Principal::Direct);
5914 let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
5915 let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
5916
5917 assert_eq!(
5918 trust_for_bind(fingerprint_a, &principal),
5919 BindTrust::Untrusted
5920 );
5921 assert_eq!(
5922 trust_for_bind(fingerprint_b, &principal),
5923 BindTrust::Untrusted
5924 );
5925 }
5926
5927 #[tokio::test]
5928 async fn persistent_cancel_resolves_when_fired_before_await() {
5929 let signal = PersistentCancelSignal::new();
5933 signal.cancel();
5934 tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
5936 .await
5937 .expect("cancelled() must resolve when cancel fired beforehand");
5938
5939 let racing = PersistentCancelSignal::new();
5941 let racing_for_task = racing.clone();
5942 let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
5943 racing.cancel();
5944 tokio::time::timeout(Duration::from_secs(1), waiter)
5945 .await
5946 .expect("cancelled() must resolve when cancel races the await")
5947 .expect("waiter task panicked");
5948 }
5949
5950 #[test]
5951 fn ingress_epoch_validation_silently_drops_every_stale_route_frame() {
5952 let installed = HashMap::from([(7, 9)]);
5953 for ty in [
5954 FrameType::Request,
5955 FrameType::Response,
5956 FrameType::Error,
5957 FrameType::Push,
5958 FrameType::Cancel,
5959 FrameType::Goodbye,
5960 ] {
5961 let body = if ty.is_pure_header() {
5962 Vec::new()
5963 } else {
5964 br#"{}"#.to_vec()
5965 };
5966 let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
5967 assert!(!ingress_route_is_current(&installed, &stale), "{ty:?}");
5968 }
5969
5970 let uninstalled = Frame::build(
5971 FrameType::Request,
5972 control_flags(),
5973 8,
5974 1,
5975 42,
5976 br#"{}"#.to_vec(),
5977 )
5978 .unwrap();
5979 assert!(!ingress_route_is_current(&installed, &uninstalled));
5980
5981 let current = Frame::build(
5982 FrameType::Request,
5983 control_flags(),
5984 7,
5985 9,
5986 43,
5987 br#"{}"#.to_vec(),
5988 )
5989 .unwrap();
5990 let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
5991 assert!(ingress_route_is_current(&installed, ¤t));
5992 assert!(ingress_route_is_current(&installed, &control));
5993 assert_eq!(installed, HashMap::from([(7, 9)]));
5994 }
5995
5996 #[tokio::test]
5997 async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
5998 let (_dir, root) = test_root("route-bind-b2-ordering");
5999 let route = route_key(7, 3);
6000 let identity = RouteIdentity(Arc::new(RouteIdentityData {
6001 root: root.clone(),
6002 project_root: root.as_path().to_path_buf(),
6003 harness: "opencode".to_string(),
6004 session: "b2-session".to_string(),
6005 trust: BindTrust::FirstParty,
6006 spawn_principal: AuthenticatedPrincipal::FirstParty,
6007 consumer_elicitation_capable: false,
6008 }));
6009 let replay_key = push::ReplayKey::from_identity(&identity);
6010 let completion = RouteBindCompletion {
6011 route,
6012 identity,
6013 bind_root_id: root.clone(),
6014 inserted_new_actor: false,
6015 configure_response: Response::success("subc-bind-7", json!({})),
6016 diagnostics_on_edit: false,
6017 ver: PROTOCOL_VERSION,
6018 corr: 91,
6019 flags: control_flags(),
6020 };
6021 let mut pending_binds = HashMap::from([(
6022 route,
6023 PendingBind {
6024 bind_root_id: root,
6025 inserted_new_actor: false,
6026 cancelled: false,
6027 configure_request_id: "subc-bind-7".to_string(),
6028 started_at: Instant::now(),
6029 warned_half_deadline: false,
6030 deadline_reported: false,
6031 corr: 91,
6032 ver: PROTOCOL_VERSION,
6033 flags: control_flags(),
6034 cancellation: crate::executor::JobCancellation::new(),
6035 },
6036 )]);
6037 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6038 let mut push_buffer =
6039 HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
6040 let (writer_tx, mut writer_rx) = mpsc::channel(8);
6041 let metrics = Arc::new(DispatchPathMetrics::new());
6042
6043 handle_route_bind_completion(
6044 &writer_tx,
6045 completion,
6046 &mut HashMap::new(),
6047 &mut HashMap::new(),
6048 &mut HashMap::new(),
6049 &mut push_buffer,
6050 &mut HashMap::new(),
6051 &mut pending_binds,
6052 &mut installed_route_epochs,
6053 &Arc::new(Executor::new()),
6054 &Arc::new(Notify::new()),
6055 &metrics,
6056 )
6057 .await
6058 .unwrap();
6059
6060 let ack = writer_rx.try_recv().expect("RouteBindAck");
6061 assert_eq!(ack.header.ty, FrameType::Response);
6062 assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
6063 let route_frame = writer_rx.try_recv().expect("post-ack route frame");
6064 assert_eq!(route_frame.header.ty, FrameType::Push);
6065 assert_eq!(
6066 (route_frame.header.channel, route_frame.header.epoch),
6067 (route.channel, route.epoch)
6068 );
6069 }
6070}