1use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, Lane};
31use crate::jsonc::strip_jsonc;
32use crate::log_ctx;
33use crate::path_identity::ProjectRootId;
34use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
35use crate::run_tool_call::{
36 run_tool_call, strip_agent_preview_arg_owned, PhaseTrace, ToolCallContext, ToolCallOutcome,
37 ToolCallResult,
38};
39use crate::runtime_drain;
40
41use subc_protocol::manifest::{
42 Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
43 ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
44};
45use subc_protocol::session::{
46 HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
47 MODULE_CONTROL_OP_HEALTH_CHECK,
48};
49use subc_protocol::{
50 ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, PROTOCOL_VERSION,
51};
52use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
53use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
54use tokio::net::TcpStream;
55use tokio::sync::{mpsc, oneshot, Notify};
56use tokio::task::JoinHandle;
57
58const AUTH_DEADLINE: Duration = Duration::from_secs(5);
61const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
62const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
63const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
64const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
65
66const HELLO_CORR: u64 = 1;
68
69const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
72
73const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
77
78const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
82
83const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
87
88const WRITER_QUEUE_CAPACITY: usize = 256;
89
90const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
93
94const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
102const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
103 MaintenanceDrainKind::Watcher,
104 MaintenanceDrainKind::Lsp,
105 MaintenanceDrainKind::ConfigureTail,
106 MaintenanceDrainKind::CompletionDrains,
107];
108#[cfg(test)]
109const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
110
111const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
112const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
113
114const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
115const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
116
117const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
120
121const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
125
126const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
128const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
129
130#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
131struct RouteChannel {
132 channel: u16,
133 epoch: u32,
134}
135
136impl fmt::Display for RouteChannel {
137 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
138 write!(f, "{}@{}", self.channel, self.epoch)
139 }
140}
141
142type PushEnvelope = (ProjectRootId, PushFrame);
143type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
144type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
145mod bash;
146mod health;
147mod manifest;
148mod push;
149mod wire;
150
151use self::health::{
152 build_health_report, warn_slow_pending_binds, DispatchPathMetrics, ResponseTaskGuard,
153};
154use self::manifest::{
155 build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
156 is_subc_agent_core_tool, is_subc_native_plumbing_tool,
157};
158pub use self::wire::SubcError;
159
160pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
164 manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
165}
166use self::wire::{
167 build_error_frame, build_goodbye_frame, build_tool_response_frame, decrement_counted_channel,
168 response_is_fatal_panic, response_message, send_counted_channel, send_frame,
169 send_reliable_writer_frame, send_traced_tool_response_frame, ToolResponseWriteTrace,
170 WriterFrame, WriterSender,
171};
172
173struct DecodedFrame {
174 frame: Frame,
175 phase_trace: PhaseTrace,
176}
177
178struct ToolCallCompletion {
179 text: String,
180 phase_trace: PhaseTrace,
181}
182
183#[derive(Clone)]
184struct PushSenders {
185 lossy_tx: mpsc::Sender<LossyPushEnvelope>,
186 reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
187 lossy_overflow: Arc<push::LossyOverflow>,
188 lossy_seq: Arc<AtomicU64>,
189}
190
191#[derive(Clone)]
192struct PersistentCancelSignal {
193 inner: Arc<PersistentCancelInner>,
194}
195
196struct PersistentCancelInner {
197 cancelled: AtomicBool,
198 notify: Notify,
199}
200
201impl PersistentCancelSignal {
202 fn new() -> Self {
203 Self {
204 inner: Arc::new(PersistentCancelInner {
205 cancelled: AtomicBool::new(false),
206 notify: Notify::new(),
207 }),
208 }
209 }
210
211 fn cancel(&self) {
212 if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
213 self.inner.notify.notify_waiters();
214 }
215 }
216
217 fn is_cancelled(&self) -> bool {
218 self.inner.cancelled.load(Ordering::SeqCst)
219 }
220
221 async fn cancelled(&self) {
222 loop {
230 let notified = self.inner.notify.notified();
231 tokio::pin!(notified);
232 notified.as_mut().enable();
233 if self.is_cancelled() {
234 return;
235 }
236 notified.await;
237 }
238 }
239}
240
241#[derive(Debug, Clone, Copy, PartialEq, Eq)]
242pub(crate) enum BindTrust {
243 FirstParty,
244 Untrusted,
245}
246
247impl BindTrust {
248 fn allows_bash_observation(self) -> bool {
249 matches!(self, Self::FirstParty)
250 }
251
252 fn label(self) -> &'static str {
253 match self {
254 Self::FirstParty => "first_party",
255 Self::Untrusted => "untrusted",
256 }
257 }
258}
259
260pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
261 match principal {
262 Some(Principal::Direct) => BindTrust::FirstParty,
263 Some(Principal::Reserved { module_id })
264 if module_id == "llm-runner"
265 || module_id == "aft"
266 || module_id == "broca"
267 || module_id == "alfonso-core" =>
268 {
269 BindTrust::FirstParty
270 }
271 Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
272 BindTrust::Untrusted
273 }
274 }
275}
276
277fn harness_forces_untrusted(harness: &str) -> bool {
278 harness.starts_with("fed:")
279}
280
281pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
282 if harness_forces_untrusted(harness) {
283 BindTrust::Untrusted
284 } else {
285 trust_for_principal(principal)
286 }
287}
288
289fn principal_label(principal: &Option<Principal>) -> String {
290 match principal {
291 Some(Principal::Direct) => "direct".to_string(),
292 Some(Principal::Reserved { module_id }) => format!("reserved:{module_id}"),
293 Some(Principal::Unverified) => "unverified".to_string(),
294 None => "absent".to_string(),
295 }
296}
297
298#[derive(Debug)]
299struct RootMeta {
305 maintenance_pending: bool,
306 maintenance_jobs_in_flight: usize,
307 maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
308 maintenance_last_submitted: Option<Instant>,
309 maintenance_poisoned: bool,
310 last_touched: Instant,
311 diagnostics_on_edit: bool,
312 active_bash_waits: usize,
313 idle_artifacts_evicted: bool,
314 unbound_quiesced: bool,
315}
316
317#[derive(Debug)]
318struct PendingBind {
319 bind_root_id: ProjectRootId,
320 inserted_new_actor: bool,
321 cancelled: bool,
322 configure_request_id: String,
323 started_at: Instant,
324 warned_half_deadline: bool,
325 deadline_reported: bool,
326 corr: u64,
327 ver: u8,
328 flags: Flags,
329 cancellation: crate::executor::JobCancellation,
334}
335
336struct RouteBindCompletion {
337 route: RouteChannel,
338 identity: RouteIdentity,
339 bind_root_id: ProjectRootId,
340 inserted_new_actor: bool,
341 configure_response: Response,
342 diagnostics_on_edit: bool,
343 ver: u8,
344 corr: u64,
345 flags: Flags,
346}
347
348#[derive(Debug, Clone)]
349struct RouteIdentity(Arc<RouteIdentityData>);
350
351#[derive(Debug)]
352struct RouteIdentityData {
353 root: ProjectRootId,
354 project_root: PathBuf,
355 harness: String,
356 session: String,
357 trust: BindTrust,
358 consumer_elicitation_capable: bool,
359}
360
361impl Deref for RouteIdentity {
362 type Target = RouteIdentityData;
363
364 fn deref(&self) -> &Self::Target {
365 &self.0
366 }
367}
368
369#[derive(Debug, Clone)]
370struct RetainedSessionIdentity {
371 harness: String,
372 trust: BindTrust,
373}
374
375#[derive(Clone, Copy)]
376struct BgSub {
377 corr: u64,
378 ver: u8,
379 flags: Flags,
380}
381
382struct MaintenanceCompletion {
383 root_id: ProjectRootId,
384 kind: MaintenanceDrainKind,
385 response: Response,
386 empty_bg_sessions: Vec<(String, u64)>,
387 requeue_kind: Option<MaintenanceDrainKind>,
388}
389
390#[derive(Clone, Copy, Debug, PartialEq, Eq)]
391enum MaintenanceDrainKind {
392 Watcher,
393 Lsp,
394 ConfigureTail,
395 CompletionDrains,
396}
397
398impl MaintenanceDrainKind {
399 fn label(self) -> &'static str {
400 match self {
401 Self::Watcher => "watcher",
402 Self::Lsp => "lsp",
403 Self::ConfigureTail => "configure-tail",
404 Self::CompletionDrains => "completion-drains",
405 }
406 }
407}
408
409#[derive(Debug, Default)]
410struct MaintenanceJobOutcome {
411 empty_bg_sessions: Vec<(String, u64)>,
412 requeue_kind: Option<MaintenanceDrainKind>,
413}
414
415#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
416struct ReverseCorrKey {
417 route: RouteChannel,
418 corr: u64,
419}
420
421struct PendingBashAsk {
422 route: RouteChannel,
423 tool_corr: u64,
424 tool_flags: Flags,
425 tool_ver: u8,
426 root: ProjectRootId,
427 project_root: PathBuf,
428 session_id: String,
429 request_id: String,
430 arguments: Value,
431 format_context: crate::subc_format::FormatContext,
432 cancel: bash::BashWaitCancel,
433 grants: Vec<String>,
434 expires_at: Instant,
435}
436
437impl RootMeta {
438 fn new(now: Instant) -> Self {
439 Self {
440 maintenance_pending: false,
441 maintenance_jobs_in_flight: 0,
442 maintenance_queued_kinds: VecDeque::new(),
443 maintenance_last_submitted: None,
444 maintenance_poisoned: false,
445 last_touched: now,
446 diagnostics_on_edit: false,
447 active_bash_waits: 0,
448 idle_artifacts_evicted: false,
449 unbound_quiesced: false,
450 }
451 }
452
453 fn note_activity(&mut self) {
454 self.last_touched = Instant::now();
455 }
456
457 fn reactivate_bound(&mut self) {
458 self.note_activity();
459 self.idle_artifacts_evicted = false;
460 self.unbound_quiesced = false;
461 }
462}
463
464fn due_maintenance_jobs(
465 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
466 executor: Option<&Executor>,
467 bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
468 bg_wake_pending: &HashSet<RouteChannel>,
469 budget: usize,
470 pending_bind_roots: &HashSet<ProjectRootId>,
471) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
472 let mut jobs = Vec::new();
473 let mut deferred = false;
474 let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
475 roots.sort_by(|left, right| {
476 let left_last = live_roots
477 .get(left)
478 .and_then(|meta| meta.maintenance_last_submitted);
479 let right_last = live_roots
480 .get(right)
481 .and_then(|meta| meta.maintenance_last_submitted);
482 left_last
483 .cmp(&right_last)
484 .then_with(|| left.as_path().cmp(right.as_path()))
485 });
486
487 for root_id in roots {
488 let Some(meta) = live_roots.get_mut(&root_id) else {
489 continue;
490 };
491 if meta.maintenance_poisoned {
492 continue;
493 }
494
495 if pending_bind_roots.contains(&root_id) {
496 if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
497 deferred = true;
498 }
499 continue;
500 }
501
502 if !meta.maintenance_pending {
503 if jobs.len() >= budget {
504 deferred = true;
505 continue;
506 }
507 let executor_actor_context =
511 executor.and_then(|executor| executor.actor_context(&root_id));
512 let root_has_pending_bg_wake =
513 bg_sub_by_session.iter().any(|((sub_root, _), channel)| {
514 sub_root == &root_id && bg_wake_pending.contains(channel)
515 });
516 let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
517 Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
518 .into_iter()
519 .filter(|kind| {
520 if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
521 return false;
522 }
523 match kind {
524 MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
525 MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
526 MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
527 MaintenanceDrainKind::CompletionDrains => {
533 root_has_pending_bg_wake || ctx.completion_drains_have_work()
534 }
535 }
536 })
537 .collect(),
538 None if meta.unbound_quiesced => Vec::new(),
539 None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
541 };
542 if kinds_with_work.is_empty() {
543 continue;
544 }
545 meta.maintenance_pending = true;
546 meta.maintenance_queued_kinds.extend(kinds_with_work);
547 }
548
549 while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
550 if jobs.len() >= budget {
551 meta.maintenance_queued_kinds.push_front(kind);
552 deferred = true;
553 break;
554 }
555 meta.maintenance_jobs_in_flight += 1;
556 meta.maintenance_last_submitted = Some(Instant::now());
557 jobs.push((root_id.clone(), kind));
558 }
559
560 meta.maintenance_pending =
561 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
562 }
563
564 (jobs, deferred)
565}
566
567fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
568 match estimate.estimated_bytes {
569 Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
570 None if estimate.status == "busy" => "busy".to_string(),
571 None => "not estimated".to_string(),
572 }
573}
574
575fn optional_memory_label(bytes: Option<u64>) -> String {
576 bytes.map_or_else(
577 || "not estimated".to_string(),
578 |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
579 )
580}
581
582fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
583 format!(
584 "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
585 optional_memory_label(relief.rss_before_bytes),
586 optional_memory_label(relief.rss_after_bytes),
587 optional_memory_label(relief.allocator_before.bytes_in_use),
588 optional_memory_label(relief.allocator_after.bytes_in_use),
589 optional_memory_label(relief.allocator_before.size_allocated),
590 optional_memory_label(relief.allocator_after.size_allocated),
591 optional_memory_label(relief.allocator_before.retained_slack_bytes),
592 optional_memory_label(relief.allocator_after.retained_slack_bytes),
593 relief.bytes_released as f64 / (1024.0 * 1024.0),
594 )
595}
596
597fn idle_root_eviction_message(
598 root_id: &ProjectRootId,
599 memory: &crate::memory::RootMemorySnapshot,
600 pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
601) -> String {
602 let freed_bytes = [
605 &memory.semantic,
606 &memory.trigram,
607 &memory.symbols,
608 &memory.callgraph,
609 &memory.inspect,
610 ]
611 .iter()
612 .filter_map(|estimate| estimate.estimated_bytes)
613 .fold(0u64, u64::saturating_add);
614 let mut message = format!(
615 "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
616 root_id.as_path().display(),
617 freed_bytes as f64 / (1024.0 * 1024.0),
618 eviction_estimate_label(&memory.semantic),
619 eviction_estimate_label(&memory.trigram),
620 eviction_estimate_label(&memory.symbols),
621 eviction_estimate_label(&memory.callgraph),
622 eviction_estimate_label(&memory.inspect),
623 eviction_estimate_label(&memory.bash),
624 eviction_estimate_label(&memory.lsp),
625 eviction_estimate_label(&memory.parser_pool),
626 );
627 if let Some(pressure_relief) = pressure_relief {
628 message.push_str(&pressure_relief_label(pressure_relief));
629 }
630 message
631}
632
633fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
634 !live_roots.is_empty()
635 && live_roots.values().all(|meta| {
636 now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
637 && meta.active_bash_waits == 0
638 && !meta.maintenance_pending
639 && meta.maintenance_queued_kinds.is_empty()
640 })
641}
642
643fn allocator_pressure_relief_after_idle_sweep(
644 now: Instant,
645 live_roots: &HashMap<ProjectRootId, RootMeta>,
646 executor: &Executor,
647) -> Option<crate::memory::AllocatorPressureRelief> {
648 if !process_has_been_idle(now, live_roots)
649 || live_roots.keys().any(|root_id| {
650 executor
651 .actor_context(root_id)
652 .is_some_and(|ctx| ctx.artifact_eviction_blocked())
653 })
654 {
655 return None;
656 }
657
658 #[cfg(target_os = "macos")]
659 {
660 Some(crate::memory::relieve_allocator_pressure())
661 }
662 #[cfg(not(target_os = "macos"))]
663 {
664 None
665 }
666}
667
668fn quiesce_unbound_root(
669 root_id: &ProjectRootId,
670 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
671 executor: &Arc<Executor>,
672) {
673 let Some(meta) = live_roots.get_mut(root_id) else {
674 return;
675 };
676
677 let ctx = executor.actor_context(root_id);
678 if let Some(ctx) = ctx.as_ref() {
679 ctx.mark_subc_unbound();
683 }
684 let cancelled = executor.cancel_queued_maintenance(root_id);
685 let discarded = ctx
693 .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
694 .unwrap_or(0);
695 meta.unbound_quiesced = true;
696 meta.maintenance_queued_kinds.clear();
697 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
698 log::debug!(
699 "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s))",
700 root_id.as_path().display(),
701 cancelled,
702 discarded
703 );
704}
705
706fn reap_idle_roots(
707 now: Instant,
708 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
709 pending_binds: &HashMap<RouteChannel, PendingBind>,
710 executor: &Arc<Executor>,
711) -> usize {
712 let pending_bind_roots = pending_binds
713 .values()
714 .map(|pending| pending.bind_root_id.clone())
715 .collect::<HashSet<_>>();
716 let candidates = live_roots
717 .iter()
718 .filter_map(|(root_id, meta)| {
719 if meta.idle_artifacts_evicted
724 || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
725 || meta.active_bash_waits > 0
726 || meta.maintenance_pending
727 || !meta.maintenance_queued_kinds.is_empty()
728 || pending_bind_roots.contains(root_id)
729 {
730 return None;
731 }
732 Some(root_id.clone())
733 })
734 .collect::<Vec<_>>();
735
736 let mut reaped = Vec::new();
737 for root_id in candidates {
738 let Some(ctx) = executor.actor_context(&root_id) else {
739 continue;
740 };
741 let taken_pending = ctx
752 .subc_unbound_quiesced()
753 .then(|| ctx.take_pending_reconciliation_state());
754 if ctx.artifact_eviction_blocked() {
755 if let Some(pending) = taken_pending {
756 ctx.restore_pending_reconciliation_state(pending);
757 }
758 continue;
759 }
760 let memory_before = ctx.memory_root_snapshot();
761 if !ctx.evict_idle_artifacts() {
762 if let Some(pending) = taken_pending {
763 ctx.restore_pending_reconciliation_state(pending);
764 }
765 continue;
766 }
767 drop(taken_pending);
768 ctx.stop_watcher_runtime_in_background();
772 ctx.invalidate_artifacts_after_watcher_gap();
777 if let Some(meta) = live_roots.get_mut(&root_id) {
778 meta.idle_artifacts_evicted = true;
779 }
780 reaped.push((root_id, memory_before));
781 }
782
783 let pressure_relief = (!reaped.is_empty())
784 .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
785 .flatten();
786 for (root_id, memory_before) in &reaped {
787 log::info!(
788 "{}",
789 idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
790 );
791 }
792 reaped.len()
793}
794
795#[allow(clippy::too_many_arguments)]
796fn submit_due_maintenance_jobs(
797 executor: &Arc<Executor>,
798 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
799 pending_binds: &HashMap<RouteChannel, PendingBind>,
800 bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
801 bg_wake_pending: &HashSet<RouteChannel>,
802 bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
803 maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
804 metrics: &Arc<DispatchPathMetrics>,
805) {
806 let pending_bind_roots = pending_binds
807 .values()
808 .map(|pending| pending.bind_root_id.clone())
809 .collect::<HashSet<_>>();
810 let (due_jobs, deferred_jobs) = due_maintenance_jobs(
811 live_roots,
812 Some(executor),
813 bg_sub_by_session,
814 bg_wake_pending,
815 MAINTENANCE_SUBMIT_BUDGET,
816 &pending_bind_roots,
817 );
818 if deferred_jobs {
819 metrics
820 .maintenance_budget_deferrals
821 .fetch_add(1, Ordering::Relaxed);
822 }
823 for (root_id, kind) in due_jobs {
824 let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
825 bg_sub_by_session
826 .iter()
827 .filter_map(|((root, session), _)| {
828 if root == &root_id {
829 Some((
830 session.clone(),
831 bg_wake_epoch
832 .get(&(root_id.clone(), session.clone()))
833 .copied()
834 .unwrap_or(0),
835 ))
836 } else {
837 None
838 }
839 })
840 .collect()
841 } else {
842 Vec::new()
843 };
844 submit_maintenance_job(
845 executor,
846 root_id,
847 kind,
848 bg_sessions_to_check,
849 maintenance_tx,
850 metrics,
851 );
852 }
853}
854
855fn should_requiesce_after_maintenance(
856 meta: &RootMeta,
857 completed_kind: MaintenanceDrainKind,
858 bind_pending: bool,
859) -> bool {
860 meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
861}
862
863fn note_maintenance_completion(
864 meta: &mut RootMeta,
865 requeue_kind: Option<MaintenanceDrainKind>,
866 fatal: bool,
867 defer_requeue: bool,
868) {
869 if fatal {
870 meta.maintenance_poisoned = true;
871 }
872
873 if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
874 meta.maintenance_queued_kinds.push_back(kind);
875 }
876
877 meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
878 meta.maintenance_pending =
879 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
880}
881
882fn route_key(channel: u16, epoch: u32) -> RouteChannel {
883 RouteChannel { channel, epoch }
884}
885
886fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
887 if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
888 installed_epochs.remove(&route.channel);
889 }
890}
891
892fn ingress_route_is_current(installed_epochs: &HashMap<u16, u32>, frame: &Frame) -> bool {
893 frame.header.channel == 0
894 || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
895}
896
897fn bash_elicitation_timeout() -> Duration {
898 if cfg!(debug_assertions) {
899 if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
900 if let Ok(ms) = raw.parse::<u64>() {
901 if ms > 0 {
902 return Duration::from_millis(ms);
903 }
904 }
905 }
906 }
907 BASH_ELICITATION_TIMEOUT
908}
909
910fn allocate_reverse_corr(
911 pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
912 route: RouteChannel,
913 next_corr: &mut u64,
914) -> u64 {
915 loop {
916 let corr = *next_corr;
917 *next_corr = (*next_corr).wrapping_add(1).max(1);
918 if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
919 return corr;
920 }
921 }
922}
923
924fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
925 match kind {
926 crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
927 crate::bash_permissions::PermissionKind::Bash => "bash",
928 }
929}
930
931fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
932 let mut patterns = Vec::new();
933 let mut seen = HashSet::new();
934 for ask in asks {
935 for pattern in ask.patterns.iter().chain(ask.always.iter()) {
936 if seen.insert(pattern.clone()) {
937 patterns.push(pattern.clone());
938 }
939 }
940 }
941 patterns
942}
943
944fn bash_elicitation_message(
945 command: &str,
946 asks: &[crate::bash_permissions::PermissionAsk],
947) -> String {
948 let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
949 let patterns = bash_elicitation_patterns(asks);
950 let pattern_text = if patterns.is_empty() {
951 "no matched permission patterns".to_string()
952 } else {
953 patterns.join(", ")
954 };
955 let ask_kinds = asks
956 .iter()
957 .map(|ask| bash_permission_kind_label(&ask.kind))
958 .collect::<HashSet<_>>()
959 .into_iter()
960 .collect::<Vec<_>>()
961 .join(", ");
962 if ask_kinds.is_empty() {
963 format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
964 } else {
965 format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
966 }
967}
968
969fn bash_elicitation_request_body(
970 command: &str,
971 asks: &[crate::bash_permissions::PermissionAsk],
972) -> Value {
973 json!({
974 "method": BASH_ELICITATION_CREATE_METHOD,
975 "params": {
976 "mode": "form",
977 "message": bash_elicitation_message(command, asks),
978 "requestedSchema": {
979 "type": "object",
980 "properties": {
981 "decision": {
982 "type": "string",
983 "enum": ["allow", "deny"],
984 "description": "Choose allow to run this bash command once, or deny to block it."
985 }
986 },
987 "required": ["decision"],
988 "additionalProperties": false
989 },
990 "_meta": {
991 "aft": {
992 "tool": "bash",
993 "command": command,
994 "asks": asks
995 }
996 }
997 }
998 })
999}
1000
1001fn build_bash_elicitation_request_frame(
1002 ver: u8,
1003 route: RouteChannel,
1004 corr: u64,
1005 flags: Flags,
1006 command: &str,
1007 asks: &[crate::bash_permissions::PermissionAsk],
1008) -> Result<Frame, SubcError> {
1009 let body = bash_elicitation_request_body(command, asks);
1010 Frame::build_with_version(
1011 ver,
1012 FrameType::Request,
1013 flags,
1014 route.channel,
1015 route.epoch,
1016 corr,
1017 serde_json::to_vec(&body).map_err(SubcError::Json)?,
1018 )
1019 .map_err(SubcError::FrameBuild)
1020}
1021
1022fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1023 let Ok(value) = serde_json::from_slice::<Value>(body) else {
1024 return false;
1025 };
1026 flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1027}
1028
1029fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1030 let Some(object) = value.as_object() else {
1031 return false;
1032 };
1033 object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1034}
1035
1036fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1037 let Some(object) = value.as_object() else {
1038 return false;
1039 };
1040 if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1041 return false;
1042 }
1043 let Some(content) = object.get("content").and_then(Value::as_object) else {
1044 return false;
1045 };
1046 content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1047}
1048
1049#[allow(clippy::too_many_arguments)]
1050async fn settle_pending_bash_ask_denied(
1051 tx: &WriterSender,
1052 pending: PendingBashAsk,
1053 routes: &HashMap<RouteChannel, RouteIdentity>,
1054 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1055 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1056 shutdown: &Arc<Notify>,
1057 metrics: &DispatchPathMetrics,
1058) -> Result<(), SubcError> {
1059 let completion = bash::bash_denied_untrusted_completion(
1060 pending.route,
1061 pending.tool_corr,
1062 pending.tool_flags,
1063 pending.tool_ver,
1064 pending.root,
1065 pending.request_id,
1066 pending.format_context,
1067 );
1068 bash::handle_bash_deferred_completion(
1069 tx,
1070 completion,
1071 routes,
1072 live_roots,
1073 route_bash_cancels,
1074 shutdown,
1075 metrics,
1076 )
1077 .await
1078}
1079
1080fn take_pending_bash_asks_for_route(
1081 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1082 route: RouteChannel,
1083) -> Vec<PendingBashAsk> {
1084 let keys = pending_bash_asks
1085 .keys()
1086 .copied()
1087 .filter(|key| key.route == route)
1088 .collect::<Vec<_>>();
1089 keys.into_iter()
1090 .filter_map(|key| pending_bash_asks.remove(&key))
1091 .collect()
1092}
1093
1094#[allow(clippy::too_many_arguments)]
1095async fn settle_pending_bash_asks_for_route(
1096 tx: &WriterSender,
1097 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1098 route: RouteChannel,
1099 routes: &HashMap<RouteChannel, RouteIdentity>,
1100 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1101 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1102 shutdown: &Arc<Notify>,
1103 metrics: &DispatchPathMetrics,
1104) -> Result<(), SubcError> {
1105 for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1106 settle_pending_bash_ask_denied(
1107 tx,
1108 pending,
1109 routes,
1110 live_roots,
1111 route_bash_cancels,
1112 shutdown,
1113 metrics,
1114 )
1115 .await?;
1116 }
1117 Ok(())
1118}
1119
1120#[allow(clippy::too_many_arguments)]
1121async fn settle_all_pending_bash_asks(
1122 tx: &WriterSender,
1123 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1124 routes: &HashMap<RouteChannel, RouteIdentity>,
1125 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1126 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1127 shutdown: &Arc<Notify>,
1128 metrics: &DispatchPathMetrics,
1129) -> Result<(), SubcError> {
1130 let pending = pending_bash_asks
1131 .drain()
1132 .map(|(_, pending)| pending)
1133 .collect::<Vec<_>>();
1134 for pending in pending {
1135 settle_pending_bash_ask_denied(
1136 tx,
1137 pending,
1138 routes,
1139 live_roots,
1140 route_bash_cancels,
1141 shutdown,
1142 metrics,
1143 )
1144 .await?;
1145 }
1146 Ok(())
1147}
1148
1149#[allow(clippy::too_many_arguments)]
1150async fn expire_pending_bash_asks(
1151 tx: &WriterSender,
1152 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1153 routes: &HashMap<RouteChannel, RouteIdentity>,
1154 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1155 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1156 shutdown: &Arc<Notify>,
1157 metrics: &DispatchPathMetrics,
1158) -> Result<(), SubcError> {
1159 let now = Instant::now();
1160 let expired = pending_bash_asks
1161 .iter()
1162 .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1163 .collect::<Vec<_>>();
1164 for key in expired {
1165 if let Some(pending) = pending_bash_asks.remove(&key) {
1166 log::debug!(
1167 "subc attach: bash elicitation request {} on route {} expired fail-closed",
1168 key.corr,
1169 pending.route
1170 );
1171 settle_pending_bash_ask_denied(
1172 tx,
1173 pending,
1174 routes,
1175 live_roots,
1176 route_bash_cancels,
1177 shutdown,
1178 metrics,
1179 )
1180 .await?;
1181 }
1182 }
1183 Ok(())
1184}
1185
1186#[allow(clippy::too_many_arguments)]
1187async fn handle_bash_elicitation_reply(
1188 tx: &WriterSender,
1189 frame: &Frame,
1190 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1191 routes: &HashMap<RouteChannel, RouteIdentity>,
1192 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1193 executor: &Arc<Executor>,
1194 shutdown: &Arc<Notify>,
1195 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1196 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1197 metrics: &Arc<DispatchPathMetrics>,
1198 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1199 dispatch: DispatchFn,
1200) -> Result<(), SubcError> {
1201 let key = ReverseCorrKey {
1202 route: route_key(frame.header.channel, frame.header.epoch),
1203 corr: frame.header.corr,
1204 };
1205 let Some(pending) = pending_bash_asks.remove(&key) else {
1206 return Ok(());
1207 };
1208
1209 if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1210 if routes.contains_key(&key.route) {
1211 bash::submit_deferred_bash(
1212 executor,
1213 bash_deferred_tx,
1214 bash_poll_touch_tx,
1215 metrics,
1216 dispatch,
1217 pending.root,
1218 pending.project_root,
1219 pending.session_id,
1220 pending.request_id,
1221 pending.route,
1222 pending.tool_corr,
1223 pending.tool_flags,
1224 pending.tool_ver,
1225 pending.arguments,
1226 pending.format_context,
1227 pending.cancel,
1228 BindTrust::Untrusted,
1229 Some(pending.grants),
1230 );
1231 return Ok(());
1232 }
1233 log::debug!(
1234 "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1235 key.corr,
1236 pending.route
1237 );
1238 }
1239
1240 settle_pending_bash_ask_denied(
1241 tx,
1242 pending,
1243 routes,
1244 live_roots,
1245 route_bash_cancels,
1246 shutdown,
1247 metrics,
1248 )
1249 .await
1250}
1251
1252#[allow(clippy::too_many_arguments)]
1253async fn cancel_pending_bash_ask_for_tool_call(
1254 tx: &WriterSender,
1255 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1256 route: RouteChannel,
1257 tool_corr: u64,
1258 routes: &HashMap<RouteChannel, RouteIdentity>,
1259 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1260 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1261 shutdown: &Arc<Notify>,
1262 metrics: &DispatchPathMetrics,
1263) -> Result<(), SubcError> {
1264 let keys = pending_bash_asks
1265 .iter()
1266 .filter_map(|(key, pending)| {
1267 (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1268 })
1269 .collect::<Vec<_>>();
1270 for key in keys {
1271 if let Some(pending) = pending_bash_asks.remove(&key) {
1272 settle_pending_bash_ask_denied(
1273 tx,
1274 pending,
1275 routes,
1276 live_roots,
1277 route_bash_cancels,
1278 shutdown,
1279 metrics,
1280 )
1281 .await?;
1282 }
1283 }
1284 Ok(())
1285}
1286
1287fn remove_root_channel(
1288 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1289 root: &ProjectRootId,
1290 channel: RouteChannel,
1291) {
1292 let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1293 channels.remove(&channel);
1294 channels.is_empty()
1295 } else {
1296 false
1297 };
1298 if remove_root {
1299 root_channels.remove(root);
1300 }
1301}
1302
1303fn remove_route_channel(
1304 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1305 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1306 channel: RouteChannel,
1307) -> Option<RouteIdentity> {
1308 let removed = routes.remove(&channel);
1309 if let Some(identity) = &removed {
1310 remove_root_channel(root_channels, &identity.root, channel);
1311 }
1312 removed
1313}
1314
1315fn insert_route_channel(
1316 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1317 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1318 channel: RouteChannel,
1319 identity: RouteIdentity,
1320) {
1321 if let Some(previous) = routes.insert(channel, identity.clone()) {
1322 remove_root_channel(root_channels, &previous.root, channel);
1323 }
1324 root_channels
1325 .entry(identity.root.clone())
1326 .or_default()
1327 .insert(channel);
1328}
1329
1330fn remove_bg_subscription_index(
1331 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1332 channel: RouteChannel,
1333 identity: Option<&RouteIdentity>,
1334) {
1335 if let Some(identity) = identity {
1336 let key = (identity.root.clone(), identity.session.clone());
1337 if bg_sub_by_session.get(&key).copied() == Some(channel) {
1338 bg_sub_by_session.remove(&key);
1339 }
1340 } else {
1341 bg_sub_by_session.retain(|_, mapped_channel| *mapped_channel != channel);
1342 }
1343}
1344
1345fn route_removal_will_quiesce_root(
1346 root: &ProjectRootId,
1347 route: RouteChannel,
1348 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1349 has_pending_bind: bool,
1350 replacement_root: Option<&ProjectRootId>,
1351) -> bool {
1352 let removes_last_route = root_channels
1353 .get(root)
1354 .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1355 removes_last_route && !has_pending_bind && replacement_root != Some(root)
1356}
1357
1358fn should_quiesce_removed_root(
1359 root: &ProjectRootId,
1360 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1361 has_pending_bind: bool,
1362 replacement_root: Option<&ProjectRootId>,
1363) -> bool {
1364 !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1365}
1366
1367async fn end_bg_subscription(
1368 writer_tx: &WriterSender,
1369 metrics: &DispatchPathMetrics,
1370 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1371 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1372 bg_wake_pending: &mut HashSet<RouteChannel>,
1373 channel: RouteChannel,
1374 identity: Option<&RouteIdentity>,
1375) -> Result<(), SubcError> {
1376 if let Some(sub) = bg_subs.remove(&channel) {
1377 bg_wake_pending.remove(&channel);
1378 remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1379 push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1380 }
1381 Ok(())
1382}
1383
1384#[allow(clippy::too_many_arguments)]
1385async fn teardown_installed_route(
1386 tx: &WriterSender,
1387 metrics: &DispatchPathMetrics,
1388 executor: &Arc<Executor>,
1389 channel: RouteChannel,
1390 cancellation_reason: &str,
1391 replacement_root: Option<&ProjectRootId>,
1392 installed_route_epochs: &mut HashMap<u16, u32>,
1393 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1394 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1395 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1396 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1397 bg_wake_pending: &mut HashSet<RouteChannel>,
1398 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1399 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1400 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1401 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1402 retry_buffer: &mut RetryBuffer,
1403 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1404 shutdown: &Arc<Notify>,
1405) -> Result<(), SubcError> {
1406 remove_installed_route(installed_route_epochs, channel);
1407 end_bg_subscription(
1408 tx,
1409 metrics,
1410 bg_subs,
1411 bg_sub_by_session,
1412 bg_wake_pending,
1413 channel,
1414 routes.get(&channel),
1415 )
1416 .await?;
1417 settle_pending_bash_asks_for_route(
1418 tx,
1419 pending_bash_asks,
1420 channel,
1421 routes,
1422 live_roots,
1423 route_bash_cancels,
1424 shutdown,
1425 metrics,
1426 )
1427 .await?;
1428 if let Some(cancel) = route_bash_cancels.remove(&channel) {
1429 cancel.token.cancel();
1430 }
1431 if let Some(pending) = pending_binds.get_mut(&channel) {
1432 pending.cancelled = true;
1433 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1434 log::debug!(
1435 "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1436 channel.channel
1437 );
1438 }
1439 let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1440 if let Some(identity) = routes.get(&channel) {
1441 let has_pending_bind = pending_binds
1442 .values()
1443 .any(|pending| pending.bind_root_id == identity.root);
1444 if route_removal_will_quiesce_root(
1445 &identity.root,
1446 channel,
1447 root_channels,
1448 has_pending_bind,
1449 replacement_root,
1450 ) {
1451 if let Some(ctx) = executor.actor_context(&identity.root) {
1452 ctx.mark_subc_unbound();
1455 }
1456 }
1457 }
1458 if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
1459 if migrated > 0 {
1460 log::debug!(
1461 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
1462 channel.channel
1463 );
1464 }
1465 if let Some(meta) = live_roots.get_mut(&identity.root) {
1466 let idle_for = meta.last_touched.elapsed();
1467 meta.note_activity();
1468 log::debug!(
1469 "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
1470 channel.channel,
1471 identity.root.as_path().display(),
1472 identity.harness,
1473 identity.session,
1474 idle_for
1475 );
1476 } else {
1477 log::debug!(
1478 "subc attach: route {} torn down for root {} harness {} session {}",
1479 channel.channel,
1480 identity.root.as_path().display(),
1481 identity.harness,
1482 identity.session
1483 );
1484 }
1485 let has_pending_bind = pending_binds
1486 .values()
1487 .any(|pending| pending.bind_root_id == identity.root);
1488 if should_quiesce_removed_root(
1489 &identity.root,
1490 root_channels,
1491 has_pending_bind,
1492 replacement_root,
1493 ) {
1494 quiesce_unbound_root(&identity.root, live_roots, executor);
1495 }
1496 } else {
1497 if migrated > 0 {
1498 log::debug!(
1499 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
1500 channel.channel
1501 );
1502 }
1503 log::debug!("subc attach: unbound route {} torn down", channel.channel);
1504 }
1505 Ok(())
1506}
1507
1508fn remember_session_identity(
1509 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1510 identity: &RouteIdentity,
1511) {
1512 let key = (identity.root.clone(), identity.session.clone());
1513 if matches!(identity.trust, BindTrust::Untrusted)
1514 && session_identity
1515 .get(&key)
1516 .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
1517 {
1518 return;
1519 }
1520
1521 session_identity.insert(
1526 key,
1527 RetainedSessionIdentity {
1528 harness: identity.harness.clone(),
1529 trust: identity.trust,
1530 },
1531 );
1532}
1533
1534fn replay_key_for_session(
1535 session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1536 root: &ProjectRootId,
1537 session: &str,
1538) -> Option<(push::ReplayKey, BindTrust)> {
1539 let retained = session_identity.get(&(root.clone(), session.to_string()))?;
1540 Some((
1541 push::ReplayKey {
1542 root: root.clone(),
1543 harness: retained.harness.clone(),
1544 session: session.to_string(),
1545 },
1546 retained.trust,
1547 ))
1548}
1549pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
1552
1553#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1554enum ModuleLoopExit {
1555 Graceful,
1556 SkipSearchFlush,
1557}
1558
1559pub fn run_subc_mode(
1564 connection_file_path: &Path,
1565 ctx: Arc<AppContext>,
1566 executor: Arc<Executor>,
1567 dispatch: DispatchFn,
1568 user_config_path: Option<PathBuf>,
1569) -> Result<(), SubcError> {
1570 run_subc_mode_inner(
1574 connection_file_path,
1575 ctx,
1576 executor,
1577 dispatch,
1578 user_config_path,
1579 false,
1580 )
1581}
1582
1583fn run_subc_mode_inner(
1584 connection_file_path: &Path,
1585 ctx: Arc<AppContext>,
1586 executor: Arc<Executor>,
1587 dispatch: DispatchFn,
1588 user_config_path: Option<PathBuf>,
1589 allow_native_passthrough: bool,
1590) -> Result<(), SubcError> {
1591 let runtime = tokio::runtime::Builder::new_current_thread()
1592 .enable_all()
1593 .build()
1594 .map_err(SubcError::Runtime)?;
1595
1596 let executor_for_loop = Arc::clone(&executor);
1597 let loop_result = runtime.block_on(async move {
1598 let shared_app = ctx.app();
1599 drop(ctx);
1600 let stream = connect_and_authenticate(connection_file_path).await?;
1601 log::info!(
1602 "subc attach: authenticated to daemon via {}",
1603 connection_file_path.display()
1604 );
1605 let (read_half, write_half) = tokio::io::split(stream);
1606 run_module_loop(
1607 read_half,
1608 write_half,
1609 shared_app,
1610 executor_for_loop,
1611 dispatch,
1612 user_config_path,
1613 allow_native_passthrough,
1614 )
1615 .await
1616 });
1617
1618 let actor_contexts = executor.actor_contexts();
1619 if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
1620 flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
1623 }
1624 for actor_ctx in &actor_contexts {
1625 actor_ctx.lsp().shutdown_all();
1626 actor_ctx.bash_background().detach();
1627 }
1628
1629 loop_result.map(|_| ())
1630}
1631
1632fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
1633 for actor_ctx in actor_contexts {
1634 let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
1635 }
1636 let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
1637}
1638
1639#[doc(hidden)]
1644pub fn run_subc_mode_for_test(
1645 connection_file_path: &Path,
1646 ctx: Arc<AppContext>,
1647 executor: Arc<Executor>,
1648 dispatch: DispatchFn,
1649 user_config_path: Option<PathBuf>,
1650) -> Result<(), SubcError> {
1651 run_subc_mode_inner(
1652 connection_file_path,
1653 ctx,
1654 executor,
1655 dispatch,
1656 user_config_path,
1657 true,
1658 )
1659}
1660
1661#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1662enum AttachErrorClass {
1663 Transient,
1664 Permanent,
1665}
1666
1667impl fmt::Display for AttachErrorClass {
1668 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1669 match self {
1670 Self::Transient => f.write_str("transient"),
1671 Self::Permanent => f.write_str("permanent"),
1672 }
1673 }
1674}
1675
1676#[derive(Clone, Copy)]
1677struct AttachRetryPolicy {
1678 budget: Duration,
1679 initial_backoff: Duration,
1680 max_backoff: Duration,
1681 jitter_percent: u64,
1682}
1683
1684const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
1685 budget: ATTACH_RETRY_BUDGET,
1686 initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
1687 max_backoff: ATTACH_RETRY_MAX_BACKOFF,
1688 jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
1689};
1690
1691fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
1694 let transient = match error {
1695 SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
1696 SubcError::Auth { source, .. } => match source {
1697 subc_transport::AuthError::Timeout { .. }
1698 | subc_transport::AuthError::UnexpectedEof { .. } => true,
1699 subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
1700 _ => false,
1701 },
1702 _ => false,
1703 };
1704 if transient {
1705 AttachErrorClass::Transient
1706 } else {
1707 AttachErrorClass::Permanent
1708 }
1709}
1710
1711fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
1712 matches!(
1713 kind,
1714 io::ErrorKind::ConnectionRefused
1715 | io::ErrorKind::TimedOut
1716 | io::ErrorKind::ConnectionReset
1717 | io::ErrorKind::ConnectionAborted
1718 | io::ErrorKind::BrokenPipe
1719 | io::ErrorKind::UnexpectedEof
1720 )
1721}
1722
1723async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
1727 connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
1728}
1729
1730async fn connect_and_authenticate_with_policy(
1731 connection_file_path: &Path,
1732 policy: AttachRetryPolicy,
1733) -> Result<TcpStream, SubcError> {
1734 let started_at = Instant::now();
1735 let deadline = started_at + policy.budget;
1736 let mut attempt = 0_u32;
1737 let mut backoff = policy.initial_backoff;
1738 let mut history = Vec::new();
1739
1740 loop {
1741 attempt = attempt.saturating_add(1);
1742 let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
1743 Ok(stream) => return Ok(stream),
1744 Err(error) => error,
1745 };
1746 let class = classify_attach_error(&error);
1747 let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
1748 history.push(format!("attempt {attempt} [{class}]: {error_text}"));
1749
1750 if class == AttachErrorClass::Permanent {
1751 log_attach_final_failure(started_at.elapsed(), &history);
1752 return Err(error);
1753 }
1754
1755 let remaining = deadline.saturating_duration_since(Instant::now());
1756 if remaining.is_zero() {
1757 log_attach_final_failure(started_at.elapsed(), &history);
1758 return Err(error);
1759 }
1760
1761 let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
1762 log::info!(
1763 "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
1764 );
1765 tokio::time::sleep(delay).await;
1766
1767 if Instant::now() >= deadline {
1768 log_attach_final_failure(started_at.elapsed(), &history);
1769 return Err(error);
1770 }
1771 backoff = backoff.saturating_mul(2).min(policy.max_backoff);
1772 }
1773}
1774
1775fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
1776 let jitter_percent = jitter_percent.min(100);
1777 if jitter_percent == 0 {
1778 return base;
1779 }
1780
1781 let mut random_bytes = [0_u8; 8];
1782 let random = if getrandom::fill(&mut random_bytes).is_ok() {
1783 u64::from_le_bytes(random_bytes)
1784 } else {
1785 let timestamp = std::time::SystemTime::now()
1786 .duration_since(std::time::UNIX_EPOCH)
1787 .unwrap_or_default()
1788 .subsec_nanos();
1789 u64::from(timestamp) ^ u64::from(attempt)
1790 };
1791 let span = jitter_percent.saturating_mul(2).saturating_add(1);
1792 let multiplier_percent = 100 - jitter_percent + random % span;
1793 let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
1794 Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
1795}
1796
1797fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
1798 log::error!(
1799 "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
1800 history.len(),
1801 history.join(" | ")
1802 );
1803}
1804
1805async fn connect_and_authenticate_once(
1806 connection_file_path: &Path,
1807 deadline: Instant,
1808) -> Result<TcpStream, SubcError> {
1809 let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
1812 SubcError::ConnectionFile {
1813 path: connection_file_path.to_path_buf(),
1814 source,
1815 }
1816 })?;
1817
1818 let endpoint = conn
1819 .endpoints
1820 .first()
1821 .ok_or_else(|| SubcError::NoEndpoint {
1822 path: connection_file_path.to_path_buf(),
1823 })?;
1824 let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
1825 let ip = endpoint
1826 .host
1827 .parse::<IpAddr>()
1828 .map_err(|_| SubcError::InvalidEndpoint {
1829 path: connection_file_path.to_path_buf(),
1830 endpoint: endpoint_label.clone(),
1831 })?;
1832 let addr = SocketAddr::new(ip, endpoint.port);
1833
1834 let connect_budget = deadline.saturating_duration_since(Instant::now());
1835 let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
1836 .await
1837 .map_err(|_| SubcError::Connect {
1838 endpoint: endpoint_label.clone(),
1839 source: io::Error::new(
1840 io::ErrorKind::TimedOut,
1841 "initial subc attach retry budget elapsed during TCP connect",
1842 ),
1843 })?
1844 .map_err(|source| SubcError::Connect {
1845 endpoint: endpoint_label.clone(),
1846 source,
1847 })?;
1848 stream
1849 .set_nodelay(true)
1850 .map_err(|source| SubcError::Connect {
1851 endpoint: endpoint_label.clone(),
1852 source,
1853 })?;
1854
1855 let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
1856 authenticate_client(&mut stream, &conn, auth_budget)
1857 .await
1858 .map_err(|source| SubcError::Auth {
1859 endpoint: endpoint_label,
1860 source,
1861 })?;
1862
1863 Ok(stream)
1864}
1865
1866#[allow(clippy::too_many_arguments)]
1867async fn process_route_bind_completion(
1868 writer_tx: &WriterSender,
1869 completion: RouteBindCompletion,
1870 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1871 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1872 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1873 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1874 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1875 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1876 installed_route_epochs: &mut HashMap<u16, u32>,
1877 executor: &Arc<Executor>,
1878 shutdown: &Arc<Notify>,
1879 metrics: &Arc<DispatchPathMetrics>,
1880) -> Result<(), SubcError> {
1881 decrement_counted_channel(&metrics.control_completion_queued);
1882 handle_route_bind_completion(
1883 writer_tx,
1884 completion,
1885 routes,
1886 root_channels,
1887 session_identity,
1888 push_buffer,
1889 live_roots,
1890 pending_binds,
1891 installed_route_epochs,
1892 executor,
1893 shutdown,
1894 metrics,
1895 )
1896 .await
1897}
1898
1899#[allow(clippy::too_many_arguments)]
1900async fn drain_pending_route_bind_completions(
1901 control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
1902 writer_tx: &WriterSender,
1903 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1904 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1905 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1906 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1907 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1908 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1909 installed_route_epochs: &mut HashMap<u16, u32>,
1910 executor: &Arc<Executor>,
1911 shutdown: &Arc<Notify>,
1912 metrics: &Arc<DispatchPathMetrics>,
1913) -> Result<usize, SubcError> {
1914 let mut drained = 0;
1915 while let Ok(completion) = control_completion_rx.try_recv() {
1916 process_route_bind_completion(
1917 writer_tx,
1918 completion,
1919 routes,
1920 root_channels,
1921 session_identity,
1922 push_buffer,
1923 live_roots,
1924 pending_binds,
1925 installed_route_epochs,
1926 executor,
1927 shutdown,
1928 metrics,
1929 )
1930 .await?;
1931 drained += 1;
1932 }
1933 Ok(drained)
1934}
1935
1936async fn run_module_loop<R, W>(
1940 mut read: R,
1941 mut write: W,
1942 shared_app: Arc<App>,
1943 executor: Arc<Executor>,
1944 dispatch: DispatchFn,
1945 user_config_path: Option<PathBuf>,
1946 allow_native_passthrough: bool,
1947) -> Result<ModuleLoopExit, SubcError>
1948where
1949 R: AsyncRead + Unpin + Send + 'static,
1950 W: AsyncWrite + Unpin + Send + 'static,
1951{
1952 let hello = ModuleHelloBody {
1956 manifest: build_manifest(),
1957 protocol_ver: PROTOCOL_VERSION,
1958 control_ops: control_ops(),
1959 launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
1960 };
1961 let hello_frame = Frame::build(
1962 FrameType::Hello,
1963 control_flags(),
1964 0,
1965 0,
1966 HELLO_CORR,
1967 serde_json::to_vec(&hello).map_err(SubcError::Json)?,
1968 )
1969 .map_err(SubcError::FrameBuild)?;
1970 write_frame(&mut write, &hello_frame)
1971 .await
1972 .map_err(SubcError::FrameIo)?;
1973
1974 match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
1976 None => return Err(SubcError::ClosedBeforeHelloAck),
1977 Some(frame) => match frame.header.ty {
1978 FrameType::HelloAck => {
1979 log::info!("subc attach: registered (HelloAck received)");
1980 }
1981 FrameType::Error => {
1982 let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
1983 return Err(SubcError::HelloRejected { body });
1984 }
1985 other => return Err(SubcError::UnexpectedFrame { ty: other }),
1986 },
1987 }
1988
1989 let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
1990 let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
1991 let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
1992 let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
1999 let reader_task = spawn_reader_task(read, reader_tx);
2000 let shutdown = Arc::new(Notify::new());
2001 let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2010 let mut next_maintenance_at = next_drain_at;
2011 let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2012 let (bash_deferred_tx, mut bash_deferred_rx) =
2013 mpsc::channel::<bash::BashDeferredCompletion>(256);
2014 let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2015 let (control_completion_tx, mut control_completion_rx) =
2016 mpsc::channel::<RouteBindCompletion>(256);
2017 let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2018 let lossy_overflow = Arc::new(push::LossyOverflow::default());
2019 let lossy_seq = Arc::new(AtomicU64::new(0));
2020 let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2021 let push_senders = PushSenders {
2022 lossy_tx,
2023 reliable_tx,
2024 lossy_overflow: Arc::clone(&lossy_overflow),
2025 lossy_seq,
2026 };
2027 let connection_cancel = PersistentCancelSignal::new();
2028 let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2029 let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2030 let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2031 let mut bg_sub_by_session: HashMap<(ProjectRootId, String), RouteChannel> = HashMap::new();
2032 let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2033 let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2034 let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2035 let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2036 HashMap::new();
2037 let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2038 let mut retry_buffer: RetryBuffer = HashMap::new();
2039 let mut completed_tasks = push::CompletedTaskIds::default();
2040 let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2041 let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2042 let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2043 let mut next_bash_ask_corr: u64 = 1;
2044 let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2045
2046 let loop_result: Result<ModuleLoopExit, SubcError> = loop {
2047 crate::logging::perf_tick(Some(&executor));
2048 dispatch_path_metrics.mark_frame_loop_tick();
2049 if let Err(error) = expire_pending_bash_asks(
2050 &writer_tx,
2051 &mut pending_bash_asks,
2052 &routes,
2053 &mut live_roots,
2054 &mut route_bash_cancels,
2055 &shutdown,
2056 &dispatch_path_metrics,
2057 )
2058 .await
2059 {
2060 break Err(error);
2061 }
2062
2063 match drain_pending_route_bind_completions(
2067 &mut control_completion_rx,
2068 &writer_tx,
2069 &mut routes,
2070 &mut root_channels,
2071 &mut session_identity,
2072 &mut push_buffer,
2073 &mut live_roots,
2074 &mut pending_binds,
2075 &mut installed_route_epochs,
2076 &executor,
2077 &shutdown,
2078 &dispatch_path_metrics,
2079 )
2080 .await
2081 {
2082 Ok(drained) => {
2083 if drained > 0 {
2084 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2085 }
2086 }
2087 Err(error) => break Err(error),
2088 }
2089
2090 if tokio::time::Instant::now() >= next_drain_at {
2091 push::emit_bg_event_wakes(
2092 &writer_tx,
2093 &dispatch_path_metrics,
2094 &bg_subs,
2095 &mut bg_wake_pending,
2096 );
2097 warn_slow_pending_binds(&mut pending_binds, &executor);
2098 if let Err(error) = expire_overdue_route_binds(
2099 &writer_tx,
2100 &executor,
2101 &mut pending_binds,
2102 &mut installed_route_epochs,
2103 &dispatch_path_metrics,
2104 )
2105 .await
2106 {
2107 break Err(error);
2108 }
2109
2110 let retried = push::drain_retry_buffers_for_bound_routes(
2111 &writer_tx,
2112 &dispatch_path_metrics,
2113 &routes,
2114 &mut retry_buffer,
2115 );
2116 if retried > 0 {
2117 log::debug!(
2118 "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2119 );
2120 }
2121
2122 next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2123 }
2124
2125 let overflow_batch = lossy_overflow.drain();
2131 if !overflow_batch.is_empty() {
2132 let (_, deferred) = push::drain_reliable_push_turn(
2133 &writer_tx,
2134 &dispatch_path_metrics,
2135 &routes,
2136 &root_channels,
2137 &session_identity,
2138 &mut retry_buffer,
2139 &mut push_buffer,
2140 &mut completed_tasks,
2141 &bg_sub_by_session,
2142 &mut bg_wake_pending,
2143 &mut bg_wake_epoch,
2144 &mut reliable_rx,
2145 None,
2146 );
2147 if deferred {
2148 tokio::task::yield_now().await;
2149 }
2150
2151 let mut batch = Vec::new();
2152 while let Ok(item) = lossy_rx.try_recv() {
2153 batch.push(item);
2154 }
2155 batch.extend(overflow_batch);
2156 push::process_lossy_push_envelope_batch(
2157 &writer_tx,
2158 &dispatch_path_metrics,
2159 &routes,
2160 &root_channels,
2161 &completed_tasks,
2162 batch,
2163 );
2164 }
2165
2166 tokio::select! {
2167 biased;
2168 Some(completion) = control_completion_rx.recv() => {
2169 if let Err(error) = process_route_bind_completion(
2170 &writer_tx,
2171 completion,
2172 &mut routes,
2173 &mut root_channels,
2174 &mut session_identity,
2175 &mut push_buffer,
2176 &mut live_roots,
2177 &mut pending_binds,
2178 &mut installed_route_epochs,
2179 &executor,
2180 &shutdown,
2181 &dispatch_path_metrics,
2182 )
2183 .await
2184 {
2185 break Err(error);
2186 }
2187 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2188 }
2189 _ = shutdown.notified() => {
2190 log::warn!("subc attach: fatal executor response requested teardown");
2191 break Ok(ModuleLoopExit::SkipSearchFlush);
2192 }
2193 maybe_frame = reader_rx.recv() => {
2194 let frame = match maybe_frame {
2195 None => {
2196 log::info!("subc attach: daemon closed connection");
2197 break Ok(ModuleLoopExit::Graceful);
2198 }
2199 Some(Err(error)) => break Err(error),
2200 Some(Ok(frame)) => frame,
2201 };
2202 let phase_trace = frame.phase_trace;
2203 let frame = frame.frame;
2204
2205 if !ingress_route_is_current(&installed_route_epochs, &frame) {
2206 log::debug!(
2207 "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2208 frame.header.ty,
2209 frame.header.channel,
2210 frame.header.epoch
2211 );
2212 continue;
2213 }
2214
2215 match frame.header.ty {
2216 FrameType::Ping if frame.header.channel == 0 => {
2217 let pong = match Frame::build_with_version(
2218 frame.header.ver,
2219 FrameType::Pong,
2220 frame.header.flags,
2221 0,
2222 0,
2223 frame.header.corr,
2224 Vec::new(),
2225 ) {
2226 Ok(pong) => pong,
2227 Err(error) => break Err(SubcError::FrameBuild(error)),
2228 };
2229 if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2230 break Err(error);
2231 }
2232 }
2233 FrameType::Goodbye if frame.header.channel == 0 => {
2234 log::info!("subc attach: received channel-0 Goodbye");
2235 break Ok(ModuleLoopExit::Graceful);
2236 }
2237 FrameType::Goodbye => {
2238 let channel = route_key(frame.header.channel, frame.header.epoch);
2239 if let Err(error) = teardown_installed_route(
2240 &writer_tx,
2241 &dispatch_path_metrics,
2242 &executor,
2243 channel,
2244 "Goodbye",
2245 None,
2246 &mut installed_route_epochs,
2247 &mut routes,
2248 &mut root_channels,
2249 &mut bg_subs,
2250 &mut bg_sub_by_session,
2251 &mut bg_wake_pending,
2252 &mut pending_bash_asks,
2253 &mut live_roots,
2254 &mut route_bash_cancels,
2255 &mut pending_binds,
2256 &mut retry_buffer,
2257 &mut push_buffer,
2258 &shutdown,
2259 )
2260 .await
2261 {
2262 break Err(error);
2263 }
2264 }
2265 FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2266 if let Err(error) = handle_bash_elicitation_reply(
2267 &writer_tx,
2268 &frame,
2269 &mut pending_bash_asks,
2270 &routes,
2271 &mut live_roots,
2272 &executor,
2273 &shutdown,
2274 &bash_deferred_tx,
2275 &bash_poll_touch_tx,
2276 &dispatch_path_metrics,
2277 &mut route_bash_cancels,
2278 dispatch,
2279 )
2280 .await
2281 {
2282 break Err(error);
2283 }
2284 }
2285 FrameType::Request if frame.header.channel == 0 => {
2286 if let Err(error) = handle_control_request(
2287 &writer_tx,
2288 &frame,
2289 &shared_app,
2290 &executor,
2291 &mut live_roots,
2292 &mut pending_binds,
2293 &mut installed_route_epochs,
2294 &mut routes,
2295 &mut root_channels,
2296 &mut bg_subs,
2297 &mut bg_sub_by_session,
2298 &mut bg_wake_pending,
2299 &mut pending_bash_asks,
2300 &mut route_bash_cancels,
2301 &mut retry_buffer,
2302 &mut push_buffer,
2303 &shutdown,
2304 &control_completion_tx,
2305 &dispatch_path_metrics,
2306 &push_senders,
2307 dispatch,
2308 user_config_path.as_deref(),
2309 )
2310 .await
2311 {
2312 break Err(error);
2313 }
2314 }
2315 FrameType::Request => {
2316 if let Err(error) = handle_tool_call(
2317 &writer_tx,
2318 &frame,
2319 phase_trace,
2320 &routes,
2321 &pending_binds,
2322 &mut live_roots,
2323 &executor,
2324 &shutdown,
2325 &connection_cancel,
2326 &bash_deferred_tx,
2327 &bash_poll_touch_tx,
2328 &dispatch_path_metrics,
2329 &mut route_bash_cancels,
2330 &mut pending_bash_asks,
2331 &mut next_bash_ask_corr,
2332 &mut bg_subs,
2333 &mut bg_sub_by_session,
2334 &mut bg_wake_pending,
2335 &mut bg_wake_epoch,
2336 dispatch,
2337 allow_native_passthrough,
2338 )
2339 .await
2340 {
2341 break Err(error);
2342 }
2343 }
2344 FrameType::Cancel => {
2345 let channel = route_key(frame.header.channel, frame.header.epoch);
2346 if bg_subs.contains_key(&channel) {
2347 if let Err(error) = end_bg_subscription(
2348 &writer_tx,
2349 &dispatch_path_metrics,
2350 &mut bg_subs,
2351 &mut bg_sub_by_session,
2352 &mut bg_wake_pending,
2353 channel,
2354 routes.get(&channel),
2355 )
2356 .await
2357 {
2358 break Err(error);
2359 }
2360 }
2361 if let Err(error) = cancel_pending_bash_ask_for_tool_call(
2362 &writer_tx,
2363 &mut pending_bash_asks,
2364 channel,
2365 frame.header.corr,
2366 &routes,
2367 &mut live_roots,
2368 &mut route_bash_cancels,
2369 &shutdown,
2370 &dispatch_path_metrics,
2371 )
2372 .await
2373 {
2374 break Err(error);
2375 }
2376 }
2377 _ => {}
2382 }
2383 }
2384 Some((root_id, frame)) = reliable_rx.recv() => {
2385 let (_, deferred) = push::drain_reliable_push_turn(
2389 &writer_tx,
2390 &dispatch_path_metrics,
2391 &routes,
2392 &root_channels,
2393 &session_identity,
2394 &mut retry_buffer,
2395 &mut push_buffer,
2396 &mut completed_tasks,
2397 &bg_sub_by_session,
2398 &mut bg_wake_pending,
2399 &mut bg_wake_epoch,
2400 &mut reliable_rx,
2401 Some((root_id, frame)),
2402 );
2403 if deferred {
2404 tokio::task::yield_now().await;
2405 }
2406 }
2407 Some((order, root_id, frame)) = lossy_rx.recv() => {
2408 let (_, deferred) = push::drain_reliable_push_turn(
2412 &writer_tx,
2413 &dispatch_path_metrics,
2414 &routes,
2415 &root_channels,
2416 &session_identity,
2417 &mut retry_buffer,
2418 &mut push_buffer,
2419 &mut completed_tasks,
2420 &bg_sub_by_session,
2421 &mut bg_wake_pending,
2422 &mut bg_wake_epoch,
2423 &mut reliable_rx,
2424 None,
2425 );
2426 if deferred {
2427 tokio::task::yield_now().await;
2428 }
2429
2430 let mut batch = vec![(order, root_id, frame)];
2437 while let Ok(item) = lossy_rx.try_recv() {
2438 batch.push(item);
2439 }
2440 batch.extend(lossy_overflow.drain());
2441 push::process_lossy_push_envelope_batch(
2442 &writer_tx,
2443 &dispatch_path_metrics,
2444 &routes,
2445 &root_channels,
2446 &completed_tasks,
2447 batch,
2448 );
2449 }
2450 Some(done) = bash_deferred_rx.recv() => {
2451 decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
2452 if let Err(error) = bash::handle_bash_deferred_completion(
2453 &writer_tx,
2454 done,
2455 &routes,
2456 &mut live_roots,
2457 &mut route_bash_cancels,
2458 &shutdown,
2459 &dispatch_path_metrics,
2460 )
2461 .await
2462 {
2463 break Err(error);
2464 }
2465 }
2466 Some(root_id) = bash_poll_touch_rx.recv() => {
2467 decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
2468 if let Some(meta) = live_roots.get_mut(&root_id) {
2469 meta.note_activity();
2470 }
2471 }
2472 Some(completion) = maintenance_rx.recv() => {
2473 decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
2474 let root_id = completion.root_id.clone();
2475 let response = completion.response;
2476 let response_is_fatal = response_is_fatal_panic(&response);
2477 let bind_pending = pending_binds
2478 .values()
2479 .any(|pending| pending.bind_root_id == root_id);
2480 let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
2481 let defer_requeue = meta.unbound_quiesced || bind_pending;
2482 note_maintenance_completion(
2483 meta,
2484 completion.requeue_kind,
2485 response_is_fatal,
2486 defer_requeue,
2487 );
2488 should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
2489 } else {
2490 false
2491 };
2492 if requiesce {
2493 quiesce_unbound_root(&root_id, &mut live_roots, &executor);
2494 }
2495 push::clear_stale_bg_wakes_for_empty_sessions(
2496 &root_id,
2497 &completion.empty_bg_sessions,
2498 &bg_sub_by_session,
2499 &mut bg_wake_pending,
2500 &bg_wake_epoch,
2501 );
2502 if response_is_fatal {
2503 if let Some(meta) = live_roots.get_mut(&root_id) {
2504 meta.maintenance_poisoned = true;
2505 }
2506 log::warn!(
2507 "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
2508 );
2509 }
2510 }
2511 _ = tokio::time::sleep_until(next_drain_at) => {
2512 }
2515 _ = tokio::time::sleep_until(next_maintenance_at) => {
2516 let reaped = reap_idle_roots(
2521 Instant::now(),
2522 &mut live_roots,
2523 &pending_binds,
2524 &executor,
2525 );
2526 if reaped > 0 {
2527 log::debug!("subc attach: reaped {reaped} idle root(s)");
2528 }
2529 submit_due_maintenance_jobs(
2530 &executor,
2531 &mut live_roots,
2532 &pending_binds,
2533 &bg_sub_by_session,
2534 &bg_wake_pending,
2535 &bg_wake_epoch,
2536 &maintenance_tx,
2537 &dispatch_path_metrics,
2538 );
2539 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2540 }
2541 }
2542 };
2543
2544 for pending in pending_binds.values() {
2548 let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
2549 }
2550
2551 let mut loop_result = loop_result;
2552 if !pending_bash_asks.is_empty() {
2553 let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2554 if let Err(error) = settle_all_pending_bash_asks(
2555 &writer_tx,
2556 &mut pending_bash_asks,
2557 &no_routes,
2558 &mut live_roots,
2559 &mut route_bash_cancels,
2560 &shutdown,
2561 &dispatch_path_metrics,
2562 )
2563 .await
2564 {
2565 loop_result = loop_result.and(Err(error));
2566 }
2567 }
2568
2569 connection_cancel.cancel();
2572 reader_task.abort();
2573 drop(writer_tx);
2574 let writer_result = finish_writer_task(writer_task).await;
2575 loop_result.and_then(|exit| writer_result.map(|_| exit))
2576}
2577
2578fn spawn_writer_task<W>(
2579 mut write: W,
2580 mut rx: mpsc::Receiver<WriterFrame>,
2581 metrics: Arc<DispatchPathMetrics>,
2582) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
2583where
2584 W: AsyncWrite + Unpin + Send + 'static,
2585{
2586 tokio::spawn(async move {
2587 let mut write_buffer = Vec::new();
2588 while let Some(mut queued) = rx.recv().await {
2589 let measure = queued.tool_response_trace.is_some();
2590 let dequeued = measure.then(Instant::now);
2591 metrics.writer_active.store(true, Ordering::Relaxed);
2592 decrement_counted_channel(&metrics.writer_queued);
2593 let write_timing =
2594 write_frame_contiguous(&mut write, &queued.frame, &mut write_buffer, measure).await;
2595 metrics.writer_active.store(false, Ordering::Relaxed);
2596 let write_timing = write_timing?;
2597
2598 if let (Some(trace), Some(dequeued), Some(write_timing)) =
2599 (queued.tool_response_trace.take(), dequeued, write_timing)
2600 {
2601 if let Some(completed) = trace.finish(
2602 dequeued,
2603 write_timing.write_started,
2604 write_timing.write_finished,
2605 write_timing.frame_bytes,
2606 ) {
2607 log_ctx::with_session(Some(completed.session), || {
2608 crate::logging::note_tool_call_trace(
2609 &completed.name,
2610 &completed.root,
2611 completed.channel,
2612 completed.corr,
2613 completed.phases,
2614 );
2615 });
2616 }
2617 }
2618 }
2619 Ok(())
2620 })
2621}
2622
2623struct FrameWriteTiming {
2624 write_started: Instant,
2625 write_finished: Instant,
2626 frame_bytes: usize,
2627}
2628
2629async fn write_frame_contiguous<W>(
2633 writer: &mut W,
2634 frame: &Frame,
2635 buffer: &mut Vec<u8>,
2636 measure: bool,
2637) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
2638where
2639 W: AsyncWrite + Unpin,
2640{
2641 if frame.header.len as usize != frame.body.len() {
2642 return Err(subc_transport::FrameIoError::BodyLengthMismatch {
2643 header_len: frame.header.len,
2644 body_len: frame.body.len(),
2645 });
2646 }
2647
2648 let header = frame.header.encode();
2649 buffer.clear();
2650 buffer.reserve(header.len() + frame.body.len());
2651 buffer.extend_from_slice(&header);
2652 buffer.extend_from_slice(&frame.body);
2653 let write_started = measure.then(Instant::now);
2654 writer
2655 .write_all(buffer)
2656 .await
2657 .map_err(subc_transport::FrameIoError::Io)?;
2658 Ok(write_started.map(|write_started| FrameWriteTiming {
2659 write_started,
2660 write_finished: Instant::now(),
2661 frame_bytes: buffer.len(),
2662 }))
2663}
2664
2665fn spawn_reader_task<R>(
2666 mut read: R,
2667 tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
2668) -> JoinHandle<()>
2669where
2670 R: AsyncRead + Unpin + Send + 'static,
2671{
2672 tokio::spawn(async move {
2673 loop {
2674 match read_frame(&mut read).await {
2675 Ok(Some(frame)) => {
2676 let decoded = DecodedFrame {
2677 frame,
2678 phase_trace: PhaseTrace::new(Instant::now()),
2679 };
2680 if tx.send(Ok(decoded)).await.is_err() {
2681 return;
2682 }
2683 }
2684 Ok(None) => {
2685 return;
2687 }
2688 Err(error) => {
2689 if let subc_transport::FrameIoError::Io(io_error) = &error {
2696 if matches!(
2697 io_error.kind(),
2698 std::io::ErrorKind::ConnectionReset
2699 | std::io::ErrorKind::ConnectionAborted
2700 ) {
2701 log::info!(
2702 "subc attach: connection reset by daemon; treating as close"
2703 );
2704 return;
2705 }
2706 }
2707 let _ = tx.send(Err(SubcError::FrameIo(error))).await;
2708 return;
2709 }
2710 }
2711 }
2712 })
2713}
2714
2715async fn finish_writer_task(
2716 mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
2717) -> Result<(), SubcError> {
2718 match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
2719 Ok(Ok(Ok(()))) => Ok(()),
2720 Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
2721 Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
2722 Err(_) => {
2723 writer_task.abort();
2724 Ok(())
2725 }
2726 }
2727}
2728
2729fn register_actor_for_bind(
2730 shared_app: &Arc<App>,
2731 executor: &Arc<Executor>,
2732 push_senders: &PushSenders,
2733 bind_root_id: &ProjectRootId,
2734 route_channel: u16,
2735 root_was_live: bool,
2736) -> bool {
2737 if executor.actor_registered(bind_root_id) {
2738 log::debug!(
2739 "subc attach: reusing actor for route {} root {}",
2740 route_channel,
2741 bind_root_id.as_path().display()
2742 );
2743 return false;
2744 }
2745
2746 if root_was_live {
2747 log::warn!(
2748 "subc attach: recreating missing actor for live root {} on route {}",
2749 bind_root_id.as_path().display(),
2750 route_channel
2751 );
2752 }
2753
2754 let actor_ctx = Arc::new(AppContext::from_app(
2755 Arc::clone(shared_app),
2756 Config::default(),
2757 ));
2758 install_bash_compressor(&actor_ctx);
2759 actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
2760 push_senders.clone(),
2761 bind_root_id.clone(),
2762 )));
2763 let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
2764 drop(actor_ctx);
2765 if inserted {
2766 log::debug!(
2770 "subc attach: registered actor for route {} root {}",
2771 route_channel,
2772 bind_root_id.as_path().display()
2773 );
2774 } else {
2775 log::debug!(
2776 "subc attach: actor appeared while binding route {} root {}; reusing it",
2777 route_channel,
2778 bind_root_id.as_path().display()
2779 );
2780 }
2781 inserted
2782}
2783
2784fn rollback_pending_bind_actor(
2785 executor: &Arc<Executor>,
2786 live_roots: &HashMap<ProjectRootId, RootMeta>,
2787 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2788 root_id: &ProjectRootId,
2789 inserted_new_actor: bool,
2790) {
2791 if !inserted_new_actor || live_roots.contains_key(root_id) {
2792 return;
2793 }
2794
2795 if let Some((route, pending)) = pending_binds
2796 .iter_mut()
2797 .find(|(_, pending)| &pending.bind_root_id == root_id)
2798 {
2799 pending.inserted_new_actor = true;
2800 log::debug!(
2801 "subc attach: transferred rollback ownership for root {} to pending route {}",
2802 root_id.as_path().display(),
2803 route
2804 );
2805 return;
2806 }
2807
2808 executor.remove_actor(root_id);
2809}
2810
2811fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
2812 match response.data.get("code").and_then(|code| code.as_str()) {
2813 Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
2818 Some("cache_key_probe_failed") => "cache_key_probe_failed",
2822 Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
2826 _ => "config_divergence",
2827 }
2828}
2829
2830fn queue_post_bind_configure_and_completion_maintenance(
2831 root_id: &ProjectRootId,
2832 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2833) {
2834 let Some(meta) = live_roots.get_mut(root_id) else {
2835 return;
2836 };
2837 if meta.maintenance_poisoned || meta.maintenance_pending {
2838 return;
2839 }
2840
2841 meta.maintenance_pending = true;
2842 meta.maintenance_queued_kinds
2843 .push_back(MaintenanceDrainKind::ConfigureTail);
2844 meta.maintenance_queued_kinds
2845 .push_back(MaintenanceDrainKind::CompletionDrains);
2846}
2847
2848#[allow(clippy::too_many_arguments)]
2849async fn handle_route_bind_completion(
2850 tx: &WriterSender,
2851 completion: RouteBindCompletion,
2852 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2853 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2854 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2855 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2856 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2857 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2858 installed_route_epochs: &mut HashMap<u16, u32>,
2859 executor: &Arc<Executor>,
2860 shutdown: &Arc<Notify>,
2861 metrics: &Arc<DispatchPathMetrics>,
2862) -> Result<(), SubcError> {
2863 let route_id = completion.route;
2864 let Some(pending) = pending_binds.remove(&route_id) else {
2865 log::warn!(
2866 "subc attach: dropping RouteBind completion for non-pending route {}",
2867 completion.route
2868 );
2869 rollback_pending_bind_actor(
2870 executor,
2871 live_roots,
2872 pending_binds,
2873 &completion.bind_root_id,
2874 completion.inserted_new_actor,
2875 );
2876 let has_pending_bind = pending_binds
2877 .values()
2878 .any(|pending| pending.bind_root_id == completion.bind_root_id);
2879 if !root_channels
2880 .get(&completion.bind_root_id)
2881 .is_some_and(|channels| !channels.is_empty())
2882 && !has_pending_bind
2883 {
2884 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
2885 }
2886 remove_installed_route(installed_route_epochs, route_id);
2887 return Ok(());
2888 };
2889
2890 if pending.bind_root_id != completion.bind_root_id {
2891 log::warn!(
2892 "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
2893 completion.route,
2894 pending.bind_root_id.as_path().display(),
2895 completion.bind_root_id.as_path().display()
2896 );
2897 }
2898
2899 let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
2900 if pending.cancelled {
2901 rollback_pending_bind_actor(
2902 executor,
2903 live_roots,
2904 pending_binds,
2905 &completion.bind_root_id,
2906 inserted_new_actor,
2907 );
2908 let has_pending_bind = pending_binds
2909 .values()
2910 .any(|pending| pending.bind_root_id == completion.bind_root_id);
2911 if !root_channels
2912 .get(&completion.bind_root_id)
2913 .is_some_and(|channels| !channels.is_empty())
2914 && !has_pending_bind
2915 {
2916 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
2917 }
2918 log::debug!(
2919 "subc attach: discarded completed RouteBind for cancelled route {} root {}",
2920 completion.route,
2921 completion.bind_root_id.as_path().display()
2922 );
2923 remove_installed_route(installed_route_epochs, route_id);
2924 return Ok(());
2925 }
2926
2927 let failure = if !completion.configure_response.success {
2928 Some((
2929 &completion.configure_response,
2930 "configure failed during route bind",
2931 ))
2932 } else {
2933 None
2934 };
2935
2936 if let Some((response, fallback)) = failure {
2937 rollback_pending_bind_actor(
2938 executor,
2939 live_roots,
2940 pending_binds,
2941 &completion.bind_root_id,
2942 inserted_new_actor,
2943 );
2944 let has_pending_bind = pending_binds
2945 .values()
2946 .any(|pending| pending.bind_root_id == completion.bind_root_id);
2947 if !root_channels
2948 .get(&completion.bind_root_id)
2949 .is_some_and(|channels| !channels.is_empty())
2950 && !has_pending_bind
2951 {
2952 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
2953 }
2954 let message = response_message(response, fallback);
2955 let fatal = response_is_fatal_panic(response);
2956 let error_code = route_bind_error_code_for_configure_response(response);
2957 send_route_bind_error_parts(
2958 tx,
2959 completion.ver,
2960 completion.corr,
2961 completion.flags,
2962 error_code,
2963 &message,
2964 metrics,
2965 )
2966 .await?;
2967 remove_installed_route(installed_route_epochs, route_id);
2968 if fatal {
2969 signal_fatal_teardown(
2970 tx,
2971 Some(completion.route),
2972 completion.ver,
2973 completion.corr,
2974 shutdown,
2975 metrics,
2976 )
2977 .await;
2978 }
2979 return Ok(());
2980 }
2981
2982 remember_session_identity(session_identity, &completion.identity);
2983 let replay_key = push::ReplayKey::from_identity(&completion.identity);
2984 let bind_trust = completion.identity.trust;
2985 insert_route_channel(routes, root_channels, route_id, completion.identity);
2986 let restore_watcher = live_roots
2987 .get(&completion.bind_root_id)
2988 .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
2989 live_roots
2990 .entry(completion.bind_root_id.clone())
2991 .and_modify(|meta| {
2992 meta.reactivate_bound();
2993 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
2994 meta.maintenance_poisoned = false;
2995 })
2996 .or_insert_with(|| RootMeta::new(Instant::now()));
2997 if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
2998 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
2999 meta.maintenance_poisoned = false;
3000 }
3001 if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3002 ctx.mark_subc_bound();
3003 if restore_watcher {
3004 crate::commands::configure::ensure_project_watcher(&ctx);
3005 }
3006 }
3007
3008 let ack =
3009 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3010 let response = Frame::build_with_version(
3011 completion.ver,
3012 FrameType::Response,
3013 control_flags(),
3014 0,
3015 0,
3016 completion.corr,
3017 ack,
3018 )
3019 .map_err(SubcError::FrameBuild)?;
3020 send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3021 queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3022 let replayed = push::replay_buffered_push_frames(
3023 tx,
3024 metrics,
3025 route_id,
3026 push_buffer,
3027 &replay_key,
3028 bind_trust,
3029 );
3030 if replayed > 0 {
3031 log::debug!(
3032 "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3033 replayed,
3034 completion.route,
3035 replay_key.root.as_path().display(),
3036 replay_key.harness,
3037 replay_key.session
3038 );
3039 }
3040 log::info!(
3041 "subc attach: route {} bound to root {}",
3042 completion.route,
3043 completion.bind_root_id.as_path().display()
3044 );
3045 Ok(())
3046}
3047
3048async fn expire_overdue_route_binds(
3049 tx: &WriterSender,
3050 executor: &Arc<Executor>,
3051 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3052 installed_route_epochs: &mut HashMap<u16, u32>,
3053 metrics: &DispatchPathMetrics,
3054) -> Result<(), SubcError> {
3055 let now = Instant::now();
3056 let expired: Vec<_> = pending_binds
3057 .iter()
3058 .filter_map(|(route, pending)| {
3059 let age = now.saturating_duration_since(pending.started_at);
3060 (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3061 (
3062 *route,
3063 pending.corr,
3064 pending.ver,
3065 pending.flags,
3066 pending.bind_root_id.clone(),
3067 pending.configure_request_id.clone(),
3068 age,
3069 )
3070 })
3071 })
3072 .collect();
3073
3074 for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3075 if let Some(pending) = pending_binds.get_mut(&route) {
3076 pending.cancelled = true;
3077 pending.deadline_reported = true;
3078 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3079 log::debug!(
3080 "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3081 );
3082 }
3083 remove_installed_route(installed_route_epochs, route);
3084 let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3085 let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3086 send_route_bind_error_parts(
3087 tx,
3088 ver,
3089 corr,
3090 flags,
3091 "actor_not_ready",
3092 &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3093 metrics,
3094 )
3095 .await?;
3096 log::warn!(
3097 "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3098 route,
3099 root_id.as_path().display(),
3100 deadline_ms,
3101 configure_request_id
3102 );
3103 }
3104
3105 Ok(())
3106}
3107
3108#[allow(clippy::too_many_arguments)]
3113async fn handle_control_request(
3114 tx: &WriterSender,
3115 frame: &Frame,
3116 shared_app: &Arc<App>,
3117 executor: &Arc<Executor>,
3118 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3119 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3120 installed_route_epochs: &mut HashMap<u16, u32>,
3121 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3122 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3123 bg_subs: &mut HashMap<RouteChannel, BgSub>,
3124 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3125 bg_wake_pending: &mut HashSet<RouteChannel>,
3126 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3127 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3128 retry_buffer: &mut RetryBuffer,
3129 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3130 shutdown: &Arc<Notify>,
3131 control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3132 metrics: &Arc<DispatchPathMetrics>,
3133 push_senders: &PushSenders,
3134 dispatch: DispatchFn,
3135 user_config_path: Option<&Path>,
3136) -> Result<(), SubcError> {
3137 let request =
3138 serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3139 match request {
3140 ModuleControlRequest::RouteBind {
3141 route_channel,
3142 epoch,
3143 target: _,
3144 identity,
3145 principal,
3146 consumer_capabilities,
3147 } => {
3148 let route_id = route_key(route_channel, epoch);
3149 if epoch == 0 {
3150 return send_route_bind_error(
3151 tx,
3152 frame,
3153 "config_divergence",
3154 "route bind uses an invalid channel generation",
3155 metrics,
3156 )
3157 .await;
3158 }
3159 let mut bind_root_id = None;
3160 if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3161 if installed_epoch >= epoch {
3162 return send_route_bind_error(
3163 tx,
3164 frame,
3165 "config_divergence",
3166 "route bind generation is not newer than the installed generation",
3167 metrics,
3168 )
3169 .await;
3170 }
3171
3172 let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3173 Ok(root_id) => root_id,
3174 Err(error) => {
3175 return send_route_bind_error(
3176 tx,
3177 frame,
3178 "config_divergence",
3179 &format!("invalid route project root: {error}"),
3180 metrics,
3181 )
3182 .await;
3183 }
3184 };
3185 teardown_installed_route(
3186 tx,
3187 metrics,
3188 executor,
3189 route_key(route_channel, installed_epoch),
3190 "higher-epoch RouteBind",
3191 Some(&replacement_root),
3192 installed_route_epochs,
3193 routes,
3194 root_channels,
3195 bg_subs,
3196 bg_sub_by_session,
3197 bg_wake_pending,
3198 pending_bash_asks,
3199 live_roots,
3200 route_bash_cancels,
3201 pending_binds,
3202 retry_buffer,
3203 push_buffer,
3204 shutdown,
3205 )
3206 .await?;
3207 bind_root_id = Some(replacement_root);
3208 }
3209 if pending_binds.contains_key(&route_id) {
3210 return send_route_bind_error(
3211 tx,
3212 frame,
3213 "config_divergence",
3214 "route bind is already pending for channel",
3215 metrics,
3216 )
3217 .await;
3218 }
3219 let bind_root_id = match bind_root_id {
3220 Some(root_id) => root_id,
3221 None => match ProjectRootId::from_path(&identity.project_root) {
3222 Ok(root_id) => root_id,
3223 Err(error) => {
3224 return send_route_bind_error(
3225 tx,
3226 frame,
3227 "config_divergence",
3228 &format!("invalid route project root: {error}"),
3229 metrics,
3230 )
3231 .await;
3232 }
3233 },
3234 };
3235
3236 let request_id = format!("subc-bind-{route_channel}");
3239 let bind_project_root = identity.project_root.clone();
3240 let bind_harness = identity.harness.clone();
3241 let bind_session = identity.session.clone();
3242 let bind_trust = trust_for_bind(&bind_harness, &principal);
3243 let consumer_elicitation_capable = consumer_capabilities
3248 .as_ref()
3249 .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
3250 log::info!(
3251 "subc attach: route {} harness={} principal={} trust={} elicitation={}",
3252 route_channel,
3253 bind_harness,
3254 principal_label(&principal),
3255 bind_trust.label(),
3256 consumer_elicitation_capable
3257 );
3258
3259 let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
3271 user_config_path,
3272 Path::new(&bind_project_root),
3273 );
3274 let config_tiers: Vec<Value> = local_tiers
3275 .iter()
3276 .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
3277 .collect();
3278 let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
3279 let configure_json = json!({
3280 "id": request_id,
3281 "command": "configure",
3282 "project_root": bind_project_root,
3283 "harness": bind_harness,
3284 "session_id": bind_session.clone(),
3285 "config": config_tiers,
3286 });
3287 let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
3288 Ok(req) => req,
3289 Err(error) => {
3290 return send_route_bind_error(
3291 tx,
3292 frame,
3293 "config_divergence",
3294 &format!("failed to build configure request: {error}"),
3295 metrics,
3296 )
3297 .await;
3298 }
3299 };
3300
3301 let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
3302 root: bind_root_id.clone(),
3303 project_root: PathBuf::from(&bind_project_root),
3304 harness: bind_harness.clone(),
3305 session: bind_session.clone(),
3306 trust: bind_trust,
3307 consumer_elicitation_capable,
3308 }));
3309 let configure_session = route_identity.session.clone();
3310 let root_was_live = live_roots.contains_key(&bind_root_id);
3311 let inserted_new_actor = register_actor_for_bind(
3312 shared_app,
3313 executor,
3314 push_senders,
3315 &bind_root_id,
3316 route_channel,
3317 root_was_live,
3318 );
3319
3320 let configure_request_id = configure_req.id.clone();
3321 installed_route_epochs.insert(route_channel, epoch);
3322 if let Some(meta) = live_roots.get_mut(&bind_root_id) {
3323 meta.maintenance_queued_kinds.clear();
3324 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
3325 }
3326 let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
3327 bind_root_id.clone(),
3328 Lane::Mutating,
3329 configure_request_id.clone(),
3330 Box::new(move |ctx| {
3331 log_ctx::with_session(Some(configure_session.clone()), || {
3332 dispatch(configure_req, ctx)
3333 })
3334 }),
3335 );
3336 pending_binds.insert(
3337 route_id,
3338 PendingBind {
3339 bind_root_id: bind_root_id.clone(),
3340 inserted_new_actor,
3341 cancelled: false,
3342 configure_request_id: configure_request_id.clone(),
3343 started_at: Instant::now(),
3344 warned_half_deadline: false,
3345 deadline_reported: false,
3346 corr: frame.header.corr,
3347 ver: frame.header.ver,
3348 flags: frame.header.flags,
3349 cancellation: configure_cancellation,
3350 },
3351 );
3352
3353 let completion_tx = control_completion_tx.clone();
3354 let completion_identity = route_identity;
3355 let completion_root = bind_root_id.clone();
3356 let completion_route_channel = route_channel;
3357 let completion_ver = frame.header.ver;
3358 let completion_corr = frame.header.corr;
3359 let completion_flags = frame.header.flags;
3360 let completion_metrics = Arc::clone(metrics);
3361 tokio::spawn(async move {
3362 let _response_task = ResponseTaskGuard::new(&completion_metrics);
3363 let configure_response =
3364 await_executor_response(configure_rx, configure_request_id.clone()).await;
3365 let completion = RouteBindCompletion {
3370 route: route_key(completion_route_channel, epoch),
3371 identity: completion_identity,
3372 bind_root_id: completion_root,
3373 inserted_new_actor,
3374 configure_response,
3375 diagnostics_on_edit,
3376 ver: completion_ver,
3377 corr: completion_corr,
3378 flags: completion_flags,
3379 };
3380 if send_counted_channel(
3381 &completion_tx,
3382 &completion_metrics.control_completion_queued,
3383 completion,
3384 )
3385 .await
3386 .is_err()
3387 {
3388 log::debug!(
3389 "subc attach: dropped RouteBind completion for route {} after loop exit",
3390 completion_route_channel
3391 );
3392 }
3393 });
3394
3395 Ok(())
3396 }
3397 ModuleControlRequest::HealthCheck {} => {
3398 let report = build_health_report(executor, pending_binds, metrics);
3399 let body = serde_json::to_vec(&ModuleControlResponse::from(report))
3400 .map_err(SubcError::Json)?;
3401 let response = Frame::build_with_version(
3402 frame.header.ver,
3403 FrameType::Response,
3404 frame.header.flags,
3405 0,
3406 0,
3407 frame.header.corr,
3408 body,
3409 )
3410 .map_err(SubcError::FrameBuild)?;
3411 send_frame(tx, metrics, response).await
3412 }
3413 }
3414}
3415
3416fn install_bash_compressor(ctx: &AppContext) {
3417 let filter_registry_handle = ctx.shared_filter_registry();
3419 let compress_flag = ctx.bash_compress_flag();
3420 ctx.bash_background().set_compressor_with_exit_code(
3421 move |command: &str, output: String, exit_code: Option<i32>| {
3422 if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
3423 return crate::compress::CompressionResult::new(output);
3424 }
3425 let registry_guard = match filter_registry_handle.read() {
3426 Ok(g) => g,
3427 Err(poisoned) => poisoned.into_inner(),
3428 };
3429 crate::compress::compress_with_registry_exit_code(
3430 command,
3431 &output,
3432 exit_code,
3433 ®istry_guard,
3434 )
3435 },
3436 );
3437}
3438
3439fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
3440 let mut diagnostics_on_edit = false;
3441 for tier in tiers {
3442 if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
3443 diagnostics_on_edit = value;
3444 }
3445 }
3446 diagnostics_on_edit
3447}
3448
3449fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
3450 let stripped = strip_jsonc(doc);
3451 let value = serde_json::from_str::<Value>(&stripped).ok()?;
3452 value
3453 .get("lsp")
3454 .and_then(Value::as_object)?
3455 .get("diagnostics_on_edit")
3456 .and_then(Value::as_bool)
3457}
3458
3459async fn send_route_bind_error(
3460 tx: &WriterSender,
3461 frame: &Frame,
3462 code: &str,
3463 message: &str,
3464 metrics: &DispatchPathMetrics,
3465) -> Result<(), SubcError> {
3466 send_route_bind_error_parts(
3467 tx,
3468 frame.header.ver,
3469 frame.header.corr,
3470 frame.header.flags,
3471 code,
3472 message,
3473 metrics,
3474 )
3475 .await
3476}
3477
3478async fn send_route_bind_error_parts(
3479 tx: &WriterSender,
3480 ver: u8,
3481 corr: u64,
3482 flags: Flags,
3483 code: &str,
3484 message: &str,
3485 metrics: &DispatchPathMetrics,
3486) -> Result<(), SubcError> {
3487 let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
3488 send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
3489 log::warn!("subc attach: route bind rejected ({code}): {message}");
3490 Ok(())
3491}
3492
3493async fn handle_tool_call(
3498 tx: &WriterSender,
3499 frame: &Frame,
3500 mut phase_trace: PhaseTrace,
3501 routes: &HashMap<RouteChannel, RouteIdentity>,
3502 pending_binds: &HashMap<RouteChannel, PendingBind>,
3503 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3504 executor: &Arc<Executor>,
3505 shutdown: &Arc<Notify>,
3506 connection_cancel: &PersistentCancelSignal,
3507 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
3508 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
3509 metrics: &Arc<DispatchPathMetrics>,
3510 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3511 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3512 next_bash_ask_corr: &mut u64,
3513 bg_subs: &mut HashMap<RouteChannel, BgSub>,
3514 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3515 bg_wake_pending: &mut HashSet<RouteChannel>,
3516 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
3517 dispatch: DispatchFn,
3518 allow_native_passthrough: bool,
3519) -> Result<(), SubcError> {
3520 let route_id = route_key(frame.header.channel, frame.header.epoch);
3521 if pending_binds.contains_key(&route_id) {
3522 let error = build_error_frame(
3523 frame.header.ver,
3524 frame.header.channel,
3525 frame.header.epoch,
3526 frame.header.corr,
3527 frame.header.flags,
3528 "route_not_bound",
3529 "route is not bound before tool call",
3530 )?;
3531 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3532 }
3533
3534 let Some(identity) = routes.get(&route_id).cloned() else {
3535 let error = build_error_frame(
3536 frame.header.ver,
3537 frame.header.channel,
3538 frame.header.epoch,
3539 frame.header.corr,
3540 frame.header.flags,
3541 "route_not_bound",
3542 "route is not bound before tool call",
3543 )?;
3544 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3545 };
3546 let restore_watcher = live_roots
3547 .get(&identity.root)
3548 .is_some_and(|meta| meta.idle_artifacts_evicted);
3549 if let Some(meta) = live_roots.get_mut(&identity.root) {
3550 meta.reactivate_bound();
3551 }
3552 if restore_watcher {
3553 if let Some(ctx) = executor.actor_context(&identity.root) {
3554 crate::commands::configure::ensure_project_watcher(&ctx);
3555 }
3556 }
3557
3558 let route_request =
3559 serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
3560 if matches!(
3561 route_request,
3562 RouteRequest::BgEvents(BgEventsRequest {
3563 op: BgEventsOp::BgEvents
3564 })
3565 ) {
3566 if let Some(old_sub) = bg_subs.get(&route_id).copied() {
3567 push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
3568 }
3569 if !identity.trust.allows_bash_observation() {
3570 bg_subs.remove(&route_id);
3571 bg_wake_pending.remove(&route_id);
3572 remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
3573 let denied_sub = BgSub {
3574 corr: frame.header.corr,
3575 ver: frame.header.ver,
3576 flags: frame.header.flags,
3577 };
3578 push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
3579 return Ok(());
3580 }
3581 bg_subs.insert(
3582 route_id,
3583 BgSub {
3584 corr: frame.header.corr,
3585 ver: frame.header.ver,
3586 flags: frame.header.flags,
3587 },
3588 );
3589 bg_sub_by_session.insert((identity.root.clone(), identity.session.clone()), route_id);
3590 push::arm_bg_wake(
3591 identity.root.clone(),
3592 identity.session.clone(),
3593 route_id,
3594 bg_wake_pending,
3595 bg_wake_epoch,
3596 );
3597 return Ok(());
3598 }
3599
3600 let RouteRequest::ToolCall(call) = route_request else {
3601 unreachable!("background event subscription returned above")
3602 };
3603 let bare_name = call.name;
3604 let arguments = strip_agent_preview_arg_owned(call.arguments);
3605 let format_context = crate::subc_format::FormatContext::from_tool_call(
3606 &bare_name,
3607 &arguments,
3608 identity.project_root.as_path(),
3609 );
3610
3611 let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
3612 let bind_trust = identity.trust;
3613 let diagnostics_on_edit = live_roots
3614 .get(&identity.root)
3615 .map(|meta| meta.diagnostics_on_edit)
3616 .unwrap_or(false);
3617
3618 if matches!(bind_trust, BindTrust::Untrusted)
3619 && is_bash_family_tool(&bare_name)
3620 && (bare_name != "bash" || !identity.consumer_elicitation_capable)
3621 {
3622 let response = bash::bash_denied_untrusted_response(request_id.clone());
3623 let text = crate::subc_format::format_response_with_context(
3624 &bare_name,
3625 &response,
3626 &format_context,
3627 );
3628 let result = ToolCallResult { text, response };
3629 let response_frame = build_tool_response_frame(
3630 frame.header.ver,
3631 route_id,
3632 frame.header.corr,
3633 frame.header.flags,
3634 &result,
3635 bind_trust,
3636 )?;
3637 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3638 }
3639
3640 if !is_subc_agent_core_tool(&bare_name)
3648 && !is_subc_native_plumbing_tool(&bare_name)
3649 && !allow_native_passthrough
3650 {
3651 log::warn!(
3652 "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
3653 bare_name,
3654 frame.header.channel
3655 );
3656 let response = Response::error(
3657 request_id.clone(),
3658 "unknown_tool",
3659 format!("tool {:?} is not in the AFT tool manifest", bare_name),
3660 );
3661 let text = crate::subc_format::format_response_with_context(
3662 &bare_name,
3663 &response,
3664 &format_context,
3665 );
3666 let result = ToolCallResult { text, response };
3667 let response_frame = build_tool_response_frame(
3668 frame.header.ver,
3669 route_id,
3670 frame.header.corr,
3671 frame.header.flags,
3672 &result,
3673 bind_trust,
3674 )?;
3675 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3676 }
3677
3678 if bare_name == "bash" {
3679 if matches!(bind_trust, BindTrust::Untrusted) {
3680 let plan = match bash::prepare_bash_elicitation_plan(
3681 &arguments,
3682 identity.project_root.as_path(),
3683 ) {
3684 Ok(plan) => plan,
3685 Err(error) => {
3686 let response = Response::error(request_id.clone(), error.code, error.message);
3687 let text = crate::subc_format::format_response_with_context(
3688 &bare_name,
3689 &response,
3690 &format_context,
3691 );
3692 let result = ToolCallResult { text, response };
3693 let response_frame = build_tool_response_frame(
3694 frame.header.ver,
3695 route_id,
3696 frame.header.corr,
3697 frame.header.flags,
3698 &result,
3699 bind_trust,
3700 )?;
3701 return send_reliable_writer_frame(
3702 tx,
3703 metrics,
3704 response_frame,
3705 "tool response",
3706 )
3707 .await;
3708 }
3709 };
3710
3711 let meta = live_roots
3712 .entry(identity.root.clone())
3713 .or_insert_with(|| RootMeta::new(Instant::now()));
3714 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
3715 meta.reactivate_bound();
3716
3717 let route_cancel =
3718 route_bash_cancels
3719 .entry(route_id)
3720 .or_insert_with(|| bash::RouteBashCancel {
3721 token: PersistentCancelSignal::new(),
3722 active_waits: 0,
3723 });
3724 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
3725 let cancel = bash::BashWaitCancel {
3726 connection: connection_cancel.clone(),
3727 route: route_cancel.token.clone(),
3728 };
3729
3730 let reverse_corr =
3731 allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
3732 let ask_frame = build_bash_elicitation_request_frame(
3733 frame.header.ver,
3734 route_id,
3735 reverse_corr,
3736 frame.header.flags,
3737 &plan.command,
3738 &plan.asks,
3739 )?;
3740 pending_bash_asks.insert(
3741 ReverseCorrKey {
3742 route: route_id,
3743 corr: reverse_corr,
3744 },
3745 PendingBashAsk {
3746 route: route_id,
3747 tool_corr: frame.header.corr,
3748 tool_flags: frame.header.flags,
3749 tool_ver: frame.header.ver,
3750 root: identity.root.clone(),
3751 project_root: identity.project_root.clone(),
3752 session_id: identity.session.clone(),
3753 request_id,
3754 arguments,
3755 format_context,
3756 cancel,
3757 grants: plan.grants,
3758 expires_at: Instant::now() + bash_elicitation_timeout(),
3759 },
3760 );
3761 return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
3762 .await;
3763 }
3764
3765 let meta = live_roots
3766 .entry(identity.root.clone())
3767 .or_insert_with(|| RootMeta::new(Instant::now()));
3768 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
3769 meta.reactivate_bound();
3770
3771 let route_cancel =
3772 route_bash_cancels
3773 .entry(route_id)
3774 .or_insert_with(|| bash::RouteBashCancel {
3775 token: PersistentCancelSignal::new(),
3776 active_waits: 0,
3777 });
3778 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
3779 let cancel = bash::BashWaitCancel {
3780 connection: connection_cancel.clone(),
3781 route: route_cancel.token.clone(),
3782 };
3783
3784 bash::submit_deferred_bash(
3785 executor,
3786 bash_deferred_tx,
3787 bash_poll_touch_tx,
3788 metrics,
3789 dispatch,
3790 identity.root.clone(),
3791 identity.project_root.clone(),
3792 identity.session.clone(),
3793 request_id,
3794 route_id,
3795 frame.header.corr,
3796 frame.header.flags,
3797 frame.header.ver,
3798 arguments,
3799 format_context,
3800 cancel,
3801 bind_trust,
3802 None,
3803 );
3804 return Ok(());
3805 }
3806
3807 let lane = command_lane(&bare_name);
3808 let tool_call_context = ToolCallContext {
3809 project_root: identity.project_root.clone(),
3810 session_id: Some(identity.session.clone()),
3811 request_id: request_id.clone(),
3812 diagnostics_on_edit,
3813 preview: call.preview,
3814 };
3815 let bare_name_for_frame = bare_name.clone();
3816 let identity_for_run = identity.clone();
3817 let completion_session = identity.session.clone();
3818 let completion_root = identity.project_root.clone();
3819 let request_id_for_force = request_id.clone();
3820 let format_context_for_frame = format_context.clone();
3821 let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
3822 phase_trace.mark_executor_submitted();
3823 let rx = executor.submit_async(
3824 identity.root.clone(),
3825 lane,
3826 request_id.clone(),
3827 Box::new(move |ctx| {
3828 phase_trace.mark_job_admitted();
3829 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
3830 let run = || {
3831 let finalizer = |response: &mut Response| {
3832 crate::response_finalize::finalize_response_with_bg_completions(
3833 response,
3834 ctx,
3835 &identity_for_run.session,
3836 &bare_name,
3837 bind_trust.allows_bash_observation(),
3838 );
3839 };
3840 match run_tool_call(
3841 &bare_name,
3842 &arguments,
3843 &format_context,
3844 &tool_call_context,
3845 ctx,
3846 &dispatch,
3847 Some(&finalizer),
3848 Some(&mut phase_trace),
3849 ) {
3850 ToolCallOutcome::Unary(result) => {
3851 let response = result.response;
3852 let _ = tool_call_tx.send(ToolCallCompletion {
3853 text: result.text,
3854 phase_trace,
3855 });
3856 response
3857 }
3858 }
3859 };
3860 if matches!(bind_trust, BindTrust::Untrusted) {
3861 ctx.with_force_restrict(&request_id_for_force, run)
3862 } else {
3863 run()
3864 }
3865 })
3866 }),
3867 );
3868 let completion_tx = tx.clone();
3869 let completion_shutdown = Arc::clone(shutdown);
3870 let route = route_id;
3871 let corr = frame.header.corr;
3872 let flags = frame.header.flags;
3873 let ver = frame.header.ver;
3874 let completion_metrics = Arc::clone(metrics);
3875 tokio::spawn(async move {
3876 let _response_task = ResponseTaskGuard::new(&completion_metrics);
3877 let response = await_executor_response(rx, request_id.clone()).await;
3878 let (text, phase_trace) = match tool_call_rx.await {
3879 Ok(completion) => (completion.text, Some(completion.phase_trace)),
3880 Err(_) => (
3881 crate::subc_format::format_response_with_context(
3882 &bare_name_for_frame,
3883 &response,
3884 &format_context_for_frame,
3885 ),
3886 None,
3887 ),
3888 };
3889 let result = ToolCallResult { text, response };
3890 let fatal = response_is_fatal_panic(&result.response);
3891 match build_tool_response_frame(ver, route, corr, flags, &result, bind_trust) {
3892 Ok(response_frame) => {
3893 let send_result = if let Some(phase_trace) = phase_trace {
3894 let trace = ToolResponseWriteTrace::new(
3895 phase_trace,
3896 bare_name_for_frame,
3897 completion_root,
3898 completion_session,
3899 route.channel,
3900 corr,
3901 );
3902 send_traced_tool_response_frame(
3903 &completion_tx,
3904 &completion_metrics,
3905 response_frame,
3906 trace,
3907 )
3908 .await
3909 } else {
3910 send_reliable_writer_frame(
3911 &completion_tx,
3912 &completion_metrics,
3913 response_frame,
3914 "tool response",
3915 )
3916 .await
3917 };
3918 if let Err(error) = send_result {
3919 log::warn!("subc attach: failed to queue tool response frame: {error}");
3920 }
3921 }
3922 Err(error) => {
3923 log::error!("subc attach: failed to build tool response frame: {error}");
3924 }
3925 }
3926 if fatal {
3927 signal_fatal_teardown(
3928 &completion_tx,
3929 Some(route),
3930 ver,
3931 corr,
3932 &completion_shutdown,
3933 &completion_metrics,
3934 )
3935 .await;
3936 }
3937 });
3938 Ok(())
3939}
3940
3941fn submit_maintenance_job(
3942 executor: &Arc<Executor>,
3943 root_id: ProjectRootId,
3944 kind: MaintenanceDrainKind,
3945 bg_sessions_to_check: Vec<(String, u64)>,
3946 completion_tx: &mpsc::Sender<MaintenanceCompletion>,
3947 metrics: &Arc<DispatchPathMetrics>,
3948) {
3949 let request_id = format!(
3950 "subc-maintenance-drain-{}-{}",
3951 kind.label(),
3952 root_id.as_path().to_string_lossy()
3953 );
3954 let response_id = request_id.clone();
3955 let completion_root_id = root_id.clone();
3956 let maintenance_generation = executor
3957 .actor_context(&root_id)
3958 .map(|ctx| ctx.configure_generation())
3959 .unwrap_or(0);
3960 let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
3961 let lane = match kind {
3966 MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
3967 MaintenanceDrainKind::Watcher
3968 | MaintenanceDrainKind::Lsp
3969 | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
3970 };
3971 let rx = executor.submit_maintenance_async(
3972 root_id,
3973 lane,
3974 request_id.clone(),
3975 Box::new(move |ctx| {
3976 let outcome = match kind {
3977 MaintenanceDrainKind::Watcher => {
3978 let drained = runtime_drain::drain_watcher_events_bounded(
3979 ctx,
3980 runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
3981 );
3982 MaintenanceJobOutcome {
3983 empty_bg_sessions: Vec::new(),
3984 requeue_kind: drained.has_more.then_some(kind),
3985 }
3986 }
3987 MaintenanceDrainKind::Lsp => {
3988 let drained = runtime_drain::drain_lsp_events_bounded(
3989 ctx,
3990 runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
3991 );
3992 MaintenanceJobOutcome {
3993 empty_bg_sessions: Vec::new(),
3994 requeue_kind: drained.has_more.then_some(kind),
3995 }
3996 }
3997 MaintenanceDrainKind::ConfigureTail => {
3998 runtime_drain::drain_deferred_configure_maintenance(ctx);
3999 runtime_drain::drain_configure_warning_events(ctx);
4000 MaintenanceJobOutcome::default()
4001 }
4002 MaintenanceDrainKind::CompletionDrains => {
4003 runtime_drain::drain_search_index_events(ctx);
4004 runtime_drain::drain_callgraph_store_events(ctx);
4005 runtime_drain::drain_semantic_index_events(ctx);
4006 runtime_drain::drain_semantic_refresh_events(ctx);
4007 runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
4008 let empty_bg_sessions = bg_sessions_to_check
4009 .into_iter()
4010 .filter(|(session, _)| {
4011 !ctx.bash_background()
4012 .has_completions_for_session(Some(session.as_str()))
4013 })
4014 .collect();
4015 MaintenanceJobOutcome {
4016 empty_bg_sessions,
4017 requeue_kind: None,
4018 }
4019 }
4020 };
4021 let requeued = outcome.requeue_kind.is_some();
4022 let _ = outcome_tx.send(outcome);
4023 Response::success(
4024 response_id,
4025 json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
4026 )
4027 }),
4028 );
4029 let completion_tx = completion_tx.clone();
4030 let completion_metrics = Arc::clone(metrics);
4031 tokio::spawn(async move {
4032 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4033 let response = await_executor_response(rx, request_id).await;
4034 let outcome = outcome_rx.await.unwrap_or_default();
4035 let _ = send_counted_channel(
4036 &completion_tx,
4037 &completion_metrics.maintenance_queued,
4038 MaintenanceCompletion {
4039 root_id: completion_root_id,
4040 kind,
4041 response,
4042 empty_bg_sessions: outcome.empty_bg_sessions,
4043 requeue_kind: outcome.requeue_kind,
4044 },
4045 )
4046 .await;
4047 });
4048}
4049
4050async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
4051 rx.await
4052 .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
4053}
4054async fn signal_fatal_teardown(
4055 tx: &WriterSender,
4056 route: Option<RouteChannel>,
4057 ver: u8,
4058 corr: u64,
4059 shutdown: &Arc<Notify>,
4060 metrics: &DispatchPathMetrics,
4061) {
4062 if let Some(route) = route {
4063 if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
4064 if let Err(error) = send_frame(tx, metrics, frame).await {
4065 log::warn!(
4066 "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
4067 );
4068 }
4069 }
4070 }
4071 if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
4072 if let Err(error) = send_frame(tx, metrics, frame).await {
4073 log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
4074 }
4075 }
4076 shutdown.notify_one();
4077}
4078#[derive(Debug, Deserialize)]
4079#[serde(untagged)]
4080enum RouteRequest {
4081 BgEvents(BgEventsRequest),
4082 ToolCall(ToolCallRequest),
4083}
4084
4085#[derive(Debug, Deserialize)]
4086struct BgEventsRequest {
4087 op: BgEventsOp,
4088}
4089
4090#[derive(Debug, Deserialize)]
4091#[serde(rename_all = "snake_case")]
4092enum BgEventsOp {
4093 BgEvents,
4094}
4095
4096#[derive(Debug, Deserialize)]
4097struct ToolCallRequest {
4098 name: String,
4099 #[serde(default)]
4100 arguments: Value,
4101 #[serde(default)]
4106 preview: bool,
4107}
4108
4109#[cfg(test)]
4110pub(crate) mod test_support {
4111 use super::*;
4112 use crate::bash_background::BgTaskStatus;
4113 use crate::protocol::{
4114 BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
4115 ProgressFrame, StatusChangedFrame,
4116 };
4117 use serde_json::json;
4118
4119 pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
4120 let dir = tempfile::Builder::new()
4121 .prefix(name)
4122 .tempdir()
4123 .expect("temp root");
4124 let root = ProjectRootId::from_path(dir.path()).expect("project root id");
4125 (dir, root)
4126 }
4127
4128 pub(super) fn test_ctx() -> Arc<AppContext> {
4129 Arc::new(AppContext::new(
4130 Box::new(crate::parser::TreeSitterProvider::new()),
4131 crate::config::Config::default(),
4132 ))
4133 }
4134
4135 pub(super) fn status_frame(seq: u64) -> PushFrame {
4136 status_frame_with_session(seq, None)
4137 }
4138
4139 pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
4140 PushFrame::StatusChanged(StatusChangedFrame {
4141 frame_type: "status_changed",
4142 session_id: session_id.map(str::to_string),
4143 snapshot: json!({ "seq": seq }),
4144 })
4145 }
4146
4147 pub(super) fn completion_frame(task_id: &str) -> PushFrame {
4148 completion_frame_with_session(task_id, "session-1")
4149 }
4150
4151 pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
4152 PushFrame::BashCompleted(BashCompletedFrame {
4153 frame_type: "bash_completed",
4154 task_id: task_id.to_string(),
4155 session_id: session_id.to_string(),
4156 status: BgTaskStatus::Completed,
4157 exit_code: Some(0),
4158 command: format!("echo {task_id}"),
4159 output_preview: String::new(),
4160 output_truncated: false,
4161 original_tokens: None,
4162 compressed_tokens: None,
4163 tokens_skipped: false,
4164 })
4165 }
4166
4167 pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
4168 long_running_frame_with_session(task_id, "session-1", elapsed_ms)
4169 }
4170
4171 pub(super) fn long_running_frame_with_session(
4172 task_id: &str,
4173 session_id: &str,
4174 elapsed_ms: u64,
4175 ) -> PushFrame {
4176 PushFrame::BashLongRunning(BashLongRunningFrame {
4177 frame_type: "bash_long_running",
4178 task_id: task_id.to_string(),
4179 session_id: session_id.to_string(),
4180 command: format!("sleep {elapsed_ms}"),
4181 elapsed_ms,
4182 })
4183 }
4184
4185 pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
4186 PushFrame::BashPatternMatch(BashPatternMatchFrame {
4187 frame_type: "bash_pattern_match",
4188 task_id: "task-pattern".to_string(),
4189 session_id: session_id.to_string(),
4190 watch_id: "watch-1".to_string(),
4191 match_text: "needle".to_string(),
4192 match_offset: 7,
4193 context: "haystack needle".to_string(),
4194 once: true,
4195 reason: "pattern_match",
4196 })
4197 }
4198
4199 pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
4200 PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
4201 frame_type: "configure_warnings",
4202 session_id: session_id.map(str::to_string),
4203 project_root: "/tmp/subc-test".to_string(),
4204 warnings: Vec::new(),
4205 })
4206 }
4207
4208 pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
4209 route_identity_with_trust(root, session_id, BindTrust::FirstParty)
4210 }
4211
4212 pub(super) fn route_identity_with_trust(
4213 root: &ProjectRootId,
4214 session_id: &str,
4215 trust: BindTrust,
4216 ) -> RouteIdentity {
4217 RouteIdentity(Arc::new(RouteIdentityData {
4218 root: root.clone(),
4219 project_root: root.as_path().to_path_buf(),
4220 harness: "opencode".to_string(),
4221 session: session_id.to_string(),
4222 trust,
4223 consumer_elicitation_capable: false,
4224 }))
4225 }
4226
4227 pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
4228 PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
4229 }
4230
4231 pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
4232 match frame {
4233 PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
4234 _ => None,
4235 }
4236 }
4237
4238 pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
4239 match frame {
4240 PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
4241 _ => None,
4242 }
4243 }
4244
4245 pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
4246 let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
4247 body.get("task_id")
4248 .and_then(serde_json::Value::as_str)
4249 .map(str::to_string)
4250 }
4251}
4252
4253#[cfg(test)]
4254mod tests {
4255 use super::test_support::{completion_frame, test_ctx, test_root};
4256 use super::*;
4257
4258 fn attach_error(kind: io::ErrorKind) -> SubcError {
4259 SubcError::Connect {
4260 endpoint: "127.0.0.1:1".to_string(),
4261 source: io::Error::new(kind, "constructed attach failure"),
4262 }
4263 }
4264
4265 fn auth_io_error(kind: io::ErrorKind) -> SubcError {
4266 SubcError::Auth {
4267 endpoint: "127.0.0.1:1".to_string(),
4268 source: subc_transport::AuthError::Io {
4269 stage: subc_transport::AuthStage::ServerProof,
4270 source: io::Error::new(kind, "constructed auth failure"),
4271 },
4272 }
4273 }
4274
4275 #[test]
4276 fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
4277 let transient_errors = vec![
4278 attach_error(io::ErrorKind::ConnectionRefused),
4279 attach_error(io::ErrorKind::TimedOut),
4280 attach_error(io::ErrorKind::ConnectionReset),
4281 auth_io_error(io::ErrorKind::ConnectionAborted),
4282 auth_io_error(io::ErrorKind::BrokenPipe),
4283 SubcError::Auth {
4284 endpoint: "127.0.0.1:1".to_string(),
4285 source: subc_transport::AuthError::UnexpectedEof {
4286 stage: subc_transport::AuthStage::ServerProof,
4287 expected: 4,
4288 actual: 0,
4289 },
4290 },
4291 SubcError::Auth {
4292 endpoint: "127.0.0.1:1".to_string(),
4293 source: subc_transport::AuthError::Timeout {
4294 stage: subc_transport::AuthStage::ServerProof,
4295 deadline: AUTH_DEADLINE,
4296 },
4297 },
4298 ];
4299 for error in &transient_errors {
4300 assert_eq!(
4301 classify_attach_error(error),
4302 AttachErrorClass::Transient,
4303 "expected transient: {error}"
4304 );
4305 }
4306
4307 let permanent_errors = vec![
4308 attach_error(io::ErrorKind::PermissionDenied),
4309 auth_io_error(io::ErrorKind::InvalidData),
4310 SubcError::Auth {
4311 endpoint: "127.0.0.1:1".to_string(),
4312 source: subc_transport::AuthError::InvalidServerProof,
4313 },
4314 SubcError::Auth {
4315 endpoint: "127.0.0.1:1".to_string(),
4316 source: subc_transport::AuthError::DaemonIdMismatch,
4317 },
4318 SubcError::ConnectionFile {
4319 path: PathBuf::from("subc-connection.json"),
4320 source: subc_transport::ConnectionFileError::Invalid {
4321 reason: "constructed invalid file".to_string(),
4322 },
4323 },
4324 SubcError::NoEndpoint {
4325 path: PathBuf::from("subc-connection.json"),
4326 },
4327 SubcError::InvalidEndpoint {
4328 path: PathBuf::from("subc-connection.json"),
4329 endpoint: "not-an-ip:1234".to_string(),
4330 },
4331 ];
4332 for error in &permanent_errors {
4333 assert_eq!(
4334 classify_attach_error(error),
4335 AttachErrorClass::Permanent,
4336 "expected permanent: {error}"
4337 );
4338 }
4339 }
4340
4341 #[test]
4342 fn incompatible_wire_version_is_rejected_before_tcp_connect() {
4343 let conn_dir = tempfile::tempdir().expect("connection tempdir");
4344 let conn_path = conn_dir.path().join("subc-connection.json");
4345 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
4346 listener
4347 .set_nonblocking(true)
4348 .expect("set listener nonblocking");
4349 let port = listener.local_addr().expect("listener addr").port();
4350 connection_file::write_atomic(
4351 &conn_path,
4352 &connection_file::ConnectionInfo {
4353 schema: connection_file::SCHEMA_VERSION,
4354 wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
4355 endpoints: vec![connection_file::Endpoint {
4356 host: "127.0.0.1".to_string(),
4357 port,
4358 }],
4359 key: vec![0x42; subc_transport::KEY_LEN],
4360 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4361 pid: std::process::id(),
4362 daemon_ver: "subc-test".to_string(),
4363 },
4364 )
4365 .expect("write connection file");
4366
4367 let runtime = tokio::runtime::Builder::new_current_thread()
4368 .enable_all()
4369 .build()
4370 .expect("test runtime");
4371 let result = runtime.block_on(connect_and_authenticate_with_policy(
4372 &conn_path,
4373 AttachRetryPolicy {
4374 budget: Duration::from_secs(1),
4375 initial_backoff: Duration::from_millis(5),
4376 max_backoff: Duration::from_millis(10),
4377 jitter_percent: 0,
4378 },
4379 ));
4380 assert!(matches!(
4381 result,
4382 Err(SubcError::ConnectionFile {
4383 source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
4384 ..
4385 })
4386 ));
4387 assert!(matches!(
4388 listener.accept(),
4389 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
4390 ));
4391 }
4392
4393 #[test]
4394 fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
4395 let conn_dir = tempfile::tempdir().expect("connection tempdir");
4396 let conn_path = conn_dir.path().join("subc-connection.json");
4397 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
4398 let port = listener.local_addr().expect("reserved addr").port();
4399 drop(listener);
4400 connection_file::write_atomic(
4401 &conn_path,
4402 &connection_file::ConnectionInfo {
4403 schema: connection_file::SCHEMA_VERSION,
4404 wire_version: Some(PROTOCOL_VERSION),
4405 endpoints: vec![connection_file::Endpoint {
4406 host: "127.0.0.1".to_string(),
4407 port,
4408 }],
4409 key: vec![0x42; subc_transport::KEY_LEN],
4410 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4411 pid: std::process::id(),
4412 daemon_ver: "subc-test".to_string(),
4413 },
4414 )
4415 .expect("write connection file");
4416
4417 let policy = AttachRetryPolicy {
4418 budget: Duration::from_millis(40),
4419 initial_backoff: Duration::from_millis(5),
4420 max_backoff: Duration::from_millis(10),
4421 jitter_percent: 0,
4422 };
4423 let runtime = tokio::runtime::Builder::new_current_thread()
4424 .enable_all()
4425 .build()
4426 .expect("test runtime");
4427 let started_at = Instant::now();
4428 let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
4429 let elapsed = started_at.elapsed();
4430 let error = match result {
4431 Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
4432 Err(error) => error,
4433 };
4434
4435 assert!(matches!(error, SubcError::Connect { .. }), "{error}");
4436 assert!(
4437 elapsed >= Duration::from_millis(35),
4438 "retry budget ended too early: {elapsed:?}"
4439 );
4440 assert!(
4441 elapsed < Duration::from_secs(1),
4442 "retry budget was not bounded: {elapsed:?}"
4443 );
4444 }
4445
4446 fn due_maintenance_jobs_without_actor_context(
4447 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4448 budget: usize,
4449 pending_bind_roots: &HashSet<ProjectRootId>,
4450 ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
4451 due_maintenance_jobs(
4452 live_roots,
4453 None,
4454 &HashMap::new(),
4455 &HashSet::new(),
4456 budget,
4457 pending_bind_roots,
4458 )
4459 }
4460
4461 fn actor_ctx_with_dirty_search_index(
4462 root: &Path,
4463 storage: &Path,
4464 file_name: &str,
4465 old_contents: &str,
4466 new_contents: &str,
4467 ) -> (Arc<AppContext>, PathBuf, PathBuf) {
4468 let file = root.join(file_name);
4469 std::fs::write(&file, old_contents).expect("write source");
4470 let canonical_root = std::fs::canonicalize(root).expect("canonical root");
4471 let ctx = Arc::new(AppContext::new(
4472 Box::new(crate::parser::TreeSitterProvider::new()),
4473 Config {
4474 project_root: Some(root.to_path_buf()),
4475 storage_dir: Some(storage.to_path_buf()),
4476 ..Config::default()
4477 },
4478 ));
4479 ctx.set_canonical_cache_root(canonical_root.clone());
4480
4481 let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
4482 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
4483 let git_head = index.stored_git_head().map(str::to_owned);
4484 index.write_to_disk(&cache_dir, git_head.as_deref());
4485
4486 std::fs::write(&file, new_contents).expect("edit source");
4487 index.update_file(&file);
4488 *ctx.search_index()
4489 .write()
4490 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
4491 (ctx, canonical_root, cache_dir)
4492 }
4493
4494 #[test]
4495 fn graceful_shutdown_flushes_every_actor_search_index() {
4496 let storage = tempfile::tempdir().expect("storage tempdir");
4497 let (root1_dir, root1) = test_root("shutdown-flush-root-1");
4498 let (root2_dir, root2) = test_root("shutdown-flush-root-2");
4499 let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
4500 root1_dir.path(),
4501 storage.path(),
4502 "alpha.txt",
4503 "old actor one token\n",
4504 "new actor one token\n",
4505 );
4506 let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
4507 root2_dir.path(),
4508 storage.path(),
4509 "beta.txt",
4510 "old actor two token\n",
4511 "new actor two token\n",
4512 );
4513
4514 let executor = Executor::new();
4515 assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
4516 assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
4517
4518 flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
4519
4520 let mut restored1 =
4521 crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
4522 .expect("load flushed root one index");
4523 restored1.ready = true;
4524 assert_eq!(
4525 restored1
4526 .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
4527 .matches
4528 .len(),
4529 1,
4530 "graceful subc shutdown should flush the first root's trigram delta"
4531 );
4532
4533 let mut restored2 =
4534 crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
4535 .expect("load flushed root two index");
4536 restored2.ready = true;
4537 assert_eq!(
4538 restored2
4539 .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
4540 .matches
4541 .len(),
4542 1,
4543 "graceful subc shutdown should flush every registered root"
4544 );
4545 }
4546
4547 #[test]
4548 fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
4549 let _ = env_logger::builder().is_test(true).try_init();
4550 let (root_dir, root) = test_root("idle-root-reaper");
4551 let storage = tempfile::tempdir().expect("storage tempdir");
4552 std::fs::write(
4553 root_dir.path().join("main.rs"),
4554 "fn entry() { leaf(); }\nfn leaf() {}\n",
4555 )
4556 .expect("source file");
4557 let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
4558 let app = App::default_shared();
4559 let ctx = Arc::new(AppContext::from_app(
4560 Arc::clone(&app),
4561 Config {
4562 project_root: Some(canonical_root.clone()),
4563 storage_dir: Some(storage.path().to_path_buf()),
4564 callgraph_store: true,
4565 search_index: true,
4566 ..Config::default()
4567 },
4568 ));
4569 ctx.set_canonical_cache_root(canonical_root.clone());
4570 assert!(ctx
4571 .ensure_callgraph_store()
4572 .expect("build callgraph store")
4573 .is_some());
4574
4575 let cache_dir =
4576 crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
4577 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
4578 let git_head = index.stored_git_head().map(str::to_owned);
4579 index.write_to_disk(&cache_dir, git_head.as_deref());
4580 *ctx.search_index()
4581 .write()
4582 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
4583 let seeded_generation =
4586 crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
4587 .expect("seeded artifact generation");
4588 crate::cache_freshness::record_verify_completed(
4589 &canonical_root,
4590 crate::cache_freshness::VerifyArtifact::Search,
4591 Some(seeded_generation),
4592 );
4593 assert!(
4594 matches!(
4595 crate::cache_freshness::warm_verify_plan(
4596 canonical_root.as_path(),
4597 crate::cache_freshness::VerifyArtifact::Search,
4598 Some(seeded_generation),
4599 ),
4600 crate::cache_freshness::WarmVerifyPlan::Skip
4601 ),
4602 "memo must be warm before eviction for the downgrade assertion to bite"
4603 );
4604
4605 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
4606 let _dispatch_tx = dispatch_tx;
4607 let shutdown = Arc::new(AtomicBool::new(false));
4608 let thread_shutdown = Arc::clone(&shutdown);
4609 let join = std::thread::spawn(move || {
4610 while !thread_shutdown.load(Ordering::SeqCst) {
4611 std::thread::yield_now();
4612 }
4613 });
4614 ctx.install_watcher_runtime(
4615 dispatch_rx,
4616 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
4617 );
4618 assert_eq!(ctx.watcher_registry_count(), 1);
4619
4620 let executor = Arc::new(Executor::new());
4621 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4622 let mut live_roots = HashMap::new();
4623 let mut meta = RootMeta::new(Instant::now());
4624 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
4625 live_roots.insert(root.clone(), meta);
4626
4627 let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
4628 assert!(message.contains("evicted idle root"));
4629 assert!(message.contains("freed ~"));
4630 assert!(message.contains("semantic"));
4631 assert!(!message.contains("semantic not estimated retained"));
4632 assert!(message.contains("trigram"));
4633 assert!(message.contains("retained: bash"));
4634 assert!(message.contains("parser_pool"));
4635
4636 assert_eq!(
4637 reap_idle_roots(Instant::now(), &mut live_roots, &HashMap::new(), &executor),
4638 1
4639 );
4640 assert!(ctx.search_index().read().unwrap().is_none());
4641 assert_eq!(ctx.watcher_registry_count(), 0);
4642 assert!(
4647 matches!(
4648 crate::cache_freshness::warm_verify_plan(
4649 canonical_root.as_path(),
4650 crate::cache_freshness::VerifyArtifact::Search,
4651 Some(seeded_generation),
4652 ),
4653 crate::cache_freshness::WarmVerifyPlan::Strict
4654 ),
4655 "idle eviction must force strict re-verification"
4656 );
4657 assert!(
4658 crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
4659 );
4660 assert!(ctx
4661 .ensure_callgraph_store()
4662 .expect("reopen callgraph store")
4663 .is_some());
4664 assert!(live_roots[&root].idle_artifacts_evicted);
4665 }
4666
4667 #[test]
4668 fn idle_root_reaper_applies_ttl_to_unbound_roots() {
4669 let (_root_dir, root) = test_root("idle-root-ttl-gate");
4670 let ctx = test_ctx();
4671 let executor = Arc::new(Executor::new());
4672 assert!(executor.register_actor(root.clone(), ctx));
4673 let now = Instant::now();
4674 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(now))]);
4675
4676 assert_eq!(
4680 reap_idle_roots(now, &mut live_roots, &HashMap::new(), &executor),
4681 0
4682 );
4683 assert!(!live_roots[&root].idle_artifacts_evicted);
4684
4685 let ctx = executor.actor_context(&root).expect("actor context");
4691 ctx.mark_subc_unbound();
4692 ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
4693 assert_eq!(
4694 reap_idle_roots(
4695 now + IDLE_ROOT_TTL,
4696 &mut live_roots,
4697 &HashMap::new(),
4698 &executor,
4699 ),
4700 1
4701 );
4702 assert!(live_roots[&root].idle_artifacts_evicted);
4703 assert!(
4704 ctx.take_pending_search_index_paths().is_empty(),
4705 "TTL eviction must dispose retained pending reconciliation paths"
4706 );
4707 }
4708
4709 #[test]
4710 fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
4711 let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
4712 let ctx = test_ctx();
4713 let executor = Arc::new(Executor::new());
4714 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4715 ctx.mark_subc_unbound();
4716
4717 let pending = root.as_path().join("edited-while-unbound.rs");
4723 ctx.add_pending_search_index_paths([pending.clone()]);
4724 let dirty_source = root.as_path().join("dirty.rs");
4725 std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
4726 let mut dirty = crate::search_index::SearchIndex::new();
4727 dirty.ready = true;
4728 dirty.update_file(&dirty_source);
4729 *ctx.search_index()
4730 .write()
4731 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
4732 assert!(ctx.artifact_eviction_blocked());
4733
4734 let mut live_roots = HashMap::new();
4735 let mut meta = RootMeta::new(Instant::now());
4736 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
4737 live_roots.insert(root.clone(), meta);
4738
4739 assert_eq!(
4740 reap_idle_roots(Instant::now(), &mut live_roots, &HashMap::new(), &executor),
4741 0,
4742 "the dirty index must still block this eviction"
4743 );
4744 assert_eq!(
4745 ctx.take_pending_search_index_paths(),
4746 vec![pending],
4747 "a blocked eviction must restore the taken pending paths"
4748 );
4749 }
4750
4751 #[test]
4752 fn unbound_root_quiesces_maintenance_without_removing_actor() {
4753 let (_root_dir, root) = test_root("unbound-root-quiesce");
4754 let ctx = test_ctx();
4755 let executor = Arc::new(Executor::new());
4756 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4757 let mut meta = RootMeta::new(Instant::now());
4758 meta.maintenance_pending = true;
4759 meta.maintenance_jobs_in_flight = 1;
4760 meta.maintenance_queued_kinds
4761 .push_back(MaintenanceDrainKind::ConfigureTail);
4762 let mut live_roots = HashMap::from([(root.clone(), meta)]);
4763 *ctx.search_index()
4765 .write()
4766 .unwrap_or_else(std::sync::PoisonError::into_inner) =
4767 Some(crate::search_index::SearchIndex::new());
4768 ctx.set_cache_writer_capabilities(true, true);
4769 let pending = root.as_path().join("pending.rs");
4770 ctx.add_pending_search_index_paths([pending.clone()]);
4771 let canonical_root = root.as_path().to_path_buf();
4775 let artifact = canonical_root.join("cache.bin");
4776 std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
4777 let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
4778 crate::cache_freshness::record_verify_completed(
4779 &canonical_root,
4780 crate::cache_freshness::VerifyArtifact::Search,
4781 seeded_generation,
4782 );
4783 assert!(matches!(
4784 crate::cache_freshness::warm_verify_plan(
4785 &canonical_root,
4786 crate::cache_freshness::VerifyArtifact::Search,
4787 seeded_generation,
4788 ),
4789 crate::cache_freshness::WarmVerifyPlan::Skip
4790 ));
4791 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
4794 let _dispatch_tx = dispatch_tx;
4795 let shutdown = Arc::new(AtomicBool::new(false));
4796 let thread_shutdown = Arc::clone(&shutdown);
4797 let join = std::thread::spawn(move || {
4798 while !thread_shutdown.load(Ordering::SeqCst) {
4799 std::thread::yield_now();
4800 }
4801 });
4802 ctx.install_watcher_runtime(
4803 dispatch_rx,
4804 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
4805 );
4806 assert!(ctx.watcher_runtime_active());
4807
4808 quiesce_unbound_root(&root, &mut live_roots, &executor);
4809 let meta = &live_roots[&root];
4810 assert!(meta.unbound_quiesced);
4811 assert!(ctx.subc_unbound_quiesced());
4812 assert!(meta.maintenance_pending);
4813 assert!(meta.maintenance_queued_kinds.is_empty());
4814 assert!(executor.actor_registered(&root));
4815 assert!(
4819 ctx.search_index()
4820 .read()
4821 .unwrap_or_else(std::sync::PoisonError::into_inner)
4822 .is_some(),
4823 "quiesce must not evict resident artifacts"
4824 );
4825 assert_eq!(
4826 ctx.pending_callgraph_store_force_token(),
4827 None,
4828 "quiesce must not force a callgraph rebuild"
4829 );
4830 assert_eq!(
4831 ctx.take_pending_search_index_paths(),
4832 vec![pending],
4833 "quiesce must retain pending watcher-derived paths"
4834 );
4835 assert!(
4836 matches!(
4837 crate::cache_freshness::warm_verify_plan(
4838 &canonical_root,
4839 crate::cache_freshness::VerifyArtifact::Search,
4840 seeded_generation,
4841 ),
4842 crate::cache_freshness::WarmVerifyPlan::Skip
4843 ),
4844 "quiesce must not invalidate the warm verify memo"
4845 );
4846 assert!(
4847 ctx.watcher_runtime_active(),
4848 "quiesce must not stop a running watcher"
4849 );
4850 ctx.stop_watcher_runtime();
4851
4852 let meta = live_roots.get_mut(&root).expect("root metadata");
4853 note_maintenance_completion(
4854 meta,
4855 Some(MaintenanceDrainKind::ConfigureTail),
4856 false,
4857 meta.unbound_quiesced,
4858 );
4859 assert!(!meta.maintenance_pending);
4860 assert!(meta.maintenance_queued_kinds.is_empty());
4861 }
4862
4863 #[test]
4864 fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
4865 let (_dir, root) = test_root("same-root-replacement");
4866 let route = route_key(7, 1);
4867 let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
4868 let root_channels = HashMap::new();
4869
4870 assert!(!route_removal_will_quiesce_root(
4871 &root,
4872 route,
4873 &installed_channels,
4874 false,
4875 Some(&root),
4876 ));
4877 assert!(route_removal_will_quiesce_root(
4878 &root,
4879 route,
4880 &installed_channels,
4881 false,
4882 None,
4883 ));
4884 assert!(!should_quiesce_removed_root(
4885 &root,
4886 &root_channels,
4887 false,
4888 Some(&root),
4889 ));
4890 assert!(should_quiesce_removed_root(
4891 &root,
4892 &root_channels,
4893 false,
4894 None,
4895 ));
4896 assert!(!should_quiesce_removed_root(
4897 &root,
4898 &root_channels,
4899 true,
4900 None,
4901 ));
4902 }
4903
4904 #[test]
4905 fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
4906 let (_root_dir, root) = test_root("unbound-root-route-count");
4907 let executor = Arc::new(Executor::new());
4908 assert!(executor.register_actor(root.clone(), test_ctx()));
4909 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
4910 let mut root_channels = HashMap::from([(
4911 root.clone(),
4912 HashSet::from([route_key(7, 1), route_key(8, 1)]),
4913 )]);
4914
4915 remove_root_channel(&mut root_channels, &root, route_key(7, 1));
4916 if !root_channels.contains_key(&root) {
4917 quiesce_unbound_root(&root, &mut live_roots, &executor);
4918 }
4919 assert!(!live_roots[&root].unbound_quiesced);
4920
4921 remove_root_channel(&mut root_channels, &root, route_key(8, 1));
4922 if !root_channels.contains_key(&root) {
4923 quiesce_unbound_root(&root, &mut live_roots, &executor);
4924 }
4925 assert!(live_roots[&root].unbound_quiesced);
4926
4927 live_roots
4928 .get_mut(&root)
4929 .expect("root metadata")
4930 .note_activity();
4931 assert!(
4932 live_roots[&root].unbound_quiesced,
4933 "late asynchronous activity must not reactivate an unbound root"
4934 );
4935
4936 live_roots
4937 .get_mut(&root)
4938 .expect("root metadata")
4939 .reactivate_bound();
4940 assert!(!live_roots[&root].unbound_quiesced);
4941 }
4942
4943 #[test]
4944 fn allocator_pressure_relief_requires_every_root_to_be_idle() {
4945 let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
4946 let (_active_dir, active_root) = test_root("allocator-relief-active");
4947 let now = Instant::now();
4948 let mut live_roots = HashMap::new();
4949 let mut idle = RootMeta::new(now);
4950 idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
4951 live_roots.insert(idle_root, idle);
4952 assert!(process_has_been_idle(now, &live_roots));
4953
4954 live_roots.insert(active_root.clone(), RootMeta::new(now));
4955 assert!(!process_has_been_idle(now, &live_roots));
4956
4957 let active = live_roots
4958 .get_mut(&active_root)
4959 .expect("active root metadata");
4960 active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
4961 active.active_bash_waits = 1;
4962 assert!(!process_has_been_idle(now, &live_roots));
4963 }
4964
4965 #[test]
4966 fn pressure_relief_log_reports_before_and_after_measurements() {
4967 let allocator = crate::memory::AllocatorMemorySnapshot {
4968 status: "measured",
4969 bytes_in_use: Some(8 * 1024 * 1024),
4970 size_allocated: Some(12 * 1024 * 1024),
4971 retained_slack_bytes: Some(4 * 1024 * 1024),
4972 not_estimated: None,
4973 };
4974 let relief = crate::memory::AllocatorPressureRelief {
4975 bytes_released: 3 * 1024 * 1024,
4976 rss_before_bytes: Some(20 * 1024 * 1024),
4977 rss_after_bytes: Some(17 * 1024 * 1024),
4978 allocator_before: allocator.clone(),
4979 allocator_after: crate::memory::AllocatorMemorySnapshot {
4980 size_allocated: Some(9 * 1024 * 1024),
4981 retained_slack_bytes: Some(1024 * 1024),
4982 ..allocator
4983 },
4984 };
4985 let message = pressure_relief_label(&relief);
4986 assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
4987 assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
4988 assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
4989 assert!(message.contains("reported 3.0 MB released"));
4990 }
4991
4992 #[test]
4993 fn due_maintenance_jobs_skip_poisoned_roots() {
4994 let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
4995 let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
4996 let mut live_roots = HashMap::new();
4997 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
4998 let mut poisoned_meta = RootMeta::new(Instant::now());
4999 poisoned_meta.maintenance_poisoned = true;
5000 live_roots.insert(poisoned_root.clone(), poisoned_meta);
5001
5002 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5003 &mut live_roots,
5004 MAINTENANCE_SUBMIT_BUDGET,
5005 &HashSet::new(),
5006 );
5007
5008 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5009 assert!(due.iter().all(|(root, _)| root == &healthy_root));
5010 assert!(!deferred);
5011 assert!(live_roots[&healthy_root].maintenance_pending);
5012 assert_eq!(
5013 live_roots[&healthy_root].maintenance_jobs_in_flight,
5014 INITIAL_MAINTENANCE_JOB_COUNT
5015 );
5016 assert!(!live_roots[&poisoned_root].maintenance_pending);
5017 }
5018
5019 #[test]
5020 fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
5021 let (_dir, root) = test_root("maintenance-unbound");
5022 let mut meta = RootMeta::new(Instant::now());
5023 meta.unbound_quiesced = true;
5024 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5025
5026 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5027 &mut live_roots,
5028 MAINTENANCE_SUBMIT_BUDGET,
5029 &HashSet::new(),
5030 );
5031
5032 assert!(due.is_empty());
5033 assert!(!deferred);
5034 assert!(!live_roots[&root].maintenance_pending);
5035 }
5036
5037 #[test]
5038 fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
5039 let (_dir, root) = test_root("maintenance-idle-bg-subscription");
5040 let ctx = test_ctx();
5041 assert!(!ctx.completion_drains_have_work());
5042
5043 let executor = Executor::new();
5044 assert!(executor.register_actor(root.clone(), ctx));
5045 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5046 let session = "idle-session".to_string();
5047 let channel = route_key(17, 1);
5048 let bg_sub_by_session = HashMap::from([((root.clone(), session.clone()), channel)]);
5049 let mut bg_wake_pending = HashSet::new();
5050
5051 let (idle_tick_jobs, deferred) = due_maintenance_jobs(
5052 &mut live_roots,
5053 Some(&executor),
5054 &bg_sub_by_session,
5055 &bg_wake_pending,
5056 MAINTENANCE_SUBMIT_BUDGET,
5057 &HashSet::new(),
5058 );
5059 assert!(idle_tick_jobs.is_empty());
5060 assert!(!deferred);
5061 assert!(!live_roots[&root].maintenance_pending);
5062
5063 let mut bg_wake_epoch = HashMap::new();
5066 push::arm_bg_wake(
5067 root.clone(),
5068 session,
5069 channel,
5070 &mut bg_wake_pending,
5071 &mut bg_wake_epoch,
5072 );
5073 let (next_tick_jobs, deferred) = due_maintenance_jobs(
5074 &mut live_roots,
5075 Some(&executor),
5076 &bg_sub_by_session,
5077 &bg_wake_pending,
5078 MAINTENANCE_SUBMIT_BUDGET,
5079 &HashSet::new(),
5080 );
5081 assert_eq!(
5082 next_tick_jobs,
5083 vec![(root, MaintenanceDrainKind::CompletionDrains)]
5084 );
5085 assert!(!deferred);
5086 }
5087
5088 #[tokio::test]
5089 async fn subc_configure_tail_precedes_completed_search_install() {
5090 let root_dir = tempfile::tempdir().unwrap();
5091 let storage = tempfile::tempdir().unwrap();
5092 let root = ProjectRootId::from_path(root_dir.path()).unwrap();
5093 let (ctx, ignored_path) =
5094 runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
5095 let ctx = Arc::new(ctx);
5096 assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
5097 &ctx,
5098 &ignored_path
5099 ));
5100
5101 let executor = Arc::new(Executor::new());
5102 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5103 let metrics = Arc::new(DispatchPathMetrics::new());
5104 let (completion_tx, mut completion_rx) = mpsc::channel(4);
5105 submit_maintenance_job(
5106 &executor,
5107 root.clone(),
5108 MaintenanceDrainKind::ConfigureTail,
5109 Vec::new(),
5110 &completion_tx,
5111 &metrics,
5112 );
5113 submit_maintenance_job(
5114 &executor,
5115 root,
5116 MaintenanceDrainKind::CompletionDrains,
5117 Vec::new(),
5118 &completion_tx,
5119 &metrics,
5120 );
5121
5122 let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
5123 .await
5124 .expect("configure-tail completion timed out")
5125 .expect("configure-tail completion channel closed");
5126 let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
5127 .await
5128 .expect("completion-drains completion timed out")
5129 .expect("completion-drains completion channel closed");
5130 assert!(first.response.id.contains("configure-tail"));
5131 assert!(second.response.id.contains("completion-drains"));
5132 assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
5133 &ctx,
5134 &ignored_path
5135 ));
5136 assert_eq!(
5137 ctx.search_index()
5138 .read()
5139 .unwrap_or_else(std::sync::PoisonError::into_inner)
5140 .as_ref()
5141 .expect("completed search index installed")
5142 .file_count(),
5143 0,
5144 "configure must install the ignore matcher before pending paths replay"
5145 );
5146 ctx.stop_watcher_runtime();
5147 }
5148
5149 #[test]
5150 fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
5151 let (_dir, root) = test_root("maintenance-post-bind");
5152 let mut live_roots = HashMap::new();
5153 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5154
5155 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
5156 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
5157
5158 let meta = live_roots.get(&root).expect("root metadata");
5159 assert!(meta.maintenance_pending);
5160 assert_eq!(meta.maintenance_jobs_in_flight, 0);
5161 assert_eq!(
5162 meta.maintenance_queued_kinds
5163 .iter()
5164 .copied()
5165 .collect::<Vec<_>>(),
5166 vec![
5167 MaintenanceDrainKind::ConfigureTail,
5168 MaintenanceDrainKind::CompletionDrains,
5169 ]
5170 );
5171
5172 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5173 &mut live_roots,
5174 MAINTENANCE_SUBMIT_BUDGET,
5175 &HashSet::new(),
5176 );
5177
5178 assert_eq!(
5179 due,
5180 vec![
5181 (root.clone(), MaintenanceDrainKind::ConfigureTail),
5182 (root.clone(), MaintenanceDrainKind::CompletionDrains),
5183 ]
5184 );
5185 assert!(!deferred);
5186 assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
5187 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
5188 }
5189
5190 #[test]
5191 fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
5192 let mut live_roots = HashMap::new();
5193 let mut root_ids = Vec::new();
5194 let mut _dirs = Vec::new();
5195 for index in 0..4 {
5196 let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
5197 live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
5198 root_ids.push(root_id);
5199 _dirs.push(dir);
5200 }
5201
5202 let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
5203 let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
5204 &mut live_roots,
5205 small_budget,
5206 &HashSet::new(),
5207 );
5208
5209 assert_eq!(first_due.len(), small_budget);
5210 assert!(first_deferred);
5211 let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
5212 assert!(first_due_set
5213 .iter()
5214 .all(|root| live_roots[root].maintenance_pending));
5215 assert!(first_due_set
5216 .iter()
5217 .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
5218
5219 let all_roots: HashSet<_> = root_ids.into_iter().collect();
5220 let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
5221 assert!(deferred_roots
5222 .iter()
5223 .all(|root| !live_roots[root].maintenance_pending));
5224 }
5225
5226 #[test]
5227 fn due_maintenance_jobs_defers_pending_bind_roots() {
5228 let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
5229 let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
5230 let mut live_roots = HashMap::new();
5231 live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
5232 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
5233 let pending_bind_roots = HashSet::from([bind_root.clone()]);
5234
5235 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5236 &mut live_roots,
5237 usize::MAX,
5238 &pending_bind_roots,
5239 );
5240
5241 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5242 assert!(due.iter().all(|(root, _)| root == &healthy_root));
5243 assert!(!deferred);
5244 assert!(!live_roots[&bind_root].maintenance_pending);
5245 assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
5246 }
5247
5248 #[test]
5249 fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
5250 let (_dir, root) = test_root("maintenance-requeue");
5251 let mut live_roots = HashMap::new();
5252 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5253 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5254 &mut live_roots,
5255 usize::MAX,
5256 &HashSet::new(),
5257 );
5258 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5259 assert!(due.iter().all(|(due_root, _)| due_root == &root));
5260 assert!(!deferred);
5261
5262 let meta = live_roots.get_mut(&root).unwrap();
5263 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
5264 assert!(meta.maintenance_pending);
5265 assert_eq!(
5266 meta.maintenance_jobs_in_flight,
5267 INITIAL_MAINTENANCE_JOB_COUNT - 1
5268 );
5269 assert_eq!(meta.maintenance_queued_kinds.len(), 1);
5270
5271 let (requeued, deferred) =
5272 due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
5273 assert_eq!(
5274 requeued,
5275 vec![(root.clone(), MaintenanceDrainKind::Watcher)]
5276 );
5277 assert!(!deferred);
5278 let meta = live_roots.get_mut(&root).unwrap();
5279 assert_eq!(
5280 meta.maintenance_jobs_in_flight,
5281 INITIAL_MAINTENANCE_JOB_COUNT
5282 );
5283 assert!(meta.maintenance_queued_kinds.is_empty());
5284
5285 for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
5286 note_maintenance_completion(meta, None, false, false);
5287 }
5288 assert!(!meta.maintenance_pending);
5289 assert_eq!(meta.maintenance_jobs_in_flight, 0);
5290 }
5291
5292 #[test]
5293 fn maintenance_requeue_drops_while_bind_is_pending() {
5294 let (_dir, root) = test_root("maintenance-bind-requeue");
5295 let mut live_roots = HashMap::new();
5296 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5297 let (due, _) = due_maintenance_jobs_without_actor_context(
5298 &mut live_roots,
5299 usize::MAX,
5300 &HashSet::new(),
5301 );
5302 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5303
5304 let meta = live_roots.get_mut(&root).unwrap();
5305 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
5306
5307 assert_eq!(
5308 meta.maintenance_jobs_in_flight,
5309 INITIAL_MAINTENANCE_JOB_COUNT - 1
5310 );
5311 assert!(meta.maintenance_queued_kinds.is_empty());
5312 assert!(meta.maintenance_pending);
5313 }
5314
5315 #[test]
5316 fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
5317 let mut meta = RootMeta::new(Instant::now());
5318 meta.unbound_quiesced = true;
5319
5320 assert!(!should_requiesce_after_maintenance(
5321 &meta,
5322 MaintenanceDrainKind::Lsp,
5323 false,
5324 ));
5325 assert!(!should_requiesce_after_maintenance(
5326 &meta,
5327 MaintenanceDrainKind::ConfigureTail,
5328 true,
5329 ));
5330 assert!(should_requiesce_after_maintenance(
5331 &meta,
5332 MaintenanceDrainKind::ConfigureTail,
5333 false,
5334 ));
5335 }
5336
5337 #[test]
5338 fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
5339 let (_dir, root) = test_root("maintenance-fatal");
5340 let mut live_roots = HashMap::new();
5341 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5342 let (due, _) = due_maintenance_jobs_without_actor_context(
5343 &mut live_roots,
5344 usize::MAX,
5345 &HashSet::new(),
5346 );
5347 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5348
5349 let meta = live_roots.get_mut(&root).unwrap();
5350 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
5351 assert!(meta.maintenance_poisoned);
5352 assert!(meta.maintenance_queued_kinds.is_empty());
5353
5354 for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
5355 note_maintenance_completion(meta, None, false, false);
5356 }
5357 assert!(!meta.maintenance_pending);
5358 assert_eq!(meta.maintenance_jobs_in_flight, 0);
5359 }
5360
5361 #[test]
5362 fn trust_for_principal_matrix() {
5363 assert_eq!(
5364 trust_for_principal(&Some(Principal::Direct)),
5365 BindTrust::FirstParty
5366 );
5367 assert_eq!(
5368 trust_for_principal(&Some(Principal::Reserved {
5369 module_id: "llm-runner".to_string(),
5370 })),
5371 BindTrust::FirstParty
5372 );
5373 assert_eq!(
5374 trust_for_principal(&Some(Principal::Reserved {
5375 module_id: "aft".to_string(),
5376 })),
5377 BindTrust::FirstParty
5378 );
5379 assert_eq!(
5380 trust_for_principal(&Some(Principal::Reserved {
5381 module_id: "subc-mcp".to_string(),
5382 })),
5383 BindTrust::Untrusted
5384 );
5385 assert_eq!(
5386 trust_for_principal(&Some(Principal::Reserved {
5387 module_id: "anything-unknown".to_string(),
5388 })),
5389 BindTrust::Untrusted
5390 );
5391 assert_eq!(
5392 trust_for_principal(&Some(Principal::Unverified)),
5393 BindTrust::Untrusted
5394 );
5395 assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
5396 }
5397
5398 #[test]
5399 fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
5400 let principal = Some(Principal::Direct);
5401 let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
5402 let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
5403
5404 assert_eq!(
5405 trust_for_bind(fingerprint_a, &principal),
5406 BindTrust::Untrusted
5407 );
5408 assert_eq!(
5409 trust_for_bind(fingerprint_b, &principal),
5410 BindTrust::Untrusted
5411 );
5412 }
5413
5414 #[tokio::test]
5415 async fn persistent_cancel_resolves_when_fired_before_await() {
5416 let signal = PersistentCancelSignal::new();
5420 signal.cancel();
5421 tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
5423 .await
5424 .expect("cancelled() must resolve when cancel fired beforehand");
5425
5426 let racing = PersistentCancelSignal::new();
5428 let racing_for_task = racing.clone();
5429 let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
5430 racing.cancel();
5431 tokio::time::timeout(Duration::from_secs(1), waiter)
5432 .await
5433 .expect("cancelled() must resolve when cancel races the await")
5434 .expect("waiter task panicked");
5435 }
5436
5437 #[test]
5438 fn ingress_epoch_validation_silently_drops_every_stale_route_frame() {
5439 let installed = HashMap::from([(7, 9)]);
5440 for ty in [
5441 FrameType::Request,
5442 FrameType::Response,
5443 FrameType::Error,
5444 FrameType::Push,
5445 FrameType::Cancel,
5446 FrameType::Goodbye,
5447 ] {
5448 let body = if ty.is_pure_header() {
5449 Vec::new()
5450 } else {
5451 br#"{}"#.to_vec()
5452 };
5453 let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
5454 assert!(!ingress_route_is_current(&installed, &stale), "{ty:?}");
5455 }
5456
5457 let uninstalled = Frame::build(
5458 FrameType::Request,
5459 control_flags(),
5460 8,
5461 1,
5462 42,
5463 br#"{}"#.to_vec(),
5464 )
5465 .unwrap();
5466 assert!(!ingress_route_is_current(&installed, &uninstalled));
5467
5468 let current = Frame::build(
5469 FrameType::Request,
5470 control_flags(),
5471 7,
5472 9,
5473 43,
5474 br#"{}"#.to_vec(),
5475 )
5476 .unwrap();
5477 let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
5478 assert!(ingress_route_is_current(&installed, ¤t));
5479 assert!(ingress_route_is_current(&installed, &control));
5480 assert_eq!(installed, HashMap::from([(7, 9)]));
5481 }
5482
5483 #[tokio::test]
5484 async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
5485 let (_dir, root) = test_root("route-bind-b2-ordering");
5486 let route = route_key(7, 3);
5487 let identity = RouteIdentity(Arc::new(RouteIdentityData {
5488 root: root.clone(),
5489 project_root: root.as_path().to_path_buf(),
5490 harness: "opencode".to_string(),
5491 session: "b2-session".to_string(),
5492 trust: BindTrust::FirstParty,
5493 consumer_elicitation_capable: false,
5494 }));
5495 let replay_key = push::ReplayKey::from_identity(&identity);
5496 let completion = RouteBindCompletion {
5497 route,
5498 identity,
5499 bind_root_id: root.clone(),
5500 inserted_new_actor: false,
5501 configure_response: Response::success("subc-bind-7", json!({})),
5502 diagnostics_on_edit: false,
5503 ver: PROTOCOL_VERSION,
5504 corr: 91,
5505 flags: control_flags(),
5506 };
5507 let mut pending_binds = HashMap::from([(
5508 route,
5509 PendingBind {
5510 bind_root_id: root,
5511 inserted_new_actor: false,
5512 cancelled: false,
5513 configure_request_id: "subc-bind-7".to_string(),
5514 started_at: Instant::now(),
5515 warned_half_deadline: false,
5516 deadline_reported: false,
5517 corr: 91,
5518 ver: PROTOCOL_VERSION,
5519 flags: control_flags(),
5520 cancellation: crate::executor::JobCancellation::new(),
5521 },
5522 )]);
5523 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5524 let mut push_buffer =
5525 HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
5526 let (writer_tx, mut writer_rx) = mpsc::channel(8);
5527 let metrics = Arc::new(DispatchPathMetrics::new());
5528
5529 handle_route_bind_completion(
5530 &writer_tx,
5531 completion,
5532 &mut HashMap::new(),
5533 &mut HashMap::new(),
5534 &mut HashMap::new(),
5535 &mut push_buffer,
5536 &mut HashMap::new(),
5537 &mut pending_binds,
5538 &mut installed_route_epochs,
5539 &Arc::new(Executor::new()),
5540 &Arc::new(Notify::new()),
5541 &metrics,
5542 )
5543 .await
5544 .unwrap();
5545
5546 let ack = writer_rx.try_recv().expect("RouteBindAck");
5547 assert_eq!(ack.header.ty, FrameType::Response);
5548 assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
5549 let route_frame = writer_rx.try_recv().expect("post-ack route frame");
5550 assert_eq!(route_frame.header.ty, FrameType::Push);
5551 assert_eq!(
5552 (route_frame.header.channel, route_frame.header.epoch),
5553 (route.channel, route.epoch)
5554 );
5555 }
5556}