1use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, Lane};
31use crate::jsonc::strip_jsonc;
32use crate::log_ctx;
33use crate::path_identity::ProjectRootId;
34use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
35use crate::run_tool_call::{
36 run_tool_call, strip_agent_preview_arg_owned, PhaseTrace, ToolCallContext, ToolCallOutcome,
37 ToolCallResult,
38};
39use crate::runtime_drain;
40use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
41
42use subc_protocol::manifest::{
43 Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
44 ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
45};
46use subc_protocol::session::{
47 HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
48 MODULE_CONTROL_OP_HEALTH_CHECK,
49};
50use subc_protocol::{
51 ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, PROTOCOL_VERSION,
52};
53use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
54use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
55use tokio::net::TcpStream;
56use tokio::sync::{mpsc, oneshot, Notify};
57use tokio::task::JoinHandle;
58
59const AUTH_DEADLINE: Duration = Duration::from_secs(5);
62const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
63const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
64const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
65const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
66
67const HELLO_CORR: u64 = 1;
69
70const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
73
74const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
78
79const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
83
84const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
88
89const WRITER_QUEUE_CAPACITY: usize = 256;
90
91const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
94
95const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
103const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
104 MaintenanceDrainKind::Watcher,
105 MaintenanceDrainKind::Lsp,
106 MaintenanceDrainKind::ConfigureTail,
107 MaintenanceDrainKind::CompletionDrains,
108];
109#[cfg(test)]
110const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
111
112const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
113const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
114
115const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
116const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
117
118const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
121
122const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
126
127const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
129const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
130
131#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
132struct RouteChannel {
133 channel: u16,
134 epoch: u32,
135}
136
137impl fmt::Display for RouteChannel {
138 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
139 write!(f, "{}@{}", self.channel, self.epoch)
140 }
141}
142
143type PushEnvelope = (ProjectRootId, PushFrame);
144type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
145type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
146mod bash;
147mod health;
148mod manifest;
149mod push;
150mod wire;
151
152use self::health::{
153 build_health_report, warn_slow_pending_binds, DispatchPathMetrics, ReapBlockerCensus,
154 ResponseTaskGuard,
155};
156use self::manifest::{
157 build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
158 is_subc_agent_core_tool, is_subc_native_plumbing_tool,
159};
160pub use self::wire::SubcError;
161
162pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
166 manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
167}
168use self::wire::{
169 build_error_frame, build_goodbye_frame, build_tool_response_frame, decrement_counted_channel,
170 response_is_fatal_panic, response_message, send_counted_channel, send_frame,
171 send_reliable_writer_frame, send_traced_tool_response_frame, ToolResponseWriteTrace,
172 WriterFrame, WriterSender,
173};
174
175struct DecodedFrame {
176 frame: Frame,
177 phase_trace: PhaseTrace,
178}
179
180struct ToolCallCompletion {
181 text: String,
182 phase_trace: PhaseTrace,
183}
184
185#[derive(Clone)]
186struct PushSenders {
187 lossy_tx: mpsc::Sender<LossyPushEnvelope>,
188 reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
189 lossy_overflow: Arc<push::LossyOverflow>,
190 lossy_seq: Arc<AtomicU64>,
191}
192
193#[derive(Clone)]
194struct PersistentCancelSignal {
195 inner: Arc<PersistentCancelInner>,
196}
197
198struct PersistentCancelInner {
199 cancelled: AtomicBool,
200 notify: Notify,
201}
202
203impl PersistentCancelSignal {
204 fn new() -> Self {
205 Self {
206 inner: Arc::new(PersistentCancelInner {
207 cancelled: AtomicBool::new(false),
208 notify: Notify::new(),
209 }),
210 }
211 }
212
213 fn cancel(&self) {
214 if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
215 self.inner.notify.notify_waiters();
216 }
217 }
218
219 fn is_cancelled(&self) -> bool {
220 self.inner.cancelled.load(Ordering::SeqCst)
221 }
222
223 async fn cancelled(&self) {
224 loop {
232 let notified = self.inner.notify.notified();
233 tokio::pin!(notified);
234 notified.as_mut().enable();
235 if self.is_cancelled() {
236 return;
237 }
238 notified.await;
239 }
240 }
241}
242
243#[derive(Debug, Clone, Copy, PartialEq, Eq)]
244pub(crate) enum BindTrust {
245 FirstParty,
246 Untrusted,
247}
248
249impl BindTrust {
250 fn allows_bash_observation(self) -> bool {
251 matches!(self, Self::FirstParty)
252 }
253
254 fn label(self) -> &'static str {
255 match self {
256 Self::FirstParty => "first_party",
257 Self::Untrusted => "untrusted",
258 }
259 }
260
261 fn sandbox_trust(self) -> PrincipalTrust {
262 match self {
263 Self::FirstParty => PrincipalTrust::FirstParty,
264 Self::Untrusted => PrincipalTrust::Untrusted,
265 }
266 }
267}
268
269pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
270 match principal {
271 Some(Principal::Direct) => BindTrust::FirstParty,
272 Some(Principal::Reserved { module_id })
283 if module_id == "llm-runner"
284 || module_id == "aft"
285 || module_id == "broca"
286 || module_id == "alfonso-core"
287 || module_id == "prefrontal"
288 || module_id == "prefrontal-core" =>
289 {
290 BindTrust::FirstParty
291 }
292 Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
293 BindTrust::Untrusted
294 }
295 }
296}
297
298fn harness_forces_untrusted(harness: &str) -> bool {
299 harness.starts_with("fed:")
300}
301
302pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
303 if harness_forces_untrusted(harness) {
304 BindTrust::Untrusted
305 } else {
306 trust_for_principal(principal)
307 }
308}
309
310fn principal_id(principal: &Option<Principal>) -> Option<String> {
311 match principal {
312 Some(Principal::Direct) => Some("direct".to_string()),
313 Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
314 Some(Principal::Unverified) => Some("unverified".to_string()),
315 None => None,
316 }
317}
318
319fn principal_label(principal: &Option<Principal>) -> String {
320 principal_id(principal).unwrap_or_else(|| "absent".to_string())
321}
322
323#[derive(Debug)]
324struct RootMeta {
330 maintenance_pending: bool,
331 maintenance_jobs_in_flight: usize,
332 maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
333 maintenance_last_submitted: Option<Instant>,
334 maintenance_poisoned: bool,
335 last_touched: Instant,
336 diagnostics_on_edit: bool,
337 active_bash_waits: usize,
338 idle_artifacts_evicted: bool,
339 unbound_quiesced: bool,
340 consecutive_missing_sweeps: u8,
341}
342
343#[derive(Debug)]
344struct PendingBind {
345 bind_root_id: ProjectRootId,
346 inserted_new_actor: bool,
347 cancelled: bool,
348 configure_request_id: String,
349 started_at: Instant,
350 warned_half_deadline: bool,
351 deadline_reported: bool,
352 corr: u64,
353 ver: u8,
354 flags: Flags,
355 cancellation: crate::executor::JobCancellation,
360}
361
362struct RouteBindCompletion {
363 route: RouteChannel,
364 identity: RouteIdentity,
365 bind_root_id: ProjectRootId,
366 inserted_new_actor: bool,
367 configure_response: Response,
368 diagnostics_on_edit: bool,
369 ver: u8,
370 corr: u64,
371 flags: Flags,
372}
373
374#[derive(Debug, Clone)]
375struct RouteIdentity(Arc<RouteIdentityData>);
376
377#[derive(Debug)]
378struct RouteIdentityData {
379 root: ProjectRootId,
380 project_root: PathBuf,
381 harness: String,
382 session: String,
383 trust: BindTrust,
384 spawn_principal: AuthenticatedPrincipal,
385 consumer_elicitation_capable: bool,
386}
387
388impl Deref for RouteIdentity {
389 type Target = RouteIdentityData;
390
391 fn deref(&self) -> &Self::Target {
392 &self.0
393 }
394}
395
396#[derive(Debug, Clone)]
397struct RetainedSessionIdentity {
398 harness: String,
399 trust: BindTrust,
400}
401
402#[derive(Clone, Copy)]
403struct BgSub {
404 corr: u64,
405 ver: u8,
406 flags: Flags,
407}
408
409struct MaintenanceCompletion {
410 root_id: ProjectRootId,
411 kind: MaintenanceDrainKind,
412 response: Response,
413 empty_bg_sessions: Vec<(String, u64)>,
414 requeue_kind: Option<MaintenanceDrainKind>,
415}
416
417#[derive(Clone, Copy, Debug, PartialEq, Eq)]
418enum MaintenanceDrainKind {
419 Watcher,
420 Lsp,
421 ConfigureTail,
422 CompletionDrains,
423}
424
425impl MaintenanceDrainKind {
426 fn label(self) -> &'static str {
427 match self {
428 Self::Watcher => "watcher",
429 Self::Lsp => "lsp",
430 Self::ConfigureTail => "configure-tail",
431 Self::CompletionDrains => "completion-drains",
432 }
433 }
434}
435
436#[derive(Debug, Default)]
437struct MaintenanceJobOutcome {
438 empty_bg_sessions: Vec<(String, u64)>,
439 requeue_kind: Option<MaintenanceDrainKind>,
440}
441
442#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
443struct ReverseCorrKey {
444 route: RouteChannel,
445 corr: u64,
446}
447
448struct PendingBashAsk {
449 route: RouteChannel,
450 tool_corr: u64,
451 tool_flags: Flags,
452 tool_ver: u8,
453 root: ProjectRootId,
454 project_root: PathBuf,
455 session_id: String,
456 spawn_principal: AuthenticatedPrincipal,
457 request_id: String,
458 arguments: Value,
459 format_context: crate::subc_format::FormatContext,
460 cancel: bash::BashWaitCancel,
461 grants: Vec<String>,
462 expires_at: Instant,
463}
464
465impl RootMeta {
466 fn new(now: Instant) -> Self {
467 Self {
468 maintenance_pending: false,
469 maintenance_jobs_in_flight: 0,
470 maintenance_queued_kinds: VecDeque::new(),
471 maintenance_last_submitted: None,
472 maintenance_poisoned: false,
473 last_touched: now,
474 diagnostics_on_edit: false,
475 active_bash_waits: 0,
476 idle_artifacts_evicted: false,
477 unbound_quiesced: false,
478 consecutive_missing_sweeps: 0,
479 }
480 }
481
482 fn note_activity(&mut self) {
483 self.last_touched = Instant::now();
484 }
485
486 fn reactivate_bound(&mut self) {
487 self.note_activity();
488 self.idle_artifacts_evicted = false;
489 self.unbound_quiesced = false;
490 }
491}
492
493fn due_maintenance_jobs(
494 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
495 executor: Option<&Executor>,
496 bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
497 bg_wake_pending: &HashSet<RouteChannel>,
498 budget: usize,
499 pending_bind_roots: &HashSet<ProjectRootId>,
500) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
501 let mut jobs = Vec::new();
502 let mut deferred = false;
503 let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
504 roots.sort_by(|left, right| {
505 let left_last = live_roots
506 .get(left)
507 .and_then(|meta| meta.maintenance_last_submitted);
508 let right_last = live_roots
509 .get(right)
510 .and_then(|meta| meta.maintenance_last_submitted);
511 left_last
512 .cmp(&right_last)
513 .then_with(|| left.as_path().cmp(right.as_path()))
514 });
515
516 for root_id in roots {
517 let Some(meta) = live_roots.get_mut(&root_id) else {
518 continue;
519 };
520 if meta.maintenance_poisoned {
521 continue;
522 }
523
524 if pending_bind_roots.contains(&root_id) {
525 if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
526 deferred = true;
527 }
528 continue;
529 }
530
531 if !meta.maintenance_pending {
532 if jobs.len() >= budget {
533 deferred = true;
534 continue;
535 }
536 let executor_actor_context =
540 executor.and_then(|executor| executor.actor_context(&root_id));
541 let root_has_pending_bg_wake =
542 bg_sub_by_session.iter().any(|((sub_root, _), channel)| {
543 sub_root == &root_id && bg_wake_pending.contains(channel)
544 });
545 let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
546 Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
547 .into_iter()
548 .filter(|kind| {
549 if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
550 return false;
551 }
552 match kind {
553 MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
554 MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
555 MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
556 MaintenanceDrainKind::CompletionDrains => {
562 root_has_pending_bg_wake || ctx.completion_drains_have_work()
563 }
564 }
565 })
566 .collect(),
567 None if meta.unbound_quiesced => Vec::new(),
568 None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
570 };
571 if kinds_with_work.is_empty() {
572 continue;
573 }
574 meta.maintenance_pending = true;
575 meta.maintenance_queued_kinds.extend(kinds_with_work);
576 }
577
578 while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
579 if jobs.len() >= budget {
580 meta.maintenance_queued_kinds.push_front(kind);
581 deferred = true;
582 break;
583 }
584 meta.maintenance_jobs_in_flight += 1;
585 meta.maintenance_last_submitted = Some(Instant::now());
586 jobs.push((root_id.clone(), kind));
587 }
588
589 meta.maintenance_pending =
590 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
591 }
592
593 (jobs, deferred)
594}
595
596fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
597 match estimate.estimated_bytes {
598 Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
599 None if estimate.status == "busy" => "busy".to_string(),
600 None => "not estimated".to_string(),
601 }
602}
603
604fn optional_memory_label(bytes: Option<u64>) -> String {
605 bytes.map_or_else(
606 || "not estimated".to_string(),
607 |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
608 )
609}
610
611fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
612 format!(
613 "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
614 optional_memory_label(relief.rss_before_bytes),
615 optional_memory_label(relief.rss_after_bytes),
616 optional_memory_label(relief.allocator_before.bytes_in_use),
617 optional_memory_label(relief.allocator_after.bytes_in_use),
618 optional_memory_label(relief.allocator_before.size_allocated),
619 optional_memory_label(relief.allocator_after.size_allocated),
620 optional_memory_label(relief.allocator_before.retained_slack_bytes),
621 optional_memory_label(relief.allocator_after.retained_slack_bytes),
622 relief.bytes_released as f64 / (1024.0 * 1024.0),
623 )
624}
625
626fn idle_root_eviction_message(
627 root_id: &ProjectRootId,
628 memory: &crate::memory::RootMemorySnapshot,
629 pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
630) -> String {
631 let freed_bytes = [
634 &memory.semantic,
635 &memory.trigram,
636 &memory.symbols,
637 &memory.callgraph,
638 &memory.inspect,
639 ]
640 .iter()
641 .filter_map(|estimate| estimate.estimated_bytes)
642 .fold(0u64, u64::saturating_add);
643 let mut message = format!(
644 "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
645 root_id.as_path().display(),
646 freed_bytes as f64 / (1024.0 * 1024.0),
647 eviction_estimate_label(&memory.semantic),
648 eviction_estimate_label(&memory.trigram),
649 eviction_estimate_label(&memory.symbols),
650 eviction_estimate_label(&memory.callgraph),
651 eviction_estimate_label(&memory.inspect),
652 eviction_estimate_label(&memory.bash),
653 eviction_estimate_label(&memory.lsp),
654 eviction_estimate_label(&memory.parser_pool),
655 );
656 if let Some(pressure_relief) = pressure_relief {
657 message.push_str(&pressure_relief_label(pressure_relief));
658 }
659 message
660}
661
662fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
663 !live_roots.is_empty()
664 && live_roots.values().all(|meta| {
665 now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
666 && meta.active_bash_waits == 0
667 && !meta.maintenance_pending
668 && meta.maintenance_queued_kinds.is_empty()
669 })
670}
671
672fn allocator_pressure_relief_after_idle_sweep(
673 now: Instant,
674 live_roots: &HashMap<ProjectRootId, RootMeta>,
675 executor: &Executor,
676) -> Option<crate::memory::AllocatorPressureRelief> {
677 if !process_has_been_idle(now, live_roots)
678 || live_roots.keys().any(|root_id| {
679 executor
680 .actor_context(root_id)
681 .is_some_and(|ctx| ctx.artifact_eviction_blocked())
682 })
683 {
684 return None;
685 }
686
687 #[cfg(target_os = "macos")]
688 {
689 Some(crate::memory::relieve_allocator_pressure())
690 }
691 #[cfg(not(target_os = "macos"))]
692 {
693 None
694 }
695}
696
697fn quiesce_unbound_root(
698 root_id: &ProjectRootId,
699 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
700 executor: &Arc<Executor>,
701) {
702 let Some(meta) = live_roots.get_mut(root_id) else {
703 return;
704 };
705
706 let ctx = executor.actor_context(root_id);
707 if let Some(ctx) = ctx.as_ref() {
708 ctx.mark_subc_unbound();
712 }
713 let cancelled = executor.cancel_queued_maintenance(root_id);
714 let discarded = ctx
722 .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
723 .unwrap_or(0);
724 meta.unbound_quiesced = true;
725 meta.maintenance_queued_kinds.clear();
726 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
727 log::debug!(
728 "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s))",
729 root_id.as_path().display(),
730 cancelled,
731 discarded
732 );
733}
734
735#[allow(clippy::too_many_arguments)]
736fn quiesce_connection_roots(
737 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
738 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
739 routes: &mut HashMap<RouteChannel, RouteIdentity>,
740 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
741 installed_route_epochs: &mut HashMap<u16, u32>,
742 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
743 executor: &Arc<Executor>,
744) {
745 for cancel in route_bash_cancels.values() {
746 cancel.token.cancel();
747 }
748 route_bash_cancels.clear();
749
750 let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
751 for pending in pending_binds.values_mut() {
752 pending.cancelled = true;
753 roots.insert(pending.bind_root_id.clone());
754 let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
755 }
756
757 for root_id in roots {
761 if live_roots.contains_key(&root_id) {
762 quiesce_unbound_root(&root_id, live_roots, executor);
763 } else if let Some(ctx) = executor.actor_context(&root_id) {
764 ctx.mark_subc_unbound();
765 executor.cancel_queued_maintenance(&root_id);
766 crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
767 }
768 }
769
770 routes.clear();
771 root_channels.clear();
772 installed_route_epochs.clear();
773}
774
775#[derive(Debug, Default)]
779struct ReclaimedRoutes {
780 highest_epoch_by_channel: HashMap<u16, u32>,
781}
782
783impl ReclaimedRoutes {
784 fn insert(&mut self, route: RouteChannel) {
785 self.highest_epoch_by_channel
786 .entry(route.channel)
787 .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
788 .or_insert(route.epoch);
789 }
790
791 fn contains(&self, route: RouteChannel) -> bool {
792 self.highest_epoch_by_channel
793 .get(&route.channel)
794 .is_some_and(|epoch| route.epoch <= *epoch)
795 }
796}
797
798#[derive(Debug, Default)]
799struct IdleReapOutcome {
800 evicted: usize,
801 forgotten_deleted_roots: Vec<ProjectRootId>,
802}
803
804fn reap_idle_roots(
805 now: Instant,
806 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
807 pending_binds: &HashMap<RouteChannel, PendingBind>,
808 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
809 executor: &Arc<Executor>,
810 metrics: &DispatchPathMetrics,
811) -> IdleReapOutcome {
812 let pending_bind_roots = pending_binds
813 .values()
814 .map(|pending| pending.bind_root_id.clone())
815 .collect::<HashSet<_>>();
816 let mut census = ReapBlockerCensus::default();
817 let mut candidates = Vec::new();
818
819 for (root_id, meta) in live_roots.iter_mut() {
820 let deleted = !root_id.as_path().exists();
821 if deleted {
822 meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
827 } else {
828 meta.consecutive_missing_sweeps = 0;
829 }
830 let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
831 let has_bound_route = root_channels
832 .get(root_id)
833 .is_some_and(|channels| !channels.is_empty());
834 let has_pending_bind = pending_bind_roots.contains(root_id);
835
836 if deleted {
837 let mut retained = false;
838 if !deletion_confirmed {
839 census.absence_unconfirmed += 1;
840 retained = true;
841 }
842 if meta.active_bash_waits > 0 {
846 census.bash_waits += 1;
847 retained = true;
848 }
849 if meta.maintenance_pending {
850 census.maintenance_pending += 1;
851 retained = true;
852 }
853 if !meta.maintenance_queued_kinds.is_empty() {
854 census.maintenance_queued += 1;
855 retained = true;
856 }
857 if has_pending_bind {
858 census.pending_binds += 1;
859 retained = true;
860 }
861 match executor.try_actor_is_idle(root_id) {
862 Some(true) => {}
863 Some(false) => {
864 census.actor_busy += 1;
865 retained = true;
866 }
867 None => {
868 census.actor_state_busy += 1;
869 retained = true;
870 }
871 }
872 if retained {
873 census.deleted_retained += 1;
874 continue;
875 }
876 } else {
877 if has_bound_route
881 || !meta.unbound_quiesced
882 || meta.idle_artifacts_evicted
883 || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
884 || meta.active_bash_waits > 0
885 || meta.maintenance_pending
886 || !meta.maintenance_queued_kinds.is_empty()
887 || has_pending_bind
888 || !executor.actor_is_idle(root_id)
889 {
890 continue;
891 }
892 }
893 candidates.push((root_id.clone(), deleted));
894 }
895
896 let mut reaped = Vec::new();
897 let mut forgotten_deleted_roots = Vec::new();
898 for (root_id, deleted) in candidates {
899 let Some(ctx) = executor.actor_context(&root_id) else {
900 if deleted {
901 census.deleted_retained += 1;
902 census.actor_busy += 1;
903 }
904 continue;
905 };
906 let taken_pending = Some(ctx.take_pending_reconciliation_state());
911 if ctx.artifact_eviction_blocked() {
912 if let Some(pending) = taken_pending {
913 ctx.restore_pending_reconciliation_state(pending);
914 }
915 if deleted {
916 census.deleted_retained += 1;
917 census.artifact_eviction_blocked += 1;
918 }
919 continue;
920 }
921 let memory_before = ctx.memory_root_snapshot();
922 if !ctx.evict_idle_artifacts() {
923 if let Some(pending) = taken_pending {
924 ctx.restore_pending_reconciliation_state(pending);
925 }
926 if deleted {
927 census.deleted_retained += 1;
928 census.artifact_eviction_failed += 1;
929 }
930 continue;
931 }
932 drop(taken_pending);
933 ctx.stop_watcher_runtime_in_background();
934 ctx.invalidate_artifacts_after_watcher_gap();
937
938 if deleted {
939 if executor.retire_idle_actor_in_background(&root_id) {
940 live_roots.remove(&root_id);
941 forgotten_deleted_roots.push(root_id.clone());
942 } else {
943 census.deleted_retained += 1;
944 census.actor_busy += 1;
945 }
946 } else {
947 if let Some(meta) = live_roots.get_mut(&root_id) {
948 meta.idle_artifacts_evicted = true;
949 }
950 ctx.release_idle_reopenable_resources_in_background();
951 }
952 reaped.push((root_id, memory_before));
953 }
954
955 metrics.record_reap(census);
956 if census.deleted_retained > 0 {
957 log::info!(
958 "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
959 census.deleted_retained,
960 census.blocker_histogram()
961 );
962 }
963
964 let pressure_relief = (!reaped.is_empty())
965 .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
966 .flatten();
967 for (root_id, memory_before) in &reaped {
968 log::info!(
969 "{}",
970 idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
971 );
972 }
973 IdleReapOutcome {
974 evicted: reaped.len(),
975 forgotten_deleted_roots,
976 }
977}
978
979#[allow(clippy::too_many_arguments)]
980fn purge_deleted_root_residents(
981 root_id: &ProjectRootId,
982 routes: &mut HashMap<RouteChannel, RouteIdentity>,
983 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
984 installed_route_epochs: &mut HashMap<u16, u32>,
985 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
986 retry_buffer: &mut RetryBuffer,
987 reclaimed_routes: &mut ReclaimedRoutes,
988 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
989 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
990 bg_subs: &mut HashMap<RouteChannel, BgSub>,
991 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
992 bg_wake_pending: &mut HashSet<RouteChannel>,
993 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
994 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
995) {
996 let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
997 stale_routes.extend(
998 routes
999 .iter()
1000 .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1001 );
1002 stale_routes.extend(
1003 bg_sub_by_session
1004 .iter()
1005 .filter_map(|((root, _), route)| (root == root_id).then_some(*route)),
1006 );
1007 stale_routes.extend(
1008 pending_bash_asks
1009 .values()
1010 .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1011 );
1012
1013 for route in stale_routes {
1014 reclaimed_routes.insert(route);
1015 remove_installed_route(installed_route_epochs, route);
1016 remove_route_channel(routes, root_channels, route);
1017 if let Some(cancel) = route_bash_cancels.remove(&route) {
1018 cancel.token.cancel();
1019 }
1020 retry_buffer.remove(&route);
1021 bg_subs.remove(&route);
1022 bg_wake_pending.remove(&route);
1023 }
1024 root_channels.remove(root_id);
1025 session_identity.retain(|(root, _), _| root != root_id);
1026 push_buffer.retain(|key, _| &key.root != root_id);
1027 bg_wake_epoch.retain(|(root, _), _| root != root_id);
1028 pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1029 bg_sub_by_session.retain(|(root, _), _| root != root_id);
1030
1031 log::info!(
1032 "subc attach: fully forgot deleted root {}",
1033 root_id.as_path().display()
1034 );
1035}
1036
1037#[allow(clippy::too_many_arguments)]
1038fn submit_due_maintenance_jobs(
1039 executor: &Arc<Executor>,
1040 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1041 pending_binds: &HashMap<RouteChannel, PendingBind>,
1042 bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
1043 bg_wake_pending: &HashSet<RouteChannel>,
1044 bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1045 maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1046 metrics: &Arc<DispatchPathMetrics>,
1047) {
1048 let pending_bind_roots = pending_binds
1049 .values()
1050 .map(|pending| pending.bind_root_id.clone())
1051 .collect::<HashSet<_>>();
1052 let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1053 live_roots,
1054 Some(executor),
1055 bg_sub_by_session,
1056 bg_wake_pending,
1057 MAINTENANCE_SUBMIT_BUDGET,
1058 &pending_bind_roots,
1059 );
1060 if deferred_jobs {
1061 metrics
1062 .maintenance_budget_deferrals
1063 .fetch_add(1, Ordering::Relaxed);
1064 }
1065 for (root_id, kind) in due_jobs {
1066 let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1067 bg_sub_by_session
1068 .iter()
1069 .filter_map(|((root, session), _)| {
1070 if root == &root_id {
1071 Some((
1072 session.clone(),
1073 bg_wake_epoch
1074 .get(&(root_id.clone(), session.clone()))
1075 .copied()
1076 .unwrap_or(0),
1077 ))
1078 } else {
1079 None
1080 }
1081 })
1082 .collect()
1083 } else {
1084 Vec::new()
1085 };
1086 submit_maintenance_job(
1087 executor,
1088 root_id,
1089 kind,
1090 bg_sessions_to_check,
1091 maintenance_tx,
1092 metrics,
1093 );
1094 }
1095}
1096
1097fn should_requiesce_after_maintenance(
1098 meta: &RootMeta,
1099 completed_kind: MaintenanceDrainKind,
1100 bind_pending: bool,
1101) -> bool {
1102 meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1103}
1104
1105fn note_maintenance_completion(
1106 meta: &mut RootMeta,
1107 requeue_kind: Option<MaintenanceDrainKind>,
1108 fatal: bool,
1109 defer_requeue: bool,
1110) {
1111 if fatal {
1112 meta.maintenance_poisoned = true;
1113 }
1114
1115 if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1116 meta.maintenance_queued_kinds.push_back(kind);
1117 }
1118
1119 meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1120 meta.maintenance_pending =
1121 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1122}
1123
1124fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1125 RouteChannel { channel, epoch }
1126}
1127
1128fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1129 if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1130 installed_epochs.remove(&route.channel);
1131 }
1132}
1133
1134fn ingress_route_should_be_processed(
1135 installed_epochs: &HashMap<u16, u32>,
1136 reclaimed_routes: &ReclaimedRoutes,
1137 frame: &Frame,
1138) -> bool {
1139 if frame.header.channel == 0
1140 || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1141 {
1142 return true;
1143 }
1144
1145 frame.header.ty == FrameType::Request
1150 && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1151}
1152
1153fn bash_elicitation_timeout() -> Duration {
1154 if cfg!(debug_assertions) {
1155 if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1156 if let Ok(ms) = raw.parse::<u64>() {
1157 if ms > 0 {
1158 return Duration::from_millis(ms);
1159 }
1160 }
1161 }
1162 }
1163 BASH_ELICITATION_TIMEOUT
1164}
1165
1166fn allocate_reverse_corr(
1167 pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1168 route: RouteChannel,
1169 next_corr: &mut u64,
1170) -> u64 {
1171 loop {
1172 let corr = *next_corr;
1173 *next_corr = (*next_corr).wrapping_add(1).max(1);
1174 if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1175 return corr;
1176 }
1177 }
1178}
1179
1180fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1181 match kind {
1182 crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1183 crate::bash_permissions::PermissionKind::Bash => "bash",
1184 }
1185}
1186
1187fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1188 let mut patterns = Vec::new();
1189 let mut seen = HashSet::new();
1190 for ask in asks {
1191 for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1192 if seen.insert(pattern.clone()) {
1193 patterns.push(pattern.clone());
1194 }
1195 }
1196 }
1197 patterns
1198}
1199
1200fn bash_elicitation_message(
1201 command: &str,
1202 asks: &[crate::bash_permissions::PermissionAsk],
1203) -> String {
1204 let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1205 let patterns = bash_elicitation_patterns(asks);
1206 let pattern_text = if patterns.is_empty() {
1207 "no matched permission patterns".to_string()
1208 } else {
1209 patterns.join(", ")
1210 };
1211 let ask_kinds = asks
1212 .iter()
1213 .map(|ask| bash_permission_kind_label(&ask.kind))
1214 .collect::<HashSet<_>>()
1215 .into_iter()
1216 .collect::<Vec<_>>()
1217 .join(", ");
1218 if ask_kinds.is_empty() {
1219 format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1220 } else {
1221 format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1222 }
1223}
1224
1225fn bash_elicitation_request_body(
1226 command: &str,
1227 asks: &[crate::bash_permissions::PermissionAsk],
1228) -> Value {
1229 json!({
1230 "method": BASH_ELICITATION_CREATE_METHOD,
1231 "params": {
1232 "mode": "form",
1233 "message": bash_elicitation_message(command, asks),
1234 "requestedSchema": {
1235 "type": "object",
1236 "properties": {
1237 "decision": {
1238 "type": "string",
1239 "enum": ["allow", "deny"],
1240 "description": "Choose allow to run this bash command once, or deny to block it."
1241 }
1242 },
1243 "required": ["decision"],
1244 "additionalProperties": false
1245 },
1246 "_meta": {
1247 "aft": {
1248 "tool": "bash",
1249 "command": command,
1250 "asks": asks
1251 }
1252 }
1253 }
1254 })
1255}
1256
1257fn build_bash_elicitation_request_frame(
1258 ver: u8,
1259 route: RouteChannel,
1260 corr: u64,
1261 flags: Flags,
1262 command: &str,
1263 asks: &[crate::bash_permissions::PermissionAsk],
1264) -> Result<Frame, SubcError> {
1265 let body = bash_elicitation_request_body(command, asks);
1266 Frame::build_with_version(
1267 ver,
1268 FrameType::Request,
1269 flags,
1270 route.channel,
1271 route.epoch,
1272 corr,
1273 serde_json::to_vec(&body).map_err(SubcError::Json)?,
1274 )
1275 .map_err(SubcError::FrameBuild)
1276}
1277
1278fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1279 let Ok(value) = serde_json::from_slice::<Value>(body) else {
1280 return false;
1281 };
1282 flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1283}
1284
1285fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1286 let Some(object) = value.as_object() else {
1287 return false;
1288 };
1289 object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1290}
1291
1292fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1293 let Some(object) = value.as_object() else {
1294 return false;
1295 };
1296 if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1297 return false;
1298 }
1299 let Some(content) = object.get("content").and_then(Value::as_object) else {
1300 return false;
1301 };
1302 content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1303}
1304
1305#[allow(clippy::too_many_arguments)]
1306async fn settle_pending_bash_ask_denied(
1307 tx: &WriterSender,
1308 pending: PendingBashAsk,
1309 routes: &HashMap<RouteChannel, RouteIdentity>,
1310 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1311 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1312 shutdown: &Arc<Notify>,
1313 metrics: &DispatchPathMetrics,
1314) -> Result<(), SubcError> {
1315 let completion = bash::bash_denied_untrusted_completion(
1316 pending.route,
1317 pending.tool_corr,
1318 pending.tool_flags,
1319 pending.tool_ver,
1320 pending.root,
1321 pending.request_id,
1322 pending.format_context,
1323 );
1324 bash::handle_bash_deferred_completion(
1325 tx,
1326 completion,
1327 routes,
1328 live_roots,
1329 route_bash_cancels,
1330 shutdown,
1331 metrics,
1332 )
1333 .await
1334}
1335
1336fn take_pending_bash_asks_for_route(
1337 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1338 route: RouteChannel,
1339) -> Vec<PendingBashAsk> {
1340 let keys = pending_bash_asks
1341 .keys()
1342 .copied()
1343 .filter(|key| key.route == route)
1344 .collect::<Vec<_>>();
1345 keys.into_iter()
1346 .filter_map(|key| pending_bash_asks.remove(&key))
1347 .collect()
1348}
1349
1350#[allow(clippy::too_many_arguments)]
1351async fn settle_pending_bash_asks_for_route(
1352 tx: &WriterSender,
1353 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1354 route: RouteChannel,
1355 routes: &HashMap<RouteChannel, RouteIdentity>,
1356 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1357 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1358 shutdown: &Arc<Notify>,
1359 metrics: &DispatchPathMetrics,
1360) -> Result<(), SubcError> {
1361 for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1362 settle_pending_bash_ask_denied(
1363 tx,
1364 pending,
1365 routes,
1366 live_roots,
1367 route_bash_cancels,
1368 shutdown,
1369 metrics,
1370 )
1371 .await?;
1372 }
1373 Ok(())
1374}
1375
1376#[allow(clippy::too_many_arguments)]
1377async fn settle_all_pending_bash_asks(
1378 tx: &WriterSender,
1379 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1380 routes: &HashMap<RouteChannel, RouteIdentity>,
1381 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1382 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1383 shutdown: &Arc<Notify>,
1384 metrics: &DispatchPathMetrics,
1385) -> Result<(), SubcError> {
1386 let pending = pending_bash_asks
1387 .drain()
1388 .map(|(_, pending)| pending)
1389 .collect::<Vec<_>>();
1390 for pending in pending {
1391 settle_pending_bash_ask_denied(
1392 tx,
1393 pending,
1394 routes,
1395 live_roots,
1396 route_bash_cancels,
1397 shutdown,
1398 metrics,
1399 )
1400 .await?;
1401 }
1402 Ok(())
1403}
1404
1405#[allow(clippy::too_many_arguments)]
1406async fn expire_pending_bash_asks(
1407 tx: &WriterSender,
1408 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1409 routes: &HashMap<RouteChannel, RouteIdentity>,
1410 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1411 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1412 shutdown: &Arc<Notify>,
1413 metrics: &DispatchPathMetrics,
1414) -> Result<(), SubcError> {
1415 let now = Instant::now();
1416 let expired = pending_bash_asks
1417 .iter()
1418 .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1419 .collect::<Vec<_>>();
1420 for key in expired {
1421 if let Some(pending) = pending_bash_asks.remove(&key) {
1422 log::debug!(
1423 "subc attach: bash elicitation request {} on route {} expired fail-closed",
1424 key.corr,
1425 pending.route
1426 );
1427 settle_pending_bash_ask_denied(
1428 tx,
1429 pending,
1430 routes,
1431 live_roots,
1432 route_bash_cancels,
1433 shutdown,
1434 metrics,
1435 )
1436 .await?;
1437 }
1438 }
1439 Ok(())
1440}
1441
1442#[allow(clippy::too_many_arguments)]
1443async fn handle_bash_elicitation_reply(
1444 tx: &WriterSender,
1445 frame: &Frame,
1446 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1447 routes: &HashMap<RouteChannel, RouteIdentity>,
1448 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1449 executor: &Arc<Executor>,
1450 shutdown: &Arc<Notify>,
1451 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1452 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1453 metrics: &Arc<DispatchPathMetrics>,
1454 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1455 dispatch: DispatchFn,
1456) -> Result<(), SubcError> {
1457 let key = ReverseCorrKey {
1458 route: route_key(frame.header.channel, frame.header.epoch),
1459 corr: frame.header.corr,
1460 };
1461 let Some(pending) = pending_bash_asks.remove(&key) else {
1462 return Ok(());
1463 };
1464
1465 if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1466 if routes.contains_key(&key.route) {
1467 bash::submit_deferred_bash(
1468 executor,
1469 bash_deferred_tx,
1470 bash_poll_touch_tx,
1471 metrics,
1472 dispatch,
1473 pending.root,
1474 pending.project_root,
1475 pending.session_id,
1476 pending.request_id,
1477 pending.route,
1478 pending.tool_corr,
1479 pending.tool_flags,
1480 pending.tool_ver,
1481 pending.arguments,
1482 pending.format_context,
1483 pending.cancel,
1484 BindTrust::Untrusted,
1485 pending.spawn_principal,
1486 Some(pending.grants),
1487 );
1488 return Ok(());
1489 }
1490 log::debug!(
1491 "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1492 key.corr,
1493 pending.route
1494 );
1495 }
1496
1497 settle_pending_bash_ask_denied(
1498 tx,
1499 pending,
1500 routes,
1501 live_roots,
1502 route_bash_cancels,
1503 shutdown,
1504 metrics,
1505 )
1506 .await
1507}
1508
1509#[allow(clippy::too_many_arguments)]
1510async fn cancel_pending_bash_ask_for_tool_call(
1511 tx: &WriterSender,
1512 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1513 route: RouteChannel,
1514 tool_corr: u64,
1515 routes: &HashMap<RouteChannel, RouteIdentity>,
1516 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1517 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1518 shutdown: &Arc<Notify>,
1519 metrics: &DispatchPathMetrics,
1520) -> Result<(), SubcError> {
1521 let keys = pending_bash_asks
1522 .iter()
1523 .filter_map(|(key, pending)| {
1524 (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1525 })
1526 .collect::<Vec<_>>();
1527 for key in keys {
1528 if let Some(pending) = pending_bash_asks.remove(&key) {
1529 settle_pending_bash_ask_denied(
1530 tx,
1531 pending,
1532 routes,
1533 live_roots,
1534 route_bash_cancels,
1535 shutdown,
1536 metrics,
1537 )
1538 .await?;
1539 }
1540 }
1541 Ok(())
1542}
1543
1544fn remove_root_channel(
1545 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1546 root: &ProjectRootId,
1547 channel: RouteChannel,
1548) {
1549 let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1550 channels.remove(&channel);
1551 channels.is_empty()
1552 } else {
1553 false
1554 };
1555 if remove_root {
1556 root_channels.remove(root);
1557 }
1558}
1559
1560fn remove_route_channel(
1561 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1562 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1563 channel: RouteChannel,
1564) -> Option<RouteIdentity> {
1565 let removed = routes.remove(&channel);
1566 if let Some(identity) = &removed {
1567 remove_root_channel(root_channels, &identity.root, channel);
1568 }
1569 removed
1570}
1571
1572fn insert_route_channel(
1573 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1574 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1575 channel: RouteChannel,
1576 identity: RouteIdentity,
1577) {
1578 if let Some(previous) = routes.insert(channel, identity.clone()) {
1579 remove_root_channel(root_channels, &previous.root, channel);
1580 }
1581 root_channels
1582 .entry(identity.root.clone())
1583 .or_default()
1584 .insert(channel);
1585}
1586
1587fn remove_bg_subscription_index(
1588 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1589 channel: RouteChannel,
1590 identity: Option<&RouteIdentity>,
1591) {
1592 if let Some(identity) = identity {
1593 let key = (identity.root.clone(), identity.session.clone());
1594 if bg_sub_by_session.get(&key).copied() == Some(channel) {
1595 bg_sub_by_session.remove(&key);
1596 }
1597 } else {
1598 bg_sub_by_session.retain(|_, mapped_channel| *mapped_channel != channel);
1599 }
1600}
1601
1602fn route_removal_will_quiesce_root(
1603 root: &ProjectRootId,
1604 route: RouteChannel,
1605 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1606 has_pending_bind: bool,
1607 replacement_root: Option<&ProjectRootId>,
1608) -> bool {
1609 let removes_last_route = root_channels
1610 .get(root)
1611 .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1612 removes_last_route && !has_pending_bind && replacement_root != Some(root)
1613}
1614
1615fn should_quiesce_removed_root(
1616 root: &ProjectRootId,
1617 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1618 has_pending_bind: bool,
1619 replacement_root: Option<&ProjectRootId>,
1620) -> bool {
1621 !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1622}
1623
1624async fn end_bg_subscription(
1625 writer_tx: &WriterSender,
1626 metrics: &DispatchPathMetrics,
1627 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1628 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1629 bg_wake_pending: &mut HashSet<RouteChannel>,
1630 channel: RouteChannel,
1631 identity: Option<&RouteIdentity>,
1632) -> Result<(), SubcError> {
1633 if let Some(sub) = bg_subs.remove(&channel) {
1634 bg_wake_pending.remove(&channel);
1635 remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1636 push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1637 }
1638 Ok(())
1639}
1640
1641#[allow(clippy::too_many_arguments)]
1642async fn teardown_installed_route(
1643 tx: &WriterSender,
1644 metrics: &DispatchPathMetrics,
1645 executor: &Arc<Executor>,
1646 channel: RouteChannel,
1647 cancellation_reason: &str,
1648 replacement_root: Option<&ProjectRootId>,
1649 installed_route_epochs: &mut HashMap<u16, u32>,
1650 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1651 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1652 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1653 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1654 bg_wake_pending: &mut HashSet<RouteChannel>,
1655 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1656 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1657 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1658 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1659 retry_buffer: &mut RetryBuffer,
1660 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1661 shutdown: &Arc<Notify>,
1662) -> Result<(), SubcError> {
1663 remove_installed_route(installed_route_epochs, channel);
1664 end_bg_subscription(
1665 tx,
1666 metrics,
1667 bg_subs,
1668 bg_sub_by_session,
1669 bg_wake_pending,
1670 channel,
1671 routes.get(&channel),
1672 )
1673 .await?;
1674 settle_pending_bash_asks_for_route(
1675 tx,
1676 pending_bash_asks,
1677 channel,
1678 routes,
1679 live_roots,
1680 route_bash_cancels,
1681 shutdown,
1682 metrics,
1683 )
1684 .await?;
1685 if let Some(cancel) = route_bash_cancels.remove(&channel) {
1686 cancel.token.cancel();
1687 }
1688 if let Some(pending) = pending_binds.get_mut(&channel) {
1689 pending.cancelled = true;
1690 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1691 log::debug!(
1692 "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1693 channel.channel
1694 );
1695 }
1696 let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1697 if let Some(identity) = routes.get(&channel) {
1698 let has_pending_bind = pending_binds
1699 .values()
1700 .any(|pending| pending.bind_root_id == identity.root);
1701 if route_removal_will_quiesce_root(
1702 &identity.root,
1703 channel,
1704 root_channels,
1705 has_pending_bind,
1706 replacement_root,
1707 ) {
1708 if let Some(ctx) = executor.actor_context(&identity.root) {
1709 ctx.mark_subc_unbound();
1712 }
1713 }
1714 }
1715 if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
1716 if migrated > 0 {
1717 log::debug!(
1718 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
1719 channel.channel
1720 );
1721 }
1722 if let Some(meta) = live_roots.get_mut(&identity.root) {
1723 let idle_for = meta.last_touched.elapsed();
1724 meta.note_activity();
1725 log::debug!(
1726 "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
1727 channel.channel,
1728 identity.root.as_path().display(),
1729 identity.harness,
1730 identity.session,
1731 idle_for
1732 );
1733 } else {
1734 log::debug!(
1735 "subc attach: route {} torn down for root {} harness {} session {}",
1736 channel.channel,
1737 identity.root.as_path().display(),
1738 identity.harness,
1739 identity.session
1740 );
1741 }
1742 let has_pending_bind = pending_binds
1743 .values()
1744 .any(|pending| pending.bind_root_id == identity.root);
1745 if should_quiesce_removed_root(
1746 &identity.root,
1747 root_channels,
1748 has_pending_bind,
1749 replacement_root,
1750 ) {
1751 quiesce_unbound_root(&identity.root, live_roots, executor);
1752 }
1753 } else {
1754 if migrated > 0 {
1755 log::debug!(
1756 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
1757 channel.channel
1758 );
1759 }
1760 log::debug!("subc attach: unbound route {} torn down", channel.channel);
1761 }
1762 Ok(())
1763}
1764
1765fn remember_session_identity(
1766 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1767 identity: &RouteIdentity,
1768) {
1769 let key = (identity.root.clone(), identity.session.clone());
1770 if matches!(identity.trust, BindTrust::Untrusted)
1771 && session_identity
1772 .get(&key)
1773 .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
1774 {
1775 return;
1776 }
1777
1778 session_identity.insert(
1783 key,
1784 RetainedSessionIdentity {
1785 harness: identity.harness.clone(),
1786 trust: identity.trust,
1787 },
1788 );
1789}
1790
1791fn replay_key_for_session(
1792 session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1793 root: &ProjectRootId,
1794 session: &str,
1795) -> Option<(push::ReplayKey, BindTrust)> {
1796 let retained = session_identity.get(&(root.clone(), session.to_string()))?;
1797 Some((
1798 push::ReplayKey {
1799 root: root.clone(),
1800 harness: retained.harness.clone(),
1801 session: session.to_string(),
1802 },
1803 retained.trust,
1804 ))
1805}
1806pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
1809
1810#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1811enum ModuleLoopExit {
1812 Graceful,
1813 SkipSearchFlush,
1814}
1815
1816pub fn run_subc_mode(
1821 connection_file_path: &Path,
1822 ctx: Arc<AppContext>,
1823 executor: Arc<Executor>,
1824 dispatch: DispatchFn,
1825 user_config_path: Option<PathBuf>,
1826) -> Result<(), SubcError> {
1827 run_subc_mode_inner(
1831 connection_file_path,
1832 ctx,
1833 executor,
1834 dispatch,
1835 user_config_path,
1836 false,
1837 )
1838}
1839
1840fn run_subc_mode_inner(
1841 connection_file_path: &Path,
1842 ctx: Arc<AppContext>,
1843 executor: Arc<Executor>,
1844 dispatch: DispatchFn,
1845 user_config_path: Option<PathBuf>,
1846 allow_native_passthrough: bool,
1847) -> Result<(), SubcError> {
1848 let runtime = tokio::runtime::Builder::new_current_thread()
1849 .enable_all()
1850 .build()
1851 .map_err(SubcError::Runtime)?;
1852
1853 let executor_for_loop = Arc::clone(&executor);
1854 let loop_result = runtime.block_on(async move {
1855 let shared_app = ctx.app();
1856 drop(ctx);
1857 let stream = connect_and_authenticate(connection_file_path).await?;
1858 log::info!(
1859 "subc attach: authenticated to daemon via {}",
1860 connection_file_path.display()
1861 );
1862 let (read_half, write_half) = tokio::io::split(stream);
1863 run_module_loop(
1864 read_half,
1865 write_half,
1866 shared_app,
1867 executor_for_loop,
1868 dispatch,
1869 user_config_path,
1870 allow_native_passthrough,
1871 )
1872 .await
1873 });
1874
1875 let actor_contexts = executor.actor_contexts();
1876 if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
1877 flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
1880 }
1881 for actor_ctx in &actor_contexts {
1882 actor_ctx.lsp().shutdown_all();
1883 actor_ctx.bash_background().detach();
1884 }
1885
1886 loop_result.map(|_| ())
1887}
1888
1889fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
1890 for actor_ctx in actor_contexts {
1891 let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
1892 }
1893 let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
1894}
1895
1896#[doc(hidden)]
1901pub fn run_subc_mode_for_test(
1902 connection_file_path: &Path,
1903 ctx: Arc<AppContext>,
1904 executor: Arc<Executor>,
1905 dispatch: DispatchFn,
1906 user_config_path: Option<PathBuf>,
1907) -> Result<(), SubcError> {
1908 run_subc_mode_inner(
1909 connection_file_path,
1910 ctx,
1911 executor,
1912 dispatch,
1913 user_config_path,
1914 true,
1915 )
1916}
1917
1918#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1919enum AttachErrorClass {
1920 Transient,
1921 Permanent,
1922}
1923
1924impl fmt::Display for AttachErrorClass {
1925 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1926 match self {
1927 Self::Transient => f.write_str("transient"),
1928 Self::Permanent => f.write_str("permanent"),
1929 }
1930 }
1931}
1932
1933#[derive(Clone, Copy)]
1934struct AttachRetryPolicy {
1935 budget: Duration,
1936 initial_backoff: Duration,
1937 max_backoff: Duration,
1938 jitter_percent: u64,
1939}
1940
1941const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
1942 budget: ATTACH_RETRY_BUDGET,
1943 initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
1944 max_backoff: ATTACH_RETRY_MAX_BACKOFF,
1945 jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
1946};
1947
1948fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
1951 let transient = match error {
1952 SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
1953 SubcError::Auth { source, .. } => match source {
1954 subc_transport::AuthError::Timeout { .. }
1955 | subc_transport::AuthError::UnexpectedEof { .. } => true,
1956 subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
1957 _ => false,
1958 },
1959 _ => false,
1960 };
1961 if transient {
1962 AttachErrorClass::Transient
1963 } else {
1964 AttachErrorClass::Permanent
1965 }
1966}
1967
1968fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
1969 matches!(
1970 kind,
1971 io::ErrorKind::ConnectionRefused
1972 | io::ErrorKind::TimedOut
1973 | io::ErrorKind::ConnectionReset
1974 | io::ErrorKind::ConnectionAborted
1975 | io::ErrorKind::BrokenPipe
1976 | io::ErrorKind::UnexpectedEof
1977 )
1978}
1979
1980async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
1984 connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
1985}
1986
1987async fn connect_and_authenticate_with_policy(
1988 connection_file_path: &Path,
1989 policy: AttachRetryPolicy,
1990) -> Result<TcpStream, SubcError> {
1991 let started_at = Instant::now();
1992 let deadline = started_at + policy.budget;
1993 let mut attempt = 0_u32;
1994 let mut backoff = policy.initial_backoff;
1995 let mut history = Vec::new();
1996
1997 loop {
1998 attempt = attempt.saturating_add(1);
1999 let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2000 Ok(stream) => return Ok(stream),
2001 Err(error) => error,
2002 };
2003 let class = classify_attach_error(&error);
2004 let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2005 history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2006
2007 if class == AttachErrorClass::Permanent {
2008 log_attach_final_failure(started_at.elapsed(), &history);
2009 return Err(error);
2010 }
2011
2012 let remaining = deadline.saturating_duration_since(Instant::now());
2013 if remaining.is_zero() {
2014 log_attach_final_failure(started_at.elapsed(), &history);
2015 return Err(error);
2016 }
2017
2018 let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2019 log::info!(
2020 "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2021 );
2022 tokio::time::sleep(delay).await;
2023
2024 if Instant::now() >= deadline {
2025 log_attach_final_failure(started_at.elapsed(), &history);
2026 return Err(error);
2027 }
2028 backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2029 }
2030}
2031
2032fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2033 let jitter_percent = jitter_percent.min(100);
2034 if jitter_percent == 0 {
2035 return base;
2036 }
2037
2038 let mut random_bytes = [0_u8; 8];
2039 let random = if getrandom::fill(&mut random_bytes).is_ok() {
2040 u64::from_le_bytes(random_bytes)
2041 } else {
2042 let timestamp = std::time::SystemTime::now()
2043 .duration_since(std::time::UNIX_EPOCH)
2044 .unwrap_or_default()
2045 .subsec_nanos();
2046 u64::from(timestamp) ^ u64::from(attempt)
2047 };
2048 let span = jitter_percent.saturating_mul(2).saturating_add(1);
2049 let multiplier_percent = 100 - jitter_percent + random % span;
2050 let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2051 Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2052}
2053
2054fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2055 log::error!(
2056 "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2057 history.len(),
2058 history.join(" | ")
2059 );
2060}
2061
2062async fn connect_and_authenticate_once(
2063 connection_file_path: &Path,
2064 deadline: Instant,
2065) -> Result<TcpStream, SubcError> {
2066 let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2069 SubcError::ConnectionFile {
2070 path: connection_file_path.to_path_buf(),
2071 source,
2072 }
2073 })?;
2074
2075 let endpoint = conn
2076 .endpoints
2077 .first()
2078 .ok_or_else(|| SubcError::NoEndpoint {
2079 path: connection_file_path.to_path_buf(),
2080 })?;
2081 let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2082 let ip = endpoint
2083 .host
2084 .parse::<IpAddr>()
2085 .map_err(|_| SubcError::InvalidEndpoint {
2086 path: connection_file_path.to_path_buf(),
2087 endpoint: endpoint_label.clone(),
2088 })?;
2089 let addr = SocketAddr::new(ip, endpoint.port);
2090
2091 let connect_budget = deadline.saturating_duration_since(Instant::now());
2092 let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2093 .await
2094 .map_err(|_| SubcError::Connect {
2095 endpoint: endpoint_label.clone(),
2096 source: io::Error::new(
2097 io::ErrorKind::TimedOut,
2098 "initial subc attach retry budget elapsed during TCP connect",
2099 ),
2100 })?
2101 .map_err(|source| SubcError::Connect {
2102 endpoint: endpoint_label.clone(),
2103 source,
2104 })?;
2105 stream
2106 .set_nodelay(true)
2107 .map_err(|source| SubcError::Connect {
2108 endpoint: endpoint_label.clone(),
2109 source,
2110 })?;
2111
2112 let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2113 authenticate_client(&mut stream, &conn, auth_budget)
2114 .await
2115 .map_err(|source| SubcError::Auth {
2116 endpoint: endpoint_label,
2117 source,
2118 })?;
2119
2120 Ok(stream)
2121}
2122
2123#[allow(clippy::too_many_arguments)]
2124async fn process_route_bind_completion(
2125 writer_tx: &WriterSender,
2126 completion: RouteBindCompletion,
2127 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2128 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2129 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2130 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2131 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2132 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2133 installed_route_epochs: &mut HashMap<u16, u32>,
2134 executor: &Arc<Executor>,
2135 shutdown: &Arc<Notify>,
2136 metrics: &Arc<DispatchPathMetrics>,
2137) -> Result<(), SubcError> {
2138 decrement_counted_channel(&metrics.control_completion_queued);
2139 handle_route_bind_completion(
2140 writer_tx,
2141 completion,
2142 routes,
2143 root_channels,
2144 session_identity,
2145 push_buffer,
2146 live_roots,
2147 pending_binds,
2148 installed_route_epochs,
2149 executor,
2150 shutdown,
2151 metrics,
2152 )
2153 .await
2154}
2155
2156#[allow(clippy::too_many_arguments)]
2157async fn drain_pending_route_bind_completions(
2158 control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2159 writer_tx: &WriterSender,
2160 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2161 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2162 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2163 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2164 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2165 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2166 installed_route_epochs: &mut HashMap<u16, u32>,
2167 executor: &Arc<Executor>,
2168 shutdown: &Arc<Notify>,
2169 metrics: &Arc<DispatchPathMetrics>,
2170) -> Result<usize, SubcError> {
2171 let mut drained = 0;
2172 while let Ok(completion) = control_completion_rx.try_recv() {
2173 process_route_bind_completion(
2174 writer_tx,
2175 completion,
2176 routes,
2177 root_channels,
2178 session_identity,
2179 push_buffer,
2180 live_roots,
2181 pending_binds,
2182 installed_route_epochs,
2183 executor,
2184 shutdown,
2185 metrics,
2186 )
2187 .await?;
2188 drained += 1;
2189 }
2190 Ok(drained)
2191}
2192
2193async fn run_module_loop<R, W>(
2197 mut read: R,
2198 mut write: W,
2199 shared_app: Arc<App>,
2200 executor: Arc<Executor>,
2201 dispatch: DispatchFn,
2202 user_config_path: Option<PathBuf>,
2203 allow_native_passthrough: bool,
2204) -> Result<ModuleLoopExit, SubcError>
2205where
2206 R: AsyncRead + Unpin + Send + 'static,
2207 W: AsyncWrite + Unpin + Send + 'static,
2208{
2209 let hello = ModuleHelloBody {
2213 manifest: build_manifest(),
2214 protocol_ver: PROTOCOL_VERSION,
2215 control_ops: control_ops(),
2216 launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2217 };
2218 let hello_frame = Frame::build(
2219 FrameType::Hello,
2220 control_flags(),
2221 0,
2222 0,
2223 HELLO_CORR,
2224 serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2225 )
2226 .map_err(SubcError::FrameBuild)?;
2227 write_frame(&mut write, &hello_frame)
2228 .await
2229 .map_err(SubcError::FrameIo)?;
2230
2231 match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2233 None => return Err(SubcError::ClosedBeforeHelloAck),
2234 Some(frame) => match frame.header.ty {
2235 FrameType::HelloAck => {
2236 log::info!("subc attach: registered (HelloAck received)");
2237 }
2238 FrameType::Error => {
2239 let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2240 return Err(SubcError::HelloRejected { body });
2241 }
2242 other => return Err(SubcError::UnexpectedFrame { ty: other }),
2243 },
2244 }
2245
2246 let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2247 let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2248 let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2249 let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2256 let reader_task = spawn_reader_task(read, reader_tx);
2257 let shutdown = Arc::new(Notify::new());
2258 let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2267 let mut next_maintenance_at = next_drain_at;
2268 #[cfg(target_os = "macos")]
2271 let mut last_slack_relief: Option<std::time::Instant> = None;
2272 let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2273 let (bash_deferred_tx, mut bash_deferred_rx) =
2274 mpsc::channel::<bash::BashDeferredCompletion>(256);
2275 let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2276 let (control_completion_tx, mut control_completion_rx) =
2277 mpsc::channel::<RouteBindCompletion>(256);
2278 let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2279 let lossy_overflow = Arc::new(push::LossyOverflow::default());
2280 let lossy_seq = Arc::new(AtomicU64::new(0));
2281 let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2282 let push_senders = PushSenders {
2283 lossy_tx,
2284 reliable_tx,
2285 lossy_overflow: Arc::clone(&lossy_overflow),
2286 lossy_seq,
2287 };
2288 let connection_cancel = PersistentCancelSignal::new();
2289 let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2290 let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2291 let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2292 let mut bg_sub_by_session: HashMap<(ProjectRootId, String), RouteChannel> = HashMap::new();
2293 let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2294 let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2295 let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2296 let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2297 HashMap::new();
2298 let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2299 let mut retry_buffer: RetryBuffer = HashMap::new();
2300 let mut reclaimed_routes = ReclaimedRoutes::default();
2301 let mut completed_tasks = push::CompletedTaskIds::default();
2302 let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2303 let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2304 let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2305 let mut next_bash_ask_corr: u64 = 1;
2306 let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2307
2308 let loop_result: Result<ModuleLoopExit, SubcError> = loop {
2309 shared_app.set_open_route_count(routes.len());
2310 crate::logging::perf_tick(Some(&executor));
2311 dispatch_path_metrics.mark_frame_loop_tick();
2312 if let Err(error) = expire_pending_bash_asks(
2313 &writer_tx,
2314 &mut pending_bash_asks,
2315 &routes,
2316 &mut live_roots,
2317 &mut route_bash_cancels,
2318 &shutdown,
2319 &dispatch_path_metrics,
2320 )
2321 .await
2322 {
2323 break Err(error);
2324 }
2325
2326 match drain_pending_route_bind_completions(
2330 &mut control_completion_rx,
2331 &writer_tx,
2332 &mut routes,
2333 &mut root_channels,
2334 &mut session_identity,
2335 &mut push_buffer,
2336 &mut live_roots,
2337 &mut pending_binds,
2338 &mut installed_route_epochs,
2339 &executor,
2340 &shutdown,
2341 &dispatch_path_metrics,
2342 )
2343 .await
2344 {
2345 Ok(drained) => {
2346 if drained > 0 {
2347 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2348 }
2349 }
2350 Err(error) => break Err(error),
2351 }
2352
2353 if tokio::time::Instant::now() >= next_drain_at {
2354 push::emit_bg_event_wakes(
2355 &writer_tx,
2356 &dispatch_path_metrics,
2357 &bg_subs,
2358 &mut bg_wake_pending,
2359 );
2360 warn_slow_pending_binds(&mut pending_binds, &executor);
2361 if let Err(error) = expire_overdue_route_binds(
2362 &writer_tx,
2363 &executor,
2364 &mut pending_binds,
2365 &mut installed_route_epochs,
2366 &dispatch_path_metrics,
2367 )
2368 .await
2369 {
2370 break Err(error);
2371 }
2372
2373 let retried = push::drain_retry_buffers_for_bound_routes(
2374 &writer_tx,
2375 &dispatch_path_metrics,
2376 &routes,
2377 &mut retry_buffer,
2378 );
2379 if retried > 0 {
2380 log::debug!(
2381 "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2382 );
2383 }
2384
2385 next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2386 }
2387
2388 let overflow_batch = lossy_overflow.drain();
2394 if !overflow_batch.is_empty() {
2395 let (_, deferred) = push::drain_reliable_push_turn(
2396 &writer_tx,
2397 &dispatch_path_metrics,
2398 &routes,
2399 &root_channels,
2400 &session_identity,
2401 &mut retry_buffer,
2402 &mut push_buffer,
2403 &mut completed_tasks,
2404 &bg_sub_by_session,
2405 &mut bg_wake_pending,
2406 &mut bg_wake_epoch,
2407 &mut reliable_rx,
2408 None,
2409 );
2410 if deferred {
2411 tokio::task::yield_now().await;
2412 }
2413
2414 let mut batch = Vec::new();
2415 while let Ok(item) = lossy_rx.try_recv() {
2416 batch.push(item);
2417 }
2418 batch.extend(overflow_batch);
2419 push::process_lossy_push_envelope_batch(
2420 &writer_tx,
2421 &dispatch_path_metrics,
2422 &routes,
2423 &root_channels,
2424 &completed_tasks,
2425 batch,
2426 );
2427 }
2428
2429 tokio::select! {
2430 biased;
2431 Some(completion) = control_completion_rx.recv() => {
2432 if let Err(error) = process_route_bind_completion(
2433 &writer_tx,
2434 completion,
2435 &mut routes,
2436 &mut root_channels,
2437 &mut session_identity,
2438 &mut push_buffer,
2439 &mut live_roots,
2440 &mut pending_binds,
2441 &mut installed_route_epochs,
2442 &executor,
2443 &shutdown,
2444 &dispatch_path_metrics,
2445 )
2446 .await
2447 {
2448 break Err(error);
2449 }
2450 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2451 }
2452 _ = shutdown.notified() => {
2453 log::warn!("subc attach: fatal executor response requested teardown");
2454 break Ok(ModuleLoopExit::SkipSearchFlush);
2455 }
2456 maybe_frame = reader_rx.recv() => {
2457 let frame = match maybe_frame {
2458 None => {
2459 log::info!("subc attach: daemon closed connection");
2460 break Ok(ModuleLoopExit::Graceful);
2461 }
2462 Some(Err(error)) => break Err(error),
2463 Some(Ok(frame)) => frame,
2464 };
2465 let phase_trace = frame.phase_trace;
2466 let frame = frame.frame;
2467
2468 if !ingress_route_should_be_processed(
2469 &installed_route_epochs,
2470 &reclaimed_routes,
2471 &frame,
2472 ) {
2473 log::debug!(
2474 "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2475 frame.header.ty,
2476 frame.header.channel,
2477 frame.header.epoch
2478 );
2479 continue;
2480 }
2481
2482 match frame.header.ty {
2483 FrameType::Ping if frame.header.channel == 0 => {
2484 let pong = match Frame::build_with_version(
2485 frame.header.ver,
2486 FrameType::Pong,
2487 frame.header.flags,
2488 0,
2489 0,
2490 frame.header.corr,
2491 Vec::new(),
2492 ) {
2493 Ok(pong) => pong,
2494 Err(error) => break Err(SubcError::FrameBuild(error)),
2495 };
2496 if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2497 break Err(error);
2498 }
2499 }
2500 FrameType::Goodbye if frame.header.channel == 0 => {
2501 log::info!("subc attach: received channel-0 Goodbye");
2502 break Ok(ModuleLoopExit::Graceful);
2503 }
2504 FrameType::Goodbye => {
2505 let channel = route_key(frame.header.channel, frame.header.epoch);
2506 if let Err(error) = teardown_installed_route(
2507 &writer_tx,
2508 &dispatch_path_metrics,
2509 &executor,
2510 channel,
2511 "Goodbye",
2512 None,
2513 &mut installed_route_epochs,
2514 &mut routes,
2515 &mut root_channels,
2516 &mut bg_subs,
2517 &mut bg_sub_by_session,
2518 &mut bg_wake_pending,
2519 &mut pending_bash_asks,
2520 &mut live_roots,
2521 &mut route_bash_cancels,
2522 &mut pending_binds,
2523 &mut retry_buffer,
2524 &mut push_buffer,
2525 &shutdown,
2526 )
2527 .await
2528 {
2529 break Err(error);
2530 }
2531 }
2532 FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2533 if let Err(error) = handle_bash_elicitation_reply(
2534 &writer_tx,
2535 &frame,
2536 &mut pending_bash_asks,
2537 &routes,
2538 &mut live_roots,
2539 &executor,
2540 &shutdown,
2541 &bash_deferred_tx,
2542 &bash_poll_touch_tx,
2543 &dispatch_path_metrics,
2544 &mut route_bash_cancels,
2545 dispatch,
2546 )
2547 .await
2548 {
2549 break Err(error);
2550 }
2551 }
2552 FrameType::Request if frame.header.channel == 0 => {
2553 if let Err(error) = handle_control_request(
2554 &writer_tx,
2555 &frame,
2556 &shared_app,
2557 &executor,
2558 &mut live_roots,
2559 &mut pending_binds,
2560 &mut installed_route_epochs,
2561 &mut routes,
2562 &mut root_channels,
2563 &mut bg_subs,
2564 &mut bg_sub_by_session,
2565 &mut bg_wake_pending,
2566 &mut pending_bash_asks,
2567 &mut route_bash_cancels,
2568 &mut retry_buffer,
2569 &mut push_buffer,
2570 &shutdown,
2571 &control_completion_tx,
2572 &dispatch_path_metrics,
2573 &push_senders,
2574 dispatch,
2575 user_config_path.as_deref(),
2576 )
2577 .await
2578 {
2579 break Err(error);
2580 }
2581 }
2582 FrameType::Request => {
2583 if let Err(error) = handle_tool_call(
2584 &writer_tx,
2585 &frame,
2586 phase_trace,
2587 &routes,
2588 &pending_binds,
2589 &mut live_roots,
2590 &executor,
2591 &shutdown,
2592 &connection_cancel,
2593 &bash_deferred_tx,
2594 &bash_poll_touch_tx,
2595 &dispatch_path_metrics,
2596 &mut route_bash_cancels,
2597 &mut pending_bash_asks,
2598 &mut next_bash_ask_corr,
2599 &mut bg_subs,
2600 &mut bg_sub_by_session,
2601 &mut bg_wake_pending,
2602 &mut bg_wake_epoch,
2603 dispatch,
2604 allow_native_passthrough,
2605 )
2606 .await
2607 {
2608 break Err(error);
2609 }
2610 }
2611 FrameType::Cancel => {
2612 let channel = route_key(frame.header.channel, frame.header.epoch);
2613 if bg_subs.contains_key(&channel) {
2614 if let Err(error) = end_bg_subscription(
2615 &writer_tx,
2616 &dispatch_path_metrics,
2617 &mut bg_subs,
2618 &mut bg_sub_by_session,
2619 &mut bg_wake_pending,
2620 channel,
2621 routes.get(&channel),
2622 )
2623 .await
2624 {
2625 break Err(error);
2626 }
2627 }
2628 if let Err(error) = cancel_pending_bash_ask_for_tool_call(
2629 &writer_tx,
2630 &mut pending_bash_asks,
2631 channel,
2632 frame.header.corr,
2633 &routes,
2634 &mut live_roots,
2635 &mut route_bash_cancels,
2636 &shutdown,
2637 &dispatch_path_metrics,
2638 )
2639 .await
2640 {
2641 break Err(error);
2642 }
2643 }
2644 _ => {}
2649 }
2650 }
2651 Some((root_id, frame)) = reliable_rx.recv() => {
2652 let (_, deferred) = push::drain_reliable_push_turn(
2656 &writer_tx,
2657 &dispatch_path_metrics,
2658 &routes,
2659 &root_channels,
2660 &session_identity,
2661 &mut retry_buffer,
2662 &mut push_buffer,
2663 &mut completed_tasks,
2664 &bg_sub_by_session,
2665 &mut bg_wake_pending,
2666 &mut bg_wake_epoch,
2667 &mut reliable_rx,
2668 Some((root_id, frame)),
2669 );
2670 if deferred {
2671 tokio::task::yield_now().await;
2672 }
2673 }
2674 Some((order, root_id, frame)) = lossy_rx.recv() => {
2675 let (_, deferred) = push::drain_reliable_push_turn(
2679 &writer_tx,
2680 &dispatch_path_metrics,
2681 &routes,
2682 &root_channels,
2683 &session_identity,
2684 &mut retry_buffer,
2685 &mut push_buffer,
2686 &mut completed_tasks,
2687 &bg_sub_by_session,
2688 &mut bg_wake_pending,
2689 &mut bg_wake_epoch,
2690 &mut reliable_rx,
2691 None,
2692 );
2693 if deferred {
2694 tokio::task::yield_now().await;
2695 }
2696
2697 let mut batch = vec![(order, root_id, frame)];
2704 while let Ok(item) = lossy_rx.try_recv() {
2705 batch.push(item);
2706 }
2707 batch.extend(lossy_overflow.drain());
2708 push::process_lossy_push_envelope_batch(
2709 &writer_tx,
2710 &dispatch_path_metrics,
2711 &routes,
2712 &root_channels,
2713 &completed_tasks,
2714 batch,
2715 );
2716 }
2717 Some(done) = bash_deferred_rx.recv() => {
2718 decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
2719 if let Err(error) = bash::handle_bash_deferred_completion(
2720 &writer_tx,
2721 done,
2722 &routes,
2723 &mut live_roots,
2724 &mut route_bash_cancels,
2725 &shutdown,
2726 &dispatch_path_metrics,
2727 )
2728 .await
2729 {
2730 break Err(error);
2731 }
2732 }
2733 Some(root_id) = bash_poll_touch_rx.recv() => {
2734 decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
2735 if let Some(meta) = live_roots.get_mut(&root_id) {
2736 meta.note_activity();
2737 }
2738 }
2739 Some(completion) = maintenance_rx.recv() => {
2740 decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
2741 let root_id = completion.root_id.clone();
2742 let response = completion.response;
2743 let response_is_fatal = response_is_fatal_panic(&response);
2744 let bind_pending = pending_binds
2745 .values()
2746 .any(|pending| pending.bind_root_id == root_id);
2747 let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
2748 let defer_requeue = meta.unbound_quiesced || bind_pending;
2749 note_maintenance_completion(
2750 meta,
2751 completion.requeue_kind,
2752 response_is_fatal,
2753 defer_requeue,
2754 );
2755 should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
2756 } else {
2757 false
2758 };
2759 if requiesce {
2760 quiesce_unbound_root(&root_id, &mut live_roots, &executor);
2761 }
2762 push::clear_stale_bg_wakes_for_empty_sessions(
2763 &root_id,
2764 &completion.empty_bg_sessions,
2765 &bg_sub_by_session,
2766 &mut bg_wake_pending,
2767 &bg_wake_epoch,
2768 );
2769 if response_is_fatal {
2770 if let Some(meta) = live_roots.get_mut(&root_id) {
2771 meta.maintenance_poisoned = true;
2772 }
2773 log::warn!(
2774 "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
2775 );
2776 }
2777 }
2778 _ = tokio::time::sleep_until(next_drain_at) => {
2779 }
2782 _ = tokio::time::sleep_until(next_maintenance_at) => {
2783 let reaped_lsp_children = shared_app
2788 .lsp_child_registry()
2789 .reap_children_with_gone_cwd();
2790 if reaped_lsp_children > 0 {
2791 log::warn!(
2792 "subc attach: reaped {reaped_lsp_children} LSP child process group(s) whose cwd no longer exists"
2793 );
2794 }
2795 let reap = reap_idle_roots(
2796 Instant::now(),
2797 &mut live_roots,
2798 &pending_binds,
2799 &root_channels,
2800 &executor,
2801 &dispatch_path_metrics,
2802 );
2803 for root_id in &reap.forgotten_deleted_roots {
2804 purge_deleted_root_residents(
2805 root_id,
2806 &mut routes,
2807 &mut root_channels,
2808 &mut installed_route_epochs,
2809 &mut route_bash_cancels,
2810 &mut retry_buffer,
2811 &mut reclaimed_routes,
2812 &mut session_identity,
2813 &mut push_buffer,
2814 &mut bg_subs,
2815 &mut bg_sub_by_session,
2816 &mut bg_wake_pending,
2817 &mut bg_wake_epoch,
2818 &mut pending_bash_asks,
2819 );
2820 }
2821 if reap.evicted > 0 {
2822 log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
2823 }
2824 submit_due_maintenance_jobs(
2825 &executor,
2826 &mut live_roots,
2827 &pending_binds,
2828 &bg_sub_by_session,
2829 &bg_wake_pending,
2830 &bg_wake_epoch,
2831 &maintenance_tx,
2832 &dispatch_path_metrics,
2833 );
2834 #[cfg(target_os = "macos")]
2841 {
2842 let now_std = std::time::Instant::now();
2843 if crate::memory::spawn_allocator_slack_relief_if_due(
2844 last_slack_relief,
2845 now_std,
2846 ) {
2847 last_slack_relief = Some(now_std);
2848 }
2849 }
2850 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2851 }
2852 }
2853 };
2854
2855 shared_app.set_open_route_count(0);
2856
2857 connection_cancel.cancel();
2858 quiesce_connection_roots(
2861 &mut live_roots,
2862 &mut pending_binds,
2863 &mut routes,
2864 &mut root_channels,
2865 &mut installed_route_epochs,
2866 &mut route_bash_cancels,
2867 &executor,
2868 );
2869
2870 let mut loop_result = loop_result;
2871 if !pending_bash_asks.is_empty() {
2872 let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2873 if let Err(error) = settle_all_pending_bash_asks(
2874 &writer_tx,
2875 &mut pending_bash_asks,
2876 &no_routes,
2877 &mut live_roots,
2878 &mut route_bash_cancels,
2879 &shutdown,
2880 &dispatch_path_metrics,
2881 )
2882 .await
2883 {
2884 loop_result = loop_result.and(Err(error));
2885 }
2886 }
2887
2888 reader_task.abort();
2891 drop(writer_tx);
2892 let writer_result = finish_writer_task(writer_task).await;
2893 loop_result.and_then(|exit| writer_result.map(|_| exit))
2894}
2895
2896fn spawn_writer_task<W>(
2897 mut write: W,
2898 mut rx: mpsc::Receiver<WriterFrame>,
2899 metrics: Arc<DispatchPathMetrics>,
2900) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
2901where
2902 W: AsyncWrite + Unpin + Send + 'static,
2903{
2904 tokio::spawn(async move {
2905 let mut write_buffer = Vec::new();
2906 while let Some(mut queued) = rx.recv().await {
2907 let measure = queued.tool_response_trace.is_some();
2908 let dequeued = measure.then(Instant::now);
2909 metrics.writer_active.store(true, Ordering::Relaxed);
2910 decrement_counted_channel(&metrics.writer_queued);
2911 let write_timing = write_frame_contiguous(
2912 &mut write,
2913 queued.frame(),
2914 queued.body(),
2915 &mut write_buffer,
2916 measure,
2917 )
2918 .await;
2919 metrics.writer_active.store(false, Ordering::Relaxed);
2920 let write_timing = write_timing?;
2921
2922 if let (Some(trace), Some(dequeued), Some(write_timing)) =
2923 (queued.tool_response_trace.take(), dequeued, write_timing)
2924 {
2925 if let Some(completed) = trace.finish(
2926 dequeued,
2927 write_timing.write_started,
2928 write_timing.write_finished,
2929 write_timing.frame_bytes,
2930 ) {
2931 log_ctx::with_session(Some(completed.session), || {
2932 crate::logging::note_tool_call_trace(
2933 &completed.name,
2934 &completed.root,
2935 completed.channel,
2936 completed.corr,
2937 completed.phases,
2938 );
2939 });
2940 }
2941 }
2942 }
2943 Ok(())
2944 })
2945}
2946
2947struct FrameWriteTiming {
2948 write_started: Instant,
2949 write_finished: Instant,
2950 frame_bytes: usize,
2951}
2952
2953async fn write_frame_contiguous<W>(
2957 writer: &mut W,
2958 frame: &Frame,
2959 body: &[u8],
2960 buffer: &mut Vec<u8>,
2961 measure: bool,
2962) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
2963where
2964 W: AsyncWrite + Unpin,
2965{
2966 if frame.header.len as usize != body.len() {
2967 return Err(subc_transport::FrameIoError::BodyLengthMismatch {
2968 header_len: frame.header.len,
2969 body_len: body.len(),
2970 });
2971 }
2972
2973 let header = frame.header.encode();
2974 buffer.clear();
2975 buffer.reserve(header.len() + body.len());
2976 buffer.extend_from_slice(&header);
2977 buffer.extend_from_slice(body);
2978 let write_started = measure.then(Instant::now);
2979 writer
2980 .write_all(buffer)
2981 .await
2982 .map_err(subc_transport::FrameIoError::Io)?;
2983 Ok(write_started.map(|write_started| FrameWriteTiming {
2984 write_started,
2985 write_finished: Instant::now(),
2986 frame_bytes: buffer.len(),
2987 }))
2988}
2989
2990fn spawn_reader_task<R>(
2991 mut read: R,
2992 tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
2993) -> JoinHandle<()>
2994where
2995 R: AsyncRead + Unpin + Send + 'static,
2996{
2997 tokio::spawn(async move {
2998 loop {
2999 match read_frame(&mut read).await {
3000 Ok(Some(frame)) => {
3001 let decoded = DecodedFrame {
3002 frame,
3003 phase_trace: PhaseTrace::new(Instant::now()),
3004 };
3005 if tx.send(Ok(decoded)).await.is_err() {
3006 return;
3007 }
3008 }
3009 Ok(None) => {
3010 return;
3012 }
3013 Err(error) => {
3014 if let subc_transport::FrameIoError::Io(io_error) = &error {
3021 if matches!(
3022 io_error.kind(),
3023 std::io::ErrorKind::ConnectionReset
3024 | std::io::ErrorKind::ConnectionAborted
3025 ) {
3026 log::info!(
3027 "subc attach: connection reset by daemon; treating as close"
3028 );
3029 return;
3030 }
3031 }
3032 let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3033 return;
3034 }
3035 }
3036 }
3037 })
3038}
3039
3040async fn finish_writer_task(
3041 mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3042) -> Result<(), SubcError> {
3043 match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3044 Ok(Ok(Ok(()))) => Ok(()),
3045 Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3046 Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3047 Err(_) => {
3048 writer_task.abort();
3049 Ok(())
3050 }
3051 }
3052}
3053
3054fn register_actor_for_bind(
3055 shared_app: &Arc<App>,
3056 executor: &Arc<Executor>,
3057 push_senders: &PushSenders,
3058 bind_root_id: &ProjectRootId,
3059 route_channel: u16,
3060 root_was_live: bool,
3061) -> bool {
3062 if executor.actor_registered(bind_root_id) {
3063 log::debug!(
3064 "subc attach: reusing actor for route {} root {}",
3065 route_channel,
3066 bind_root_id.as_path().display()
3067 );
3068 return false;
3069 }
3070
3071 if root_was_live {
3072 log::warn!(
3073 "subc attach: recreating missing actor for live root {} on route {}",
3074 bind_root_id.as_path().display(),
3075 route_channel
3076 );
3077 }
3078
3079 let actor_ctx = Arc::new(AppContext::from_app(
3080 Arc::clone(shared_app),
3081 Config::default(),
3082 ));
3083 install_bash_compressor(&actor_ctx);
3084 actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
3085 push_senders.clone(),
3086 bind_root_id.clone(),
3087 )));
3088 let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
3089 drop(actor_ctx);
3090 if inserted {
3091 log::debug!(
3095 "subc attach: registered actor for route {} root {}",
3096 route_channel,
3097 bind_root_id.as_path().display()
3098 );
3099 } else {
3100 log::debug!(
3101 "subc attach: actor appeared while binding route {} root {}; reusing it",
3102 route_channel,
3103 bind_root_id.as_path().display()
3104 );
3105 }
3106 inserted
3107}
3108
3109fn rollback_pending_bind_actor(
3110 executor: &Arc<Executor>,
3111 live_roots: &HashMap<ProjectRootId, RootMeta>,
3112 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3113 root_id: &ProjectRootId,
3114 inserted_new_actor: bool,
3115) {
3116 if !inserted_new_actor || live_roots.contains_key(root_id) {
3117 return;
3118 }
3119
3120 if let Some((route, pending)) = pending_binds
3121 .iter_mut()
3122 .find(|(_, pending)| &pending.bind_root_id == root_id)
3123 {
3124 pending.inserted_new_actor = true;
3125 log::debug!(
3126 "subc attach: transferred rollback ownership for root {} to pending route {}",
3127 root_id.as_path().display(),
3128 route
3129 );
3130 return;
3131 }
3132
3133 executor.remove_actor(root_id);
3134}
3135
3136fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
3137 match response.data.get("code").and_then(|code| code.as_str()) {
3138 Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
3143 Some("cache_key_probe_failed") => "cache_key_probe_failed",
3147 Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
3151 _ => "config_divergence",
3152 }
3153}
3154
3155fn queue_post_bind_configure_and_completion_maintenance(
3156 root_id: &ProjectRootId,
3157 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3158) {
3159 let Some(meta) = live_roots.get_mut(root_id) else {
3160 return;
3161 };
3162 if meta.maintenance_poisoned || meta.maintenance_pending {
3163 return;
3164 }
3165
3166 meta.maintenance_pending = true;
3167 meta.maintenance_queued_kinds
3168 .push_back(MaintenanceDrainKind::ConfigureTail);
3169 meta.maintenance_queued_kinds
3170 .push_back(MaintenanceDrainKind::CompletionDrains);
3171}
3172
3173#[allow(clippy::too_many_arguments)]
3174async fn handle_route_bind_completion(
3175 tx: &WriterSender,
3176 completion: RouteBindCompletion,
3177 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3178 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3179 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
3180 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3181 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3182 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3183 installed_route_epochs: &mut HashMap<u16, u32>,
3184 executor: &Arc<Executor>,
3185 shutdown: &Arc<Notify>,
3186 metrics: &Arc<DispatchPathMetrics>,
3187) -> Result<(), SubcError> {
3188 let route_id = completion.route;
3189 let Some(pending) = pending_binds.remove(&route_id) else {
3190 log::warn!(
3191 "subc attach: dropping RouteBind completion for non-pending route {}",
3192 completion.route
3193 );
3194 rollback_pending_bind_actor(
3195 executor,
3196 live_roots,
3197 pending_binds,
3198 &completion.bind_root_id,
3199 completion.inserted_new_actor,
3200 );
3201 let has_pending_bind = pending_binds
3202 .values()
3203 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3204 if !root_channels
3205 .get(&completion.bind_root_id)
3206 .is_some_and(|channels| !channels.is_empty())
3207 && !has_pending_bind
3208 {
3209 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3210 }
3211 remove_installed_route(installed_route_epochs, route_id);
3212 return Ok(());
3213 };
3214
3215 if pending.bind_root_id != completion.bind_root_id {
3216 log::warn!(
3217 "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
3218 completion.route,
3219 pending.bind_root_id.as_path().display(),
3220 completion.bind_root_id.as_path().display()
3221 );
3222 }
3223
3224 let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
3225 if pending.cancelled {
3226 rollback_pending_bind_actor(
3227 executor,
3228 live_roots,
3229 pending_binds,
3230 &completion.bind_root_id,
3231 inserted_new_actor,
3232 );
3233 let has_pending_bind = pending_binds
3234 .values()
3235 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3236 if !root_channels
3237 .get(&completion.bind_root_id)
3238 .is_some_and(|channels| !channels.is_empty())
3239 && !has_pending_bind
3240 {
3241 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3242 }
3243 log::debug!(
3244 "subc attach: discarded completed RouteBind for cancelled route {} root {}",
3245 completion.route,
3246 completion.bind_root_id.as_path().display()
3247 );
3248 remove_installed_route(installed_route_epochs, route_id);
3249 return Ok(());
3250 }
3251
3252 let failure = if !completion.configure_response.success {
3253 Some((
3254 &completion.configure_response,
3255 "configure failed during route bind",
3256 ))
3257 } else {
3258 None
3259 };
3260
3261 if let Some((response, fallback)) = failure {
3262 rollback_pending_bind_actor(
3263 executor,
3264 live_roots,
3265 pending_binds,
3266 &completion.bind_root_id,
3267 inserted_new_actor,
3268 );
3269 let has_pending_bind = pending_binds
3270 .values()
3271 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3272 if !root_channels
3273 .get(&completion.bind_root_id)
3274 .is_some_and(|channels| !channels.is_empty())
3275 && !has_pending_bind
3276 {
3277 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3278 }
3279 let message = response_message(response, fallback);
3280 let fatal = response_is_fatal_panic(response);
3281 let error_code = route_bind_error_code_for_configure_response(response);
3282 send_route_bind_error_parts(
3283 tx,
3284 completion.ver,
3285 completion.corr,
3286 completion.flags,
3287 error_code,
3288 &message,
3289 metrics,
3290 )
3291 .await?;
3292 remove_installed_route(installed_route_epochs, route_id);
3293 if fatal {
3294 signal_fatal_teardown(
3295 tx,
3296 Some(completion.route),
3297 completion.ver,
3298 completion.corr,
3299 shutdown,
3300 metrics,
3301 )
3302 .await;
3303 }
3304 return Ok(());
3305 }
3306
3307 remember_session_identity(session_identity, &completion.identity);
3308 let replay_key = push::ReplayKey::from_identity(&completion.identity);
3309 let bind_trust = completion.identity.trust;
3310 insert_route_channel(routes, root_channels, route_id, completion.identity);
3311 let restore_watcher = live_roots
3312 .get(&completion.bind_root_id)
3313 .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3314 live_roots
3315 .entry(completion.bind_root_id.clone())
3316 .and_modify(|meta| {
3317 meta.reactivate_bound();
3318 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3319 meta.maintenance_poisoned = false;
3320 })
3321 .or_insert_with(|| RootMeta::new(Instant::now()));
3322 if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3323 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3324 meta.maintenance_poisoned = false;
3325 }
3326 if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3327 ctx.mark_subc_bound();
3328 if restore_watcher {
3329 crate::commands::configure::ensure_project_watcher(&ctx);
3330 }
3331 }
3332
3333 let ack =
3334 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3335 let response = Frame::build_with_version(
3336 completion.ver,
3337 FrameType::Response,
3338 control_flags(),
3339 0,
3340 0,
3341 completion.corr,
3342 ack,
3343 )
3344 .map_err(SubcError::FrameBuild)?;
3345 send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3346 queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3347 let replayed = push::replay_buffered_push_frames(
3348 tx,
3349 metrics,
3350 route_id,
3351 push_buffer,
3352 &replay_key,
3353 bind_trust,
3354 );
3355 if replayed > 0 {
3356 log::debug!(
3357 "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3358 replayed,
3359 completion.route,
3360 replay_key.root.as_path().display(),
3361 replay_key.harness,
3362 replay_key.session
3363 );
3364 }
3365 log::info!(
3366 "subc attach: route {} bound to root {}",
3367 completion.route,
3368 completion.bind_root_id.as_path().display()
3369 );
3370 Ok(())
3371}
3372
3373async fn expire_overdue_route_binds(
3374 tx: &WriterSender,
3375 executor: &Arc<Executor>,
3376 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3377 installed_route_epochs: &mut HashMap<u16, u32>,
3378 metrics: &DispatchPathMetrics,
3379) -> Result<(), SubcError> {
3380 let now = Instant::now();
3381 let expired: Vec<_> = pending_binds
3382 .iter()
3383 .filter_map(|(route, pending)| {
3384 let age = now.saturating_duration_since(pending.started_at);
3385 (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3386 (
3387 *route,
3388 pending.corr,
3389 pending.ver,
3390 pending.flags,
3391 pending.bind_root_id.clone(),
3392 pending.configure_request_id.clone(),
3393 age,
3394 )
3395 })
3396 })
3397 .collect();
3398
3399 for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3400 if let Some(pending) = pending_binds.get_mut(&route) {
3401 pending.cancelled = true;
3402 pending.deadline_reported = true;
3403 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3404 log::debug!(
3405 "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3406 );
3407 }
3408 remove_installed_route(installed_route_epochs, route);
3409 let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3410 let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3411 send_route_bind_error_parts(
3412 tx,
3413 ver,
3414 corr,
3415 flags,
3416 "actor_not_ready",
3417 &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3418 metrics,
3419 )
3420 .await?;
3421 log::warn!(
3422 "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3423 route,
3424 root_id.as_path().display(),
3425 deadline_ms,
3426 configure_request_id
3427 );
3428 }
3429
3430 Ok(())
3431}
3432
3433#[allow(clippy::too_many_arguments)]
3438async fn handle_control_request(
3439 tx: &WriterSender,
3440 frame: &Frame,
3441 shared_app: &Arc<App>,
3442 executor: &Arc<Executor>,
3443 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3444 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3445 installed_route_epochs: &mut HashMap<u16, u32>,
3446 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3447 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3448 bg_subs: &mut HashMap<RouteChannel, BgSub>,
3449 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3450 bg_wake_pending: &mut HashSet<RouteChannel>,
3451 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3452 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3453 retry_buffer: &mut RetryBuffer,
3454 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3455 shutdown: &Arc<Notify>,
3456 control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3457 metrics: &Arc<DispatchPathMetrics>,
3458 push_senders: &PushSenders,
3459 dispatch: DispatchFn,
3460 user_config_path: Option<&Path>,
3461) -> Result<(), SubcError> {
3462 let request =
3463 serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3464 match request {
3465 ModuleControlRequest::RouteBind {
3466 route_channel,
3467 epoch,
3468 target: _,
3469 identity,
3470 principal,
3471 consumer_capabilities,
3472 } => {
3473 let route_id = route_key(route_channel, epoch);
3474 if epoch == 0 {
3475 return send_route_bind_error(
3476 tx,
3477 frame,
3478 "config_divergence",
3479 "route bind uses an invalid channel generation",
3480 metrics,
3481 )
3482 .await;
3483 }
3484 let mut bind_root_id = None;
3485 if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3486 if installed_epoch >= epoch {
3487 return send_route_bind_error(
3488 tx,
3489 frame,
3490 "config_divergence",
3491 "route bind generation is not newer than the installed generation",
3492 metrics,
3493 )
3494 .await;
3495 }
3496
3497 let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3498 Ok(root_id) => root_id,
3499 Err(error) => {
3500 return send_route_bind_error(
3501 tx,
3502 frame,
3503 "config_divergence",
3504 &format!("invalid route project root: {error}"),
3505 metrics,
3506 )
3507 .await;
3508 }
3509 };
3510 teardown_installed_route(
3511 tx,
3512 metrics,
3513 executor,
3514 route_key(route_channel, installed_epoch),
3515 "higher-epoch RouteBind",
3516 Some(&replacement_root),
3517 installed_route_epochs,
3518 routes,
3519 root_channels,
3520 bg_subs,
3521 bg_sub_by_session,
3522 bg_wake_pending,
3523 pending_bash_asks,
3524 live_roots,
3525 route_bash_cancels,
3526 pending_binds,
3527 retry_buffer,
3528 push_buffer,
3529 shutdown,
3530 )
3531 .await?;
3532 bind_root_id = Some(replacement_root);
3533 }
3534 if pending_binds.contains_key(&route_id) {
3535 return send_route_bind_error(
3536 tx,
3537 frame,
3538 "config_divergence",
3539 "route bind is already pending for channel",
3540 metrics,
3541 )
3542 .await;
3543 }
3544 let bind_root_id = match bind_root_id {
3545 Some(root_id) => root_id,
3546 None => match ProjectRootId::from_path(&identity.project_root) {
3547 Ok(root_id) => root_id,
3548 Err(error) => {
3549 return send_route_bind_error(
3550 tx,
3551 frame,
3552 "config_divergence",
3553 &format!("invalid route project root: {error}"),
3554 metrics,
3555 )
3556 .await;
3557 }
3558 },
3559 };
3560
3561 let request_id = format!("subc-bind-{route_channel}");
3564 let bind_project_root = identity.project_root.clone();
3565 let bind_harness = identity.harness.clone();
3566 let bind_session = identity.session.clone();
3567 let bind_trust = trust_for_bind(&bind_harness, &principal);
3568 let bind_principal_id = principal_id(&principal);
3569 let consumer_elicitation_capable = consumer_capabilities
3574 .as_ref()
3575 .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
3576 log::info!(
3577 "subc attach: route {} harness={} principal={} trust={} elicitation={}",
3578 route_channel,
3579 bind_harness,
3580 principal_label(&principal),
3581 bind_trust.label(),
3582 consumer_elicitation_capable
3583 );
3584
3585 let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
3597 user_config_path,
3598 Path::new(&bind_project_root),
3599 );
3600 let config_tiers: Vec<Value> = local_tiers
3601 .iter()
3602 .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
3603 .collect();
3604 let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
3605 let configure_json = json!({
3606 "id": request_id,
3607 "command": "configure",
3608 "project_root": bind_project_root,
3609 "harness": bind_harness,
3610 "session_id": bind_session.clone(),
3611 "config": config_tiers,
3612 });
3613 let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
3614 Ok(req) => req,
3615 Err(error) => {
3616 return send_route_bind_error(
3617 tx,
3618 frame,
3619 "config_divergence",
3620 &format!("failed to build configure request: {error}"),
3621 metrics,
3622 )
3623 .await;
3624 }
3625 };
3626
3627 let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
3628 root: bind_root_id.clone(),
3629 project_root: PathBuf::from(&bind_project_root),
3630 harness: bind_harness.clone(),
3631 session: bind_session.clone(),
3632 trust: bind_trust,
3633 spawn_principal: AuthenticatedPrincipal::RouteBind {
3634 trust: bind_trust.sandbox_trust(),
3635 route_channel,
3636 route_epoch: epoch,
3637 project_root: PathBuf::from(&bind_project_root),
3638 harness: bind_harness.clone(),
3639 session_id: bind_session.clone(),
3640 principal_id: bind_principal_id,
3641 },
3642 consumer_elicitation_capable,
3643 }));
3644 let configure_session = route_identity.session.clone();
3645 let root_was_live = live_roots.contains_key(&bind_root_id);
3646 let inserted_new_actor = register_actor_for_bind(
3647 shared_app,
3648 executor,
3649 push_senders,
3650 &bind_root_id,
3651 route_channel,
3652 root_was_live,
3653 );
3654
3655 let configure_request_id = configure_req.id.clone();
3656 installed_route_epochs.insert(route_channel, epoch);
3657 if let Some(meta) = live_roots.get_mut(&bind_root_id) {
3658 meta.maintenance_queued_kinds.clear();
3659 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
3660 }
3661 let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
3662 bind_root_id.clone(),
3663 Lane::Mutating,
3664 configure_request_id.clone(),
3665 Box::new(move |ctx| {
3666 log_ctx::with_session(Some(configure_session.clone()), || {
3667 dispatch(configure_req, ctx)
3668 })
3669 }),
3670 );
3671 pending_binds.insert(
3672 route_id,
3673 PendingBind {
3674 bind_root_id: bind_root_id.clone(),
3675 inserted_new_actor,
3676 cancelled: false,
3677 configure_request_id: configure_request_id.clone(),
3678 started_at: Instant::now(),
3679 warned_half_deadline: false,
3680 deadline_reported: false,
3681 corr: frame.header.corr,
3682 ver: frame.header.ver,
3683 flags: frame.header.flags,
3684 cancellation: configure_cancellation,
3685 },
3686 );
3687
3688 let completion_tx = control_completion_tx.clone();
3689 let completion_identity = route_identity;
3690 let completion_root = bind_root_id.clone();
3691 let completion_route_channel = route_channel;
3692 let completion_ver = frame.header.ver;
3693 let completion_corr = frame.header.corr;
3694 let completion_flags = frame.header.flags;
3695 let completion_metrics = Arc::clone(metrics);
3696 tokio::spawn(async move {
3697 let _response_task = ResponseTaskGuard::new(&completion_metrics);
3698 let configure_response =
3699 await_executor_response(configure_rx, configure_request_id.clone()).await;
3700 let completion = RouteBindCompletion {
3705 route: route_key(completion_route_channel, epoch),
3706 identity: completion_identity,
3707 bind_root_id: completion_root,
3708 inserted_new_actor,
3709 configure_response,
3710 diagnostics_on_edit,
3711 ver: completion_ver,
3712 corr: completion_corr,
3713 flags: completion_flags,
3714 };
3715 if send_counted_channel(
3716 &completion_tx,
3717 &completion_metrics.control_completion_queued,
3718 completion,
3719 )
3720 .await
3721 .is_err()
3722 {
3723 log::debug!(
3724 "subc attach: dropped RouteBind completion for route {} after loop exit",
3725 completion_route_channel
3726 );
3727 }
3728 });
3729
3730 Ok(())
3731 }
3732 ModuleControlRequest::HealthCheck {} => {
3733 let report = build_health_report(executor, pending_binds, metrics, shared_app);
3734 let body = serde_json::to_vec(&ModuleControlResponse::from(report))
3735 .map_err(SubcError::Json)?;
3736 let response = Frame::build_with_version(
3737 frame.header.ver,
3738 FrameType::Response,
3739 frame.header.flags,
3740 0,
3741 0,
3742 frame.header.corr,
3743 body,
3744 )
3745 .map_err(SubcError::FrameBuild)?;
3746 send_frame(tx, metrics, response).await
3747 }
3748 }
3749}
3750
3751fn install_bash_compressor(ctx: &AppContext) {
3752 let filter_registry_handle = ctx.shared_filter_registry();
3754 let compress_flag = ctx.bash_compress_flag();
3755 ctx.bash_background().set_compressor_with_exit_code(
3756 move |command: &str, output: String, exit_code: Option<i32>| {
3757 if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
3758 return crate::compress::CompressionResult::new(output);
3759 }
3760 let registry_guard = match filter_registry_handle.read() {
3761 Ok(g) => g,
3762 Err(poisoned) => poisoned.into_inner(),
3763 };
3764 crate::compress::compress_with_registry_exit_code(
3765 command,
3766 &output,
3767 exit_code,
3768 ®istry_guard,
3769 )
3770 },
3771 );
3772}
3773
3774fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
3775 let mut diagnostics_on_edit = false;
3776 for tier in tiers {
3777 if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
3778 diagnostics_on_edit = value;
3779 }
3780 }
3781 diagnostics_on_edit
3782}
3783
3784fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
3785 let stripped = strip_jsonc(doc);
3786 let value = serde_json::from_str::<Value>(&stripped).ok()?;
3787 value
3788 .get("lsp")
3789 .and_then(Value::as_object)?
3790 .get("diagnostics_on_edit")
3791 .and_then(Value::as_bool)
3792}
3793
3794async fn send_route_bind_error(
3795 tx: &WriterSender,
3796 frame: &Frame,
3797 code: &str,
3798 message: &str,
3799 metrics: &DispatchPathMetrics,
3800) -> Result<(), SubcError> {
3801 send_route_bind_error_parts(
3802 tx,
3803 frame.header.ver,
3804 frame.header.corr,
3805 frame.header.flags,
3806 code,
3807 message,
3808 metrics,
3809 )
3810 .await
3811}
3812
3813async fn send_route_bind_error_parts(
3814 tx: &WriterSender,
3815 ver: u8,
3816 corr: u64,
3817 flags: Flags,
3818 code: &str,
3819 message: &str,
3820 metrics: &DispatchPathMetrics,
3821) -> Result<(), SubcError> {
3822 let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
3823 send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
3824 log::warn!("subc attach: route bind rejected ({code}): {message}");
3825 Ok(())
3826}
3827
3828async fn handle_tool_call(
3833 tx: &WriterSender,
3834 frame: &Frame,
3835 mut phase_trace: PhaseTrace,
3836 routes: &HashMap<RouteChannel, RouteIdentity>,
3837 pending_binds: &HashMap<RouteChannel, PendingBind>,
3838 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3839 executor: &Arc<Executor>,
3840 shutdown: &Arc<Notify>,
3841 connection_cancel: &PersistentCancelSignal,
3842 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
3843 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
3844 metrics: &Arc<DispatchPathMetrics>,
3845 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3846 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3847 next_bash_ask_corr: &mut u64,
3848 bg_subs: &mut HashMap<RouteChannel, BgSub>,
3849 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3850 bg_wake_pending: &mut HashSet<RouteChannel>,
3851 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
3852 dispatch: DispatchFn,
3853 allow_native_passthrough: bool,
3854) -> Result<(), SubcError> {
3855 let route_id = route_key(frame.header.channel, frame.header.epoch);
3856 if pending_binds.contains_key(&route_id) {
3857 let error = build_error_frame(
3858 frame.header.ver,
3859 frame.header.channel,
3860 frame.header.epoch,
3861 frame.header.corr,
3862 frame.header.flags,
3863 "route_not_bound",
3864 "route is not bound before tool call",
3865 )?;
3866 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3867 }
3868
3869 let Some(identity) = routes.get(&route_id).cloned() else {
3870 let error = build_error_frame(
3871 frame.header.ver,
3872 frame.header.channel,
3873 frame.header.epoch,
3874 frame.header.corr,
3875 frame.header.flags,
3876 "route_not_bound",
3877 "route is not bound before tool call",
3878 )?;
3879 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3880 };
3881 let restore_watcher = live_roots
3882 .get(&identity.root)
3883 .is_some_and(|meta| meta.idle_artifacts_evicted);
3884 if let Some(meta) = live_roots.get_mut(&identity.root) {
3885 meta.reactivate_bound();
3886 }
3887 if restore_watcher {
3888 if let Some(ctx) = executor.actor_context(&identity.root) {
3889 crate::commands::configure::ensure_project_watcher(&ctx);
3890 }
3891 }
3892
3893 let route_request =
3894 serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
3895 if matches!(
3896 route_request,
3897 RouteRequest::BgEvents(BgEventsRequest {
3898 op: BgEventsOp::BgEvents
3899 })
3900 ) {
3901 if let Some(old_sub) = bg_subs.get(&route_id).copied() {
3902 push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
3903 }
3904 if !identity.trust.allows_bash_observation() {
3905 bg_subs.remove(&route_id);
3906 bg_wake_pending.remove(&route_id);
3907 remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
3908 let denied_sub = BgSub {
3909 corr: frame.header.corr,
3910 ver: frame.header.ver,
3911 flags: frame.header.flags,
3912 };
3913 push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
3914 return Ok(());
3915 }
3916 bg_subs.insert(
3917 route_id,
3918 BgSub {
3919 corr: frame.header.corr,
3920 ver: frame.header.ver,
3921 flags: frame.header.flags,
3922 },
3923 );
3924 bg_sub_by_session.insert((identity.root.clone(), identity.session.clone()), route_id);
3925 push::arm_bg_wake(
3926 identity.root.clone(),
3927 identity.session.clone(),
3928 route_id,
3929 bg_wake_pending,
3930 bg_wake_epoch,
3931 );
3932 return Ok(());
3933 }
3934
3935 let RouteRequest::ToolCall(call) = route_request else {
3936 unreachable!("background event subscription returned above")
3937 };
3938 let bare_name = call.name;
3939 let arguments = strip_agent_preview_arg_owned(call.arguments);
3940 let format_context = crate::subc_format::FormatContext::from_tool_call(
3941 &bare_name,
3942 &arguments,
3943 identity.project_root.as_path(),
3944 );
3945
3946 let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
3947 let bind_trust = identity.trust;
3948 let diagnostics_on_edit = live_roots
3949 .get(&identity.root)
3950 .map(|meta| meta.diagnostics_on_edit)
3951 .unwrap_or(false);
3952
3953 let requests_host = bare_name == "bash"
3954 && arguments
3955 .get("sandbox")
3956 .or_else(|| {
3957 arguments
3958 .get("params")
3959 .and_then(|params| params.get("sandbox"))
3960 })
3961 .and_then(Value::as_str)
3962 == Some("host");
3963 if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
3964 let response = Response::error(
3965 request_id.clone(),
3966 "sandbox_escalation_denied",
3967 "sandbox host escalation is unavailable to untrusted principals",
3968 );
3969 let text = crate::subc_format::format_response_with_context(
3970 &bare_name,
3971 &response,
3972 &format_context,
3973 );
3974 let result = ToolCallResult { text, response };
3975 let response_frame = build_tool_response_frame(
3976 frame.header.ver,
3977 route_id,
3978 frame.header.corr,
3979 frame.header.flags,
3980 &result,
3981 bind_trust,
3982 )?;
3983 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3984 }
3985
3986 if matches!(bind_trust, BindTrust::Untrusted)
3987 && is_bash_family_tool(&bare_name)
3988 && (bare_name != "bash" || !identity.consumer_elicitation_capable)
3989 {
3990 let response = bash::bash_denied_untrusted_response(request_id.clone());
3991 let text = crate::subc_format::format_response_with_context(
3992 &bare_name,
3993 &response,
3994 &format_context,
3995 );
3996 let result = ToolCallResult { text, response };
3997 let response_frame = build_tool_response_frame(
3998 frame.header.ver,
3999 route_id,
4000 frame.header.corr,
4001 frame.header.flags,
4002 &result,
4003 bind_trust,
4004 )?;
4005 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4006 }
4007
4008 if !is_subc_agent_core_tool(&bare_name)
4016 && !is_subc_native_plumbing_tool(&bare_name)
4017 && !allow_native_passthrough
4018 {
4019 log::warn!(
4020 "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
4021 bare_name,
4022 frame.header.channel
4023 );
4024 let response = Response::error(
4025 request_id.clone(),
4026 "unknown_tool",
4027 format!("tool {:?} is not in the AFT tool manifest", bare_name),
4028 );
4029 let text = crate::subc_format::format_response_with_context(
4030 &bare_name,
4031 &response,
4032 &format_context,
4033 );
4034 let result = ToolCallResult { text, response };
4035 let response_frame = build_tool_response_frame(
4036 frame.header.ver,
4037 route_id,
4038 frame.header.corr,
4039 frame.header.flags,
4040 &result,
4041 bind_trust,
4042 )?;
4043 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4044 }
4045
4046 if bare_name == "bash" {
4047 if matches!(bind_trust, BindTrust::Untrusted) {
4048 let plan = match bash::prepare_bash_elicitation_plan(
4049 &arguments,
4050 identity.project_root.as_path(),
4051 ) {
4052 Ok(plan) => plan,
4053 Err(error) => {
4054 let response = Response::error(request_id.clone(), error.code, error.message);
4055 let text = crate::subc_format::format_response_with_context(
4056 &bare_name,
4057 &response,
4058 &format_context,
4059 );
4060 let result = ToolCallResult { text, response };
4061 let response_frame = build_tool_response_frame(
4062 frame.header.ver,
4063 route_id,
4064 frame.header.corr,
4065 frame.header.flags,
4066 &result,
4067 bind_trust,
4068 )?;
4069 return send_reliable_writer_frame(
4070 tx,
4071 metrics,
4072 response_frame,
4073 "tool response",
4074 )
4075 .await;
4076 }
4077 };
4078
4079 let reverse_corr =
4080 allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
4081 let ask_frame = build_bash_elicitation_request_frame(
4082 frame.header.ver,
4083 route_id,
4084 reverse_corr,
4085 frame.header.flags,
4086 &plan.command,
4087 &plan.asks,
4088 )?;
4089
4090 let meta = live_roots
4091 .entry(identity.root.clone())
4092 .or_insert_with(|| RootMeta::new(Instant::now()));
4093 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4094 meta.reactivate_bound();
4095
4096 let route_cancel =
4097 route_bash_cancels
4098 .entry(route_id)
4099 .or_insert_with(|| bash::RouteBashCancel {
4100 token: PersistentCancelSignal::new(),
4101 active_waits: 0,
4102 });
4103 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4104 let cancel = bash::BashWaitCancel {
4105 connection: connection_cancel.clone(),
4106 route: route_cancel.token.clone(),
4107 };
4108 pending_bash_asks.insert(
4109 ReverseCorrKey {
4110 route: route_id,
4111 corr: reverse_corr,
4112 },
4113 PendingBashAsk {
4114 route: route_id,
4115 tool_corr: frame.header.corr,
4116 tool_flags: frame.header.flags,
4117 tool_ver: frame.header.ver,
4118 root: identity.root.clone(),
4119 project_root: identity.project_root.clone(),
4120 session_id: identity.session.clone(),
4121 spawn_principal: identity.spawn_principal.clone(),
4122 request_id,
4123 arguments,
4124 format_context,
4125 cancel,
4126 grants: plan.grants,
4127 expires_at: Instant::now() + bash_elicitation_timeout(),
4128 },
4129 );
4130 return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
4131 .await;
4132 }
4133
4134 let meta = live_roots
4135 .entry(identity.root.clone())
4136 .or_insert_with(|| RootMeta::new(Instant::now()));
4137 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4138 meta.reactivate_bound();
4139
4140 let route_cancel =
4141 route_bash_cancels
4142 .entry(route_id)
4143 .or_insert_with(|| bash::RouteBashCancel {
4144 token: PersistentCancelSignal::new(),
4145 active_waits: 0,
4146 });
4147 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4148 let cancel = bash::BashWaitCancel {
4149 connection: connection_cancel.clone(),
4150 route: route_cancel.token.clone(),
4151 };
4152
4153 bash::submit_deferred_bash(
4154 executor,
4155 bash_deferred_tx,
4156 bash_poll_touch_tx,
4157 metrics,
4158 dispatch,
4159 identity.root.clone(),
4160 identity.project_root.clone(),
4161 identity.session.clone(),
4162 request_id,
4163 route_id,
4164 frame.header.corr,
4165 frame.header.flags,
4166 frame.header.ver,
4167 arguments,
4168 format_context,
4169 cancel,
4170 bind_trust,
4171 identity.spawn_principal.clone(),
4172 None,
4173 );
4174 return Ok(());
4175 }
4176
4177 let lane = command_lane(&bare_name);
4178 let tool_call_context = ToolCallContext {
4179 project_root: identity.project_root.clone(),
4180 session_id: Some(identity.session.clone()),
4181 request_id: request_id.clone(),
4182 diagnostics_on_edit,
4183 preview: call.preview,
4184 };
4185 let bare_name_for_frame = bare_name.clone();
4186 let identity_for_run = identity.clone();
4187 let completion_session = identity.session.clone();
4188 let completion_root = identity.project_root.clone();
4189 let request_id_for_force = request_id.clone();
4190 let format_context_for_frame = format_context.clone();
4191 let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
4192 phase_trace.mark_executor_submitted();
4193 let rx = executor.submit_async(
4194 identity.root.clone(),
4195 lane,
4196 request_id.clone(),
4197 Box::new(move |ctx| {
4198 phase_trace.mark_job_admitted();
4199 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4200 let run = || {
4201 let finalizer = |response: &mut Response| {
4202 crate::response_finalize::finalize_response_with_bg_completions(
4203 response,
4204 ctx,
4205 &identity_for_run.session,
4206 &bare_name,
4207 bind_trust.allows_bash_observation(),
4208 );
4209 };
4210 match run_tool_call(
4211 &bare_name,
4212 arguments,
4213 &format_context,
4214 &tool_call_context,
4215 ctx,
4216 &dispatch,
4217 Some(&finalizer),
4218 Some(&mut phase_trace),
4219 ) {
4220 ToolCallOutcome::Unary(result) => {
4221 let response = result.response;
4222 let _ = tool_call_tx.send(ToolCallCompletion {
4223 text: result.text,
4224 phase_trace,
4225 });
4226 response
4227 }
4228 }
4229 };
4230 if matches!(bind_trust, BindTrust::Untrusted) {
4231 ctx.with_force_restrict(&request_id_for_force, run)
4232 } else {
4233 run()
4234 }
4235 })
4236 }),
4237 );
4238 let completion_tx = tx.clone();
4239 let completion_shutdown = Arc::clone(shutdown);
4240 let route = route_id;
4241 let corr = frame.header.corr;
4242 let flags = frame.header.flags;
4243 let ver = frame.header.ver;
4244 let completion_metrics = Arc::clone(metrics);
4245 tokio::spawn(async move {
4246 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4247 let response = await_executor_response(rx, request_id.clone()).await;
4248 let (text, phase_trace) = match tool_call_rx.await {
4249 Ok(completion) => (completion.text, Some(completion.phase_trace)),
4250 Err(_) => (
4251 crate::subc_format::format_response_with_context(
4252 &bare_name_for_frame,
4253 &response,
4254 &format_context_for_frame,
4255 ),
4256 None,
4257 ),
4258 };
4259 let result = ToolCallResult { text, response };
4260 let fatal = response_is_fatal_panic(&result.response);
4261 match build_tool_response_frame(ver, route, corr, flags, &result, bind_trust) {
4262 Ok(response_frame) => {
4263 let send_result = if let Some(phase_trace) = phase_trace {
4264 let trace = ToolResponseWriteTrace::new(
4265 phase_trace,
4266 bare_name_for_frame,
4267 completion_root,
4268 completion_session,
4269 route.channel,
4270 corr,
4271 );
4272 send_traced_tool_response_frame(
4273 &completion_tx,
4274 &completion_metrics,
4275 response_frame,
4276 trace,
4277 )
4278 .await
4279 } else {
4280 send_reliable_writer_frame(
4281 &completion_tx,
4282 &completion_metrics,
4283 response_frame,
4284 "tool response",
4285 )
4286 .await
4287 };
4288 if let Err(error) = send_result {
4289 log::warn!("subc attach: failed to queue tool response frame: {error}");
4290 }
4291 }
4292 Err(error) => {
4293 log::error!("subc attach: failed to build tool response frame: {error}");
4294 }
4295 }
4296 if fatal {
4297 signal_fatal_teardown(
4298 &completion_tx,
4299 Some(route),
4300 ver,
4301 corr,
4302 &completion_shutdown,
4303 &completion_metrics,
4304 )
4305 .await;
4306 }
4307 });
4308 Ok(())
4309}
4310
4311fn submit_maintenance_job(
4312 executor: &Arc<Executor>,
4313 root_id: ProjectRootId,
4314 kind: MaintenanceDrainKind,
4315 bg_sessions_to_check: Vec<(String, u64)>,
4316 completion_tx: &mpsc::Sender<MaintenanceCompletion>,
4317 metrics: &Arc<DispatchPathMetrics>,
4318) {
4319 let request_id = format!(
4320 "subc-maintenance-drain-{}-{}",
4321 kind.label(),
4322 root_id.as_path().to_string_lossy()
4323 );
4324 let response_id = request_id.clone();
4325 let completion_root_id = root_id.clone();
4326 let maintenance_generation = executor
4327 .actor_context(&root_id)
4328 .map(|ctx| ctx.configure_generation())
4329 .unwrap_or(0);
4330 let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
4331 let lane = match kind {
4336 MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
4337 MaintenanceDrainKind::Watcher
4338 | MaintenanceDrainKind::Lsp
4339 | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
4340 };
4341 let rx = executor.submit_maintenance_async(
4342 root_id,
4343 lane,
4344 request_id.clone(),
4345 Box::new(move |ctx| {
4346 let outcome = match kind {
4347 MaintenanceDrainKind::Watcher => {
4348 let drained = runtime_drain::drain_watcher_events_bounded(
4349 ctx,
4350 runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
4351 );
4352 MaintenanceJobOutcome {
4353 empty_bg_sessions: Vec::new(),
4354 requeue_kind: drained.has_more.then_some(kind),
4355 }
4356 }
4357 MaintenanceDrainKind::Lsp => {
4358 let drained = runtime_drain::drain_lsp_events_bounded(
4359 ctx,
4360 runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
4361 );
4362 MaintenanceJobOutcome {
4363 empty_bg_sessions: Vec::new(),
4364 requeue_kind: drained.has_more.then_some(kind),
4365 }
4366 }
4367 MaintenanceDrainKind::ConfigureTail => {
4368 runtime_drain::drain_deferred_configure_maintenance(ctx);
4369 runtime_drain::drain_configure_warning_events(ctx);
4370 MaintenanceJobOutcome::default()
4371 }
4372 MaintenanceDrainKind::CompletionDrains => {
4373 runtime_drain::drain_search_index_events(ctx);
4374 runtime_drain::drain_callgraph_store_events(ctx);
4375 runtime_drain::drain_semantic_index_events(ctx);
4376 runtime_drain::drain_semantic_refresh_events(ctx);
4377 runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
4378 let empty_bg_sessions = bg_sessions_to_check
4379 .into_iter()
4380 .filter(|(session, _)| {
4381 !ctx.bash_background()
4382 .has_completions_for_session(Some(session.as_str()))
4383 })
4384 .collect();
4385 MaintenanceJobOutcome {
4386 empty_bg_sessions,
4387 requeue_kind: None,
4388 }
4389 }
4390 };
4391 let requeued = outcome.requeue_kind.is_some();
4392 let _ = outcome_tx.send(outcome);
4393 Response::success(
4394 response_id,
4395 json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
4396 )
4397 }),
4398 );
4399 let completion_tx = completion_tx.clone();
4400 let completion_metrics = Arc::clone(metrics);
4401 tokio::spawn(async move {
4402 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4403 let response = await_executor_response(rx, request_id).await;
4404 let outcome = outcome_rx.await.unwrap_or_default();
4405 let _ = send_counted_channel(
4406 &completion_tx,
4407 &completion_metrics.maintenance_queued,
4408 MaintenanceCompletion {
4409 root_id: completion_root_id,
4410 kind,
4411 response,
4412 empty_bg_sessions: outcome.empty_bg_sessions,
4413 requeue_kind: outcome.requeue_kind,
4414 },
4415 )
4416 .await;
4417 });
4418}
4419
4420async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
4421 rx.await
4422 .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
4423}
4424async fn signal_fatal_teardown(
4425 tx: &WriterSender,
4426 route: Option<RouteChannel>,
4427 ver: u8,
4428 corr: u64,
4429 shutdown: &Arc<Notify>,
4430 metrics: &DispatchPathMetrics,
4431) {
4432 if let Some(route) = route {
4433 if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
4434 if let Err(error) = send_frame(tx, metrics, frame).await {
4435 log::warn!(
4436 "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
4437 );
4438 }
4439 }
4440 }
4441 if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
4442 if let Err(error) = send_frame(tx, metrics, frame).await {
4443 log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
4444 }
4445 }
4446 shutdown.notify_one();
4447}
4448#[derive(Debug, Deserialize)]
4449#[serde(untagged)]
4450enum RouteRequest {
4451 BgEvents(BgEventsRequest),
4452 ToolCall(ToolCallRequest),
4453}
4454
4455#[derive(Debug, Deserialize)]
4456struct BgEventsRequest {
4457 op: BgEventsOp,
4458}
4459
4460#[derive(Debug, Deserialize)]
4461#[serde(rename_all = "snake_case")]
4462enum BgEventsOp {
4463 BgEvents,
4464}
4465
4466#[derive(Debug, Deserialize)]
4467struct ToolCallRequest {
4468 name: String,
4469 #[serde(default)]
4470 arguments: Value,
4471 #[serde(default)]
4476 preview: bool,
4477}
4478
4479#[cfg(test)]
4480pub(crate) mod test_support {
4481 use super::*;
4482 use crate::bash_background::BgTaskStatus;
4483 use crate::protocol::{
4484 BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
4485 ProgressFrame, StatusChangedFrame,
4486 };
4487 use serde_json::json;
4488
4489 pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
4490 let dir = tempfile::Builder::new()
4491 .prefix(name)
4492 .tempdir()
4493 .expect("temp root");
4494 let root = ProjectRootId::from_path(dir.path()).expect("project root id");
4495 (dir, root)
4496 }
4497
4498 pub(super) fn test_ctx() -> Arc<AppContext> {
4499 Arc::new(AppContext::new(
4500 Box::new(crate::parser::TreeSitterProvider::new()),
4501 crate::config::Config::default(),
4502 ))
4503 }
4504
4505 pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
4506 let deadline = Instant::now() + Duration::from_secs(30);
4507 loop {
4508 let observed = ctx.watcher_registry_count();
4509 if observed == expected {
4510 return;
4511 }
4512 assert!(
4513 Instant::now() < deadline,
4514 "watcher count did not settle before deadline: expected={expected}, observed={observed}"
4515 );
4516 std::thread::sleep(Duration::from_millis(50));
4517 }
4518 }
4519
4520 pub(super) fn reap_until_forgotten(
4530 root: &ProjectRootId,
4531 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4532 pending_binds: &HashMap<RouteChannel, PendingBind>,
4533 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4534 executor: &Arc<Executor>,
4535 metrics: &DispatchPathMetrics,
4536 ) -> IdleReapOutcome {
4537 let deadline = Instant::now() + Duration::from_secs(30);
4538 loop {
4539 let outcome = reap_idle_roots(
4540 Instant::now(),
4541 live_roots,
4542 pending_binds,
4543 root_channels,
4544 executor,
4545 metrics,
4546 );
4547 if outcome.forgotten_deleted_roots.contains(root) {
4548 return outcome;
4549 }
4550 assert!(
4551 Instant::now() < deadline,
4552 "deleted root was never forgotten: {root:?}"
4553 );
4554 std::thread::sleep(Duration::from_millis(10));
4555 }
4556 }
4557
4558 pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
4559 let deadline = Instant::now() + Duration::from_secs(30);
4560 loop {
4561 let observed = app.actor_root_count();
4562 if observed == expected {
4563 return;
4564 }
4565 assert!(
4566 Instant::now() < deadline,
4567 "actor root count did not settle before deadline: expected={expected}, observed={observed}"
4568 );
4569 std::thread::sleep(Duration::from_millis(50));
4570 }
4571 }
4572
4573 pub(super) fn status_frame(seq: u64) -> PushFrame {
4574 status_frame_with_session(seq, None)
4575 }
4576
4577 pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
4578 PushFrame::StatusChanged(StatusChangedFrame {
4579 frame_type: "status_changed",
4580 session_id: session_id.map(str::to_string),
4581 snapshot: json!({ "seq": seq }),
4582 })
4583 }
4584
4585 pub(super) fn completion_frame(task_id: &str) -> PushFrame {
4586 completion_frame_with_session(task_id, "session-1")
4587 }
4588
4589 pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
4590 PushFrame::BashCompleted(BashCompletedFrame {
4591 frame_type: "bash_completed",
4592 task_id: task_id.to_string(),
4593 session_id: session_id.to_string(),
4594 status: BgTaskStatus::Completed,
4595 exit_code: Some(0),
4596 command: format!("echo {task_id}"),
4597 output_preview: String::new(),
4598 output_truncated: false,
4599 original_tokens: None,
4600 compressed_tokens: None,
4601 tokens_skipped: false,
4602 })
4603 }
4604
4605 pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
4606 long_running_frame_with_session(task_id, "session-1", elapsed_ms)
4607 }
4608
4609 pub(super) fn long_running_frame_with_session(
4610 task_id: &str,
4611 session_id: &str,
4612 elapsed_ms: u64,
4613 ) -> PushFrame {
4614 PushFrame::BashLongRunning(BashLongRunningFrame {
4615 frame_type: "bash_long_running",
4616 task_id: task_id.to_string(),
4617 session_id: session_id.to_string(),
4618 command: format!("sleep {elapsed_ms}"),
4619 elapsed_ms,
4620 })
4621 }
4622
4623 pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
4624 PushFrame::BashPatternMatch(BashPatternMatchFrame {
4625 frame_type: "bash_pattern_match",
4626 task_id: "task-pattern".to_string(),
4627 session_id: session_id.to_string(),
4628 watch_id: "watch-1".to_string(),
4629 match_text: "needle".to_string(),
4630 match_offset: 7,
4631 context: "haystack needle".to_string(),
4632 once: true,
4633 reason: "pattern_match",
4634 })
4635 }
4636
4637 pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
4638 PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
4639 frame_type: "configure_warnings",
4640 session_id: session_id.map(str::to_string),
4641 project_root: "/tmp/subc-test".to_string(),
4642 warnings: Vec::new(),
4643 })
4644 }
4645
4646 pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
4647 route_identity_with_trust(root, session_id, BindTrust::FirstParty)
4648 }
4649
4650 pub(super) fn route_identity_with_trust(
4651 root: &ProjectRootId,
4652 session_id: &str,
4653 trust: BindTrust,
4654 ) -> RouteIdentity {
4655 RouteIdentity(Arc::new(RouteIdentityData {
4656 root: root.clone(),
4657 project_root: root.as_path().to_path_buf(),
4658 harness: "opencode".to_string(),
4659 session: session_id.to_string(),
4660 trust,
4661 spawn_principal: AuthenticatedPrincipal::RouteBind {
4662 trust: trust.sandbox_trust(),
4663 route_channel: 0,
4664 route_epoch: 0,
4665 project_root: root.as_path().to_path_buf(),
4666 harness: "opencode".to_string(),
4667 session_id: session_id.to_string(),
4668 principal_id: Some(match trust {
4669 BindTrust::FirstParty => "direct".to_string(),
4670 BindTrust::Untrusted => "unverified".to_string(),
4671 }),
4672 },
4673 consumer_elicitation_capable: false,
4674 }))
4675 }
4676
4677 pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
4678 PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
4679 }
4680
4681 pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
4682 match frame {
4683 PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
4684 _ => None,
4685 }
4686 }
4687
4688 pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
4689 match frame {
4690 PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
4691 _ => None,
4692 }
4693 }
4694
4695 pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
4696 let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
4697 body.get("task_id")
4698 .and_then(serde_json::Value::as_str)
4699 .map(str::to_string)
4700 }
4701}
4702
4703#[cfg(test)]
4704mod tests {
4705 use super::test_support::{
4706 completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
4707 wait_for_actor_root_count, wait_for_watcher_count,
4708 };
4709 use super::*;
4710
4711 fn attach_error(kind: io::ErrorKind) -> SubcError {
4712 SubcError::Connect {
4713 endpoint: "127.0.0.1:1".to_string(),
4714 source: io::Error::new(kind, "constructed attach failure"),
4715 }
4716 }
4717
4718 fn auth_io_error(kind: io::ErrorKind) -> SubcError {
4719 SubcError::Auth {
4720 endpoint: "127.0.0.1:1".to_string(),
4721 source: subc_transport::AuthError::Io {
4722 stage: subc_transport::AuthStage::ServerProof,
4723 source: io::Error::new(kind, "constructed auth failure"),
4724 },
4725 }
4726 }
4727
4728 #[test]
4729 fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
4730 let transient_errors = vec![
4731 attach_error(io::ErrorKind::ConnectionRefused),
4732 attach_error(io::ErrorKind::TimedOut),
4733 attach_error(io::ErrorKind::ConnectionReset),
4734 auth_io_error(io::ErrorKind::ConnectionAborted),
4735 auth_io_error(io::ErrorKind::BrokenPipe),
4736 SubcError::Auth {
4737 endpoint: "127.0.0.1:1".to_string(),
4738 source: subc_transport::AuthError::UnexpectedEof {
4739 stage: subc_transport::AuthStage::ServerProof,
4740 expected: 4,
4741 actual: 0,
4742 },
4743 },
4744 SubcError::Auth {
4745 endpoint: "127.0.0.1:1".to_string(),
4746 source: subc_transport::AuthError::Timeout {
4747 stage: subc_transport::AuthStage::ServerProof,
4748 deadline: AUTH_DEADLINE,
4749 },
4750 },
4751 ];
4752 for error in &transient_errors {
4753 assert_eq!(
4754 classify_attach_error(error),
4755 AttachErrorClass::Transient,
4756 "expected transient: {error}"
4757 );
4758 }
4759
4760 let permanent_errors = vec![
4761 attach_error(io::ErrorKind::PermissionDenied),
4762 auth_io_error(io::ErrorKind::InvalidData),
4763 SubcError::Auth {
4764 endpoint: "127.0.0.1:1".to_string(),
4765 source: subc_transport::AuthError::InvalidServerProof,
4766 },
4767 SubcError::Auth {
4768 endpoint: "127.0.0.1:1".to_string(),
4769 source: subc_transport::AuthError::DaemonIdMismatch,
4770 },
4771 SubcError::ConnectionFile {
4772 path: PathBuf::from("subc-connection.json"),
4773 source: subc_transport::ConnectionFileError::Invalid {
4774 reason: "constructed invalid file".to_string(),
4775 },
4776 },
4777 SubcError::NoEndpoint {
4778 path: PathBuf::from("subc-connection.json"),
4779 },
4780 SubcError::InvalidEndpoint {
4781 path: PathBuf::from("subc-connection.json"),
4782 endpoint: "not-an-ip:1234".to_string(),
4783 },
4784 ];
4785 for error in &permanent_errors {
4786 assert_eq!(
4787 classify_attach_error(error),
4788 AttachErrorClass::Permanent,
4789 "expected permanent: {error}"
4790 );
4791 }
4792 }
4793
4794 #[test]
4795 fn incompatible_wire_version_is_rejected_before_tcp_connect() {
4796 let conn_dir = tempfile::tempdir().expect("connection tempdir");
4797 let conn_path = conn_dir.path().join("subc-connection.json");
4798 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
4799 listener
4800 .set_nonblocking(true)
4801 .expect("set listener nonblocking");
4802 let port = listener.local_addr().expect("listener addr").port();
4803 connection_file::write_atomic(
4804 &conn_path,
4805 &connection_file::ConnectionInfo {
4806 schema: connection_file::SCHEMA_VERSION,
4807 wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
4808 endpoints: vec![connection_file::Endpoint {
4809 host: "127.0.0.1".to_string(),
4810 port,
4811 }],
4812 key: vec![0x42; subc_transport::KEY_LEN],
4813 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4814 pid: std::process::id(),
4815 daemon_ver: "subc-test".to_string(),
4816 },
4817 )
4818 .expect("write connection file");
4819
4820 let runtime = tokio::runtime::Builder::new_current_thread()
4821 .enable_all()
4822 .build()
4823 .expect("test runtime");
4824 let result = runtime.block_on(connect_and_authenticate_with_policy(
4825 &conn_path,
4826 AttachRetryPolicy {
4827 budget: Duration::from_secs(1),
4828 initial_backoff: Duration::from_millis(5),
4829 max_backoff: Duration::from_millis(10),
4830 jitter_percent: 0,
4831 },
4832 ));
4833 assert!(matches!(
4834 result,
4835 Err(SubcError::ConnectionFile {
4836 source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
4837 ..
4838 })
4839 ));
4840 assert!(matches!(
4841 listener.accept(),
4842 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
4843 ));
4844 }
4845
4846 #[test]
4847 fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
4848 let conn_dir = tempfile::tempdir().expect("connection tempdir");
4849 let conn_path = conn_dir.path().join("subc-connection.json");
4850 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
4851 let port = listener.local_addr().expect("reserved addr").port();
4852 drop(listener);
4853 connection_file::write_atomic(
4854 &conn_path,
4855 &connection_file::ConnectionInfo {
4856 schema: connection_file::SCHEMA_VERSION,
4857 wire_version: Some(PROTOCOL_VERSION),
4858 endpoints: vec![connection_file::Endpoint {
4859 host: "127.0.0.1".to_string(),
4860 port,
4861 }],
4862 key: vec![0x42; subc_transport::KEY_LEN],
4863 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4864 pid: std::process::id(),
4865 daemon_ver: "subc-test".to_string(),
4866 },
4867 )
4868 .expect("write connection file");
4869
4870 let policy = AttachRetryPolicy {
4871 budget: Duration::from_millis(40),
4872 initial_backoff: Duration::from_millis(5),
4873 max_backoff: Duration::from_millis(10),
4874 jitter_percent: 0,
4875 };
4876 let runtime = tokio::runtime::Builder::new_current_thread()
4877 .enable_all()
4878 .build()
4879 .expect("test runtime");
4880 let started_at = Instant::now();
4881 let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
4882 let elapsed = started_at.elapsed();
4883 let error = match result {
4884 Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
4885 Err(error) => error,
4886 };
4887
4888 assert!(matches!(error, SubcError::Connect { .. }), "{error}");
4889 assert!(
4890 elapsed >= Duration::from_millis(35),
4891 "retry budget ended too early: {elapsed:?}"
4892 );
4893 assert!(
4894 elapsed < Duration::from_secs(1),
4895 "retry budget was not bounded: {elapsed:?}"
4896 );
4897 }
4898
4899 fn due_maintenance_jobs_without_actor_context(
4900 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4901 budget: usize,
4902 pending_bind_roots: &HashSet<ProjectRootId>,
4903 ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
4904 due_maintenance_jobs(
4905 live_roots,
4906 None,
4907 &HashMap::new(),
4908 &HashSet::new(),
4909 budget,
4910 pending_bind_roots,
4911 )
4912 }
4913
4914 fn actor_ctx_with_dirty_search_index(
4915 root: &Path,
4916 storage: &Path,
4917 file_name: &str,
4918 old_contents: &str,
4919 new_contents: &str,
4920 ) -> (Arc<AppContext>, PathBuf, PathBuf) {
4921 let file = root.join(file_name);
4922 std::fs::write(&file, old_contents).expect("write source");
4923 let canonical_root = std::fs::canonicalize(root).expect("canonical root");
4924 let ctx = Arc::new(AppContext::new(
4925 Box::new(crate::parser::TreeSitterProvider::new()),
4926 Config {
4927 project_root: Some(root.to_path_buf()),
4928 storage_dir: Some(storage.to_path_buf()),
4929 ..Config::default()
4930 },
4931 ));
4932 ctx.set_canonical_cache_root(canonical_root.clone());
4933
4934 let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
4935 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
4936 let git_head = index.stored_git_head().map(str::to_owned);
4937 index.write_to_disk(&cache_dir, git_head.as_deref());
4938
4939 std::fs::write(&file, new_contents).expect("edit source");
4940 index.update_file(&file);
4941 *ctx.search_index()
4942 .write()
4943 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
4944 (ctx, canonical_root, cache_dir)
4945 }
4946
4947 #[test]
4948 fn graceful_shutdown_flushes_every_actor_search_index() {
4949 let storage = tempfile::tempdir().expect("storage tempdir");
4950 let (root1_dir, root1) = test_root("shutdown-flush-root-1");
4951 let (root2_dir, root2) = test_root("shutdown-flush-root-2");
4952 let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
4953 root1_dir.path(),
4954 storage.path(),
4955 "alpha.txt",
4956 "old actor one token\n",
4957 "new actor one token\n",
4958 );
4959 let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
4960 root2_dir.path(),
4961 storage.path(),
4962 "beta.txt",
4963 "old actor two token\n",
4964 "new actor two token\n",
4965 );
4966
4967 let executor = Executor::new();
4968 assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
4969 assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
4970
4971 flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
4972
4973 let mut restored1 =
4974 crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
4975 .expect("load flushed root one index");
4976 restored1.ready = true;
4977 assert_eq!(
4978 restored1
4979 .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
4980 .matches
4981 .len(),
4982 1,
4983 "graceful subc shutdown should flush the first root's trigram delta"
4984 );
4985
4986 let mut restored2 =
4987 crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
4988 .expect("load flushed root two index");
4989 restored2.ready = true;
4990 assert_eq!(
4991 restored2
4992 .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
4993 .matches
4994 .len(),
4995 1,
4996 "graceful subc shutdown should flush every registered root"
4997 );
4998 }
4999
5000 #[test]
5001 fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
5002 let _ = env_logger::builder().is_test(true).try_init();
5003 let (root_dir, root) = test_root("idle-root-reaper");
5004 let storage = tempfile::tempdir().expect("storage tempdir");
5005 std::fs::write(
5006 root_dir.path().join("main.rs"),
5007 "fn entry() { leaf(); }\nfn leaf() {}\n",
5008 )
5009 .expect("source file");
5010 let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
5011 let app = App::default_shared();
5012 let ctx = Arc::new(AppContext::from_app(
5013 Arc::clone(&app),
5014 Config {
5015 project_root: Some(canonical_root.clone()),
5016 storage_dir: Some(storage.path().to_path_buf()),
5017 callgraph_store: true,
5018 search_index: true,
5019 ..Config::default()
5020 },
5021 ));
5022 ctx.set_canonical_cache_root(canonical_root.clone());
5023 assert!(ctx
5024 .ensure_callgraph_store()
5025 .expect("build callgraph store")
5026 .is_some());
5027
5028 let cache_dir =
5029 crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
5030 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
5031 let git_head = index.stored_git_head().map(str::to_owned);
5032 index.write_to_disk(&cache_dir, git_head.as_deref());
5033 *ctx.search_index()
5034 .write()
5035 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
5036 let seeded_generation =
5039 crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
5040 .expect("seeded artifact generation");
5041 crate::cache_freshness::record_verify_completed(
5042 &canonical_root,
5043 crate::cache_freshness::VerifyArtifact::Search,
5044 Some(seeded_generation),
5045 );
5046 assert!(
5047 matches!(
5048 crate::cache_freshness::warm_verify_plan(
5049 canonical_root.as_path(),
5050 crate::cache_freshness::VerifyArtifact::Search,
5051 Some(seeded_generation),
5052 ),
5053 crate::cache_freshness::WarmVerifyPlan::Skip
5054 ),
5055 "memo must be warm before eviction for the downgrade assertion to bite"
5056 );
5057
5058 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5059 let _dispatch_tx = dispatch_tx;
5060 let shutdown = Arc::new(AtomicBool::new(false));
5061 let thread_shutdown = Arc::clone(&shutdown);
5062 let join = std::thread::spawn(move || {
5063 while !thread_shutdown.load(Ordering::SeqCst) {
5064 std::thread::yield_now();
5065 }
5066 });
5067 ctx.install_watcher_runtime(
5068 dispatch_rx,
5069 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5070 );
5071 wait_for_watcher_count(&ctx, 1);
5072
5073 let executor = Arc::new(Executor::new());
5074 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5075 ctx.mark_subc_unbound();
5076 let mut live_roots = HashMap::new();
5077 let mut meta = RootMeta::new(Instant::now());
5078 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5079 meta.unbound_quiesced = true;
5080 live_roots.insert(root.clone(), meta);
5081
5082 let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
5083 assert!(message.contains("evicted idle root"));
5084 assert!(message.contains("freed ~"));
5085 assert!(message.contains("semantic"));
5086 assert!(!message.contains("semantic not estimated retained"));
5087 assert!(message.contains("trigram"));
5088 assert!(message.contains("retained: bash"));
5089 assert!(message.contains("parser_pool"));
5090
5091 assert_eq!(
5092 reap_idle_roots(
5093 Instant::now(),
5094 &mut live_roots,
5095 &HashMap::new(),
5096 &HashMap::new(),
5097 &executor,
5098 &DispatchPathMetrics::new(),
5099 )
5100 .evicted,
5101 1
5102 );
5103 assert!(ctx.search_index().read().unwrap().is_none());
5104 wait_for_watcher_count(&ctx, 0);
5105 assert!(
5110 matches!(
5111 crate::cache_freshness::warm_verify_plan(
5112 canonical_root.as_path(),
5113 crate::cache_freshness::VerifyArtifact::Search,
5114 Some(seeded_generation),
5115 ),
5116 crate::cache_freshness::WarmVerifyPlan::Strict
5117 ),
5118 "idle eviction must force strict re-verification"
5119 );
5120 assert!(
5121 crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
5122 );
5123 ctx.mark_subc_bound();
5124 assert!(ctx
5125 .ensure_callgraph_store()
5126 .expect("reopen callgraph store")
5127 .is_some());
5128 assert!(live_roots[&root].idle_artifacts_evicted);
5129 }
5130
5131 #[test]
5132 fn idle_root_reaper_applies_ttl_to_unbound_roots() {
5133 let (_root_dir, root) = test_root("idle-root-ttl-gate");
5134 let ctx = test_ctx();
5135 let executor = Arc::new(Executor::new());
5136 assert!(executor.register_actor(root.clone(), ctx));
5137 let ctx = executor.actor_context(&root).expect("actor context");
5138 ctx.mark_subc_unbound();
5139 let now = Instant::now();
5140 let mut meta = RootMeta::new(now);
5141 meta.unbound_quiesced = true;
5142 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5143
5144 assert_eq!(
5148 reap_idle_roots(
5149 now,
5150 &mut live_roots,
5151 &HashMap::new(),
5152 &HashMap::new(),
5153 &executor,
5154 &DispatchPathMetrics::new(),
5155 )
5156 .evicted,
5157 0
5158 );
5159 assert!(!live_roots[&root].idle_artifacts_evicted);
5160
5161 ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
5167 assert_eq!(
5168 reap_idle_roots(
5169 now + IDLE_ROOT_TTL,
5170 &mut live_roots,
5171 &HashMap::new(),
5172 &HashMap::new(),
5173 &executor,
5174 &DispatchPathMetrics::new(),
5175 )
5176 .evicted,
5177 1
5178 );
5179 assert!(live_roots[&root].idle_artifacts_evicted);
5180 assert!(
5181 ctx.take_pending_search_index_paths().is_empty(),
5182 "TTL eviction must dispose retained pending reconciliation paths"
5183 );
5184 }
5185
5186 #[test]
5187 fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
5188 let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
5189 let ctx = test_ctx();
5190 let executor = Arc::new(Executor::new());
5191 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5192 ctx.mark_subc_unbound();
5193
5194 let pending = root.as_path().join("edited-while-unbound.rs");
5200 ctx.add_pending_search_index_paths([pending.clone()]);
5201 let dirty_source = root.as_path().join("dirty.rs");
5202 std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
5203 let mut dirty = crate::search_index::SearchIndex::new();
5204 dirty.ready = true;
5205 dirty.update_file(&dirty_source);
5206 *ctx.search_index()
5207 .write()
5208 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
5209 assert!(ctx.artifact_eviction_blocked());
5210
5211 let mut live_roots = HashMap::new();
5212 let mut meta = RootMeta::new(Instant::now());
5213 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5214 meta.unbound_quiesced = true;
5215 live_roots.insert(root.clone(), meta);
5216
5217 assert_eq!(
5218 reap_idle_roots(
5219 Instant::now(),
5220 &mut live_roots,
5221 &HashMap::new(),
5222 &HashMap::new(),
5223 &executor,
5224 &DispatchPathMetrics::new(),
5225 )
5226 .evicted,
5227 0,
5228 "the dirty index must still block this eviction"
5229 );
5230 assert_eq!(
5231 ctx.take_pending_search_index_paths(),
5232 vec![pending],
5233 "a blocked eviction must restore the taken pending paths"
5234 );
5235 }
5236
5237 #[test]
5238 fn idle_reap_with_bound_route_keeps_watcher_running() {
5239 let (_root_dir, root) = test_root("bound-root-reap-gate");
5240 let ctx = test_ctx();
5241 let executor = Arc::new(Executor::new());
5242 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5243
5244 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5245 let _dispatch_tx = dispatch_tx;
5246 let shutdown = Arc::new(AtomicBool::new(false));
5247 let thread_shutdown = Arc::clone(&shutdown);
5248 let join = std::thread::spawn(move || {
5249 while !thread_shutdown.load(Ordering::SeqCst) {
5250 std::thread::yield_now();
5251 }
5252 });
5253 ctx.install_watcher_runtime(
5254 dispatch_rx,
5255 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5256 );
5257
5258 let mut meta = RootMeta::new(Instant::now());
5259 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5260 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5261 let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
5262 assert_eq!(
5263 reap_idle_roots(
5264 Instant::now(),
5265 &mut live_roots,
5266 &HashMap::new(),
5267 &bound,
5268 &executor,
5269 &DispatchPathMetrics::new(),
5270 )
5271 .evicted,
5272 0
5273 );
5274 wait_for_watcher_count(&ctx, 1);
5275 ctx.stop_watcher_runtime_in_background();
5276 wait_for_watcher_count(&ctx, 0);
5277 }
5278
5279 #[test]
5280 fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
5281 let (root_dir, root) = test_root("deleted-bound-root-reap");
5282 let executor = Arc::new(Executor::new());
5283 assert!(executor.register_actor(root.clone(), test_ctx()));
5284 root_dir.close().expect("delete project root");
5285
5286 let route = route_key(19, 3);
5287 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5288 let cancel_signal = PersistentCancelSignal::new();
5289 let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
5290 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5291 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5292 let mut route_bash_cancels = HashMap::from([(
5293 route,
5294 bash::RouteBashCancel {
5295 token: cancel_signal.clone(),
5296 active_waits: 0,
5297 },
5298 )]);
5299 let metrics = DispatchPathMetrics::new();
5300
5301 let first = reap_idle_roots(
5302 Instant::now(),
5303 &mut live_roots,
5304 &HashMap::new(),
5305 &root_channels,
5306 &executor,
5307 &metrics,
5308 );
5309 assert!(first.forgotten_deleted_roots.is_empty());
5310 assert!(executor.actor_registered(&root));
5311
5312 let mut forgotten = Vec::new();
5313 for _ in 0..100 {
5314 let outcome = reap_idle_roots(
5315 Instant::now(),
5316 &mut live_roots,
5317 &HashMap::new(),
5318 &root_channels,
5319 &executor,
5320 &metrics,
5321 );
5322 if !outcome.forgotten_deleted_roots.is_empty() {
5323 forgotten = outcome.forgotten_deleted_roots;
5324 break;
5325 }
5326 std::thread::sleep(Duration::from_millis(10));
5327 }
5328 assert_eq!(forgotten, vec![root.clone()]);
5329 assert!(!executor.actor_registered(&root));
5330
5331 let mut retry_buffer = HashMap::new();
5332 let mut reclaimed_routes = ReclaimedRoutes::default();
5333 let mut session_identity = HashMap::new();
5334 let mut push_buffer = HashMap::new();
5335 let mut bg_subs = HashMap::new();
5336 let mut bg_sub_by_session = HashMap::new();
5337 let mut bg_wake_pending = HashSet::new();
5338 let mut bg_wake_epoch = HashMap::new();
5339 let mut pending_bash_asks = HashMap::new();
5340 purge_deleted_root_residents(
5341 &root,
5342 &mut routes,
5343 &mut root_channels,
5344 &mut installed_route_epochs,
5345 &mut route_bash_cancels,
5346 &mut retry_buffer,
5347 &mut reclaimed_routes,
5348 &mut session_identity,
5349 &mut push_buffer,
5350 &mut bg_subs,
5351 &mut bg_sub_by_session,
5352 &mut bg_wake_pending,
5353 &mut bg_wake_epoch,
5354 &mut pending_bash_asks,
5355 );
5356
5357 assert!(routes.is_empty());
5358 assert!(root_channels.is_empty());
5359 assert!(installed_route_epochs.is_empty());
5360 assert!(route_bash_cancels.is_empty());
5361 assert!(reclaimed_routes.contains(route));
5362 assert!(cancel_signal.is_cancelled());
5363 }
5364
5365 #[test]
5372 fn live_root_with_bound_route_is_never_reclaimed() {
5373 let (_root_dir, root) = test_root("live-bound-root-retained");
5374 let ctx = test_ctx();
5375 ctx.mark_subc_unbound();
5376 let executor = Arc::new(Executor::new());
5377 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5378
5379 let route = RouteChannel {
5380 channel: 7,
5381 epoch: 1,
5382 };
5383 let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
5384 meta.unbound_quiesced = true;
5385 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5386 let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5387
5388 for _ in 0..3 {
5393 let outcome = reap_idle_roots(
5394 Instant::now(),
5395 &mut live_roots,
5396 &HashMap::new(),
5397 &root_channels,
5398 &executor,
5399 &DispatchPathMetrics::new(),
5400 );
5401 assert!(
5402 outcome.forgotten_deleted_roots.is_empty(),
5403 "a root whose directory exists must never be forgotten"
5404 );
5405 }
5406
5407 assert!(live_roots.contains_key(&root), "live root must be retained");
5408 assert!(
5409 executor.actor_registered(&root),
5410 "live root's actor must survive"
5411 );
5412 assert!(
5413 root.as_path().exists(),
5414 "test vehicle must keep the directory alive; otherwise this control proves nothing"
5415 );
5416 }
5417
5418 #[test]
5419 fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
5420 let (root_dir, root) = test_root("deleted-root-first-observation");
5421 let ctx = test_ctx();
5422 ctx.mark_subc_unbound();
5423 let executor = Arc::new(Executor::new());
5424 assert!(executor.register_actor(root.clone(), ctx));
5425 root_dir.close().expect("delete project root");
5426
5427 let mut meta = RootMeta::new(Instant::now());
5428 meta.unbound_quiesced = true;
5429 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5430 let outcome = reap_idle_roots(
5431 Instant::now(),
5432 &mut live_roots,
5433 &HashMap::new(),
5434 &HashMap::new(),
5435 &executor,
5436 &DispatchPathMetrics::new(),
5437 );
5438
5439 assert!(outcome.forgotten_deleted_roots.is_empty());
5440 assert!(live_roots.contains_key(&root));
5441 assert!(executor.actor_registered(&root));
5442 }
5443
5444 #[test]
5445 fn observing_root_again_resets_deleted_sweep_confirmation() {
5446 let (root_dir, root) = test_root("deleted-root-observation-reset");
5447 let ctx = test_ctx();
5448 ctx.mark_subc_unbound();
5449 let executor = Arc::new(Executor::new());
5450 assert!(executor.register_actor(root.clone(), ctx));
5451 root_dir.close().expect("delete project root");
5452
5453 let mut meta = RootMeta::new(Instant::now());
5454 meta.unbound_quiesced = true;
5455 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5456 let pending_binds = HashMap::new();
5457 let root_channels = HashMap::new();
5458 let metrics = DispatchPathMetrics::new();
5459
5460 let first = reap_idle_roots(
5461 Instant::now(),
5462 &mut live_roots,
5463 &pending_binds,
5464 &root_channels,
5465 &executor,
5466 &metrics,
5467 );
5468 assert!(first.forgotten_deleted_roots.is_empty());
5469
5470 std::fs::create_dir_all(root.as_path()).expect("restore project root");
5471 reap_idle_roots(
5472 Instant::now(),
5473 &mut live_roots,
5474 &pending_binds,
5475 &root_channels,
5476 &executor,
5477 &metrics,
5478 );
5479 std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
5480
5481 let after_reset = reap_idle_roots(
5482 Instant::now(),
5483 &mut live_roots,
5484 &pending_binds,
5485 &root_channels,
5486 &executor,
5487 &metrics,
5488 );
5489 assert!(after_reset.forgotten_deleted_roots.is_empty());
5490 assert!(live_roots.contains_key(&root));
5491 assert!(executor.actor_registered(&root));
5492 }
5493
5494 #[test]
5495 fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
5496 let (root_dir, root) = test_root("deleted-root-reap");
5497 let app = App::default_shared();
5498 let ctx = Arc::new(AppContext::from_app(
5499 Arc::clone(&app),
5500 Config {
5501 project_root: Some(root.as_path().to_path_buf()),
5502 ..Config::default()
5503 },
5504 ));
5505 ctx.set_canonical_cache_root(root.as_path().to_path_buf());
5506 ctx.mark_subc_unbound();
5507 let executor = Arc::new(Executor::new());
5508 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5509 assert_eq!(app.actor_root_count(), 1);
5510 drop(ctx);
5511 root_dir.close().expect("delete project root");
5512
5513 let mut meta = RootMeta::new(Instant::now());
5514 meta.unbound_quiesced = true;
5515 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5516 let outcome = reap_until_forgotten(
5517 &root,
5518 &mut live_roots,
5519 &HashMap::new(),
5520 &HashMap::new(),
5521 &executor,
5522 &DispatchPathMetrics::new(),
5523 );
5524 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5525 assert!(!executor.actor_registered(&root));
5526 assert!(!live_roots.contains_key(&root));
5527 wait_for_actor_root_count(&app, 0);
5528
5529 let status_ctx = AppContext::from_app(app, Config::default());
5530 let status = status_ctx.build_status_snapshot();
5531 assert_eq!(status["runtime"]["live_actor_roots"], 0);
5532 assert_eq!(status["runtime"]["open_routes"], 0);
5533 }
5534
5535 #[test]
5536 fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
5537 let (root_dir, root) = test_root("deleted-root-reap-census");
5538 let executor = Arc::new(Executor::new());
5539 assert!(executor.register_actor(root.clone(), test_ctx()));
5540 root_dir.close().expect("delete project root");
5541
5542 let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
5543 let metrics = DispatchPathMetrics::new();
5544 let outcome = reap_idle_roots(
5545 Instant::now(),
5546 &mut live_roots,
5547 &HashMap::new(),
5548 &HashMap::new(),
5549 &executor,
5550 &metrics,
5551 );
5552 assert_eq!(outcome.evicted, 0);
5553
5554 let report = build_health_report(
5555 &executor,
5556 &HashMap::new(),
5557 &metrics,
5558 &crate::context::App::default_shared(),
5559 );
5560 let reap = report
5561 .metrics
5562 .as_ref()
5563 .and_then(|metrics| metrics.get("reap"))
5564 .expect("reap health metrics");
5565 assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
5566 assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
5567 assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
5568 assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
5569 }
5570
5571 #[test]
5572 fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
5573 let (root_dir, root) = test_root("connection-exit-deleted-root");
5574 let executor = Arc::new(Executor::new());
5575 assert!(executor.register_actor(root.clone(), test_ctx()));
5576
5577 let route = route_key(11, 1);
5578 let mut meta = RootMeta::new(Instant::now());
5579 meta.maintenance_pending = true;
5580 meta.maintenance_queued_kinds
5581 .push_back(MaintenanceDrainKind::CompletionDrains);
5582 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5583 let mut pending_binds = HashMap::new();
5584 let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
5585 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5586 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5587 let mut route_bash_cancels = HashMap::new();
5588
5589 quiesce_connection_roots(
5590 &mut live_roots,
5591 &mut pending_binds,
5592 &mut routes,
5593 &mut root_channels,
5594 &mut installed_route_epochs,
5595 &mut route_bash_cancels,
5596 &executor,
5597 );
5598 assert!(live_roots[&root].unbound_quiesced);
5599 assert!(!live_roots[&root].maintenance_pending);
5600 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
5601 assert!(routes.is_empty());
5602 assert!(root_channels.is_empty());
5603
5604 root_dir.close().expect("delete project root");
5605 let metrics = DispatchPathMetrics::new();
5606 let outcome = reap_until_forgotten(
5607 &root,
5608 &mut live_roots,
5609 &pending_binds,
5610 &root_channels,
5611 &executor,
5612 &metrics,
5613 );
5614 let mut session_identity = HashMap::new();
5615 let mut push_buffer = HashMap::new();
5616 let mut bg_subs = HashMap::new();
5617 let mut bg_sub_by_session = HashMap::new();
5618 let mut bg_wake_pending = HashSet::new();
5619 let mut bg_wake_epoch = HashMap::new();
5620 let mut pending_bash_asks = HashMap::new();
5621 let mut retry_buffer = HashMap::new();
5622 let mut reclaimed_routes = ReclaimedRoutes::default();
5623 for forgotten in &outcome.forgotten_deleted_roots {
5624 purge_deleted_root_residents(
5625 forgotten,
5626 &mut routes,
5627 &mut root_channels,
5628 &mut installed_route_epochs,
5629 &mut route_bash_cancels,
5630 &mut retry_buffer,
5631 &mut reclaimed_routes,
5632 &mut session_identity,
5633 &mut push_buffer,
5634 &mut bg_subs,
5635 &mut bg_sub_by_session,
5636 &mut bg_wake_pending,
5637 &mut bg_wake_epoch,
5638 &mut pending_bash_asks,
5639 );
5640 }
5641
5642 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5643 assert!(!executor.actor_registered(&root));
5644 assert!(!live_roots.contains_key(&root));
5645 }
5646
5647 #[test]
5648 fn unbound_root_quiesces_maintenance_without_removing_actor() {
5649 let (_root_dir, root) = test_root("unbound-root-quiesce");
5650 let ctx = test_ctx();
5651 let executor = Arc::new(Executor::new());
5652 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5653 let mut meta = RootMeta::new(Instant::now());
5654 meta.maintenance_pending = true;
5655 meta.maintenance_jobs_in_flight = 1;
5656 meta.maintenance_queued_kinds
5657 .push_back(MaintenanceDrainKind::ConfigureTail);
5658 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5659 *ctx.search_index()
5661 .write()
5662 .unwrap_or_else(std::sync::PoisonError::into_inner) =
5663 Some(crate::search_index::SearchIndex::new());
5664 ctx.set_cache_writer_capabilities(true, true);
5665 let pending = root.as_path().join("pending.rs");
5666 ctx.add_pending_search_index_paths([pending.clone()]);
5667 let canonical_root = root.as_path().to_path_buf();
5671 let artifact = canonical_root.join("cache.bin");
5672 std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
5673 let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
5674 crate::cache_freshness::record_verify_completed(
5675 &canonical_root,
5676 crate::cache_freshness::VerifyArtifact::Search,
5677 seeded_generation,
5678 );
5679 assert!(matches!(
5680 crate::cache_freshness::warm_verify_plan(
5681 &canonical_root,
5682 crate::cache_freshness::VerifyArtifact::Search,
5683 seeded_generation,
5684 ),
5685 crate::cache_freshness::WarmVerifyPlan::Skip
5686 ));
5687 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5690 let _dispatch_tx = dispatch_tx;
5691 let shutdown = Arc::new(AtomicBool::new(false));
5692 let thread_shutdown = Arc::clone(&shutdown);
5693 let join = std::thread::spawn(move || {
5694 while !thread_shutdown.load(Ordering::SeqCst) {
5695 std::thread::yield_now();
5696 }
5697 });
5698 ctx.install_watcher_runtime(
5699 dispatch_rx,
5700 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5701 );
5702 assert!(ctx.watcher_runtime_active());
5703
5704 quiesce_unbound_root(&root, &mut live_roots, &executor);
5705 let meta = &live_roots[&root];
5706 assert!(meta.unbound_quiesced);
5707 assert!(ctx.subc_unbound_quiesced());
5708 assert!(meta.maintenance_pending);
5709 assert!(meta.maintenance_queued_kinds.is_empty());
5710 assert!(executor.actor_registered(&root));
5711 assert!(
5715 ctx.search_index()
5716 .read()
5717 .unwrap_or_else(std::sync::PoisonError::into_inner)
5718 .is_some(),
5719 "quiesce must not evict resident artifacts"
5720 );
5721 assert_eq!(
5722 ctx.pending_callgraph_store_force_token(),
5723 None,
5724 "quiesce must not force a callgraph rebuild"
5725 );
5726 assert_eq!(
5727 ctx.take_pending_search_index_paths(),
5728 vec![pending],
5729 "quiesce must retain pending watcher-derived paths"
5730 );
5731 assert!(
5732 matches!(
5733 crate::cache_freshness::warm_verify_plan(
5734 &canonical_root,
5735 crate::cache_freshness::VerifyArtifact::Search,
5736 seeded_generation,
5737 ),
5738 crate::cache_freshness::WarmVerifyPlan::Skip
5739 ),
5740 "quiesce must not invalidate the warm verify memo"
5741 );
5742 assert!(
5743 ctx.watcher_runtime_active(),
5744 "quiesce must not stop a running watcher"
5745 );
5746 ctx.stop_watcher_runtime();
5747
5748 let meta = live_roots.get_mut(&root).expect("root metadata");
5749 note_maintenance_completion(
5750 meta,
5751 Some(MaintenanceDrainKind::ConfigureTail),
5752 false,
5753 meta.unbound_quiesced,
5754 );
5755 assert!(!meta.maintenance_pending);
5756 assert!(meta.maintenance_queued_kinds.is_empty());
5757 }
5758
5759 #[test]
5760 fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
5761 let (_dir, root) = test_root("same-root-replacement");
5762 let route = route_key(7, 1);
5763 let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5764 let root_channels = HashMap::new();
5765
5766 assert!(!route_removal_will_quiesce_root(
5767 &root,
5768 route,
5769 &installed_channels,
5770 false,
5771 Some(&root),
5772 ));
5773 assert!(route_removal_will_quiesce_root(
5774 &root,
5775 route,
5776 &installed_channels,
5777 false,
5778 None,
5779 ));
5780 assert!(!should_quiesce_removed_root(
5781 &root,
5782 &root_channels,
5783 false,
5784 Some(&root),
5785 ));
5786 assert!(should_quiesce_removed_root(
5787 &root,
5788 &root_channels,
5789 false,
5790 None,
5791 ));
5792 assert!(!should_quiesce_removed_root(
5793 &root,
5794 &root_channels,
5795 true,
5796 None,
5797 ));
5798 }
5799
5800 #[test]
5801 fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
5802 let (_root_dir, root) = test_root("unbound-root-route-count");
5803 let executor = Arc::new(Executor::new());
5804 assert!(executor.register_actor(root.clone(), test_ctx()));
5805 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5806 let mut root_channels = HashMap::from([(
5807 root.clone(),
5808 HashSet::from([route_key(7, 1), route_key(8, 1)]),
5809 )]);
5810
5811 remove_root_channel(&mut root_channels, &root, route_key(7, 1));
5812 if !root_channels.contains_key(&root) {
5813 quiesce_unbound_root(&root, &mut live_roots, &executor);
5814 }
5815 assert!(!live_roots[&root].unbound_quiesced);
5816
5817 remove_root_channel(&mut root_channels, &root, route_key(8, 1));
5818 if !root_channels.contains_key(&root) {
5819 quiesce_unbound_root(&root, &mut live_roots, &executor);
5820 }
5821 assert!(live_roots[&root].unbound_quiesced);
5822
5823 live_roots
5824 .get_mut(&root)
5825 .expect("root metadata")
5826 .note_activity();
5827 assert!(
5828 live_roots[&root].unbound_quiesced,
5829 "late asynchronous activity must not reactivate an unbound root"
5830 );
5831
5832 live_roots
5833 .get_mut(&root)
5834 .expect("root metadata")
5835 .reactivate_bound();
5836 assert!(!live_roots[&root].unbound_quiesced);
5837 }
5838
5839 #[test]
5840 fn allocator_pressure_relief_requires_every_root_to_be_idle() {
5841 let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
5842 let (_active_dir, active_root) = test_root("allocator-relief-active");
5843 let now = Instant::now();
5844 let mut live_roots = HashMap::new();
5845 let mut idle = RootMeta::new(now);
5846 idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
5847 live_roots.insert(idle_root, idle);
5848 assert!(process_has_been_idle(now, &live_roots));
5849
5850 live_roots.insert(active_root.clone(), RootMeta::new(now));
5851 assert!(!process_has_been_idle(now, &live_roots));
5852
5853 let active = live_roots
5854 .get_mut(&active_root)
5855 .expect("active root metadata");
5856 active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
5857 active.active_bash_waits = 1;
5858 assert!(!process_has_been_idle(now, &live_roots));
5859 }
5860
5861 #[test]
5862 fn pressure_relief_log_reports_before_and_after_measurements() {
5863 let allocator = crate::memory::AllocatorMemorySnapshot {
5864 status: "measured",
5865 bytes_in_use: Some(8 * 1024 * 1024),
5866 size_allocated: Some(12 * 1024 * 1024),
5867 retained_slack_bytes: Some(4 * 1024 * 1024),
5868 not_estimated: None,
5869 };
5870 let relief = crate::memory::AllocatorPressureRelief {
5871 bytes_released: 3 * 1024 * 1024,
5872 rss_before_bytes: Some(20 * 1024 * 1024),
5873 rss_after_bytes: Some(17 * 1024 * 1024),
5874 allocator_before: allocator.clone(),
5875 allocator_after: crate::memory::AllocatorMemorySnapshot {
5876 size_allocated: Some(9 * 1024 * 1024),
5877 retained_slack_bytes: Some(1024 * 1024),
5878 ..allocator
5879 },
5880 };
5881 let message = pressure_relief_label(&relief);
5882 assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
5883 assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
5884 assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
5885 assert!(message.contains("reported 3.0 MB released"));
5886 }
5887
5888 #[test]
5889 fn due_maintenance_jobs_skip_poisoned_roots() {
5890 let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
5891 let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
5892 let mut live_roots = HashMap::new();
5893 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
5894 let mut poisoned_meta = RootMeta::new(Instant::now());
5895 poisoned_meta.maintenance_poisoned = true;
5896 live_roots.insert(poisoned_root.clone(), poisoned_meta);
5897
5898 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5899 &mut live_roots,
5900 MAINTENANCE_SUBMIT_BUDGET,
5901 &HashSet::new(),
5902 );
5903
5904 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5905 assert!(due.iter().all(|(root, _)| root == &healthy_root));
5906 assert!(!deferred);
5907 assert!(live_roots[&healthy_root].maintenance_pending);
5908 assert_eq!(
5909 live_roots[&healthy_root].maintenance_jobs_in_flight,
5910 INITIAL_MAINTENANCE_JOB_COUNT
5911 );
5912 assert!(!live_roots[&poisoned_root].maintenance_pending);
5913 }
5914
5915 #[test]
5916 fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
5917 let (_dir, root) = test_root("maintenance-unbound");
5918 let mut meta = RootMeta::new(Instant::now());
5919 meta.unbound_quiesced = true;
5920 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5921
5922 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5923 &mut live_roots,
5924 MAINTENANCE_SUBMIT_BUDGET,
5925 &HashSet::new(),
5926 );
5927
5928 assert!(due.is_empty());
5929 assert!(!deferred);
5930 assert!(!live_roots[&root].maintenance_pending);
5931 }
5932
5933 #[test]
5934 fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
5935 let (_dir, root) = test_root("maintenance-idle-bg-subscription");
5936 let ctx = test_ctx();
5937 assert!(!ctx.completion_drains_have_work());
5938
5939 let executor = Executor::new();
5940 assert!(executor.register_actor(root.clone(), ctx));
5941 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5942 let session = "idle-session".to_string();
5943 let channel = route_key(17, 1);
5944 let bg_sub_by_session = HashMap::from([((root.clone(), session.clone()), channel)]);
5945 let mut bg_wake_pending = HashSet::new();
5946
5947 let (idle_tick_jobs, deferred) = due_maintenance_jobs(
5948 &mut live_roots,
5949 Some(&executor),
5950 &bg_sub_by_session,
5951 &bg_wake_pending,
5952 MAINTENANCE_SUBMIT_BUDGET,
5953 &HashSet::new(),
5954 );
5955 assert!(idle_tick_jobs.is_empty());
5956 assert!(!deferred);
5957 assert!(!live_roots[&root].maintenance_pending);
5958
5959 let mut bg_wake_epoch = HashMap::new();
5962 push::arm_bg_wake(
5963 root.clone(),
5964 session,
5965 channel,
5966 &mut bg_wake_pending,
5967 &mut bg_wake_epoch,
5968 );
5969 let (next_tick_jobs, deferred) = due_maintenance_jobs(
5970 &mut live_roots,
5971 Some(&executor),
5972 &bg_sub_by_session,
5973 &bg_wake_pending,
5974 MAINTENANCE_SUBMIT_BUDGET,
5975 &HashSet::new(),
5976 );
5977 assert_eq!(
5978 next_tick_jobs,
5979 vec![(root, MaintenanceDrainKind::CompletionDrains)]
5980 );
5981 assert!(!deferred);
5982 }
5983
5984 #[tokio::test]
5985 async fn subc_configure_tail_precedes_completed_search_install() {
5986 let root_dir = tempfile::tempdir().unwrap();
5987 let storage = tempfile::tempdir().unwrap();
5988 let root = ProjectRootId::from_path(root_dir.path()).unwrap();
5989 let (ctx, ignored_path) =
5990 runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
5991 let ctx = Arc::new(ctx);
5992 assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
5993 &ctx,
5994 &ignored_path
5995 ));
5996
5997 let executor = Arc::new(Executor::new());
5998 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5999 let metrics = Arc::new(DispatchPathMetrics::new());
6000 let (completion_tx, mut completion_rx) = mpsc::channel(4);
6001 submit_maintenance_job(
6002 &executor,
6003 root.clone(),
6004 MaintenanceDrainKind::ConfigureTail,
6005 Vec::new(),
6006 &completion_tx,
6007 &metrics,
6008 );
6009 submit_maintenance_job(
6010 &executor,
6011 root,
6012 MaintenanceDrainKind::CompletionDrains,
6013 Vec::new(),
6014 &completion_tx,
6015 &metrics,
6016 );
6017
6018 let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6019 .await
6020 .expect("configure-tail completion timed out")
6021 .expect("configure-tail completion channel closed");
6022 let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6023 .await
6024 .expect("completion-drains completion timed out")
6025 .expect("completion-drains completion channel closed");
6026 assert!(first.response.id.contains("configure-tail"));
6027 assert!(second.response.id.contains("completion-drains"));
6028 assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
6029 &ctx,
6030 &ignored_path
6031 ));
6032 assert_eq!(
6033 ctx.search_index()
6034 .read()
6035 .unwrap_or_else(std::sync::PoisonError::into_inner)
6036 .as_ref()
6037 .expect("completed search index installed")
6038 .file_count(),
6039 0,
6040 "configure must install the ignore matcher before pending paths replay"
6041 );
6042 ctx.stop_watcher_runtime();
6043 }
6044
6045 #[test]
6046 fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
6047 let (_dir, root) = test_root("maintenance-post-bind");
6048 let mut live_roots = HashMap::new();
6049 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6050
6051 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6052 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6053
6054 let meta = live_roots.get(&root).expect("root metadata");
6055 assert!(meta.maintenance_pending);
6056 assert_eq!(meta.maintenance_jobs_in_flight, 0);
6057 assert_eq!(
6058 meta.maintenance_queued_kinds
6059 .iter()
6060 .copied()
6061 .collect::<Vec<_>>(),
6062 vec![
6063 MaintenanceDrainKind::ConfigureTail,
6064 MaintenanceDrainKind::CompletionDrains,
6065 ]
6066 );
6067
6068 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6069 &mut live_roots,
6070 MAINTENANCE_SUBMIT_BUDGET,
6071 &HashSet::new(),
6072 );
6073
6074 assert_eq!(
6075 due,
6076 vec![
6077 (root.clone(), MaintenanceDrainKind::ConfigureTail),
6078 (root.clone(), MaintenanceDrainKind::CompletionDrains),
6079 ]
6080 );
6081 assert!(!deferred);
6082 assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
6083 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
6084 }
6085
6086 #[test]
6087 fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
6088 let mut live_roots = HashMap::new();
6089 let mut root_ids = Vec::new();
6090 let mut _dirs = Vec::new();
6091 for index in 0..4 {
6092 let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
6093 live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
6094 root_ids.push(root_id);
6095 _dirs.push(dir);
6096 }
6097
6098 let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
6099 let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
6100 &mut live_roots,
6101 small_budget,
6102 &HashSet::new(),
6103 );
6104
6105 assert_eq!(first_due.len(), small_budget);
6106 assert!(first_deferred);
6107 let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
6108 assert!(first_due_set
6109 .iter()
6110 .all(|root| live_roots[root].maintenance_pending));
6111 assert!(first_due_set
6112 .iter()
6113 .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
6114
6115 let all_roots: HashSet<_> = root_ids.into_iter().collect();
6116 let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
6117 assert!(deferred_roots
6118 .iter()
6119 .all(|root| !live_roots[root].maintenance_pending));
6120 }
6121
6122 #[test]
6123 fn due_maintenance_jobs_defers_pending_bind_roots() {
6124 let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
6125 let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
6126 let mut live_roots = HashMap::new();
6127 live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
6128 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6129 let pending_bind_roots = HashSet::from([bind_root.clone()]);
6130
6131 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6132 &mut live_roots,
6133 usize::MAX,
6134 &pending_bind_roots,
6135 );
6136
6137 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6138 assert!(due.iter().all(|(root, _)| root == &healthy_root));
6139 assert!(!deferred);
6140 assert!(!live_roots[&bind_root].maintenance_pending);
6141 assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
6142 }
6143
6144 #[test]
6145 fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
6146 let (_dir, root) = test_root("maintenance-requeue");
6147 let mut live_roots = HashMap::new();
6148 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6149 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6150 &mut live_roots,
6151 usize::MAX,
6152 &HashSet::new(),
6153 );
6154 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6155 assert!(due.iter().all(|(due_root, _)| due_root == &root));
6156 assert!(!deferred);
6157
6158 let meta = live_roots.get_mut(&root).unwrap();
6159 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
6160 assert!(meta.maintenance_pending);
6161 assert_eq!(
6162 meta.maintenance_jobs_in_flight,
6163 INITIAL_MAINTENANCE_JOB_COUNT - 1
6164 );
6165 assert_eq!(meta.maintenance_queued_kinds.len(), 1);
6166
6167 let (requeued, deferred) =
6168 due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
6169 assert_eq!(
6170 requeued,
6171 vec![(root.clone(), MaintenanceDrainKind::Watcher)]
6172 );
6173 assert!(!deferred);
6174 let meta = live_roots.get_mut(&root).unwrap();
6175 assert_eq!(
6176 meta.maintenance_jobs_in_flight,
6177 INITIAL_MAINTENANCE_JOB_COUNT
6178 );
6179 assert!(meta.maintenance_queued_kinds.is_empty());
6180
6181 for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
6182 note_maintenance_completion(meta, None, false, false);
6183 }
6184 assert!(!meta.maintenance_pending);
6185 assert_eq!(meta.maintenance_jobs_in_flight, 0);
6186 }
6187
6188 #[test]
6189 fn maintenance_requeue_drops_while_bind_is_pending() {
6190 let (_dir, root) = test_root("maintenance-bind-requeue");
6191 let mut live_roots = HashMap::new();
6192 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6193 let (due, _) = due_maintenance_jobs_without_actor_context(
6194 &mut live_roots,
6195 usize::MAX,
6196 &HashSet::new(),
6197 );
6198 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6199
6200 let meta = live_roots.get_mut(&root).unwrap();
6201 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
6202
6203 assert_eq!(
6204 meta.maintenance_jobs_in_flight,
6205 INITIAL_MAINTENANCE_JOB_COUNT - 1
6206 );
6207 assert!(meta.maintenance_queued_kinds.is_empty());
6208 assert!(meta.maintenance_pending);
6209 }
6210
6211 #[test]
6212 fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
6213 let mut meta = RootMeta::new(Instant::now());
6214 meta.unbound_quiesced = true;
6215
6216 assert!(!should_requiesce_after_maintenance(
6217 &meta,
6218 MaintenanceDrainKind::Lsp,
6219 false,
6220 ));
6221 assert!(!should_requiesce_after_maintenance(
6222 &meta,
6223 MaintenanceDrainKind::ConfigureTail,
6224 true,
6225 ));
6226 assert!(should_requiesce_after_maintenance(
6227 &meta,
6228 MaintenanceDrainKind::ConfigureTail,
6229 false,
6230 ));
6231 }
6232
6233 #[test]
6234 fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
6235 let (_dir, root) = test_root("maintenance-fatal");
6236 let mut live_roots = HashMap::new();
6237 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6238 let (due, _) = due_maintenance_jobs_without_actor_context(
6239 &mut live_roots,
6240 usize::MAX,
6241 &HashSet::new(),
6242 );
6243 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6244
6245 let meta = live_roots.get_mut(&root).unwrap();
6246 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
6247 assert!(meta.maintenance_poisoned);
6248 assert!(meta.maintenance_queued_kinds.is_empty());
6249
6250 for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
6251 note_maintenance_completion(meta, None, false, false);
6252 }
6253 assert!(!meta.maintenance_pending);
6254 assert_eq!(meta.maintenance_jobs_in_flight, 0);
6255 }
6256
6257 #[test]
6258 fn trust_for_principal_matrix() {
6259 assert_eq!(
6260 trust_for_principal(&Some(Principal::Direct)),
6261 BindTrust::FirstParty
6262 );
6263 for module_id in [
6270 "llm-runner",
6271 "aft",
6272 "broca",
6273 "alfonso-core",
6274 "prefrontal",
6275 "prefrontal-core",
6276 ] {
6277 assert_eq!(
6278 trust_for_principal(&Some(Principal::Reserved {
6279 module_id: module_id.to_string(),
6280 })),
6281 BindTrust::FirstParty,
6282 "reserved module id '{module_id}' must resolve to first-party trust"
6283 );
6284 }
6285 assert_eq!(
6286 trust_for_principal(&Some(Principal::Reserved {
6287 module_id: "subc-mcp".to_string(),
6288 })),
6289 BindTrust::Untrusted
6290 );
6291 assert_eq!(
6292 trust_for_principal(&Some(Principal::Reserved {
6293 module_id: "anything-unknown".to_string(),
6294 })),
6295 BindTrust::Untrusted
6296 );
6297 assert_eq!(
6298 trust_for_principal(&Some(Principal::Unverified)),
6299 BindTrust::Untrusted
6300 );
6301 assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
6302 }
6303
6304 #[test]
6305 fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
6306 let principal = Some(Principal::Direct);
6307 let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
6308 let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
6309
6310 assert_eq!(
6311 trust_for_bind(fingerprint_a, &principal),
6312 BindTrust::Untrusted
6313 );
6314 assert_eq!(
6315 trust_for_bind(fingerprint_b, &principal),
6316 BindTrust::Untrusted
6317 );
6318 }
6319
6320 #[test]
6327 fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
6328 for harness in ["opencode", "pi", "runner", "mcp:claude"] {
6329 assert_eq!(
6330 trust_for_bind(harness, &Some(Principal::Direct)),
6331 BindTrust::FirstParty,
6332 "a direct principal must stay first-party on {harness}"
6333 );
6334 assert_eq!(
6335 trust_for_bind(harness, &Some(Principal::Unverified)),
6336 BindTrust::Untrusted,
6337 "an unverified principal must stay untrusted on {harness}"
6338 );
6339 assert_eq!(
6340 trust_for_bind(harness, &None),
6341 BindTrust::Untrusted,
6342 "an absent principal must fail closed on {harness}"
6343 );
6344 assert_eq!(
6345 trust_for_bind(
6346 harness,
6347 &Some(Principal::Reserved {
6348 module_id: "subc-mcp".to_string(),
6349 })
6350 ),
6351 BindTrust::Untrusted,
6352 "a non-allowlisted reserved module must stay untrusted on {harness}"
6353 );
6354 }
6355 }
6356
6357 #[tokio::test]
6358 async fn persistent_cancel_resolves_when_fired_before_await() {
6359 let signal = PersistentCancelSignal::new();
6363 signal.cancel();
6364 tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
6366 .await
6367 .expect("cancelled() must resolve when cancel fired beforehand");
6368
6369 let racing = PersistentCancelSignal::new();
6371 let racing_for_task = racing.clone();
6372 let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
6373 racing.cancel();
6374 tokio::time::timeout(Duration::from_secs(1), waiter)
6375 .await
6376 .expect("cancelled() must resolve when cancel races the await")
6377 .expect("waiter task panicked");
6378 }
6379
6380 #[test]
6381 fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
6382 let installed = HashMap::from([(7, 9)]);
6383 let mut reclaimed = ReclaimedRoutes::default();
6384 reclaimed.insert(route_key(8, 1));
6385 for ty in [
6386 FrameType::Request,
6387 FrameType::Response,
6388 FrameType::Error,
6389 FrameType::Push,
6390 FrameType::Cancel,
6391 FrameType::Goodbye,
6392 ] {
6393 let body = if ty.is_pure_header() {
6394 Vec::new()
6395 } else {
6396 br#"{}"#.to_vec()
6397 };
6398 let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
6399 assert!(
6400 !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
6401 "{ty:?}"
6402 );
6403 }
6404
6405 let reclaimed_request = Frame::build(
6406 FrameType::Request,
6407 control_flags(),
6408 8,
6409 1,
6410 42,
6411 br#"{}"#.to_vec(),
6412 )
6413 .unwrap();
6414 assert!(ingress_route_should_be_processed(
6415 &installed,
6416 &reclaimed,
6417 &reclaimed_request
6418 ));
6419
6420 let never_installed = Frame::build(
6421 FrameType::Request,
6422 control_flags(),
6423 9,
6424 1,
6425 43,
6426 br#"{}"#.to_vec(),
6427 )
6428 .unwrap();
6429 assert!(!ingress_route_should_be_processed(
6430 &installed,
6431 &reclaimed,
6432 &never_installed
6433 ));
6434
6435 let current = Frame::build(
6436 FrameType::Request,
6437 control_flags(),
6438 7,
6439 9,
6440 43,
6441 br#"{}"#.to_vec(),
6442 )
6443 .unwrap();
6444 let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
6445 assert!(ingress_route_should_be_processed(
6446 &installed, &reclaimed, ¤t
6447 ));
6448 assert!(ingress_route_should_be_processed(
6449 &installed, &reclaimed, &control
6450 ));
6451 assert_eq!(installed, HashMap::from([(7, 9)]));
6452 }
6453
6454 #[tokio::test]
6455 async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
6456 let (_dir, root) = test_root("route-bind-b2-ordering");
6457 let route = route_key(7, 3);
6458 let identity = RouteIdentity(Arc::new(RouteIdentityData {
6459 root: root.clone(),
6460 project_root: root.as_path().to_path_buf(),
6461 harness: "opencode".to_string(),
6462 session: "b2-session".to_string(),
6463 trust: BindTrust::FirstParty,
6464 spawn_principal: AuthenticatedPrincipal::FirstParty,
6465 consumer_elicitation_capable: false,
6466 }));
6467 let replay_key = push::ReplayKey::from_identity(&identity);
6468 let completion = RouteBindCompletion {
6469 route,
6470 identity,
6471 bind_root_id: root.clone(),
6472 inserted_new_actor: false,
6473 configure_response: Response::success("subc-bind-7", json!({})),
6474 diagnostics_on_edit: false,
6475 ver: PROTOCOL_VERSION,
6476 corr: 91,
6477 flags: control_flags(),
6478 };
6479 let mut pending_binds = HashMap::from([(
6480 route,
6481 PendingBind {
6482 bind_root_id: root,
6483 inserted_new_actor: false,
6484 cancelled: false,
6485 configure_request_id: "subc-bind-7".to_string(),
6486 started_at: Instant::now(),
6487 warned_half_deadline: false,
6488 deadline_reported: false,
6489 corr: 91,
6490 ver: PROTOCOL_VERSION,
6491 flags: control_flags(),
6492 cancellation: crate::executor::JobCancellation::new(),
6493 },
6494 )]);
6495 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6496 let mut push_buffer =
6497 HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
6498 let (writer_tx, mut writer_rx) = mpsc::channel(8);
6499 let metrics = Arc::new(DispatchPathMetrics::new());
6500
6501 handle_route_bind_completion(
6502 &writer_tx,
6503 completion,
6504 &mut HashMap::new(),
6505 &mut HashMap::new(),
6506 &mut HashMap::new(),
6507 &mut push_buffer,
6508 &mut HashMap::new(),
6509 &mut pending_binds,
6510 &mut installed_route_epochs,
6511 &Arc::new(Executor::new()),
6512 &Arc::new(Notify::new()),
6513 &metrics,
6514 )
6515 .await
6516 .unwrap();
6517
6518 let ack = writer_rx.try_recv().expect("RouteBindAck");
6519 assert_eq!(ack.header.ty, FrameType::Response);
6520 assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
6521 let route_frame = writer_rx.try_recv().expect("post-ack route frame");
6522 assert_eq!(route_frame.header.ty, FrameType::Push);
6523 assert_eq!(
6524 (route_frame.header.channel, route_frame.header.epoch),
6525 (route.channel, route.epoch)
6526 );
6527 }
6528}