1use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, Lane};
31use crate::jsonc::strip_jsonc;
32use crate::log_ctx;
33use crate::path_identity::ProjectRootId;
34use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
35use crate::run_tool_call::{
36 run_tool_call, strip_agent_preview_arg_owned, PhaseTrace, ToolCallContext, ToolCallOutcome,
37 ToolCallResult,
38};
39use crate::runtime_drain;
40use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
41
42use subc_protocol::manifest::{
43 Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
44 ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
45};
46use subc_protocol::session::{
47 HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
48 MODULE_CONTROL_OP_HEALTH_CHECK,
49};
50use subc_protocol::{
51 ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, PROTOCOL_VERSION,
52};
53use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
54use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
55use tokio::net::TcpStream;
56use tokio::sync::{mpsc, oneshot, Notify};
57use tokio::task::JoinHandle;
58
59const AUTH_DEADLINE: Duration = Duration::from_secs(5);
62const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
63const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
64const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
65const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
66
67const HELLO_CORR: u64 = 1;
69
70const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
73
74const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
78
79const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
83
84const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
88
89const WRITER_QUEUE_CAPACITY: usize = 256;
90
91const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
94
95const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
103const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
104 MaintenanceDrainKind::Watcher,
105 MaintenanceDrainKind::Lsp,
106 MaintenanceDrainKind::ConfigureTail,
107 MaintenanceDrainKind::CompletionDrains,
108];
109#[cfg(test)]
110const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
111
112const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
113const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
114
115const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
116const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
117
118const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
121
122const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
126
127const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
129const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
130
131#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
132struct RouteChannel {
133 channel: u16,
134 epoch: u32,
135}
136
137impl fmt::Display for RouteChannel {
138 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
139 write!(f, "{}@{}", self.channel, self.epoch)
140 }
141}
142
143type PushEnvelope = (ProjectRootId, PushFrame);
144type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
145type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
146mod bash;
147mod health;
148mod manifest;
149mod push;
150mod wire;
151
152use self::health::{
153 build_health_report, warn_slow_pending_binds, DispatchPathMetrics, ReapBlockerCensus,
154 ResponseTaskGuard,
155};
156use self::manifest::{
157 build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
158 is_subc_agent_core_tool, is_subc_native_plumbing_tool,
159};
160pub use self::wire::SubcError;
161
162pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
166 manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
167}
168use self::wire::{
169 build_error_frame, build_goodbye_frame, build_tool_response_frame, decrement_counted_channel,
170 response_is_fatal_panic, response_message, send_counted_channel, send_frame,
171 send_reliable_writer_frame, send_traced_tool_response_frame, ToolResponseWriteTrace,
172 WriterFrame, WriterSender,
173};
174
175struct DecodedFrame {
176 frame: Frame,
177 phase_trace: PhaseTrace,
178}
179
180struct ToolCallCompletion {
181 text: String,
182 phase_trace: PhaseTrace,
183}
184
185#[derive(Clone)]
186struct PushSenders {
187 lossy_tx: mpsc::Sender<LossyPushEnvelope>,
188 reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
189 lossy_overflow: Arc<push::LossyOverflow>,
190 lossy_seq: Arc<AtomicU64>,
191}
192
193#[derive(Clone)]
194struct PersistentCancelSignal {
195 inner: Arc<PersistentCancelInner>,
196}
197
198struct PersistentCancelInner {
199 cancelled: AtomicBool,
200 notify: Notify,
201}
202
203impl PersistentCancelSignal {
204 fn new() -> Self {
205 Self {
206 inner: Arc::new(PersistentCancelInner {
207 cancelled: AtomicBool::new(false),
208 notify: Notify::new(),
209 }),
210 }
211 }
212
213 fn cancel(&self) {
214 if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
215 self.inner.notify.notify_waiters();
216 }
217 }
218
219 fn is_cancelled(&self) -> bool {
220 self.inner.cancelled.load(Ordering::SeqCst)
221 }
222
223 async fn cancelled(&self) {
224 loop {
232 let notified = self.inner.notify.notified();
233 tokio::pin!(notified);
234 notified.as_mut().enable();
235 if self.is_cancelled() {
236 return;
237 }
238 notified.await;
239 }
240 }
241}
242
243#[derive(Debug, Clone, Copy, PartialEq, Eq)]
244pub(crate) enum BindTrust {
245 FirstParty,
246 Untrusted,
247}
248
249impl BindTrust {
250 fn allows_bash_observation(self) -> bool {
251 matches!(self, Self::FirstParty)
252 }
253
254 fn label(self) -> &'static str {
255 match self {
256 Self::FirstParty => "first_party",
257 Self::Untrusted => "untrusted",
258 }
259 }
260
261 fn sandbox_trust(self) -> PrincipalTrust {
262 match self {
263 Self::FirstParty => PrincipalTrust::FirstParty,
264 Self::Untrusted => PrincipalTrust::Untrusted,
265 }
266 }
267}
268
269pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
270 match principal {
271 Some(Principal::Direct) => BindTrust::FirstParty,
272 Some(Principal::Reserved { module_id })
283 if module_id == "llm-runner"
284 || module_id == "aft"
285 || module_id == "broca"
286 || module_id == "alfonso-core"
287 || module_id == "prefrontal"
288 || module_id == "prefrontal-core" =>
289 {
290 BindTrust::FirstParty
291 }
292 Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
293 BindTrust::Untrusted
294 }
295 }
296}
297
298fn harness_forces_untrusted(harness: &str) -> bool {
299 harness.starts_with("fed:")
300}
301
302pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
303 if harness_forces_untrusted(harness) {
304 BindTrust::Untrusted
305 } else {
306 trust_for_principal(principal)
307 }
308}
309
310fn principal_id(principal: &Option<Principal>) -> Option<String> {
311 match principal {
312 Some(Principal::Direct) => Some("direct".to_string()),
313 Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
314 Some(Principal::Unverified) => Some("unverified".to_string()),
315 None => None,
316 }
317}
318
319fn principal_label(principal: &Option<Principal>) -> String {
320 principal_id(principal).unwrap_or_else(|| "absent".to_string())
321}
322
323#[derive(Debug)]
324struct RootMeta {
330 maintenance_pending: bool,
331 maintenance_jobs_in_flight: usize,
332 maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
333 maintenance_last_submitted: Option<Instant>,
334 maintenance_poisoned: bool,
335 last_touched: Instant,
336 diagnostics_on_edit: bool,
337 active_bash_waits: usize,
338 idle_artifacts_evicted: bool,
339 unbound_quiesced: bool,
340 consecutive_missing_sweeps: u8,
341}
342
343#[derive(Debug)]
344struct PendingBind {
345 bind_root_id: ProjectRootId,
346 inserted_new_actor: bool,
347 cancelled: bool,
348 configure_request_id: String,
349 started_at: Instant,
350 warned_half_deadline: bool,
351 deadline_reported: bool,
352 corr: u64,
353 ver: u8,
354 flags: Flags,
355 cancellation: crate::executor::JobCancellation,
360}
361
362struct RouteBindCompletion {
363 route: RouteChannel,
364 identity: RouteIdentity,
365 bind_root_id: ProjectRootId,
366 inserted_new_actor: bool,
367 configure_response: Response,
368 diagnostics_on_edit: bool,
369 ver: u8,
370 corr: u64,
371 flags: Flags,
372}
373
374#[derive(Debug, Clone)]
375struct RouteIdentity(Arc<RouteIdentityData>);
376
377#[derive(Debug)]
378struct RouteIdentityData {
379 root: ProjectRootId,
380 project_root: PathBuf,
381 harness: String,
382 session: String,
383 trust: BindTrust,
384 spawn_principal: AuthenticatedPrincipal,
385 consumer_elicitation_capable: bool,
386}
387
388impl Deref for RouteIdentity {
389 type Target = RouteIdentityData;
390
391 fn deref(&self) -> &Self::Target {
392 &self.0
393 }
394}
395
396#[derive(Debug, Clone)]
397struct RetainedSessionIdentity {
398 harness: String,
399 trust: BindTrust,
400}
401
402#[derive(Clone, Copy)]
403struct BgSub {
404 corr: u64,
405 ver: u8,
406 flags: Flags,
407}
408
409struct MaintenanceCompletion {
410 root_id: ProjectRootId,
411 kind: MaintenanceDrainKind,
412 response: Response,
413 empty_bg_sessions: Vec<(String, u64)>,
414 requeue_kind: Option<MaintenanceDrainKind>,
415}
416
417#[derive(Clone, Copy, Debug, PartialEq, Eq)]
418enum MaintenanceDrainKind {
419 Watcher,
420 Lsp,
421 ConfigureTail,
422 CompletionDrains,
423}
424
425impl MaintenanceDrainKind {
426 fn label(self) -> &'static str {
427 match self {
428 Self::Watcher => "watcher",
429 Self::Lsp => "lsp",
430 Self::ConfigureTail => "configure-tail",
431 Self::CompletionDrains => "completion-drains",
432 }
433 }
434}
435
436#[derive(Debug, Default)]
437struct MaintenanceJobOutcome {
438 empty_bg_sessions: Vec<(String, u64)>,
439 requeue_kind: Option<MaintenanceDrainKind>,
440}
441
442#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
443struct ReverseCorrKey {
444 route: RouteChannel,
445 corr: u64,
446}
447
448struct PendingBashAsk {
449 route: RouteChannel,
450 tool_corr: u64,
451 tool_flags: Flags,
452 tool_ver: u8,
453 root: ProjectRootId,
454 project_root: PathBuf,
455 session_id: String,
456 spawn_principal: AuthenticatedPrincipal,
457 request_id: String,
458 arguments: Value,
459 format_context: crate::subc_format::FormatContext,
460 cancel: bash::BashWaitCancel,
461 grants: Vec<String>,
462 expires_at: Instant,
463}
464
465impl RootMeta {
466 fn new(now: Instant) -> Self {
467 Self {
468 maintenance_pending: false,
469 maintenance_jobs_in_flight: 0,
470 maintenance_queued_kinds: VecDeque::new(),
471 maintenance_last_submitted: None,
472 maintenance_poisoned: false,
473 last_touched: now,
474 diagnostics_on_edit: false,
475 active_bash_waits: 0,
476 idle_artifacts_evicted: false,
477 unbound_quiesced: false,
478 consecutive_missing_sweeps: 0,
479 }
480 }
481
482 fn note_activity(&mut self) {
483 self.last_touched = Instant::now();
484 }
485
486 fn reactivate_bound(&mut self) {
487 self.note_activity();
488 self.idle_artifacts_evicted = false;
489 self.unbound_quiesced = false;
490 }
491}
492
493fn due_maintenance_jobs(
494 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
495 executor: Option<&Executor>,
496 bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
497 bg_wake_pending: &HashSet<RouteChannel>,
498 budget: usize,
499 pending_bind_roots: &HashSet<ProjectRootId>,
500) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
501 let mut jobs = Vec::new();
502 let mut deferred = false;
503 let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
504 roots.sort_by(|left, right| {
505 let left_last = live_roots
506 .get(left)
507 .and_then(|meta| meta.maintenance_last_submitted);
508 let right_last = live_roots
509 .get(right)
510 .and_then(|meta| meta.maintenance_last_submitted);
511 left_last
512 .cmp(&right_last)
513 .then_with(|| left.as_path().cmp(right.as_path()))
514 });
515
516 for root_id in roots {
517 let Some(meta) = live_roots.get_mut(&root_id) else {
518 continue;
519 };
520 if meta.maintenance_poisoned {
521 continue;
522 }
523
524 if pending_bind_roots.contains(&root_id) {
525 if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
526 deferred = true;
527 }
528 continue;
529 }
530
531 if !meta.maintenance_pending {
532 if jobs.len() >= budget {
533 deferred = true;
534 continue;
535 }
536 let executor_actor_context =
540 executor.and_then(|executor| executor.actor_context(&root_id));
541 let root_has_pending_bg_wake =
542 bg_sub_by_session.iter().any(|((sub_root, _), channel)| {
543 sub_root == &root_id && bg_wake_pending.contains(channel)
544 });
545 let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
546 Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
547 .into_iter()
548 .filter(|kind| {
549 if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
550 return false;
551 }
552 match kind {
553 MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
554 MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
555 MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
556 MaintenanceDrainKind::CompletionDrains => {
562 root_has_pending_bg_wake || ctx.completion_drains_have_work()
563 }
564 }
565 })
566 .collect(),
567 None if meta.unbound_quiesced => Vec::new(),
568 None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
570 };
571 if kinds_with_work.is_empty() {
572 continue;
573 }
574 meta.maintenance_pending = true;
575 meta.maintenance_queued_kinds.extend(kinds_with_work);
576 }
577
578 while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
579 if jobs.len() >= budget {
580 meta.maintenance_queued_kinds.push_front(kind);
581 deferred = true;
582 break;
583 }
584 meta.maintenance_jobs_in_flight += 1;
585 meta.maintenance_last_submitted = Some(Instant::now());
586 jobs.push((root_id.clone(), kind));
587 }
588
589 meta.maintenance_pending =
590 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
591 }
592
593 (jobs, deferred)
594}
595
596fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
597 match estimate.estimated_bytes {
598 Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
599 None if estimate.status == "busy" => "busy".to_string(),
600 None => "not estimated".to_string(),
601 }
602}
603
604fn optional_memory_label(bytes: Option<u64>) -> String {
605 bytes.map_or_else(
606 || "not estimated".to_string(),
607 |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
608 )
609}
610
611fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
612 format!(
613 "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
614 optional_memory_label(relief.rss_before_bytes),
615 optional_memory_label(relief.rss_after_bytes),
616 optional_memory_label(relief.allocator_before.bytes_in_use),
617 optional_memory_label(relief.allocator_after.bytes_in_use),
618 optional_memory_label(relief.allocator_before.size_allocated),
619 optional_memory_label(relief.allocator_after.size_allocated),
620 optional_memory_label(relief.allocator_before.retained_slack_bytes),
621 optional_memory_label(relief.allocator_after.retained_slack_bytes),
622 relief.bytes_released as f64 / (1024.0 * 1024.0),
623 )
624}
625
626fn idle_root_eviction_message(
627 root_id: &ProjectRootId,
628 memory: &crate::memory::RootMemorySnapshot,
629 pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
630) -> String {
631 let freed_bytes = [
634 &memory.semantic,
635 &memory.trigram,
636 &memory.symbols,
637 &memory.callgraph,
638 &memory.inspect,
639 ]
640 .iter()
641 .filter_map(|estimate| estimate.estimated_bytes)
642 .fold(0u64, u64::saturating_add);
643 let mut message = format!(
644 "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
645 root_id.as_path().display(),
646 freed_bytes as f64 / (1024.0 * 1024.0),
647 eviction_estimate_label(&memory.semantic),
648 eviction_estimate_label(&memory.trigram),
649 eviction_estimate_label(&memory.symbols),
650 eviction_estimate_label(&memory.callgraph),
651 eviction_estimate_label(&memory.inspect),
652 eviction_estimate_label(&memory.bash),
653 eviction_estimate_label(&memory.lsp),
654 eviction_estimate_label(&memory.parser_pool),
655 );
656 if let Some(pressure_relief) = pressure_relief {
657 message.push_str(&pressure_relief_label(pressure_relief));
658 }
659 message
660}
661
662fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
663 !live_roots.is_empty()
664 && live_roots.values().all(|meta| {
665 now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
666 && meta.active_bash_waits == 0
667 && !meta.maintenance_pending
668 && meta.maintenance_queued_kinds.is_empty()
669 })
670}
671
672fn allocator_pressure_relief_after_idle_sweep(
673 now: Instant,
674 live_roots: &HashMap<ProjectRootId, RootMeta>,
675 executor: &Executor,
676) -> Option<crate::memory::AllocatorPressureRelief> {
677 if !process_has_been_idle(now, live_roots)
678 || live_roots.keys().any(|root_id| {
679 executor
680 .actor_context(root_id)
681 .is_some_and(|ctx| ctx.artifact_eviction_blocked())
682 })
683 {
684 return None;
685 }
686
687 #[cfg(target_os = "macos")]
688 {
689 Some(crate::memory::relieve_allocator_pressure())
690 }
691 #[cfg(not(target_os = "macos"))]
692 {
693 None
694 }
695}
696
697fn quiesce_unbound_root(
698 root_id: &ProjectRootId,
699 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
700 executor: &Arc<Executor>,
701) {
702 let Some(meta) = live_roots.get_mut(root_id) else {
703 return;
704 };
705
706 let ctx = executor.actor_context(root_id);
707 if let Some(ctx) = ctx.as_ref() {
708 ctx.mark_subc_unbound();
712 }
713 let cancelled = executor.cancel_queued_maintenance(root_id);
714 let discarded = ctx
722 .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
723 .unwrap_or(0);
724 meta.unbound_quiesced = true;
725 meta.maintenance_queued_kinds.clear();
726 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
727 log::debug!(
728 "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s))",
729 root_id.as_path().display(),
730 cancelled,
731 discarded
732 );
733}
734
735#[allow(clippy::too_many_arguments)]
736fn quiesce_connection_roots(
737 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
738 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
739 routes: &mut HashMap<RouteChannel, RouteIdentity>,
740 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
741 installed_route_epochs: &mut HashMap<u16, u32>,
742 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
743 executor: &Arc<Executor>,
744) {
745 for cancel in route_bash_cancels.values() {
746 cancel.token.cancel();
747 }
748 route_bash_cancels.clear();
749
750 let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
751 for pending in pending_binds.values_mut() {
752 pending.cancelled = true;
753 roots.insert(pending.bind_root_id.clone());
754 let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
755 }
756
757 for root_id in roots {
761 if live_roots.contains_key(&root_id) {
762 quiesce_unbound_root(&root_id, live_roots, executor);
763 } else if let Some(ctx) = executor.actor_context(&root_id) {
764 ctx.mark_subc_unbound();
765 executor.cancel_queued_maintenance(&root_id);
766 crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
767 }
768 }
769
770 routes.clear();
771 root_channels.clear();
772 installed_route_epochs.clear();
773}
774
775#[derive(Debug, Default)]
779struct ReclaimedRoutes {
780 highest_epoch_by_channel: HashMap<u16, u32>,
781}
782
783impl ReclaimedRoutes {
784 fn insert(&mut self, route: RouteChannel) {
785 self.highest_epoch_by_channel
786 .entry(route.channel)
787 .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
788 .or_insert(route.epoch);
789 }
790
791 fn contains(&self, route: RouteChannel) -> bool {
792 self.highest_epoch_by_channel
793 .get(&route.channel)
794 .is_some_and(|epoch| route.epoch <= *epoch)
795 }
796}
797
798#[derive(Debug, Default)]
799struct IdleReapOutcome {
800 evicted: usize,
801 forgotten_deleted_roots: Vec<ProjectRootId>,
802}
803
804fn reap_idle_roots(
805 now: Instant,
806 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
807 pending_binds: &HashMap<RouteChannel, PendingBind>,
808 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
809 executor: &Arc<Executor>,
810 metrics: &DispatchPathMetrics,
811) -> IdleReapOutcome {
812 let pending_bind_roots = pending_binds
813 .values()
814 .map(|pending| pending.bind_root_id.clone())
815 .collect::<HashSet<_>>();
816 let mut census = ReapBlockerCensus::default();
817 let mut candidates = Vec::new();
818
819 for (root_id, meta) in live_roots.iter_mut() {
820 let deleted = !root_id.as_path().exists();
821 if deleted {
822 meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
832 } else {
833 meta.consecutive_missing_sweeps = 0;
834 }
835 let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
836 let has_bound_route = root_channels
837 .get(root_id)
838 .is_some_and(|channels| !channels.is_empty());
839 let has_pending_bind = pending_bind_roots.contains(root_id);
840
841 if deleted {
842 let mut retained = false;
843 if !deletion_confirmed {
844 census.absence_unconfirmed += 1;
845 retained = true;
846 }
847 if meta.active_bash_waits > 0 {
851 census.bash_waits += 1;
852 retained = true;
853 }
854 if meta.maintenance_pending {
855 census.maintenance_pending += 1;
856 retained = true;
857 }
858 if !meta.maintenance_queued_kinds.is_empty() {
859 census.maintenance_queued += 1;
860 retained = true;
861 }
862 if has_pending_bind {
863 census.pending_binds += 1;
864 retained = true;
865 }
866 match executor.try_actor_is_idle(root_id) {
867 Some(true) => {}
868 Some(false) => {
869 census.actor_busy += 1;
870 retained = true;
871 }
872 None => {
873 census.actor_state_busy += 1;
874 retained = true;
875 }
876 }
877 if retained {
878 census.deleted_retained += 1;
879 continue;
880 }
881 } else {
882 if has_bound_route
886 || !meta.unbound_quiesced
887 || meta.idle_artifacts_evicted
888 || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
889 || meta.active_bash_waits > 0
890 || meta.maintenance_pending
891 || !meta.maintenance_queued_kinds.is_empty()
892 || has_pending_bind
893 || !executor.actor_is_idle(root_id)
894 {
895 continue;
896 }
897 }
898 candidates.push((root_id.clone(), deleted));
899 }
900
901 let mut reaped = Vec::new();
902 let mut forgotten_deleted_roots = Vec::new();
903 for (root_id, deleted) in candidates {
904 let Some(ctx) = executor.actor_context(&root_id) else {
905 if deleted {
906 census.deleted_retained += 1;
907 census.actor_busy += 1;
908 }
909 continue;
910 };
911 if deleted {
922 ctx.bash_background()
923 .kill_running_tasks_for_root(root_id.as_path());
924 }
925 let taken_pending = Some(ctx.take_pending_reconciliation_state());
926 if ctx.artifact_eviction_blocked() {
927 if let Some(pending) = taken_pending {
928 ctx.restore_pending_reconciliation_state(pending);
929 }
930 if deleted {
931 census.deleted_retained += 1;
932 census.artifact_eviction_blocked += 1;
933 }
934 continue;
935 }
936 let memory_before = ctx.memory_root_snapshot();
937 if !ctx.evict_idle_artifacts() {
938 if let Some(pending) = taken_pending {
939 ctx.restore_pending_reconciliation_state(pending);
940 }
941 if deleted {
942 census.deleted_retained += 1;
943 census.artifact_eviction_failed += 1;
944 }
945 continue;
946 }
947 drop(taken_pending);
948 ctx.stop_watcher_runtime_in_background();
949 ctx.invalidate_artifacts_after_watcher_gap();
952
953 if deleted {
954 if executor.retire_idle_actor_in_background(&root_id) {
955 live_roots.remove(&root_id);
956 forgotten_deleted_roots.push(root_id.clone());
957 } else {
958 census.deleted_retained += 1;
959 census.actor_busy += 1;
960 }
961 } else {
962 if let Some(meta) = live_roots.get_mut(&root_id) {
963 meta.idle_artifacts_evicted = true;
964 }
965 ctx.release_idle_reopenable_resources_in_background();
966 }
967 reaped.push((root_id, memory_before));
968 }
969
970 metrics.record_reap(census);
971 if census.deleted_retained > 0 {
972 log::info!(
973 "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
974 census.deleted_retained,
975 census.blocker_histogram()
976 );
977 }
978
979 let pressure_relief = (!reaped.is_empty())
980 .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
981 .flatten();
982 for (root_id, memory_before) in &reaped {
983 log::info!(
984 "{}",
985 idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
986 );
987 }
988 IdleReapOutcome {
989 evicted: reaped.len(),
990 forgotten_deleted_roots,
991 }
992}
993
994#[allow(clippy::too_many_arguments)]
995fn purge_deleted_root_residents(
996 root_id: &ProjectRootId,
997 routes: &mut HashMap<RouteChannel, RouteIdentity>,
998 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
999 installed_route_epochs: &mut HashMap<u16, u32>,
1000 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1001 retry_buffer: &mut RetryBuffer,
1002 reclaimed_routes: &mut ReclaimedRoutes,
1003 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1004 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1005 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1006 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1007 bg_wake_pending: &mut HashSet<RouteChannel>,
1008 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1009 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1010) {
1011 let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1012 stale_routes.extend(
1013 routes
1014 .iter()
1015 .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1016 );
1017 stale_routes.extend(
1018 bg_sub_by_session
1019 .iter()
1020 .filter_map(|((root, _), route)| (root == root_id).then_some(*route)),
1021 );
1022 stale_routes.extend(
1023 pending_bash_asks
1024 .values()
1025 .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1026 );
1027
1028 for route in stale_routes {
1029 reclaimed_routes.insert(route);
1030 remove_installed_route(installed_route_epochs, route);
1031 remove_route_channel(routes, root_channels, route);
1032 if let Some(cancel) = route_bash_cancels.remove(&route) {
1033 cancel.token.cancel();
1034 }
1035 retry_buffer.remove(&route);
1036 bg_subs.remove(&route);
1037 bg_wake_pending.remove(&route);
1038 }
1039 root_channels.remove(root_id);
1040 session_identity.retain(|(root, _), _| root != root_id);
1041 push_buffer.retain(|key, _| &key.root != root_id);
1042 bg_wake_epoch.retain(|(root, _), _| root != root_id);
1043 pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1044 bg_sub_by_session.retain(|(root, _), _| root != root_id);
1045
1046 log::info!(
1047 "subc attach: fully forgot deleted root {}",
1048 root_id.as_path().display()
1049 );
1050}
1051
1052#[allow(clippy::too_many_arguments)]
1053fn submit_due_maintenance_jobs(
1054 executor: &Arc<Executor>,
1055 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1056 pending_binds: &HashMap<RouteChannel, PendingBind>,
1057 bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
1058 bg_wake_pending: &HashSet<RouteChannel>,
1059 bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1060 maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1061 metrics: &Arc<DispatchPathMetrics>,
1062) {
1063 let pending_bind_roots = pending_binds
1064 .values()
1065 .map(|pending| pending.bind_root_id.clone())
1066 .collect::<HashSet<_>>();
1067 let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1068 live_roots,
1069 Some(executor),
1070 bg_sub_by_session,
1071 bg_wake_pending,
1072 MAINTENANCE_SUBMIT_BUDGET,
1073 &pending_bind_roots,
1074 );
1075 if deferred_jobs {
1076 metrics
1077 .maintenance_budget_deferrals
1078 .fetch_add(1, Ordering::Relaxed);
1079 }
1080 for (root_id, kind) in due_jobs {
1081 let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1082 bg_sub_by_session
1083 .iter()
1084 .filter_map(|((root, session), _)| {
1085 if root == &root_id {
1086 Some((
1087 session.clone(),
1088 bg_wake_epoch
1089 .get(&(root_id.clone(), session.clone()))
1090 .copied()
1091 .unwrap_or(0),
1092 ))
1093 } else {
1094 None
1095 }
1096 })
1097 .collect()
1098 } else {
1099 Vec::new()
1100 };
1101 submit_maintenance_job(
1102 executor,
1103 root_id,
1104 kind,
1105 bg_sessions_to_check,
1106 maintenance_tx,
1107 metrics,
1108 );
1109 }
1110}
1111
1112fn should_requiesce_after_maintenance(
1113 meta: &RootMeta,
1114 completed_kind: MaintenanceDrainKind,
1115 bind_pending: bool,
1116) -> bool {
1117 meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1118}
1119
1120fn note_maintenance_completion(
1121 meta: &mut RootMeta,
1122 requeue_kind: Option<MaintenanceDrainKind>,
1123 fatal: bool,
1124 defer_requeue: bool,
1125) {
1126 if fatal {
1127 meta.maintenance_poisoned = true;
1128 }
1129
1130 if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1131 meta.maintenance_queued_kinds.push_back(kind);
1132 }
1133
1134 meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1135 meta.maintenance_pending =
1136 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1137}
1138
1139fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1140 RouteChannel { channel, epoch }
1141}
1142
1143fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1144 if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1145 installed_epochs.remove(&route.channel);
1146 }
1147}
1148
1149fn ingress_route_should_be_processed(
1150 installed_epochs: &HashMap<u16, u32>,
1151 reclaimed_routes: &ReclaimedRoutes,
1152 frame: &Frame,
1153) -> bool {
1154 if frame.header.channel == 0
1155 || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1156 {
1157 return true;
1158 }
1159
1160 frame.header.ty == FrameType::Request
1165 && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1166}
1167
1168fn bash_elicitation_timeout() -> Duration {
1169 if cfg!(debug_assertions) {
1170 if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1171 if let Ok(ms) = raw.parse::<u64>() {
1172 if ms > 0 {
1173 return Duration::from_millis(ms);
1174 }
1175 }
1176 }
1177 }
1178 BASH_ELICITATION_TIMEOUT
1179}
1180
1181fn allocate_reverse_corr(
1182 pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1183 route: RouteChannel,
1184 next_corr: &mut u64,
1185) -> u64 {
1186 loop {
1187 let corr = *next_corr;
1188 *next_corr = (*next_corr).wrapping_add(1).max(1);
1189 if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1190 return corr;
1191 }
1192 }
1193}
1194
1195fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1196 match kind {
1197 crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1198 crate::bash_permissions::PermissionKind::Bash => "bash",
1199 }
1200}
1201
1202fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1203 let mut patterns = Vec::new();
1204 let mut seen = HashSet::new();
1205 for ask in asks {
1206 for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1207 if seen.insert(pattern.clone()) {
1208 patterns.push(pattern.clone());
1209 }
1210 }
1211 }
1212 patterns
1213}
1214
1215fn bash_elicitation_message(
1216 command: &str,
1217 asks: &[crate::bash_permissions::PermissionAsk],
1218) -> String {
1219 let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1220 let patterns = bash_elicitation_patterns(asks);
1221 let pattern_text = if patterns.is_empty() {
1222 "no matched permission patterns".to_string()
1223 } else {
1224 patterns.join(", ")
1225 };
1226 let ask_kinds = asks
1227 .iter()
1228 .map(|ask| bash_permission_kind_label(&ask.kind))
1229 .collect::<HashSet<_>>()
1230 .into_iter()
1231 .collect::<Vec<_>>()
1232 .join(", ");
1233 if ask_kinds.is_empty() {
1234 format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1235 } else {
1236 format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1237 }
1238}
1239
1240fn bash_elicitation_request_body(
1241 command: &str,
1242 asks: &[crate::bash_permissions::PermissionAsk],
1243) -> Value {
1244 json!({
1245 "method": BASH_ELICITATION_CREATE_METHOD,
1246 "params": {
1247 "mode": "form",
1248 "message": bash_elicitation_message(command, asks),
1249 "requestedSchema": {
1250 "type": "object",
1251 "properties": {
1252 "decision": {
1253 "type": "string",
1254 "enum": ["allow", "deny"],
1255 "description": "Choose allow to run this bash command once, or deny to block it."
1256 }
1257 },
1258 "required": ["decision"],
1259 "additionalProperties": false
1260 },
1261 "_meta": {
1262 "aft": {
1263 "tool": "bash",
1264 "command": command,
1265 "asks": asks
1266 }
1267 }
1268 }
1269 })
1270}
1271
1272fn build_bash_elicitation_request_frame(
1273 ver: u8,
1274 route: RouteChannel,
1275 corr: u64,
1276 flags: Flags,
1277 command: &str,
1278 asks: &[crate::bash_permissions::PermissionAsk],
1279) -> Result<Frame, SubcError> {
1280 let body = bash_elicitation_request_body(command, asks);
1281 Frame::build_with_version(
1282 ver,
1283 FrameType::Request,
1284 flags,
1285 route.channel,
1286 route.epoch,
1287 corr,
1288 serde_json::to_vec(&body).map_err(SubcError::Json)?,
1289 )
1290 .map_err(SubcError::FrameBuild)
1291}
1292
1293fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1294 let Ok(value) = serde_json::from_slice::<Value>(body) else {
1295 return false;
1296 };
1297 flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1298}
1299
1300fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1301 let Some(object) = value.as_object() else {
1302 return false;
1303 };
1304 object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1305}
1306
1307fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1308 let Some(object) = value.as_object() else {
1309 return false;
1310 };
1311 if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1312 return false;
1313 }
1314 let Some(content) = object.get("content").and_then(Value::as_object) else {
1315 return false;
1316 };
1317 content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1318}
1319
1320#[allow(clippy::too_many_arguments)]
1321async fn settle_pending_bash_ask_denied(
1322 tx: &WriterSender,
1323 pending: PendingBashAsk,
1324 routes: &HashMap<RouteChannel, RouteIdentity>,
1325 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1326 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1327 shutdown: &Arc<Notify>,
1328 metrics: &DispatchPathMetrics,
1329) -> Result<(), SubcError> {
1330 let completion = bash::bash_denied_untrusted_completion(
1331 pending.route,
1332 pending.tool_corr,
1333 pending.tool_flags,
1334 pending.tool_ver,
1335 pending.root,
1336 pending.request_id,
1337 pending.format_context,
1338 );
1339 bash::handle_bash_deferred_completion(
1340 tx,
1341 completion,
1342 routes,
1343 live_roots,
1344 route_bash_cancels,
1345 shutdown,
1346 metrics,
1347 )
1348 .await
1349}
1350
1351fn take_pending_bash_asks_for_route(
1352 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1353 route: RouteChannel,
1354) -> Vec<PendingBashAsk> {
1355 let keys = pending_bash_asks
1356 .keys()
1357 .copied()
1358 .filter(|key| key.route == route)
1359 .collect::<Vec<_>>();
1360 keys.into_iter()
1361 .filter_map(|key| pending_bash_asks.remove(&key))
1362 .collect()
1363}
1364
1365#[allow(clippy::too_many_arguments)]
1366async fn settle_pending_bash_asks_for_route(
1367 tx: &WriterSender,
1368 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1369 route: RouteChannel,
1370 routes: &HashMap<RouteChannel, RouteIdentity>,
1371 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1372 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1373 shutdown: &Arc<Notify>,
1374 metrics: &DispatchPathMetrics,
1375) -> Result<(), SubcError> {
1376 for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1377 settle_pending_bash_ask_denied(
1378 tx,
1379 pending,
1380 routes,
1381 live_roots,
1382 route_bash_cancels,
1383 shutdown,
1384 metrics,
1385 )
1386 .await?;
1387 }
1388 Ok(())
1389}
1390
1391#[allow(clippy::too_many_arguments)]
1392async fn settle_all_pending_bash_asks(
1393 tx: &WriterSender,
1394 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1395 routes: &HashMap<RouteChannel, RouteIdentity>,
1396 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1397 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1398 shutdown: &Arc<Notify>,
1399 metrics: &DispatchPathMetrics,
1400) -> Result<(), SubcError> {
1401 let pending = pending_bash_asks
1402 .drain()
1403 .map(|(_, pending)| pending)
1404 .collect::<Vec<_>>();
1405 for pending in pending {
1406 settle_pending_bash_ask_denied(
1407 tx,
1408 pending,
1409 routes,
1410 live_roots,
1411 route_bash_cancels,
1412 shutdown,
1413 metrics,
1414 )
1415 .await?;
1416 }
1417 Ok(())
1418}
1419
1420#[allow(clippy::too_many_arguments)]
1421async fn expire_pending_bash_asks(
1422 tx: &WriterSender,
1423 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1424 routes: &HashMap<RouteChannel, RouteIdentity>,
1425 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1426 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1427 shutdown: &Arc<Notify>,
1428 metrics: &DispatchPathMetrics,
1429) -> Result<(), SubcError> {
1430 let now = Instant::now();
1431 let expired = pending_bash_asks
1432 .iter()
1433 .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1434 .collect::<Vec<_>>();
1435 for key in expired {
1436 if let Some(pending) = pending_bash_asks.remove(&key) {
1437 log::debug!(
1438 "subc attach: bash elicitation request {} on route {} expired fail-closed",
1439 key.corr,
1440 pending.route
1441 );
1442 settle_pending_bash_ask_denied(
1443 tx,
1444 pending,
1445 routes,
1446 live_roots,
1447 route_bash_cancels,
1448 shutdown,
1449 metrics,
1450 )
1451 .await?;
1452 }
1453 }
1454 Ok(())
1455}
1456
1457#[allow(clippy::too_many_arguments)]
1458async fn handle_bash_elicitation_reply(
1459 tx: &WriterSender,
1460 frame: &Frame,
1461 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1462 routes: &HashMap<RouteChannel, RouteIdentity>,
1463 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1464 executor: &Arc<Executor>,
1465 shutdown: &Arc<Notify>,
1466 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1467 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1468 metrics: &Arc<DispatchPathMetrics>,
1469 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1470 dispatch: DispatchFn,
1471) -> Result<(), SubcError> {
1472 let key = ReverseCorrKey {
1473 route: route_key(frame.header.channel, frame.header.epoch),
1474 corr: frame.header.corr,
1475 };
1476 let Some(pending) = pending_bash_asks.remove(&key) else {
1477 return Ok(());
1478 };
1479
1480 if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1481 if routes.contains_key(&key.route) {
1482 bash::submit_deferred_bash(
1483 executor,
1484 bash_deferred_tx,
1485 bash_poll_touch_tx,
1486 metrics,
1487 dispatch,
1488 pending.root,
1489 pending.project_root,
1490 pending.session_id,
1491 pending.request_id,
1492 pending.route,
1493 pending.tool_corr,
1494 pending.tool_flags,
1495 pending.tool_ver,
1496 pending.arguments,
1497 pending.format_context,
1498 pending.cancel,
1499 BindTrust::Untrusted,
1500 pending.spawn_principal,
1501 Some(pending.grants),
1502 );
1503 return Ok(());
1504 }
1505 log::debug!(
1506 "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1507 key.corr,
1508 pending.route
1509 );
1510 }
1511
1512 settle_pending_bash_ask_denied(
1513 tx,
1514 pending,
1515 routes,
1516 live_roots,
1517 route_bash_cancels,
1518 shutdown,
1519 metrics,
1520 )
1521 .await
1522}
1523
1524#[allow(clippy::too_many_arguments)]
1525async fn cancel_pending_bash_ask_for_tool_call(
1526 tx: &WriterSender,
1527 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1528 route: RouteChannel,
1529 tool_corr: u64,
1530 routes: &HashMap<RouteChannel, RouteIdentity>,
1531 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1532 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1533 shutdown: &Arc<Notify>,
1534 metrics: &DispatchPathMetrics,
1535) -> Result<(), SubcError> {
1536 let keys = pending_bash_asks
1537 .iter()
1538 .filter_map(|(key, pending)| {
1539 (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1540 })
1541 .collect::<Vec<_>>();
1542 for key in keys {
1543 if let Some(pending) = pending_bash_asks.remove(&key) {
1544 settle_pending_bash_ask_denied(
1545 tx,
1546 pending,
1547 routes,
1548 live_roots,
1549 route_bash_cancels,
1550 shutdown,
1551 metrics,
1552 )
1553 .await?;
1554 }
1555 }
1556 Ok(())
1557}
1558
1559fn remove_root_channel(
1560 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1561 root: &ProjectRootId,
1562 channel: RouteChannel,
1563) {
1564 let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1565 channels.remove(&channel);
1566 channels.is_empty()
1567 } else {
1568 false
1569 };
1570 if remove_root {
1571 root_channels.remove(root);
1572 }
1573}
1574
1575fn remove_route_channel(
1576 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1577 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1578 channel: RouteChannel,
1579) -> Option<RouteIdentity> {
1580 let removed = routes.remove(&channel);
1581 if let Some(identity) = &removed {
1582 remove_root_channel(root_channels, &identity.root, channel);
1583 }
1584 removed
1585}
1586
1587fn insert_route_channel(
1588 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1589 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1590 channel: RouteChannel,
1591 identity: RouteIdentity,
1592) {
1593 if let Some(previous) = routes.insert(channel, identity.clone()) {
1594 remove_root_channel(root_channels, &previous.root, channel);
1595 }
1596 root_channels
1597 .entry(identity.root.clone())
1598 .or_default()
1599 .insert(channel);
1600}
1601
1602fn remove_bg_subscription_index(
1603 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1604 channel: RouteChannel,
1605 identity: Option<&RouteIdentity>,
1606) {
1607 if let Some(identity) = identity {
1608 let key = (identity.root.clone(), identity.session.clone());
1609 if bg_sub_by_session.get(&key).copied() == Some(channel) {
1610 bg_sub_by_session.remove(&key);
1611 }
1612 } else {
1613 bg_sub_by_session.retain(|_, mapped_channel| *mapped_channel != channel);
1614 }
1615}
1616
1617fn route_removal_will_quiesce_root(
1618 root: &ProjectRootId,
1619 route: RouteChannel,
1620 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1621 has_pending_bind: bool,
1622 replacement_root: Option<&ProjectRootId>,
1623) -> bool {
1624 let removes_last_route = root_channels
1625 .get(root)
1626 .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1627 removes_last_route && !has_pending_bind && replacement_root != Some(root)
1628}
1629
1630fn should_quiesce_removed_root(
1631 root: &ProjectRootId,
1632 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1633 has_pending_bind: bool,
1634 replacement_root: Option<&ProjectRootId>,
1635) -> bool {
1636 !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1637}
1638
1639async fn end_bg_subscription(
1640 writer_tx: &WriterSender,
1641 metrics: &DispatchPathMetrics,
1642 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1643 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1644 bg_wake_pending: &mut HashSet<RouteChannel>,
1645 channel: RouteChannel,
1646 identity: Option<&RouteIdentity>,
1647) -> Result<(), SubcError> {
1648 if let Some(sub) = bg_subs.remove(&channel) {
1649 bg_wake_pending.remove(&channel);
1650 remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1651 push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1652 }
1653 Ok(())
1654}
1655
1656#[allow(clippy::too_many_arguments)]
1657async fn teardown_installed_route(
1658 tx: &WriterSender,
1659 metrics: &DispatchPathMetrics,
1660 executor: &Arc<Executor>,
1661 channel: RouteChannel,
1662 cancellation_reason: &str,
1663 replacement_root: Option<&ProjectRootId>,
1664 installed_route_epochs: &mut HashMap<u16, u32>,
1665 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1666 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1667 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1668 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1669 bg_wake_pending: &mut HashSet<RouteChannel>,
1670 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1671 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1672 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1673 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1674 retry_buffer: &mut RetryBuffer,
1675 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1676 shutdown: &Arc<Notify>,
1677) -> Result<(), SubcError> {
1678 remove_installed_route(installed_route_epochs, channel);
1679 end_bg_subscription(
1680 tx,
1681 metrics,
1682 bg_subs,
1683 bg_sub_by_session,
1684 bg_wake_pending,
1685 channel,
1686 routes.get(&channel),
1687 )
1688 .await?;
1689 settle_pending_bash_asks_for_route(
1690 tx,
1691 pending_bash_asks,
1692 channel,
1693 routes,
1694 live_roots,
1695 route_bash_cancels,
1696 shutdown,
1697 metrics,
1698 )
1699 .await?;
1700 if let Some(cancel) = route_bash_cancels.remove(&channel) {
1701 cancel.token.cancel();
1702 }
1703 if let Some(pending) = pending_binds.get_mut(&channel) {
1704 pending.cancelled = true;
1705 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1706 log::debug!(
1707 "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1708 channel.channel
1709 );
1710 }
1711 let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1712 if let Some(identity) = routes.get(&channel) {
1713 let has_pending_bind = pending_binds
1714 .values()
1715 .any(|pending| pending.bind_root_id == identity.root);
1716 if route_removal_will_quiesce_root(
1717 &identity.root,
1718 channel,
1719 root_channels,
1720 has_pending_bind,
1721 replacement_root,
1722 ) {
1723 if let Some(ctx) = executor.actor_context(&identity.root) {
1724 ctx.mark_subc_unbound();
1727 }
1728 }
1729 }
1730 if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
1731 if migrated > 0 {
1732 log::debug!(
1733 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
1734 channel.channel
1735 );
1736 }
1737 if let Some(meta) = live_roots.get_mut(&identity.root) {
1738 let idle_for = meta.last_touched.elapsed();
1739 meta.note_activity();
1740 log::debug!(
1741 "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
1742 channel.channel,
1743 identity.root.as_path().display(),
1744 identity.harness,
1745 identity.session,
1746 idle_for
1747 );
1748 } else {
1749 log::debug!(
1750 "subc attach: route {} torn down for root {} harness {} session {}",
1751 channel.channel,
1752 identity.root.as_path().display(),
1753 identity.harness,
1754 identity.session
1755 );
1756 }
1757 let has_pending_bind = pending_binds
1758 .values()
1759 .any(|pending| pending.bind_root_id == identity.root);
1760 if should_quiesce_removed_root(
1761 &identity.root,
1762 root_channels,
1763 has_pending_bind,
1764 replacement_root,
1765 ) {
1766 quiesce_unbound_root(&identity.root, live_roots, executor);
1767 }
1768 } else {
1769 if migrated > 0 {
1770 log::debug!(
1771 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
1772 channel.channel
1773 );
1774 }
1775 log::debug!("subc attach: unbound route {} torn down", channel.channel);
1776 }
1777 Ok(())
1778}
1779
1780fn remember_session_identity(
1781 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1782 identity: &RouteIdentity,
1783) {
1784 let key = (identity.root.clone(), identity.session.clone());
1785 if matches!(identity.trust, BindTrust::Untrusted)
1786 && session_identity
1787 .get(&key)
1788 .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
1789 {
1790 return;
1791 }
1792
1793 session_identity.insert(
1798 key,
1799 RetainedSessionIdentity {
1800 harness: identity.harness.clone(),
1801 trust: identity.trust,
1802 },
1803 );
1804}
1805
1806fn replay_key_for_session(
1807 session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1808 root: &ProjectRootId,
1809 session: &str,
1810) -> Option<(push::ReplayKey, BindTrust)> {
1811 let retained = session_identity.get(&(root.clone(), session.to_string()))?;
1812 Some((
1813 push::ReplayKey {
1814 root: root.clone(),
1815 harness: retained.harness.clone(),
1816 session: session.to_string(),
1817 },
1818 retained.trust,
1819 ))
1820}
1821pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
1824
1825#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1826enum ModuleLoopExit {
1827 Graceful,
1828 SkipSearchFlush,
1829}
1830
1831pub fn run_subc_mode(
1836 connection_file_path: &Path,
1837 ctx: Arc<AppContext>,
1838 executor: Arc<Executor>,
1839 dispatch: DispatchFn,
1840 user_config_path: Option<PathBuf>,
1841) -> Result<(), SubcError> {
1842 run_subc_mode_inner(
1846 connection_file_path,
1847 ctx,
1848 executor,
1849 dispatch,
1850 user_config_path,
1851 false,
1852 )
1853}
1854
1855fn run_subc_mode_inner(
1856 connection_file_path: &Path,
1857 ctx: Arc<AppContext>,
1858 executor: Arc<Executor>,
1859 dispatch: DispatchFn,
1860 user_config_path: Option<PathBuf>,
1861 allow_native_passthrough: bool,
1862) -> Result<(), SubcError> {
1863 let runtime = tokio::runtime::Builder::new_current_thread()
1864 .enable_all()
1865 .build()
1866 .map_err(SubcError::Runtime)?;
1867
1868 let executor_for_loop = Arc::clone(&executor);
1869 let loop_result = runtime.block_on(async move {
1870 let shared_app = ctx.app();
1871 drop(ctx);
1872 let stream = connect_and_authenticate(connection_file_path).await?;
1873 log::info!(
1874 "subc attach: authenticated to daemon via {}",
1875 connection_file_path.display()
1876 );
1877 let (read_half, write_half) = tokio::io::split(stream);
1878 run_module_loop(
1879 read_half,
1880 write_half,
1881 shared_app,
1882 executor_for_loop,
1883 dispatch,
1884 user_config_path,
1885 allow_native_passthrough,
1886 )
1887 .await
1888 });
1889
1890 let actor_contexts = executor.actor_contexts();
1891 if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
1892 flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
1895 }
1896 for actor_ctx in &actor_contexts {
1897 actor_ctx.lsp().shutdown_all();
1898 actor_ctx.bash_background().detach();
1899 }
1900
1901 loop_result.map(|_| ())
1902}
1903
1904fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
1905 for actor_ctx in actor_contexts {
1906 let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
1907 }
1908 let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
1909}
1910
1911#[doc(hidden)]
1916pub fn run_subc_mode_for_test(
1917 connection_file_path: &Path,
1918 ctx: Arc<AppContext>,
1919 executor: Arc<Executor>,
1920 dispatch: DispatchFn,
1921 user_config_path: Option<PathBuf>,
1922) -> Result<(), SubcError> {
1923 run_subc_mode_inner(
1924 connection_file_path,
1925 ctx,
1926 executor,
1927 dispatch,
1928 user_config_path,
1929 true,
1930 )
1931}
1932
1933#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1934enum AttachErrorClass {
1935 Transient,
1936 Permanent,
1937}
1938
1939impl fmt::Display for AttachErrorClass {
1940 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1941 match self {
1942 Self::Transient => f.write_str("transient"),
1943 Self::Permanent => f.write_str("permanent"),
1944 }
1945 }
1946}
1947
1948#[derive(Clone, Copy)]
1949struct AttachRetryPolicy {
1950 budget: Duration,
1951 initial_backoff: Duration,
1952 max_backoff: Duration,
1953 jitter_percent: u64,
1954}
1955
1956const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
1957 budget: ATTACH_RETRY_BUDGET,
1958 initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
1959 max_backoff: ATTACH_RETRY_MAX_BACKOFF,
1960 jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
1961};
1962
1963fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
1966 let transient = match error {
1967 SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
1968 SubcError::Auth { source, .. } => match source {
1969 subc_transport::AuthError::Timeout { .. }
1970 | subc_transport::AuthError::UnexpectedEof { .. } => true,
1971 subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
1972 _ => false,
1973 },
1974 _ => false,
1975 };
1976 if transient {
1977 AttachErrorClass::Transient
1978 } else {
1979 AttachErrorClass::Permanent
1980 }
1981}
1982
1983fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
1984 matches!(
1985 kind,
1986 io::ErrorKind::ConnectionRefused
1987 | io::ErrorKind::TimedOut
1988 | io::ErrorKind::ConnectionReset
1989 | io::ErrorKind::ConnectionAborted
1990 | io::ErrorKind::BrokenPipe
1991 | io::ErrorKind::UnexpectedEof
1992 )
1993}
1994
1995async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
1999 connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
2000}
2001
2002async fn connect_and_authenticate_with_policy(
2003 connection_file_path: &Path,
2004 policy: AttachRetryPolicy,
2005) -> Result<TcpStream, SubcError> {
2006 let started_at = Instant::now();
2007 let deadline = started_at + policy.budget;
2008 let mut attempt = 0_u32;
2009 let mut backoff = policy.initial_backoff;
2010 let mut history = Vec::new();
2011
2012 loop {
2013 attempt = attempt.saturating_add(1);
2014 let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2015 Ok(stream) => return Ok(stream),
2016 Err(error) => error,
2017 };
2018 let class = classify_attach_error(&error);
2019 let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2020 history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2021
2022 if class == AttachErrorClass::Permanent {
2023 log_attach_final_failure(started_at.elapsed(), &history);
2024 return Err(error);
2025 }
2026
2027 let remaining = deadline.saturating_duration_since(Instant::now());
2028 if remaining.is_zero() {
2029 log_attach_final_failure(started_at.elapsed(), &history);
2030 return Err(error);
2031 }
2032
2033 let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2034 log::info!(
2035 "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2036 );
2037 tokio::time::sleep(delay).await;
2038
2039 if Instant::now() >= deadline {
2040 log_attach_final_failure(started_at.elapsed(), &history);
2041 return Err(error);
2042 }
2043 backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2044 }
2045}
2046
2047fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2048 let jitter_percent = jitter_percent.min(100);
2049 if jitter_percent == 0 {
2050 return base;
2051 }
2052
2053 let mut random_bytes = [0_u8; 8];
2054 let random = if getrandom::fill(&mut random_bytes).is_ok() {
2055 u64::from_le_bytes(random_bytes)
2056 } else {
2057 let timestamp = std::time::SystemTime::now()
2058 .duration_since(std::time::UNIX_EPOCH)
2059 .unwrap_or_default()
2060 .subsec_nanos();
2061 u64::from(timestamp) ^ u64::from(attempt)
2062 };
2063 let span = jitter_percent.saturating_mul(2).saturating_add(1);
2064 let multiplier_percent = 100 - jitter_percent + random % span;
2065 let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2066 Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2067}
2068
2069fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2070 log::error!(
2071 "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2072 history.len(),
2073 history.join(" | ")
2074 );
2075}
2076
2077async fn connect_and_authenticate_once(
2078 connection_file_path: &Path,
2079 deadline: Instant,
2080) -> Result<TcpStream, SubcError> {
2081 let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2084 SubcError::ConnectionFile {
2085 path: connection_file_path.to_path_buf(),
2086 source,
2087 }
2088 })?;
2089
2090 let endpoint = conn
2091 .endpoints
2092 .first()
2093 .ok_or_else(|| SubcError::NoEndpoint {
2094 path: connection_file_path.to_path_buf(),
2095 })?;
2096 let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2097 let ip = endpoint
2098 .host
2099 .parse::<IpAddr>()
2100 .map_err(|_| SubcError::InvalidEndpoint {
2101 path: connection_file_path.to_path_buf(),
2102 endpoint: endpoint_label.clone(),
2103 })?;
2104 let addr = SocketAddr::new(ip, endpoint.port);
2105
2106 let connect_budget = deadline.saturating_duration_since(Instant::now());
2107 let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2108 .await
2109 .map_err(|_| SubcError::Connect {
2110 endpoint: endpoint_label.clone(),
2111 source: io::Error::new(
2112 io::ErrorKind::TimedOut,
2113 "initial subc attach retry budget elapsed during TCP connect",
2114 ),
2115 })?
2116 .map_err(|source| SubcError::Connect {
2117 endpoint: endpoint_label.clone(),
2118 source,
2119 })?;
2120 stream
2121 .set_nodelay(true)
2122 .map_err(|source| SubcError::Connect {
2123 endpoint: endpoint_label.clone(),
2124 source,
2125 })?;
2126
2127 let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2128 authenticate_client(&mut stream, &conn, auth_budget)
2129 .await
2130 .map_err(|source| SubcError::Auth {
2131 endpoint: endpoint_label,
2132 source,
2133 })?;
2134
2135 Ok(stream)
2136}
2137
2138#[allow(clippy::too_many_arguments)]
2139async fn process_route_bind_completion(
2140 writer_tx: &WriterSender,
2141 completion: RouteBindCompletion,
2142 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2143 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2144 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2145 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2146 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2147 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2148 installed_route_epochs: &mut HashMap<u16, u32>,
2149 executor: &Arc<Executor>,
2150 shutdown: &Arc<Notify>,
2151 metrics: &Arc<DispatchPathMetrics>,
2152) -> Result<(), SubcError> {
2153 decrement_counted_channel(&metrics.control_completion_queued);
2154 handle_route_bind_completion(
2155 writer_tx,
2156 completion,
2157 routes,
2158 root_channels,
2159 session_identity,
2160 push_buffer,
2161 live_roots,
2162 pending_binds,
2163 installed_route_epochs,
2164 executor,
2165 shutdown,
2166 metrics,
2167 )
2168 .await
2169}
2170
2171#[allow(clippy::too_many_arguments)]
2172async fn drain_pending_route_bind_completions(
2173 control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2174 writer_tx: &WriterSender,
2175 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2176 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2177 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2178 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2179 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2180 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2181 installed_route_epochs: &mut HashMap<u16, u32>,
2182 executor: &Arc<Executor>,
2183 shutdown: &Arc<Notify>,
2184 metrics: &Arc<DispatchPathMetrics>,
2185) -> Result<usize, SubcError> {
2186 let mut drained = 0;
2187 while let Ok(completion) = control_completion_rx.try_recv() {
2188 process_route_bind_completion(
2189 writer_tx,
2190 completion,
2191 routes,
2192 root_channels,
2193 session_identity,
2194 push_buffer,
2195 live_roots,
2196 pending_binds,
2197 installed_route_epochs,
2198 executor,
2199 shutdown,
2200 metrics,
2201 )
2202 .await?;
2203 drained += 1;
2204 }
2205 Ok(drained)
2206}
2207
2208async fn run_module_loop<R, W>(
2212 mut read: R,
2213 mut write: W,
2214 shared_app: Arc<App>,
2215 executor: Arc<Executor>,
2216 dispatch: DispatchFn,
2217 user_config_path: Option<PathBuf>,
2218 allow_native_passthrough: bool,
2219) -> Result<ModuleLoopExit, SubcError>
2220where
2221 R: AsyncRead + Unpin + Send + 'static,
2222 W: AsyncWrite + Unpin + Send + 'static,
2223{
2224 let hello = ModuleHelloBody {
2228 manifest: build_manifest(),
2229 protocol_ver: PROTOCOL_VERSION,
2230 control_ops: control_ops(),
2231 launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2232 };
2233 let hello_frame = Frame::build(
2234 FrameType::Hello,
2235 control_flags(),
2236 0,
2237 0,
2238 HELLO_CORR,
2239 serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2240 )
2241 .map_err(SubcError::FrameBuild)?;
2242 write_frame(&mut write, &hello_frame)
2243 .await
2244 .map_err(SubcError::FrameIo)?;
2245
2246 match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2248 None => return Err(SubcError::ClosedBeforeHelloAck),
2249 Some(frame) => match frame.header.ty {
2250 FrameType::HelloAck => {
2251 log::info!("subc attach: registered (HelloAck received)");
2252 }
2253 FrameType::Error => {
2254 let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2255 return Err(SubcError::HelloRejected { body });
2256 }
2257 other => return Err(SubcError::UnexpectedFrame { ty: other }),
2258 },
2259 }
2260
2261 let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2262 let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2263 let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2264 let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2271 let reader_task = spawn_reader_task(read, reader_tx);
2272 let shutdown = Arc::new(Notify::new());
2273 let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2282 let mut next_maintenance_at = next_drain_at;
2283 #[cfg(target_os = "macos")]
2286 let mut last_slack_relief: Option<std::time::Instant> = None;
2287 let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2288 let (bash_deferred_tx, mut bash_deferred_rx) =
2289 mpsc::channel::<bash::BashDeferredCompletion>(256);
2290 let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2291 let (control_completion_tx, mut control_completion_rx) =
2292 mpsc::channel::<RouteBindCompletion>(256);
2293 let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2294 let lossy_overflow = Arc::new(push::LossyOverflow::default());
2295 let lossy_seq = Arc::new(AtomicU64::new(0));
2296 let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2297 let push_senders = PushSenders {
2298 lossy_tx,
2299 reliable_tx,
2300 lossy_overflow: Arc::clone(&lossy_overflow),
2301 lossy_seq,
2302 };
2303 let connection_cancel = PersistentCancelSignal::new();
2304 let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2305 let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2306 let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2307 let mut bg_sub_by_session: HashMap<(ProjectRootId, String), RouteChannel> = HashMap::new();
2308 let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2309 let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2310 let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2311 let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2312 HashMap::new();
2313 let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2314 let mut retry_buffer: RetryBuffer = HashMap::new();
2315 let mut reclaimed_routes = ReclaimedRoutes::default();
2316 let mut completed_tasks = push::CompletedTaskIds::default();
2317 let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2318 let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2319 let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2320 let mut next_bash_ask_corr: u64 = 1;
2321 let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2322
2323 let loop_result: Result<ModuleLoopExit, SubcError> = loop {
2324 shared_app.set_open_route_count(routes.len());
2325 crate::logging::perf_tick(Some(&executor));
2326 dispatch_path_metrics.mark_frame_loop_tick();
2327 if let Err(error) = expire_pending_bash_asks(
2328 &writer_tx,
2329 &mut pending_bash_asks,
2330 &routes,
2331 &mut live_roots,
2332 &mut route_bash_cancels,
2333 &shutdown,
2334 &dispatch_path_metrics,
2335 )
2336 .await
2337 {
2338 break Err(error);
2339 }
2340
2341 match drain_pending_route_bind_completions(
2345 &mut control_completion_rx,
2346 &writer_tx,
2347 &mut routes,
2348 &mut root_channels,
2349 &mut session_identity,
2350 &mut push_buffer,
2351 &mut live_roots,
2352 &mut pending_binds,
2353 &mut installed_route_epochs,
2354 &executor,
2355 &shutdown,
2356 &dispatch_path_metrics,
2357 )
2358 .await
2359 {
2360 Ok(drained) => {
2361 if drained > 0 {
2362 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2363 }
2364 }
2365 Err(error) => break Err(error),
2366 }
2367
2368 if tokio::time::Instant::now() >= next_drain_at {
2369 push::emit_bg_event_wakes(
2370 &writer_tx,
2371 &dispatch_path_metrics,
2372 &bg_subs,
2373 &mut bg_wake_pending,
2374 );
2375 warn_slow_pending_binds(&mut pending_binds, &executor);
2376 if let Err(error) = expire_overdue_route_binds(
2377 &writer_tx,
2378 &executor,
2379 &mut pending_binds,
2380 &mut installed_route_epochs,
2381 &dispatch_path_metrics,
2382 )
2383 .await
2384 {
2385 break Err(error);
2386 }
2387
2388 let retried = push::drain_retry_buffers_for_bound_routes(
2389 &writer_tx,
2390 &dispatch_path_metrics,
2391 &routes,
2392 &mut retry_buffer,
2393 );
2394 if retried > 0 {
2395 log::debug!(
2396 "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2397 );
2398 }
2399
2400 next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2401 }
2402
2403 let overflow_batch = lossy_overflow.drain();
2409 if !overflow_batch.is_empty() {
2410 let (_, deferred) = push::drain_reliable_push_turn(
2411 &writer_tx,
2412 &dispatch_path_metrics,
2413 &routes,
2414 &root_channels,
2415 &session_identity,
2416 &mut retry_buffer,
2417 &mut push_buffer,
2418 &mut completed_tasks,
2419 &bg_sub_by_session,
2420 &mut bg_wake_pending,
2421 &mut bg_wake_epoch,
2422 &mut reliable_rx,
2423 None,
2424 );
2425 if deferred {
2426 tokio::task::yield_now().await;
2427 }
2428
2429 let mut batch = Vec::new();
2430 while let Ok(item) = lossy_rx.try_recv() {
2431 batch.push(item);
2432 }
2433 batch.extend(overflow_batch);
2434 push::process_lossy_push_envelope_batch(
2435 &writer_tx,
2436 &dispatch_path_metrics,
2437 &routes,
2438 &root_channels,
2439 &completed_tasks,
2440 batch,
2441 );
2442 }
2443
2444 tokio::select! {
2445 biased;
2446 Some(completion) = control_completion_rx.recv() => {
2447 if let Err(error) = process_route_bind_completion(
2448 &writer_tx,
2449 completion,
2450 &mut routes,
2451 &mut root_channels,
2452 &mut session_identity,
2453 &mut push_buffer,
2454 &mut live_roots,
2455 &mut pending_binds,
2456 &mut installed_route_epochs,
2457 &executor,
2458 &shutdown,
2459 &dispatch_path_metrics,
2460 )
2461 .await
2462 {
2463 break Err(error);
2464 }
2465 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2466 }
2467 _ = shutdown.notified() => {
2468 log::warn!("subc attach: fatal executor response requested teardown");
2469 break Ok(ModuleLoopExit::SkipSearchFlush);
2470 }
2471 maybe_frame = reader_rx.recv() => {
2472 let frame = match maybe_frame {
2473 None => {
2474 log::info!("subc attach: daemon closed connection");
2475 break Ok(ModuleLoopExit::Graceful);
2476 }
2477 Some(Err(error)) => break Err(error),
2478 Some(Ok(frame)) => frame,
2479 };
2480 let phase_trace = frame.phase_trace;
2481 let frame = frame.frame;
2482
2483 if !ingress_route_should_be_processed(
2484 &installed_route_epochs,
2485 &reclaimed_routes,
2486 &frame,
2487 ) {
2488 log::debug!(
2489 "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2490 frame.header.ty,
2491 frame.header.channel,
2492 frame.header.epoch
2493 );
2494 continue;
2495 }
2496
2497 match frame.header.ty {
2498 FrameType::Ping if frame.header.channel == 0 => {
2499 let pong = match Frame::build_with_version(
2500 frame.header.ver,
2501 FrameType::Pong,
2502 frame.header.flags,
2503 0,
2504 0,
2505 frame.header.corr,
2506 Vec::new(),
2507 ) {
2508 Ok(pong) => pong,
2509 Err(error) => break Err(SubcError::FrameBuild(error)),
2510 };
2511 if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2512 break Err(error);
2513 }
2514 }
2515 FrameType::Goodbye if frame.header.channel == 0 => {
2516 log::info!("subc attach: received channel-0 Goodbye");
2517 break Ok(ModuleLoopExit::Graceful);
2518 }
2519 FrameType::Goodbye => {
2520 let channel = route_key(frame.header.channel, frame.header.epoch);
2521 if let Err(error) = teardown_installed_route(
2522 &writer_tx,
2523 &dispatch_path_metrics,
2524 &executor,
2525 channel,
2526 "Goodbye",
2527 None,
2528 &mut installed_route_epochs,
2529 &mut routes,
2530 &mut root_channels,
2531 &mut bg_subs,
2532 &mut bg_sub_by_session,
2533 &mut bg_wake_pending,
2534 &mut pending_bash_asks,
2535 &mut live_roots,
2536 &mut route_bash_cancels,
2537 &mut pending_binds,
2538 &mut retry_buffer,
2539 &mut push_buffer,
2540 &shutdown,
2541 )
2542 .await
2543 {
2544 break Err(error);
2545 }
2546 }
2547 FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2548 if let Err(error) = handle_bash_elicitation_reply(
2549 &writer_tx,
2550 &frame,
2551 &mut pending_bash_asks,
2552 &routes,
2553 &mut live_roots,
2554 &executor,
2555 &shutdown,
2556 &bash_deferred_tx,
2557 &bash_poll_touch_tx,
2558 &dispatch_path_metrics,
2559 &mut route_bash_cancels,
2560 dispatch,
2561 )
2562 .await
2563 {
2564 break Err(error);
2565 }
2566 }
2567 FrameType::Request if frame.header.channel == 0 => {
2568 if let Err(error) = handle_control_request(
2569 &writer_tx,
2570 &frame,
2571 &shared_app,
2572 &executor,
2573 &mut live_roots,
2574 &mut pending_binds,
2575 &mut installed_route_epochs,
2576 &mut routes,
2577 &mut root_channels,
2578 &mut bg_subs,
2579 &mut bg_sub_by_session,
2580 &mut bg_wake_pending,
2581 &mut pending_bash_asks,
2582 &mut route_bash_cancels,
2583 &mut retry_buffer,
2584 &mut push_buffer,
2585 &shutdown,
2586 &control_completion_tx,
2587 &dispatch_path_metrics,
2588 &push_senders,
2589 dispatch,
2590 user_config_path.as_deref(),
2591 )
2592 .await
2593 {
2594 break Err(error);
2595 }
2596 }
2597 FrameType::Request => {
2598 if let Err(error) = handle_tool_call(
2599 &writer_tx,
2600 &frame,
2601 phase_trace,
2602 &routes,
2603 &pending_binds,
2604 &mut live_roots,
2605 &executor,
2606 &shutdown,
2607 &connection_cancel,
2608 &bash_deferred_tx,
2609 &bash_poll_touch_tx,
2610 &dispatch_path_metrics,
2611 &mut route_bash_cancels,
2612 &mut pending_bash_asks,
2613 &mut next_bash_ask_corr,
2614 &mut bg_subs,
2615 &mut bg_sub_by_session,
2616 &mut bg_wake_pending,
2617 &mut bg_wake_epoch,
2618 dispatch,
2619 allow_native_passthrough,
2620 )
2621 .await
2622 {
2623 break Err(error);
2624 }
2625 }
2626 FrameType::Cancel => {
2627 let channel = route_key(frame.header.channel, frame.header.epoch);
2628 if bg_subs.contains_key(&channel) {
2629 if let Err(error) = end_bg_subscription(
2630 &writer_tx,
2631 &dispatch_path_metrics,
2632 &mut bg_subs,
2633 &mut bg_sub_by_session,
2634 &mut bg_wake_pending,
2635 channel,
2636 routes.get(&channel),
2637 )
2638 .await
2639 {
2640 break Err(error);
2641 }
2642 }
2643 if let Err(error) = cancel_pending_bash_ask_for_tool_call(
2644 &writer_tx,
2645 &mut pending_bash_asks,
2646 channel,
2647 frame.header.corr,
2648 &routes,
2649 &mut live_roots,
2650 &mut route_bash_cancels,
2651 &shutdown,
2652 &dispatch_path_metrics,
2653 )
2654 .await
2655 {
2656 break Err(error);
2657 }
2658 }
2659 _ => {}
2664 }
2665 }
2666 Some((root_id, frame)) = reliable_rx.recv() => {
2667 let (_, deferred) = push::drain_reliable_push_turn(
2671 &writer_tx,
2672 &dispatch_path_metrics,
2673 &routes,
2674 &root_channels,
2675 &session_identity,
2676 &mut retry_buffer,
2677 &mut push_buffer,
2678 &mut completed_tasks,
2679 &bg_sub_by_session,
2680 &mut bg_wake_pending,
2681 &mut bg_wake_epoch,
2682 &mut reliable_rx,
2683 Some((root_id, frame)),
2684 );
2685 if deferred {
2686 tokio::task::yield_now().await;
2687 }
2688 }
2689 Some((order, root_id, frame)) = lossy_rx.recv() => {
2690 let (_, deferred) = push::drain_reliable_push_turn(
2694 &writer_tx,
2695 &dispatch_path_metrics,
2696 &routes,
2697 &root_channels,
2698 &session_identity,
2699 &mut retry_buffer,
2700 &mut push_buffer,
2701 &mut completed_tasks,
2702 &bg_sub_by_session,
2703 &mut bg_wake_pending,
2704 &mut bg_wake_epoch,
2705 &mut reliable_rx,
2706 None,
2707 );
2708 if deferred {
2709 tokio::task::yield_now().await;
2710 }
2711
2712 let mut batch = vec![(order, root_id, frame)];
2719 while let Ok(item) = lossy_rx.try_recv() {
2720 batch.push(item);
2721 }
2722 batch.extend(lossy_overflow.drain());
2723 push::process_lossy_push_envelope_batch(
2724 &writer_tx,
2725 &dispatch_path_metrics,
2726 &routes,
2727 &root_channels,
2728 &completed_tasks,
2729 batch,
2730 );
2731 }
2732 Some(done) = bash_deferred_rx.recv() => {
2733 decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
2734 if let Err(error) = bash::handle_bash_deferred_completion(
2735 &writer_tx,
2736 done,
2737 &routes,
2738 &mut live_roots,
2739 &mut route_bash_cancels,
2740 &shutdown,
2741 &dispatch_path_metrics,
2742 )
2743 .await
2744 {
2745 break Err(error);
2746 }
2747 }
2748 Some(root_id) = bash_poll_touch_rx.recv() => {
2749 decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
2750 if let Some(meta) = live_roots.get_mut(&root_id) {
2751 meta.note_activity();
2752 }
2753 }
2754 Some(completion) = maintenance_rx.recv() => {
2755 decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
2756 let root_id = completion.root_id.clone();
2757 let response = completion.response;
2758 let response_is_fatal = response_is_fatal_panic(&response);
2759 let bind_pending = pending_binds
2760 .values()
2761 .any(|pending| pending.bind_root_id == root_id);
2762 let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
2763 let defer_requeue = meta.unbound_quiesced || bind_pending;
2764 note_maintenance_completion(
2765 meta,
2766 completion.requeue_kind,
2767 response_is_fatal,
2768 defer_requeue,
2769 );
2770 should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
2771 } else {
2772 false
2773 };
2774 if requiesce {
2775 quiesce_unbound_root(&root_id, &mut live_roots, &executor);
2776 }
2777 push::clear_stale_bg_wakes_for_empty_sessions(
2778 &root_id,
2779 &completion.empty_bg_sessions,
2780 &bg_sub_by_session,
2781 &mut bg_wake_pending,
2782 &bg_wake_epoch,
2783 );
2784 if response_is_fatal {
2785 if let Some(meta) = live_roots.get_mut(&root_id) {
2786 meta.maintenance_poisoned = true;
2787 }
2788 log::warn!(
2789 "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
2790 );
2791 }
2792 }
2793 _ = tokio::time::sleep_until(next_drain_at) => {
2794 }
2797 _ = tokio::time::sleep_until(next_maintenance_at) => {
2798 let reaped_lsp_children = shared_app
2803 .lsp_child_registry()
2804 .reap_children_with_gone_cwd();
2805 if reaped_lsp_children > 0 {
2806 log::warn!(
2807 "subc attach: reaped {reaped_lsp_children} LSP child process group(s) whose cwd no longer exists"
2808 );
2809 }
2810 let reap = reap_idle_roots(
2811 Instant::now(),
2812 &mut live_roots,
2813 &pending_binds,
2814 &root_channels,
2815 &executor,
2816 &dispatch_path_metrics,
2817 );
2818 for root_id in &reap.forgotten_deleted_roots {
2819 purge_deleted_root_residents(
2820 root_id,
2821 &mut routes,
2822 &mut root_channels,
2823 &mut installed_route_epochs,
2824 &mut route_bash_cancels,
2825 &mut retry_buffer,
2826 &mut reclaimed_routes,
2827 &mut session_identity,
2828 &mut push_buffer,
2829 &mut bg_subs,
2830 &mut bg_sub_by_session,
2831 &mut bg_wake_pending,
2832 &mut bg_wake_epoch,
2833 &mut pending_bash_asks,
2834 );
2835 }
2836 if reap.evicted > 0 {
2837 log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
2838 }
2839 submit_due_maintenance_jobs(
2840 &executor,
2841 &mut live_roots,
2842 &pending_binds,
2843 &bg_sub_by_session,
2844 &bg_wake_pending,
2845 &bg_wake_epoch,
2846 &maintenance_tx,
2847 &dispatch_path_metrics,
2848 );
2849 #[cfg(target_os = "macos")]
2856 {
2857 let now_std = std::time::Instant::now();
2858 if crate::memory::spawn_allocator_slack_relief_if_due(
2859 last_slack_relief,
2860 now_std,
2861 ) {
2862 last_slack_relief = Some(now_std);
2863 }
2864 }
2865 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2866 }
2867 }
2868 };
2869
2870 shared_app.set_open_route_count(0);
2871
2872 connection_cancel.cancel();
2873 quiesce_connection_roots(
2876 &mut live_roots,
2877 &mut pending_binds,
2878 &mut routes,
2879 &mut root_channels,
2880 &mut installed_route_epochs,
2881 &mut route_bash_cancels,
2882 &executor,
2883 );
2884
2885 let mut loop_result = loop_result;
2886 if !pending_bash_asks.is_empty() {
2887 let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2888 if let Err(error) = settle_all_pending_bash_asks(
2889 &writer_tx,
2890 &mut pending_bash_asks,
2891 &no_routes,
2892 &mut live_roots,
2893 &mut route_bash_cancels,
2894 &shutdown,
2895 &dispatch_path_metrics,
2896 )
2897 .await
2898 {
2899 loop_result = loop_result.and(Err(error));
2900 }
2901 }
2902
2903 reader_task.abort();
2906 drop(writer_tx);
2907 let writer_result = finish_writer_task(writer_task).await;
2908 loop_result.and_then(|exit| writer_result.map(|_| exit))
2909}
2910
2911fn spawn_writer_task<W>(
2912 mut write: W,
2913 mut rx: mpsc::Receiver<WriterFrame>,
2914 metrics: Arc<DispatchPathMetrics>,
2915) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
2916where
2917 W: AsyncWrite + Unpin + Send + 'static,
2918{
2919 tokio::spawn(async move {
2920 let mut write_buffer = Vec::new();
2921 while let Some(mut queued) = rx.recv().await {
2922 let measure = queued.tool_response_trace.is_some();
2923 let dequeued = measure.then(Instant::now);
2924 metrics.writer_active.store(true, Ordering::Relaxed);
2925 decrement_counted_channel(&metrics.writer_queued);
2926 let write_timing = write_frame_contiguous(
2927 &mut write,
2928 queued.frame(),
2929 queued.body(),
2930 &mut write_buffer,
2931 measure,
2932 )
2933 .await;
2934 metrics.writer_active.store(false, Ordering::Relaxed);
2935 let write_timing = write_timing?;
2936
2937 if let (Some(trace), Some(dequeued), Some(write_timing)) =
2938 (queued.tool_response_trace.take(), dequeued, write_timing)
2939 {
2940 if let Some(completed) = trace.finish(
2941 dequeued,
2942 write_timing.write_started,
2943 write_timing.write_finished,
2944 write_timing.frame_bytes,
2945 ) {
2946 log_ctx::with_session(Some(completed.session), || {
2947 crate::logging::note_tool_call_trace(
2948 &completed.name,
2949 &completed.root,
2950 completed.channel,
2951 completed.corr,
2952 completed.phases,
2953 );
2954 });
2955 }
2956 }
2957 }
2958 Ok(())
2959 })
2960}
2961
2962struct FrameWriteTiming {
2963 write_started: Instant,
2964 write_finished: Instant,
2965 frame_bytes: usize,
2966}
2967
2968async fn write_frame_contiguous<W>(
2972 writer: &mut W,
2973 frame: &Frame,
2974 body: &[u8],
2975 buffer: &mut Vec<u8>,
2976 measure: bool,
2977) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
2978where
2979 W: AsyncWrite + Unpin,
2980{
2981 if frame.header.len as usize != body.len() {
2982 return Err(subc_transport::FrameIoError::BodyLengthMismatch {
2983 header_len: frame.header.len,
2984 body_len: body.len(),
2985 });
2986 }
2987
2988 let header = frame.header.encode();
2989 buffer.clear();
2990 buffer.reserve(header.len() + body.len());
2991 buffer.extend_from_slice(&header);
2992 buffer.extend_from_slice(body);
2993 let write_started = measure.then(Instant::now);
2994 writer
2995 .write_all(buffer)
2996 .await
2997 .map_err(subc_transport::FrameIoError::Io)?;
2998 Ok(write_started.map(|write_started| FrameWriteTiming {
2999 write_started,
3000 write_finished: Instant::now(),
3001 frame_bytes: buffer.len(),
3002 }))
3003}
3004
3005fn spawn_reader_task<R>(
3006 mut read: R,
3007 tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3008) -> JoinHandle<()>
3009where
3010 R: AsyncRead + Unpin + Send + 'static,
3011{
3012 tokio::spawn(async move {
3013 loop {
3014 match read_frame(&mut read).await {
3015 Ok(Some(frame)) => {
3016 let decoded = DecodedFrame {
3017 frame,
3018 phase_trace: PhaseTrace::new(Instant::now()),
3019 };
3020 if tx.send(Ok(decoded)).await.is_err() {
3021 return;
3022 }
3023 }
3024 Ok(None) => {
3025 return;
3027 }
3028 Err(error) => {
3029 if let subc_transport::FrameIoError::Io(io_error) = &error {
3036 if matches!(
3037 io_error.kind(),
3038 std::io::ErrorKind::ConnectionReset
3039 | std::io::ErrorKind::ConnectionAborted
3040 ) {
3041 log::info!(
3042 "subc attach: connection reset by daemon; treating as close"
3043 );
3044 return;
3045 }
3046 }
3047 let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3048 return;
3049 }
3050 }
3051 }
3052 })
3053}
3054
3055async fn finish_writer_task(
3056 mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3057) -> Result<(), SubcError> {
3058 match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3059 Ok(Ok(Ok(()))) => Ok(()),
3060 Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3061 Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3062 Err(_) => {
3063 writer_task.abort();
3064 Ok(())
3065 }
3066 }
3067}
3068
3069fn register_actor_for_bind(
3070 shared_app: &Arc<App>,
3071 executor: &Arc<Executor>,
3072 push_senders: &PushSenders,
3073 bind_root_id: &ProjectRootId,
3074 route_channel: u16,
3075 root_was_live: bool,
3076) -> bool {
3077 if executor.actor_registered(bind_root_id) {
3078 log::debug!(
3079 "subc attach: reusing actor for route {} root {}",
3080 route_channel,
3081 bind_root_id.as_path().display()
3082 );
3083 return false;
3084 }
3085
3086 if root_was_live {
3087 log::warn!(
3088 "subc attach: recreating missing actor for live root {} on route {}",
3089 bind_root_id.as_path().display(),
3090 route_channel
3091 );
3092 }
3093
3094 let actor_ctx = Arc::new(AppContext::from_app(
3095 Arc::clone(shared_app),
3096 Config::default(),
3097 ));
3098 install_bash_compressor(&actor_ctx);
3099 actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
3100 push_senders.clone(),
3101 bind_root_id.clone(),
3102 )));
3103 let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
3104 drop(actor_ctx);
3105 if inserted {
3106 log::debug!(
3110 "subc attach: registered actor for route {} root {}",
3111 route_channel,
3112 bind_root_id.as_path().display()
3113 );
3114 } else {
3115 log::debug!(
3116 "subc attach: actor appeared while binding route {} root {}; reusing it",
3117 route_channel,
3118 bind_root_id.as_path().display()
3119 );
3120 }
3121 inserted
3122}
3123
3124fn rollback_pending_bind_actor(
3125 executor: &Arc<Executor>,
3126 live_roots: &HashMap<ProjectRootId, RootMeta>,
3127 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3128 root_id: &ProjectRootId,
3129 inserted_new_actor: bool,
3130) {
3131 if !inserted_new_actor || live_roots.contains_key(root_id) {
3132 return;
3133 }
3134
3135 if let Some((route, pending)) = pending_binds
3136 .iter_mut()
3137 .find(|(_, pending)| &pending.bind_root_id == root_id)
3138 {
3139 pending.inserted_new_actor = true;
3140 log::debug!(
3141 "subc attach: transferred rollback ownership for root {} to pending route {}",
3142 root_id.as_path().display(),
3143 route
3144 );
3145 return;
3146 }
3147
3148 executor.remove_actor(root_id);
3149}
3150
3151fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
3152 match response.data.get("code").and_then(|code| code.as_str()) {
3153 Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
3158 Some("cache_key_probe_failed") => "cache_key_probe_failed",
3162 Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
3166 _ => "config_divergence",
3167 }
3168}
3169
3170fn queue_post_bind_configure_and_completion_maintenance(
3171 root_id: &ProjectRootId,
3172 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3173) {
3174 let Some(meta) = live_roots.get_mut(root_id) else {
3175 return;
3176 };
3177 if meta.maintenance_poisoned || meta.maintenance_pending {
3178 return;
3179 }
3180
3181 meta.maintenance_pending = true;
3182 meta.maintenance_queued_kinds
3183 .push_back(MaintenanceDrainKind::ConfigureTail);
3184 meta.maintenance_queued_kinds
3185 .push_back(MaintenanceDrainKind::CompletionDrains);
3186}
3187
3188#[allow(clippy::too_many_arguments)]
3189async fn handle_route_bind_completion(
3190 tx: &WriterSender,
3191 completion: RouteBindCompletion,
3192 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3193 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3194 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
3195 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3196 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3197 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3198 installed_route_epochs: &mut HashMap<u16, u32>,
3199 executor: &Arc<Executor>,
3200 shutdown: &Arc<Notify>,
3201 metrics: &Arc<DispatchPathMetrics>,
3202) -> Result<(), SubcError> {
3203 let route_id = completion.route;
3204 let Some(pending) = pending_binds.remove(&route_id) else {
3205 log::warn!(
3206 "subc attach: dropping RouteBind completion for non-pending route {}",
3207 completion.route
3208 );
3209 rollback_pending_bind_actor(
3210 executor,
3211 live_roots,
3212 pending_binds,
3213 &completion.bind_root_id,
3214 completion.inserted_new_actor,
3215 );
3216 let has_pending_bind = pending_binds
3217 .values()
3218 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3219 if !root_channels
3220 .get(&completion.bind_root_id)
3221 .is_some_and(|channels| !channels.is_empty())
3222 && !has_pending_bind
3223 {
3224 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3225 }
3226 remove_installed_route(installed_route_epochs, route_id);
3227 return Ok(());
3228 };
3229
3230 if pending.bind_root_id != completion.bind_root_id {
3231 log::warn!(
3232 "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
3233 completion.route,
3234 pending.bind_root_id.as_path().display(),
3235 completion.bind_root_id.as_path().display()
3236 );
3237 }
3238
3239 let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
3240 if pending.cancelled {
3241 rollback_pending_bind_actor(
3242 executor,
3243 live_roots,
3244 pending_binds,
3245 &completion.bind_root_id,
3246 inserted_new_actor,
3247 );
3248 let has_pending_bind = pending_binds
3249 .values()
3250 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3251 if !root_channels
3252 .get(&completion.bind_root_id)
3253 .is_some_and(|channels| !channels.is_empty())
3254 && !has_pending_bind
3255 {
3256 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3257 }
3258 log::debug!(
3259 "subc attach: discarded completed RouteBind for cancelled route {} root {}",
3260 completion.route,
3261 completion.bind_root_id.as_path().display()
3262 );
3263 remove_installed_route(installed_route_epochs, route_id);
3264 return Ok(());
3265 }
3266
3267 let failure = if !completion.configure_response.success {
3268 Some((
3269 &completion.configure_response,
3270 "configure failed during route bind",
3271 ))
3272 } else {
3273 None
3274 };
3275
3276 if let Some((response, fallback)) = failure {
3277 rollback_pending_bind_actor(
3278 executor,
3279 live_roots,
3280 pending_binds,
3281 &completion.bind_root_id,
3282 inserted_new_actor,
3283 );
3284 let has_pending_bind = pending_binds
3285 .values()
3286 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3287 if !root_channels
3288 .get(&completion.bind_root_id)
3289 .is_some_and(|channels| !channels.is_empty())
3290 && !has_pending_bind
3291 {
3292 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3293 }
3294 let message = response_message(response, fallback);
3295 let fatal = response_is_fatal_panic(response);
3296 let error_code = route_bind_error_code_for_configure_response(response);
3297 send_route_bind_error_parts(
3298 tx,
3299 completion.ver,
3300 completion.corr,
3301 completion.flags,
3302 error_code,
3303 &message,
3304 metrics,
3305 )
3306 .await?;
3307 remove_installed_route(installed_route_epochs, route_id);
3308 if fatal {
3309 signal_fatal_teardown(
3310 tx,
3311 Some(completion.route),
3312 completion.ver,
3313 completion.corr,
3314 shutdown,
3315 metrics,
3316 )
3317 .await;
3318 }
3319 return Ok(());
3320 }
3321
3322 remember_session_identity(session_identity, &completion.identity);
3323 let replay_key = push::ReplayKey::from_identity(&completion.identity);
3324 let bind_trust = completion.identity.trust;
3325 insert_route_channel(routes, root_channels, route_id, completion.identity);
3326 let restore_watcher = live_roots
3327 .get(&completion.bind_root_id)
3328 .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3329 live_roots
3330 .entry(completion.bind_root_id.clone())
3331 .and_modify(|meta| {
3332 meta.reactivate_bound();
3333 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3334 meta.maintenance_poisoned = false;
3335 })
3336 .or_insert_with(|| RootMeta::new(Instant::now()));
3337 if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3338 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3339 meta.maintenance_poisoned = false;
3340 }
3341 if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3342 ctx.mark_subc_bound();
3343 if restore_watcher {
3344 crate::commands::configure::ensure_project_watcher(&ctx);
3345 }
3346 }
3347
3348 let ack =
3349 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3350 let response = Frame::build_with_version(
3351 completion.ver,
3352 FrameType::Response,
3353 control_flags(),
3354 0,
3355 0,
3356 completion.corr,
3357 ack,
3358 )
3359 .map_err(SubcError::FrameBuild)?;
3360 send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3361 queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3362 let replayed = push::replay_buffered_push_frames(
3363 tx,
3364 metrics,
3365 route_id,
3366 push_buffer,
3367 &replay_key,
3368 bind_trust,
3369 );
3370 if replayed > 0 {
3371 log::debug!(
3372 "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3373 replayed,
3374 completion.route,
3375 replay_key.root.as_path().display(),
3376 replay_key.harness,
3377 replay_key.session
3378 );
3379 }
3380 log::info!(
3381 "subc attach: route {} bound to root {}",
3382 completion.route,
3383 completion.bind_root_id.as_path().display()
3384 );
3385 Ok(())
3386}
3387
3388async fn expire_overdue_route_binds(
3389 tx: &WriterSender,
3390 executor: &Arc<Executor>,
3391 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3392 installed_route_epochs: &mut HashMap<u16, u32>,
3393 metrics: &DispatchPathMetrics,
3394) -> Result<(), SubcError> {
3395 let now = Instant::now();
3396 let expired: Vec<_> = pending_binds
3397 .iter()
3398 .filter_map(|(route, pending)| {
3399 let age = now.saturating_duration_since(pending.started_at);
3400 (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3401 (
3402 *route,
3403 pending.corr,
3404 pending.ver,
3405 pending.flags,
3406 pending.bind_root_id.clone(),
3407 pending.configure_request_id.clone(),
3408 age,
3409 )
3410 })
3411 })
3412 .collect();
3413
3414 for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3415 if let Some(pending) = pending_binds.get_mut(&route) {
3416 pending.cancelled = true;
3417 pending.deadline_reported = true;
3418 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3419 log::debug!(
3420 "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3421 );
3422 }
3423 remove_installed_route(installed_route_epochs, route);
3424 let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3425 let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3426 send_route_bind_error_parts(
3427 tx,
3428 ver,
3429 corr,
3430 flags,
3431 "actor_not_ready",
3432 &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3433 metrics,
3434 )
3435 .await?;
3436 log::warn!(
3437 "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3438 route,
3439 root_id.as_path().display(),
3440 deadline_ms,
3441 configure_request_id
3442 );
3443 }
3444
3445 Ok(())
3446}
3447
3448#[allow(clippy::too_many_arguments)]
3453async fn handle_control_request(
3454 tx: &WriterSender,
3455 frame: &Frame,
3456 shared_app: &Arc<App>,
3457 executor: &Arc<Executor>,
3458 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3459 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3460 installed_route_epochs: &mut HashMap<u16, u32>,
3461 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3462 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3463 bg_subs: &mut HashMap<RouteChannel, BgSub>,
3464 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3465 bg_wake_pending: &mut HashSet<RouteChannel>,
3466 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3467 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3468 retry_buffer: &mut RetryBuffer,
3469 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3470 shutdown: &Arc<Notify>,
3471 control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3472 metrics: &Arc<DispatchPathMetrics>,
3473 push_senders: &PushSenders,
3474 dispatch: DispatchFn,
3475 user_config_path: Option<&Path>,
3476) -> Result<(), SubcError> {
3477 let request =
3478 serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3479 match request {
3480 ModuleControlRequest::RouteBind {
3481 route_channel,
3482 epoch,
3483 target: _,
3484 identity,
3485 principal,
3486 consumer_capabilities,
3487 } => {
3488 let route_id = route_key(route_channel, epoch);
3489 if epoch == 0 {
3490 return send_route_bind_error(
3491 tx,
3492 frame,
3493 "config_divergence",
3494 "route bind uses an invalid channel generation",
3495 metrics,
3496 )
3497 .await;
3498 }
3499 let mut bind_root_id = None;
3500 if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3501 if installed_epoch >= epoch {
3502 return send_route_bind_error(
3503 tx,
3504 frame,
3505 "config_divergence",
3506 "route bind generation is not newer than the installed generation",
3507 metrics,
3508 )
3509 .await;
3510 }
3511
3512 let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3513 Ok(root_id) => root_id,
3514 Err(error) => {
3515 return send_route_bind_error(
3516 tx,
3517 frame,
3518 "config_divergence",
3519 &format!("invalid route project root: {error}"),
3520 metrics,
3521 )
3522 .await;
3523 }
3524 };
3525 teardown_installed_route(
3526 tx,
3527 metrics,
3528 executor,
3529 route_key(route_channel, installed_epoch),
3530 "higher-epoch RouteBind",
3531 Some(&replacement_root),
3532 installed_route_epochs,
3533 routes,
3534 root_channels,
3535 bg_subs,
3536 bg_sub_by_session,
3537 bg_wake_pending,
3538 pending_bash_asks,
3539 live_roots,
3540 route_bash_cancels,
3541 pending_binds,
3542 retry_buffer,
3543 push_buffer,
3544 shutdown,
3545 )
3546 .await?;
3547 bind_root_id = Some(replacement_root);
3548 }
3549 if pending_binds.contains_key(&route_id) {
3550 return send_route_bind_error(
3551 tx,
3552 frame,
3553 "config_divergence",
3554 "route bind is already pending for channel",
3555 metrics,
3556 )
3557 .await;
3558 }
3559 let bind_root_id = match bind_root_id {
3560 Some(root_id) => root_id,
3561 None => match ProjectRootId::from_path(&identity.project_root) {
3562 Ok(root_id) => root_id,
3563 Err(error) => {
3564 return send_route_bind_error(
3565 tx,
3566 frame,
3567 "config_divergence",
3568 &format!("invalid route project root: {error}"),
3569 metrics,
3570 )
3571 .await;
3572 }
3573 },
3574 };
3575
3576 let request_id = format!("subc-bind-{route_channel}");
3579 let bind_project_root = identity.project_root.clone();
3580 let bind_harness = identity.harness.clone();
3581 let bind_session = identity.session.clone();
3582 let bind_trust = trust_for_bind(&bind_harness, &principal);
3583 let bind_principal_id = principal_id(&principal);
3584 let consumer_elicitation_capable = consumer_capabilities
3589 .as_ref()
3590 .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
3591 log::info!(
3592 "subc attach: route {} harness={} principal={} trust={} elicitation={}",
3593 route_channel,
3594 bind_harness,
3595 principal_label(&principal),
3596 bind_trust.label(),
3597 consumer_elicitation_capable
3598 );
3599
3600 let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
3612 user_config_path,
3613 Path::new(&bind_project_root),
3614 );
3615 let config_tiers: Vec<Value> = local_tiers
3616 .iter()
3617 .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
3618 .collect();
3619 let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
3620 let configure_json = json!({
3621 "id": request_id,
3622 "command": "configure",
3623 "project_root": bind_project_root,
3624 "harness": bind_harness,
3625 "session_id": bind_session.clone(),
3626 "config": config_tiers,
3627 });
3628 let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
3629 Ok(req) => req,
3630 Err(error) => {
3631 return send_route_bind_error(
3632 tx,
3633 frame,
3634 "config_divergence",
3635 &format!("failed to build configure request: {error}"),
3636 metrics,
3637 )
3638 .await;
3639 }
3640 };
3641
3642 let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
3643 root: bind_root_id.clone(),
3644 project_root: PathBuf::from(&bind_project_root),
3645 harness: bind_harness.clone(),
3646 session: bind_session.clone(),
3647 trust: bind_trust,
3648 spawn_principal: AuthenticatedPrincipal::RouteBind {
3649 trust: bind_trust.sandbox_trust(),
3650 route_channel,
3651 route_epoch: epoch,
3652 project_root: PathBuf::from(&bind_project_root),
3653 harness: bind_harness.clone(),
3654 session_id: bind_session.clone(),
3655 principal_id: bind_principal_id,
3656 },
3657 consumer_elicitation_capable,
3658 }));
3659 let configure_session = route_identity.session.clone();
3660 let root_was_live = live_roots.contains_key(&bind_root_id);
3661 let inserted_new_actor = register_actor_for_bind(
3662 shared_app,
3663 executor,
3664 push_senders,
3665 &bind_root_id,
3666 route_channel,
3667 root_was_live,
3668 );
3669
3670 let configure_request_id = configure_req.id.clone();
3671 installed_route_epochs.insert(route_channel, epoch);
3672 if let Some(meta) = live_roots.get_mut(&bind_root_id) {
3673 meta.maintenance_queued_kinds.clear();
3674 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
3675 }
3676 let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
3677 bind_root_id.clone(),
3678 Lane::Mutating,
3679 configure_request_id.clone(),
3680 Box::new(move |ctx| {
3681 log_ctx::with_session(Some(configure_session.clone()), || {
3682 dispatch(configure_req, ctx)
3683 })
3684 }),
3685 );
3686 pending_binds.insert(
3687 route_id,
3688 PendingBind {
3689 bind_root_id: bind_root_id.clone(),
3690 inserted_new_actor,
3691 cancelled: false,
3692 configure_request_id: configure_request_id.clone(),
3693 started_at: Instant::now(),
3694 warned_half_deadline: false,
3695 deadline_reported: false,
3696 corr: frame.header.corr,
3697 ver: frame.header.ver,
3698 flags: frame.header.flags,
3699 cancellation: configure_cancellation,
3700 },
3701 );
3702
3703 let completion_tx = control_completion_tx.clone();
3704 let completion_identity = route_identity;
3705 let completion_root = bind_root_id.clone();
3706 let completion_route_channel = route_channel;
3707 let completion_ver = frame.header.ver;
3708 let completion_corr = frame.header.corr;
3709 let completion_flags = frame.header.flags;
3710 let completion_metrics = Arc::clone(metrics);
3711 tokio::spawn(async move {
3712 let _response_task = ResponseTaskGuard::new(&completion_metrics);
3713 let configure_response =
3714 await_executor_response(configure_rx, configure_request_id.clone()).await;
3715 let completion = RouteBindCompletion {
3720 route: route_key(completion_route_channel, epoch),
3721 identity: completion_identity,
3722 bind_root_id: completion_root,
3723 inserted_new_actor,
3724 configure_response,
3725 diagnostics_on_edit,
3726 ver: completion_ver,
3727 corr: completion_corr,
3728 flags: completion_flags,
3729 };
3730 if send_counted_channel(
3731 &completion_tx,
3732 &completion_metrics.control_completion_queued,
3733 completion,
3734 )
3735 .await
3736 .is_err()
3737 {
3738 log::debug!(
3739 "subc attach: dropped RouteBind completion for route {} after loop exit",
3740 completion_route_channel
3741 );
3742 }
3743 });
3744
3745 Ok(())
3746 }
3747 ModuleControlRequest::HealthCheck {} => {
3748 let report = build_health_report(executor, pending_binds, metrics, shared_app);
3749 let body = serde_json::to_vec(&ModuleControlResponse::from(report))
3750 .map_err(SubcError::Json)?;
3751 let response = Frame::build_with_version(
3752 frame.header.ver,
3753 FrameType::Response,
3754 frame.header.flags,
3755 0,
3756 0,
3757 frame.header.corr,
3758 body,
3759 )
3760 .map_err(SubcError::FrameBuild)?;
3761 send_frame(tx, metrics, response).await
3762 }
3763 }
3764}
3765
3766fn install_bash_compressor(ctx: &AppContext) {
3767 let filter_registry_handle = ctx.shared_filter_registry();
3769 let compress_flag = ctx.bash_compress_flag();
3770 ctx.bash_background().set_compressor_with_exit_code(
3771 move |command: &str, output: String, exit_code: Option<i32>| {
3772 if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
3773 return crate::compress::CompressionResult::new(output);
3774 }
3775 let registry_guard = match filter_registry_handle.read() {
3776 Ok(g) => g,
3777 Err(poisoned) => poisoned.into_inner(),
3778 };
3779 crate::compress::compress_with_registry_exit_code(
3780 command,
3781 &output,
3782 exit_code,
3783 ®istry_guard,
3784 )
3785 },
3786 );
3787}
3788
3789fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
3790 let mut diagnostics_on_edit = false;
3791 for tier in tiers {
3792 if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
3793 diagnostics_on_edit = value;
3794 }
3795 }
3796 diagnostics_on_edit
3797}
3798
3799fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
3800 let stripped = strip_jsonc(doc);
3801 let value = serde_json::from_str::<Value>(&stripped).ok()?;
3802 value
3803 .get("lsp")
3804 .and_then(Value::as_object)?
3805 .get("diagnostics_on_edit")
3806 .and_then(Value::as_bool)
3807}
3808
3809async fn send_route_bind_error(
3810 tx: &WriterSender,
3811 frame: &Frame,
3812 code: &str,
3813 message: &str,
3814 metrics: &DispatchPathMetrics,
3815) -> Result<(), SubcError> {
3816 send_route_bind_error_parts(
3817 tx,
3818 frame.header.ver,
3819 frame.header.corr,
3820 frame.header.flags,
3821 code,
3822 message,
3823 metrics,
3824 )
3825 .await
3826}
3827
3828async fn send_route_bind_error_parts(
3829 tx: &WriterSender,
3830 ver: u8,
3831 corr: u64,
3832 flags: Flags,
3833 code: &str,
3834 message: &str,
3835 metrics: &DispatchPathMetrics,
3836) -> Result<(), SubcError> {
3837 let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
3838 send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
3839 log::warn!("subc attach: route bind rejected ({code}): {message}");
3840 Ok(())
3841}
3842
3843async fn handle_tool_call(
3848 tx: &WriterSender,
3849 frame: &Frame,
3850 mut phase_trace: PhaseTrace,
3851 routes: &HashMap<RouteChannel, RouteIdentity>,
3852 pending_binds: &HashMap<RouteChannel, PendingBind>,
3853 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3854 executor: &Arc<Executor>,
3855 shutdown: &Arc<Notify>,
3856 connection_cancel: &PersistentCancelSignal,
3857 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
3858 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
3859 metrics: &Arc<DispatchPathMetrics>,
3860 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3861 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3862 next_bash_ask_corr: &mut u64,
3863 bg_subs: &mut HashMap<RouteChannel, BgSub>,
3864 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3865 bg_wake_pending: &mut HashSet<RouteChannel>,
3866 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
3867 dispatch: DispatchFn,
3868 allow_native_passthrough: bool,
3869) -> Result<(), SubcError> {
3870 let route_id = route_key(frame.header.channel, frame.header.epoch);
3871 if pending_binds.contains_key(&route_id) {
3872 let error = build_error_frame(
3873 frame.header.ver,
3874 frame.header.channel,
3875 frame.header.epoch,
3876 frame.header.corr,
3877 frame.header.flags,
3878 "route_not_bound",
3879 "route is not bound before tool call",
3880 )?;
3881 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3882 }
3883
3884 let Some(identity) = routes.get(&route_id).cloned() else {
3885 let error = build_error_frame(
3886 frame.header.ver,
3887 frame.header.channel,
3888 frame.header.epoch,
3889 frame.header.corr,
3890 frame.header.flags,
3891 "route_not_bound",
3892 "route is not bound before tool call",
3893 )?;
3894 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3895 };
3896 let restore_watcher = live_roots
3897 .get(&identity.root)
3898 .is_some_and(|meta| meta.idle_artifacts_evicted);
3899 if let Some(meta) = live_roots.get_mut(&identity.root) {
3900 meta.reactivate_bound();
3901 }
3902 if restore_watcher {
3903 if let Some(ctx) = executor.actor_context(&identity.root) {
3904 crate::commands::configure::ensure_project_watcher(&ctx);
3905 }
3906 }
3907
3908 let route_request =
3909 serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
3910 if matches!(
3911 route_request,
3912 RouteRequest::BgEvents(BgEventsRequest {
3913 op: BgEventsOp::BgEvents
3914 })
3915 ) {
3916 if let Some(old_sub) = bg_subs.get(&route_id).copied() {
3917 push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
3918 }
3919 if !identity.trust.allows_bash_observation() {
3920 bg_subs.remove(&route_id);
3921 bg_wake_pending.remove(&route_id);
3922 remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
3923 let denied_sub = BgSub {
3924 corr: frame.header.corr,
3925 ver: frame.header.ver,
3926 flags: frame.header.flags,
3927 };
3928 push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
3929 return Ok(());
3930 }
3931 bg_subs.insert(
3932 route_id,
3933 BgSub {
3934 corr: frame.header.corr,
3935 ver: frame.header.ver,
3936 flags: frame.header.flags,
3937 },
3938 );
3939 bg_sub_by_session.insert((identity.root.clone(), identity.session.clone()), route_id);
3940 push::arm_bg_wake(
3941 identity.root.clone(),
3942 identity.session.clone(),
3943 route_id,
3944 bg_wake_pending,
3945 bg_wake_epoch,
3946 );
3947 return Ok(());
3948 }
3949
3950 let RouteRequest::ToolCall(call) = route_request else {
3951 unreachable!("background event subscription returned above")
3952 };
3953 let bare_name = call.name;
3954 let arguments = strip_agent_preview_arg_owned(call.arguments);
3955 let format_context = crate::subc_format::FormatContext::from_tool_call(
3956 &bare_name,
3957 &arguments,
3958 identity.project_root.as_path(),
3959 );
3960
3961 let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
3962 let bind_trust = identity.trust;
3963 let diagnostics_on_edit = live_roots
3964 .get(&identity.root)
3965 .map(|meta| meta.diagnostics_on_edit)
3966 .unwrap_or(false);
3967
3968 let requests_host = bare_name == "bash"
3969 && arguments
3970 .get("sandbox")
3971 .or_else(|| {
3972 arguments
3973 .get("params")
3974 .and_then(|params| params.get("sandbox"))
3975 })
3976 .and_then(Value::as_str)
3977 == Some("host");
3978 if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
3979 let response = Response::error(
3980 request_id.clone(),
3981 "sandbox_escalation_denied",
3982 "sandbox host escalation is unavailable to untrusted principals",
3983 );
3984 let text = crate::subc_format::format_response_with_context(
3985 &bare_name,
3986 &response,
3987 &format_context,
3988 );
3989 let result = ToolCallResult { text, response };
3990 let response_frame = build_tool_response_frame(
3991 frame.header.ver,
3992 route_id,
3993 frame.header.corr,
3994 frame.header.flags,
3995 &result,
3996 bind_trust,
3997 )?;
3998 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3999 }
4000
4001 if matches!(bind_trust, BindTrust::Untrusted)
4002 && is_bash_family_tool(&bare_name)
4003 && (bare_name != "bash" || !identity.consumer_elicitation_capable)
4004 {
4005 let response = bash::bash_denied_untrusted_response(request_id.clone());
4006 let text = crate::subc_format::format_response_with_context(
4007 &bare_name,
4008 &response,
4009 &format_context,
4010 );
4011 let result = ToolCallResult { text, response };
4012 let response_frame = build_tool_response_frame(
4013 frame.header.ver,
4014 route_id,
4015 frame.header.corr,
4016 frame.header.flags,
4017 &result,
4018 bind_trust,
4019 )?;
4020 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4021 }
4022
4023 if !is_subc_agent_core_tool(&bare_name)
4031 && !is_subc_native_plumbing_tool(&bare_name)
4032 && !allow_native_passthrough
4033 {
4034 log::warn!(
4035 "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
4036 bare_name,
4037 frame.header.channel
4038 );
4039 let response = Response::error(
4040 request_id.clone(),
4041 "unknown_tool",
4042 format!("tool {:?} is not in the AFT tool manifest", bare_name),
4043 );
4044 let text = crate::subc_format::format_response_with_context(
4045 &bare_name,
4046 &response,
4047 &format_context,
4048 );
4049 let result = ToolCallResult { text, response };
4050 let response_frame = build_tool_response_frame(
4051 frame.header.ver,
4052 route_id,
4053 frame.header.corr,
4054 frame.header.flags,
4055 &result,
4056 bind_trust,
4057 )?;
4058 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4059 }
4060
4061 if bare_name == "bash" {
4062 if matches!(bind_trust, BindTrust::Untrusted) {
4063 let plan = match bash::prepare_bash_elicitation_plan(
4064 &arguments,
4065 identity.project_root.as_path(),
4066 ) {
4067 Ok(plan) => plan,
4068 Err(error) => {
4069 let response = Response::error(request_id.clone(), error.code, error.message);
4070 let text = crate::subc_format::format_response_with_context(
4071 &bare_name,
4072 &response,
4073 &format_context,
4074 );
4075 let result = ToolCallResult { text, response };
4076 let response_frame = build_tool_response_frame(
4077 frame.header.ver,
4078 route_id,
4079 frame.header.corr,
4080 frame.header.flags,
4081 &result,
4082 bind_trust,
4083 )?;
4084 return send_reliable_writer_frame(
4085 tx,
4086 metrics,
4087 response_frame,
4088 "tool response",
4089 )
4090 .await;
4091 }
4092 };
4093
4094 let reverse_corr =
4095 allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
4096 let ask_frame = build_bash_elicitation_request_frame(
4097 frame.header.ver,
4098 route_id,
4099 reverse_corr,
4100 frame.header.flags,
4101 &plan.command,
4102 &plan.asks,
4103 )?;
4104
4105 let meta = live_roots
4106 .entry(identity.root.clone())
4107 .or_insert_with(|| RootMeta::new(Instant::now()));
4108 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4109 meta.reactivate_bound();
4110
4111 let route_cancel =
4112 route_bash_cancels
4113 .entry(route_id)
4114 .or_insert_with(|| bash::RouteBashCancel {
4115 token: PersistentCancelSignal::new(),
4116 active_waits: 0,
4117 });
4118 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4119 let cancel = bash::BashWaitCancel {
4120 connection: connection_cancel.clone(),
4121 route: route_cancel.token.clone(),
4122 };
4123 pending_bash_asks.insert(
4124 ReverseCorrKey {
4125 route: route_id,
4126 corr: reverse_corr,
4127 },
4128 PendingBashAsk {
4129 route: route_id,
4130 tool_corr: frame.header.corr,
4131 tool_flags: frame.header.flags,
4132 tool_ver: frame.header.ver,
4133 root: identity.root.clone(),
4134 project_root: identity.project_root.clone(),
4135 session_id: identity.session.clone(),
4136 spawn_principal: identity.spawn_principal.clone(),
4137 request_id,
4138 arguments,
4139 format_context,
4140 cancel,
4141 grants: plan.grants,
4142 expires_at: Instant::now() + bash_elicitation_timeout(),
4143 },
4144 );
4145 return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
4146 .await;
4147 }
4148
4149 let meta = live_roots
4150 .entry(identity.root.clone())
4151 .or_insert_with(|| RootMeta::new(Instant::now()));
4152 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4153 meta.reactivate_bound();
4154
4155 let route_cancel =
4156 route_bash_cancels
4157 .entry(route_id)
4158 .or_insert_with(|| bash::RouteBashCancel {
4159 token: PersistentCancelSignal::new(),
4160 active_waits: 0,
4161 });
4162 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4163 let cancel = bash::BashWaitCancel {
4164 connection: connection_cancel.clone(),
4165 route: route_cancel.token.clone(),
4166 };
4167
4168 bash::submit_deferred_bash(
4169 executor,
4170 bash_deferred_tx,
4171 bash_poll_touch_tx,
4172 metrics,
4173 dispatch,
4174 identity.root.clone(),
4175 identity.project_root.clone(),
4176 identity.session.clone(),
4177 request_id,
4178 route_id,
4179 frame.header.corr,
4180 frame.header.flags,
4181 frame.header.ver,
4182 arguments,
4183 format_context,
4184 cancel,
4185 bind_trust,
4186 identity.spawn_principal.clone(),
4187 None,
4188 );
4189 return Ok(());
4190 }
4191
4192 let lane = command_lane(&bare_name);
4193 let tool_call_context = ToolCallContext {
4194 project_root: identity.project_root.clone(),
4195 session_id: Some(identity.session.clone()),
4196 request_id: request_id.clone(),
4197 diagnostics_on_edit,
4198 preview: call.preview,
4199 };
4200 let bare_name_for_frame = bare_name.clone();
4201 let identity_for_run = identity.clone();
4202 let completion_session = identity.session.clone();
4203 let completion_root = identity.project_root.clone();
4204 let request_id_for_force = request_id.clone();
4205 let format_context_for_frame = format_context.clone();
4206 let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
4207 phase_trace.mark_executor_submitted();
4208 let rx = executor.submit_async(
4209 identity.root.clone(),
4210 lane,
4211 request_id.clone(),
4212 Box::new(move |ctx| {
4213 phase_trace.mark_job_admitted();
4214 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4215 let run = || {
4216 let finalizer = |response: &mut Response| {
4217 crate::response_finalize::finalize_response_with_bg_completions(
4218 response,
4219 ctx,
4220 &identity_for_run.session,
4221 &bare_name,
4222 bind_trust.allows_bash_observation(),
4223 );
4224 };
4225 match run_tool_call(
4226 &bare_name,
4227 arguments,
4228 &format_context,
4229 &tool_call_context,
4230 ctx,
4231 &dispatch,
4232 Some(&finalizer),
4233 Some(&mut phase_trace),
4234 ) {
4235 ToolCallOutcome::Unary(result) => {
4236 let response = result.response;
4237 let _ = tool_call_tx.send(ToolCallCompletion {
4238 text: result.text,
4239 phase_trace,
4240 });
4241 response
4242 }
4243 }
4244 };
4245 if matches!(bind_trust, BindTrust::Untrusted) {
4246 ctx.with_force_restrict(&request_id_for_force, run)
4247 } else {
4248 run()
4249 }
4250 })
4251 }),
4252 );
4253 let completion_tx = tx.clone();
4254 let completion_shutdown = Arc::clone(shutdown);
4255 let route = route_id;
4256 let corr = frame.header.corr;
4257 let flags = frame.header.flags;
4258 let ver = frame.header.ver;
4259 let completion_metrics = Arc::clone(metrics);
4260 tokio::spawn(async move {
4261 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4262 let response = await_executor_response(rx, request_id.clone()).await;
4263 let (text, phase_trace) = match tool_call_rx.await {
4264 Ok(completion) => (completion.text, Some(completion.phase_trace)),
4265 Err(_) => (
4266 crate::subc_format::format_response_with_context(
4267 &bare_name_for_frame,
4268 &response,
4269 &format_context_for_frame,
4270 ),
4271 None,
4272 ),
4273 };
4274 let result = ToolCallResult { text, response };
4275 let fatal = response_is_fatal_panic(&result.response);
4276 match build_tool_response_frame(ver, route, corr, flags, &result, bind_trust) {
4277 Ok(response_frame) => {
4278 let send_result = if let Some(phase_trace) = phase_trace {
4279 let trace = ToolResponseWriteTrace::new(
4280 phase_trace,
4281 bare_name_for_frame,
4282 completion_root,
4283 completion_session,
4284 route.channel,
4285 corr,
4286 );
4287 send_traced_tool_response_frame(
4288 &completion_tx,
4289 &completion_metrics,
4290 response_frame,
4291 trace,
4292 )
4293 .await
4294 } else {
4295 send_reliable_writer_frame(
4296 &completion_tx,
4297 &completion_metrics,
4298 response_frame,
4299 "tool response",
4300 )
4301 .await
4302 };
4303 if let Err(error) = send_result {
4304 log::warn!("subc attach: failed to queue tool response frame: {error}");
4305 }
4306 }
4307 Err(error) => {
4308 log::error!("subc attach: failed to build tool response frame: {error}");
4309 }
4310 }
4311 if fatal {
4312 signal_fatal_teardown(
4313 &completion_tx,
4314 Some(route),
4315 ver,
4316 corr,
4317 &completion_shutdown,
4318 &completion_metrics,
4319 )
4320 .await;
4321 }
4322 });
4323 Ok(())
4324}
4325
4326fn submit_maintenance_job(
4327 executor: &Arc<Executor>,
4328 root_id: ProjectRootId,
4329 kind: MaintenanceDrainKind,
4330 bg_sessions_to_check: Vec<(String, u64)>,
4331 completion_tx: &mpsc::Sender<MaintenanceCompletion>,
4332 metrics: &Arc<DispatchPathMetrics>,
4333) {
4334 let request_id = format!(
4335 "subc-maintenance-drain-{}-{}",
4336 kind.label(),
4337 root_id.as_path().to_string_lossy()
4338 );
4339 let response_id = request_id.clone();
4340 let completion_root_id = root_id.clone();
4341 let maintenance_generation = executor
4342 .actor_context(&root_id)
4343 .map(|ctx| ctx.configure_generation())
4344 .unwrap_or(0);
4345 let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
4346 let lane = match kind {
4351 MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
4352 MaintenanceDrainKind::Watcher
4353 | MaintenanceDrainKind::Lsp
4354 | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
4355 };
4356 let rx = executor.submit_maintenance_async(
4357 root_id,
4358 lane,
4359 request_id.clone(),
4360 Box::new(move |ctx| {
4361 let outcome = match kind {
4362 MaintenanceDrainKind::Watcher => {
4363 let drained = runtime_drain::drain_watcher_events_bounded(
4364 ctx,
4365 runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
4366 );
4367 MaintenanceJobOutcome {
4368 empty_bg_sessions: Vec::new(),
4369 requeue_kind: drained.has_more.then_some(kind),
4370 }
4371 }
4372 MaintenanceDrainKind::Lsp => {
4373 let drained = runtime_drain::drain_lsp_events_bounded(
4374 ctx,
4375 runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
4376 );
4377 MaintenanceJobOutcome {
4378 empty_bg_sessions: Vec::new(),
4379 requeue_kind: drained.has_more.then_some(kind),
4380 }
4381 }
4382 MaintenanceDrainKind::ConfigureTail => {
4383 runtime_drain::drain_deferred_configure_maintenance(ctx);
4384 runtime_drain::drain_configure_warning_events(ctx);
4385 MaintenanceJobOutcome::default()
4386 }
4387 MaintenanceDrainKind::CompletionDrains => {
4388 runtime_drain::drain_search_index_events(ctx);
4389 runtime_drain::drain_callgraph_store_events(ctx);
4390 runtime_drain::drain_semantic_index_events(ctx);
4391 runtime_drain::drain_semantic_refresh_events(ctx);
4392 runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
4393 let empty_bg_sessions = bg_sessions_to_check
4394 .into_iter()
4395 .filter(|(session, _)| {
4396 !ctx.bash_background()
4397 .has_completions_for_session(Some(session.as_str()))
4398 })
4399 .collect();
4400 MaintenanceJobOutcome {
4401 empty_bg_sessions,
4402 requeue_kind: None,
4403 }
4404 }
4405 };
4406 let requeued = outcome.requeue_kind.is_some();
4407 let _ = outcome_tx.send(outcome);
4408 Response::success(
4409 response_id,
4410 json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
4411 )
4412 }),
4413 );
4414 let completion_tx = completion_tx.clone();
4415 let completion_metrics = Arc::clone(metrics);
4416 tokio::spawn(async move {
4417 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4418 let response = await_executor_response(rx, request_id).await;
4419 let outcome = outcome_rx.await.unwrap_or_default();
4420 let _ = send_counted_channel(
4421 &completion_tx,
4422 &completion_metrics.maintenance_queued,
4423 MaintenanceCompletion {
4424 root_id: completion_root_id,
4425 kind,
4426 response,
4427 empty_bg_sessions: outcome.empty_bg_sessions,
4428 requeue_kind: outcome.requeue_kind,
4429 },
4430 )
4431 .await;
4432 });
4433}
4434
4435async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
4436 rx.await
4437 .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
4438}
4439async fn signal_fatal_teardown(
4440 tx: &WriterSender,
4441 route: Option<RouteChannel>,
4442 ver: u8,
4443 corr: u64,
4444 shutdown: &Arc<Notify>,
4445 metrics: &DispatchPathMetrics,
4446) {
4447 if let Some(route) = route {
4448 if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
4449 if let Err(error) = send_frame(tx, metrics, frame).await {
4450 log::warn!(
4451 "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
4452 );
4453 }
4454 }
4455 }
4456 if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
4457 if let Err(error) = send_frame(tx, metrics, frame).await {
4458 log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
4459 }
4460 }
4461 shutdown.notify_one();
4462}
4463#[derive(Debug, Deserialize)]
4464#[serde(untagged)]
4465enum RouteRequest {
4466 BgEvents(BgEventsRequest),
4467 ToolCall(ToolCallRequest),
4468}
4469
4470#[derive(Debug, Deserialize)]
4471struct BgEventsRequest {
4472 op: BgEventsOp,
4473}
4474
4475#[derive(Debug, Deserialize)]
4476#[serde(rename_all = "snake_case")]
4477enum BgEventsOp {
4478 BgEvents,
4479}
4480
4481#[derive(Debug, Deserialize)]
4482struct ToolCallRequest {
4483 name: String,
4484 #[serde(default)]
4485 arguments: Value,
4486 #[serde(default)]
4491 preview: bool,
4492}
4493
4494#[cfg(test)]
4495pub(crate) mod test_support {
4496 use super::*;
4497 use crate::bash_background::BgTaskStatus;
4498 use crate::protocol::{
4499 BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
4500 ProgressFrame, StatusChangedFrame,
4501 };
4502 use serde_json::json;
4503
4504 pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
4505 let dir = tempfile::Builder::new()
4506 .prefix(name)
4507 .tempdir()
4508 .expect("temp root");
4509 let root = ProjectRootId::from_path(dir.path()).expect("project root id");
4510 (dir, root)
4511 }
4512
4513 pub(super) fn test_ctx() -> Arc<AppContext> {
4514 Arc::new(AppContext::new(
4515 Box::new(crate::parser::TreeSitterProvider::new()),
4516 crate::config::Config::default(),
4517 ))
4518 }
4519
4520 pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
4521 let deadline = Instant::now() + Duration::from_secs(30);
4522 loop {
4523 let observed = ctx.watcher_registry_count();
4524 if observed == expected {
4525 return;
4526 }
4527 assert!(
4528 Instant::now() < deadline,
4529 "watcher count did not settle before deadline: expected={expected}, observed={observed}"
4530 );
4531 std::thread::sleep(Duration::from_millis(50));
4532 }
4533 }
4534
4535 pub(super) fn reap_until_forgotten(
4545 root: &ProjectRootId,
4546 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4547 pending_binds: &HashMap<RouteChannel, PendingBind>,
4548 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4549 executor: &Arc<Executor>,
4550 metrics: &DispatchPathMetrics,
4551 ) -> IdleReapOutcome {
4552 let deadline = Instant::now() + Duration::from_secs(30);
4553 loop {
4554 let outcome = reap_idle_roots(
4555 Instant::now(),
4556 live_roots,
4557 pending_binds,
4558 root_channels,
4559 executor,
4560 metrics,
4561 );
4562 if outcome.forgotten_deleted_roots.contains(root) {
4563 return outcome;
4564 }
4565 assert!(
4566 Instant::now() < deadline,
4567 "deleted root was never forgotten: {root:?}"
4568 );
4569 std::thread::sleep(Duration::from_millis(10));
4570 }
4571 }
4572
4573 pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
4574 let deadline = Instant::now() + Duration::from_secs(30);
4575 loop {
4576 let observed = app.actor_root_count();
4577 if observed == expected {
4578 return;
4579 }
4580 assert!(
4581 Instant::now() < deadline,
4582 "actor root count did not settle before deadline: expected={expected}, observed={observed}"
4583 );
4584 std::thread::sleep(Duration::from_millis(50));
4585 }
4586 }
4587
4588 pub(super) fn status_frame(seq: u64) -> PushFrame {
4589 status_frame_with_session(seq, None)
4590 }
4591
4592 pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
4593 PushFrame::StatusChanged(StatusChangedFrame {
4594 frame_type: "status_changed",
4595 session_id: session_id.map(str::to_string),
4596 snapshot: json!({ "seq": seq }),
4597 })
4598 }
4599
4600 pub(super) fn completion_frame(task_id: &str) -> PushFrame {
4601 completion_frame_with_session(task_id, "session-1")
4602 }
4603
4604 pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
4605 PushFrame::BashCompleted(BashCompletedFrame {
4606 frame_type: "bash_completed",
4607 task_id: task_id.to_string(),
4608 session_id: session_id.to_string(),
4609 status: BgTaskStatus::Completed,
4610 exit_code: Some(0),
4611 command: format!("echo {task_id}"),
4612 output_preview: String::new(),
4613 output_truncated: false,
4614 original_tokens: None,
4615 compressed_tokens: None,
4616 tokens_skipped: false,
4617 status_reason: None,
4618 })
4619 }
4620
4621 pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
4622 long_running_frame_with_session(task_id, "session-1", elapsed_ms)
4623 }
4624
4625 pub(super) fn long_running_frame_with_session(
4626 task_id: &str,
4627 session_id: &str,
4628 elapsed_ms: u64,
4629 ) -> PushFrame {
4630 PushFrame::BashLongRunning(BashLongRunningFrame {
4631 frame_type: "bash_long_running",
4632 task_id: task_id.to_string(),
4633 session_id: session_id.to_string(),
4634 command: format!("sleep {elapsed_ms}"),
4635 elapsed_ms,
4636 })
4637 }
4638
4639 pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
4640 PushFrame::BashPatternMatch(BashPatternMatchFrame {
4641 frame_type: "bash_pattern_match",
4642 task_id: "task-pattern".to_string(),
4643 session_id: session_id.to_string(),
4644 watch_id: "watch-1".to_string(),
4645 match_text: "needle".to_string(),
4646 match_offset: 7,
4647 context: "haystack needle".to_string(),
4648 once: true,
4649 reason: "pattern_match",
4650 })
4651 }
4652
4653 pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
4654 PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
4655 frame_type: "configure_warnings",
4656 session_id: session_id.map(str::to_string),
4657 project_root: "/tmp/subc-test".to_string(),
4658 warnings: Vec::new(),
4659 })
4660 }
4661
4662 pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
4663 route_identity_with_trust(root, session_id, BindTrust::FirstParty)
4664 }
4665
4666 pub(super) fn route_identity_with_trust(
4667 root: &ProjectRootId,
4668 session_id: &str,
4669 trust: BindTrust,
4670 ) -> RouteIdentity {
4671 RouteIdentity(Arc::new(RouteIdentityData {
4672 root: root.clone(),
4673 project_root: root.as_path().to_path_buf(),
4674 harness: "opencode".to_string(),
4675 session: session_id.to_string(),
4676 trust,
4677 spawn_principal: AuthenticatedPrincipal::RouteBind {
4678 trust: trust.sandbox_trust(),
4679 route_channel: 0,
4680 route_epoch: 0,
4681 project_root: root.as_path().to_path_buf(),
4682 harness: "opencode".to_string(),
4683 session_id: session_id.to_string(),
4684 principal_id: Some(match trust {
4685 BindTrust::FirstParty => "direct".to_string(),
4686 BindTrust::Untrusted => "unverified".to_string(),
4687 }),
4688 },
4689 consumer_elicitation_capable: false,
4690 }))
4691 }
4692
4693 pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
4694 PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
4695 }
4696
4697 pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
4698 match frame {
4699 PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
4700 _ => None,
4701 }
4702 }
4703
4704 pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
4705 match frame {
4706 PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
4707 _ => None,
4708 }
4709 }
4710
4711 pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
4712 let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
4713 body.get("task_id")
4714 .and_then(serde_json::Value::as_str)
4715 .map(str::to_string)
4716 }
4717}
4718
4719#[cfg(test)]
4720mod tests {
4721 use super::test_support::{
4722 completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
4723 wait_for_actor_root_count, wait_for_watcher_count,
4724 };
4725 use super::*;
4726 use crate::bash_background::BgTaskStatus;
4727
4728 fn attach_error(kind: io::ErrorKind) -> SubcError {
4729 SubcError::Connect {
4730 endpoint: "127.0.0.1:1".to_string(),
4731 source: io::Error::new(kind, "constructed attach failure"),
4732 }
4733 }
4734
4735 fn auth_io_error(kind: io::ErrorKind) -> SubcError {
4736 SubcError::Auth {
4737 endpoint: "127.0.0.1:1".to_string(),
4738 source: subc_transport::AuthError::Io {
4739 stage: subc_transport::AuthStage::ServerProof,
4740 source: io::Error::new(kind, "constructed auth failure"),
4741 },
4742 }
4743 }
4744
4745 #[test]
4746 fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
4747 let transient_errors = vec![
4748 attach_error(io::ErrorKind::ConnectionRefused),
4749 attach_error(io::ErrorKind::TimedOut),
4750 attach_error(io::ErrorKind::ConnectionReset),
4751 auth_io_error(io::ErrorKind::ConnectionAborted),
4752 auth_io_error(io::ErrorKind::BrokenPipe),
4753 SubcError::Auth {
4754 endpoint: "127.0.0.1:1".to_string(),
4755 source: subc_transport::AuthError::UnexpectedEof {
4756 stage: subc_transport::AuthStage::ServerProof,
4757 expected: 4,
4758 actual: 0,
4759 },
4760 },
4761 SubcError::Auth {
4762 endpoint: "127.0.0.1:1".to_string(),
4763 source: subc_transport::AuthError::Timeout {
4764 stage: subc_transport::AuthStage::ServerProof,
4765 deadline: AUTH_DEADLINE,
4766 },
4767 },
4768 ];
4769 for error in &transient_errors {
4770 assert_eq!(
4771 classify_attach_error(error),
4772 AttachErrorClass::Transient,
4773 "expected transient: {error}"
4774 );
4775 }
4776
4777 let permanent_errors = vec![
4778 attach_error(io::ErrorKind::PermissionDenied),
4779 auth_io_error(io::ErrorKind::InvalidData),
4780 SubcError::Auth {
4781 endpoint: "127.0.0.1:1".to_string(),
4782 source: subc_transport::AuthError::InvalidServerProof,
4783 },
4784 SubcError::Auth {
4785 endpoint: "127.0.0.1:1".to_string(),
4786 source: subc_transport::AuthError::DaemonIdMismatch,
4787 },
4788 SubcError::ConnectionFile {
4789 path: PathBuf::from("subc-connection.json"),
4790 source: subc_transport::ConnectionFileError::Invalid {
4791 reason: "constructed invalid file".to_string(),
4792 },
4793 },
4794 SubcError::NoEndpoint {
4795 path: PathBuf::from("subc-connection.json"),
4796 },
4797 SubcError::InvalidEndpoint {
4798 path: PathBuf::from("subc-connection.json"),
4799 endpoint: "not-an-ip:1234".to_string(),
4800 },
4801 ];
4802 for error in &permanent_errors {
4803 assert_eq!(
4804 classify_attach_error(error),
4805 AttachErrorClass::Permanent,
4806 "expected permanent: {error}"
4807 );
4808 }
4809 }
4810
4811 #[test]
4812 fn incompatible_wire_version_is_rejected_before_tcp_connect() {
4813 let conn_dir = tempfile::tempdir().expect("connection tempdir");
4814 let conn_path = conn_dir.path().join("subc-connection.json");
4815 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
4816 listener
4817 .set_nonblocking(true)
4818 .expect("set listener nonblocking");
4819 let port = listener.local_addr().expect("listener addr").port();
4820 connection_file::write_atomic(
4821 &conn_path,
4822 &connection_file::ConnectionInfo {
4823 schema: connection_file::SCHEMA_VERSION,
4824 wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
4825 endpoints: vec![connection_file::Endpoint {
4826 host: "127.0.0.1".to_string(),
4827 port,
4828 }],
4829 key: vec![0x42; subc_transport::KEY_LEN],
4830 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4831 pid: std::process::id(),
4832 daemon_ver: "subc-test".to_string(),
4833 },
4834 )
4835 .expect("write connection file");
4836
4837 let runtime = tokio::runtime::Builder::new_current_thread()
4838 .enable_all()
4839 .build()
4840 .expect("test runtime");
4841 let result = runtime.block_on(connect_and_authenticate_with_policy(
4842 &conn_path,
4843 AttachRetryPolicy {
4844 budget: Duration::from_secs(1),
4845 initial_backoff: Duration::from_millis(5),
4846 max_backoff: Duration::from_millis(10),
4847 jitter_percent: 0,
4848 },
4849 ));
4850 assert!(matches!(
4851 result,
4852 Err(SubcError::ConnectionFile {
4853 source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
4854 ..
4855 })
4856 ));
4857 assert!(matches!(
4858 listener.accept(),
4859 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
4860 ));
4861 }
4862
4863 #[test]
4864 fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
4865 let conn_dir = tempfile::tempdir().expect("connection tempdir");
4866 let conn_path = conn_dir.path().join("subc-connection.json");
4867 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
4868 let port = listener.local_addr().expect("reserved addr").port();
4869 drop(listener);
4870 connection_file::write_atomic(
4871 &conn_path,
4872 &connection_file::ConnectionInfo {
4873 schema: connection_file::SCHEMA_VERSION,
4874 wire_version: Some(PROTOCOL_VERSION),
4875 endpoints: vec![connection_file::Endpoint {
4876 host: "127.0.0.1".to_string(),
4877 port,
4878 }],
4879 key: vec![0x42; subc_transport::KEY_LEN],
4880 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4881 pid: std::process::id(),
4882 daemon_ver: "subc-test".to_string(),
4883 },
4884 )
4885 .expect("write connection file");
4886
4887 let policy = AttachRetryPolicy {
4888 budget: Duration::from_millis(40),
4889 initial_backoff: Duration::from_millis(5),
4890 max_backoff: Duration::from_millis(10),
4891 jitter_percent: 0,
4892 };
4893 let runtime = tokio::runtime::Builder::new_current_thread()
4894 .enable_all()
4895 .build()
4896 .expect("test runtime");
4897 let started_at = Instant::now();
4898 let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
4899 let elapsed = started_at.elapsed();
4900 let error = match result {
4901 Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
4902 Err(error) => error,
4903 };
4904
4905 assert!(matches!(error, SubcError::Connect { .. }), "{error}");
4906 assert!(
4907 elapsed >= Duration::from_millis(35),
4908 "retry budget ended too early: {elapsed:?}"
4909 );
4910 assert!(
4911 elapsed < Duration::from_secs(1),
4912 "retry budget was not bounded: {elapsed:?}"
4913 );
4914 }
4915
4916 fn due_maintenance_jobs_without_actor_context(
4917 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4918 budget: usize,
4919 pending_bind_roots: &HashSet<ProjectRootId>,
4920 ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
4921 due_maintenance_jobs(
4922 live_roots,
4923 None,
4924 &HashMap::new(),
4925 &HashSet::new(),
4926 budget,
4927 pending_bind_roots,
4928 )
4929 }
4930
4931 fn actor_ctx_with_dirty_search_index(
4932 root: &Path,
4933 storage: &Path,
4934 file_name: &str,
4935 old_contents: &str,
4936 new_contents: &str,
4937 ) -> (Arc<AppContext>, PathBuf, PathBuf) {
4938 let file = root.join(file_name);
4939 std::fs::write(&file, old_contents).expect("write source");
4940 let canonical_root = std::fs::canonicalize(root).expect("canonical root");
4941 let ctx = Arc::new(AppContext::new(
4942 Box::new(crate::parser::TreeSitterProvider::new()),
4943 Config {
4944 project_root: Some(root.to_path_buf()),
4945 storage_dir: Some(storage.to_path_buf()),
4946 ..Config::default()
4947 },
4948 ));
4949 ctx.set_canonical_cache_root(canonical_root.clone());
4950
4951 let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
4952 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
4953 let git_head = index.stored_git_head().map(str::to_owned);
4954 index.write_to_disk(&cache_dir, git_head.as_deref());
4955
4956 std::fs::write(&file, new_contents).expect("edit source");
4957 index.update_file(&file);
4958 *ctx.search_index()
4959 .write()
4960 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
4961 (ctx, canonical_root, cache_dir)
4962 }
4963
4964 #[test]
4965 fn graceful_shutdown_flushes_every_actor_search_index() {
4966 let storage = tempfile::tempdir().expect("storage tempdir");
4967 let (root1_dir, root1) = test_root("shutdown-flush-root-1");
4968 let (root2_dir, root2) = test_root("shutdown-flush-root-2");
4969 let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
4970 root1_dir.path(),
4971 storage.path(),
4972 "alpha.txt",
4973 "old actor one token\n",
4974 "new actor one token\n",
4975 );
4976 let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
4977 root2_dir.path(),
4978 storage.path(),
4979 "beta.txt",
4980 "old actor two token\n",
4981 "new actor two token\n",
4982 );
4983
4984 let executor = Executor::new();
4985 assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
4986 assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
4987
4988 flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
4989
4990 let mut restored1 =
4991 crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
4992 .expect("load flushed root one index");
4993 restored1.ready = true;
4994 assert_eq!(
4995 restored1
4996 .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
4997 .matches
4998 .len(),
4999 1,
5000 "graceful subc shutdown should flush the first root's trigram delta"
5001 );
5002
5003 let mut restored2 =
5004 crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
5005 .expect("load flushed root two index");
5006 restored2.ready = true;
5007 assert_eq!(
5008 restored2
5009 .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
5010 .matches
5011 .len(),
5012 1,
5013 "graceful subc shutdown should flush every registered root"
5014 );
5015 }
5016
5017 #[test]
5018 fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
5019 let _ = env_logger::builder().is_test(true).try_init();
5020 let (root_dir, root) = test_root("idle-root-reaper");
5021 let storage = tempfile::tempdir().expect("storage tempdir");
5022 std::fs::write(
5023 root_dir.path().join("main.rs"),
5024 "fn entry() { leaf(); }\nfn leaf() {}\n",
5025 )
5026 .expect("source file");
5027 let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
5028 let app = App::default_shared();
5029 let ctx = Arc::new(AppContext::from_app(
5030 Arc::clone(&app),
5031 Config {
5032 project_root: Some(canonical_root.clone()),
5033 storage_dir: Some(storage.path().to_path_buf()),
5034 callgraph_store: true,
5035 search_index: true,
5036 ..Config::default()
5037 },
5038 ));
5039 ctx.set_canonical_cache_root(canonical_root.clone());
5040 assert!(ctx
5041 .ensure_callgraph_store()
5042 .expect("build callgraph store")
5043 .is_some());
5044
5045 let cache_dir =
5046 crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
5047 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
5048 let git_head = index.stored_git_head().map(str::to_owned);
5049 index.write_to_disk(&cache_dir, git_head.as_deref());
5050 *ctx.search_index()
5051 .write()
5052 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
5053 let seeded_generation =
5056 crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
5057 .expect("seeded artifact generation");
5058 crate::cache_freshness::record_verify_completed(
5059 &canonical_root,
5060 crate::cache_freshness::VerifyArtifact::Search,
5061 Some(seeded_generation),
5062 );
5063 assert!(
5064 matches!(
5065 crate::cache_freshness::warm_verify_plan(
5066 canonical_root.as_path(),
5067 crate::cache_freshness::VerifyArtifact::Search,
5068 Some(seeded_generation),
5069 ),
5070 crate::cache_freshness::WarmVerifyPlan::Skip
5071 ),
5072 "memo must be warm before eviction for the downgrade assertion to bite"
5073 );
5074
5075 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5076 let _dispatch_tx = dispatch_tx;
5077 let shutdown = Arc::new(AtomicBool::new(false));
5078 let thread_shutdown = Arc::clone(&shutdown);
5079 let join = std::thread::spawn(move || {
5080 while !thread_shutdown.load(Ordering::SeqCst) {
5081 std::thread::yield_now();
5082 }
5083 });
5084 ctx.install_watcher_runtime(
5085 dispatch_rx,
5086 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5087 );
5088 wait_for_watcher_count(&ctx, 1);
5089
5090 let executor = Arc::new(Executor::new());
5091 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5092 ctx.mark_subc_unbound();
5093 let mut live_roots = HashMap::new();
5094 let mut meta = RootMeta::new(Instant::now());
5095 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5096 meta.unbound_quiesced = true;
5097 live_roots.insert(root.clone(), meta);
5098
5099 let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
5100 assert!(message.contains("evicted idle root"));
5101 assert!(message.contains("freed ~"));
5102 assert!(message.contains("semantic"));
5103 assert!(!message.contains("semantic not estimated retained"));
5104 assert!(message.contains("trigram"));
5105 assert!(message.contains("retained: bash"));
5106 assert!(message.contains("parser_pool"));
5107
5108 assert_eq!(
5109 reap_idle_roots(
5110 Instant::now(),
5111 &mut live_roots,
5112 &HashMap::new(),
5113 &HashMap::new(),
5114 &executor,
5115 &DispatchPathMetrics::new(),
5116 )
5117 .evicted,
5118 1
5119 );
5120 assert!(ctx.search_index().read().unwrap().is_none());
5121 wait_for_watcher_count(&ctx, 0);
5122 assert!(
5127 matches!(
5128 crate::cache_freshness::warm_verify_plan(
5129 canonical_root.as_path(),
5130 crate::cache_freshness::VerifyArtifact::Search,
5131 Some(seeded_generation),
5132 ),
5133 crate::cache_freshness::WarmVerifyPlan::Strict
5134 ),
5135 "idle eviction must force strict re-verification"
5136 );
5137 assert!(
5138 crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
5139 );
5140 ctx.mark_subc_bound();
5141 assert!(ctx
5142 .ensure_callgraph_store()
5143 .expect("reopen callgraph store")
5144 .is_some());
5145 assert!(live_roots[&root].idle_artifacts_evicted);
5146 }
5147
5148 #[test]
5149 fn idle_root_reaper_applies_ttl_to_unbound_roots() {
5150 let (_root_dir, root) = test_root("idle-root-ttl-gate");
5151 let ctx = test_ctx();
5152 let executor = Arc::new(Executor::new());
5153 assert!(executor.register_actor(root.clone(), ctx));
5154 let ctx = executor.actor_context(&root).expect("actor context");
5155 ctx.mark_subc_unbound();
5156 let now = Instant::now();
5157 let mut meta = RootMeta::new(now);
5158 meta.unbound_quiesced = true;
5159 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5160
5161 assert_eq!(
5165 reap_idle_roots(
5166 now,
5167 &mut live_roots,
5168 &HashMap::new(),
5169 &HashMap::new(),
5170 &executor,
5171 &DispatchPathMetrics::new(),
5172 )
5173 .evicted,
5174 0
5175 );
5176 assert!(!live_roots[&root].idle_artifacts_evicted);
5177
5178 ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
5184 assert_eq!(
5185 reap_idle_roots(
5186 now + IDLE_ROOT_TTL,
5187 &mut live_roots,
5188 &HashMap::new(),
5189 &HashMap::new(),
5190 &executor,
5191 &DispatchPathMetrics::new(),
5192 )
5193 .evicted,
5194 1
5195 );
5196 assert!(live_roots[&root].idle_artifacts_evicted);
5197 assert!(
5198 ctx.take_pending_search_index_paths().is_empty(),
5199 "TTL eviction must dispose retained pending reconciliation paths"
5200 );
5201 }
5202
5203 #[test]
5204 fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
5205 let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
5206 let ctx = test_ctx();
5207 let executor = Arc::new(Executor::new());
5208 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5209 ctx.mark_subc_unbound();
5210
5211 let pending = root.as_path().join("edited-while-unbound.rs");
5217 ctx.add_pending_search_index_paths([pending.clone()]);
5218 let dirty_source = root.as_path().join("dirty.rs");
5219 std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
5220 let mut dirty = crate::search_index::SearchIndex::new();
5221 dirty.ready = true;
5222 dirty.update_file(&dirty_source);
5223 *ctx.search_index()
5224 .write()
5225 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
5226 assert!(ctx.artifact_eviction_blocked());
5227
5228 let mut live_roots = HashMap::new();
5229 let mut meta = RootMeta::new(Instant::now());
5230 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5231 meta.unbound_quiesced = true;
5232 live_roots.insert(root.clone(), meta);
5233
5234 assert_eq!(
5235 reap_idle_roots(
5236 Instant::now(),
5237 &mut live_roots,
5238 &HashMap::new(),
5239 &HashMap::new(),
5240 &executor,
5241 &DispatchPathMetrics::new(),
5242 )
5243 .evicted,
5244 0,
5245 "the dirty index must still block this eviction"
5246 );
5247 assert_eq!(
5248 ctx.take_pending_search_index_paths(),
5249 vec![pending],
5250 "a blocked eviction must restore the taken pending paths"
5251 );
5252 }
5253
5254 #[test]
5255 fn idle_reap_with_bound_route_keeps_watcher_running() {
5256 let (_root_dir, root) = test_root("bound-root-reap-gate");
5257 let ctx = test_ctx();
5258 let executor = Arc::new(Executor::new());
5259 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5260
5261 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5262 let _dispatch_tx = dispatch_tx;
5263 let shutdown = Arc::new(AtomicBool::new(false));
5264 let thread_shutdown = Arc::clone(&shutdown);
5265 let join = std::thread::spawn(move || {
5266 while !thread_shutdown.load(Ordering::SeqCst) {
5267 std::thread::yield_now();
5268 }
5269 });
5270 ctx.install_watcher_runtime(
5271 dispatch_rx,
5272 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5273 );
5274
5275 let mut meta = RootMeta::new(Instant::now());
5276 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5277 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5278 let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
5279 assert_eq!(
5280 reap_idle_roots(
5281 Instant::now(),
5282 &mut live_roots,
5283 &HashMap::new(),
5284 &bound,
5285 &executor,
5286 &DispatchPathMetrics::new(),
5287 )
5288 .evicted,
5289 0
5290 );
5291 wait_for_watcher_count(&ctx, 1);
5292 ctx.stop_watcher_runtime_in_background();
5293 wait_for_watcher_count(&ctx, 0);
5294 }
5295
5296 #[test]
5297 fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
5298 let (root_dir, root) = test_root("deleted-bound-root-reap");
5299 let executor = Arc::new(Executor::new());
5300 assert!(executor.register_actor(root.clone(), test_ctx()));
5301 root_dir.close().expect("delete project root");
5302
5303 let route = route_key(19, 3);
5304 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5305 let cancel_signal = PersistentCancelSignal::new();
5306 let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
5307 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5308 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5309 let mut route_bash_cancels = HashMap::from([(
5310 route,
5311 bash::RouteBashCancel {
5312 token: cancel_signal.clone(),
5313 active_waits: 0,
5314 },
5315 )]);
5316 let metrics = DispatchPathMetrics::new();
5317
5318 let first = reap_idle_roots(
5319 Instant::now(),
5320 &mut live_roots,
5321 &HashMap::new(),
5322 &root_channels,
5323 &executor,
5324 &metrics,
5325 );
5326 assert!(first.forgotten_deleted_roots.is_empty());
5327 assert!(executor.actor_registered(&root));
5328
5329 let mut forgotten = Vec::new();
5330 for _ in 0..100 {
5331 let outcome = reap_idle_roots(
5332 Instant::now(),
5333 &mut live_roots,
5334 &HashMap::new(),
5335 &root_channels,
5336 &executor,
5337 &metrics,
5338 );
5339 if !outcome.forgotten_deleted_roots.is_empty() {
5340 forgotten = outcome.forgotten_deleted_roots;
5341 break;
5342 }
5343 std::thread::sleep(Duration::from_millis(10));
5344 }
5345 assert_eq!(forgotten, vec![root.clone()]);
5346 assert!(!executor.actor_registered(&root));
5347
5348 let mut retry_buffer = HashMap::new();
5349 let mut reclaimed_routes = ReclaimedRoutes::default();
5350 let mut session_identity = HashMap::new();
5351 let mut push_buffer = HashMap::new();
5352 let mut bg_subs = HashMap::new();
5353 let mut bg_sub_by_session = HashMap::new();
5354 let mut bg_wake_pending = HashSet::new();
5355 let mut bg_wake_epoch = HashMap::new();
5356 let mut pending_bash_asks = HashMap::new();
5357 purge_deleted_root_residents(
5358 &root,
5359 &mut routes,
5360 &mut root_channels,
5361 &mut installed_route_epochs,
5362 &mut route_bash_cancels,
5363 &mut retry_buffer,
5364 &mut reclaimed_routes,
5365 &mut session_identity,
5366 &mut push_buffer,
5367 &mut bg_subs,
5368 &mut bg_sub_by_session,
5369 &mut bg_wake_pending,
5370 &mut bg_wake_epoch,
5371 &mut pending_bash_asks,
5372 );
5373
5374 assert!(routes.is_empty());
5375 assert!(root_channels.is_empty());
5376 assert!(installed_route_epochs.is_empty());
5377 assert!(route_bash_cancels.is_empty());
5378 assert!(reclaimed_routes.contains(route));
5379 assert!(cancel_signal.is_cancelled());
5380 }
5381
5382 #[test]
5389 fn live_root_with_bound_route_is_never_reclaimed() {
5390 let (_root_dir, root) = test_root("live-bound-root-retained");
5391 let ctx = test_ctx();
5392 ctx.mark_subc_unbound();
5393 let executor = Arc::new(Executor::new());
5394 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5395
5396 let route = RouteChannel {
5397 channel: 7,
5398 epoch: 1,
5399 };
5400 let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
5401 meta.unbound_quiesced = true;
5402 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5403 let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5404
5405 for _ in 0..3 {
5410 let outcome = reap_idle_roots(
5411 Instant::now(),
5412 &mut live_roots,
5413 &HashMap::new(),
5414 &root_channels,
5415 &executor,
5416 &DispatchPathMetrics::new(),
5417 );
5418 assert!(
5419 outcome.forgotten_deleted_roots.is_empty(),
5420 "a root whose directory exists must never be forgotten"
5421 );
5422 }
5423
5424 assert!(live_roots.contains_key(&root), "live root must be retained");
5425 assert!(
5426 executor.actor_registered(&root),
5427 "live root's actor must survive"
5428 );
5429 assert!(
5430 root.as_path().exists(),
5431 "test vehicle must keep the directory alive; otherwise this control proves nothing"
5432 );
5433 }
5434
5435 #[test]
5436 fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
5437 let (root_dir, root) = test_root("deleted-root-first-observation");
5438 let ctx = test_ctx();
5439 ctx.mark_subc_unbound();
5440 let executor = Arc::new(Executor::new());
5441 assert!(executor.register_actor(root.clone(), ctx));
5442 root_dir.close().expect("delete project root");
5443
5444 let mut meta = RootMeta::new(Instant::now());
5445 meta.unbound_quiesced = true;
5446 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5447 let outcome = reap_idle_roots(
5448 Instant::now(),
5449 &mut live_roots,
5450 &HashMap::new(),
5451 &HashMap::new(),
5452 &executor,
5453 &DispatchPathMetrics::new(),
5454 );
5455
5456 assert!(outcome.forgotten_deleted_roots.is_empty());
5457 assert!(live_roots.contains_key(&root));
5458 assert!(executor.actor_registered(&root));
5459 }
5460
5461 fn spawn_background_for_root(
5462 ctx: &AppContext,
5463 root: &ProjectRootId,
5464 storage: &tempfile::TempDir,
5465 session_id: &str,
5466 ) -> (String, u32) {
5467 let command = if cfg!(windows) {
5473 "ping -n 31 127.0.0.1 > nul"
5475 } else {
5476 "sleep 30"
5477 };
5478 let task_id = ctx
5479 .bash_background()
5480 .spawn(
5481 crate::sandbox_spawn::SpawnPlan::Unsandboxed,
5482 command,
5483 session_id.to_string(),
5484 storage.path().to_path_buf(),
5485 HashMap::new(),
5486 Some(Duration::from_secs(60)),
5487 storage.path().to_path_buf(),
5488 8,
5489 true,
5490 false,
5491 Some(root.as_path().to_path_buf()),
5492 )
5493 .expect("spawn background task");
5494 let snapshot = ctx
5495 .bash_background()
5496 .status(
5497 &task_id,
5498 session_id,
5499 Some(root.as_path()),
5500 Some(storage.path()),
5501 0,
5502 )
5503 .expect("background task status");
5504 (task_id, snapshot.child_pid.expect("background child pid"))
5505 }
5506
5507 fn wait_for_background_exit(pid: u32) {
5508 let deadline = Instant::now() + Duration::from_secs(5);
5509 while crate::bash_background::process::is_process_alive(pid) {
5510 assert!(
5511 Instant::now() < deadline,
5512 "background task process survived kill"
5513 );
5514 std::thread::sleep(Duration::from_millis(20));
5515 }
5516 }
5517
5518 #[test]
5519 fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
5520 let (root_dir, root) = test_root("deleted-root-background-task");
5521 let storage = tempfile::tempdir().expect("task storage");
5522 let ctx = test_ctx();
5523 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
5524 assert!(crate::bash_background::process::is_process_alive(pid));
5525
5526 let executor = Arc::new(Executor::new());
5527 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5528 assert!(executor.actor_is_idle(&root));
5529 root_dir.close().expect("delete project root");
5530 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5531 let pending_binds = HashMap::new();
5532 let root_channels = HashMap::new();
5533 let metrics = DispatchPathMetrics::new();
5534
5535 let first = reap_idle_roots(
5536 Instant::now(),
5537 &mut live_roots,
5538 &pending_binds,
5539 &root_channels,
5540 &executor,
5541 &metrics,
5542 );
5543 assert!(first.forgotten_deleted_roots.is_empty());
5544 assert!(crate::bash_background::process::is_process_alive(pid));
5545
5546 let outcome = reap_until_forgotten(
5547 &root,
5548 &mut live_roots,
5549 &pending_binds,
5550 &root_channels,
5551 &executor,
5552 &metrics,
5553 );
5554 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5555 wait_for_background_exit(pid);
5556
5557 let snapshot = ctx
5558 .bash_background()
5559 .status(
5560 &task_id,
5561 "reclaim-session",
5562 Some(root.as_path()),
5563 Some(storage.path()),
5564 0,
5565 )
5566 .expect("reclaimed task status");
5567 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
5568 assert_eq!(
5569 snapshot.info.status_reason.as_deref(),
5570 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
5571 );
5572 assert_eq!(
5573 serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
5574 crate::bash_background::registry::ROOT_RECLAIMED_REASON
5575 );
5576 let completion = ctx
5577 .bash_background()
5578 .drain_completions_for_session(Some("reclaim-session"))
5579 .pop()
5580 .expect("reclaimed task completion");
5581 assert_eq!(
5582 completion.status_reason.as_deref(),
5583 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
5584 );
5585 }
5586
5587 #[test]
5588 fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
5589 let (root_dir, root) = test_root("existing-root-background-task");
5590 let storage = tempfile::tempdir().expect("task storage");
5591 let ctx = test_ctx();
5592 ctx.mark_subc_unbound();
5593 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
5594
5595 let executor = Arc::new(Executor::new());
5596 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5597 let mut meta = RootMeta::new(
5598 Instant::now()
5599 .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
5600 .expect("old root timestamp"),
5601 );
5602 meta.unbound_quiesced = true;
5603 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5604 let pending_binds = HashMap::new();
5605 let root_channels = HashMap::new();
5606 let metrics = DispatchPathMetrics::new();
5607
5608 for _ in 0..8 {
5609 reap_idle_roots(
5610 Instant::now(),
5611 &mut live_roots,
5612 &pending_binds,
5613 &root_channels,
5614 &executor,
5615 &metrics,
5616 );
5617 std::thread::sleep(Duration::from_millis(10));
5618 }
5619 assert!(root_dir.path().exists());
5620 assert!(crate::bash_background::process::is_process_alive(pid));
5621 let snapshot = ctx
5622 .bash_background()
5623 .status(
5624 &task_id,
5625 "existing-session",
5626 Some(root.as_path()),
5627 Some(storage.path()),
5628 0,
5629 )
5630 .expect("existing task status");
5631 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5632 let _ = ctx.bash_background().kill(&task_id, "existing-session");
5633 wait_for_background_exit(pid);
5634 }
5635
5636 #[test]
5637 fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
5638 let (root_dir, root) = test_root("restored-root-background-task");
5639 let storage = tempfile::tempdir().expect("task storage");
5640 let ctx = test_ctx();
5641 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
5642
5643 let executor = Arc::new(Executor::new());
5644 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5645 root_dir.close().expect("delete project root");
5646 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5647 let pending_binds = HashMap::new();
5648 let root_channels = HashMap::new();
5649 let metrics = DispatchPathMetrics::new();
5650
5651 let first = reap_idle_roots(
5652 Instant::now(),
5653 &mut live_roots,
5654 &pending_binds,
5655 &root_channels,
5656 &executor,
5657 &metrics,
5658 );
5659 assert!(first.forgotten_deleted_roots.is_empty());
5660 std::fs::create_dir_all(root.as_path()).expect("restore project root");
5661 let second = reap_idle_roots(
5662 Instant::now(),
5663 &mut live_roots,
5664 &pending_binds,
5665 &root_channels,
5666 &executor,
5667 &metrics,
5668 );
5669 assert!(second.forgotten_deleted_roots.is_empty());
5670 assert!(crate::bash_background::process::is_process_alive(pid));
5671 let _ = ctx.bash_background().kill(&task_id, "restored-session");
5672 wait_for_background_exit(pid);
5673 }
5674
5675 #[test]
5676 fn observing_root_again_resets_deleted_sweep_confirmation() {
5677 let (root_dir, root) = test_root("deleted-root-observation-reset");
5678 let ctx = test_ctx();
5679 ctx.mark_subc_unbound();
5680 let executor = Arc::new(Executor::new());
5681 assert!(executor.register_actor(root.clone(), ctx));
5682 root_dir.close().expect("delete project root");
5683
5684 let mut meta = RootMeta::new(Instant::now());
5685 meta.unbound_quiesced = true;
5686 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5687 let pending_binds = HashMap::new();
5688 let root_channels = HashMap::new();
5689 let metrics = DispatchPathMetrics::new();
5690
5691 let first = reap_idle_roots(
5692 Instant::now(),
5693 &mut live_roots,
5694 &pending_binds,
5695 &root_channels,
5696 &executor,
5697 &metrics,
5698 );
5699 assert!(first.forgotten_deleted_roots.is_empty());
5700
5701 std::fs::create_dir_all(root.as_path()).expect("restore project root");
5702 reap_idle_roots(
5703 Instant::now(),
5704 &mut live_roots,
5705 &pending_binds,
5706 &root_channels,
5707 &executor,
5708 &metrics,
5709 );
5710 std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
5711
5712 let after_reset = reap_idle_roots(
5713 Instant::now(),
5714 &mut live_roots,
5715 &pending_binds,
5716 &root_channels,
5717 &executor,
5718 &metrics,
5719 );
5720 assert!(after_reset.forgotten_deleted_roots.is_empty());
5721 assert!(live_roots.contains_key(&root));
5722 assert!(executor.actor_registered(&root));
5723 }
5724
5725 #[test]
5726 fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
5727 let (root_dir, root) = test_root("deleted-root-reap");
5728 let app = App::default_shared();
5729 let ctx = Arc::new(AppContext::from_app(
5730 Arc::clone(&app),
5731 Config {
5732 project_root: Some(root.as_path().to_path_buf()),
5733 ..Config::default()
5734 },
5735 ));
5736 ctx.set_canonical_cache_root(root.as_path().to_path_buf());
5737 ctx.mark_subc_unbound();
5738 let executor = Arc::new(Executor::new());
5739 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5740 assert_eq!(app.actor_root_count(), 1);
5741 drop(ctx);
5742 root_dir.close().expect("delete project root");
5743
5744 let mut meta = RootMeta::new(Instant::now());
5745 meta.unbound_quiesced = true;
5746 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5747 let outcome = reap_until_forgotten(
5748 &root,
5749 &mut live_roots,
5750 &HashMap::new(),
5751 &HashMap::new(),
5752 &executor,
5753 &DispatchPathMetrics::new(),
5754 );
5755 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5756 assert!(!executor.actor_registered(&root));
5757 assert!(!live_roots.contains_key(&root));
5758 wait_for_actor_root_count(&app, 0);
5759
5760 let status_ctx = AppContext::from_app(app, Config::default());
5761 let status = status_ctx.build_status_snapshot();
5762 assert_eq!(status["runtime"]["live_actor_roots"], 0);
5763 assert_eq!(status["runtime"]["open_routes"], 0);
5764 }
5765
5766 #[test]
5767 fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
5768 let (root_dir, root) = test_root("deleted-root-reap-census");
5769 let executor = Arc::new(Executor::new());
5770 assert!(executor.register_actor(root.clone(), test_ctx()));
5771 root_dir.close().expect("delete project root");
5772
5773 let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
5774 let metrics = DispatchPathMetrics::new();
5775 let outcome = reap_idle_roots(
5776 Instant::now(),
5777 &mut live_roots,
5778 &HashMap::new(),
5779 &HashMap::new(),
5780 &executor,
5781 &metrics,
5782 );
5783 assert_eq!(outcome.evicted, 0);
5784
5785 let report = build_health_report(
5786 &executor,
5787 &HashMap::new(),
5788 &metrics,
5789 &crate::context::App::default_shared(),
5790 );
5791 let reap = report
5792 .metrics
5793 .as_ref()
5794 .and_then(|metrics| metrics.get("reap"))
5795 .expect("reap health metrics");
5796 assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
5797 assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
5798 assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
5799 assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
5800 }
5801
5802 #[test]
5803 fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
5804 let (root_dir, root) = test_root("connection-exit-deleted-root");
5805 let executor = Arc::new(Executor::new());
5806 assert!(executor.register_actor(root.clone(), test_ctx()));
5807
5808 let route = route_key(11, 1);
5809 let mut meta = RootMeta::new(Instant::now());
5810 meta.maintenance_pending = true;
5811 meta.maintenance_queued_kinds
5812 .push_back(MaintenanceDrainKind::CompletionDrains);
5813 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5814 let mut pending_binds = HashMap::new();
5815 let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
5816 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5817 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5818 let mut route_bash_cancels = HashMap::new();
5819
5820 quiesce_connection_roots(
5821 &mut live_roots,
5822 &mut pending_binds,
5823 &mut routes,
5824 &mut root_channels,
5825 &mut installed_route_epochs,
5826 &mut route_bash_cancels,
5827 &executor,
5828 );
5829 assert!(live_roots[&root].unbound_quiesced);
5830 assert!(!live_roots[&root].maintenance_pending);
5831 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
5832 assert!(routes.is_empty());
5833 assert!(root_channels.is_empty());
5834
5835 root_dir.close().expect("delete project root");
5836 let metrics = DispatchPathMetrics::new();
5837 let outcome = reap_until_forgotten(
5838 &root,
5839 &mut live_roots,
5840 &pending_binds,
5841 &root_channels,
5842 &executor,
5843 &metrics,
5844 );
5845 let mut session_identity = HashMap::new();
5846 let mut push_buffer = HashMap::new();
5847 let mut bg_subs = HashMap::new();
5848 let mut bg_sub_by_session = HashMap::new();
5849 let mut bg_wake_pending = HashSet::new();
5850 let mut bg_wake_epoch = HashMap::new();
5851 let mut pending_bash_asks = HashMap::new();
5852 let mut retry_buffer = HashMap::new();
5853 let mut reclaimed_routes = ReclaimedRoutes::default();
5854 for forgotten in &outcome.forgotten_deleted_roots {
5855 purge_deleted_root_residents(
5856 forgotten,
5857 &mut routes,
5858 &mut root_channels,
5859 &mut installed_route_epochs,
5860 &mut route_bash_cancels,
5861 &mut retry_buffer,
5862 &mut reclaimed_routes,
5863 &mut session_identity,
5864 &mut push_buffer,
5865 &mut bg_subs,
5866 &mut bg_sub_by_session,
5867 &mut bg_wake_pending,
5868 &mut bg_wake_epoch,
5869 &mut pending_bash_asks,
5870 );
5871 }
5872
5873 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5874 assert!(!executor.actor_registered(&root));
5875 assert!(!live_roots.contains_key(&root));
5876 }
5877
5878 #[test]
5879 fn unbound_root_quiesces_maintenance_without_removing_actor() {
5880 let (_root_dir, root) = test_root("unbound-root-quiesce");
5881 let ctx = test_ctx();
5882 let executor = Arc::new(Executor::new());
5883 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5884 let mut meta = RootMeta::new(Instant::now());
5885 meta.maintenance_pending = true;
5886 meta.maintenance_jobs_in_flight = 1;
5887 meta.maintenance_queued_kinds
5888 .push_back(MaintenanceDrainKind::ConfigureTail);
5889 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5890 *ctx.search_index()
5892 .write()
5893 .unwrap_or_else(std::sync::PoisonError::into_inner) =
5894 Some(crate::search_index::SearchIndex::new());
5895 ctx.set_cache_writer_capabilities(true, true);
5896 let pending = root.as_path().join("pending.rs");
5897 ctx.add_pending_search_index_paths([pending.clone()]);
5898 let canonical_root = root.as_path().to_path_buf();
5902 let artifact = canonical_root.join("cache.bin");
5903 std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
5904 let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
5905 crate::cache_freshness::record_verify_completed(
5906 &canonical_root,
5907 crate::cache_freshness::VerifyArtifact::Search,
5908 seeded_generation,
5909 );
5910 assert!(matches!(
5911 crate::cache_freshness::warm_verify_plan(
5912 &canonical_root,
5913 crate::cache_freshness::VerifyArtifact::Search,
5914 seeded_generation,
5915 ),
5916 crate::cache_freshness::WarmVerifyPlan::Skip
5917 ));
5918 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5921 let _dispatch_tx = dispatch_tx;
5922 let shutdown = Arc::new(AtomicBool::new(false));
5923 let thread_shutdown = Arc::clone(&shutdown);
5924 let join = std::thread::spawn(move || {
5925 while !thread_shutdown.load(Ordering::SeqCst) {
5926 std::thread::yield_now();
5927 }
5928 });
5929 ctx.install_watcher_runtime(
5930 dispatch_rx,
5931 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5932 );
5933 assert!(ctx.watcher_runtime_active());
5934
5935 quiesce_unbound_root(&root, &mut live_roots, &executor);
5936 let meta = &live_roots[&root];
5937 assert!(meta.unbound_quiesced);
5938 assert!(ctx.subc_unbound_quiesced());
5939 assert!(meta.maintenance_pending);
5940 assert!(meta.maintenance_queued_kinds.is_empty());
5941 assert!(executor.actor_registered(&root));
5942 assert!(
5946 ctx.search_index()
5947 .read()
5948 .unwrap_or_else(std::sync::PoisonError::into_inner)
5949 .is_some(),
5950 "quiesce must not evict resident artifacts"
5951 );
5952 assert_eq!(
5953 ctx.pending_callgraph_store_force_token(),
5954 None,
5955 "quiesce must not force a callgraph rebuild"
5956 );
5957 assert_eq!(
5958 ctx.take_pending_search_index_paths(),
5959 vec![pending],
5960 "quiesce must retain pending watcher-derived paths"
5961 );
5962 assert!(
5963 matches!(
5964 crate::cache_freshness::warm_verify_plan(
5965 &canonical_root,
5966 crate::cache_freshness::VerifyArtifact::Search,
5967 seeded_generation,
5968 ),
5969 crate::cache_freshness::WarmVerifyPlan::Skip
5970 ),
5971 "quiesce must not invalidate the warm verify memo"
5972 );
5973 assert!(
5974 ctx.watcher_runtime_active(),
5975 "quiesce must not stop a running watcher"
5976 );
5977 ctx.stop_watcher_runtime();
5978
5979 let meta = live_roots.get_mut(&root).expect("root metadata");
5980 note_maintenance_completion(
5981 meta,
5982 Some(MaintenanceDrainKind::ConfigureTail),
5983 false,
5984 meta.unbound_quiesced,
5985 );
5986 assert!(!meta.maintenance_pending);
5987 assert!(meta.maintenance_queued_kinds.is_empty());
5988 }
5989
5990 #[test]
5991 fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
5992 let (_dir, root) = test_root("same-root-replacement");
5993 let route = route_key(7, 1);
5994 let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5995 let root_channels = HashMap::new();
5996
5997 assert!(!route_removal_will_quiesce_root(
5998 &root,
5999 route,
6000 &installed_channels,
6001 false,
6002 Some(&root),
6003 ));
6004 assert!(route_removal_will_quiesce_root(
6005 &root,
6006 route,
6007 &installed_channels,
6008 false,
6009 None,
6010 ));
6011 assert!(!should_quiesce_removed_root(
6012 &root,
6013 &root_channels,
6014 false,
6015 Some(&root),
6016 ));
6017 assert!(should_quiesce_removed_root(
6018 &root,
6019 &root_channels,
6020 false,
6021 None,
6022 ));
6023 assert!(!should_quiesce_removed_root(
6024 &root,
6025 &root_channels,
6026 true,
6027 None,
6028 ));
6029 }
6030
6031 #[test]
6032 fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
6033 let (_root_dir, root) = test_root("unbound-root-route-count");
6034 let executor = Arc::new(Executor::new());
6035 assert!(executor.register_actor(root.clone(), test_ctx()));
6036 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6037 let mut root_channels = HashMap::from([(
6038 root.clone(),
6039 HashSet::from([route_key(7, 1), route_key(8, 1)]),
6040 )]);
6041
6042 remove_root_channel(&mut root_channels, &root, route_key(7, 1));
6043 if !root_channels.contains_key(&root) {
6044 quiesce_unbound_root(&root, &mut live_roots, &executor);
6045 }
6046 assert!(!live_roots[&root].unbound_quiesced);
6047
6048 remove_root_channel(&mut root_channels, &root, route_key(8, 1));
6049 if !root_channels.contains_key(&root) {
6050 quiesce_unbound_root(&root, &mut live_roots, &executor);
6051 }
6052 assert!(live_roots[&root].unbound_quiesced);
6053
6054 live_roots
6055 .get_mut(&root)
6056 .expect("root metadata")
6057 .note_activity();
6058 assert!(
6059 live_roots[&root].unbound_quiesced,
6060 "late asynchronous activity must not reactivate an unbound root"
6061 );
6062
6063 live_roots
6064 .get_mut(&root)
6065 .expect("root metadata")
6066 .reactivate_bound();
6067 assert!(!live_roots[&root].unbound_quiesced);
6068 }
6069
6070 #[test]
6071 fn allocator_pressure_relief_requires_every_root_to_be_idle() {
6072 let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
6073 let (_active_dir, active_root) = test_root("allocator-relief-active");
6074 let now = Instant::now();
6075 let mut live_roots = HashMap::new();
6076 let mut idle = RootMeta::new(now);
6077 idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
6078 live_roots.insert(idle_root, idle);
6079 assert!(process_has_been_idle(now, &live_roots));
6080
6081 live_roots.insert(active_root.clone(), RootMeta::new(now));
6082 assert!(!process_has_been_idle(now, &live_roots));
6083
6084 let active = live_roots
6085 .get_mut(&active_root)
6086 .expect("active root metadata");
6087 active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
6088 active.active_bash_waits = 1;
6089 assert!(!process_has_been_idle(now, &live_roots));
6090 }
6091
6092 #[test]
6093 fn pressure_relief_log_reports_before_and_after_measurements() {
6094 let allocator = crate::memory::AllocatorMemorySnapshot {
6095 status: "measured",
6096 bytes_in_use: Some(8 * 1024 * 1024),
6097 size_allocated: Some(12 * 1024 * 1024),
6098 retained_slack_bytes: Some(4 * 1024 * 1024),
6099 not_estimated: None,
6100 };
6101 let relief = crate::memory::AllocatorPressureRelief {
6102 bytes_released: 3 * 1024 * 1024,
6103 rss_before_bytes: Some(20 * 1024 * 1024),
6104 rss_after_bytes: Some(17 * 1024 * 1024),
6105 allocator_before: allocator.clone(),
6106 allocator_after: crate::memory::AllocatorMemorySnapshot {
6107 size_allocated: Some(9 * 1024 * 1024),
6108 retained_slack_bytes: Some(1024 * 1024),
6109 ..allocator
6110 },
6111 };
6112 let message = pressure_relief_label(&relief);
6113 assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
6114 assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
6115 assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
6116 assert!(message.contains("reported 3.0 MB released"));
6117 }
6118
6119 #[test]
6120 fn due_maintenance_jobs_skip_poisoned_roots() {
6121 let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
6122 let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
6123 let mut live_roots = HashMap::new();
6124 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6125 let mut poisoned_meta = RootMeta::new(Instant::now());
6126 poisoned_meta.maintenance_poisoned = true;
6127 live_roots.insert(poisoned_root.clone(), poisoned_meta);
6128
6129 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6130 &mut live_roots,
6131 MAINTENANCE_SUBMIT_BUDGET,
6132 &HashSet::new(),
6133 );
6134
6135 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6136 assert!(due.iter().all(|(root, _)| root == &healthy_root));
6137 assert!(!deferred);
6138 assert!(live_roots[&healthy_root].maintenance_pending);
6139 assert_eq!(
6140 live_roots[&healthy_root].maintenance_jobs_in_flight,
6141 INITIAL_MAINTENANCE_JOB_COUNT
6142 );
6143 assert!(!live_roots[&poisoned_root].maintenance_pending);
6144 }
6145
6146 #[test]
6147 fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
6148 let (_dir, root) = test_root("maintenance-unbound");
6149 let mut meta = RootMeta::new(Instant::now());
6150 meta.unbound_quiesced = true;
6151 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6152
6153 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6154 &mut live_roots,
6155 MAINTENANCE_SUBMIT_BUDGET,
6156 &HashSet::new(),
6157 );
6158
6159 assert!(due.is_empty());
6160 assert!(!deferred);
6161 assert!(!live_roots[&root].maintenance_pending);
6162 }
6163
6164 #[test]
6165 fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
6166 let (_dir, root) = test_root("maintenance-idle-bg-subscription");
6167 let ctx = test_ctx();
6168 assert!(!ctx.completion_drains_have_work());
6169
6170 let executor = Executor::new();
6171 assert!(executor.register_actor(root.clone(), ctx));
6172 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6173 let session = "idle-session".to_string();
6174 let channel = route_key(17, 1);
6175 let bg_sub_by_session = HashMap::from([((root.clone(), session.clone()), channel)]);
6176 let mut bg_wake_pending = HashSet::new();
6177
6178 let (idle_tick_jobs, deferred) = due_maintenance_jobs(
6179 &mut live_roots,
6180 Some(&executor),
6181 &bg_sub_by_session,
6182 &bg_wake_pending,
6183 MAINTENANCE_SUBMIT_BUDGET,
6184 &HashSet::new(),
6185 );
6186 assert!(idle_tick_jobs.is_empty());
6187 assert!(!deferred);
6188 assert!(!live_roots[&root].maintenance_pending);
6189
6190 let mut bg_wake_epoch = HashMap::new();
6193 push::arm_bg_wake(
6194 root.clone(),
6195 session,
6196 channel,
6197 &mut bg_wake_pending,
6198 &mut bg_wake_epoch,
6199 );
6200 let (next_tick_jobs, deferred) = due_maintenance_jobs(
6201 &mut live_roots,
6202 Some(&executor),
6203 &bg_sub_by_session,
6204 &bg_wake_pending,
6205 MAINTENANCE_SUBMIT_BUDGET,
6206 &HashSet::new(),
6207 );
6208 assert_eq!(
6209 next_tick_jobs,
6210 vec![(root, MaintenanceDrainKind::CompletionDrains)]
6211 );
6212 assert!(!deferred);
6213 }
6214
6215 #[tokio::test]
6216 async fn subc_configure_tail_precedes_completed_search_install() {
6217 let root_dir = tempfile::tempdir().unwrap();
6218 let storage = tempfile::tempdir().unwrap();
6219 let root = ProjectRootId::from_path(root_dir.path()).unwrap();
6220 let (ctx, ignored_path) =
6221 runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
6222 let ctx = Arc::new(ctx);
6223 assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
6224 &ctx,
6225 &ignored_path
6226 ));
6227
6228 let executor = Arc::new(Executor::new());
6229 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6230 let metrics = Arc::new(DispatchPathMetrics::new());
6231 let (completion_tx, mut completion_rx) = mpsc::channel(4);
6232 submit_maintenance_job(
6233 &executor,
6234 root.clone(),
6235 MaintenanceDrainKind::ConfigureTail,
6236 Vec::new(),
6237 &completion_tx,
6238 &metrics,
6239 );
6240 submit_maintenance_job(
6241 &executor,
6242 root,
6243 MaintenanceDrainKind::CompletionDrains,
6244 Vec::new(),
6245 &completion_tx,
6246 &metrics,
6247 );
6248
6249 let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6250 .await
6251 .expect("configure-tail completion timed out")
6252 .expect("configure-tail completion channel closed");
6253 let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6254 .await
6255 .expect("completion-drains completion timed out")
6256 .expect("completion-drains completion channel closed");
6257 assert!(first.response.id.contains("configure-tail"));
6258 assert!(second.response.id.contains("completion-drains"));
6259 assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
6260 &ctx,
6261 &ignored_path
6262 ));
6263 assert_eq!(
6264 ctx.search_index()
6265 .read()
6266 .unwrap_or_else(std::sync::PoisonError::into_inner)
6267 .as_ref()
6268 .expect("completed search index installed")
6269 .file_count(),
6270 0,
6271 "configure must install the ignore matcher before pending paths replay"
6272 );
6273 ctx.stop_watcher_runtime();
6274 }
6275
6276 #[test]
6277 fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
6278 let (_dir, root) = test_root("maintenance-post-bind");
6279 let mut live_roots = HashMap::new();
6280 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6281
6282 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6283 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6284
6285 let meta = live_roots.get(&root).expect("root metadata");
6286 assert!(meta.maintenance_pending);
6287 assert_eq!(meta.maintenance_jobs_in_flight, 0);
6288 assert_eq!(
6289 meta.maintenance_queued_kinds
6290 .iter()
6291 .copied()
6292 .collect::<Vec<_>>(),
6293 vec![
6294 MaintenanceDrainKind::ConfigureTail,
6295 MaintenanceDrainKind::CompletionDrains,
6296 ]
6297 );
6298
6299 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6300 &mut live_roots,
6301 MAINTENANCE_SUBMIT_BUDGET,
6302 &HashSet::new(),
6303 );
6304
6305 assert_eq!(
6306 due,
6307 vec![
6308 (root.clone(), MaintenanceDrainKind::ConfigureTail),
6309 (root.clone(), MaintenanceDrainKind::CompletionDrains),
6310 ]
6311 );
6312 assert!(!deferred);
6313 assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
6314 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
6315 }
6316
6317 #[test]
6318 fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
6319 let mut live_roots = HashMap::new();
6320 let mut root_ids = Vec::new();
6321 let mut _dirs = Vec::new();
6322 for index in 0..4 {
6323 let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
6324 live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
6325 root_ids.push(root_id);
6326 _dirs.push(dir);
6327 }
6328
6329 let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
6330 let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
6331 &mut live_roots,
6332 small_budget,
6333 &HashSet::new(),
6334 );
6335
6336 assert_eq!(first_due.len(), small_budget);
6337 assert!(first_deferred);
6338 let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
6339 assert!(first_due_set
6340 .iter()
6341 .all(|root| live_roots[root].maintenance_pending));
6342 assert!(first_due_set
6343 .iter()
6344 .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
6345
6346 let all_roots: HashSet<_> = root_ids.into_iter().collect();
6347 let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
6348 assert!(deferred_roots
6349 .iter()
6350 .all(|root| !live_roots[root].maintenance_pending));
6351 }
6352
6353 #[test]
6354 fn due_maintenance_jobs_defers_pending_bind_roots() {
6355 let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
6356 let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
6357 let mut live_roots = HashMap::new();
6358 live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
6359 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6360 let pending_bind_roots = HashSet::from([bind_root.clone()]);
6361
6362 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6363 &mut live_roots,
6364 usize::MAX,
6365 &pending_bind_roots,
6366 );
6367
6368 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6369 assert!(due.iter().all(|(root, _)| root == &healthy_root));
6370 assert!(!deferred);
6371 assert!(!live_roots[&bind_root].maintenance_pending);
6372 assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
6373 }
6374
6375 #[test]
6376 fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
6377 let (_dir, root) = test_root("maintenance-requeue");
6378 let mut live_roots = HashMap::new();
6379 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6380 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6381 &mut live_roots,
6382 usize::MAX,
6383 &HashSet::new(),
6384 );
6385 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6386 assert!(due.iter().all(|(due_root, _)| due_root == &root));
6387 assert!(!deferred);
6388
6389 let meta = live_roots.get_mut(&root).unwrap();
6390 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
6391 assert!(meta.maintenance_pending);
6392 assert_eq!(
6393 meta.maintenance_jobs_in_flight,
6394 INITIAL_MAINTENANCE_JOB_COUNT - 1
6395 );
6396 assert_eq!(meta.maintenance_queued_kinds.len(), 1);
6397
6398 let (requeued, deferred) =
6399 due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
6400 assert_eq!(
6401 requeued,
6402 vec![(root.clone(), MaintenanceDrainKind::Watcher)]
6403 );
6404 assert!(!deferred);
6405 let meta = live_roots.get_mut(&root).unwrap();
6406 assert_eq!(
6407 meta.maintenance_jobs_in_flight,
6408 INITIAL_MAINTENANCE_JOB_COUNT
6409 );
6410 assert!(meta.maintenance_queued_kinds.is_empty());
6411
6412 for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
6413 note_maintenance_completion(meta, None, false, false);
6414 }
6415 assert!(!meta.maintenance_pending);
6416 assert_eq!(meta.maintenance_jobs_in_flight, 0);
6417 }
6418
6419 #[test]
6420 fn maintenance_requeue_drops_while_bind_is_pending() {
6421 let (_dir, root) = test_root("maintenance-bind-requeue");
6422 let mut live_roots = HashMap::new();
6423 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6424 let (due, _) = due_maintenance_jobs_without_actor_context(
6425 &mut live_roots,
6426 usize::MAX,
6427 &HashSet::new(),
6428 );
6429 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6430
6431 let meta = live_roots.get_mut(&root).unwrap();
6432 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
6433
6434 assert_eq!(
6435 meta.maintenance_jobs_in_flight,
6436 INITIAL_MAINTENANCE_JOB_COUNT - 1
6437 );
6438 assert!(meta.maintenance_queued_kinds.is_empty());
6439 assert!(meta.maintenance_pending);
6440 }
6441
6442 #[test]
6443 fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
6444 let mut meta = RootMeta::new(Instant::now());
6445 meta.unbound_quiesced = true;
6446
6447 assert!(!should_requiesce_after_maintenance(
6448 &meta,
6449 MaintenanceDrainKind::Lsp,
6450 false,
6451 ));
6452 assert!(!should_requiesce_after_maintenance(
6453 &meta,
6454 MaintenanceDrainKind::ConfigureTail,
6455 true,
6456 ));
6457 assert!(should_requiesce_after_maintenance(
6458 &meta,
6459 MaintenanceDrainKind::ConfigureTail,
6460 false,
6461 ));
6462 }
6463
6464 #[test]
6465 fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
6466 let (_dir, root) = test_root("maintenance-fatal");
6467 let mut live_roots = HashMap::new();
6468 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6469 let (due, _) = due_maintenance_jobs_without_actor_context(
6470 &mut live_roots,
6471 usize::MAX,
6472 &HashSet::new(),
6473 );
6474 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6475
6476 let meta = live_roots.get_mut(&root).unwrap();
6477 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
6478 assert!(meta.maintenance_poisoned);
6479 assert!(meta.maintenance_queued_kinds.is_empty());
6480
6481 for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
6482 note_maintenance_completion(meta, None, false, false);
6483 }
6484 assert!(!meta.maintenance_pending);
6485 assert_eq!(meta.maintenance_jobs_in_flight, 0);
6486 }
6487
6488 #[test]
6489 fn trust_for_principal_matrix() {
6490 assert_eq!(
6491 trust_for_principal(&Some(Principal::Direct)),
6492 BindTrust::FirstParty
6493 );
6494 for module_id in [
6501 "llm-runner",
6502 "aft",
6503 "broca",
6504 "alfonso-core",
6505 "prefrontal",
6506 "prefrontal-core",
6507 ] {
6508 assert_eq!(
6509 trust_for_principal(&Some(Principal::Reserved {
6510 module_id: module_id.to_string(),
6511 })),
6512 BindTrust::FirstParty,
6513 "reserved module id '{module_id}' must resolve to first-party trust"
6514 );
6515 }
6516 assert_eq!(
6517 trust_for_principal(&Some(Principal::Reserved {
6518 module_id: "subc-mcp".to_string(),
6519 })),
6520 BindTrust::Untrusted
6521 );
6522 assert_eq!(
6523 trust_for_principal(&Some(Principal::Reserved {
6524 module_id: "anything-unknown".to_string(),
6525 })),
6526 BindTrust::Untrusted
6527 );
6528 assert_eq!(
6529 trust_for_principal(&Some(Principal::Unverified)),
6530 BindTrust::Untrusted
6531 );
6532 assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
6533 }
6534
6535 #[test]
6536 fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
6537 let principal = Some(Principal::Direct);
6538 let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
6539 let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
6540
6541 assert_eq!(
6542 trust_for_bind(fingerprint_a, &principal),
6543 BindTrust::Untrusted
6544 );
6545 assert_eq!(
6546 trust_for_bind(fingerprint_b, &principal),
6547 BindTrust::Untrusted
6548 );
6549 }
6550
6551 #[test]
6558 fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
6559 for harness in ["opencode", "pi", "runner", "mcp:claude"] {
6560 assert_eq!(
6561 trust_for_bind(harness, &Some(Principal::Direct)),
6562 BindTrust::FirstParty,
6563 "a direct principal must stay first-party on {harness}"
6564 );
6565 assert_eq!(
6566 trust_for_bind(harness, &Some(Principal::Unverified)),
6567 BindTrust::Untrusted,
6568 "an unverified principal must stay untrusted on {harness}"
6569 );
6570 assert_eq!(
6571 trust_for_bind(harness, &None),
6572 BindTrust::Untrusted,
6573 "an absent principal must fail closed on {harness}"
6574 );
6575 assert_eq!(
6576 trust_for_bind(
6577 harness,
6578 &Some(Principal::Reserved {
6579 module_id: "subc-mcp".to_string(),
6580 })
6581 ),
6582 BindTrust::Untrusted,
6583 "a non-allowlisted reserved module must stay untrusted on {harness}"
6584 );
6585 }
6586 }
6587
6588 #[tokio::test]
6589 async fn persistent_cancel_resolves_when_fired_before_await() {
6590 let signal = PersistentCancelSignal::new();
6594 signal.cancel();
6595 tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
6597 .await
6598 .expect("cancelled() must resolve when cancel fired beforehand");
6599
6600 let racing = PersistentCancelSignal::new();
6602 let racing_for_task = racing.clone();
6603 let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
6604 racing.cancel();
6605 tokio::time::timeout(Duration::from_secs(1), waiter)
6606 .await
6607 .expect("cancelled() must resolve when cancel races the await")
6608 .expect("waiter task panicked");
6609 }
6610
6611 #[test]
6612 fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
6613 let installed = HashMap::from([(7, 9)]);
6614 let mut reclaimed = ReclaimedRoutes::default();
6615 reclaimed.insert(route_key(8, 1));
6616 for ty in [
6617 FrameType::Request,
6618 FrameType::Response,
6619 FrameType::Error,
6620 FrameType::Push,
6621 FrameType::Cancel,
6622 FrameType::Goodbye,
6623 ] {
6624 let body = if ty.is_pure_header() {
6625 Vec::new()
6626 } else {
6627 br#"{}"#.to_vec()
6628 };
6629 let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
6630 assert!(
6631 !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
6632 "{ty:?}"
6633 );
6634 }
6635
6636 let reclaimed_request = Frame::build(
6637 FrameType::Request,
6638 control_flags(),
6639 8,
6640 1,
6641 42,
6642 br#"{}"#.to_vec(),
6643 )
6644 .unwrap();
6645 assert!(ingress_route_should_be_processed(
6646 &installed,
6647 &reclaimed,
6648 &reclaimed_request
6649 ));
6650
6651 let never_installed = Frame::build(
6652 FrameType::Request,
6653 control_flags(),
6654 9,
6655 1,
6656 43,
6657 br#"{}"#.to_vec(),
6658 )
6659 .unwrap();
6660 assert!(!ingress_route_should_be_processed(
6661 &installed,
6662 &reclaimed,
6663 &never_installed
6664 ));
6665
6666 let current = Frame::build(
6667 FrameType::Request,
6668 control_flags(),
6669 7,
6670 9,
6671 43,
6672 br#"{}"#.to_vec(),
6673 )
6674 .unwrap();
6675 let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
6676 assert!(ingress_route_should_be_processed(
6677 &installed, &reclaimed, ¤t
6678 ));
6679 assert!(ingress_route_should_be_processed(
6680 &installed, &reclaimed, &control
6681 ));
6682 assert_eq!(installed, HashMap::from([(7, 9)]));
6683 }
6684
6685 #[tokio::test]
6686 async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
6687 let (_dir, root) = test_root("route-bind-b2-ordering");
6688 let route = route_key(7, 3);
6689 let identity = RouteIdentity(Arc::new(RouteIdentityData {
6690 root: root.clone(),
6691 project_root: root.as_path().to_path_buf(),
6692 harness: "opencode".to_string(),
6693 session: "b2-session".to_string(),
6694 trust: BindTrust::FirstParty,
6695 spawn_principal: AuthenticatedPrincipal::FirstParty,
6696 consumer_elicitation_capable: false,
6697 }));
6698 let replay_key = push::ReplayKey::from_identity(&identity);
6699 let completion = RouteBindCompletion {
6700 route,
6701 identity,
6702 bind_root_id: root.clone(),
6703 inserted_new_actor: false,
6704 configure_response: Response::success("subc-bind-7", json!({})),
6705 diagnostics_on_edit: false,
6706 ver: PROTOCOL_VERSION,
6707 corr: 91,
6708 flags: control_flags(),
6709 };
6710 let mut pending_binds = HashMap::from([(
6711 route,
6712 PendingBind {
6713 bind_root_id: root,
6714 inserted_new_actor: false,
6715 cancelled: false,
6716 configure_request_id: "subc-bind-7".to_string(),
6717 started_at: Instant::now(),
6718 warned_half_deadline: false,
6719 deadline_reported: false,
6720 corr: 91,
6721 ver: PROTOCOL_VERSION,
6722 flags: control_flags(),
6723 cancellation: crate::executor::JobCancellation::new(),
6724 },
6725 )]);
6726 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6727 let mut push_buffer =
6728 HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
6729 let (writer_tx, mut writer_rx) = mpsc::channel(8);
6730 let metrics = Arc::new(DispatchPathMetrics::new());
6731
6732 handle_route_bind_completion(
6733 &writer_tx,
6734 completion,
6735 &mut HashMap::new(),
6736 &mut HashMap::new(),
6737 &mut HashMap::new(),
6738 &mut push_buffer,
6739 &mut HashMap::new(),
6740 &mut pending_binds,
6741 &mut installed_route_epochs,
6742 &Arc::new(Executor::new()),
6743 &Arc::new(Notify::new()),
6744 &metrics,
6745 )
6746 .await
6747 .unwrap();
6748
6749 let ack = writer_rx.try_recv().expect("RouteBindAck");
6750 assert_eq!(ack.header.ty, FrameType::Response);
6751 assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
6752 let route_frame = writer_rx.try_recv().expect("post-ack route frame");
6753 assert_eq!(route_frame.header.ty, FrameType::Push);
6754 assert_eq!(
6755 (route_frame.header.channel, route_frame.header.epoch),
6756 (route.channel, route.epoch)
6757 );
6758 }
6759}