1use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock, Mutex as StdMutex, OnceLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, Lane};
31use crate::fleet_status::{spawn_fleet_status_dial, FleetStatusClient};
32use crate::jsonc::strip_jsonc;
33use crate::log_ctx;
34use crate::path_identity::ProjectRootId;
35use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
36use crate::run_tool_call::{
37 run_tool_call, strip_agent_preview_arg_owned, PhaseTrace, ToolCallContext, ToolCallOutcome,
38 ToolCallResult,
39};
40use crate::runtime_drain;
41use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
42
43use subc_protocol::manifest::{
44 Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
45 ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
46};
47use subc_protocol::session::{
48 HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
49 MODULE_CONTROL_OP_HEALTH_CHECK,
50};
51use subc_protocol::{
52 ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, MAX_FRAME_BODY_LEN,
53 PROTOCOL_VERSION,
54};
55use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
56use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
57use tokio::net::TcpStream;
58use tokio::sync::{mpsc, oneshot, Notify};
59use tokio::task::JoinHandle;
60
61const AUTH_DEADLINE: Duration = Duration::from_secs(5);
64const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
65const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
66const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
67const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
68
69const HELLO_CORR: u64 = 1;
71
72const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
75
76const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
80
81const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
85
86const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
90
91const WRITER_QUEUE_CAPACITY: usize = 256;
92
93const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
96
97const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
105const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
106 MaintenanceDrainKind::Watcher,
107 MaintenanceDrainKind::Lsp,
108 MaintenanceDrainKind::ConfigureTail,
109 MaintenanceDrainKind::CompletionDrains,
110];
111#[cfg(test)]
112const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
113
114const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
115const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
116
117const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
118const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
119
120const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
123
124const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
128
129const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
131const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
132
133#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
134struct RouteChannel {
135 channel: u16,
136 epoch: u32,
137}
138
139impl fmt::Display for RouteChannel {
140 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
141 write!(f, "{}@{}", self.channel, self.epoch)
142 }
143}
144
145type PushEnvelope = (ProjectRootId, PushFrame);
146type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
147type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
148mod bash;
149mod health;
150mod manifest;
151mod push;
152mod wire;
153
154use self::health::{
155 build_health_report, warn_slow_pending_binds, DispatchPathMetrics, HealthRollupCache,
156 ReapBlockerCensus, ResponseTaskGuard, HEALTH_ROLLUP_TTL,
157};
158use self::manifest::{
159 build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
160 is_subc_agent_core_tool, is_subc_native_plumbing_tool,
161};
162pub use self::wire::SubcError;
163
164pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
168 manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
169}
170use self::wire::{
171 build_error_frame, build_goodbye_frame, build_tool_response_frame,
172 build_tool_response_frame_with_limit, decrement_counted_channel, response_is_fatal_panic,
173 response_message, send_counted_channel, send_frame, send_reliable_writer_frame,
174 send_traced_tool_response_frame, ToolResponseWriteTrace, WriterFrame, WriterSender,
175};
176
177struct DecodedFrame {
178 frame: Frame,
179 phase_trace: PhaseTrace,
180}
181
182struct ToolCallCompletion {
183 text: String,
184 phase_trace: PhaseTrace,
185}
186
187#[derive(Clone)]
188struct PushSenders {
189 lossy_tx: mpsc::Sender<LossyPushEnvelope>,
190 reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
191 lossy_overflow: Arc<push::LossyOverflow>,
192 lossy_seq: Arc<AtomicU64>,
193 fleet_status_client: FleetStatusClient,
194}
195
196#[derive(Clone)]
197struct PersistentCancelSignal {
198 inner: Arc<PersistentCancelInner>,
199}
200
201struct PersistentCancelInner {
202 cancelled: AtomicBool,
203 notify: Notify,
204}
205
206impl PersistentCancelSignal {
207 fn new() -> Self {
208 Self {
209 inner: Arc::new(PersistentCancelInner {
210 cancelled: AtomicBool::new(false),
211 notify: Notify::new(),
212 }),
213 }
214 }
215
216 fn cancel(&self) {
217 if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
218 self.inner.notify.notify_waiters();
219 }
220 }
221
222 fn is_cancelled(&self) -> bool {
223 self.inner.cancelled.load(Ordering::SeqCst)
224 }
225
226 async fn cancelled(&self) {
227 loop {
235 let notified = self.inner.notify.notified();
236 tokio::pin!(notified);
237 notified.as_mut().enable();
238 if self.is_cancelled() {
239 return;
240 }
241 notified.await;
242 }
243 }
244}
245
246#[derive(Debug, Clone, Copy, PartialEq, Eq)]
247pub(crate) enum BindTrust {
248 FirstParty,
249 Untrusted,
250}
251
252impl BindTrust {
253 fn allows_bash_observation(self) -> bool {
254 matches!(self, Self::FirstParty)
255 }
256
257 fn label(self) -> &'static str {
258 match self {
259 Self::FirstParty => "first_party",
260 Self::Untrusted => "untrusted",
261 }
262 }
263
264 fn sandbox_trust(self) -> PrincipalTrust {
265 match self {
266 Self::FirstParty => PrincipalTrust::FirstParty,
267 Self::Untrusted => PrincipalTrust::Untrusted,
268 }
269 }
270}
271
272pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
273 match principal {
274 Some(Principal::Direct) => BindTrust::FirstParty,
275 Some(Principal::Reserved { module_id })
286 if module_id == "llm-runner"
287 || module_id == "aft"
288 || module_id == "broca"
289 || module_id == "alfonso-core"
290 || module_id == "prefrontal"
291 || module_id == "prefrontal-core" =>
292 {
293 BindTrust::FirstParty
294 }
295 Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
296 BindTrust::Untrusted
297 }
298 }
299}
300
301fn harness_forces_untrusted(harness: &str) -> bool {
302 harness.starts_with("fed:")
303}
304
305pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
306 if harness_forces_untrusted(harness) {
307 BindTrust::Untrusted
308 } else {
309 trust_for_principal(principal)
310 }
311}
312
313fn principal_id(principal: &Option<Principal>) -> Option<String> {
314 match principal {
315 Some(Principal::Direct) => Some("direct".to_string()),
316 Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
317 Some(Principal::Unverified) => Some("unverified".to_string()),
318 None => None,
319 }
320}
321
322fn principal_label(principal: &Option<Principal>) -> String {
323 principal_id(principal).unwrap_or_else(|| "absent".to_string())
324}
325
326#[derive(Debug)]
327struct RootMeta {
333 maintenance_pending: bool,
334 maintenance_jobs_in_flight: usize,
335 maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
336 maintenance_last_submitted: Option<Instant>,
337 maintenance_poisoned: bool,
338 last_touched: Instant,
339 diagnostics_on_edit: bool,
340 active_bash_waits: usize,
341 idle_artifacts_evicted: bool,
342 unbound_quiesced: bool,
343 consecutive_missing_sweeps: u8,
344}
345
346#[derive(Debug)]
347struct PendingBind {
348 bind_root_id: ProjectRootId,
349 inserted_new_actor: bool,
350 cancelled: bool,
351 configure_request_id: String,
352 started_at: Instant,
353 warned_half_deadline: bool,
354 deadline_reported: bool,
355 corr: u64,
356 ver: u8,
357 flags: Flags,
358 cancellation: crate::executor::JobCancellation,
363}
364
365struct RouteBindCompletion {
366 route: RouteChannel,
367 identity: RouteIdentity,
368 bind_root_id: ProjectRootId,
369 inserted_new_actor: bool,
370 configure_response: Response,
371 diagnostics_on_edit: bool,
372 ver: u8,
373 corr: u64,
374 flags: Flags,
375}
376
377#[derive(Debug, Clone)]
378struct RouteIdentity(Arc<RouteIdentityData>);
379
380#[derive(Debug)]
381struct RouteIdentityData {
382 root: ProjectRootId,
383 project_root: PathBuf,
384 harness: String,
385 session: String,
386 trust: BindTrust,
387 spawn_principal: AuthenticatedPrincipal,
388 consumer_elicitation_capable: bool,
389}
390
391impl Deref for RouteIdentity {
392 type Target = RouteIdentityData;
393
394 fn deref(&self) -> &Self::Target {
395 &self.0
396 }
397}
398
399#[derive(Debug, Clone)]
400struct RetainedSessionIdentity {
401 harness: String,
402 trust: BindTrust,
403}
404
405#[derive(Clone)]
406struct BgSub {
407 corr: u64,
408 ver: u8,
409 flags: Flags,
410 root: ProjectRootId,
411 session: String,
412}
413
414type BgSubsBySession = HashMap<(ProjectRootId, String), HashSet<RouteChannel>>;
417
418struct MaintenanceCompletion {
419 root_id: ProjectRootId,
420 kind: MaintenanceDrainKind,
421 response: Response,
422 empty_bg_sessions: Vec<(String, u64)>,
423 requeue_kind: Option<MaintenanceDrainKind>,
424}
425
426#[derive(Clone, Copy, Debug, PartialEq, Eq)]
427enum MaintenanceDrainKind {
428 Watcher,
429 Lsp,
430 ConfigureTail,
431 CompletionDrains,
432}
433
434impl MaintenanceDrainKind {
435 fn label(self) -> &'static str {
436 match self {
437 Self::Watcher => "watcher",
438 Self::Lsp => "lsp",
439 Self::ConfigureTail => "configure-tail",
440 Self::CompletionDrains => "completion-drains",
441 }
442 }
443}
444
445#[derive(Debug, Default)]
446struct MaintenanceJobOutcome {
447 empty_bg_sessions: Vec<(String, u64)>,
448 requeue_kind: Option<MaintenanceDrainKind>,
449}
450
451#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
452struct ReverseCorrKey {
453 route: RouteChannel,
454 corr: u64,
455}
456
457struct PendingBashAsk {
458 route: RouteChannel,
459 tool_corr: u64,
460 tool_flags: Flags,
461 tool_ver: u8,
462 root: ProjectRootId,
463 project_root: PathBuf,
464 session_id: String,
465 spawn_principal: AuthenticatedPrincipal,
466 edit_slot_survives: Option<bool>,
467 request_id: String,
468 arguments: Value,
469 format_context: crate::subc_format::FormatContext,
470 cancel: bash::BashWaitCancel,
471 grants: Vec<String>,
472 expires_at: Instant,
473}
474
475impl RootMeta {
476 fn new(now: Instant) -> Self {
477 Self {
478 maintenance_pending: false,
479 maintenance_jobs_in_flight: 0,
480 maintenance_queued_kinds: VecDeque::new(),
481 maintenance_last_submitted: None,
482 maintenance_poisoned: false,
483 last_touched: now,
484 diagnostics_on_edit: false,
485 active_bash_waits: 0,
486 idle_artifacts_evicted: false,
487 unbound_quiesced: false,
488 consecutive_missing_sweeps: 0,
489 }
490 }
491
492 fn note_activity(&mut self) {
493 self.last_touched = Instant::now();
494 }
495
496 fn reactivate_bound(&mut self) {
497 self.note_activity();
498 self.idle_artifacts_evicted = false;
499 self.unbound_quiesced = false;
500 }
501}
502
503fn due_maintenance_jobs(
504 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
505 executor: Option<&Executor>,
506 bg_sub_by_session: &BgSubsBySession,
507 bg_wake_pending: &HashSet<RouteChannel>,
508 budget: usize,
509 pending_bind_roots: &HashSet<ProjectRootId>,
510) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
511 let mut jobs = Vec::new();
512 let mut deferred = false;
513 let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
514 roots.sort_by(|left, right| {
515 let left_last = live_roots
516 .get(left)
517 .and_then(|meta| meta.maintenance_last_submitted);
518 let right_last = live_roots
519 .get(right)
520 .and_then(|meta| meta.maintenance_last_submitted);
521 left_last
522 .cmp(&right_last)
523 .then_with(|| left.as_path().cmp(right.as_path()))
524 });
525
526 for root_id in roots {
527 let Some(meta) = live_roots.get_mut(&root_id) else {
528 continue;
529 };
530 if meta.maintenance_poisoned {
531 continue;
532 }
533
534 if pending_bind_roots.contains(&root_id) {
535 if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
536 deferred = true;
537 }
538 continue;
539 }
540
541 if !meta.maintenance_pending {
542 if jobs.len() >= budget {
543 deferred = true;
544 continue;
545 }
546 let executor_actor_context =
550 executor.and_then(|executor| executor.actor_context(&root_id));
551 let root_has_pending_bg_wake =
552 bg_sub_by_session.iter().any(|((sub_root, _), channels)| {
553 sub_root == &root_id
554 && channels
555 .iter()
556 .any(|channel| bg_wake_pending.contains(channel))
557 });
558 let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
559 Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
560 .into_iter()
561 .filter(|kind| {
562 if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
563 return false;
564 }
565 match kind {
566 MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
567 MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
568 MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
569 MaintenanceDrainKind::CompletionDrains => {
575 root_has_pending_bg_wake || ctx.completion_drains_have_work()
576 }
577 }
578 })
579 .collect(),
580 None if meta.unbound_quiesced => Vec::new(),
581 None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
583 };
584 if kinds_with_work.is_empty() {
585 continue;
586 }
587 meta.maintenance_pending = true;
588 meta.maintenance_queued_kinds.extend(kinds_with_work);
589 }
590
591 while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
592 if jobs.len() >= budget {
593 meta.maintenance_queued_kinds.push_front(kind);
594 deferred = true;
595 break;
596 }
597 meta.maintenance_jobs_in_flight += 1;
598 meta.maintenance_last_submitted = Some(Instant::now());
599 jobs.push((root_id.clone(), kind));
600 }
601
602 meta.maintenance_pending =
603 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
604 }
605
606 (jobs, deferred)
607}
608
609fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
610 match estimate.estimated_bytes {
611 Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
612 None if estimate.status == "busy" => "busy".to_string(),
613 None => "not estimated".to_string(),
614 }
615}
616
617fn optional_memory_label(bytes: Option<u64>) -> String {
618 bytes.map_or_else(
619 || "not estimated".to_string(),
620 |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
621 )
622}
623
624fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
625 format!(
626 "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
627 optional_memory_label(relief.rss_before_bytes),
628 optional_memory_label(relief.rss_after_bytes),
629 optional_memory_label(relief.allocator_before.bytes_in_use),
630 optional_memory_label(relief.allocator_after.bytes_in_use),
631 optional_memory_label(relief.allocator_before.size_allocated),
632 optional_memory_label(relief.allocator_after.size_allocated),
633 optional_memory_label(relief.allocator_before.retained_slack_bytes),
634 optional_memory_label(relief.allocator_after.retained_slack_bytes),
635 relief.bytes_released as f64 / (1024.0 * 1024.0),
636 )
637}
638
639fn idle_root_eviction_message(
640 root_id: &ProjectRootId,
641 memory: &crate::memory::RootMemorySnapshot,
642 pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
643) -> String {
644 let freed_bytes = [
647 &memory.semantic,
648 &memory.trigram,
649 &memory.symbols,
650 &memory.callgraph,
651 &memory.inspect,
652 ]
653 .iter()
654 .filter_map(|estimate| estimate.estimated_bytes)
655 .fold(0u64, u64::saturating_add);
656 let mut message = format!(
657 "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
658 root_id.as_path().display(),
659 freed_bytes as f64 / (1024.0 * 1024.0),
660 eviction_estimate_label(&memory.semantic),
661 eviction_estimate_label(&memory.trigram),
662 eviction_estimate_label(&memory.symbols),
663 eviction_estimate_label(&memory.callgraph),
664 eviction_estimate_label(&memory.inspect),
665 eviction_estimate_label(&memory.bash),
666 eviction_estimate_label(&memory.lsp),
667 eviction_estimate_label(&memory.parser_pool),
668 );
669 if let Some(pressure_relief) = pressure_relief {
670 message.push_str(&pressure_relief_label(pressure_relief));
671 }
672 message
673}
674
675fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
676 !live_roots.is_empty()
677 && live_roots.values().all(|meta| {
678 now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
679 && meta.active_bash_waits == 0
680 && !meta.maintenance_pending
681 && meta.maintenance_queued_kinds.is_empty()
682 })
683}
684
685fn allocator_pressure_relief_after_idle_sweep(
686 now: Instant,
687 live_roots: &HashMap<ProjectRootId, RootMeta>,
688 executor: &Executor,
689) -> Option<crate::memory::AllocatorPressureRelief> {
690 if !process_has_been_idle(now, live_roots)
691 || live_roots.keys().any(|root_id| {
692 executor
693 .actor_context(root_id)
694 .is_some_and(|ctx| ctx.artifact_eviction_blocked())
695 })
696 {
697 return None;
698 }
699
700 #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))]
701 {
702 Some(crate::memory::relieve_allocator_pressure())
703 }
704 #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))]
705 {
706 None
707 }
708}
709
710fn quiesce_unbound_root(
711 root_id: &ProjectRootId,
712 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
713 executor: &Arc<Executor>,
714) {
715 let Some(meta) = live_roots.get_mut(root_id) else {
716 return;
717 };
718
719 let ctx = executor.actor_context(root_id);
720 if let Some(ctx) = ctx.as_ref() {
721 ctx.mark_subc_unbound();
725 }
726 let cancelled = executor.cancel_queued_maintenance(root_id);
727 let discarded = ctx
735 .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
736 .unwrap_or(0);
737 meta.unbound_quiesced = true;
738 meta.maintenance_queued_kinds.clear();
739 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
740 log::info!(
741 "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s)); cause=goodbye_unbound",
742 root_id.as_path().display(),
743 cancelled,
744 discarded
745 );
746}
747
748#[allow(clippy::too_many_arguments)]
749fn quiesce_connection_roots(
750 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
751 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
752 routes: &mut HashMap<RouteChannel, RouteIdentity>,
753 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
754 installed_route_epochs: &mut HashMap<u16, u32>,
755 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
756 executor: &Arc<Executor>,
757) {
758 for cancel in route_bash_cancels.values() {
759 cancel.token.cancel();
760 }
761 route_bash_cancels.clear();
762
763 let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
764 for pending in pending_binds.values_mut() {
765 pending.cancelled = true;
766 roots.insert(pending.bind_root_id.clone());
767 let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
768 }
769
770 for root_id in roots {
774 if live_roots.contains_key(&root_id) {
775 quiesce_unbound_root(&root_id, live_roots, executor);
776 } else if let Some(ctx) = executor.actor_context(&root_id) {
777 ctx.mark_subc_unbound();
778 executor.cancel_queued_maintenance(&root_id);
779 crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
780 }
781 }
782
783 routes.clear();
784 root_channels.clear();
785 installed_route_epochs.clear();
786}
787
788#[derive(Debug, Default)]
792struct ReclaimedRoutes {
793 highest_epoch_by_channel: HashMap<u16, u32>,
794}
795
796impl ReclaimedRoutes {
797 fn insert(&mut self, route: RouteChannel) {
798 self.highest_epoch_by_channel
799 .entry(route.channel)
800 .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
801 .or_insert(route.epoch);
802 }
803
804 fn contains(&self, route: RouteChannel) -> bool {
805 self.highest_epoch_by_channel
806 .get(&route.channel)
807 .is_some_and(|epoch| route.epoch <= *epoch)
808 }
809}
810
811#[derive(Debug, Default)]
812struct IdleReapOutcome {
813 evicted: usize,
814 forgotten_deleted_roots: Vec<ProjectRootId>,
815}
816
817fn reap_idle_roots(
818 now: Instant,
819 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
820 pending_binds: &HashMap<RouteChannel, PendingBind>,
821 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
822 executor: &Arc<Executor>,
823 metrics: &DispatchPathMetrics,
824) -> IdleReapOutcome {
825 let pending_bind_roots = pending_binds
826 .values()
827 .map(|pending| pending.bind_root_id.clone())
828 .collect::<HashSet<_>>();
829 let mut census = ReapBlockerCensus::default();
830 let mut candidates = Vec::new();
831
832 for (root_id, meta) in live_roots.iter_mut() {
833 let deleted = !root_id.as_path().exists();
834 if deleted {
835 meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
845 } else {
846 meta.consecutive_missing_sweeps = 0;
847 }
848 let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
849 let has_bound_route = root_channels
850 .get(root_id)
851 .is_some_and(|channels| !channels.is_empty());
852 let has_pending_bind = pending_bind_roots.contains(root_id);
853
854 if deleted {
855 let mut retained = false;
856 if !deletion_confirmed {
857 census.absence_unconfirmed += 1;
858 retained = true;
859 }
860 if meta.active_bash_waits > 0 {
864 census.bash_waits += 1;
865 retained = true;
866 }
867 if meta.maintenance_pending {
868 census.maintenance_pending += 1;
869 retained = true;
870 }
871 if !meta.maintenance_queued_kinds.is_empty() {
872 census.maintenance_queued += 1;
873 retained = true;
874 }
875 if has_pending_bind {
876 census.pending_binds += 1;
877 retained = true;
878 }
879 match executor.try_actor_is_idle(root_id) {
880 Some(true) => {}
881 Some(false) => {
882 census.actor_busy += 1;
883 retained = true;
884 }
885 None => {
886 census.actor_state_busy += 1;
887 retained = true;
888 }
889 }
890 if retained {
891 census.deleted_retained += 1;
892 continue;
893 }
894 } else {
895 if has_bound_route
899 || !meta.unbound_quiesced
900 || meta.idle_artifacts_evicted
901 || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
902 || meta.active_bash_waits > 0
903 || meta.maintenance_pending
904 || !meta.maintenance_queued_kinds.is_empty()
905 || has_pending_bind
906 || !executor.actor_is_idle(root_id)
907 {
908 continue;
909 }
910 }
911 candidates.push((root_id.clone(), deleted));
912 }
913
914 let mut reaped = Vec::new();
915 let mut forgotten_deleted_roots = Vec::new();
916 for (root_id, deleted) in candidates {
917 let Some(ctx) = executor.actor_context(&root_id) else {
918 if deleted {
919 census.deleted_retained += 1;
920 census.actor_busy += 1;
921 }
922 continue;
923 };
924 if deleted {
935 ctx.bash_background()
936 .kill_running_tasks_for_root(root_id.as_path());
937 }
938 let taken_pending = Some(ctx.take_pending_reconciliation_state());
939 if ctx.artifact_eviction_blocked() {
940 if let Some(pending) = taken_pending {
941 ctx.restore_pending_reconciliation_state(pending);
942 }
943 if deleted {
944 census.deleted_retained += 1;
945 census.artifact_eviction_blocked += 1;
946 }
947 continue;
948 }
949 let memory_before = ctx.memory_root_snapshot();
950 if !ctx.evict_idle_artifacts() {
951 if let Some(pending) = taken_pending {
952 ctx.restore_pending_reconciliation_state(pending);
953 }
954 if deleted {
955 census.deleted_retained += 1;
956 census.artifact_eviction_failed += 1;
957 }
958 continue;
959 }
960 drop(taken_pending);
961 ctx.stop_watcher_runtime_in_background();
962 ctx.invalidate_artifacts_after_watcher_gap();
965
966 if deleted {
967 if executor.retire_idle_actor_in_background(&root_id) {
968 live_roots.remove(&root_id);
969 forgotten_deleted_roots.push(root_id.clone());
970 } else {
971 census.deleted_retained += 1;
972 census.actor_busy += 1;
973 }
974 } else {
975 if let Some(meta) = live_roots.get_mut(&root_id) {
976 meta.idle_artifacts_evicted = true;
977 }
978 ctx.release_idle_reopenable_resources_in_background();
979 }
980 reaped.push((root_id, memory_before));
981 }
982
983 metrics.record_reap(census);
984 if census.deleted_retained > 0 {
985 log::info!(
986 "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
987 census.deleted_retained,
988 census.blocker_histogram()
989 );
990 }
991
992 let pressure_relief = (!reaped.is_empty())
993 .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
994 .flatten();
995 for (root_id, memory_before) in &reaped {
996 log::info!(
997 "{}",
998 idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
999 );
1000 }
1001 IdleReapOutcome {
1002 evicted: reaped.len(),
1003 forgotten_deleted_roots,
1004 }
1005}
1006
1007#[allow(clippy::too_many_arguments)]
1008fn purge_deleted_root_residents(
1009 root_id: &ProjectRootId,
1010 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1011 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1012 installed_route_epochs: &mut HashMap<u16, u32>,
1013 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1014 retry_buffer: &mut RetryBuffer,
1015 reclaimed_routes: &mut ReclaimedRoutes,
1016 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1017 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1018 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1019 bg_sub_by_session: &mut BgSubsBySession,
1020 bg_wake_pending: &mut HashSet<RouteChannel>,
1021 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1022 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1023 metrics: &DispatchPathMetrics,
1024) {
1025 let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1026 stale_routes.extend(
1027 routes
1028 .iter()
1029 .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1030 );
1031 stale_routes.extend(
1032 bg_sub_by_session
1033 .iter()
1034 .filter(|((root, _), _)| root == root_id)
1035 .flat_map(|(_, routes)| routes.iter().copied()),
1036 );
1037 stale_routes.extend(
1038 pending_bash_asks
1039 .values()
1040 .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1041 );
1042
1043 for route in stale_routes {
1044 reclaimed_routes.insert(route);
1045 remove_installed_route(installed_route_epochs, route);
1046 remove_route_channel(routes, root_channels, route);
1047 if let Some(cancel) = route_bash_cancels.remove(&route) {
1048 cancel.token.cancel();
1049 }
1050 retry_buffer.remove(&route);
1051 if let Some(sub) = bg_subs.remove(&route) {
1052 metrics.record_bg_subscription_ended(&sub.root, &sub.session, route, "root-reclaim");
1053 }
1054 bg_wake_pending.remove(&route);
1055 }
1056 root_channels.remove(root_id);
1057 session_identity.retain(|(root, _), _| root != root_id);
1058 push_buffer.retain(|key, _| &key.root != root_id);
1059 bg_wake_epoch.retain(|(root, _), _| root != root_id);
1060 pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1061 bg_sub_by_session.retain(|(root, _), _| root != root_id);
1062
1063 log::info!(
1064 "subc attach: fully forgot deleted root {}; cause=absence_reclaim",
1065 root_id.as_path().display()
1066 );
1067}
1068
1069#[allow(clippy::too_many_arguments)]
1070fn submit_due_maintenance_jobs(
1071 executor: &Arc<Executor>,
1072 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1073 pending_binds: &HashMap<RouteChannel, PendingBind>,
1074 bg_sub_by_session: &BgSubsBySession,
1075 bg_wake_pending: &HashSet<RouteChannel>,
1076 bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1077 maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1078 metrics: &Arc<DispatchPathMetrics>,
1079) {
1080 let pending_bind_roots = pending_binds
1081 .values()
1082 .map(|pending| pending.bind_root_id.clone())
1083 .collect::<HashSet<_>>();
1084 let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1085 live_roots,
1086 Some(executor),
1087 bg_sub_by_session,
1088 bg_wake_pending,
1089 MAINTENANCE_SUBMIT_BUDGET,
1090 &pending_bind_roots,
1091 );
1092 if deferred_jobs {
1093 metrics
1094 .maintenance_budget_deferrals
1095 .fetch_add(1, Ordering::Relaxed);
1096 }
1097 for (root_id, kind) in due_jobs {
1098 let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1099 bg_sub_by_session
1100 .iter()
1101 .filter_map(|((root, session), _)| {
1102 if root == &root_id {
1103 Some((
1104 session.clone(),
1105 bg_wake_epoch
1106 .get(&(root_id.clone(), session.clone()))
1107 .copied()
1108 .unwrap_or(0),
1109 ))
1110 } else {
1111 None
1112 }
1113 })
1114 .collect()
1115 } else {
1116 Vec::new()
1117 };
1118 submit_maintenance_job(
1119 executor,
1120 root_id,
1121 kind,
1122 bg_sessions_to_check,
1123 maintenance_tx,
1124 metrics,
1125 );
1126 }
1127}
1128
1129fn should_requiesce_after_maintenance(
1130 meta: &RootMeta,
1131 completed_kind: MaintenanceDrainKind,
1132 bind_pending: bool,
1133) -> bool {
1134 meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1135}
1136
1137fn note_maintenance_completion(
1138 meta: &mut RootMeta,
1139 requeue_kind: Option<MaintenanceDrainKind>,
1140 fatal: bool,
1141 defer_requeue: bool,
1142) {
1143 if fatal {
1144 meta.maintenance_poisoned = true;
1145 }
1146
1147 if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1148 meta.maintenance_queued_kinds.push_back(kind);
1149 }
1150
1151 meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1152 meta.maintenance_pending =
1153 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1154}
1155
1156fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1157 RouteChannel { channel, epoch }
1158}
1159
1160fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1161 if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1162 installed_epochs.remove(&route.channel);
1163 }
1164}
1165
1166fn ingress_route_should_be_processed(
1167 installed_epochs: &HashMap<u16, u32>,
1168 reclaimed_routes: &ReclaimedRoutes,
1169 frame: &Frame,
1170) -> bool {
1171 if frame.header.channel == 0
1172 || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1173 {
1174 return true;
1175 }
1176
1177 frame.header.ty == FrameType::Request
1182 && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1183}
1184
1185fn bash_elicitation_timeout() -> Duration {
1186 if cfg!(debug_assertions) {
1187 if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1188 if let Ok(ms) = raw.parse::<u64>() {
1189 if ms > 0 {
1190 return Duration::from_millis(ms);
1191 }
1192 }
1193 }
1194 }
1195 BASH_ELICITATION_TIMEOUT
1196}
1197
1198fn allocate_reverse_corr(
1199 pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1200 route: RouteChannel,
1201 next_corr: &mut u64,
1202) -> u64 {
1203 loop {
1204 let corr = *next_corr;
1205 *next_corr = (*next_corr).wrapping_add(1).max(1);
1206 if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1207 return corr;
1208 }
1209 }
1210}
1211
1212fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1213 match kind {
1214 crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1215 crate::bash_permissions::PermissionKind::Bash => "bash",
1216 }
1217}
1218
1219fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1220 let mut patterns = Vec::new();
1221 let mut seen = HashSet::new();
1222 for ask in asks {
1223 for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1224 if seen.insert(pattern.clone()) {
1225 patterns.push(pattern.clone());
1226 }
1227 }
1228 }
1229 patterns
1230}
1231
1232fn bash_elicitation_message(
1233 command: &str,
1234 asks: &[crate::bash_permissions::PermissionAsk],
1235) -> String {
1236 let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1237 let patterns = bash_elicitation_patterns(asks);
1238 let pattern_text = if patterns.is_empty() {
1239 "no matched permission patterns".to_string()
1240 } else {
1241 patterns.join(", ")
1242 };
1243 let ask_kinds = asks
1244 .iter()
1245 .map(|ask| bash_permission_kind_label(&ask.kind))
1246 .collect::<HashSet<_>>()
1247 .into_iter()
1248 .collect::<Vec<_>>()
1249 .join(", ");
1250 if ask_kinds.is_empty() {
1251 format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1252 } else {
1253 format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1254 }
1255}
1256
1257fn bash_elicitation_request_body(
1258 command: &str,
1259 asks: &[crate::bash_permissions::PermissionAsk],
1260) -> Value {
1261 json!({
1262 "method": BASH_ELICITATION_CREATE_METHOD,
1263 "params": {
1264 "mode": "form",
1265 "message": bash_elicitation_message(command, asks),
1266 "requestedSchema": {
1267 "type": "object",
1268 "properties": {
1269 "decision": {
1270 "type": "string",
1271 "enum": ["allow", "deny"],
1272 "description": "Choose allow to run this bash command once, or deny to block it."
1273 }
1274 },
1275 "required": ["decision"],
1276 "additionalProperties": false
1277 },
1278 "_meta": {
1279 "aft": {
1280 "tool": "bash",
1281 "command": command,
1282 "asks": asks
1283 }
1284 }
1285 }
1286 })
1287}
1288
1289fn build_bash_elicitation_request_frame(
1290 ver: u8,
1291 route: RouteChannel,
1292 corr: u64,
1293 flags: Flags,
1294 command: &str,
1295 asks: &[crate::bash_permissions::PermissionAsk],
1296) -> Result<Frame, SubcError> {
1297 let body = bash_elicitation_request_body(command, asks);
1298 Frame::build_with_version(
1299 ver,
1300 FrameType::Request,
1301 flags,
1302 route.channel,
1303 route.epoch,
1304 corr,
1305 serde_json::to_vec(&body).map_err(SubcError::Json)?,
1306 )
1307 .map_err(SubcError::FrameBuild)
1308}
1309
1310fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1311 let Ok(value) = serde_json::from_slice::<Value>(body) else {
1312 return false;
1313 };
1314 flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1315}
1316
1317fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1318 let Some(object) = value.as_object() else {
1319 return false;
1320 };
1321 object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1322}
1323
1324fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1325 let Some(object) = value.as_object() else {
1326 return false;
1327 };
1328 if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1329 return false;
1330 }
1331 let Some(content) = object.get("content").and_then(Value::as_object) else {
1332 return false;
1333 };
1334 content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1335}
1336
1337#[allow(clippy::too_many_arguments)]
1338async fn settle_pending_bash_ask_denied(
1339 tx: &WriterSender,
1340 pending: PendingBashAsk,
1341 routes: &HashMap<RouteChannel, RouteIdentity>,
1342 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1343 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1344 shutdown: &Arc<Notify>,
1345 metrics: &DispatchPathMetrics,
1346) -> Result<(), SubcError> {
1347 let completion = bash::bash_denied_untrusted_completion(
1348 pending.route,
1349 pending.tool_corr,
1350 pending.tool_flags,
1351 pending.tool_ver,
1352 pending.root,
1353 pending.request_id,
1354 pending.format_context,
1355 );
1356 bash::handle_bash_deferred_completion(
1357 tx,
1358 completion,
1359 routes,
1360 live_roots,
1361 route_bash_cancels,
1362 shutdown,
1363 metrics,
1364 )
1365 .await
1366}
1367
1368fn take_pending_bash_asks_for_route(
1369 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1370 route: RouteChannel,
1371) -> Vec<PendingBashAsk> {
1372 let keys = pending_bash_asks
1373 .keys()
1374 .copied()
1375 .filter(|key| key.route == route)
1376 .collect::<Vec<_>>();
1377 keys.into_iter()
1378 .filter_map(|key| pending_bash_asks.remove(&key))
1379 .collect()
1380}
1381
1382#[allow(clippy::too_many_arguments)]
1383async fn settle_pending_bash_asks_for_route(
1384 tx: &WriterSender,
1385 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1386 route: RouteChannel,
1387 routes: &HashMap<RouteChannel, RouteIdentity>,
1388 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1389 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1390 shutdown: &Arc<Notify>,
1391 metrics: &DispatchPathMetrics,
1392) -> Result<(), SubcError> {
1393 for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1394 settle_pending_bash_ask_denied(
1395 tx,
1396 pending,
1397 routes,
1398 live_roots,
1399 route_bash_cancels,
1400 shutdown,
1401 metrics,
1402 )
1403 .await?;
1404 }
1405 Ok(())
1406}
1407
1408#[allow(clippy::too_many_arguments)]
1409async fn settle_all_pending_bash_asks(
1410 tx: &WriterSender,
1411 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1412 routes: &HashMap<RouteChannel, RouteIdentity>,
1413 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1414 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1415 shutdown: &Arc<Notify>,
1416 metrics: &DispatchPathMetrics,
1417) -> Result<(), SubcError> {
1418 let pending = pending_bash_asks
1419 .drain()
1420 .map(|(_, pending)| pending)
1421 .collect::<Vec<_>>();
1422 for pending in pending {
1423 settle_pending_bash_ask_denied(
1424 tx,
1425 pending,
1426 routes,
1427 live_roots,
1428 route_bash_cancels,
1429 shutdown,
1430 metrics,
1431 )
1432 .await?;
1433 }
1434 Ok(())
1435}
1436
1437#[allow(clippy::too_many_arguments)]
1438async fn expire_pending_bash_asks(
1439 tx: &WriterSender,
1440 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1441 routes: &HashMap<RouteChannel, RouteIdentity>,
1442 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1443 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1444 shutdown: &Arc<Notify>,
1445 metrics: &DispatchPathMetrics,
1446) -> Result<(), SubcError> {
1447 let now = Instant::now();
1448 let expired = pending_bash_asks
1449 .iter()
1450 .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1451 .collect::<Vec<_>>();
1452 for key in expired {
1453 if let Some(pending) = pending_bash_asks.remove(&key) {
1454 log::debug!(
1455 "subc attach: bash elicitation request {} on route {} expired fail-closed",
1456 key.corr,
1457 pending.route
1458 );
1459 settle_pending_bash_ask_denied(
1460 tx,
1461 pending,
1462 routes,
1463 live_roots,
1464 route_bash_cancels,
1465 shutdown,
1466 metrics,
1467 )
1468 .await?;
1469 }
1470 }
1471 Ok(())
1472}
1473
1474#[allow(clippy::too_many_arguments)]
1475async fn handle_bash_elicitation_reply(
1476 tx: &WriterSender,
1477 frame: &Frame,
1478 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1479 routes: &HashMap<RouteChannel, RouteIdentity>,
1480 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1481 executor: &Arc<Executor>,
1482 shutdown: &Arc<Notify>,
1483 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1484 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1485 metrics: &Arc<DispatchPathMetrics>,
1486 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1487 dispatch: DispatchFn,
1488) -> Result<(), SubcError> {
1489 let key = ReverseCorrKey {
1490 route: route_key(frame.header.channel, frame.header.epoch),
1491 corr: frame.header.corr,
1492 };
1493 let Some(pending) = pending_bash_asks.remove(&key) else {
1494 return Ok(());
1495 };
1496
1497 if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1498 if routes.contains_key(&key.route) {
1499 bash::submit_deferred_bash(
1500 executor,
1501 bash_deferred_tx,
1502 bash_poll_touch_tx,
1503 metrics,
1504 dispatch,
1505 pending.root,
1506 pending.project_root,
1507 pending.session_id,
1508 pending.request_id,
1509 pending.route,
1510 pending.tool_corr,
1511 pending.tool_flags,
1512 pending.tool_ver,
1513 pending.arguments,
1514 pending.format_context,
1515 pending.cancel,
1516 BindTrust::Untrusted,
1517 pending.spawn_principal,
1518 pending.edit_slot_survives,
1519 Some(pending.grants),
1520 );
1521 return Ok(());
1522 }
1523 log::debug!(
1524 "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1525 key.corr,
1526 pending.route
1527 );
1528 }
1529
1530 settle_pending_bash_ask_denied(
1531 tx,
1532 pending,
1533 routes,
1534 live_roots,
1535 route_bash_cancels,
1536 shutdown,
1537 metrics,
1538 )
1539 .await
1540}
1541
1542#[allow(clippy::too_many_arguments)]
1543async fn cancel_pending_bash_ask_for_tool_call(
1544 tx: &WriterSender,
1545 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1546 route: RouteChannel,
1547 tool_corr: u64,
1548 routes: &HashMap<RouteChannel, RouteIdentity>,
1549 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1550 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1551 shutdown: &Arc<Notify>,
1552 metrics: &DispatchPathMetrics,
1553) -> Result<(), SubcError> {
1554 let keys = pending_bash_asks
1555 .iter()
1556 .filter_map(|(key, pending)| {
1557 (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1558 })
1559 .collect::<Vec<_>>();
1560 for key in keys {
1561 if let Some(pending) = pending_bash_asks.remove(&key) {
1562 settle_pending_bash_ask_denied(
1563 tx,
1564 pending,
1565 routes,
1566 live_roots,
1567 route_bash_cancels,
1568 shutdown,
1569 metrics,
1570 )
1571 .await?;
1572 }
1573 }
1574 Ok(())
1575}
1576
1577fn remove_root_channel(
1578 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1579 root: &ProjectRootId,
1580 channel: RouteChannel,
1581) {
1582 let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1583 channels.remove(&channel);
1584 channels.is_empty()
1585 } else {
1586 false
1587 };
1588 if remove_root {
1589 root_channels.remove(root);
1590 }
1591}
1592
1593fn remove_route_channel(
1594 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1595 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1596 channel: RouteChannel,
1597) -> Option<RouteIdentity> {
1598 let removed = routes.remove(&channel);
1599 if let Some(identity) = &removed {
1600 remove_root_channel(root_channels, &identity.root, channel);
1601 }
1602 removed
1603}
1604
1605fn insert_route_channel(
1606 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1607 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1608 channel: RouteChannel,
1609 identity: RouteIdentity,
1610) {
1611 if let Some(previous) = routes.insert(channel, identity.clone()) {
1612 remove_root_channel(root_channels, &previous.root, channel);
1613 }
1614 root_channels
1615 .entry(identity.root.clone())
1616 .or_default()
1617 .insert(channel);
1618}
1619
1620fn insert_bg_subscription_index(
1621 bg_sub_by_session: &mut BgSubsBySession,
1622 root: ProjectRootId,
1623 session: String,
1624 channel: RouteChannel,
1625) {
1626 bg_sub_by_session
1627 .entry((root, session))
1628 .or_default()
1629 .insert(channel);
1630}
1631
1632fn remove_bg_subscription_index(
1633 bg_sub_by_session: &mut BgSubsBySession,
1634 channel: RouteChannel,
1635 identity: Option<&RouteIdentity>,
1636) {
1637 if let Some(identity) = identity {
1638 let key = (identity.root.clone(), identity.session.clone());
1639 let remove_key = bg_sub_by_session.get_mut(&key).is_some_and(|channels| {
1640 channels.remove(&channel);
1641 channels.is_empty()
1642 });
1643 if remove_key {
1644 bg_sub_by_session.remove(&key);
1645 }
1646 } else {
1647 bg_sub_by_session.retain(|_, channels| {
1648 channels.remove(&channel);
1649 !channels.is_empty()
1650 });
1651 }
1652}
1653
1654fn route_removal_will_quiesce_root(
1655 root: &ProjectRootId,
1656 route: RouteChannel,
1657 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1658 has_pending_bind: bool,
1659 replacement_root: Option<&ProjectRootId>,
1660) -> bool {
1661 let removes_last_route = root_channels
1662 .get(root)
1663 .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1664 removes_last_route && !has_pending_bind && replacement_root != Some(root)
1665}
1666
1667fn should_quiesce_removed_root(
1668 root: &ProjectRootId,
1669 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1670 has_pending_bind: bool,
1671 replacement_root: Option<&ProjectRootId>,
1672) -> bool {
1673 !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1674}
1675
1676async fn end_bg_subscription(
1677 writer_tx: &WriterSender,
1678 metrics: &DispatchPathMetrics,
1679 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1680 bg_sub_by_session: &mut BgSubsBySession,
1681 bg_wake_pending: &mut HashSet<RouteChannel>,
1682 channel: RouteChannel,
1683 identity: Option<&RouteIdentity>,
1684 cause: &str,
1685) -> Result<(), SubcError> {
1686 if let Some(sub) = bg_subs.remove(&channel) {
1687 bg_wake_pending.remove(&channel);
1688 remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1689 metrics.record_bg_subscription_ended(&sub.root, &sub.session, channel, cause);
1690 push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1691 }
1692 Ok(())
1693}
1694
1695#[allow(clippy::too_many_arguments)]
1696async fn teardown_installed_route(
1697 tx: &WriterSender,
1698 metrics: &DispatchPathMetrics,
1699 executor: &Arc<Executor>,
1700 channel: RouteChannel,
1701 cancellation_reason: &str,
1702 replacement_root: Option<&ProjectRootId>,
1703 installed_route_epochs: &mut HashMap<u16, u32>,
1704 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1705 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1706 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1707 bg_sub_by_session: &mut BgSubsBySession,
1708 bg_wake_pending: &mut HashSet<RouteChannel>,
1709 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1710 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1711 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1712 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1713 retry_buffer: &mut RetryBuffer,
1714 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1715 shutdown: &Arc<Notify>,
1716) -> Result<(), SubcError> {
1717 remove_installed_route(installed_route_epochs, channel);
1718 let bg_end_cause = match cancellation_reason {
1719 "Goodbye" => "goodbye",
1720 "higher-epoch RouteBind" => "higher-epoch",
1721 other => other,
1722 };
1723 end_bg_subscription(
1724 tx,
1725 metrics,
1726 bg_subs,
1727 bg_sub_by_session,
1728 bg_wake_pending,
1729 channel,
1730 routes.get(&channel),
1731 bg_end_cause,
1732 )
1733 .await?;
1734 settle_pending_bash_asks_for_route(
1735 tx,
1736 pending_bash_asks,
1737 channel,
1738 routes,
1739 live_roots,
1740 route_bash_cancels,
1741 shutdown,
1742 metrics,
1743 )
1744 .await?;
1745 if let Some(cancel) = route_bash_cancels.remove(&channel) {
1746 cancel.token.cancel();
1747 }
1748 if let Some(pending) = pending_binds.get_mut(&channel) {
1749 pending.cancelled = true;
1750 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1751 log::debug!(
1752 "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1753 channel.channel
1754 );
1755 }
1756 let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1757 if let Some(identity) = routes.get(&channel) {
1758 let has_pending_bind = pending_binds
1759 .values()
1760 .any(|pending| pending.bind_root_id == identity.root);
1761 if route_removal_will_quiesce_root(
1762 &identity.root,
1763 channel,
1764 root_channels,
1765 has_pending_bind,
1766 replacement_root,
1767 ) {
1768 if let Some(ctx) = executor.actor_context(&identity.root) {
1769 ctx.mark_subc_unbound();
1772 }
1773 }
1774 }
1775 if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
1776 let session_still_routed = routes
1777 .values()
1778 .any(|route| route.root == identity.root && route.session == identity.session);
1779 if !session_still_routed {
1780 if let Some(ctx) = executor.actor_context(&identity.root) {
1781 ctx.hashline_bindings()
1782 .teardown(identity.root.as_path(), &identity.session);
1783 }
1784 }
1785 if migrated > 0 {
1786 log::debug!(
1787 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
1788 channel.channel
1789 );
1790 }
1791 if let Some(meta) = live_roots.get_mut(&identity.root) {
1792 let idle_for = meta.last_touched.elapsed();
1793 meta.note_activity();
1794 log::debug!(
1795 "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
1796 channel.channel,
1797 identity.root.as_path().display(),
1798 identity.harness,
1799 identity.session,
1800 idle_for
1801 );
1802 } else {
1803 log::debug!(
1804 "subc attach: route {} torn down for root {} harness {} session {}",
1805 channel.channel,
1806 identity.root.as_path().display(),
1807 identity.harness,
1808 identity.session
1809 );
1810 }
1811 let has_pending_bind = pending_binds
1812 .values()
1813 .any(|pending| pending.bind_root_id == identity.root);
1814 if should_quiesce_removed_root(
1815 &identity.root,
1816 root_channels,
1817 has_pending_bind,
1818 replacement_root,
1819 ) {
1820 quiesce_unbound_root(&identity.root, live_roots, executor);
1821 }
1822 } else {
1823 if migrated > 0 {
1824 log::debug!(
1825 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
1826 channel.channel
1827 );
1828 }
1829 log::debug!("subc attach: unbound route {} torn down", channel.channel);
1830 }
1831 Ok(())
1832}
1833
1834fn remember_session_identity(
1835 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1836 identity: &RouteIdentity,
1837) {
1838 let key = (identity.root.clone(), identity.session.clone());
1839 if matches!(identity.trust, BindTrust::Untrusted)
1840 && session_identity
1841 .get(&key)
1842 .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
1843 {
1844 return;
1845 }
1846
1847 session_identity.insert(
1852 key,
1853 RetainedSessionIdentity {
1854 harness: identity.harness.clone(),
1855 trust: identity.trust,
1856 },
1857 );
1858}
1859
1860fn replay_key_for_session(
1861 session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1862 root: &ProjectRootId,
1863 session: &str,
1864) -> Option<(push::ReplayKey, BindTrust)> {
1865 let retained = session_identity.get(&(root.clone(), session.to_string()))?;
1866 Some((
1867 push::ReplayKey {
1868 root: root.clone(),
1869 harness: retained.harness.clone(),
1870 session: session.to_string(),
1871 },
1872 retained.trust,
1873 ))
1874}
1875pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
1878
1879#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1880enum ModuleLoopExit {
1881 Graceful,
1882 SkipSearchFlush,
1883}
1884
1885pub fn run_subc_mode(
1890 connection_file_path: &Path,
1891 ctx: Arc<AppContext>,
1892 executor: Arc<Executor>,
1893 dispatch: DispatchFn,
1894 user_config_path: Option<PathBuf>,
1895) -> Result<(), SubcError> {
1896 run_subc_mode_inner(
1900 connection_file_path,
1901 ctx,
1902 executor,
1903 dispatch,
1904 user_config_path,
1905 false,
1906 MAX_FRAME_BODY_LEN as usize,
1907 )
1908}
1909
1910fn run_subc_mode_inner(
1911 connection_file_path: &Path,
1912 ctx: Arc<AppContext>,
1913 executor: Arc<Executor>,
1914 dispatch: DispatchFn,
1915 user_config_path: Option<PathBuf>,
1916 allow_native_passthrough: bool,
1917 tool_response_body_limit: usize,
1918) -> Result<(), SubcError> {
1919 let runtime = tokio::runtime::Builder::new_current_thread()
1920 .enable_all()
1921 .build()
1922 .map_err(SubcError::Runtime)?;
1923
1924 let executor_for_loop = Arc::clone(&executor);
1925 let loop_result = runtime.block_on(async move {
1926 let shared_app = ctx.app();
1927 drop(ctx);
1928 let stream = connect_and_authenticate(connection_file_path).await?;
1929 log::info!(
1930 "subc attach: authenticated to daemon via {}",
1931 connection_file_path.display()
1932 );
1933 let (read_half, write_half) = tokio::io::split(stream);
1934 run_module_loop(
1935 read_half,
1936 write_half,
1937 connection_file_path,
1938 shared_app,
1939 executor_for_loop,
1940 dispatch,
1941 user_config_path,
1942 allow_native_passthrough,
1943 tool_response_body_limit,
1944 )
1945 .await
1946 });
1947
1948 let actor_contexts = executor.actor_contexts();
1949 if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
1950 flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
1953 }
1954 for actor_ctx in &actor_contexts {
1955 actor_ctx.lsp().shutdown_all();
1956 actor_ctx.bash_background().detach();
1957 }
1958
1959 loop_result.map(|_| ())
1960}
1961
1962fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
1963 for actor_ctx in actor_contexts {
1964 let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
1965 }
1966 let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
1967}
1968
1969#[doc(hidden)]
1974pub fn run_subc_mode_for_test(
1975 connection_file_path: &Path,
1976 ctx: Arc<AppContext>,
1977 executor: Arc<Executor>,
1978 dispatch: DispatchFn,
1979 user_config_path: Option<PathBuf>,
1980) -> Result<(), SubcError> {
1981 run_subc_mode_inner(
1982 connection_file_path,
1983 ctx,
1984 executor,
1985 dispatch,
1986 user_config_path,
1987 true,
1988 MAX_FRAME_BODY_LEN as usize,
1989 )
1990}
1991
1992#[doc(hidden)]
1995pub fn run_subc_mode_for_test_with_response_body_limit(
1996 connection_file_path: &Path,
1997 ctx: Arc<AppContext>,
1998 executor: Arc<Executor>,
1999 dispatch: DispatchFn,
2000 user_config_path: Option<PathBuf>,
2001 tool_response_body_limit: usize,
2002) -> Result<(), SubcError> {
2003 assert!((4 * 1_024..=MAX_FRAME_BODY_LEN as usize).contains(&tool_response_body_limit));
2004 run_subc_mode_inner(
2005 connection_file_path,
2006 ctx,
2007 executor,
2008 dispatch,
2009 user_config_path,
2010 true,
2011 tool_response_body_limit,
2012 )
2013}
2014
2015#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2016enum AttachErrorClass {
2017 Transient,
2018 Permanent,
2019}
2020
2021impl fmt::Display for AttachErrorClass {
2022 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2023 match self {
2024 Self::Transient => f.write_str("transient"),
2025 Self::Permanent => f.write_str("permanent"),
2026 }
2027 }
2028}
2029
2030#[derive(Clone, Copy)]
2031struct AttachRetryPolicy {
2032 budget: Duration,
2033 initial_backoff: Duration,
2034 max_backoff: Duration,
2035 jitter_percent: u64,
2036}
2037
2038const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
2039 budget: ATTACH_RETRY_BUDGET,
2040 initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
2041 max_backoff: ATTACH_RETRY_MAX_BACKOFF,
2042 jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
2043};
2044
2045fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
2048 let transient = match error {
2049 SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
2050 SubcError::Auth { source, .. } => match source {
2051 subc_transport::AuthError::Timeout { .. }
2052 | subc_transport::AuthError::UnexpectedEof { .. } => true,
2053 subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
2054 _ => false,
2055 },
2056 _ => false,
2057 };
2058 if transient {
2059 AttachErrorClass::Transient
2060 } else {
2061 AttachErrorClass::Permanent
2062 }
2063}
2064
2065fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
2066 matches!(
2067 kind,
2068 io::ErrorKind::ConnectionRefused
2069 | io::ErrorKind::TimedOut
2070 | io::ErrorKind::ConnectionReset
2071 | io::ErrorKind::ConnectionAborted
2072 | io::ErrorKind::BrokenPipe
2073 | io::ErrorKind::UnexpectedEof
2074 )
2075}
2076
2077async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
2081 connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
2082}
2083
2084async fn connect_and_authenticate_with_policy(
2085 connection_file_path: &Path,
2086 policy: AttachRetryPolicy,
2087) -> Result<TcpStream, SubcError> {
2088 let started_at = Instant::now();
2089 let deadline = started_at + policy.budget;
2090 let mut attempt = 0_u32;
2091 let mut backoff = policy.initial_backoff;
2092 let mut history = Vec::new();
2093
2094 loop {
2095 attempt = attempt.saturating_add(1);
2096 let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2097 Ok(stream) => return Ok(stream),
2098 Err(error) => error,
2099 };
2100 let class = classify_attach_error(&error);
2101 let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2102 history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2103
2104 if class == AttachErrorClass::Permanent {
2105 log_attach_final_failure(started_at.elapsed(), &history);
2106 return Err(error);
2107 }
2108
2109 let remaining = deadline.saturating_duration_since(Instant::now());
2110 if remaining.is_zero() {
2111 log_attach_final_failure(started_at.elapsed(), &history);
2112 return Err(error);
2113 }
2114
2115 let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2116 log::info!(
2117 "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2118 );
2119 tokio::time::sleep(delay).await;
2120
2121 if Instant::now() >= deadline {
2122 log_attach_final_failure(started_at.elapsed(), &history);
2123 return Err(error);
2124 }
2125 backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2126 }
2127}
2128
2129fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2130 let jitter_percent = jitter_percent.min(100);
2131 if jitter_percent == 0 {
2132 return base;
2133 }
2134
2135 let mut random_bytes = [0_u8; 8];
2136 let random = if getrandom::fill(&mut random_bytes).is_ok() {
2137 u64::from_le_bytes(random_bytes)
2138 } else {
2139 let timestamp = std::time::SystemTime::now()
2140 .duration_since(std::time::UNIX_EPOCH)
2141 .unwrap_or_default()
2142 .subsec_nanos();
2143 u64::from(timestamp) ^ u64::from(attempt)
2144 };
2145 let span = jitter_percent.saturating_mul(2).saturating_add(1);
2146 let multiplier_percent = 100 - jitter_percent + random % span;
2147 let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2148 Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2149}
2150
2151fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2152 log::error!(
2153 "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2154 history.len(),
2155 history.join(" | ")
2156 );
2157}
2158
2159async fn connect_and_authenticate_once(
2160 connection_file_path: &Path,
2161 deadline: Instant,
2162) -> Result<TcpStream, SubcError> {
2163 let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2166 SubcError::ConnectionFile {
2167 path: connection_file_path.to_path_buf(),
2168 source,
2169 }
2170 })?;
2171
2172 let endpoint = conn
2173 .endpoints
2174 .first()
2175 .ok_or_else(|| SubcError::NoEndpoint {
2176 path: connection_file_path.to_path_buf(),
2177 })?;
2178 let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2179 let ip = endpoint
2180 .host
2181 .parse::<IpAddr>()
2182 .map_err(|_| SubcError::InvalidEndpoint {
2183 path: connection_file_path.to_path_buf(),
2184 endpoint: endpoint_label.clone(),
2185 })?;
2186 let addr = SocketAddr::new(ip, endpoint.port);
2187
2188 let connect_budget = deadline.saturating_duration_since(Instant::now());
2189 let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2190 .await
2191 .map_err(|_| SubcError::Connect {
2192 endpoint: endpoint_label.clone(),
2193 source: io::Error::new(
2194 io::ErrorKind::TimedOut,
2195 "initial subc attach retry budget elapsed during TCP connect",
2196 ),
2197 })?
2198 .map_err(|source| SubcError::Connect {
2199 endpoint: endpoint_label.clone(),
2200 source,
2201 })?;
2202 stream
2203 .set_nodelay(true)
2204 .map_err(|source| SubcError::Connect {
2205 endpoint: endpoint_label.clone(),
2206 source,
2207 })?;
2208
2209 let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2210 authenticate_client(&mut stream, &conn, auth_budget)
2211 .await
2212 .map_err(|source| SubcError::Auth {
2213 endpoint: endpoint_label,
2214 source,
2215 })?;
2216
2217 Ok(stream)
2218}
2219
2220#[allow(clippy::too_many_arguments)]
2221async fn process_route_bind_completion(
2222 writer_tx: &WriterSender,
2223 completion: RouteBindCompletion,
2224 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2225 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2226 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2227 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2228 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2229 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2230 installed_route_epochs: &mut HashMap<u16, u32>,
2231 executor: &Arc<Executor>,
2232 shutdown: &Arc<Notify>,
2233 metrics: &Arc<DispatchPathMetrics>,
2234) -> Result<(), SubcError> {
2235 decrement_counted_channel(&metrics.control_completion_queued);
2236 handle_route_bind_completion(
2237 writer_tx,
2238 completion,
2239 routes,
2240 root_channels,
2241 session_identity,
2242 push_buffer,
2243 live_roots,
2244 pending_binds,
2245 installed_route_epochs,
2246 executor,
2247 shutdown,
2248 metrics,
2249 )
2250 .await
2251}
2252
2253#[allow(clippy::too_many_arguments)]
2254async fn drain_pending_route_bind_completions(
2255 control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2256 writer_tx: &WriterSender,
2257 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2258 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2259 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2260 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2261 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2262 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2263 installed_route_epochs: &mut HashMap<u16, u32>,
2264 executor: &Arc<Executor>,
2265 shutdown: &Arc<Notify>,
2266 metrics: &Arc<DispatchPathMetrics>,
2267) -> Result<usize, SubcError> {
2268 let mut drained = 0;
2269 while let Ok(completion) = control_completion_rx.try_recv() {
2270 process_route_bind_completion(
2271 writer_tx,
2272 completion,
2273 routes,
2274 root_channels,
2275 session_identity,
2276 push_buffer,
2277 live_roots,
2278 pending_binds,
2279 installed_route_epochs,
2280 executor,
2281 shutdown,
2282 metrics,
2283 )
2284 .await?;
2285 drained += 1;
2286 }
2287 Ok(drained)
2288}
2289
2290async fn run_module_loop<R, W>(
2294 mut read: R,
2295 mut write: W,
2296 connection_file_path: &Path,
2297 shared_app: Arc<App>,
2298 executor: Arc<Executor>,
2299 dispatch: DispatchFn,
2300 user_config_path: Option<PathBuf>,
2301 allow_native_passthrough: bool,
2302 tool_response_body_limit: usize,
2303) -> Result<ModuleLoopExit, SubcError>
2304where
2305 R: AsyncRead + Unpin + Send + 'static,
2306 W: AsyncWrite + Unpin + Send + 'static,
2307{
2308 let hello = ModuleHelloBody {
2312 manifest: build_manifest(),
2313 protocol_ver: PROTOCOL_VERSION,
2314 control_ops: control_ops(),
2315 launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2316 };
2317 let hello_frame = Frame::build(
2318 FrameType::Hello,
2319 control_flags(),
2320 0,
2321 0,
2322 HELLO_CORR,
2323 serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2324 )
2325 .map_err(SubcError::FrameBuild)?;
2326 write_frame(&mut write, &hello_frame)
2327 .await
2328 .map_err(SubcError::FrameIo)?;
2329
2330 match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2332 None => return Err(SubcError::ClosedBeforeHelloAck),
2333 Some(frame) => match frame.header.ty {
2334 FrameType::HelloAck => {
2335 log::info!("subc attach: registered (HelloAck received)");
2336 }
2337 FrameType::Error => {
2338 let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2339 return Err(SubcError::HelloRejected { body });
2340 }
2341 other => return Err(SubcError::UnexpectedFrame { ty: other }),
2342 },
2343 }
2344
2345 let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2346 let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2347 let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2348 let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2355 let reader_task = spawn_reader_task(read, reader_tx);
2356 let shutdown = Arc::new(Notify::new());
2357 let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2366 let mut next_maintenance_at = next_drain_at;
2367 #[cfg(any(target_os = "macos", target_os = "linux"))]
2370 let mut last_slack_relief: Option<std::time::Instant> = None;
2371 let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2372 let (bash_deferred_tx, mut bash_deferred_rx) =
2373 mpsc::channel::<bash::BashDeferredCompletion>(256);
2374 let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2375 let (control_completion_tx, mut control_completion_rx) =
2376 mpsc::channel::<RouteBindCompletion>(256);
2377 let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2378 let lossy_overflow = Arc::new(push::LossyOverflow::default());
2379 let lossy_seq = Arc::new(AtomicU64::new(0));
2380 let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2381 let (fleet_status_client, fleet_status_task) =
2382 spawn_fleet_status_dial(connection_file_path, 64);
2383 let push_senders = PushSenders {
2384 lossy_tx,
2385 reliable_tx,
2386 lossy_overflow: Arc::clone(&lossy_overflow),
2387 lossy_seq,
2388 fleet_status_client: fleet_status_client.clone(),
2389 };
2390 let connection_cancel = PersistentCancelSignal::new();
2391 let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2392 let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2393 let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2394 let mut bg_sub_by_session: BgSubsBySession = HashMap::new();
2395 let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2396 let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2397 let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2398 let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2399 HashMap::new();
2400 let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2401 let mut retry_buffer: RetryBuffer = HashMap::new();
2402 let mut reclaimed_routes = ReclaimedRoutes::default();
2403 let mut completed_tasks = push::CompletedTaskIds::default();
2404 let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2405 let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2406 let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2407 let mut next_bash_ask_corr: u64 = 1;
2408 let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2409 let health_rollup_cache = HealthRollupCache::new();
2410 health_rollup_cache.refresh(&executor, &shared_app);
2411 let mut next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2412
2413 let loop_result: Result<ModuleLoopExit, SubcError> = loop {
2414 shared_app.set_open_route_count(routes.len());
2415 if tokio::time::Instant::now() >= next_health_rollup_at {
2416 health_rollup_cache.refresh(&executor, &shared_app);
2417 next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2418 }
2419 crate::logging::perf_tick(Some(&executor));
2420 dispatch_path_metrics.mark_frame_loop_tick();
2421 if let Err(error) = expire_pending_bash_asks(
2422 &writer_tx,
2423 &mut pending_bash_asks,
2424 &routes,
2425 &mut live_roots,
2426 &mut route_bash_cancels,
2427 &shutdown,
2428 &dispatch_path_metrics,
2429 )
2430 .await
2431 {
2432 break Err(error);
2433 }
2434
2435 match drain_pending_route_bind_completions(
2439 &mut control_completion_rx,
2440 &writer_tx,
2441 &mut routes,
2442 &mut root_channels,
2443 &mut session_identity,
2444 &mut push_buffer,
2445 &mut live_roots,
2446 &mut pending_binds,
2447 &mut installed_route_epochs,
2448 &executor,
2449 &shutdown,
2450 &dispatch_path_metrics,
2451 )
2452 .await
2453 {
2454 Ok(drained) => {
2455 if drained > 0 {
2456 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2457 health_rollup_cache.refresh(&executor, &shared_app);
2458 next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2459 }
2460 }
2461 Err(error) => break Err(error),
2462 }
2463
2464 if tokio::time::Instant::now() >= next_drain_at {
2465 push::emit_bg_event_wakes(
2466 &writer_tx,
2467 &dispatch_path_metrics,
2468 &bg_subs,
2469 &mut bg_wake_pending,
2470 );
2471 warn_slow_pending_binds(&mut pending_binds, &executor);
2472 if let Err(error) = expire_overdue_route_binds(
2473 &writer_tx,
2474 &executor,
2475 &mut pending_binds,
2476 &mut installed_route_epochs,
2477 &dispatch_path_metrics,
2478 )
2479 .await
2480 {
2481 break Err(error);
2482 }
2483
2484 let retried = push::drain_retry_buffers_for_bound_routes(
2485 &writer_tx,
2486 &dispatch_path_metrics,
2487 &routes,
2488 &mut retry_buffer,
2489 );
2490 if retried > 0 {
2491 log::debug!(
2492 "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2493 );
2494 }
2495
2496 next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2497 }
2498
2499 let overflow_batch = lossy_overflow.drain();
2505 if !overflow_batch.is_empty() {
2506 let (_, deferred) = push::drain_reliable_push_turn(
2507 &writer_tx,
2508 &dispatch_path_metrics,
2509 &routes,
2510 &root_channels,
2511 &session_identity,
2512 &mut retry_buffer,
2513 &mut push_buffer,
2514 &mut completed_tasks,
2515 &bg_sub_by_session,
2516 &mut bg_wake_pending,
2517 &mut bg_wake_epoch,
2518 &mut reliable_rx,
2519 None,
2520 );
2521 if deferred {
2522 tokio::task::yield_now().await;
2523 }
2524
2525 let mut batch = Vec::new();
2526 while let Ok(item) = lossy_rx.try_recv() {
2527 batch.push(item);
2528 }
2529 batch.extend(overflow_batch);
2530 push::process_lossy_push_envelope_batch(
2531 &writer_tx,
2532 &dispatch_path_metrics,
2533 &routes,
2534 &root_channels,
2535 &completed_tasks,
2536 batch,
2537 );
2538 }
2539
2540 tokio::select! {
2541 biased;
2542 Some(completion) = control_completion_rx.recv() => {
2543 if let Err(error) = process_route_bind_completion(
2544 &writer_tx,
2545 completion,
2546 &mut routes,
2547 &mut root_channels,
2548 &mut session_identity,
2549 &mut push_buffer,
2550 &mut live_roots,
2551 &mut pending_binds,
2552 &mut installed_route_epochs,
2553 &executor,
2554 &shutdown,
2555 &dispatch_path_metrics,
2556 )
2557 .await
2558 {
2559 break Err(error);
2560 }
2561 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2562 next_health_rollup_at = tokio::time::Instant::now();
2563 }
2564 _ = shutdown.notified() => {
2565 log::warn!("subc attach: fatal executor response requested teardown");
2566 break Ok(ModuleLoopExit::SkipSearchFlush);
2567 }
2568 maybe_frame = reader_rx.recv() => {
2569 let frame = match maybe_frame {
2570 None => {
2571 log::info!("subc attach: daemon closed connection");
2572 break Ok(ModuleLoopExit::Graceful);
2573 }
2574 Some(Err(error)) => break Err(error),
2575 Some(Ok(frame)) => frame,
2576 };
2577 let phase_trace = frame.phase_trace;
2578 let frame = frame.frame;
2579
2580 if !ingress_route_should_be_processed(
2581 &installed_route_epochs,
2582 &reclaimed_routes,
2583 &frame,
2584 ) {
2585 log::debug!(
2586 "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2587 frame.header.ty,
2588 frame.header.channel,
2589 frame.header.epoch
2590 );
2591 continue;
2592 }
2593
2594 match frame.header.ty {
2595 FrameType::Ping if frame.header.channel == 0 => {
2596 let pong = match Frame::build_with_version(
2597 frame.header.ver,
2598 FrameType::Pong,
2599 frame.header.flags,
2600 0,
2601 0,
2602 frame.header.corr,
2603 Vec::new(),
2604 ) {
2605 Ok(pong) => pong,
2606 Err(error) => break Err(SubcError::FrameBuild(error)),
2607 };
2608 if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2609 break Err(error);
2610 }
2611 }
2612 FrameType::Goodbye if frame.header.channel == 0 => {
2613 log::info!("subc attach: received channel-0 Goodbye");
2614 break Ok(ModuleLoopExit::Graceful);
2615 }
2616 FrameType::Goodbye => {
2617 let channel = route_key(frame.header.channel, frame.header.epoch);
2618 if let Err(error) = teardown_installed_route(
2619 &writer_tx,
2620 &dispatch_path_metrics,
2621 &executor,
2622 channel,
2623 "Goodbye",
2624 None,
2625 &mut installed_route_epochs,
2626 &mut routes,
2627 &mut root_channels,
2628 &mut bg_subs,
2629 &mut bg_sub_by_session,
2630 &mut bg_wake_pending,
2631 &mut pending_bash_asks,
2632 &mut live_roots,
2633 &mut route_bash_cancels,
2634 &mut pending_binds,
2635 &mut retry_buffer,
2636 &mut push_buffer,
2637 &shutdown,
2638 )
2639 .await
2640 {
2641 break Err(error);
2642 }
2643 }
2644 FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2645 if let Err(error) = handle_bash_elicitation_reply(
2646 &writer_tx,
2647 &frame,
2648 &mut pending_bash_asks,
2649 &routes,
2650 &mut live_roots,
2651 &executor,
2652 &shutdown,
2653 &bash_deferred_tx,
2654 &bash_poll_touch_tx,
2655 &dispatch_path_metrics,
2656 &mut route_bash_cancels,
2657 dispatch,
2658 )
2659 .await
2660 {
2661 break Err(error);
2662 }
2663 }
2664 FrameType::Request if frame.header.channel == 0 => {
2665 if let Err(error) = handle_control_request(
2666 &writer_tx,
2667 &frame,
2668 &shared_app,
2669 &executor,
2670 &mut live_roots,
2671 &mut pending_binds,
2672 &mut installed_route_epochs,
2673 &mut routes,
2674 &mut root_channels,
2675 &mut bg_subs,
2676 &mut bg_sub_by_session,
2677 &mut bg_wake_pending,
2678 &mut pending_bash_asks,
2679 &mut route_bash_cancels,
2680 &mut retry_buffer,
2681 &mut push_buffer,
2682 &shutdown,
2683 &control_completion_tx,
2684 &dispatch_path_metrics,
2685 &health_rollup_cache,
2686 &push_senders,
2687 dispatch,
2688 user_config_path.as_deref(),
2689 )
2690 .await
2691 {
2692 break Err(error);
2693 }
2694 }
2695 FrameType::Request => {
2696 if let Err(error) = handle_tool_call(
2697 &writer_tx,
2698 &frame,
2699 phase_trace,
2700 &routes,
2701 &pending_binds,
2702 &mut live_roots,
2703 &executor,
2704 &shutdown,
2705 &connection_cancel,
2706 &bash_deferred_tx,
2707 &bash_poll_touch_tx,
2708 &dispatch_path_metrics,
2709 &mut route_bash_cancels,
2710 &mut pending_bash_asks,
2711 &mut next_bash_ask_corr,
2712 &mut bg_subs,
2713 &mut bg_sub_by_session,
2714 &mut bg_wake_pending,
2715 &mut bg_wake_epoch,
2716 dispatch,
2717 allow_native_passthrough,
2718 tool_response_body_limit,
2719 )
2720 .await
2721 {
2722 break Err(error);
2723 }
2724 }
2725 FrameType::Cancel => {
2726 let channel = route_key(frame.header.channel, frame.header.epoch);
2727 if bg_subs.contains_key(&channel) {
2728 if let Err(error) = end_bg_subscription(
2729 &writer_tx,
2730 &dispatch_path_metrics,
2731 &mut bg_subs,
2732 &mut bg_sub_by_session,
2733 &mut bg_wake_pending,
2734 channel,
2735 routes.get(&channel),
2736 "cancel",
2737 )
2738 .await
2739 {
2740 break Err(error);
2741 }
2742 }
2743 if let Err(error) = cancel_pending_bash_ask_for_tool_call(
2744 &writer_tx,
2745 &mut pending_bash_asks,
2746 channel,
2747 frame.header.corr,
2748 &routes,
2749 &mut live_roots,
2750 &mut route_bash_cancels,
2751 &shutdown,
2752 &dispatch_path_metrics,
2753 )
2754 .await
2755 {
2756 break Err(error);
2757 }
2758 }
2759 _ => {}
2764 }
2765 }
2766 Some((root_id, frame)) = reliable_rx.recv() => {
2767 let (_, deferred) = push::drain_reliable_push_turn(
2771 &writer_tx,
2772 &dispatch_path_metrics,
2773 &routes,
2774 &root_channels,
2775 &session_identity,
2776 &mut retry_buffer,
2777 &mut push_buffer,
2778 &mut completed_tasks,
2779 &bg_sub_by_session,
2780 &mut bg_wake_pending,
2781 &mut bg_wake_epoch,
2782 &mut reliable_rx,
2783 Some((root_id, frame)),
2784 );
2785 if deferred {
2786 tokio::task::yield_now().await;
2787 }
2788 }
2789 Some((order, root_id, frame)) = lossy_rx.recv() => {
2790 let (_, deferred) = push::drain_reliable_push_turn(
2794 &writer_tx,
2795 &dispatch_path_metrics,
2796 &routes,
2797 &root_channels,
2798 &session_identity,
2799 &mut retry_buffer,
2800 &mut push_buffer,
2801 &mut completed_tasks,
2802 &bg_sub_by_session,
2803 &mut bg_wake_pending,
2804 &mut bg_wake_epoch,
2805 &mut reliable_rx,
2806 None,
2807 );
2808 if deferred {
2809 tokio::task::yield_now().await;
2810 }
2811
2812 let mut batch = vec![(order, root_id, frame)];
2819 while let Ok(item) = lossy_rx.try_recv() {
2820 batch.push(item);
2821 }
2822 batch.extend(lossy_overflow.drain());
2823 push::process_lossy_push_envelope_batch(
2824 &writer_tx,
2825 &dispatch_path_metrics,
2826 &routes,
2827 &root_channels,
2828 &completed_tasks,
2829 batch,
2830 );
2831 }
2832 Some(done) = bash_deferred_rx.recv() => {
2833 decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
2834 if let Err(error) = bash::handle_bash_deferred_completion(
2835 &writer_tx,
2836 done,
2837 &routes,
2838 &mut live_roots,
2839 &mut route_bash_cancels,
2840 &shutdown,
2841 &dispatch_path_metrics,
2842 )
2843 .await
2844 {
2845 break Err(error);
2846 }
2847 }
2848 Some(root_id) = bash_poll_touch_rx.recv() => {
2849 decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
2850 if let Some(meta) = live_roots.get_mut(&root_id) {
2851 meta.note_activity();
2852 }
2853 }
2854 Some(completion) = maintenance_rx.recv() => {
2855 decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
2856 let root_id = completion.root_id.clone();
2857 let response = completion.response;
2858 let response_is_fatal = response_is_fatal_panic(&response);
2859 let bind_pending = pending_binds
2860 .values()
2861 .any(|pending| pending.bind_root_id == root_id);
2862 let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
2863 let defer_requeue = meta.unbound_quiesced || bind_pending;
2864 note_maintenance_completion(
2865 meta,
2866 completion.requeue_kind,
2867 response_is_fatal,
2868 defer_requeue,
2869 );
2870 should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
2871 } else {
2872 false
2873 };
2874 if requiesce {
2875 quiesce_unbound_root(&root_id, &mut live_roots, &executor);
2876 }
2877 push::clear_stale_bg_wakes_for_empty_sessions(
2878 &root_id,
2879 &completion.empty_bg_sessions,
2880 &bg_sub_by_session,
2881 &mut bg_wake_pending,
2882 &bg_wake_epoch,
2883 );
2884 if response_is_fatal {
2885 if let Some(meta) = live_roots.get_mut(&root_id) {
2886 meta.maintenance_poisoned = true;
2887 }
2888 log::warn!(
2889 "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
2890 );
2891 }
2892 }
2893 _ = tokio::time::sleep_until(next_drain_at) => {
2894 }
2897 _ = tokio::time::sleep_until(next_maintenance_at) => {
2898 crate::logging::maybe_sweep_logs();
2903 let reaped_lsp_children = shared_app
2904 .lsp_child_registry()
2905 .reap_children_with_gone_cwd_or_reclaimed_root();
2906 if reaped_lsp_children > 0 {
2907 log::warn!(
2908 "subc attach: reaped {reaped_lsp_children} LSP child process group(s) with a deleted cwd or reclaimed root"
2909 );
2910 }
2911 let reap = reap_idle_roots(
2912 Instant::now(),
2913 &mut live_roots,
2914 &pending_binds,
2915 &root_channels,
2916 &executor,
2917 &dispatch_path_metrics,
2918 );
2919 for root_id in &reap.forgotten_deleted_roots {
2920 purge_deleted_root_residents(
2921 root_id,
2922 &mut routes,
2923 &mut root_channels,
2924 &mut installed_route_epochs,
2925 &mut route_bash_cancels,
2926 &mut retry_buffer,
2927 &mut reclaimed_routes,
2928 &mut session_identity,
2929 &mut push_buffer,
2930 &mut bg_subs,
2931 &mut bg_sub_by_session,
2932 &mut bg_wake_pending,
2933 &mut bg_wake_epoch,
2934 &mut pending_bash_asks,
2935 &dispatch_path_metrics,
2936 );
2937 }
2938 if reap.evicted > 0 {
2939 log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
2940 }
2941 submit_due_maintenance_jobs(
2942 &executor,
2943 &mut live_roots,
2944 &pending_binds,
2945 &bg_sub_by_session,
2946 &bg_wake_pending,
2947 &bg_wake_epoch,
2948 &maintenance_tx,
2949 &dispatch_path_metrics,
2950 );
2951 #[cfg(any(target_os = "macos", target_os = "linux"))]
2958 {
2959 let now_std = std::time::Instant::now();
2960 if crate::memory::spawn_allocator_slack_relief_if_due(
2961 last_slack_relief,
2962 now_std,
2963 ) {
2964 last_slack_relief = Some(now_std);
2965 }
2966 }
2967 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2968 }
2969 }
2970 };
2971
2972 shared_app.set_open_route_count(0);
2973
2974 connection_cancel.cancel();
2975 quiesce_connection_roots(
2978 &mut live_roots,
2979 &mut pending_binds,
2980 &mut routes,
2981 &mut root_channels,
2982 &mut installed_route_epochs,
2983 &mut route_bash_cancels,
2984 &executor,
2985 );
2986
2987 fleet_status_client.set_route_live(false);
2988 fleet_status_task.abort();
2989 let _ = fleet_status_task.await;
2990
2991 let mut loop_result = loop_result;
2992 if !pending_bash_asks.is_empty() {
2993 let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2994 if let Err(error) = settle_all_pending_bash_asks(
2995 &writer_tx,
2996 &mut pending_bash_asks,
2997 &no_routes,
2998 &mut live_roots,
2999 &mut route_bash_cancels,
3000 &shutdown,
3001 &dispatch_path_metrics,
3002 )
3003 .await
3004 {
3005 loop_result = loop_result.and(Err(error));
3006 }
3007 }
3008
3009 reader_task.abort();
3012 drop(writer_tx);
3013 let writer_result = finish_writer_task(writer_task).await;
3014 loop_result.and_then(|exit| writer_result.map(|_| exit))
3015}
3016
3017fn spawn_writer_task<W>(
3018 mut write: W,
3019 mut rx: mpsc::Receiver<WriterFrame>,
3020 metrics: Arc<DispatchPathMetrics>,
3021) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
3022where
3023 W: AsyncWrite + Unpin + Send + 'static,
3024{
3025 tokio::spawn(async move {
3026 let mut write_buffer = Vec::new();
3027 while let Some(mut queued) = rx.recv().await {
3028 let measure = queued.tool_response_trace.is_some();
3029 let dequeued = measure.then(Instant::now);
3030 metrics.writer_active.store(true, Ordering::Relaxed);
3031 decrement_counted_channel(&metrics.writer_queued);
3032 let write_timing = write_frame_contiguous(
3033 &mut write,
3034 queued.frame(),
3035 queued.body(),
3036 &mut write_buffer,
3037 measure,
3038 )
3039 .await;
3040 metrics.writer_active.store(false, Ordering::Relaxed);
3041 let write_timing = write_timing?;
3042
3043 if let (Some(trace), Some(dequeued), Some(write_timing)) =
3044 (queued.tool_response_trace.take(), dequeued, write_timing)
3045 {
3046 if let Some(completed) = trace.finish(
3047 dequeued,
3048 write_timing.write_started,
3049 write_timing.write_finished,
3050 write_timing.frame_bytes,
3051 ) {
3052 log_ctx::with_session(Some(completed.session), || {
3053 crate::logging::note_tool_call_trace(
3054 &completed.name,
3055 &completed.root,
3056 completed.channel,
3057 completed.corr,
3058 completed.phases,
3059 );
3060 });
3061 }
3062 }
3063 }
3064 Ok(())
3065 })
3066}
3067
3068struct FrameWriteTiming {
3069 write_started: Instant,
3070 write_finished: Instant,
3071 frame_bytes: usize,
3072}
3073
3074async fn write_frame_contiguous<W>(
3078 writer: &mut W,
3079 frame: &Frame,
3080 body: &[u8],
3081 buffer: &mut Vec<u8>,
3082 measure: bool,
3083) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
3084where
3085 W: AsyncWrite + Unpin,
3086{
3087 if frame.header.len as usize != body.len() {
3088 return Err(subc_transport::FrameIoError::BodyLengthMismatch {
3089 header_len: frame.header.len,
3090 body_len: body.len(),
3091 });
3092 }
3093
3094 let header = frame.header.encode();
3095 buffer.clear();
3096 buffer.reserve(header.len() + body.len());
3097 buffer.extend_from_slice(&header);
3098 buffer.extend_from_slice(body);
3099 let write_started = measure.then(Instant::now);
3100 writer
3101 .write_all(buffer)
3102 .await
3103 .map_err(subc_transport::FrameIoError::Io)?;
3104 Ok(write_started.map(|write_started| FrameWriteTiming {
3105 write_started,
3106 write_finished: Instant::now(),
3107 frame_bytes: buffer.len(),
3108 }))
3109}
3110
3111fn spawn_reader_task<R>(
3112 mut read: R,
3113 tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3114) -> JoinHandle<()>
3115where
3116 R: AsyncRead + Unpin + Send + 'static,
3117{
3118 tokio::spawn(async move {
3119 loop {
3120 match read_frame(&mut read).await {
3121 Ok(Some(frame)) => {
3122 let decoded = DecodedFrame {
3123 frame,
3124 phase_trace: PhaseTrace::new(Instant::now()),
3125 };
3126 if tx.send(Ok(decoded)).await.is_err() {
3127 return;
3128 }
3129 }
3130 Ok(None) => {
3131 return;
3133 }
3134 Err(error) => {
3135 if let subc_transport::FrameIoError::Io(io_error) = &error {
3142 if matches!(
3143 io_error.kind(),
3144 std::io::ErrorKind::ConnectionReset
3145 | std::io::ErrorKind::ConnectionAborted
3146 ) {
3147 log::info!(
3148 "subc attach: connection reset by daemon; treating as close"
3149 );
3150 return;
3151 }
3152 }
3153 let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3154 return;
3155 }
3156 }
3157 }
3158 })
3159}
3160
3161async fn finish_writer_task(
3162 mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3163) -> Result<(), SubcError> {
3164 match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3165 Ok(Ok(Ok(()))) => Ok(()),
3166 Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3167 Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3168 Err(_) => {
3169 writer_task.abort();
3170 Ok(())
3171 }
3172 }
3173}
3174
3175fn register_actor_for_bind(
3176 shared_app: &Arc<App>,
3177 executor: &Arc<Executor>,
3178 push_senders: &PushSenders,
3179 bind_root_id: &ProjectRootId,
3180 route_channel: u16,
3181 root_was_live: bool,
3182) -> bool {
3183 if executor.actor_registered(bind_root_id) {
3184 log::debug!(
3185 "subc attach: reusing actor for route {} root {}",
3186 route_channel,
3187 bind_root_id.as_path().display()
3188 );
3189 return false;
3190 }
3191
3192 if root_was_live {
3193 log::warn!(
3194 "subc attach: recreating missing actor for live root {} on route {}",
3195 bind_root_id.as_path().display(),
3196 route_channel
3197 );
3198 }
3199
3200 let actor_ctx = Arc::new(AppContext::from_app(
3201 Arc::clone(shared_app),
3202 Config::default(),
3203 ));
3204 install_bash_compressor(&actor_ctx);
3205 actor_ctx.install_fleet_status_client(Some(push_senders.fleet_status_client.clone()));
3206 actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
3207 push_senders.clone(),
3208 bind_root_id.clone(),
3209 )));
3210 let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
3211 drop(actor_ctx);
3212 if inserted {
3213 log::debug!(
3217 "subc attach: registered actor for route {} root {}",
3218 route_channel,
3219 bind_root_id.as_path().display()
3220 );
3221 } else {
3222 log::debug!(
3223 "subc attach: actor appeared while binding route {} root {}; reusing it",
3224 route_channel,
3225 bind_root_id.as_path().display()
3226 );
3227 }
3228 inserted
3229}
3230
3231fn rollback_pending_bind_actor(
3232 executor: &Arc<Executor>,
3233 live_roots: &HashMap<ProjectRootId, RootMeta>,
3234 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3235 root_id: &ProjectRootId,
3236 inserted_new_actor: bool,
3237) {
3238 if !inserted_new_actor || live_roots.contains_key(root_id) {
3239 return;
3240 }
3241
3242 if let Some((route, pending)) = pending_binds
3243 .iter_mut()
3244 .find(|(_, pending)| &pending.bind_root_id == root_id)
3245 {
3246 pending.inserted_new_actor = true;
3247 log::debug!(
3248 "subc attach: transferred rollback ownership for root {} to pending route {}",
3249 root_id.as_path().display(),
3250 route
3251 );
3252 return;
3253 }
3254
3255 executor.remove_actor(root_id);
3256}
3257
3258fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
3259 match response.data.get("code").and_then(|code| code.as_str()) {
3260 Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
3265 Some("cache_key_probe_failed") => "cache_key_probe_failed",
3269 Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
3273 _ => "config_divergence",
3274 }
3275}
3276
3277fn queue_post_bind_configure_and_completion_maintenance(
3278 root_id: &ProjectRootId,
3279 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3280) {
3281 let Some(meta) = live_roots.get_mut(root_id) else {
3282 return;
3283 };
3284 if meta.maintenance_poisoned || meta.maintenance_pending {
3285 return;
3286 }
3287
3288 meta.maintenance_pending = true;
3289 meta.maintenance_queued_kinds
3290 .push_back(MaintenanceDrainKind::ConfigureTail);
3291 meta.maintenance_queued_kinds
3292 .push_back(MaintenanceDrainKind::CompletionDrains);
3293}
3294
3295#[allow(clippy::too_many_arguments)]
3296async fn handle_route_bind_completion(
3297 tx: &WriterSender,
3298 completion: RouteBindCompletion,
3299 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3300 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3301 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
3302 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3303 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3304 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3305 installed_route_epochs: &mut HashMap<u16, u32>,
3306 executor: &Arc<Executor>,
3307 shutdown: &Arc<Notify>,
3308 metrics: &Arc<DispatchPathMetrics>,
3309) -> Result<(), SubcError> {
3310 let route_id = completion.route;
3311 let Some(pending) = pending_binds.remove(&route_id) else {
3312 log::warn!(
3313 "subc attach: dropping RouteBind completion for non-pending route {}",
3314 completion.route
3315 );
3316 rollback_pending_bind_actor(
3317 executor,
3318 live_roots,
3319 pending_binds,
3320 &completion.bind_root_id,
3321 completion.inserted_new_actor,
3322 );
3323 let has_pending_bind = pending_binds
3324 .values()
3325 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3326 if !root_channels
3327 .get(&completion.bind_root_id)
3328 .is_some_and(|channels| !channels.is_empty())
3329 && !has_pending_bind
3330 {
3331 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3332 }
3333 remove_installed_route(installed_route_epochs, route_id);
3334 return Ok(());
3335 };
3336
3337 if pending.bind_root_id != completion.bind_root_id {
3338 log::warn!(
3339 "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
3340 completion.route,
3341 pending.bind_root_id.as_path().display(),
3342 completion.bind_root_id.as_path().display()
3343 );
3344 }
3345
3346 let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
3347 if pending.cancelled {
3348 rollback_pending_bind_actor(
3349 executor,
3350 live_roots,
3351 pending_binds,
3352 &completion.bind_root_id,
3353 inserted_new_actor,
3354 );
3355 let has_pending_bind = pending_binds
3356 .values()
3357 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3358 if !root_channels
3359 .get(&completion.bind_root_id)
3360 .is_some_and(|channels| !channels.is_empty())
3361 && !has_pending_bind
3362 {
3363 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3364 }
3365 log::debug!(
3366 "subc attach: discarded completed RouteBind for cancelled route {} root {}",
3367 completion.route,
3368 completion.bind_root_id.as_path().display()
3369 );
3370 remove_installed_route(installed_route_epochs, route_id);
3371 return Ok(());
3372 }
3373
3374 let failure = if !completion.configure_response.success {
3375 Some((
3376 &completion.configure_response,
3377 "configure failed during route bind",
3378 ))
3379 } else {
3380 None
3381 };
3382
3383 if let Some((response, fallback)) = failure {
3384 rollback_pending_bind_actor(
3385 executor,
3386 live_roots,
3387 pending_binds,
3388 &completion.bind_root_id,
3389 inserted_new_actor,
3390 );
3391 let has_pending_bind = pending_binds
3392 .values()
3393 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3394 if !root_channels
3395 .get(&completion.bind_root_id)
3396 .is_some_and(|channels| !channels.is_empty())
3397 && !has_pending_bind
3398 {
3399 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3400 }
3401 let message = response_message(response, fallback);
3402 let fatal = response_is_fatal_panic(response);
3403 let error_code = route_bind_error_code_for_configure_response(response);
3404 send_route_bind_error_parts(
3405 tx,
3406 completion.ver,
3407 completion.corr,
3408 completion.flags,
3409 error_code,
3410 &message,
3411 metrics,
3412 )
3413 .await?;
3414 remove_installed_route(installed_route_epochs, route_id);
3415 if fatal {
3416 signal_fatal_teardown(
3417 tx,
3418 Some(completion.route),
3419 completion.ver,
3420 completion.corr,
3421 shutdown,
3422 metrics,
3423 )
3424 .await;
3425 }
3426 return Ok(());
3427 }
3428
3429 remember_session_identity(session_identity, &completion.identity);
3430 let replay_key = push::ReplayKey::from_identity(&completion.identity);
3431 let bind_trust = completion.identity.trust;
3432 insert_route_channel(routes, root_channels, route_id, completion.identity);
3433 let restore_watcher = live_roots
3434 .get(&completion.bind_root_id)
3435 .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3436 live_roots
3437 .entry(completion.bind_root_id.clone())
3438 .and_modify(|meta| {
3439 meta.reactivate_bound();
3440 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3441 meta.maintenance_poisoned = false;
3442 })
3443 .or_insert_with(|| RootMeta::new(Instant::now()));
3444 if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3445 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3446 meta.maintenance_poisoned = false;
3447 }
3448 if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3449 ctx.mark_subc_bound();
3450 if restore_watcher {
3451 crate::commands::configure::ensure_project_watcher(&ctx);
3452 }
3453 }
3454
3455 let ack =
3456 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3457 let response = Frame::build_with_version(
3458 completion.ver,
3459 FrameType::Response,
3460 control_flags(),
3461 0,
3462 0,
3463 completion.corr,
3464 ack,
3465 )
3466 .map_err(SubcError::FrameBuild)?;
3467 send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3468 queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3469 let replayed = push::replay_buffered_push_frames(
3470 tx,
3471 metrics,
3472 route_id,
3473 push_buffer,
3474 &replay_key,
3475 bind_trust,
3476 );
3477 if replayed > 0 {
3478 log::debug!(
3479 "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3480 replayed,
3481 completion.route,
3482 replay_key.root.as_path().display(),
3483 replay_key.harness,
3484 replay_key.session
3485 );
3486 }
3487 log::info!(
3488 "subc attach: route {} bound to root {}",
3489 completion.route,
3490 completion.bind_root_id.as_path().display()
3491 );
3492 Ok(())
3493}
3494
3495async fn expire_overdue_route_binds(
3496 tx: &WriterSender,
3497 executor: &Arc<Executor>,
3498 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3499 installed_route_epochs: &mut HashMap<u16, u32>,
3500 metrics: &DispatchPathMetrics,
3501) -> Result<(), SubcError> {
3502 let now = Instant::now();
3503 let expired: Vec<_> = pending_binds
3504 .iter()
3505 .filter_map(|(route, pending)| {
3506 let age = now.saturating_duration_since(pending.started_at);
3507 (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3508 (
3509 *route,
3510 pending.corr,
3511 pending.ver,
3512 pending.flags,
3513 pending.bind_root_id.clone(),
3514 pending.configure_request_id.clone(),
3515 age,
3516 )
3517 })
3518 })
3519 .collect();
3520
3521 for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3522 if let Some(pending) = pending_binds.get_mut(&route) {
3523 pending.cancelled = true;
3524 pending.deadline_reported = true;
3525 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3526 log::debug!(
3527 "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3528 );
3529 }
3530 remove_installed_route(installed_route_epochs, route);
3531 let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3532 let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3533 send_route_bind_error_parts(
3534 tx,
3535 ver,
3536 corr,
3537 flags,
3538 "actor_not_ready",
3539 &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3540 metrics,
3541 )
3542 .await?;
3543 log::warn!(
3544 "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3545 route,
3546 root_id.as_path().display(),
3547 deadline_ms,
3548 configure_request_id
3549 );
3550 }
3551
3552 Ok(())
3553}
3554
3555#[allow(clippy::too_many_arguments)]
3560async fn handle_control_request(
3561 tx: &WriterSender,
3562 frame: &Frame,
3563 shared_app: &Arc<App>,
3564 executor: &Arc<Executor>,
3565 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3566 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3567 installed_route_epochs: &mut HashMap<u16, u32>,
3568 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3569 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3570 bg_subs: &mut HashMap<RouteChannel, BgSub>,
3571 bg_sub_by_session: &mut BgSubsBySession,
3572 bg_wake_pending: &mut HashSet<RouteChannel>,
3573 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3574 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3575 retry_buffer: &mut RetryBuffer,
3576 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3577 shutdown: &Arc<Notify>,
3578 control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3579 metrics: &Arc<DispatchPathMetrics>,
3580 health_rollup_cache: &HealthRollupCache,
3581 push_senders: &PushSenders,
3582 dispatch: DispatchFn,
3583 user_config_path: Option<&Path>,
3584) -> Result<(), SubcError> {
3585 let request =
3586 serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3587 match request {
3588 ModuleControlRequest::RouteBind {
3589 route_channel,
3590 epoch,
3591 target: _,
3592 identity,
3593 principal,
3594 consumer_capabilities,
3595 admission_facts: _,
3596 } => {
3597 let route_id = route_key(route_channel, epoch);
3598 if epoch == 0 {
3599 return send_route_bind_error(
3600 tx,
3601 frame,
3602 "config_divergence",
3603 "route bind uses an invalid channel generation",
3604 metrics,
3605 )
3606 .await;
3607 }
3608 let mut bind_root_id = None;
3609 if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3610 if installed_epoch >= epoch {
3611 return send_route_bind_error(
3612 tx,
3613 frame,
3614 "config_divergence",
3615 "route bind generation is not newer than the installed generation",
3616 metrics,
3617 )
3618 .await;
3619 }
3620
3621 let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3622 Ok(root_id) => root_id,
3623 Err(error) => {
3624 return send_route_bind_error(
3625 tx,
3626 frame,
3627 "config_divergence",
3628 &format!("invalid route project root: {error}"),
3629 metrics,
3630 )
3631 .await;
3632 }
3633 };
3634 teardown_installed_route(
3635 tx,
3636 metrics,
3637 executor,
3638 route_key(route_channel, installed_epoch),
3639 "higher-epoch RouteBind",
3640 Some(&replacement_root),
3641 installed_route_epochs,
3642 routes,
3643 root_channels,
3644 bg_subs,
3645 bg_sub_by_session,
3646 bg_wake_pending,
3647 pending_bash_asks,
3648 live_roots,
3649 route_bash_cancels,
3650 pending_binds,
3651 retry_buffer,
3652 push_buffer,
3653 shutdown,
3654 )
3655 .await?;
3656 bind_root_id = Some(replacement_root);
3657 }
3658 if pending_binds.contains_key(&route_id) {
3659 return send_route_bind_error(
3660 tx,
3661 frame,
3662 "config_divergence",
3663 "route bind is already pending for channel",
3664 metrics,
3665 )
3666 .await;
3667 }
3668 let bind_root_id = match bind_root_id {
3669 Some(root_id) => root_id,
3670 None => match ProjectRootId::from_path(&identity.project_root) {
3671 Ok(root_id) => root_id,
3672 Err(error) => {
3673 return send_route_bind_error(
3674 tx,
3675 frame,
3676 "config_divergence",
3677 &format!("invalid route project root: {error}"),
3678 metrics,
3679 )
3680 .await;
3681 }
3682 },
3683 };
3684
3685 let request_id = format!("subc-bind-{route_channel}");
3688 let bind_project_root = identity.project_root.clone();
3689 let bind_harness = identity.harness.clone();
3690 let bind_session = identity.session.clone();
3691 let bind_trust = trust_for_bind(&bind_harness, &principal);
3692 let bind_principal_id = principal_id(&principal);
3693 let consumer_elicitation_capable = consumer_capabilities
3698 .as_ref()
3699 .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
3700 log::info!(
3701 "subc attach: route {} harness={} principal={} trust={} elicitation={}",
3702 route_channel,
3703 bind_harness,
3704 principal_label(&principal),
3705 bind_trust.label(),
3706 consumer_elicitation_capable
3707 );
3708
3709 let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
3721 user_config_path,
3722 Path::new(&bind_project_root),
3723 );
3724 let config_tiers: Vec<Value> = local_tiers
3725 .iter()
3726 .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
3727 .collect();
3728 let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
3729 let configure_json = json!({
3730 "id": request_id,
3731 "command": "configure",
3732 "project_root": bind_project_root,
3733 "harness": bind_harness,
3734 "session_id": bind_session.clone(),
3735 "config": config_tiers,
3736 });
3737 let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
3738 Ok(req) => req,
3739 Err(error) => {
3740 return send_route_bind_error(
3741 tx,
3742 frame,
3743 "config_divergence",
3744 &format!("failed to build configure request: {error}"),
3745 metrics,
3746 )
3747 .await;
3748 }
3749 };
3750
3751 let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
3752 root: bind_root_id.clone(),
3753 project_root: PathBuf::from(&bind_project_root),
3754 harness: bind_harness.clone(),
3755 session: bind_session.clone(),
3756 trust: bind_trust,
3757 spawn_principal: AuthenticatedPrincipal::RouteBind {
3758 trust: bind_trust.sandbox_trust(),
3759 route_channel,
3760 route_epoch: epoch,
3761 project_root: PathBuf::from(&bind_project_root),
3762 harness: bind_harness.clone(),
3763 session_id: bind_session.clone(),
3764 principal_id: bind_principal_id,
3765 },
3766 consumer_elicitation_capable,
3767 }));
3768 let configure_session = route_identity.session.clone();
3769 let root_was_live = live_roots.contains_key(&bind_root_id);
3770 let inserted_new_actor = register_actor_for_bind(
3771 shared_app,
3772 executor,
3773 push_senders,
3774 &bind_root_id,
3775 route_channel,
3776 root_was_live,
3777 );
3778
3779 let configure_request_id = configure_req.id.clone();
3780 installed_route_epochs.insert(route_channel, epoch);
3781 if let Some(meta) = live_roots.get_mut(&bind_root_id) {
3782 meta.maintenance_queued_kinds.clear();
3783 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
3784 }
3785 let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
3786 bind_root_id.clone(),
3787 Lane::Mutating,
3788 configure_request_id.clone(),
3789 Box::new(move |ctx| {
3790 log_ctx::with_session(Some(configure_session.clone()), || {
3791 dispatch(configure_req, ctx)
3792 })
3793 }),
3794 );
3795 pending_binds.insert(
3796 route_id,
3797 PendingBind {
3798 bind_root_id: bind_root_id.clone(),
3799 inserted_new_actor,
3800 cancelled: false,
3801 configure_request_id: configure_request_id.clone(),
3802 started_at: Instant::now(),
3803 warned_half_deadline: false,
3804 deadline_reported: false,
3805 corr: frame.header.corr,
3806 ver: frame.header.ver,
3807 flags: frame.header.flags,
3808 cancellation: configure_cancellation,
3809 },
3810 );
3811
3812 let completion_tx = control_completion_tx.clone();
3813 let completion_identity = route_identity;
3814 let completion_root = bind_root_id.clone();
3815 let completion_route_channel = route_channel;
3816 let completion_ver = frame.header.ver;
3817 let completion_corr = frame.header.corr;
3818 let completion_flags = frame.header.flags;
3819 let completion_metrics = Arc::clone(metrics);
3820 tokio::spawn(async move {
3821 let _response_task = ResponseTaskGuard::new(&completion_metrics);
3822 let configure_response =
3823 await_executor_response(configure_rx, configure_request_id.clone()).await;
3824 let completion = RouteBindCompletion {
3829 route: route_key(completion_route_channel, epoch),
3830 identity: completion_identity,
3831 bind_root_id: completion_root,
3832 inserted_new_actor,
3833 configure_response,
3834 diagnostics_on_edit,
3835 ver: completion_ver,
3836 corr: completion_corr,
3837 flags: completion_flags,
3838 };
3839 if send_counted_channel(
3840 &completion_tx,
3841 &completion_metrics.control_completion_queued,
3842 completion,
3843 )
3844 .await
3845 .is_err()
3846 {
3847 log::debug!(
3848 "subc attach: dropped RouteBind completion for route {} after loop exit",
3849 completion_route_channel
3850 );
3851 }
3852 });
3853
3854 health_rollup_cache.refresh(executor, shared_app);
3855 Ok(())
3856 }
3857 ModuleControlRequest::HealthCheck {} => {
3858 metrics.record_bg_runtime(bg_subs.len(), bg_wake_pending.len());
3859 let report = build_health_report(
3860 health_rollup_cache,
3861 executor,
3862 pending_binds,
3863 metrics,
3864 shared_app,
3865 );
3866 let body = serde_json::to_vec(&ModuleControlResponse::from(report))
3867 .map_err(SubcError::Json)?;
3868 let response = Frame::build_with_version(
3869 frame.header.ver,
3870 FrameType::Response,
3871 frame.header.flags,
3872 0,
3873 0,
3874 frame.header.corr,
3875 body,
3876 )
3877 .map_err(SubcError::FrameBuild)?;
3878 send_frame(tx, metrics, response).await
3879 }
3880 }
3881}
3882
3883fn install_bash_compressor(ctx: &AppContext) {
3884 let filter_registry_handle = ctx.shared_filter_registry();
3886 let compress_flag = ctx.bash_compress_flag();
3887 ctx.bash_background().set_compressor_with_exit_code(
3888 move |command: &str, output: String, exit_code: Option<i32>| {
3889 if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
3890 return crate::compress::CompressionResult::new(output);
3891 }
3892 let registry_guard = match filter_registry_handle.read() {
3893 Ok(g) => g,
3894 Err(poisoned) => poisoned.into_inner(),
3895 };
3896 crate::compress::compress_with_registry_exit_code(
3897 command,
3898 &output,
3899 exit_code,
3900 ®istry_guard,
3901 )
3902 },
3903 );
3904}
3905
3906fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
3907 let mut diagnostics_on_edit = false;
3908 for tier in tiers {
3909 if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
3910 diagnostics_on_edit = value;
3911 }
3912 }
3913 diagnostics_on_edit
3914}
3915
3916fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
3917 let stripped = strip_jsonc(doc);
3918 let value = serde_json::from_str::<Value>(&stripped).ok()?;
3919 value
3920 .get("lsp")
3921 .and_then(Value::as_object)?
3922 .get("diagnostics_on_edit")
3923 .and_then(Value::as_bool)
3924}
3925
3926async fn send_route_bind_error(
3927 tx: &WriterSender,
3928 frame: &Frame,
3929 code: &str,
3930 message: &str,
3931 metrics: &DispatchPathMetrics,
3932) -> Result<(), SubcError> {
3933 send_route_bind_error_parts(
3934 tx,
3935 frame.header.ver,
3936 frame.header.corr,
3937 frame.header.flags,
3938 code,
3939 message,
3940 metrics,
3941 )
3942 .await
3943}
3944
3945async fn send_route_bind_error_parts(
3946 tx: &WriterSender,
3947 ver: u8,
3948 corr: u64,
3949 flags: Flags,
3950 code: &str,
3951 message: &str,
3952 metrics: &DispatchPathMetrics,
3953) -> Result<(), SubcError> {
3954 let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
3955 send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
3956 log_route_bind_rejection(code, message);
3957 Ok(())
3958}
3959
3960fn log_route_bind_rejection(code: &str, message: &str) {
3968 const WINDOW: Duration = Duration::from_secs(60);
3969 static SUPPRESSED: OnceLock<StdMutex<HashMap<String, (Instant, u64)>>> = OnceLock::new();
3970 let map = SUPPRESSED.get_or_init(|| StdMutex::new(HashMap::new()));
3971 let mut map = match map.try_lock() {
3972 Ok(map) => map,
3973 Err(_) => {
3975 log::warn!("subc attach: route bind rejected ({code}): {message}");
3976 return;
3977 }
3978 };
3979 let now = Instant::now();
3980 if map.len() > 512 {
3984 map.retain(|_, (start, _)| now.duration_since(*start) < WINDOW);
3985 }
3986 match map.get_mut(message) {
3987 Some((window_start, suppressed)) if now.duration_since(*window_start) < WINDOW => {
3988 *suppressed += 1;
3989 }
3990 Some((window_start, suppressed)) => {
3991 if *suppressed > 0 {
3992 log::warn!(
3993 "subc attach: route bind rejected ({code}): {message} (repeated {}x in last 60s)",
3994 *suppressed
3995 );
3996 } else {
3997 log::warn!("subc attach: route bind rejected ({code}): {message}");
3998 }
3999 *window_start = now;
4000 *suppressed = 0;
4001 }
4002 None => {
4003 log::warn!("subc attach: route bind rejected ({code}): {message}");
4004 map.insert(message.to_string(), (now, 0));
4005 }
4006 }
4007}
4008
4009async fn handle_tool_call(
4014 tx: &WriterSender,
4015 frame: &Frame,
4016 mut phase_trace: PhaseTrace,
4017 routes: &HashMap<RouteChannel, RouteIdentity>,
4018 pending_binds: &HashMap<RouteChannel, PendingBind>,
4019 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4020 executor: &Arc<Executor>,
4021 shutdown: &Arc<Notify>,
4022 connection_cancel: &PersistentCancelSignal,
4023 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
4024 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
4025 metrics: &Arc<DispatchPathMetrics>,
4026 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
4027 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
4028 next_bash_ask_corr: &mut u64,
4029 bg_subs: &mut HashMap<RouteChannel, BgSub>,
4030 bg_sub_by_session: &mut BgSubsBySession,
4031 bg_wake_pending: &mut HashSet<RouteChannel>,
4032 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
4033 dispatch: DispatchFn,
4034 allow_native_passthrough: bool,
4035 tool_response_body_limit: usize,
4036) -> Result<(), SubcError> {
4037 let route_id = route_key(frame.header.channel, frame.header.epoch);
4038 if pending_binds.contains_key(&route_id) {
4039 let error = build_error_frame(
4040 frame.header.ver,
4041 frame.header.channel,
4042 frame.header.epoch,
4043 frame.header.corr,
4044 frame.header.flags,
4045 "route_not_bound",
4046 "route is not bound before tool call",
4047 )?;
4048 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
4049 }
4050
4051 let Some(identity) = routes.get(&route_id).cloned() else {
4052 let error = build_error_frame(
4053 frame.header.ver,
4054 frame.header.channel,
4055 frame.header.epoch,
4056 frame.header.corr,
4057 frame.header.flags,
4058 "route_not_bound",
4059 "route is not bound before tool call",
4060 )?;
4061 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
4062 };
4063 let restore_watcher = live_roots
4064 .get(&identity.root)
4065 .is_some_and(|meta| meta.idle_artifacts_evicted);
4066 if let Some(meta) = live_roots.get_mut(&identity.root) {
4067 meta.reactivate_bound();
4068 }
4069 if restore_watcher {
4070 if let Some(ctx) = executor.actor_context(&identity.root) {
4071 crate::commands::configure::ensure_project_watcher(&ctx);
4072 }
4073 }
4074
4075 let route_request =
4076 serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
4077 if matches!(
4078 route_request,
4079 RouteRequest::BgEvents(BgEventsRequest {
4080 op: BgEventsOp::BgEvents
4081 })
4082 ) {
4083 if let Some(old_sub) = bg_subs.get(&route_id).cloned() {
4084 metrics.record_bg_subscription_ended(
4085 &old_sub.root,
4086 &old_sub.session,
4087 route_id,
4088 "resubscribe",
4089 );
4090 push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
4091 }
4092 if !identity.trust.allows_bash_observation() {
4093 bg_subs.remove(&route_id);
4094 bg_wake_pending.remove(&route_id);
4095 remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
4096 let denied_sub = BgSub {
4097 corr: frame.header.corr,
4098 ver: frame.header.ver,
4099 flags: frame.header.flags,
4100 root: identity.root.clone(),
4101 session: identity.session.clone(),
4102 };
4103 metrics.record_bg_subscription_ended(
4104 &identity.root,
4105 &identity.session,
4106 route_id,
4107 "subscribe-denied",
4108 );
4109 push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
4110 return Ok(());
4111 }
4112 bg_subs.insert(
4113 route_id,
4114 BgSub {
4115 corr: frame.header.corr,
4116 ver: frame.header.ver,
4117 flags: frame.header.flags,
4118 root: identity.root.clone(),
4119 session: identity.session.clone(),
4120 },
4121 );
4122 insert_bg_subscription_index(
4123 bg_sub_by_session,
4124 identity.root.clone(),
4125 identity.session.clone(),
4126 route_id,
4127 );
4128 metrics.record_bg_subscription_installed(&identity.root, &identity.session, route_id);
4129 push::arm_bg_wake(
4130 identity.root.clone(),
4131 identity.session.clone(),
4132 route_id,
4133 bg_wake_pending,
4134 bg_wake_epoch,
4135 metrics,
4136 );
4137 return Ok(());
4138 }
4139
4140 let RouteRequest::ToolCall(call) = route_request else {
4141 unreachable!("background event subscription returned above")
4142 };
4143 let bare_name = call.name;
4144 let arguments = strip_agent_preview_arg_owned(call.arguments);
4145 let format_context = crate::subc_format::FormatContext::from_tool_call(
4146 &bare_name,
4147 &arguments,
4148 identity.project_root.as_path(),
4149 );
4150
4151 let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
4152 let bind_trust = identity.trust;
4153 let diagnostics_on_edit = live_roots
4154 .get(&identity.root)
4155 .map(|meta| meta.diagnostics_on_edit)
4156 .unwrap_or(false);
4157
4158 let requests_host = bare_name == "bash"
4159 && arguments
4160 .get("sandbox")
4161 .or_else(|| {
4162 arguments
4163 .get("params")
4164 .and_then(|params| params.get("sandbox"))
4165 })
4166 .and_then(Value::as_str)
4167 == Some("host");
4168 if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
4169 let response = Response::error(
4170 request_id.clone(),
4171 "sandbox_escalation_denied",
4172 "sandbox host escalation is unavailable to untrusted principals",
4173 );
4174 let text = crate::subc_format::format_response_with_context(
4175 &bare_name,
4176 &response,
4177 &format_context,
4178 );
4179 let result = ToolCallResult { text, response };
4180 let response_frame = build_tool_response_frame_with_limit(
4181 frame.header.ver,
4182 route_id,
4183 frame.header.corr,
4184 frame.header.flags,
4185 &result,
4186 bind_trust,
4187 tool_response_body_limit,
4188 )?;
4189 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4190 }
4191
4192 if matches!(bind_trust, BindTrust::Untrusted)
4193 && is_bash_family_tool(&bare_name)
4194 && (bare_name != "bash" || !identity.consumer_elicitation_capable)
4195 {
4196 let response = bash::bash_denied_untrusted_response(request_id.clone());
4197 let text = crate::subc_format::format_response_with_context(
4198 &bare_name,
4199 &response,
4200 &format_context,
4201 );
4202 let result = ToolCallResult { text, response };
4203 let response_frame = build_tool_response_frame_with_limit(
4204 frame.header.ver,
4205 route_id,
4206 frame.header.corr,
4207 frame.header.flags,
4208 &result,
4209 bind_trust,
4210 tool_response_body_limit,
4211 )?;
4212 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4213 }
4214
4215 if !is_subc_agent_core_tool(&bare_name)
4223 && !is_subc_native_plumbing_tool(&bare_name)
4224 && !allow_native_passthrough
4225 {
4226 log::warn!(
4227 "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
4228 bare_name,
4229 frame.header.channel
4230 );
4231 let response = Response::error(
4232 request_id.clone(),
4233 "unknown_tool",
4234 format!("tool {:?} is not in the AFT tool manifest", bare_name),
4235 );
4236 let text = crate::subc_format::format_response_with_context(
4237 &bare_name,
4238 &response,
4239 &format_context,
4240 );
4241 let result = ToolCallResult { text, response };
4242 let response_frame = build_tool_response_frame_with_limit(
4243 frame.header.ver,
4244 route_id,
4245 frame.header.corr,
4246 frame.header.flags,
4247 &result,
4248 bind_trust,
4249 tool_response_body_limit,
4250 )?;
4251 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4252 }
4253
4254 if bare_name == "bash" {
4255 if matches!(bind_trust, BindTrust::Untrusted) {
4256 let plan = match bash::prepare_bash_elicitation_plan(
4257 &arguments,
4258 identity.project_root.as_path(),
4259 ) {
4260 Ok(plan) => plan,
4261 Err(error) => {
4262 let response = Response::error(request_id.clone(), error.code, error.message);
4263 let text = crate::subc_format::format_response_with_context(
4264 &bare_name,
4265 &response,
4266 &format_context,
4267 );
4268 let result = ToolCallResult { text, response };
4269 let response_frame = build_tool_response_frame_with_limit(
4270 frame.header.ver,
4271 route_id,
4272 frame.header.corr,
4273 frame.header.flags,
4274 &result,
4275 bind_trust,
4276 tool_response_body_limit,
4277 )?;
4278 return send_reliable_writer_frame(
4279 tx,
4280 metrics,
4281 response_frame,
4282 "tool response",
4283 )
4284 .await;
4285 }
4286 };
4287
4288 let reverse_corr =
4289 allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
4290 let ask_frame = build_bash_elicitation_request_frame(
4291 frame.header.ver,
4292 route_id,
4293 reverse_corr,
4294 frame.header.flags,
4295 &plan.command,
4296 &plan.asks,
4297 )?;
4298
4299 let meta = live_roots
4300 .entry(identity.root.clone())
4301 .or_insert_with(|| RootMeta::new(Instant::now()));
4302 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4303 meta.reactivate_bound();
4304
4305 let route_cancel =
4306 route_bash_cancels
4307 .entry(route_id)
4308 .or_insert_with(|| bash::RouteBashCancel {
4309 token: PersistentCancelSignal::new(),
4310 active_waits: 0,
4311 });
4312 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4313 let cancel = bash::BashWaitCancel {
4314 connection: connection_cancel.clone(),
4315 route: route_cancel.token.clone(),
4316 };
4317 pending_bash_asks.insert(
4318 ReverseCorrKey {
4319 route: route_id,
4320 corr: reverse_corr,
4321 },
4322 PendingBashAsk {
4323 route: route_id,
4324 tool_corr: frame.header.corr,
4325 tool_flags: frame.header.flags,
4326 tool_ver: frame.header.ver,
4327 root: identity.root.clone(),
4328 project_root: identity.project_root.clone(),
4329 session_id: identity.session.clone(),
4330 spawn_principal: identity.spawn_principal.clone(),
4331 edit_slot_survives: call.edit_slot_survives,
4332 request_id,
4333 arguments,
4334 format_context,
4335 cancel,
4336 grants: plan.grants,
4337 expires_at: Instant::now() + bash_elicitation_timeout(),
4338 },
4339 );
4340 return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
4341 .await;
4342 }
4343
4344 let meta = live_roots
4345 .entry(identity.root.clone())
4346 .or_insert_with(|| RootMeta::new(Instant::now()));
4347 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4348 meta.reactivate_bound();
4349
4350 let route_cancel =
4351 route_bash_cancels
4352 .entry(route_id)
4353 .or_insert_with(|| bash::RouteBashCancel {
4354 token: PersistentCancelSignal::new(),
4355 active_waits: 0,
4356 });
4357 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4358 let cancel = bash::BashWaitCancel {
4359 connection: connection_cancel.clone(),
4360 route: route_cancel.token.clone(),
4361 };
4362
4363 bash::submit_deferred_bash(
4364 executor,
4365 bash_deferred_tx,
4366 bash_poll_touch_tx,
4367 metrics,
4368 dispatch,
4369 identity.root.clone(),
4370 identity.project_root.clone(),
4371 identity.session.clone(),
4372 request_id,
4373 route_id,
4374 frame.header.corr,
4375 frame.header.flags,
4376 frame.header.ver,
4377 arguments,
4378 format_context,
4379 cancel,
4380 bind_trust,
4381 identity.spawn_principal.clone(),
4382 call.edit_slot_survives,
4383 None,
4384 );
4385 return Ok(());
4386 }
4387
4388 let lane = command_lane(&bare_name);
4389 let tool_call_context = ToolCallContext {
4390 project_root: identity.project_root.clone(),
4391 session_id: Some(identity.session.clone()),
4392 request_id: request_id.clone(),
4393 diagnostics_on_edit,
4394 preview: call.preview,
4395 edit_slot_survives: call.edit_slot_survives,
4396 report_registration_downgrade: true,
4397 };
4398 let bare_name_for_frame = bare_name.clone();
4399 let identity_for_run = identity.clone();
4400 let completion_session = identity.session.clone();
4401 let completion_root = identity.project_root.clone();
4402 let request_id_for_force = request_id.clone();
4403 let format_context_for_frame = format_context.clone();
4404 let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
4405 phase_trace.mark_executor_submitted();
4406 let rx = executor.submit_async(
4407 identity.root.clone(),
4408 lane,
4409 request_id.clone(),
4410 Box::new(move |ctx| {
4411 phase_trace.mark_job_admitted();
4412 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4413 let run = || {
4414 let finalizer = |response: &mut Response| {
4415 crate::response_finalize::finalize_response_with_bg_completions(
4416 response,
4417 ctx,
4418 &identity_for_run.session,
4419 &bare_name,
4420 bind_trust.allows_bash_observation(),
4421 );
4422 };
4423 match run_tool_call(
4424 &bare_name,
4425 arguments,
4426 &format_context,
4427 &tool_call_context,
4428 ctx,
4429 &dispatch,
4430 Some(&finalizer),
4431 Some(&mut phase_trace),
4432 ) {
4433 ToolCallOutcome::Unary(result) => {
4434 let response = result.response;
4435 let _ = tool_call_tx.send(ToolCallCompletion {
4436 text: result.text,
4437 phase_trace,
4438 });
4439 response
4440 }
4441 }
4442 };
4443 if matches!(bind_trust, BindTrust::Untrusted) {
4444 ctx.with_force_restrict(&request_id_for_force, run)
4445 } else {
4446 run()
4447 }
4448 })
4449 }),
4450 );
4451 let completion_tx = tx.clone();
4452 let completion_shutdown = Arc::clone(shutdown);
4453 let route = route_id;
4454 let corr = frame.header.corr;
4455 let flags = frame.header.flags;
4456 let ver = frame.header.ver;
4457 let completion_metrics = Arc::clone(metrics);
4458 tokio::spawn(async move {
4459 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4460 let response = await_executor_response(rx, request_id.clone()).await;
4461 let (text, phase_trace) = match tool_call_rx.await {
4462 Ok(completion) => (completion.text, Some(completion.phase_trace)),
4463 Err(_) => (
4464 crate::subc_format::format_response_with_context(
4465 &bare_name_for_frame,
4466 &response,
4467 &format_context_for_frame,
4468 ),
4469 None,
4470 ),
4471 };
4472 let result = ToolCallResult { text, response };
4473 let fatal = response_is_fatal_panic(&result.response);
4474 match build_tool_response_frame_with_limit(
4475 ver,
4476 route,
4477 corr,
4478 flags,
4479 &result,
4480 bind_trust,
4481 tool_response_body_limit,
4482 ) {
4483 Ok(response_frame) => {
4484 let send_result = if let Some(phase_trace) = phase_trace {
4485 let trace = ToolResponseWriteTrace::new(
4486 phase_trace,
4487 bare_name_for_frame,
4488 completion_root,
4489 completion_session,
4490 route.channel,
4491 corr,
4492 );
4493 send_traced_tool_response_frame(
4494 &completion_tx,
4495 &completion_metrics,
4496 response_frame,
4497 trace,
4498 )
4499 .await
4500 } else {
4501 send_reliable_writer_frame(
4502 &completion_tx,
4503 &completion_metrics,
4504 response_frame,
4505 "tool response",
4506 )
4507 .await
4508 };
4509 if let Err(error) = send_result {
4510 log::warn!("subc attach: failed to queue tool response frame: {error}");
4511 }
4512 }
4513 Err(error) => {
4514 log::error!("subc attach: failed to build tool response frame: {error}");
4515 }
4516 }
4517 if fatal {
4518 signal_fatal_teardown(
4519 &completion_tx,
4520 Some(route),
4521 ver,
4522 corr,
4523 &completion_shutdown,
4524 &completion_metrics,
4525 )
4526 .await;
4527 }
4528 });
4529 Ok(())
4530}
4531
4532fn submit_maintenance_job(
4533 executor: &Arc<Executor>,
4534 root_id: ProjectRootId,
4535 kind: MaintenanceDrainKind,
4536 bg_sessions_to_check: Vec<(String, u64)>,
4537 completion_tx: &mpsc::Sender<MaintenanceCompletion>,
4538 metrics: &Arc<DispatchPathMetrics>,
4539) {
4540 let request_id = format!(
4541 "subc-maintenance-drain-{}-{}",
4542 kind.label(),
4543 root_id.as_path().to_string_lossy()
4544 );
4545 let response_id = request_id.clone();
4546 let completion_root_id = root_id.clone();
4547 let maintenance_generation = executor
4548 .actor_context(&root_id)
4549 .map(|ctx| ctx.configure_generation())
4550 .unwrap_or(0);
4551 let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
4552 let lane = match kind {
4557 MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
4558 MaintenanceDrainKind::Watcher
4559 | MaintenanceDrainKind::Lsp
4560 | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
4561 };
4562 let job: crate::executor::ExecutorJob = Box::new(move |ctx: &AppContext| {
4563 let outcome = match kind {
4564 MaintenanceDrainKind::Watcher => {
4565 let drained = runtime_drain::drain_watcher_events_bounded(
4566 ctx,
4567 runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
4568 );
4569 MaintenanceJobOutcome {
4570 empty_bg_sessions: Vec::new(),
4571 requeue_kind: drained.has_more.then_some(kind),
4572 }
4573 }
4574 MaintenanceDrainKind::Lsp => {
4575 let drained = runtime_drain::drain_lsp_events_bounded(
4576 ctx,
4577 runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
4578 );
4579 MaintenanceJobOutcome {
4580 empty_bg_sessions: Vec::new(),
4581 requeue_kind: drained.has_more.then_some(kind),
4582 }
4583 }
4584 MaintenanceDrainKind::ConfigureTail => {
4585 runtime_drain::drain_deferred_configure_maintenance(ctx);
4586 runtime_drain::drain_configure_warning_events(ctx);
4587 MaintenanceJobOutcome::default()
4588 }
4589 MaintenanceDrainKind::CompletionDrains => {
4590 runtime_drain::drain_search_index_events(ctx);
4591 runtime_drain::drain_callgraph_store_events(ctx);
4592 runtime_drain::drain_semantic_index_events(ctx);
4593 runtime_drain::drain_semantic_refresh_events(ctx);
4594 runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
4595 let empty_bg_sessions = bg_sessions_to_check
4596 .into_iter()
4597 .filter(|(session, _)| {
4598 !ctx.bash_background()
4599 .has_completions_for_session(Some(session.as_str()))
4600 })
4601 .collect();
4602 MaintenanceJobOutcome {
4603 empty_bg_sessions,
4604 requeue_kind: None,
4605 }
4606 }
4607 };
4608 let requeued = outcome.requeue_kind.is_some();
4609 let _ = outcome_tx.send(outcome);
4610 Response::success(
4611 response_id,
4612 json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
4613 )
4614 });
4615 let rx = match kind {
4616 MaintenanceDrainKind::Watcher => executor.submit_coalescable_maintenance_async(
4617 root_id,
4618 lane,
4619 request_id.clone(),
4620 crate::executor::MaintenanceCoalesceKey::WatcherDrain,
4621 job,
4622 ),
4623 MaintenanceDrainKind::Lsp => executor.submit_coalescable_maintenance_async(
4624 root_id,
4625 lane,
4626 request_id.clone(),
4627 crate::executor::MaintenanceCoalesceKey::LspDrain,
4628 job,
4629 ),
4630 MaintenanceDrainKind::ConfigureTail | MaintenanceDrainKind::CompletionDrains => {
4631 executor.submit_maintenance_async(root_id, lane, request_id.clone(), job)
4632 }
4633 };
4634 let completion_tx = completion_tx.clone();
4635 let completion_metrics = Arc::clone(metrics);
4636 tokio::spawn(async move {
4637 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4638 let response = await_executor_response(rx, request_id).await;
4639 let outcome = outcome_rx.await.unwrap_or_default();
4640 let _ = send_counted_channel(
4641 &completion_tx,
4642 &completion_metrics.maintenance_queued,
4643 MaintenanceCompletion {
4644 root_id: completion_root_id,
4645 kind,
4646 response,
4647 empty_bg_sessions: outcome.empty_bg_sessions,
4648 requeue_kind: outcome.requeue_kind,
4649 },
4650 )
4651 .await;
4652 });
4653}
4654
4655async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
4656 rx.await
4657 .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
4658}
4659async fn signal_fatal_teardown(
4660 tx: &WriterSender,
4661 route: Option<RouteChannel>,
4662 ver: u8,
4663 corr: u64,
4664 shutdown: &Arc<Notify>,
4665 metrics: &DispatchPathMetrics,
4666) {
4667 if let Some(route) = route {
4668 if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
4669 if let Err(error) = send_frame(tx, metrics, frame).await {
4670 log::warn!(
4671 "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
4672 );
4673 }
4674 }
4675 }
4676 if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
4677 if let Err(error) = send_frame(tx, metrics, frame).await {
4678 log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
4679 }
4680 }
4681 shutdown.notify_one();
4682}
4683#[derive(Debug, Deserialize)]
4684#[serde(untagged)]
4685enum RouteRequest {
4686 BgEvents(BgEventsRequest),
4687 ToolCall(ToolCallRequest),
4688}
4689
4690#[derive(Debug, Deserialize)]
4691struct BgEventsRequest {
4692 op: BgEventsOp,
4693}
4694
4695#[derive(Debug, Deserialize)]
4696#[serde(rename_all = "snake_case")]
4697enum BgEventsOp {
4698 BgEvents,
4699}
4700
4701#[derive(Debug, Deserialize)]
4702struct ToolCallRequest {
4703 name: String,
4704 #[serde(default)]
4705 arguments: Value,
4706 #[serde(default)]
4708 edit_slot_survives: Option<bool>,
4709 #[serde(default)]
4714 preview: bool,
4715}
4716
4717#[cfg(test)]
4718pub(crate) mod test_support {
4719 use super::*;
4720 use crate::bash_background::BgTaskStatus;
4721 use crate::protocol::{
4722 BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
4723 ProgressFrame, StatusChangedFrame,
4724 };
4725 use serde_json::json;
4726
4727 pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
4728 let dir = tempfile::Builder::new()
4729 .prefix(name)
4730 .tempdir()
4731 .expect("temp root");
4732 let root = ProjectRootId::from_path(dir.path()).expect("project root id");
4733 (dir, root)
4734 }
4735
4736 pub(super) fn test_ctx() -> Arc<AppContext> {
4737 Arc::new(AppContext::new(
4738 Box::new(crate::parser::TreeSitterProvider::new()),
4739 crate::config::Config::default(),
4740 ))
4741 }
4742
4743 pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
4744 let deadline = Instant::now() + Duration::from_secs(30);
4745 loop {
4746 let observed = ctx.watcher_registry_count();
4747 if observed == expected {
4748 return;
4749 }
4750 assert!(
4751 Instant::now() < deadline,
4752 "watcher count did not settle before deadline: expected={expected}, observed={observed}"
4753 );
4754 std::thread::sleep(Duration::from_millis(50));
4755 }
4756 }
4757
4758 pub(super) fn reap_until_forgotten(
4768 root: &ProjectRootId,
4769 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4770 pending_binds: &HashMap<RouteChannel, PendingBind>,
4771 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4772 executor: &Arc<Executor>,
4773 metrics: &DispatchPathMetrics,
4774 ) -> IdleReapOutcome {
4775 let deadline = Instant::now() + Duration::from_secs(30);
4776 loop {
4777 let outcome = reap_idle_roots(
4778 Instant::now(),
4779 live_roots,
4780 pending_binds,
4781 root_channels,
4782 executor,
4783 metrics,
4784 );
4785 if outcome.forgotten_deleted_roots.contains(root) {
4786 return outcome;
4787 }
4788 assert!(
4789 Instant::now() < deadline,
4790 "deleted root was never forgotten: {root:?}"
4791 );
4792 std::thread::sleep(Duration::from_millis(10));
4793 }
4794 }
4795
4796 pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
4797 let deadline = Instant::now() + Duration::from_secs(30);
4798 loop {
4799 let observed = app.actor_root_count();
4800 if observed == expected {
4801 return;
4802 }
4803 assert!(
4804 Instant::now() < deadline,
4805 "actor root count did not settle before deadline: expected={expected}, observed={observed}"
4806 );
4807 std::thread::sleep(Duration::from_millis(50));
4808 }
4809 }
4810
4811 pub(super) fn status_frame(seq: u64) -> PushFrame {
4812 status_frame_with_session(seq, None)
4813 }
4814
4815 pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
4816 PushFrame::StatusChanged(StatusChangedFrame {
4817 frame_type: "status_changed",
4818 session_id: session_id.map(str::to_string),
4819 snapshot: json!({ "seq": seq }),
4820 })
4821 }
4822
4823 pub(super) fn completion_frame(task_id: &str) -> PushFrame {
4824 completion_frame_with_session(task_id, "session-1")
4825 }
4826
4827 pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
4828 PushFrame::BashCompleted(BashCompletedFrame {
4829 frame_type: "bash_completed",
4830 task_id: task_id.to_string(),
4831 session_id: session_id.to_string(),
4832 status: BgTaskStatus::Completed,
4833 exit_code: Some(0),
4834 command: format!("echo {task_id}"),
4835 output_preview: String::new(),
4836 output_truncated: false,
4837 original_tokens: None,
4838 compressed_tokens: None,
4839 tokens_skipped: false,
4840 status_reason: None,
4841 })
4842 }
4843
4844 pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
4845 long_running_frame_with_session(task_id, "session-1", elapsed_ms)
4846 }
4847
4848 pub(super) fn long_running_frame_with_session(
4849 task_id: &str,
4850 session_id: &str,
4851 elapsed_ms: u64,
4852 ) -> PushFrame {
4853 PushFrame::BashLongRunning(BashLongRunningFrame {
4854 frame_type: "bash_long_running",
4855 task_id: task_id.to_string(),
4856 session_id: session_id.to_string(),
4857 command: format!("sleep {elapsed_ms}"),
4858 elapsed_ms,
4859 })
4860 }
4861
4862 pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
4863 PushFrame::BashPatternMatch(BashPatternMatchFrame {
4864 frame_type: "bash_pattern_match",
4865 task_id: "task-pattern".to_string(),
4866 session_id: session_id.to_string(),
4867 watch_id: "watch-1".to_string(),
4868 match_text: "needle".to_string(),
4869 match_offset: 7,
4870 context: "haystack needle".to_string(),
4871 once: true,
4872 reason: "pattern_match",
4873 })
4874 }
4875
4876 pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
4877 PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
4878 frame_type: "configure_warnings",
4879 session_id: session_id.map(str::to_string),
4880 project_root: "/tmp/subc-test".to_string(),
4881 warnings: Vec::new(),
4882 })
4883 }
4884
4885 pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
4886 route_identity_with_trust(root, session_id, BindTrust::FirstParty)
4887 }
4888
4889 pub(super) fn route_identity_with_trust(
4890 root: &ProjectRootId,
4891 session_id: &str,
4892 trust: BindTrust,
4893 ) -> RouteIdentity {
4894 RouteIdentity(Arc::new(RouteIdentityData {
4895 root: root.clone(),
4896 project_root: root.as_path().to_path_buf(),
4897 harness: "opencode".to_string(),
4898 session: session_id.to_string(),
4899 trust,
4900 spawn_principal: AuthenticatedPrincipal::RouteBind {
4901 trust: trust.sandbox_trust(),
4902 route_channel: 0,
4903 route_epoch: 0,
4904 project_root: root.as_path().to_path_buf(),
4905 harness: "opencode".to_string(),
4906 session_id: session_id.to_string(),
4907 principal_id: Some(match trust {
4908 BindTrust::FirstParty => "direct".to_string(),
4909 BindTrust::Untrusted => "unverified".to_string(),
4910 }),
4911 },
4912 consumer_elicitation_capable: false,
4913 }))
4914 }
4915
4916 pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
4917 PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
4918 }
4919
4920 pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
4921 match frame {
4922 PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
4923 _ => None,
4924 }
4925 }
4926
4927 pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
4928 match frame {
4929 PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
4930 _ => None,
4931 }
4932 }
4933
4934 pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
4935 let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
4936 body.get("task_id")
4937 .and_then(serde_json::Value::as_str)
4938 .map(str::to_string)
4939 }
4940}
4941
4942#[cfg(test)]
4943mod tests {
4944 use super::test_support::{
4945 completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
4946 wait_for_actor_root_count, wait_for_watcher_count,
4947 };
4948 use super::*;
4949 use crate::bash_background::BgTaskStatus;
4950
4951 fn attach_error(kind: io::ErrorKind) -> SubcError {
4952 SubcError::Connect {
4953 endpoint: "127.0.0.1:1".to_string(),
4954 source: io::Error::new(kind, "constructed attach failure"),
4955 }
4956 }
4957
4958 fn auth_io_error(kind: io::ErrorKind) -> SubcError {
4959 SubcError::Auth {
4960 endpoint: "127.0.0.1:1".to_string(),
4961 source: subc_transport::AuthError::Io {
4962 stage: subc_transport::AuthStage::ServerProof,
4963 source: io::Error::new(kind, "constructed auth failure"),
4964 },
4965 }
4966 }
4967
4968 #[test]
4969 fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
4970 let transient_errors = vec![
4971 attach_error(io::ErrorKind::ConnectionRefused),
4972 attach_error(io::ErrorKind::TimedOut),
4973 attach_error(io::ErrorKind::ConnectionReset),
4974 auth_io_error(io::ErrorKind::ConnectionAborted),
4975 auth_io_error(io::ErrorKind::BrokenPipe),
4976 SubcError::Auth {
4977 endpoint: "127.0.0.1:1".to_string(),
4978 source: subc_transport::AuthError::UnexpectedEof {
4979 stage: subc_transport::AuthStage::ServerProof,
4980 expected: 4,
4981 actual: 0,
4982 },
4983 },
4984 SubcError::Auth {
4985 endpoint: "127.0.0.1:1".to_string(),
4986 source: subc_transport::AuthError::Timeout {
4987 stage: subc_transport::AuthStage::ServerProof,
4988 deadline: AUTH_DEADLINE,
4989 },
4990 },
4991 ];
4992 for error in &transient_errors {
4993 assert_eq!(
4994 classify_attach_error(error),
4995 AttachErrorClass::Transient,
4996 "expected transient: {error}"
4997 );
4998 }
4999
5000 let permanent_errors = vec![
5001 attach_error(io::ErrorKind::PermissionDenied),
5002 auth_io_error(io::ErrorKind::InvalidData),
5003 SubcError::Auth {
5004 endpoint: "127.0.0.1:1".to_string(),
5005 source: subc_transport::AuthError::InvalidServerProof,
5006 },
5007 SubcError::Auth {
5008 endpoint: "127.0.0.1:1".to_string(),
5009 source: subc_transport::AuthError::DaemonIdMismatch,
5010 },
5011 SubcError::ConnectionFile {
5012 path: PathBuf::from("subc-connection.json"),
5013 source: subc_transport::ConnectionFileError::Invalid {
5014 reason: "constructed invalid file".to_string(),
5015 },
5016 },
5017 SubcError::NoEndpoint {
5018 path: PathBuf::from("subc-connection.json"),
5019 },
5020 SubcError::InvalidEndpoint {
5021 path: PathBuf::from("subc-connection.json"),
5022 endpoint: "not-an-ip:1234".to_string(),
5023 },
5024 ];
5025 for error in &permanent_errors {
5026 assert_eq!(
5027 classify_attach_error(error),
5028 AttachErrorClass::Permanent,
5029 "expected permanent: {error}"
5030 );
5031 }
5032 }
5033
5034 #[test]
5035 fn incompatible_wire_version_is_rejected_before_tcp_connect() {
5036 let conn_dir = tempfile::tempdir().expect("connection tempdir");
5037 let conn_path = conn_dir.path().join("subc-connection.json");
5038 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
5039 listener
5040 .set_nonblocking(true)
5041 .expect("set listener nonblocking");
5042 let port = listener.local_addr().expect("listener addr").port();
5043 connection_file::write_atomic(
5044 &conn_path,
5045 &connection_file::ConnectionInfo {
5046 schema: connection_file::SCHEMA_VERSION,
5047 wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
5048 endpoints: vec![connection_file::Endpoint {
5049 host: "127.0.0.1".to_string(),
5050 port,
5051 }],
5052 key: vec![0x42; subc_transport::KEY_LEN],
5053 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
5054 pid: std::process::id(),
5055 daemon_ver: "subc-test".to_string(),
5056 },
5057 )
5058 .expect("write connection file");
5059
5060 let runtime = tokio::runtime::Builder::new_current_thread()
5061 .enable_all()
5062 .build()
5063 .expect("test runtime");
5064 let result = runtime.block_on(connect_and_authenticate_with_policy(
5065 &conn_path,
5066 AttachRetryPolicy {
5067 budget: Duration::from_secs(1),
5068 initial_backoff: Duration::from_millis(5),
5069 max_backoff: Duration::from_millis(10),
5070 jitter_percent: 0,
5071 },
5072 ));
5073 assert!(matches!(
5074 result,
5075 Err(SubcError::ConnectionFile {
5076 source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
5077 ..
5078 })
5079 ));
5080 assert!(matches!(
5081 listener.accept(),
5082 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
5083 ));
5084 }
5085
5086 #[test]
5087 fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
5088 let conn_dir = tempfile::tempdir().expect("connection tempdir");
5089 let conn_path = conn_dir.path().join("subc-connection.json");
5090 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
5091 let port = listener.local_addr().expect("reserved addr").port();
5092 drop(listener);
5093 connection_file::write_atomic(
5094 &conn_path,
5095 &connection_file::ConnectionInfo {
5096 schema: connection_file::SCHEMA_VERSION,
5097 wire_version: Some(PROTOCOL_VERSION),
5098 endpoints: vec![connection_file::Endpoint {
5099 host: "127.0.0.1".to_string(),
5100 port,
5101 }],
5102 key: vec![0x42; subc_transport::KEY_LEN],
5103 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
5104 pid: std::process::id(),
5105 daemon_ver: "subc-test".to_string(),
5106 },
5107 )
5108 .expect("write connection file");
5109
5110 let policy = AttachRetryPolicy {
5111 budget: Duration::from_millis(40),
5112 initial_backoff: Duration::from_millis(5),
5113 max_backoff: Duration::from_millis(10),
5114 jitter_percent: 0,
5115 };
5116 let runtime = tokio::runtime::Builder::new_current_thread()
5117 .enable_all()
5118 .build()
5119 .expect("test runtime");
5120 let started_at = Instant::now();
5121 let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
5122 let elapsed = started_at.elapsed();
5123 let error = match result {
5124 Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
5125 Err(error) => error,
5126 };
5127
5128 assert!(matches!(error, SubcError::Connect { .. }), "{error}");
5129 assert!(
5130 elapsed >= Duration::from_millis(35),
5131 "retry budget ended too early: {elapsed:?}"
5132 );
5133 assert!(
5134 elapsed < Duration::from_secs(1),
5135 "retry budget was not bounded: {elapsed:?}"
5136 );
5137 }
5138
5139 fn due_maintenance_jobs_without_actor_context(
5140 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
5141 budget: usize,
5142 pending_bind_roots: &HashSet<ProjectRootId>,
5143 ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
5144 due_maintenance_jobs(
5145 live_roots,
5146 None,
5147 &HashMap::new(),
5148 &HashSet::new(),
5149 budget,
5150 pending_bind_roots,
5151 )
5152 }
5153
5154 fn actor_ctx_with_dirty_search_index(
5155 root: &Path,
5156 storage: &Path,
5157 file_name: &str,
5158 old_contents: &str,
5159 new_contents: &str,
5160 ) -> (Arc<AppContext>, PathBuf, PathBuf) {
5161 let file = root.join(file_name);
5162 std::fs::write(&file, old_contents).expect("write source");
5163 let canonical_root = std::fs::canonicalize(root).expect("canonical root");
5164 let ctx = Arc::new(AppContext::new(
5165 Box::new(crate::parser::TreeSitterProvider::new()),
5166 Config {
5167 project_root: Some(root.to_path_buf()),
5168 storage_dir: Some(storage.to_path_buf()),
5169 ..Config::default()
5170 },
5171 ));
5172 ctx.set_canonical_cache_root(canonical_root.clone());
5173
5174 let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
5175 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
5176 let git_head = index.stored_git_head().map(str::to_owned);
5177 index.write_to_disk(&cache_dir, git_head.as_deref());
5178
5179 std::fs::write(&file, new_contents).expect("edit source");
5180 index.update_file(&file);
5181 *ctx.search_index()
5182 .write()
5183 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
5184 (ctx, canonical_root, cache_dir)
5185 }
5186
5187 #[test]
5188 fn graceful_shutdown_flushes_every_actor_search_index() {
5189 let storage = tempfile::tempdir().expect("storage tempdir");
5190 let (root1_dir, root1) = test_root("shutdown-flush-root-1");
5191 let (root2_dir, root2) = test_root("shutdown-flush-root-2");
5192 let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
5193 root1_dir.path(),
5194 storage.path(),
5195 "alpha.txt",
5196 "old actor one token\n",
5197 "new actor one token\n",
5198 );
5199 let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
5200 root2_dir.path(),
5201 storage.path(),
5202 "beta.txt",
5203 "old actor two token\n",
5204 "new actor two token\n",
5205 );
5206
5207 let executor = Executor::new();
5208 assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
5209 assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
5210
5211 flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
5212
5213 let mut restored1 =
5214 crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
5215 .expect("load flushed root one index");
5216 restored1.ready = true;
5217 assert_eq!(
5218 restored1
5219 .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
5220 .matches
5221 .len(),
5222 1,
5223 "graceful subc shutdown should flush the first root's trigram delta"
5224 );
5225
5226 let mut restored2 =
5227 crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
5228 .expect("load flushed root two index");
5229 restored2.ready = true;
5230 assert_eq!(
5231 restored2
5232 .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
5233 .matches
5234 .len(),
5235 1,
5236 "graceful subc shutdown should flush every registered root"
5237 );
5238 }
5239
5240 #[test]
5241 fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
5242 let _ = env_logger::builder().is_test(true).try_init();
5243 let (root_dir, root) = test_root("idle-root-reaper");
5244 let storage = tempfile::tempdir().expect("storage tempdir");
5245 std::fs::write(
5246 root_dir.path().join("main.rs"),
5247 "fn entry() { leaf(); }\nfn leaf() {}\n",
5248 )
5249 .expect("source file");
5250 let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
5251 let app = App::default_shared();
5252 let ctx = Arc::new(AppContext::from_app(
5253 Arc::clone(&app),
5254 Config {
5255 project_root: Some(canonical_root.clone()),
5256 storage_dir: Some(storage.path().to_path_buf()),
5257 callgraph_store: true,
5258 search_index: true,
5259 ..Config::default()
5260 },
5261 ));
5262 ctx.set_canonical_cache_root(canonical_root.clone());
5263 let project_key = crate::search_index::artifact_cache_key(&canonical_root);
5264 crate::root_cache::configure_artifact_access(&canonical_root, &project_key, false);
5265 assert!(ctx
5266 .ensure_callgraph_store()
5267 .expect("build callgraph store")
5268 .is_some());
5269
5270 let cache_dir =
5271 crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
5272 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
5273 let git_head = index.stored_git_head().map(str::to_owned);
5274 index.write_to_disk(&cache_dir, git_head.as_deref());
5275 *ctx.search_index()
5276 .write()
5277 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
5278 let seeded_generation =
5281 crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
5282 .expect("seeded artifact generation");
5283 crate::cache_freshness::record_verify_completed(
5284 &canonical_root,
5285 crate::cache_freshness::VerifyArtifact::Search,
5286 Some(seeded_generation),
5287 );
5288 assert!(
5289 matches!(
5290 crate::cache_freshness::warm_verify_plan(
5291 canonical_root.as_path(),
5292 crate::cache_freshness::VerifyArtifact::Search,
5293 Some(seeded_generation),
5294 ),
5295 crate::cache_freshness::WarmVerifyPlan::Skip
5296 ),
5297 "memo must be warm before eviction for the downgrade assertion to bite"
5298 );
5299
5300 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5301 let _dispatch_tx = dispatch_tx;
5302 let shutdown = Arc::new(AtomicBool::new(false));
5303 let thread_shutdown = Arc::clone(&shutdown);
5304 let join = std::thread::spawn(move || {
5305 while !thread_shutdown.load(Ordering::SeqCst) {
5306 std::thread::yield_now();
5307 }
5308 });
5309 ctx.install_watcher_runtime(
5310 dispatch_rx,
5311 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5312 );
5313 wait_for_watcher_count(&ctx, 1);
5314
5315 let executor = Arc::new(Executor::new());
5316 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5317 ctx.mark_subc_unbound();
5318 let mut live_roots = HashMap::new();
5319 let mut meta = RootMeta::new(Instant::now());
5320 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5321 meta.unbound_quiesced = true;
5322 live_roots.insert(root.clone(), meta);
5323
5324 let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
5325 assert!(message.contains("evicted idle root"));
5326 assert!(message.contains("freed ~"));
5327 assert!(message.contains("semantic"));
5328 assert!(!message.contains("semantic not estimated retained"));
5329 assert!(message.contains("trigram"));
5330 assert!(message.contains("retained: bash"));
5331 assert!(message.contains("parser_pool"));
5332
5333 assert_eq!(
5334 reap_idle_roots(
5335 Instant::now(),
5336 &mut live_roots,
5337 &HashMap::new(),
5338 &HashMap::new(),
5339 &executor,
5340 &DispatchPathMetrics::new(),
5341 )
5342 .evicted,
5343 1
5344 );
5345 assert!(ctx.search_index().read().unwrap().is_none());
5346 wait_for_watcher_count(&ctx, 0);
5347 assert!(
5352 matches!(
5353 crate::cache_freshness::warm_verify_plan(
5354 canonical_root.as_path(),
5355 crate::cache_freshness::VerifyArtifact::Search,
5356 Some(seeded_generation),
5357 ),
5358 crate::cache_freshness::WarmVerifyPlan::Strict
5359 ),
5360 "idle eviction must force strict re-verification"
5361 );
5362 assert!(
5363 crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
5364 );
5365 ctx.mark_subc_bound();
5366 assert!(ctx
5367 .ensure_callgraph_store()
5368 .expect("reopen callgraph store")
5369 .is_some());
5370 assert!(live_roots[&root].idle_artifacts_evicted);
5371 }
5372
5373 #[test]
5374 fn idle_root_reaper_applies_ttl_to_unbound_roots() {
5375 let (_root_dir, root) = test_root("idle-root-ttl-gate");
5376 let ctx = test_ctx();
5377 let executor = Arc::new(Executor::new());
5378 assert!(executor.register_actor(root.clone(), ctx));
5379 let ctx = executor.actor_context(&root).expect("actor context");
5380 ctx.mark_subc_unbound();
5381 let now = Instant::now();
5382 let mut meta = RootMeta::new(now);
5383 meta.unbound_quiesced = true;
5384 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5385
5386 assert_eq!(
5390 reap_idle_roots(
5391 now,
5392 &mut live_roots,
5393 &HashMap::new(),
5394 &HashMap::new(),
5395 &executor,
5396 &DispatchPathMetrics::new(),
5397 )
5398 .evicted,
5399 0
5400 );
5401 assert!(!live_roots[&root].idle_artifacts_evicted);
5402
5403 ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
5409 assert_eq!(
5410 reap_idle_roots(
5411 now + IDLE_ROOT_TTL,
5412 &mut live_roots,
5413 &HashMap::new(),
5414 &HashMap::new(),
5415 &executor,
5416 &DispatchPathMetrics::new(),
5417 )
5418 .evicted,
5419 1
5420 );
5421 assert!(live_roots[&root].idle_artifacts_evicted);
5422 assert!(
5423 ctx.take_pending_search_index_paths().is_empty(),
5424 "TTL eviction must dispose retained pending reconciliation paths"
5425 );
5426 }
5427
5428 #[test]
5429 fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
5430 let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
5431 let ctx = test_ctx();
5432 let executor = Arc::new(Executor::new());
5433 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5434 ctx.mark_subc_unbound();
5435
5436 let pending = root.as_path().join("edited-while-unbound.rs");
5442 ctx.add_pending_search_index_paths([pending.clone()]);
5443 let dirty_source = root.as_path().join("dirty.rs");
5444 std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
5445 let mut dirty = crate::search_index::SearchIndex::new();
5446 dirty.ready = true;
5447 dirty.update_file(&dirty_source);
5448 *ctx.search_index()
5449 .write()
5450 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
5451 assert!(ctx.artifact_eviction_blocked());
5452
5453 let mut live_roots = HashMap::new();
5454 let mut meta = RootMeta::new(Instant::now());
5455 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5456 meta.unbound_quiesced = true;
5457 live_roots.insert(root.clone(), meta);
5458
5459 assert_eq!(
5460 reap_idle_roots(
5461 Instant::now(),
5462 &mut live_roots,
5463 &HashMap::new(),
5464 &HashMap::new(),
5465 &executor,
5466 &DispatchPathMetrics::new(),
5467 )
5468 .evicted,
5469 0,
5470 "the dirty index must still block this eviction"
5471 );
5472 assert_eq!(
5473 ctx.take_pending_search_index_paths(),
5474 vec![pending],
5475 "a blocked eviction must restore the taken pending paths"
5476 );
5477 }
5478
5479 #[test]
5480 fn idle_reap_with_bound_route_keeps_watcher_running() {
5481 let (_root_dir, root) = test_root("bound-root-reap-gate");
5482 let ctx = test_ctx();
5483 let executor = Arc::new(Executor::new());
5484 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5485
5486 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5487 let _dispatch_tx = dispatch_tx;
5488 let shutdown = Arc::new(AtomicBool::new(false));
5489 let thread_shutdown = Arc::clone(&shutdown);
5490 let join = std::thread::spawn(move || {
5491 while !thread_shutdown.load(Ordering::SeqCst) {
5492 std::thread::yield_now();
5493 }
5494 });
5495 ctx.install_watcher_runtime(
5496 dispatch_rx,
5497 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5498 );
5499
5500 let mut meta = RootMeta::new(Instant::now());
5501 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5502 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5503 let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
5504 assert_eq!(
5505 reap_idle_roots(
5506 Instant::now(),
5507 &mut live_roots,
5508 &HashMap::new(),
5509 &bound,
5510 &executor,
5511 &DispatchPathMetrics::new(),
5512 )
5513 .evicted,
5514 0
5515 );
5516 wait_for_watcher_count(&ctx, 1);
5517 ctx.stop_watcher_runtime_in_background();
5518 wait_for_watcher_count(&ctx, 0);
5519 }
5520
5521 #[test]
5522 fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
5523 let (root_dir, root) = test_root("deleted-bound-root-reap");
5524 let executor = Arc::new(Executor::new());
5525 assert!(executor.register_actor(root.clone(), test_ctx()));
5526 root_dir.close().expect("delete project root");
5527
5528 let route = route_key(19, 3);
5529 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5530 let cancel_signal = PersistentCancelSignal::new();
5531 let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
5532 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5533 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5534 let mut route_bash_cancels = HashMap::from([(
5535 route,
5536 bash::RouteBashCancel {
5537 token: cancel_signal.clone(),
5538 active_waits: 0,
5539 },
5540 )]);
5541 let metrics = DispatchPathMetrics::new();
5542
5543 let first = reap_idle_roots(
5544 Instant::now(),
5545 &mut live_roots,
5546 &HashMap::new(),
5547 &root_channels,
5548 &executor,
5549 &metrics,
5550 );
5551 assert!(first.forgotten_deleted_roots.is_empty());
5552 assert!(executor.actor_registered(&root));
5553
5554 let mut forgotten = Vec::new();
5555 for _ in 0..100 {
5556 let outcome = reap_idle_roots(
5557 Instant::now(),
5558 &mut live_roots,
5559 &HashMap::new(),
5560 &root_channels,
5561 &executor,
5562 &metrics,
5563 );
5564 if !outcome.forgotten_deleted_roots.is_empty() {
5565 forgotten = outcome.forgotten_deleted_roots;
5566 break;
5567 }
5568 std::thread::sleep(Duration::from_millis(10));
5569 }
5570 assert_eq!(forgotten, vec![root.clone()]);
5571 assert!(!executor.actor_registered(&root));
5572
5573 let mut retry_buffer = HashMap::new();
5574 let mut reclaimed_routes = ReclaimedRoutes::default();
5575 let mut session_identity = HashMap::new();
5576 let mut push_buffer = HashMap::new();
5577 let mut bg_subs = HashMap::from([(
5578 route,
5579 BgSub {
5580 corr: 77,
5581 ver: PROTOCOL_VERSION,
5582 flags: control_flags(),
5583 root: root.clone(),
5584 session: "deleted-route".to_string(),
5585 },
5586 )]);
5587 let mut bg_sub_by_session = HashMap::from([(
5588 (root.clone(), "deleted-route".to_string()),
5589 HashSet::from([route]),
5590 )]);
5591 let mut bg_wake_pending = HashSet::from([route]);
5592 let mut bg_wake_epoch = HashMap::new();
5593 let mut pending_bash_asks = HashMap::new();
5594 health::take_bg_observability_logs_for_test();
5595 purge_deleted_root_residents(
5596 &root,
5597 &mut routes,
5598 &mut root_channels,
5599 &mut installed_route_epochs,
5600 &mut route_bash_cancels,
5601 &mut retry_buffer,
5602 &mut reclaimed_routes,
5603 &mut session_identity,
5604 &mut push_buffer,
5605 &mut bg_subs,
5606 &mut bg_sub_by_session,
5607 &mut bg_wake_pending,
5608 &mut bg_wake_epoch,
5609 &mut pending_bash_asks,
5610 &metrics,
5611 );
5612
5613 assert!(routes.is_empty());
5614 assert!(root_channels.is_empty());
5615 assert!(installed_route_epochs.is_empty());
5616 assert!(route_bash_cancels.is_empty());
5617 assert!(reclaimed_routes.contains(route));
5618 assert!(cancel_signal.is_cancelled());
5619 assert_eq!(
5620 health::take_bg_observability_logs_for_test(),
5621 vec![format!(
5622 "subc bg subscription: ended root={} session=deleted-route channel=19@3 cause=root-reclaim suppressed=0",
5623 root.as_path().display()
5624 )]
5625 );
5626 }
5627
5628 #[test]
5635 fn live_root_with_bound_route_is_never_reclaimed() {
5636 let (_root_dir, root) = test_root("live-bound-root-retained");
5637 let ctx = test_ctx();
5638 ctx.mark_subc_unbound();
5639 let executor = Arc::new(Executor::new());
5640 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5641
5642 let route = RouteChannel {
5643 channel: 7,
5644 epoch: 1,
5645 };
5646 let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
5647 meta.unbound_quiesced = true;
5648 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5649 let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5650
5651 for _ in 0..3 {
5656 let outcome = reap_idle_roots(
5657 Instant::now(),
5658 &mut live_roots,
5659 &HashMap::new(),
5660 &root_channels,
5661 &executor,
5662 &DispatchPathMetrics::new(),
5663 );
5664 assert!(
5665 outcome.forgotten_deleted_roots.is_empty(),
5666 "a root whose directory exists must never be forgotten"
5667 );
5668 }
5669
5670 assert!(live_roots.contains_key(&root), "live root must be retained");
5671 assert!(
5672 executor.actor_registered(&root),
5673 "live root's actor must survive"
5674 );
5675 assert!(
5676 root.as_path().exists(),
5677 "test vehicle must keep the directory alive; otherwise this control proves nothing"
5678 );
5679 }
5680
5681 #[test]
5682 fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
5683 let (root_dir, root) = test_root("deleted-root-first-observation");
5684 let ctx = test_ctx();
5685 ctx.mark_subc_unbound();
5686 let executor = Arc::new(Executor::new());
5687 assert!(executor.register_actor(root.clone(), ctx));
5688 root_dir.close().expect("delete project root");
5689
5690 let mut meta = RootMeta::new(Instant::now());
5691 meta.unbound_quiesced = true;
5692 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5693 let outcome = reap_idle_roots(
5694 Instant::now(),
5695 &mut live_roots,
5696 &HashMap::new(),
5697 &HashMap::new(),
5698 &executor,
5699 &DispatchPathMetrics::new(),
5700 );
5701
5702 assert!(outcome.forgotten_deleted_roots.is_empty());
5703 assert!(live_roots.contains_key(&root));
5704 assert!(executor.actor_registered(&root));
5705 }
5706
5707 fn spawn_background_for_root(
5708 ctx: &AppContext,
5709 root: &ProjectRootId,
5710 storage: &tempfile::TempDir,
5711 session_id: &str,
5712 ) -> (String, u32) {
5713 let command = if cfg!(windows) {
5719 "ping -n 31 127.0.0.1 > nul"
5721 } else {
5722 "sleep 30"
5723 };
5724 let task_id = ctx
5725 .bash_background()
5726 .spawn(
5727 crate::sandbox_spawn::SpawnPlan::Unsandboxed,
5728 command,
5729 session_id.to_string(),
5730 storage.path().to_path_buf(),
5731 HashMap::new(),
5732 Some(Duration::from_secs(60)),
5733 storage.path().to_path_buf(),
5734 8,
5735 true,
5736 false,
5737 Some(root.as_path().to_path_buf()),
5738 )
5739 .expect("spawn background task");
5740 let snapshot = ctx
5741 .bash_background()
5742 .status(
5743 &task_id,
5744 session_id,
5745 Some(root.as_path()),
5746 Some(storage.path()),
5747 0,
5748 )
5749 .expect("background task status");
5750 (task_id, snapshot.child_pid.expect("background child pid"))
5751 }
5752
5753 fn wait_for_background_exit(pid: u32) {
5754 let deadline = Instant::now() + Duration::from_secs(5);
5755 while crate::bash_background::process::is_process_alive(pid) {
5756 assert!(
5757 Instant::now() < deadline,
5758 "background task process survived kill"
5759 );
5760 std::thread::sleep(Duration::from_millis(20));
5761 }
5762 }
5763
5764 #[test]
5765 fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
5766 let (root_dir, root) = test_root("deleted-root-background-task");
5767 let storage = tempfile::tempdir().expect("task storage");
5768 let ctx = test_ctx();
5769 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
5770 assert!(crate::bash_background::process::is_process_alive(pid));
5771
5772 let executor = Arc::new(Executor::new());
5773 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5774 assert!(executor.actor_is_idle(&root));
5775 root_dir.close().expect("delete project root");
5776 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5777 let pending_binds = HashMap::new();
5778 let root_channels = HashMap::new();
5779 let metrics = DispatchPathMetrics::new();
5780
5781 let first = reap_idle_roots(
5782 Instant::now(),
5783 &mut live_roots,
5784 &pending_binds,
5785 &root_channels,
5786 &executor,
5787 &metrics,
5788 );
5789 assert!(first.forgotten_deleted_roots.is_empty());
5790 assert!(crate::bash_background::process::is_process_alive(pid));
5791
5792 let outcome = reap_until_forgotten(
5793 &root,
5794 &mut live_roots,
5795 &pending_binds,
5796 &root_channels,
5797 &executor,
5798 &metrics,
5799 );
5800 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5801 wait_for_background_exit(pid);
5802
5803 let snapshot = ctx
5804 .bash_background()
5805 .status(
5806 &task_id,
5807 "reclaim-session",
5808 Some(root.as_path()),
5809 Some(storage.path()),
5810 0,
5811 )
5812 .expect("reclaimed task status");
5813 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
5814 assert_eq!(
5815 snapshot.info.status_reason.as_deref(),
5816 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
5817 );
5818 assert_eq!(
5819 serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
5820 crate::bash_background::registry::ROOT_RECLAIMED_REASON
5821 );
5822 let completion = ctx
5823 .bash_background()
5824 .drain_completions_for_session(Some("reclaim-session"))
5825 .pop()
5826 .expect("reclaimed task completion");
5827 assert_eq!(
5828 completion.status_reason.as_deref(),
5829 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
5830 );
5831 }
5832
5833 #[test]
5834 fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
5835 let (root_dir, root) = test_root("existing-root-background-task");
5836 let storage = tempfile::tempdir().expect("task storage");
5837 let ctx = test_ctx();
5838 ctx.mark_subc_unbound();
5839 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
5840
5841 let executor = Arc::new(Executor::new());
5842 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5843 let mut meta = RootMeta::new(
5844 Instant::now()
5845 .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
5846 .expect("old root timestamp"),
5847 );
5848 meta.unbound_quiesced = true;
5849 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5850 let pending_binds = HashMap::new();
5851 let root_channels = HashMap::new();
5852 let metrics = DispatchPathMetrics::new();
5853
5854 for _ in 0..8 {
5855 reap_idle_roots(
5856 Instant::now(),
5857 &mut live_roots,
5858 &pending_binds,
5859 &root_channels,
5860 &executor,
5861 &metrics,
5862 );
5863 std::thread::sleep(Duration::from_millis(10));
5864 }
5865 assert!(root_dir.path().exists());
5866 assert!(crate::bash_background::process::is_process_alive(pid));
5867 let snapshot = ctx
5868 .bash_background()
5869 .status(
5870 &task_id,
5871 "existing-session",
5872 Some(root.as_path()),
5873 Some(storage.path()),
5874 0,
5875 )
5876 .expect("existing task status");
5877 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5878 let _ = ctx.bash_background().kill(&task_id, "existing-session");
5879 wait_for_background_exit(pid);
5880 }
5881
5882 #[test]
5883 fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
5884 let (root_dir, root) = test_root("restored-root-background-task");
5885 let storage = tempfile::tempdir().expect("task storage");
5886 let ctx = test_ctx();
5887 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
5888
5889 let executor = Arc::new(Executor::new());
5890 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5891 root_dir.close().expect("delete project root");
5892 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5893 let pending_binds = HashMap::new();
5894 let root_channels = HashMap::new();
5895 let metrics = DispatchPathMetrics::new();
5896
5897 let first = reap_idle_roots(
5898 Instant::now(),
5899 &mut live_roots,
5900 &pending_binds,
5901 &root_channels,
5902 &executor,
5903 &metrics,
5904 );
5905 assert!(first.forgotten_deleted_roots.is_empty());
5906 std::fs::create_dir_all(root.as_path()).expect("restore project root");
5907 let second = reap_idle_roots(
5908 Instant::now(),
5909 &mut live_roots,
5910 &pending_binds,
5911 &root_channels,
5912 &executor,
5913 &metrics,
5914 );
5915 assert!(second.forgotten_deleted_roots.is_empty());
5916 assert!(crate::bash_background::process::is_process_alive(pid));
5917 let _ = ctx.bash_background().kill(&task_id, "restored-session");
5918 wait_for_background_exit(pid);
5919 }
5920
5921 #[test]
5922 fn observing_root_again_resets_deleted_sweep_confirmation() {
5923 let (root_dir, root) = test_root("deleted-root-observation-reset");
5924 let ctx = test_ctx();
5925 ctx.mark_subc_unbound();
5926 let executor = Arc::new(Executor::new());
5927 assert!(executor.register_actor(root.clone(), ctx));
5928 root_dir.close().expect("delete project root");
5929
5930 let mut meta = RootMeta::new(Instant::now());
5931 meta.unbound_quiesced = true;
5932 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5933 let pending_binds = HashMap::new();
5934 let root_channels = HashMap::new();
5935 let metrics = DispatchPathMetrics::new();
5936
5937 let first = reap_idle_roots(
5938 Instant::now(),
5939 &mut live_roots,
5940 &pending_binds,
5941 &root_channels,
5942 &executor,
5943 &metrics,
5944 );
5945 assert!(first.forgotten_deleted_roots.is_empty());
5946
5947 std::fs::create_dir_all(root.as_path()).expect("restore project root");
5948 reap_idle_roots(
5949 Instant::now(),
5950 &mut live_roots,
5951 &pending_binds,
5952 &root_channels,
5953 &executor,
5954 &metrics,
5955 );
5956 std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
5957
5958 let after_reset = reap_idle_roots(
5959 Instant::now(),
5960 &mut live_roots,
5961 &pending_binds,
5962 &root_channels,
5963 &executor,
5964 &metrics,
5965 );
5966 assert!(after_reset.forgotten_deleted_roots.is_empty());
5967 assert!(live_roots.contains_key(&root));
5968 assert!(executor.actor_registered(&root));
5969 }
5970
5971 #[test]
5972 fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
5973 let (root_dir, root) = test_root("deleted-root-reap");
5974 let app = App::default_shared();
5975 let ctx = Arc::new(AppContext::from_app(
5976 Arc::clone(&app),
5977 Config {
5978 project_root: Some(root.as_path().to_path_buf()),
5979 ..Config::default()
5980 },
5981 ));
5982 ctx.set_canonical_cache_root(root.as_path().to_path_buf());
5983 ctx.mark_subc_unbound();
5984 let executor = Arc::new(Executor::new());
5985 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5986 assert_eq!(app.actor_root_count(), 1);
5987 drop(ctx);
5988 root_dir.close().expect("delete project root");
5989
5990 let mut meta = RootMeta::new(Instant::now());
5991 meta.unbound_quiesced = true;
5992 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5993 let outcome = reap_until_forgotten(
5994 &root,
5995 &mut live_roots,
5996 &HashMap::new(),
5997 &HashMap::new(),
5998 &executor,
5999 &DispatchPathMetrics::new(),
6000 );
6001 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
6002 assert!(!executor.actor_registered(&root));
6003 assert!(!live_roots.contains_key(&root));
6004 wait_for_actor_root_count(&app, 0);
6005
6006 let status_ctx = AppContext::from_app(app, Config::default());
6007 let status = status_ctx.build_status_snapshot();
6008 assert_eq!(status["runtime"]["live_actor_roots"], 0);
6009 assert_eq!(status["runtime"]["open_routes"], 0);
6010 }
6011
6012 #[test]
6013 fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
6014 let (root_dir, root) = test_root("deleted-root-reap-census");
6015 let executor = Arc::new(Executor::new());
6016 assert!(executor.register_actor(root.clone(), test_ctx()));
6017 root_dir.close().expect("delete project root");
6018
6019 let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
6020 let metrics = DispatchPathMetrics::new();
6021 let outcome = reap_idle_roots(
6022 Instant::now(),
6023 &mut live_roots,
6024 &HashMap::new(),
6025 &HashMap::new(),
6026 &executor,
6027 &metrics,
6028 );
6029 assert_eq!(outcome.evicted, 0);
6030
6031 let app = crate::context::App::default_shared();
6032 let health_rollup_cache = HealthRollupCache::new();
6033 health_rollup_cache.refresh(&executor, &app);
6034 let report = build_health_report(
6035 &health_rollup_cache,
6036 &executor,
6037 &HashMap::new(),
6038 &metrics,
6039 &app,
6040 );
6041 let reap = report
6042 .metrics
6043 .as_ref()
6044 .and_then(|metrics| metrics.get("reap"))
6045 .expect("reap health metrics");
6046 assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
6047 assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
6048 assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
6049 assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
6050 }
6051
6052 #[test]
6053 fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
6054 let (root_dir, root) = test_root("connection-exit-deleted-root");
6055 let executor = Arc::new(Executor::new());
6056 assert!(executor.register_actor(root.clone(), test_ctx()));
6057
6058 let route = route_key(11, 1);
6059 let mut meta = RootMeta::new(Instant::now());
6060 meta.maintenance_pending = true;
6061 meta.maintenance_queued_kinds
6062 .push_back(MaintenanceDrainKind::CompletionDrains);
6063 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6064 let mut pending_binds = HashMap::new();
6065 let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
6066 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
6067 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6068 let mut route_bash_cancels = HashMap::new();
6069
6070 quiesce_connection_roots(
6071 &mut live_roots,
6072 &mut pending_binds,
6073 &mut routes,
6074 &mut root_channels,
6075 &mut installed_route_epochs,
6076 &mut route_bash_cancels,
6077 &executor,
6078 );
6079 assert!(live_roots[&root].unbound_quiesced);
6080 assert!(!live_roots[&root].maintenance_pending);
6081 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
6082 assert!(routes.is_empty());
6083 assert!(root_channels.is_empty());
6084
6085 root_dir.close().expect("delete project root");
6086 let metrics = DispatchPathMetrics::new();
6087 let outcome = reap_until_forgotten(
6088 &root,
6089 &mut live_roots,
6090 &pending_binds,
6091 &root_channels,
6092 &executor,
6093 &metrics,
6094 );
6095 let mut session_identity = HashMap::new();
6096 let mut push_buffer = HashMap::new();
6097 let mut bg_subs = HashMap::new();
6098 let mut bg_sub_by_session = HashMap::new();
6099 let mut bg_wake_pending = HashSet::new();
6100 let mut bg_wake_epoch = HashMap::new();
6101 let mut pending_bash_asks = HashMap::new();
6102 let mut retry_buffer = HashMap::new();
6103 let mut reclaimed_routes = ReclaimedRoutes::default();
6104 for forgotten in &outcome.forgotten_deleted_roots {
6105 purge_deleted_root_residents(
6106 forgotten,
6107 &mut routes,
6108 &mut root_channels,
6109 &mut installed_route_epochs,
6110 &mut route_bash_cancels,
6111 &mut retry_buffer,
6112 &mut reclaimed_routes,
6113 &mut session_identity,
6114 &mut push_buffer,
6115 &mut bg_subs,
6116 &mut bg_sub_by_session,
6117 &mut bg_wake_pending,
6118 &mut bg_wake_epoch,
6119 &mut pending_bash_asks,
6120 &metrics,
6121 );
6122 }
6123
6124 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
6125 assert!(!executor.actor_registered(&root));
6126 assert!(!live_roots.contains_key(&root));
6127 }
6128
6129 #[test]
6130 fn unbound_root_quiesces_maintenance_without_removing_actor() {
6131 let (_root_dir, root) = test_root("unbound-root-quiesce");
6132 let ctx = test_ctx();
6133 let executor = Arc::new(Executor::new());
6134 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6135 let mut meta = RootMeta::new(Instant::now());
6136 meta.maintenance_pending = true;
6137 meta.maintenance_jobs_in_flight = 1;
6138 meta.maintenance_queued_kinds
6139 .push_back(MaintenanceDrainKind::ConfigureTail);
6140 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6141 *ctx.search_index()
6143 .write()
6144 .unwrap_or_else(std::sync::PoisonError::into_inner) =
6145 Some(crate::search_index::SearchIndex::new());
6146 ctx.set_cache_writer_capabilities(true, true);
6147 let pending = root.as_path().join("pending.rs");
6148 ctx.add_pending_search_index_paths([pending.clone()]);
6149 let canonical_root = root.as_path().to_path_buf();
6153 let artifact = canonical_root.join("cache.bin");
6154 std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
6155 let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
6156 crate::cache_freshness::record_verify_completed(
6157 &canonical_root,
6158 crate::cache_freshness::VerifyArtifact::Search,
6159 seeded_generation,
6160 );
6161 assert!(matches!(
6162 crate::cache_freshness::warm_verify_plan(
6163 &canonical_root,
6164 crate::cache_freshness::VerifyArtifact::Search,
6165 seeded_generation,
6166 ),
6167 crate::cache_freshness::WarmVerifyPlan::Skip
6168 ));
6169 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
6172 let _dispatch_tx = dispatch_tx;
6173 let shutdown = Arc::new(AtomicBool::new(false));
6174 let thread_shutdown = Arc::clone(&shutdown);
6175 let join = std::thread::spawn(move || {
6176 while !thread_shutdown.load(Ordering::SeqCst) {
6177 std::thread::yield_now();
6178 }
6179 });
6180 ctx.install_watcher_runtime(
6181 dispatch_rx,
6182 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
6183 );
6184 assert!(ctx.watcher_runtime_active());
6185
6186 quiesce_unbound_root(&root, &mut live_roots, &executor);
6187 let meta = &live_roots[&root];
6188 assert!(meta.unbound_quiesced);
6189 assert!(ctx.subc_unbound_quiesced());
6190 assert!(meta.maintenance_pending);
6191 assert!(meta.maintenance_queued_kinds.is_empty());
6192 assert!(executor.actor_registered(&root));
6193 assert!(
6197 ctx.search_index()
6198 .read()
6199 .unwrap_or_else(std::sync::PoisonError::into_inner)
6200 .is_some(),
6201 "quiesce must not evict resident artifacts"
6202 );
6203 assert_eq!(
6204 ctx.pending_callgraph_store_force_token(),
6205 None,
6206 "quiesce must not force a callgraph rebuild"
6207 );
6208 assert_eq!(
6209 ctx.take_pending_search_index_paths(),
6210 vec![pending],
6211 "quiesce must retain pending watcher-derived paths"
6212 );
6213 assert!(
6214 matches!(
6215 crate::cache_freshness::warm_verify_plan(
6216 &canonical_root,
6217 crate::cache_freshness::VerifyArtifact::Search,
6218 seeded_generation,
6219 ),
6220 crate::cache_freshness::WarmVerifyPlan::Skip
6221 ),
6222 "quiesce must not invalidate the warm verify memo"
6223 );
6224 assert!(
6225 ctx.watcher_runtime_active(),
6226 "quiesce must not stop a running watcher"
6227 );
6228 ctx.stop_watcher_runtime();
6229
6230 let meta = live_roots.get_mut(&root).expect("root metadata");
6231 note_maintenance_completion(
6232 meta,
6233 Some(MaintenanceDrainKind::ConfigureTail),
6234 false,
6235 meta.unbound_quiesced,
6236 );
6237 assert!(!meta.maintenance_pending);
6238 assert!(meta.maintenance_queued_kinds.is_empty());
6239 }
6240
6241 #[test]
6242 fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
6243 let (_dir, root) = test_root("same-root-replacement");
6244 let route = route_key(7, 1);
6245 let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
6246 let root_channels = HashMap::new();
6247
6248 assert!(!route_removal_will_quiesce_root(
6249 &root,
6250 route,
6251 &installed_channels,
6252 false,
6253 Some(&root),
6254 ));
6255 assert!(route_removal_will_quiesce_root(
6256 &root,
6257 route,
6258 &installed_channels,
6259 false,
6260 None,
6261 ));
6262 assert!(!should_quiesce_removed_root(
6263 &root,
6264 &root_channels,
6265 false,
6266 Some(&root),
6267 ));
6268 assert!(should_quiesce_removed_root(
6269 &root,
6270 &root_channels,
6271 false,
6272 None,
6273 ));
6274 assert!(!should_quiesce_removed_root(
6275 &root,
6276 &root_channels,
6277 true,
6278 None,
6279 ));
6280 }
6281
6282 #[test]
6283 fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
6284 let (_root_dir, root) = test_root("unbound-root-route-count");
6285 let executor = Arc::new(Executor::new());
6286 assert!(executor.register_actor(root.clone(), test_ctx()));
6287 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6288 let mut root_channels = HashMap::from([(
6289 root.clone(),
6290 HashSet::from([route_key(7, 1), route_key(8, 1)]),
6291 )]);
6292
6293 remove_root_channel(&mut root_channels, &root, route_key(7, 1));
6294 if !root_channels.contains_key(&root) {
6295 quiesce_unbound_root(&root, &mut live_roots, &executor);
6296 }
6297 assert!(!live_roots[&root].unbound_quiesced);
6298
6299 remove_root_channel(&mut root_channels, &root, route_key(8, 1));
6300 if !root_channels.contains_key(&root) {
6301 quiesce_unbound_root(&root, &mut live_roots, &executor);
6302 }
6303 assert!(live_roots[&root].unbound_quiesced);
6304
6305 live_roots
6306 .get_mut(&root)
6307 .expect("root metadata")
6308 .note_activity();
6309 assert!(
6310 live_roots[&root].unbound_quiesced,
6311 "late asynchronous activity must not reactivate an unbound root"
6312 );
6313
6314 live_roots
6315 .get_mut(&root)
6316 .expect("root metadata")
6317 .reactivate_bound();
6318 assert!(!live_roots[&root].unbound_quiesced);
6319 }
6320
6321 #[test]
6322 fn allocator_pressure_relief_requires_every_root_to_be_idle() {
6323 let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
6324 let (_active_dir, active_root) = test_root("allocator-relief-active");
6325 let now = Instant::now();
6326 let mut live_roots = HashMap::new();
6327 let mut idle = RootMeta::new(now);
6328 idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
6329 live_roots.insert(idle_root, idle);
6330 assert!(process_has_been_idle(now, &live_roots));
6331
6332 live_roots.insert(active_root.clone(), RootMeta::new(now));
6333 assert!(!process_has_been_idle(now, &live_roots));
6334
6335 let active = live_roots
6336 .get_mut(&active_root)
6337 .expect("active root metadata");
6338 active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
6339 active.active_bash_waits = 1;
6340 assert!(!process_has_been_idle(now, &live_roots));
6341 }
6342
6343 #[test]
6344 fn pressure_relief_log_reports_before_and_after_measurements() {
6345 let allocator = crate::memory::AllocatorMemorySnapshot {
6346 status: "measured",
6347 bytes_in_use: Some(8 * 1024 * 1024),
6348 size_allocated: Some(12 * 1024 * 1024),
6349 retained_slack_bytes: Some(4 * 1024 * 1024),
6350 not_estimated: None,
6351 };
6352 let relief = crate::memory::AllocatorPressureRelief {
6353 bytes_released: 3 * 1024 * 1024,
6354 rss_before_bytes: Some(20 * 1024 * 1024),
6355 rss_after_bytes: Some(17 * 1024 * 1024),
6356 allocator_before: allocator.clone(),
6357 allocator_after: crate::memory::AllocatorMemorySnapshot {
6358 size_allocated: Some(9 * 1024 * 1024),
6359 retained_slack_bytes: Some(1024 * 1024),
6360 ..allocator
6361 },
6362 };
6363 let message = pressure_relief_label(&relief);
6364 assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
6365 assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
6366 assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
6367 assert!(message.contains("reported 3.0 MB released"));
6368 }
6369
6370 #[test]
6371 fn due_maintenance_jobs_skip_poisoned_roots() {
6372 let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
6373 let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
6374 let mut live_roots = HashMap::new();
6375 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6376 let mut poisoned_meta = RootMeta::new(Instant::now());
6377 poisoned_meta.maintenance_poisoned = true;
6378 live_roots.insert(poisoned_root.clone(), poisoned_meta);
6379
6380 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6381 &mut live_roots,
6382 MAINTENANCE_SUBMIT_BUDGET,
6383 &HashSet::new(),
6384 );
6385
6386 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6387 assert!(due.iter().all(|(root, _)| root == &healthy_root));
6388 assert!(!deferred);
6389 assert!(live_roots[&healthy_root].maintenance_pending);
6390 assert_eq!(
6391 live_roots[&healthy_root].maintenance_jobs_in_flight,
6392 INITIAL_MAINTENANCE_JOB_COUNT
6393 );
6394 assert!(!live_roots[&poisoned_root].maintenance_pending);
6395 }
6396
6397 #[test]
6398 fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
6399 let (_dir, root) = test_root("maintenance-unbound");
6400 let mut meta = RootMeta::new(Instant::now());
6401 meta.unbound_quiesced = true;
6402 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6403
6404 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6405 &mut live_roots,
6406 MAINTENANCE_SUBMIT_BUDGET,
6407 &HashSet::new(),
6408 );
6409
6410 assert!(due.is_empty());
6411 assert!(!deferred);
6412 assert!(!live_roots[&root].maintenance_pending);
6413 }
6414
6415 #[test]
6416 fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
6417 let (_dir, root) = test_root("maintenance-idle-bg-subscription");
6418 let ctx = test_ctx();
6419 assert!(!ctx.completion_drains_have_work());
6420
6421 let executor = Executor::new();
6422 assert!(executor.register_actor(root.clone(), ctx));
6423 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6424 let session = "idle-session".to_string();
6425 let channel = route_key(17, 1);
6426 let metrics = DispatchPathMetrics::new();
6427 let bg_sub_by_session =
6428 HashMap::from([((root.clone(), session.clone()), HashSet::from([channel]))]);
6429 let mut bg_wake_pending = HashSet::new();
6430
6431 let (idle_tick_jobs, deferred) = due_maintenance_jobs(
6432 &mut live_roots,
6433 Some(&executor),
6434 &bg_sub_by_session,
6435 &bg_wake_pending,
6436 MAINTENANCE_SUBMIT_BUDGET,
6437 &HashSet::new(),
6438 );
6439 assert!(idle_tick_jobs.is_empty());
6440 assert!(!deferred);
6441 assert!(!live_roots[&root].maintenance_pending);
6442
6443 let mut bg_wake_epoch = HashMap::new();
6446 push::arm_bg_wake(
6447 root.clone(),
6448 session,
6449 channel,
6450 &mut bg_wake_pending,
6451 &mut bg_wake_epoch,
6452 &metrics,
6453 );
6454 let (next_tick_jobs, deferred) = due_maintenance_jobs(
6455 &mut live_roots,
6456 Some(&executor),
6457 &bg_sub_by_session,
6458 &bg_wake_pending,
6459 MAINTENANCE_SUBMIT_BUDGET,
6460 &HashSet::new(),
6461 );
6462 assert_eq!(
6463 next_tick_jobs,
6464 vec![(root, MaintenanceDrainKind::CompletionDrains)]
6465 );
6466 assert!(!deferred);
6467 }
6468
6469 #[tokio::test]
6470 async fn subc_configure_tail_precedes_completed_search_install() {
6471 let root_dir = tempfile::tempdir().unwrap();
6472 let storage = tempfile::tempdir().unwrap();
6473 let root = ProjectRootId::from_path(root_dir.path()).unwrap();
6474 let (ctx, ignored_path) =
6475 runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
6476 let ctx = Arc::new(ctx);
6477 assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
6478 &ctx,
6479 &ignored_path
6480 ));
6481
6482 let executor = Arc::new(Executor::new());
6483 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6484 let metrics = Arc::new(DispatchPathMetrics::new());
6485 let (completion_tx, mut completion_rx) = mpsc::channel(4);
6486 submit_maintenance_job(
6487 &executor,
6488 root.clone(),
6489 MaintenanceDrainKind::ConfigureTail,
6490 Vec::new(),
6491 &completion_tx,
6492 &metrics,
6493 );
6494 submit_maintenance_job(
6495 &executor,
6496 root,
6497 MaintenanceDrainKind::CompletionDrains,
6498 Vec::new(),
6499 &completion_tx,
6500 &metrics,
6501 );
6502
6503 let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6504 .await
6505 .expect("configure-tail completion timed out")
6506 .expect("configure-tail completion channel closed");
6507 let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6508 .await
6509 .expect("completion-drains completion timed out")
6510 .expect("completion-drains completion channel closed");
6511 assert!(first.response.id.contains("configure-tail"));
6512 assert!(second.response.id.contains("completion-drains"));
6513 assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
6514 &ctx,
6515 &ignored_path
6516 ));
6517 assert_eq!(
6518 ctx.search_index()
6519 .read()
6520 .unwrap_or_else(std::sync::PoisonError::into_inner)
6521 .as_ref()
6522 .expect("completed search index installed")
6523 .file_count(),
6524 0,
6525 "configure must install the ignore matcher before pending paths replay"
6526 );
6527 ctx.stop_watcher_runtime();
6528 }
6529
6530 #[test]
6531 fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
6532 let (_dir, root) = test_root("maintenance-post-bind");
6533 let mut live_roots = HashMap::new();
6534 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6535
6536 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6537 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6538
6539 let meta = live_roots.get(&root).expect("root metadata");
6540 assert!(meta.maintenance_pending);
6541 assert_eq!(meta.maintenance_jobs_in_flight, 0);
6542 assert_eq!(
6543 meta.maintenance_queued_kinds
6544 .iter()
6545 .copied()
6546 .collect::<Vec<_>>(),
6547 vec![
6548 MaintenanceDrainKind::ConfigureTail,
6549 MaintenanceDrainKind::CompletionDrains,
6550 ]
6551 );
6552
6553 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6554 &mut live_roots,
6555 MAINTENANCE_SUBMIT_BUDGET,
6556 &HashSet::new(),
6557 );
6558
6559 assert_eq!(
6560 due,
6561 vec![
6562 (root.clone(), MaintenanceDrainKind::ConfigureTail),
6563 (root.clone(), MaintenanceDrainKind::CompletionDrains),
6564 ]
6565 );
6566 assert!(!deferred);
6567 assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
6568 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
6569 }
6570
6571 #[test]
6572 fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
6573 let mut live_roots = HashMap::new();
6574 let mut root_ids = Vec::new();
6575 let mut _dirs = Vec::new();
6576 for index in 0..4 {
6577 let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
6578 live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
6579 root_ids.push(root_id);
6580 _dirs.push(dir);
6581 }
6582
6583 let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
6584 let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
6585 &mut live_roots,
6586 small_budget,
6587 &HashSet::new(),
6588 );
6589
6590 assert_eq!(first_due.len(), small_budget);
6591 assert!(first_deferred);
6592 let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
6593 assert!(first_due_set
6594 .iter()
6595 .all(|root| live_roots[root].maintenance_pending));
6596 assert!(first_due_set
6597 .iter()
6598 .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
6599
6600 let all_roots: HashSet<_> = root_ids.into_iter().collect();
6601 let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
6602 assert!(deferred_roots
6603 .iter()
6604 .all(|root| !live_roots[root].maintenance_pending));
6605 }
6606
6607 #[test]
6608 fn due_maintenance_jobs_defers_pending_bind_roots() {
6609 let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
6610 let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
6611 let mut live_roots = HashMap::new();
6612 live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
6613 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6614 let pending_bind_roots = HashSet::from([bind_root.clone()]);
6615
6616 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6617 &mut live_roots,
6618 usize::MAX,
6619 &pending_bind_roots,
6620 );
6621
6622 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6623 assert!(due.iter().all(|(root, _)| root == &healthy_root));
6624 assert!(!deferred);
6625 assert!(!live_roots[&bind_root].maintenance_pending);
6626 assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
6627 }
6628
6629 #[test]
6630 fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
6631 let (_dir, root) = test_root("maintenance-requeue");
6632 let mut live_roots = HashMap::new();
6633 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6634 let (due, deferred) = due_maintenance_jobs_without_actor_context(
6635 &mut live_roots,
6636 usize::MAX,
6637 &HashSet::new(),
6638 );
6639 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6640 assert!(due.iter().all(|(due_root, _)| due_root == &root));
6641 assert!(!deferred);
6642
6643 let meta = live_roots.get_mut(&root).unwrap();
6644 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
6645 assert!(meta.maintenance_pending);
6646 assert_eq!(
6647 meta.maintenance_jobs_in_flight,
6648 INITIAL_MAINTENANCE_JOB_COUNT - 1
6649 );
6650 assert_eq!(meta.maintenance_queued_kinds.len(), 1);
6651
6652 let (requeued, deferred) =
6653 due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
6654 assert_eq!(
6655 requeued,
6656 vec![(root.clone(), MaintenanceDrainKind::Watcher)]
6657 );
6658 assert!(!deferred);
6659 let meta = live_roots.get_mut(&root).unwrap();
6660 assert_eq!(
6661 meta.maintenance_jobs_in_flight,
6662 INITIAL_MAINTENANCE_JOB_COUNT
6663 );
6664 assert!(meta.maintenance_queued_kinds.is_empty());
6665
6666 for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
6667 note_maintenance_completion(meta, None, false, false);
6668 }
6669 assert!(!meta.maintenance_pending);
6670 assert_eq!(meta.maintenance_jobs_in_flight, 0);
6671 }
6672
6673 #[test]
6674 fn maintenance_requeue_drops_while_bind_is_pending() {
6675 let (_dir, root) = test_root("maintenance-bind-requeue");
6676 let mut live_roots = HashMap::new();
6677 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6678 let (due, _) = due_maintenance_jobs_without_actor_context(
6679 &mut live_roots,
6680 usize::MAX,
6681 &HashSet::new(),
6682 );
6683 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6684
6685 let meta = live_roots.get_mut(&root).unwrap();
6686 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
6687
6688 assert_eq!(
6689 meta.maintenance_jobs_in_flight,
6690 INITIAL_MAINTENANCE_JOB_COUNT - 1
6691 );
6692 assert!(meta.maintenance_queued_kinds.is_empty());
6693 assert!(meta.maintenance_pending);
6694 }
6695
6696 #[test]
6697 fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
6698 let mut meta = RootMeta::new(Instant::now());
6699 meta.unbound_quiesced = true;
6700
6701 assert!(!should_requiesce_after_maintenance(
6702 &meta,
6703 MaintenanceDrainKind::Lsp,
6704 false,
6705 ));
6706 assert!(!should_requiesce_after_maintenance(
6707 &meta,
6708 MaintenanceDrainKind::ConfigureTail,
6709 true,
6710 ));
6711 assert!(should_requiesce_after_maintenance(
6712 &meta,
6713 MaintenanceDrainKind::ConfigureTail,
6714 false,
6715 ));
6716 }
6717
6718 #[test]
6719 fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
6720 let (_dir, root) = test_root("maintenance-fatal");
6721 let mut live_roots = HashMap::new();
6722 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6723 let (due, _) = due_maintenance_jobs_without_actor_context(
6724 &mut live_roots,
6725 usize::MAX,
6726 &HashSet::new(),
6727 );
6728 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6729
6730 let meta = live_roots.get_mut(&root).unwrap();
6731 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
6732 assert!(meta.maintenance_poisoned);
6733 assert!(meta.maintenance_queued_kinds.is_empty());
6734
6735 for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
6736 note_maintenance_completion(meta, None, false, false);
6737 }
6738 assert!(!meta.maintenance_pending);
6739 assert_eq!(meta.maintenance_jobs_in_flight, 0);
6740 }
6741
6742 #[test]
6743 fn trust_for_principal_matrix() {
6744 assert_eq!(
6745 trust_for_principal(&Some(Principal::Direct)),
6746 BindTrust::FirstParty
6747 );
6748 for module_id in [
6755 "llm-runner",
6756 "aft",
6757 "broca",
6758 "alfonso-core",
6759 "prefrontal",
6760 "prefrontal-core",
6761 ] {
6762 assert_eq!(
6763 trust_for_principal(&Some(Principal::Reserved {
6764 module_id: module_id.to_string(),
6765 })),
6766 BindTrust::FirstParty,
6767 "reserved module id '{module_id}' must resolve to first-party trust"
6768 );
6769 }
6770 assert_eq!(
6771 trust_for_principal(&Some(Principal::Reserved {
6772 module_id: "subc-mcp".to_string(),
6773 })),
6774 BindTrust::Untrusted
6775 );
6776 assert_eq!(
6777 trust_for_principal(&Some(Principal::Reserved {
6778 module_id: "anything-unknown".to_string(),
6779 })),
6780 BindTrust::Untrusted
6781 );
6782 assert_eq!(
6783 trust_for_principal(&Some(Principal::Unverified)),
6784 BindTrust::Untrusted
6785 );
6786 assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
6787 }
6788
6789 #[test]
6790 fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
6791 let principal = Some(Principal::Direct);
6792 let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
6793 let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
6794
6795 assert_eq!(
6796 trust_for_bind(fingerprint_a, &principal),
6797 BindTrust::Untrusted
6798 );
6799 assert_eq!(
6800 trust_for_bind(fingerprint_b, &principal),
6801 BindTrust::Untrusted
6802 );
6803 }
6804
6805 #[test]
6812 fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
6813 for harness in ["opencode", "pi", "runner", "mcp:claude"] {
6814 assert_eq!(
6815 trust_for_bind(harness, &Some(Principal::Direct)),
6816 BindTrust::FirstParty,
6817 "a direct principal must stay first-party on {harness}"
6818 );
6819 assert_eq!(
6820 trust_for_bind(harness, &Some(Principal::Unverified)),
6821 BindTrust::Untrusted,
6822 "an unverified principal must stay untrusted on {harness}"
6823 );
6824 assert_eq!(
6825 trust_for_bind(harness, &None),
6826 BindTrust::Untrusted,
6827 "an absent principal must fail closed on {harness}"
6828 );
6829 assert_eq!(
6830 trust_for_bind(
6831 harness,
6832 &Some(Principal::Reserved {
6833 module_id: "subc-mcp".to_string(),
6834 })
6835 ),
6836 BindTrust::Untrusted,
6837 "a non-allowlisted reserved module must stay untrusted on {harness}"
6838 );
6839 }
6840 }
6841
6842 #[tokio::test]
6843 async fn persistent_cancel_resolves_when_fired_before_await() {
6844 let signal = PersistentCancelSignal::new();
6848 signal.cancel();
6849 tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
6851 .await
6852 .expect("cancelled() must resolve when cancel fired beforehand");
6853
6854 let racing = PersistentCancelSignal::new();
6856 let racing_for_task = racing.clone();
6857 let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
6858 racing.cancel();
6859 tokio::time::timeout(Duration::from_secs(1), waiter)
6860 .await
6861 .expect("cancelled() must resolve when cancel races the await")
6862 .expect("waiter task panicked");
6863 }
6864
6865 #[test]
6866 fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
6867 let installed = HashMap::from([(7, 9)]);
6868 let mut reclaimed = ReclaimedRoutes::default();
6869 reclaimed.insert(route_key(8, 1));
6870 for ty in [
6871 FrameType::Request,
6872 FrameType::Response,
6873 FrameType::Error,
6874 FrameType::Push,
6875 FrameType::Cancel,
6876 FrameType::Goodbye,
6877 ] {
6878 let body = if ty.is_pure_header() {
6879 Vec::new()
6880 } else {
6881 br#"{}"#.to_vec()
6882 };
6883 let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
6884 assert!(
6885 !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
6886 "{ty:?}"
6887 );
6888 }
6889
6890 let reclaimed_request = Frame::build(
6891 FrameType::Request,
6892 control_flags(),
6893 8,
6894 1,
6895 42,
6896 br#"{}"#.to_vec(),
6897 )
6898 .unwrap();
6899 assert!(ingress_route_should_be_processed(
6900 &installed,
6901 &reclaimed,
6902 &reclaimed_request
6903 ));
6904
6905 let never_installed = Frame::build(
6906 FrameType::Request,
6907 control_flags(),
6908 9,
6909 1,
6910 43,
6911 br#"{}"#.to_vec(),
6912 )
6913 .unwrap();
6914 assert!(!ingress_route_should_be_processed(
6915 &installed,
6916 &reclaimed,
6917 &never_installed
6918 ));
6919
6920 let current = Frame::build(
6921 FrameType::Request,
6922 control_flags(),
6923 7,
6924 9,
6925 43,
6926 br#"{}"#.to_vec(),
6927 )
6928 .unwrap();
6929 let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
6930 assert!(ingress_route_should_be_processed(
6931 &installed, &reclaimed, ¤t
6932 ));
6933 assert!(ingress_route_should_be_processed(
6934 &installed, &reclaimed, &control
6935 ));
6936 assert_eq!(installed, HashMap::from([(7, 9)]));
6937 }
6938
6939 #[tokio::test]
6940 async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
6941 let (_dir, root) = test_root("route-bind-b2-ordering");
6942 let route = route_key(7, 3);
6943 let identity = RouteIdentity(Arc::new(RouteIdentityData {
6944 root: root.clone(),
6945 project_root: root.as_path().to_path_buf(),
6946 harness: "opencode".to_string(),
6947 session: "b2-session".to_string(),
6948 trust: BindTrust::FirstParty,
6949 spawn_principal: AuthenticatedPrincipal::FirstParty,
6950 consumer_elicitation_capable: false,
6951 }));
6952 let replay_key = push::ReplayKey::from_identity(&identity);
6953 let completion = RouteBindCompletion {
6954 route,
6955 identity,
6956 bind_root_id: root.clone(),
6957 inserted_new_actor: false,
6958 configure_response: Response::success("subc-bind-7", json!({})),
6959 diagnostics_on_edit: false,
6960 ver: PROTOCOL_VERSION,
6961 corr: 91,
6962 flags: control_flags(),
6963 };
6964 let mut pending_binds = HashMap::from([(
6965 route,
6966 PendingBind {
6967 bind_root_id: root,
6968 inserted_new_actor: false,
6969 cancelled: false,
6970 configure_request_id: "subc-bind-7".to_string(),
6971 started_at: Instant::now(),
6972 warned_half_deadline: false,
6973 deadline_reported: false,
6974 corr: 91,
6975 ver: PROTOCOL_VERSION,
6976 flags: control_flags(),
6977 cancellation: crate::executor::JobCancellation::new(),
6978 },
6979 )]);
6980 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6981 let mut push_buffer =
6982 HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
6983 let (writer_tx, mut writer_rx) = mpsc::channel(8);
6984 let metrics = Arc::new(DispatchPathMetrics::new());
6985
6986 handle_route_bind_completion(
6987 &writer_tx,
6988 completion,
6989 &mut HashMap::new(),
6990 &mut HashMap::new(),
6991 &mut HashMap::new(),
6992 &mut push_buffer,
6993 &mut HashMap::new(),
6994 &mut pending_binds,
6995 &mut installed_route_epochs,
6996 &Arc::new(Executor::new()),
6997 &Arc::new(Notify::new()),
6998 &metrics,
6999 )
7000 .await
7001 .unwrap();
7002
7003 let ack = writer_rx.try_recv().expect("RouteBindAck");
7004 assert_eq!(ack.header.ty, FrameType::Response);
7005 assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
7006 let route_frame = writer_rx.try_recv().expect("post-ack route frame");
7007 assert_eq!(route_frame.header.ty, FrameType::Push);
7008 assert_eq!(
7009 (route_frame.header.channel, route_frame.header.epoch),
7010 (route.channel, route.epoch)
7011 );
7012 }
7013}