1use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, Lane};
31use crate::jsonc::strip_jsonc;
32use crate::log_ctx;
33use crate::path_identity::ProjectRootId;
34use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
35use crate::run_tool_call::{
36 run_tool_call, strip_agent_preview_arg_owned, PhaseTrace, ToolCallContext, ToolCallOutcome,
37 ToolCallResult,
38};
39use crate::runtime_drain;
40
41use subc_protocol::manifest::{
42 Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
43 ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
44};
45use subc_protocol::session::{
46 HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
47 MODULE_CONTROL_OP_HEALTH_CHECK,
48};
49use subc_protocol::{
50 ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, PROTOCOL_VERSION,
51};
52use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
53use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
54use tokio::net::TcpStream;
55use tokio::sync::{mpsc, oneshot, Notify};
56use tokio::task::JoinHandle;
57
58const AUTH_DEADLINE: Duration = Duration::from_secs(5);
61const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
62const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
63const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
64const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
65
66const HELLO_CORR: u64 = 1;
68
69const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
72
73const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
77
78const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
82
83const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
87
88const WRITER_QUEUE_CAPACITY: usize = 256;
89
90const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
93
94const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
102const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
103 MaintenanceDrainKind::Watcher,
104 MaintenanceDrainKind::Lsp,
105 MaintenanceDrainKind::ConfigureTail,
106 MaintenanceDrainKind::CompletionDrains,
107];
108#[cfg(test)]
109const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
110
111const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
112const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
113
114const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
115const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
116
117const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
120
121const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
125
126const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
128const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
129
130#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
131struct RouteChannel {
132 channel: u16,
133 epoch: u32,
134}
135
136impl fmt::Display for RouteChannel {
137 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
138 write!(f, "{}@{}", self.channel, self.epoch)
139 }
140}
141
142type PushEnvelope = (ProjectRootId, PushFrame);
143type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
144type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
145mod bash;
146mod health;
147mod manifest;
148mod push;
149mod wire;
150
151use self::health::{
152 build_health_report, warn_slow_pending_binds, DispatchPathMetrics, ResponseTaskGuard,
153};
154use self::manifest::{
155 build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
156 is_subc_agent_core_tool, is_subc_native_plumbing_tool,
157};
158pub use self::wire::SubcError;
159
160pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
164 manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
165}
166use self::wire::{
167 build_error_frame, build_goodbye_frame, build_tool_response_frame, decrement_counted_channel,
168 response_is_fatal_panic, response_message, send_counted_channel, send_frame,
169 send_reliable_writer_frame, send_traced_tool_response_frame, ToolResponseWriteTrace,
170 WriterFrame, WriterSender,
171};
172
173struct DecodedFrame {
174 frame: Frame,
175 phase_trace: PhaseTrace,
176}
177
178struct ToolCallCompletion {
179 text: String,
180 phase_trace: PhaseTrace,
181}
182
183#[derive(Clone)]
184struct PushSenders {
185 lossy_tx: mpsc::Sender<LossyPushEnvelope>,
186 reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
187 lossy_overflow: Arc<push::LossyOverflow>,
188 lossy_seq: Arc<AtomicU64>,
189}
190
191#[derive(Clone)]
192struct PersistentCancelSignal {
193 inner: Arc<PersistentCancelInner>,
194}
195
196struct PersistentCancelInner {
197 cancelled: AtomicBool,
198 notify: Notify,
199}
200
201impl PersistentCancelSignal {
202 fn new() -> Self {
203 Self {
204 inner: Arc::new(PersistentCancelInner {
205 cancelled: AtomicBool::new(false),
206 notify: Notify::new(),
207 }),
208 }
209 }
210
211 fn cancel(&self) {
212 if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
213 self.inner.notify.notify_waiters();
214 }
215 }
216
217 fn is_cancelled(&self) -> bool {
218 self.inner.cancelled.load(Ordering::SeqCst)
219 }
220
221 async fn cancelled(&self) {
222 loop {
230 let notified = self.inner.notify.notified();
231 tokio::pin!(notified);
232 notified.as_mut().enable();
233 if self.is_cancelled() {
234 return;
235 }
236 notified.await;
237 }
238 }
239}
240
241#[derive(Debug, Clone, Copy, PartialEq, Eq)]
242pub(crate) enum BindTrust {
243 FirstParty,
244 Untrusted,
245}
246
247impl BindTrust {
248 fn allows_bash_observation(self) -> bool {
249 matches!(self, Self::FirstParty)
250 }
251
252 fn label(self) -> &'static str {
253 match self {
254 Self::FirstParty => "first_party",
255 Self::Untrusted => "untrusted",
256 }
257 }
258}
259
260pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
261 match principal {
262 Some(Principal::Direct) => BindTrust::FirstParty,
263 Some(Principal::Reserved { module_id })
264 if module_id == "llm-runner"
265 || module_id == "aft"
266 || module_id == "broca"
267 || module_id == "alfonso-core" =>
268 {
269 BindTrust::FirstParty
270 }
271 Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
272 BindTrust::Untrusted
273 }
274 }
275}
276
277fn harness_forces_untrusted(harness: &str) -> bool {
278 harness.starts_with("fed:")
279}
280
281pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
282 if harness_forces_untrusted(harness) {
283 BindTrust::Untrusted
284 } else {
285 trust_for_principal(principal)
286 }
287}
288
289fn principal_label(principal: &Option<Principal>) -> String {
290 match principal {
291 Some(Principal::Direct) => "direct".to_string(),
292 Some(Principal::Reserved { module_id }) => format!("reserved:{module_id}"),
293 Some(Principal::Unverified) => "unverified".to_string(),
294 None => "absent".to_string(),
295 }
296}
297
298#[derive(Debug)]
299struct RootMeta {
305 maintenance_pending: bool,
306 maintenance_jobs_in_flight: usize,
307 maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
308 maintenance_last_submitted: Option<Instant>,
309 maintenance_poisoned: bool,
310 last_touched: Instant,
311 diagnostics_on_edit: bool,
312 active_bash_waits: usize,
313 idle_artifacts_evicted: bool,
314 unbound_quiesced: bool,
315}
316
317#[derive(Debug)]
318struct PendingBind {
319 bind_root_id: ProjectRootId,
320 inserted_new_actor: bool,
321 cancelled: bool,
322 configure_request_id: String,
323 started_at: Instant,
324 warned_half_deadline: bool,
325 deadline_reported: bool,
326 corr: u64,
327 ver: u8,
328 flags: Flags,
329 cancellation: crate::executor::JobCancellation,
334}
335
336struct RouteBindCompletion {
337 route: RouteChannel,
338 identity: RouteIdentity,
339 bind_root_id: ProjectRootId,
340 inserted_new_actor: bool,
341 configure_response: Response,
342 diagnostics_on_edit: bool,
343 ver: u8,
344 corr: u64,
345 flags: Flags,
346}
347
348#[derive(Debug, Clone)]
349struct RouteIdentity(Arc<RouteIdentityData>);
350
351#[derive(Debug)]
352struct RouteIdentityData {
353 root: ProjectRootId,
354 project_root: PathBuf,
355 harness: String,
356 session: String,
357 trust: BindTrust,
358 consumer_elicitation_capable: bool,
359}
360
361impl Deref for RouteIdentity {
362 type Target = RouteIdentityData;
363
364 fn deref(&self) -> &Self::Target {
365 &self.0
366 }
367}
368
369#[derive(Debug, Clone)]
370struct RetainedSessionIdentity {
371 harness: String,
372 trust: BindTrust,
373}
374
375#[derive(Clone, Copy)]
376struct BgSub {
377 corr: u64,
378 ver: u8,
379 flags: Flags,
380}
381
382struct MaintenanceCompletion {
383 root_id: ProjectRootId,
384 kind: MaintenanceDrainKind,
385 response: Response,
386 empty_bg_sessions: Vec<(String, u64)>,
387 requeue_kind: Option<MaintenanceDrainKind>,
388}
389
390#[derive(Clone, Copy, Debug, PartialEq, Eq)]
391enum MaintenanceDrainKind {
392 Watcher,
393 Lsp,
394 ConfigureTail,
395 CompletionDrains,
396}
397
398impl MaintenanceDrainKind {
399 fn label(self) -> &'static str {
400 match self {
401 Self::Watcher => "watcher",
402 Self::Lsp => "lsp",
403 Self::ConfigureTail => "configure-tail",
404 Self::CompletionDrains => "completion-drains",
405 }
406 }
407}
408
409#[derive(Debug, Default)]
410struct MaintenanceJobOutcome {
411 empty_bg_sessions: Vec<(String, u64)>,
412 requeue_kind: Option<MaintenanceDrainKind>,
413}
414
415#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
416struct ReverseCorrKey {
417 route: RouteChannel,
418 corr: u64,
419}
420
421struct PendingBashAsk {
422 route: RouteChannel,
423 tool_corr: u64,
424 tool_flags: Flags,
425 tool_ver: u8,
426 root: ProjectRootId,
427 project_root: PathBuf,
428 session_id: String,
429 request_id: String,
430 arguments: Value,
431 format_context: crate::subc_format::FormatContext,
432 cancel: bash::BashWaitCancel,
433 grants: Vec<String>,
434 expires_at: Instant,
435}
436
437impl RootMeta {
438 fn new(now: Instant) -> Self {
439 Self {
440 maintenance_pending: false,
441 maintenance_jobs_in_flight: 0,
442 maintenance_queued_kinds: VecDeque::new(),
443 maintenance_last_submitted: None,
444 maintenance_poisoned: false,
445 last_touched: now,
446 diagnostics_on_edit: false,
447 active_bash_waits: 0,
448 idle_artifacts_evicted: false,
449 unbound_quiesced: false,
450 }
451 }
452
453 fn note_activity(&mut self) {
454 self.last_touched = Instant::now();
455 }
456
457 fn reactivate_bound(&mut self) {
458 self.note_activity();
459 self.idle_artifacts_evicted = false;
460 self.unbound_quiesced = false;
461 }
462}
463
464fn due_maintenance_jobs(
465 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
466 executor: Option<&Executor>,
467 bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
468 bg_wake_pending: &HashSet<RouteChannel>,
469 budget: usize,
470 pending_bind_roots: &HashSet<ProjectRootId>,
471) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
472 let mut jobs = Vec::new();
473 let mut deferred = false;
474 let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
475 roots.sort_by(|left, right| {
476 let left_last = live_roots
477 .get(left)
478 .and_then(|meta| meta.maintenance_last_submitted);
479 let right_last = live_roots
480 .get(right)
481 .and_then(|meta| meta.maintenance_last_submitted);
482 left_last
483 .cmp(&right_last)
484 .then_with(|| left.as_path().cmp(right.as_path()))
485 });
486
487 for root_id in roots {
488 let Some(meta) = live_roots.get_mut(&root_id) else {
489 continue;
490 };
491 if meta.maintenance_poisoned {
492 continue;
493 }
494
495 if pending_bind_roots.contains(&root_id) {
496 if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
497 deferred = true;
498 }
499 continue;
500 }
501
502 if !meta.maintenance_pending {
503 if jobs.len() >= budget {
504 deferred = true;
505 continue;
506 }
507 let executor_actor_context =
511 executor.and_then(|executor| executor.actor_context(&root_id));
512 let root_has_pending_bg_wake =
513 bg_sub_by_session.iter().any(|((sub_root, _), channel)| {
514 sub_root == &root_id && bg_wake_pending.contains(channel)
515 });
516 let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
517 Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
518 .into_iter()
519 .filter(|kind| {
520 if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
521 return false;
522 }
523 match kind {
524 MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
525 MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
526 MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
527 MaintenanceDrainKind::CompletionDrains => {
533 root_has_pending_bg_wake || ctx.completion_drains_have_work()
534 }
535 }
536 })
537 .collect(),
538 None if meta.unbound_quiesced => Vec::new(),
539 None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
541 };
542 if kinds_with_work.is_empty() {
543 continue;
544 }
545 meta.maintenance_pending = true;
546 meta.maintenance_queued_kinds.extend(kinds_with_work);
547 }
548
549 while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
550 if jobs.len() >= budget {
551 meta.maintenance_queued_kinds.push_front(kind);
552 deferred = true;
553 break;
554 }
555 meta.maintenance_jobs_in_flight += 1;
556 meta.maintenance_last_submitted = Some(Instant::now());
557 jobs.push((root_id.clone(), kind));
558 }
559
560 meta.maintenance_pending =
561 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
562 }
563
564 (jobs, deferred)
565}
566
567fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
568 match estimate.estimated_bytes {
569 Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
570 None if estimate.status == "busy" => "busy".to_string(),
571 None => "not estimated".to_string(),
572 }
573}
574
575fn optional_memory_label(bytes: Option<u64>) -> String {
576 bytes.map_or_else(
577 || "not estimated".to_string(),
578 |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
579 )
580}
581
582fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
583 format!(
584 "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
585 optional_memory_label(relief.rss_before_bytes),
586 optional_memory_label(relief.rss_after_bytes),
587 optional_memory_label(relief.allocator_before.bytes_in_use),
588 optional_memory_label(relief.allocator_after.bytes_in_use),
589 optional_memory_label(relief.allocator_before.size_allocated),
590 optional_memory_label(relief.allocator_after.size_allocated),
591 optional_memory_label(relief.allocator_before.retained_slack_bytes),
592 optional_memory_label(relief.allocator_after.retained_slack_bytes),
593 relief.bytes_released as f64 / (1024.0 * 1024.0),
594 )
595}
596
597fn idle_root_eviction_message(
598 root_id: &ProjectRootId,
599 memory: &crate::memory::RootMemorySnapshot,
600 pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
601) -> String {
602 let freed_bytes = [
605 &memory.semantic,
606 &memory.trigram,
607 &memory.symbols,
608 &memory.callgraph,
609 &memory.inspect,
610 ]
611 .iter()
612 .filter_map(|estimate| estimate.estimated_bytes)
613 .fold(0u64, u64::saturating_add);
614 let mut message = format!(
615 "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
616 root_id.as_path().display(),
617 freed_bytes as f64 / (1024.0 * 1024.0),
618 eviction_estimate_label(&memory.semantic),
619 eviction_estimate_label(&memory.trigram),
620 eviction_estimate_label(&memory.symbols),
621 eviction_estimate_label(&memory.callgraph),
622 eviction_estimate_label(&memory.inspect),
623 eviction_estimate_label(&memory.bash),
624 eviction_estimate_label(&memory.lsp),
625 eviction_estimate_label(&memory.parser_pool),
626 );
627 if let Some(pressure_relief) = pressure_relief {
628 message.push_str(&pressure_relief_label(pressure_relief));
629 }
630 message
631}
632
633fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
634 !live_roots.is_empty()
635 && live_roots.values().all(|meta| {
636 now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
637 && meta.active_bash_waits == 0
638 && !meta.maintenance_pending
639 && meta.maintenance_queued_kinds.is_empty()
640 })
641}
642
643fn allocator_pressure_relief_after_idle_sweep(
644 now: Instant,
645 live_roots: &HashMap<ProjectRootId, RootMeta>,
646 executor: &Executor,
647) -> Option<crate::memory::AllocatorPressureRelief> {
648 if !process_has_been_idle(now, live_roots)
649 || live_roots.keys().any(|root_id| {
650 executor
651 .actor_context(root_id)
652 .is_some_and(|ctx| ctx.artifact_eviction_blocked())
653 })
654 {
655 return None;
656 }
657
658 #[cfg(target_os = "macos")]
659 {
660 Some(crate::memory::relieve_allocator_pressure())
661 }
662 #[cfg(not(target_os = "macos"))]
663 {
664 None
665 }
666}
667
668fn quiesce_unbound_root(
669 root_id: &ProjectRootId,
670 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
671 executor: &Arc<Executor>,
672) {
673 let Some(meta) = live_roots.get_mut(root_id) else {
674 return;
675 };
676
677 let ctx = executor.actor_context(root_id);
678 if let Some(ctx) = ctx.as_ref() {
679 ctx.mark_subc_unbound();
683 }
684 let cancelled = executor.cancel_queued_maintenance(root_id);
685 let discarded = ctx
693 .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
694 .unwrap_or(0);
695 meta.unbound_quiesced = true;
696 meta.maintenance_queued_kinds.clear();
697 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
698 log::debug!(
699 "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s))",
700 root_id.as_path().display(),
701 cancelled,
702 discarded
703 );
704}
705
706#[derive(Debug, Default)]
707struct IdleReapOutcome {
708 evicted: usize,
709 forgotten_deleted_roots: Vec<ProjectRootId>,
710}
711
712fn reap_idle_roots(
713 now: Instant,
714 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
715 pending_binds: &HashMap<RouteChannel, PendingBind>,
716 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
717 executor: &Arc<Executor>,
718) -> IdleReapOutcome {
719 let pending_bind_roots = pending_binds
720 .values()
721 .map(|pending| pending.bind_root_id.clone())
722 .collect::<HashSet<_>>();
723 let candidates = live_roots
724 .iter()
725 .filter_map(|(root_id, meta)| {
726 let deleted = !root_id.as_path().exists();
727 let has_bound_route = root_channels
728 .get(root_id)
729 .is_some_and(|channels| !channels.is_empty());
730 if has_bound_route
734 || !meta.unbound_quiesced
735 || (!deleted
736 && (meta.idle_artifacts_evicted
737 || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL))
738 || meta.active_bash_waits > 0
739 || meta.maintenance_pending
740 || !meta.maintenance_queued_kinds.is_empty()
741 || pending_bind_roots.contains(root_id)
742 || !executor.actor_is_idle(root_id)
743 {
744 return None;
745 }
746 Some((root_id.clone(), deleted))
747 })
748 .collect::<Vec<_>>();
749
750 let mut reaped = Vec::new();
751 let mut forgotten_deleted_roots = Vec::new();
752 for (root_id, deleted) in candidates {
753 let Some(ctx) = executor.actor_context(&root_id) else {
754 continue;
755 };
756 let taken_pending = Some(ctx.take_pending_reconciliation_state());
761 if ctx.artifact_eviction_blocked() {
762 if let Some(pending) = taken_pending {
763 ctx.restore_pending_reconciliation_state(pending);
764 }
765 continue;
766 }
767 let memory_before = ctx.memory_root_snapshot();
768 if !ctx.evict_idle_artifacts() {
769 if let Some(pending) = taken_pending {
770 ctx.restore_pending_reconciliation_state(pending);
771 }
772 continue;
773 }
774 drop(taken_pending);
775 ctx.stop_watcher_runtime_in_background();
776 ctx.invalidate_artifacts_after_watcher_gap();
779
780 if deleted {
781 if executor.retire_idle_actor_in_background(&root_id) {
782 live_roots.remove(&root_id);
783 forgotten_deleted_roots.push(root_id.clone());
784 }
785 } else {
786 if let Some(meta) = live_roots.get_mut(&root_id) {
787 meta.idle_artifacts_evicted = true;
788 }
789 ctx.release_idle_reopenable_resources_in_background();
790 }
791 reaped.push((root_id, memory_before));
792 }
793
794 let pressure_relief = (!reaped.is_empty())
795 .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
796 .flatten();
797 for (root_id, memory_before) in &reaped {
798 log::info!(
799 "{}",
800 idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
801 );
802 }
803 IdleReapOutcome {
804 evicted: reaped.len(),
805 forgotten_deleted_roots,
806 }
807}
808
809#[allow(clippy::too_many_arguments)]
810fn purge_deleted_root_residents(
811 root_id: &ProjectRootId,
812 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
813 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
814 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
815 bg_subs: &mut HashMap<RouteChannel, BgSub>,
816 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
817 bg_wake_pending: &mut HashSet<RouteChannel>,
818 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
819 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
820) {
821 root_channels.remove(root_id);
822 session_identity.retain(|(root, _), _| root != root_id);
823 push_buffer.retain(|key, _| &key.root != root_id);
824 bg_wake_epoch.retain(|(root, _), _| root != root_id);
825 pending_bash_asks.retain(|_, ask| &ask.root != root_id);
826
827 let stale_subscriptions = bg_sub_by_session
828 .iter()
829 .filter_map(|((root, _), channel)| (root == root_id).then_some(*channel))
830 .collect::<Vec<_>>();
831 bg_sub_by_session.retain(|(root, _), _| root != root_id);
832 for channel in stale_subscriptions {
833 bg_subs.remove(&channel);
834 bg_wake_pending.remove(&channel);
835 }
836 log::info!(
837 "subc attach: fully forgot deleted root {}",
838 root_id.as_path().display()
839 );
840}
841
842#[allow(clippy::too_many_arguments)]
843fn submit_due_maintenance_jobs(
844 executor: &Arc<Executor>,
845 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
846 pending_binds: &HashMap<RouteChannel, PendingBind>,
847 bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
848 bg_wake_pending: &HashSet<RouteChannel>,
849 bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
850 maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
851 metrics: &Arc<DispatchPathMetrics>,
852) {
853 let pending_bind_roots = pending_binds
854 .values()
855 .map(|pending| pending.bind_root_id.clone())
856 .collect::<HashSet<_>>();
857 let (due_jobs, deferred_jobs) = due_maintenance_jobs(
858 live_roots,
859 Some(executor),
860 bg_sub_by_session,
861 bg_wake_pending,
862 MAINTENANCE_SUBMIT_BUDGET,
863 &pending_bind_roots,
864 );
865 if deferred_jobs {
866 metrics
867 .maintenance_budget_deferrals
868 .fetch_add(1, Ordering::Relaxed);
869 }
870 for (root_id, kind) in due_jobs {
871 let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
872 bg_sub_by_session
873 .iter()
874 .filter_map(|((root, session), _)| {
875 if root == &root_id {
876 Some((
877 session.clone(),
878 bg_wake_epoch
879 .get(&(root_id.clone(), session.clone()))
880 .copied()
881 .unwrap_or(0),
882 ))
883 } else {
884 None
885 }
886 })
887 .collect()
888 } else {
889 Vec::new()
890 };
891 submit_maintenance_job(
892 executor,
893 root_id,
894 kind,
895 bg_sessions_to_check,
896 maintenance_tx,
897 metrics,
898 );
899 }
900}
901
902fn should_requiesce_after_maintenance(
903 meta: &RootMeta,
904 completed_kind: MaintenanceDrainKind,
905 bind_pending: bool,
906) -> bool {
907 meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
908}
909
910fn note_maintenance_completion(
911 meta: &mut RootMeta,
912 requeue_kind: Option<MaintenanceDrainKind>,
913 fatal: bool,
914 defer_requeue: bool,
915) {
916 if fatal {
917 meta.maintenance_poisoned = true;
918 }
919
920 if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
921 meta.maintenance_queued_kinds.push_back(kind);
922 }
923
924 meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
925 meta.maintenance_pending =
926 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
927}
928
929fn route_key(channel: u16, epoch: u32) -> RouteChannel {
930 RouteChannel { channel, epoch }
931}
932
933fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
934 if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
935 installed_epochs.remove(&route.channel);
936 }
937}
938
939fn ingress_route_is_current(installed_epochs: &HashMap<u16, u32>, frame: &Frame) -> bool {
940 frame.header.channel == 0
941 || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
942}
943
944fn bash_elicitation_timeout() -> Duration {
945 if cfg!(debug_assertions) {
946 if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
947 if let Ok(ms) = raw.parse::<u64>() {
948 if ms > 0 {
949 return Duration::from_millis(ms);
950 }
951 }
952 }
953 }
954 BASH_ELICITATION_TIMEOUT
955}
956
957fn allocate_reverse_corr(
958 pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
959 route: RouteChannel,
960 next_corr: &mut u64,
961) -> u64 {
962 loop {
963 let corr = *next_corr;
964 *next_corr = (*next_corr).wrapping_add(1).max(1);
965 if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
966 return corr;
967 }
968 }
969}
970
971fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
972 match kind {
973 crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
974 crate::bash_permissions::PermissionKind::Bash => "bash",
975 }
976}
977
978fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
979 let mut patterns = Vec::new();
980 let mut seen = HashSet::new();
981 for ask in asks {
982 for pattern in ask.patterns.iter().chain(ask.always.iter()) {
983 if seen.insert(pattern.clone()) {
984 patterns.push(pattern.clone());
985 }
986 }
987 }
988 patterns
989}
990
991fn bash_elicitation_message(
992 command: &str,
993 asks: &[crate::bash_permissions::PermissionAsk],
994) -> String {
995 let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
996 let patterns = bash_elicitation_patterns(asks);
997 let pattern_text = if patterns.is_empty() {
998 "no matched permission patterns".to_string()
999 } else {
1000 patterns.join(", ")
1001 };
1002 let ask_kinds = asks
1003 .iter()
1004 .map(|ask| bash_permission_kind_label(&ask.kind))
1005 .collect::<HashSet<_>>()
1006 .into_iter()
1007 .collect::<Vec<_>>()
1008 .join(", ");
1009 if ask_kinds.is_empty() {
1010 format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1011 } else {
1012 format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1013 }
1014}
1015
1016fn bash_elicitation_request_body(
1017 command: &str,
1018 asks: &[crate::bash_permissions::PermissionAsk],
1019) -> Value {
1020 json!({
1021 "method": BASH_ELICITATION_CREATE_METHOD,
1022 "params": {
1023 "mode": "form",
1024 "message": bash_elicitation_message(command, asks),
1025 "requestedSchema": {
1026 "type": "object",
1027 "properties": {
1028 "decision": {
1029 "type": "string",
1030 "enum": ["allow", "deny"],
1031 "description": "Choose allow to run this bash command once, or deny to block it."
1032 }
1033 },
1034 "required": ["decision"],
1035 "additionalProperties": false
1036 },
1037 "_meta": {
1038 "aft": {
1039 "tool": "bash",
1040 "command": command,
1041 "asks": asks
1042 }
1043 }
1044 }
1045 })
1046}
1047
1048fn build_bash_elicitation_request_frame(
1049 ver: u8,
1050 route: RouteChannel,
1051 corr: u64,
1052 flags: Flags,
1053 command: &str,
1054 asks: &[crate::bash_permissions::PermissionAsk],
1055) -> Result<Frame, SubcError> {
1056 let body = bash_elicitation_request_body(command, asks);
1057 Frame::build_with_version(
1058 ver,
1059 FrameType::Request,
1060 flags,
1061 route.channel,
1062 route.epoch,
1063 corr,
1064 serde_json::to_vec(&body).map_err(SubcError::Json)?,
1065 )
1066 .map_err(SubcError::FrameBuild)
1067}
1068
1069fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1070 let Ok(value) = serde_json::from_slice::<Value>(body) else {
1071 return false;
1072 };
1073 flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1074}
1075
1076fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1077 let Some(object) = value.as_object() else {
1078 return false;
1079 };
1080 object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1081}
1082
1083fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1084 let Some(object) = value.as_object() else {
1085 return false;
1086 };
1087 if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1088 return false;
1089 }
1090 let Some(content) = object.get("content").and_then(Value::as_object) else {
1091 return false;
1092 };
1093 content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1094}
1095
1096#[allow(clippy::too_many_arguments)]
1097async fn settle_pending_bash_ask_denied(
1098 tx: &WriterSender,
1099 pending: PendingBashAsk,
1100 routes: &HashMap<RouteChannel, RouteIdentity>,
1101 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1102 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1103 shutdown: &Arc<Notify>,
1104 metrics: &DispatchPathMetrics,
1105) -> Result<(), SubcError> {
1106 let completion = bash::bash_denied_untrusted_completion(
1107 pending.route,
1108 pending.tool_corr,
1109 pending.tool_flags,
1110 pending.tool_ver,
1111 pending.root,
1112 pending.request_id,
1113 pending.format_context,
1114 );
1115 bash::handle_bash_deferred_completion(
1116 tx,
1117 completion,
1118 routes,
1119 live_roots,
1120 route_bash_cancels,
1121 shutdown,
1122 metrics,
1123 )
1124 .await
1125}
1126
1127fn take_pending_bash_asks_for_route(
1128 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1129 route: RouteChannel,
1130) -> Vec<PendingBashAsk> {
1131 let keys = pending_bash_asks
1132 .keys()
1133 .copied()
1134 .filter(|key| key.route == route)
1135 .collect::<Vec<_>>();
1136 keys.into_iter()
1137 .filter_map(|key| pending_bash_asks.remove(&key))
1138 .collect()
1139}
1140
1141#[allow(clippy::too_many_arguments)]
1142async fn settle_pending_bash_asks_for_route(
1143 tx: &WriterSender,
1144 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1145 route: RouteChannel,
1146 routes: &HashMap<RouteChannel, RouteIdentity>,
1147 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1148 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1149 shutdown: &Arc<Notify>,
1150 metrics: &DispatchPathMetrics,
1151) -> Result<(), SubcError> {
1152 for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1153 settle_pending_bash_ask_denied(
1154 tx,
1155 pending,
1156 routes,
1157 live_roots,
1158 route_bash_cancels,
1159 shutdown,
1160 metrics,
1161 )
1162 .await?;
1163 }
1164 Ok(())
1165}
1166
1167#[allow(clippy::too_many_arguments)]
1168async fn settle_all_pending_bash_asks(
1169 tx: &WriterSender,
1170 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1171 routes: &HashMap<RouteChannel, RouteIdentity>,
1172 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1173 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1174 shutdown: &Arc<Notify>,
1175 metrics: &DispatchPathMetrics,
1176) -> Result<(), SubcError> {
1177 let pending = pending_bash_asks
1178 .drain()
1179 .map(|(_, pending)| pending)
1180 .collect::<Vec<_>>();
1181 for pending in pending {
1182 settle_pending_bash_ask_denied(
1183 tx,
1184 pending,
1185 routes,
1186 live_roots,
1187 route_bash_cancels,
1188 shutdown,
1189 metrics,
1190 )
1191 .await?;
1192 }
1193 Ok(())
1194}
1195
1196#[allow(clippy::too_many_arguments)]
1197async fn expire_pending_bash_asks(
1198 tx: &WriterSender,
1199 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1200 routes: &HashMap<RouteChannel, RouteIdentity>,
1201 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1202 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1203 shutdown: &Arc<Notify>,
1204 metrics: &DispatchPathMetrics,
1205) -> Result<(), SubcError> {
1206 let now = Instant::now();
1207 let expired = pending_bash_asks
1208 .iter()
1209 .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1210 .collect::<Vec<_>>();
1211 for key in expired {
1212 if let Some(pending) = pending_bash_asks.remove(&key) {
1213 log::debug!(
1214 "subc attach: bash elicitation request {} on route {} expired fail-closed",
1215 key.corr,
1216 pending.route
1217 );
1218 settle_pending_bash_ask_denied(
1219 tx,
1220 pending,
1221 routes,
1222 live_roots,
1223 route_bash_cancels,
1224 shutdown,
1225 metrics,
1226 )
1227 .await?;
1228 }
1229 }
1230 Ok(())
1231}
1232
1233#[allow(clippy::too_many_arguments)]
1234async fn handle_bash_elicitation_reply(
1235 tx: &WriterSender,
1236 frame: &Frame,
1237 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1238 routes: &HashMap<RouteChannel, RouteIdentity>,
1239 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1240 executor: &Arc<Executor>,
1241 shutdown: &Arc<Notify>,
1242 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1243 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1244 metrics: &Arc<DispatchPathMetrics>,
1245 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1246 dispatch: DispatchFn,
1247) -> Result<(), SubcError> {
1248 let key = ReverseCorrKey {
1249 route: route_key(frame.header.channel, frame.header.epoch),
1250 corr: frame.header.corr,
1251 };
1252 let Some(pending) = pending_bash_asks.remove(&key) else {
1253 return Ok(());
1254 };
1255
1256 if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1257 if routes.contains_key(&key.route) {
1258 bash::submit_deferred_bash(
1259 executor,
1260 bash_deferred_tx,
1261 bash_poll_touch_tx,
1262 metrics,
1263 dispatch,
1264 pending.root,
1265 pending.project_root,
1266 pending.session_id,
1267 pending.request_id,
1268 pending.route,
1269 pending.tool_corr,
1270 pending.tool_flags,
1271 pending.tool_ver,
1272 pending.arguments,
1273 pending.format_context,
1274 pending.cancel,
1275 BindTrust::Untrusted,
1276 Some(pending.grants),
1277 );
1278 return Ok(());
1279 }
1280 log::debug!(
1281 "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1282 key.corr,
1283 pending.route
1284 );
1285 }
1286
1287 settle_pending_bash_ask_denied(
1288 tx,
1289 pending,
1290 routes,
1291 live_roots,
1292 route_bash_cancels,
1293 shutdown,
1294 metrics,
1295 )
1296 .await
1297}
1298
1299#[allow(clippy::too_many_arguments)]
1300async fn cancel_pending_bash_ask_for_tool_call(
1301 tx: &WriterSender,
1302 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1303 route: RouteChannel,
1304 tool_corr: u64,
1305 routes: &HashMap<RouteChannel, RouteIdentity>,
1306 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1307 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1308 shutdown: &Arc<Notify>,
1309 metrics: &DispatchPathMetrics,
1310) -> Result<(), SubcError> {
1311 let keys = pending_bash_asks
1312 .iter()
1313 .filter_map(|(key, pending)| {
1314 (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1315 })
1316 .collect::<Vec<_>>();
1317 for key in keys {
1318 if let Some(pending) = pending_bash_asks.remove(&key) {
1319 settle_pending_bash_ask_denied(
1320 tx,
1321 pending,
1322 routes,
1323 live_roots,
1324 route_bash_cancels,
1325 shutdown,
1326 metrics,
1327 )
1328 .await?;
1329 }
1330 }
1331 Ok(())
1332}
1333
1334fn remove_root_channel(
1335 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1336 root: &ProjectRootId,
1337 channel: RouteChannel,
1338) {
1339 let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1340 channels.remove(&channel);
1341 channels.is_empty()
1342 } else {
1343 false
1344 };
1345 if remove_root {
1346 root_channels.remove(root);
1347 }
1348}
1349
1350fn remove_route_channel(
1351 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1352 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1353 channel: RouteChannel,
1354) -> Option<RouteIdentity> {
1355 let removed = routes.remove(&channel);
1356 if let Some(identity) = &removed {
1357 remove_root_channel(root_channels, &identity.root, channel);
1358 }
1359 removed
1360}
1361
1362fn insert_route_channel(
1363 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1364 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1365 channel: RouteChannel,
1366 identity: RouteIdentity,
1367) {
1368 if let Some(previous) = routes.insert(channel, identity.clone()) {
1369 remove_root_channel(root_channels, &previous.root, channel);
1370 }
1371 root_channels
1372 .entry(identity.root.clone())
1373 .or_default()
1374 .insert(channel);
1375}
1376
1377fn remove_bg_subscription_index(
1378 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1379 channel: RouteChannel,
1380 identity: Option<&RouteIdentity>,
1381) {
1382 if let Some(identity) = identity {
1383 let key = (identity.root.clone(), identity.session.clone());
1384 if bg_sub_by_session.get(&key).copied() == Some(channel) {
1385 bg_sub_by_session.remove(&key);
1386 }
1387 } else {
1388 bg_sub_by_session.retain(|_, mapped_channel| *mapped_channel != channel);
1389 }
1390}
1391
1392fn route_removal_will_quiesce_root(
1393 root: &ProjectRootId,
1394 route: RouteChannel,
1395 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1396 has_pending_bind: bool,
1397 replacement_root: Option<&ProjectRootId>,
1398) -> bool {
1399 let removes_last_route = root_channels
1400 .get(root)
1401 .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1402 removes_last_route && !has_pending_bind && replacement_root != Some(root)
1403}
1404
1405fn should_quiesce_removed_root(
1406 root: &ProjectRootId,
1407 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1408 has_pending_bind: bool,
1409 replacement_root: Option<&ProjectRootId>,
1410) -> bool {
1411 !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1412}
1413
1414async fn end_bg_subscription(
1415 writer_tx: &WriterSender,
1416 metrics: &DispatchPathMetrics,
1417 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1418 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1419 bg_wake_pending: &mut HashSet<RouteChannel>,
1420 channel: RouteChannel,
1421 identity: Option<&RouteIdentity>,
1422) -> Result<(), SubcError> {
1423 if let Some(sub) = bg_subs.remove(&channel) {
1424 bg_wake_pending.remove(&channel);
1425 remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1426 push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1427 }
1428 Ok(())
1429}
1430
1431#[allow(clippy::too_many_arguments)]
1432async fn teardown_installed_route(
1433 tx: &WriterSender,
1434 metrics: &DispatchPathMetrics,
1435 executor: &Arc<Executor>,
1436 channel: RouteChannel,
1437 cancellation_reason: &str,
1438 replacement_root: Option<&ProjectRootId>,
1439 installed_route_epochs: &mut HashMap<u16, u32>,
1440 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1441 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1442 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1443 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1444 bg_wake_pending: &mut HashSet<RouteChannel>,
1445 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1446 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1447 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1448 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1449 retry_buffer: &mut RetryBuffer,
1450 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1451 shutdown: &Arc<Notify>,
1452) -> Result<(), SubcError> {
1453 remove_installed_route(installed_route_epochs, channel);
1454 end_bg_subscription(
1455 tx,
1456 metrics,
1457 bg_subs,
1458 bg_sub_by_session,
1459 bg_wake_pending,
1460 channel,
1461 routes.get(&channel),
1462 )
1463 .await?;
1464 settle_pending_bash_asks_for_route(
1465 tx,
1466 pending_bash_asks,
1467 channel,
1468 routes,
1469 live_roots,
1470 route_bash_cancels,
1471 shutdown,
1472 metrics,
1473 )
1474 .await?;
1475 if let Some(cancel) = route_bash_cancels.remove(&channel) {
1476 cancel.token.cancel();
1477 }
1478 if let Some(pending) = pending_binds.get_mut(&channel) {
1479 pending.cancelled = true;
1480 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1481 log::debug!(
1482 "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1483 channel.channel
1484 );
1485 }
1486 let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1487 if let Some(identity) = routes.get(&channel) {
1488 let has_pending_bind = pending_binds
1489 .values()
1490 .any(|pending| pending.bind_root_id == identity.root);
1491 if route_removal_will_quiesce_root(
1492 &identity.root,
1493 channel,
1494 root_channels,
1495 has_pending_bind,
1496 replacement_root,
1497 ) {
1498 if let Some(ctx) = executor.actor_context(&identity.root) {
1499 ctx.mark_subc_unbound();
1502 }
1503 }
1504 }
1505 if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
1506 if migrated > 0 {
1507 log::debug!(
1508 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
1509 channel.channel
1510 );
1511 }
1512 if let Some(meta) = live_roots.get_mut(&identity.root) {
1513 let idle_for = meta.last_touched.elapsed();
1514 meta.note_activity();
1515 log::debug!(
1516 "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
1517 channel.channel,
1518 identity.root.as_path().display(),
1519 identity.harness,
1520 identity.session,
1521 idle_for
1522 );
1523 } else {
1524 log::debug!(
1525 "subc attach: route {} torn down for root {} harness {} session {}",
1526 channel.channel,
1527 identity.root.as_path().display(),
1528 identity.harness,
1529 identity.session
1530 );
1531 }
1532 let has_pending_bind = pending_binds
1533 .values()
1534 .any(|pending| pending.bind_root_id == identity.root);
1535 if should_quiesce_removed_root(
1536 &identity.root,
1537 root_channels,
1538 has_pending_bind,
1539 replacement_root,
1540 ) {
1541 quiesce_unbound_root(&identity.root, live_roots, executor);
1542 }
1543 } else {
1544 if migrated > 0 {
1545 log::debug!(
1546 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
1547 channel.channel
1548 );
1549 }
1550 log::debug!("subc attach: unbound route {} torn down", channel.channel);
1551 }
1552 Ok(())
1553}
1554
1555fn remember_session_identity(
1556 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1557 identity: &RouteIdentity,
1558) {
1559 let key = (identity.root.clone(), identity.session.clone());
1560 if matches!(identity.trust, BindTrust::Untrusted)
1561 && session_identity
1562 .get(&key)
1563 .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
1564 {
1565 return;
1566 }
1567
1568 session_identity.insert(
1573 key,
1574 RetainedSessionIdentity {
1575 harness: identity.harness.clone(),
1576 trust: identity.trust,
1577 },
1578 );
1579}
1580
1581fn replay_key_for_session(
1582 session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1583 root: &ProjectRootId,
1584 session: &str,
1585) -> Option<(push::ReplayKey, BindTrust)> {
1586 let retained = session_identity.get(&(root.clone(), session.to_string()))?;
1587 Some((
1588 push::ReplayKey {
1589 root: root.clone(),
1590 harness: retained.harness.clone(),
1591 session: session.to_string(),
1592 },
1593 retained.trust,
1594 ))
1595}
1596pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
1599
1600#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1601enum ModuleLoopExit {
1602 Graceful,
1603 SkipSearchFlush,
1604}
1605
1606pub fn run_subc_mode(
1611 connection_file_path: &Path,
1612 ctx: Arc<AppContext>,
1613 executor: Arc<Executor>,
1614 dispatch: DispatchFn,
1615 user_config_path: Option<PathBuf>,
1616) -> Result<(), SubcError> {
1617 run_subc_mode_inner(
1621 connection_file_path,
1622 ctx,
1623 executor,
1624 dispatch,
1625 user_config_path,
1626 false,
1627 )
1628}
1629
1630fn run_subc_mode_inner(
1631 connection_file_path: &Path,
1632 ctx: Arc<AppContext>,
1633 executor: Arc<Executor>,
1634 dispatch: DispatchFn,
1635 user_config_path: Option<PathBuf>,
1636 allow_native_passthrough: bool,
1637) -> Result<(), SubcError> {
1638 let runtime = tokio::runtime::Builder::new_current_thread()
1639 .enable_all()
1640 .build()
1641 .map_err(SubcError::Runtime)?;
1642
1643 let executor_for_loop = Arc::clone(&executor);
1644 let loop_result = runtime.block_on(async move {
1645 let shared_app = ctx.app();
1646 drop(ctx);
1647 let stream = connect_and_authenticate(connection_file_path).await?;
1648 log::info!(
1649 "subc attach: authenticated to daemon via {}",
1650 connection_file_path.display()
1651 );
1652 let (read_half, write_half) = tokio::io::split(stream);
1653 run_module_loop(
1654 read_half,
1655 write_half,
1656 shared_app,
1657 executor_for_loop,
1658 dispatch,
1659 user_config_path,
1660 allow_native_passthrough,
1661 )
1662 .await
1663 });
1664
1665 let actor_contexts = executor.actor_contexts();
1666 if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
1667 flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
1670 }
1671 for actor_ctx in &actor_contexts {
1672 actor_ctx.lsp().shutdown_all();
1673 actor_ctx.bash_background().detach();
1674 }
1675
1676 loop_result.map(|_| ())
1677}
1678
1679fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
1680 for actor_ctx in actor_contexts {
1681 let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
1682 }
1683 let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
1684}
1685
1686#[doc(hidden)]
1691pub fn run_subc_mode_for_test(
1692 connection_file_path: &Path,
1693 ctx: Arc<AppContext>,
1694 executor: Arc<Executor>,
1695 dispatch: DispatchFn,
1696 user_config_path: Option<PathBuf>,
1697) -> Result<(), SubcError> {
1698 run_subc_mode_inner(
1699 connection_file_path,
1700 ctx,
1701 executor,
1702 dispatch,
1703 user_config_path,
1704 true,
1705 )
1706}
1707
1708#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1709enum AttachErrorClass {
1710 Transient,
1711 Permanent,
1712}
1713
1714impl fmt::Display for AttachErrorClass {
1715 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1716 match self {
1717 Self::Transient => f.write_str("transient"),
1718 Self::Permanent => f.write_str("permanent"),
1719 }
1720 }
1721}
1722
1723#[derive(Clone, Copy)]
1724struct AttachRetryPolicy {
1725 budget: Duration,
1726 initial_backoff: Duration,
1727 max_backoff: Duration,
1728 jitter_percent: u64,
1729}
1730
1731const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
1732 budget: ATTACH_RETRY_BUDGET,
1733 initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
1734 max_backoff: ATTACH_RETRY_MAX_BACKOFF,
1735 jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
1736};
1737
1738fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
1741 let transient = match error {
1742 SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
1743 SubcError::Auth { source, .. } => match source {
1744 subc_transport::AuthError::Timeout { .. }
1745 | subc_transport::AuthError::UnexpectedEof { .. } => true,
1746 subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
1747 _ => false,
1748 },
1749 _ => false,
1750 };
1751 if transient {
1752 AttachErrorClass::Transient
1753 } else {
1754 AttachErrorClass::Permanent
1755 }
1756}
1757
1758fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
1759 matches!(
1760 kind,
1761 io::ErrorKind::ConnectionRefused
1762 | io::ErrorKind::TimedOut
1763 | io::ErrorKind::ConnectionReset
1764 | io::ErrorKind::ConnectionAborted
1765 | io::ErrorKind::BrokenPipe
1766 | io::ErrorKind::UnexpectedEof
1767 )
1768}
1769
1770async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
1774 connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
1775}
1776
1777async fn connect_and_authenticate_with_policy(
1778 connection_file_path: &Path,
1779 policy: AttachRetryPolicy,
1780) -> Result<TcpStream, SubcError> {
1781 let started_at = Instant::now();
1782 let deadline = started_at + policy.budget;
1783 let mut attempt = 0_u32;
1784 let mut backoff = policy.initial_backoff;
1785 let mut history = Vec::new();
1786
1787 loop {
1788 attempt = attempt.saturating_add(1);
1789 let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
1790 Ok(stream) => return Ok(stream),
1791 Err(error) => error,
1792 };
1793 let class = classify_attach_error(&error);
1794 let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
1795 history.push(format!("attempt {attempt} [{class}]: {error_text}"));
1796
1797 if class == AttachErrorClass::Permanent {
1798 log_attach_final_failure(started_at.elapsed(), &history);
1799 return Err(error);
1800 }
1801
1802 let remaining = deadline.saturating_duration_since(Instant::now());
1803 if remaining.is_zero() {
1804 log_attach_final_failure(started_at.elapsed(), &history);
1805 return Err(error);
1806 }
1807
1808 let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
1809 log::info!(
1810 "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
1811 );
1812 tokio::time::sleep(delay).await;
1813
1814 if Instant::now() >= deadline {
1815 log_attach_final_failure(started_at.elapsed(), &history);
1816 return Err(error);
1817 }
1818 backoff = backoff.saturating_mul(2).min(policy.max_backoff);
1819 }
1820}
1821
1822fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
1823 let jitter_percent = jitter_percent.min(100);
1824 if jitter_percent == 0 {
1825 return base;
1826 }
1827
1828 let mut random_bytes = [0_u8; 8];
1829 let random = if getrandom::fill(&mut random_bytes).is_ok() {
1830 u64::from_le_bytes(random_bytes)
1831 } else {
1832 let timestamp = std::time::SystemTime::now()
1833 .duration_since(std::time::UNIX_EPOCH)
1834 .unwrap_or_default()
1835 .subsec_nanos();
1836 u64::from(timestamp) ^ u64::from(attempt)
1837 };
1838 let span = jitter_percent.saturating_mul(2).saturating_add(1);
1839 let multiplier_percent = 100 - jitter_percent + random % span;
1840 let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
1841 Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
1842}
1843
1844fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
1845 log::error!(
1846 "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
1847 history.len(),
1848 history.join(" | ")
1849 );
1850}
1851
1852async fn connect_and_authenticate_once(
1853 connection_file_path: &Path,
1854 deadline: Instant,
1855) -> Result<TcpStream, SubcError> {
1856 let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
1859 SubcError::ConnectionFile {
1860 path: connection_file_path.to_path_buf(),
1861 source,
1862 }
1863 })?;
1864
1865 let endpoint = conn
1866 .endpoints
1867 .first()
1868 .ok_or_else(|| SubcError::NoEndpoint {
1869 path: connection_file_path.to_path_buf(),
1870 })?;
1871 let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
1872 let ip = endpoint
1873 .host
1874 .parse::<IpAddr>()
1875 .map_err(|_| SubcError::InvalidEndpoint {
1876 path: connection_file_path.to_path_buf(),
1877 endpoint: endpoint_label.clone(),
1878 })?;
1879 let addr = SocketAddr::new(ip, endpoint.port);
1880
1881 let connect_budget = deadline.saturating_duration_since(Instant::now());
1882 let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
1883 .await
1884 .map_err(|_| SubcError::Connect {
1885 endpoint: endpoint_label.clone(),
1886 source: io::Error::new(
1887 io::ErrorKind::TimedOut,
1888 "initial subc attach retry budget elapsed during TCP connect",
1889 ),
1890 })?
1891 .map_err(|source| SubcError::Connect {
1892 endpoint: endpoint_label.clone(),
1893 source,
1894 })?;
1895 stream
1896 .set_nodelay(true)
1897 .map_err(|source| SubcError::Connect {
1898 endpoint: endpoint_label.clone(),
1899 source,
1900 })?;
1901
1902 let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
1903 authenticate_client(&mut stream, &conn, auth_budget)
1904 .await
1905 .map_err(|source| SubcError::Auth {
1906 endpoint: endpoint_label,
1907 source,
1908 })?;
1909
1910 Ok(stream)
1911}
1912
1913#[allow(clippy::too_many_arguments)]
1914async fn process_route_bind_completion(
1915 writer_tx: &WriterSender,
1916 completion: RouteBindCompletion,
1917 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1918 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1919 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1920 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1921 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1922 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1923 installed_route_epochs: &mut HashMap<u16, u32>,
1924 executor: &Arc<Executor>,
1925 shutdown: &Arc<Notify>,
1926 metrics: &Arc<DispatchPathMetrics>,
1927) -> Result<(), SubcError> {
1928 decrement_counted_channel(&metrics.control_completion_queued);
1929 handle_route_bind_completion(
1930 writer_tx,
1931 completion,
1932 routes,
1933 root_channels,
1934 session_identity,
1935 push_buffer,
1936 live_roots,
1937 pending_binds,
1938 installed_route_epochs,
1939 executor,
1940 shutdown,
1941 metrics,
1942 )
1943 .await
1944}
1945
1946#[allow(clippy::too_many_arguments)]
1947async fn drain_pending_route_bind_completions(
1948 control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
1949 writer_tx: &WriterSender,
1950 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1951 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1952 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1953 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1954 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1955 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1956 installed_route_epochs: &mut HashMap<u16, u32>,
1957 executor: &Arc<Executor>,
1958 shutdown: &Arc<Notify>,
1959 metrics: &Arc<DispatchPathMetrics>,
1960) -> Result<usize, SubcError> {
1961 let mut drained = 0;
1962 while let Ok(completion) = control_completion_rx.try_recv() {
1963 process_route_bind_completion(
1964 writer_tx,
1965 completion,
1966 routes,
1967 root_channels,
1968 session_identity,
1969 push_buffer,
1970 live_roots,
1971 pending_binds,
1972 installed_route_epochs,
1973 executor,
1974 shutdown,
1975 metrics,
1976 )
1977 .await?;
1978 drained += 1;
1979 }
1980 Ok(drained)
1981}
1982
1983async fn run_module_loop<R, W>(
1987 mut read: R,
1988 mut write: W,
1989 shared_app: Arc<App>,
1990 executor: Arc<Executor>,
1991 dispatch: DispatchFn,
1992 user_config_path: Option<PathBuf>,
1993 allow_native_passthrough: bool,
1994) -> Result<ModuleLoopExit, SubcError>
1995where
1996 R: AsyncRead + Unpin + Send + 'static,
1997 W: AsyncWrite + Unpin + Send + 'static,
1998{
1999 let hello = ModuleHelloBody {
2003 manifest: build_manifest(),
2004 protocol_ver: PROTOCOL_VERSION,
2005 control_ops: control_ops(),
2006 launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2007 };
2008 let hello_frame = Frame::build(
2009 FrameType::Hello,
2010 control_flags(),
2011 0,
2012 0,
2013 HELLO_CORR,
2014 serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2015 )
2016 .map_err(SubcError::FrameBuild)?;
2017 write_frame(&mut write, &hello_frame)
2018 .await
2019 .map_err(SubcError::FrameIo)?;
2020
2021 match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2023 None => return Err(SubcError::ClosedBeforeHelloAck),
2024 Some(frame) => match frame.header.ty {
2025 FrameType::HelloAck => {
2026 log::info!("subc attach: registered (HelloAck received)");
2027 }
2028 FrameType::Error => {
2029 let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2030 return Err(SubcError::HelloRejected { body });
2031 }
2032 other => return Err(SubcError::UnexpectedFrame { ty: other }),
2033 },
2034 }
2035
2036 let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2037 let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2038 let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2039 let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2046 let reader_task = spawn_reader_task(read, reader_tx);
2047 let shutdown = Arc::new(Notify::new());
2048 let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2057 let mut next_maintenance_at = next_drain_at;
2058 let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2059 let (bash_deferred_tx, mut bash_deferred_rx) =
2060 mpsc::channel::<bash::BashDeferredCompletion>(256);
2061 let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2062 let (control_completion_tx, mut control_completion_rx) =
2063 mpsc::channel::<RouteBindCompletion>(256);
2064 let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2065 let lossy_overflow = Arc::new(push::LossyOverflow::default());
2066 let lossy_seq = Arc::new(AtomicU64::new(0));
2067 let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2068 let push_senders = PushSenders {
2069 lossy_tx,
2070 reliable_tx,
2071 lossy_overflow: Arc::clone(&lossy_overflow),
2072 lossy_seq,
2073 };
2074 let connection_cancel = PersistentCancelSignal::new();
2075 let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2076 let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2077 let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2078 let mut bg_sub_by_session: HashMap<(ProjectRootId, String), RouteChannel> = HashMap::new();
2079 let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2080 let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2081 let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2082 let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2083 HashMap::new();
2084 let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2085 let mut retry_buffer: RetryBuffer = HashMap::new();
2086 let mut completed_tasks = push::CompletedTaskIds::default();
2087 let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2088 let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2089 let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2090 let mut next_bash_ask_corr: u64 = 1;
2091 let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2092
2093 let loop_result: Result<ModuleLoopExit, SubcError> = loop {
2094 shared_app.set_open_route_count(routes.len());
2095 crate::logging::perf_tick(Some(&executor));
2096 dispatch_path_metrics.mark_frame_loop_tick();
2097 if let Err(error) = expire_pending_bash_asks(
2098 &writer_tx,
2099 &mut pending_bash_asks,
2100 &routes,
2101 &mut live_roots,
2102 &mut route_bash_cancels,
2103 &shutdown,
2104 &dispatch_path_metrics,
2105 )
2106 .await
2107 {
2108 break Err(error);
2109 }
2110
2111 match drain_pending_route_bind_completions(
2115 &mut control_completion_rx,
2116 &writer_tx,
2117 &mut routes,
2118 &mut root_channels,
2119 &mut session_identity,
2120 &mut push_buffer,
2121 &mut live_roots,
2122 &mut pending_binds,
2123 &mut installed_route_epochs,
2124 &executor,
2125 &shutdown,
2126 &dispatch_path_metrics,
2127 )
2128 .await
2129 {
2130 Ok(drained) => {
2131 if drained > 0 {
2132 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2133 }
2134 }
2135 Err(error) => break Err(error),
2136 }
2137
2138 if tokio::time::Instant::now() >= next_drain_at {
2139 push::emit_bg_event_wakes(
2140 &writer_tx,
2141 &dispatch_path_metrics,
2142 &bg_subs,
2143 &mut bg_wake_pending,
2144 );
2145 warn_slow_pending_binds(&mut pending_binds, &executor);
2146 if let Err(error) = expire_overdue_route_binds(
2147 &writer_tx,
2148 &executor,
2149 &mut pending_binds,
2150 &mut installed_route_epochs,
2151 &dispatch_path_metrics,
2152 )
2153 .await
2154 {
2155 break Err(error);
2156 }
2157
2158 let retried = push::drain_retry_buffers_for_bound_routes(
2159 &writer_tx,
2160 &dispatch_path_metrics,
2161 &routes,
2162 &mut retry_buffer,
2163 );
2164 if retried > 0 {
2165 log::debug!(
2166 "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2167 );
2168 }
2169
2170 next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2171 }
2172
2173 let overflow_batch = lossy_overflow.drain();
2179 if !overflow_batch.is_empty() {
2180 let (_, deferred) = push::drain_reliable_push_turn(
2181 &writer_tx,
2182 &dispatch_path_metrics,
2183 &routes,
2184 &root_channels,
2185 &session_identity,
2186 &mut retry_buffer,
2187 &mut push_buffer,
2188 &mut completed_tasks,
2189 &bg_sub_by_session,
2190 &mut bg_wake_pending,
2191 &mut bg_wake_epoch,
2192 &mut reliable_rx,
2193 None,
2194 );
2195 if deferred {
2196 tokio::task::yield_now().await;
2197 }
2198
2199 let mut batch = Vec::new();
2200 while let Ok(item) = lossy_rx.try_recv() {
2201 batch.push(item);
2202 }
2203 batch.extend(overflow_batch);
2204 push::process_lossy_push_envelope_batch(
2205 &writer_tx,
2206 &dispatch_path_metrics,
2207 &routes,
2208 &root_channels,
2209 &completed_tasks,
2210 batch,
2211 );
2212 }
2213
2214 tokio::select! {
2215 biased;
2216 Some(completion) = control_completion_rx.recv() => {
2217 if let Err(error) = process_route_bind_completion(
2218 &writer_tx,
2219 completion,
2220 &mut routes,
2221 &mut root_channels,
2222 &mut session_identity,
2223 &mut push_buffer,
2224 &mut live_roots,
2225 &mut pending_binds,
2226 &mut installed_route_epochs,
2227 &executor,
2228 &shutdown,
2229 &dispatch_path_metrics,
2230 )
2231 .await
2232 {
2233 break Err(error);
2234 }
2235 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2236 }
2237 _ = shutdown.notified() => {
2238 log::warn!("subc attach: fatal executor response requested teardown");
2239 break Ok(ModuleLoopExit::SkipSearchFlush);
2240 }
2241 maybe_frame = reader_rx.recv() => {
2242 let frame = match maybe_frame {
2243 None => {
2244 log::info!("subc attach: daemon closed connection");
2245 break Ok(ModuleLoopExit::Graceful);
2246 }
2247 Some(Err(error)) => break Err(error),
2248 Some(Ok(frame)) => frame,
2249 };
2250 let phase_trace = frame.phase_trace;
2251 let frame = frame.frame;
2252
2253 if !ingress_route_is_current(&installed_route_epochs, &frame) {
2254 log::debug!(
2255 "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2256 frame.header.ty,
2257 frame.header.channel,
2258 frame.header.epoch
2259 );
2260 continue;
2261 }
2262
2263 match frame.header.ty {
2264 FrameType::Ping if frame.header.channel == 0 => {
2265 let pong = match Frame::build_with_version(
2266 frame.header.ver,
2267 FrameType::Pong,
2268 frame.header.flags,
2269 0,
2270 0,
2271 frame.header.corr,
2272 Vec::new(),
2273 ) {
2274 Ok(pong) => pong,
2275 Err(error) => break Err(SubcError::FrameBuild(error)),
2276 };
2277 if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2278 break Err(error);
2279 }
2280 }
2281 FrameType::Goodbye if frame.header.channel == 0 => {
2282 log::info!("subc attach: received channel-0 Goodbye");
2283 break Ok(ModuleLoopExit::Graceful);
2284 }
2285 FrameType::Goodbye => {
2286 let channel = route_key(frame.header.channel, frame.header.epoch);
2287 if let Err(error) = teardown_installed_route(
2288 &writer_tx,
2289 &dispatch_path_metrics,
2290 &executor,
2291 channel,
2292 "Goodbye",
2293 None,
2294 &mut installed_route_epochs,
2295 &mut routes,
2296 &mut root_channels,
2297 &mut bg_subs,
2298 &mut bg_sub_by_session,
2299 &mut bg_wake_pending,
2300 &mut pending_bash_asks,
2301 &mut live_roots,
2302 &mut route_bash_cancels,
2303 &mut pending_binds,
2304 &mut retry_buffer,
2305 &mut push_buffer,
2306 &shutdown,
2307 )
2308 .await
2309 {
2310 break Err(error);
2311 }
2312 }
2313 FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2314 if let Err(error) = handle_bash_elicitation_reply(
2315 &writer_tx,
2316 &frame,
2317 &mut pending_bash_asks,
2318 &routes,
2319 &mut live_roots,
2320 &executor,
2321 &shutdown,
2322 &bash_deferred_tx,
2323 &bash_poll_touch_tx,
2324 &dispatch_path_metrics,
2325 &mut route_bash_cancels,
2326 dispatch,
2327 )
2328 .await
2329 {
2330 break Err(error);
2331 }
2332 }
2333 FrameType::Request if frame.header.channel == 0 => {
2334 if let Err(error) = handle_control_request(
2335 &writer_tx,
2336 &frame,
2337 &shared_app,
2338 &executor,
2339 &mut live_roots,
2340 &mut pending_binds,
2341 &mut installed_route_epochs,
2342 &mut routes,
2343 &mut root_channels,
2344 &mut bg_subs,
2345 &mut bg_sub_by_session,
2346 &mut bg_wake_pending,
2347 &mut pending_bash_asks,
2348 &mut route_bash_cancels,
2349 &mut retry_buffer,
2350 &mut push_buffer,
2351 &shutdown,
2352 &control_completion_tx,
2353 &dispatch_path_metrics,
2354 &push_senders,
2355 dispatch,
2356 user_config_path.as_deref(),
2357 )
2358 .await
2359 {
2360 break Err(error);
2361 }
2362 }
2363 FrameType::Request => {
2364 if let Err(error) = handle_tool_call(
2365 &writer_tx,
2366 &frame,
2367 phase_trace,
2368 &routes,
2369 &pending_binds,
2370 &mut live_roots,
2371 &executor,
2372 &shutdown,
2373 &connection_cancel,
2374 &bash_deferred_tx,
2375 &bash_poll_touch_tx,
2376 &dispatch_path_metrics,
2377 &mut route_bash_cancels,
2378 &mut pending_bash_asks,
2379 &mut next_bash_ask_corr,
2380 &mut bg_subs,
2381 &mut bg_sub_by_session,
2382 &mut bg_wake_pending,
2383 &mut bg_wake_epoch,
2384 dispatch,
2385 allow_native_passthrough,
2386 )
2387 .await
2388 {
2389 break Err(error);
2390 }
2391 }
2392 FrameType::Cancel => {
2393 let channel = route_key(frame.header.channel, frame.header.epoch);
2394 if bg_subs.contains_key(&channel) {
2395 if let Err(error) = end_bg_subscription(
2396 &writer_tx,
2397 &dispatch_path_metrics,
2398 &mut bg_subs,
2399 &mut bg_sub_by_session,
2400 &mut bg_wake_pending,
2401 channel,
2402 routes.get(&channel),
2403 )
2404 .await
2405 {
2406 break Err(error);
2407 }
2408 }
2409 if let Err(error) = cancel_pending_bash_ask_for_tool_call(
2410 &writer_tx,
2411 &mut pending_bash_asks,
2412 channel,
2413 frame.header.corr,
2414 &routes,
2415 &mut live_roots,
2416 &mut route_bash_cancels,
2417 &shutdown,
2418 &dispatch_path_metrics,
2419 )
2420 .await
2421 {
2422 break Err(error);
2423 }
2424 }
2425 _ => {}
2430 }
2431 }
2432 Some((root_id, frame)) = reliable_rx.recv() => {
2433 let (_, deferred) = push::drain_reliable_push_turn(
2437 &writer_tx,
2438 &dispatch_path_metrics,
2439 &routes,
2440 &root_channels,
2441 &session_identity,
2442 &mut retry_buffer,
2443 &mut push_buffer,
2444 &mut completed_tasks,
2445 &bg_sub_by_session,
2446 &mut bg_wake_pending,
2447 &mut bg_wake_epoch,
2448 &mut reliable_rx,
2449 Some((root_id, frame)),
2450 );
2451 if deferred {
2452 tokio::task::yield_now().await;
2453 }
2454 }
2455 Some((order, root_id, frame)) = lossy_rx.recv() => {
2456 let (_, deferred) = push::drain_reliable_push_turn(
2460 &writer_tx,
2461 &dispatch_path_metrics,
2462 &routes,
2463 &root_channels,
2464 &session_identity,
2465 &mut retry_buffer,
2466 &mut push_buffer,
2467 &mut completed_tasks,
2468 &bg_sub_by_session,
2469 &mut bg_wake_pending,
2470 &mut bg_wake_epoch,
2471 &mut reliable_rx,
2472 None,
2473 );
2474 if deferred {
2475 tokio::task::yield_now().await;
2476 }
2477
2478 let mut batch = vec![(order, root_id, frame)];
2485 while let Ok(item) = lossy_rx.try_recv() {
2486 batch.push(item);
2487 }
2488 batch.extend(lossy_overflow.drain());
2489 push::process_lossy_push_envelope_batch(
2490 &writer_tx,
2491 &dispatch_path_metrics,
2492 &routes,
2493 &root_channels,
2494 &completed_tasks,
2495 batch,
2496 );
2497 }
2498 Some(done) = bash_deferred_rx.recv() => {
2499 decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
2500 if let Err(error) = bash::handle_bash_deferred_completion(
2501 &writer_tx,
2502 done,
2503 &routes,
2504 &mut live_roots,
2505 &mut route_bash_cancels,
2506 &shutdown,
2507 &dispatch_path_metrics,
2508 )
2509 .await
2510 {
2511 break Err(error);
2512 }
2513 }
2514 Some(root_id) = bash_poll_touch_rx.recv() => {
2515 decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
2516 if let Some(meta) = live_roots.get_mut(&root_id) {
2517 meta.note_activity();
2518 }
2519 }
2520 Some(completion) = maintenance_rx.recv() => {
2521 decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
2522 let root_id = completion.root_id.clone();
2523 let response = completion.response;
2524 let response_is_fatal = response_is_fatal_panic(&response);
2525 let bind_pending = pending_binds
2526 .values()
2527 .any(|pending| pending.bind_root_id == root_id);
2528 let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
2529 let defer_requeue = meta.unbound_quiesced || bind_pending;
2530 note_maintenance_completion(
2531 meta,
2532 completion.requeue_kind,
2533 response_is_fatal,
2534 defer_requeue,
2535 );
2536 should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
2537 } else {
2538 false
2539 };
2540 if requiesce {
2541 quiesce_unbound_root(&root_id, &mut live_roots, &executor);
2542 }
2543 push::clear_stale_bg_wakes_for_empty_sessions(
2544 &root_id,
2545 &completion.empty_bg_sessions,
2546 &bg_sub_by_session,
2547 &mut bg_wake_pending,
2548 &bg_wake_epoch,
2549 );
2550 if response_is_fatal {
2551 if let Some(meta) = live_roots.get_mut(&root_id) {
2552 meta.maintenance_poisoned = true;
2553 }
2554 log::warn!(
2555 "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
2556 );
2557 }
2558 }
2559 _ = tokio::time::sleep_until(next_drain_at) => {
2560 }
2563 _ = tokio::time::sleep_until(next_maintenance_at) => {
2564 let reap = reap_idle_roots(
2569 Instant::now(),
2570 &mut live_roots,
2571 &pending_binds,
2572 &root_channels,
2573 &executor,
2574 );
2575 for root_id in &reap.forgotten_deleted_roots {
2576 purge_deleted_root_residents(
2577 root_id,
2578 &mut root_channels,
2579 &mut session_identity,
2580 &mut push_buffer,
2581 &mut bg_subs,
2582 &mut bg_sub_by_session,
2583 &mut bg_wake_pending,
2584 &mut bg_wake_epoch,
2585 &mut pending_bash_asks,
2586 );
2587 }
2588 if reap.evicted > 0 {
2589 log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
2590 }
2591 submit_due_maintenance_jobs(
2592 &executor,
2593 &mut live_roots,
2594 &pending_binds,
2595 &bg_sub_by_session,
2596 &bg_wake_pending,
2597 &bg_wake_epoch,
2598 &maintenance_tx,
2599 &dispatch_path_metrics,
2600 );
2601 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2602 }
2603 }
2604 };
2605
2606 shared_app.set_open_route_count(0);
2607
2608 for pending in pending_binds.values() {
2612 let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
2613 }
2614
2615 let mut loop_result = loop_result;
2616 if !pending_bash_asks.is_empty() {
2617 let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2618 if let Err(error) = settle_all_pending_bash_asks(
2619 &writer_tx,
2620 &mut pending_bash_asks,
2621 &no_routes,
2622 &mut live_roots,
2623 &mut route_bash_cancels,
2624 &shutdown,
2625 &dispatch_path_metrics,
2626 )
2627 .await
2628 {
2629 loop_result = loop_result.and(Err(error));
2630 }
2631 }
2632
2633 connection_cancel.cancel();
2636 reader_task.abort();
2637 drop(writer_tx);
2638 let writer_result = finish_writer_task(writer_task).await;
2639 loop_result.and_then(|exit| writer_result.map(|_| exit))
2640}
2641
2642fn spawn_writer_task<W>(
2643 mut write: W,
2644 mut rx: mpsc::Receiver<WriterFrame>,
2645 metrics: Arc<DispatchPathMetrics>,
2646) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
2647where
2648 W: AsyncWrite + Unpin + Send + 'static,
2649{
2650 tokio::spawn(async move {
2651 let mut write_buffer = Vec::new();
2652 while let Some(mut queued) = rx.recv().await {
2653 let measure = queued.tool_response_trace.is_some();
2654 let dequeued = measure.then(Instant::now);
2655 metrics.writer_active.store(true, Ordering::Relaxed);
2656 decrement_counted_channel(&metrics.writer_queued);
2657 let write_timing =
2658 write_frame_contiguous(&mut write, &queued.frame, &mut write_buffer, measure).await;
2659 metrics.writer_active.store(false, Ordering::Relaxed);
2660 let write_timing = write_timing?;
2661
2662 if let (Some(trace), Some(dequeued), Some(write_timing)) =
2663 (queued.tool_response_trace.take(), dequeued, write_timing)
2664 {
2665 if let Some(completed) = trace.finish(
2666 dequeued,
2667 write_timing.write_started,
2668 write_timing.write_finished,
2669 write_timing.frame_bytes,
2670 ) {
2671 log_ctx::with_session(Some(completed.session), || {
2672 crate::logging::note_tool_call_trace(
2673 &completed.name,
2674 &completed.root,
2675 completed.channel,
2676 completed.corr,
2677 completed.phases,
2678 );
2679 });
2680 }
2681 }
2682 }
2683 Ok(())
2684 })
2685}
2686
2687struct FrameWriteTiming {
2688 write_started: Instant,
2689 write_finished: Instant,
2690 frame_bytes: usize,
2691}
2692
2693async fn write_frame_contiguous<W>(
2697 writer: &mut W,
2698 frame: &Frame,
2699 buffer: &mut Vec<u8>,
2700 measure: bool,
2701) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
2702where
2703 W: AsyncWrite + Unpin,
2704{
2705 if frame.header.len as usize != frame.body.len() {
2706 return Err(subc_transport::FrameIoError::BodyLengthMismatch {
2707 header_len: frame.header.len,
2708 body_len: frame.body.len(),
2709 });
2710 }
2711
2712 let header = frame.header.encode();
2713 buffer.clear();
2714 buffer.reserve(header.len() + frame.body.len());
2715 buffer.extend_from_slice(&header);
2716 buffer.extend_from_slice(&frame.body);
2717 let write_started = measure.then(Instant::now);
2718 writer
2719 .write_all(buffer)
2720 .await
2721 .map_err(subc_transport::FrameIoError::Io)?;
2722 Ok(write_started.map(|write_started| FrameWriteTiming {
2723 write_started,
2724 write_finished: Instant::now(),
2725 frame_bytes: buffer.len(),
2726 }))
2727}
2728
2729fn spawn_reader_task<R>(
2730 mut read: R,
2731 tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
2732) -> JoinHandle<()>
2733where
2734 R: AsyncRead + Unpin + Send + 'static,
2735{
2736 tokio::spawn(async move {
2737 loop {
2738 match read_frame(&mut read).await {
2739 Ok(Some(frame)) => {
2740 let decoded = DecodedFrame {
2741 frame,
2742 phase_trace: PhaseTrace::new(Instant::now()),
2743 };
2744 if tx.send(Ok(decoded)).await.is_err() {
2745 return;
2746 }
2747 }
2748 Ok(None) => {
2749 return;
2751 }
2752 Err(error) => {
2753 if let subc_transport::FrameIoError::Io(io_error) = &error {
2760 if matches!(
2761 io_error.kind(),
2762 std::io::ErrorKind::ConnectionReset
2763 | std::io::ErrorKind::ConnectionAborted
2764 ) {
2765 log::info!(
2766 "subc attach: connection reset by daemon; treating as close"
2767 );
2768 return;
2769 }
2770 }
2771 let _ = tx.send(Err(SubcError::FrameIo(error))).await;
2772 return;
2773 }
2774 }
2775 }
2776 })
2777}
2778
2779async fn finish_writer_task(
2780 mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
2781) -> Result<(), SubcError> {
2782 match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
2783 Ok(Ok(Ok(()))) => Ok(()),
2784 Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
2785 Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
2786 Err(_) => {
2787 writer_task.abort();
2788 Ok(())
2789 }
2790 }
2791}
2792
2793fn register_actor_for_bind(
2794 shared_app: &Arc<App>,
2795 executor: &Arc<Executor>,
2796 push_senders: &PushSenders,
2797 bind_root_id: &ProjectRootId,
2798 route_channel: u16,
2799 root_was_live: bool,
2800) -> bool {
2801 if executor.actor_registered(bind_root_id) {
2802 log::debug!(
2803 "subc attach: reusing actor for route {} root {}",
2804 route_channel,
2805 bind_root_id.as_path().display()
2806 );
2807 return false;
2808 }
2809
2810 if root_was_live {
2811 log::warn!(
2812 "subc attach: recreating missing actor for live root {} on route {}",
2813 bind_root_id.as_path().display(),
2814 route_channel
2815 );
2816 }
2817
2818 let actor_ctx = Arc::new(AppContext::from_app(
2819 Arc::clone(shared_app),
2820 Config::default(),
2821 ));
2822 install_bash_compressor(&actor_ctx);
2823 actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
2824 push_senders.clone(),
2825 bind_root_id.clone(),
2826 )));
2827 let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
2828 drop(actor_ctx);
2829 if inserted {
2830 log::debug!(
2834 "subc attach: registered actor for route {} root {}",
2835 route_channel,
2836 bind_root_id.as_path().display()
2837 );
2838 } else {
2839 log::debug!(
2840 "subc attach: actor appeared while binding route {} root {}; reusing it",
2841 route_channel,
2842 bind_root_id.as_path().display()
2843 );
2844 }
2845 inserted
2846}
2847
2848fn rollback_pending_bind_actor(
2849 executor: &Arc<Executor>,
2850 live_roots: &HashMap<ProjectRootId, RootMeta>,
2851 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2852 root_id: &ProjectRootId,
2853 inserted_new_actor: bool,
2854) {
2855 if !inserted_new_actor || live_roots.contains_key(root_id) {
2856 return;
2857 }
2858
2859 if let Some((route, pending)) = pending_binds
2860 .iter_mut()
2861 .find(|(_, pending)| &pending.bind_root_id == root_id)
2862 {
2863 pending.inserted_new_actor = true;
2864 log::debug!(
2865 "subc attach: transferred rollback ownership for root {} to pending route {}",
2866 root_id.as_path().display(),
2867 route
2868 );
2869 return;
2870 }
2871
2872 executor.remove_actor(root_id);
2873}
2874
2875fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
2876 match response.data.get("code").and_then(|code| code.as_str()) {
2877 Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
2882 Some("cache_key_probe_failed") => "cache_key_probe_failed",
2886 Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
2890 _ => "config_divergence",
2891 }
2892}
2893
2894fn queue_post_bind_configure_and_completion_maintenance(
2895 root_id: &ProjectRootId,
2896 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2897) {
2898 let Some(meta) = live_roots.get_mut(root_id) else {
2899 return;
2900 };
2901 if meta.maintenance_poisoned || meta.maintenance_pending {
2902 return;
2903 }
2904
2905 meta.maintenance_pending = true;
2906 meta.maintenance_queued_kinds
2907 .push_back(MaintenanceDrainKind::ConfigureTail);
2908 meta.maintenance_queued_kinds
2909 .push_back(MaintenanceDrainKind::CompletionDrains);
2910}
2911
2912#[allow(clippy::too_many_arguments)]
2913async fn handle_route_bind_completion(
2914 tx: &WriterSender,
2915 completion: RouteBindCompletion,
2916 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2917 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2918 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2919 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2920 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2921 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2922 installed_route_epochs: &mut HashMap<u16, u32>,
2923 executor: &Arc<Executor>,
2924 shutdown: &Arc<Notify>,
2925 metrics: &Arc<DispatchPathMetrics>,
2926) -> Result<(), SubcError> {
2927 let route_id = completion.route;
2928 let Some(pending) = pending_binds.remove(&route_id) else {
2929 log::warn!(
2930 "subc attach: dropping RouteBind completion for non-pending route {}",
2931 completion.route
2932 );
2933 rollback_pending_bind_actor(
2934 executor,
2935 live_roots,
2936 pending_binds,
2937 &completion.bind_root_id,
2938 completion.inserted_new_actor,
2939 );
2940 let has_pending_bind = pending_binds
2941 .values()
2942 .any(|pending| pending.bind_root_id == completion.bind_root_id);
2943 if !root_channels
2944 .get(&completion.bind_root_id)
2945 .is_some_and(|channels| !channels.is_empty())
2946 && !has_pending_bind
2947 {
2948 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
2949 }
2950 remove_installed_route(installed_route_epochs, route_id);
2951 return Ok(());
2952 };
2953
2954 if pending.bind_root_id != completion.bind_root_id {
2955 log::warn!(
2956 "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
2957 completion.route,
2958 pending.bind_root_id.as_path().display(),
2959 completion.bind_root_id.as_path().display()
2960 );
2961 }
2962
2963 let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
2964 if pending.cancelled {
2965 rollback_pending_bind_actor(
2966 executor,
2967 live_roots,
2968 pending_binds,
2969 &completion.bind_root_id,
2970 inserted_new_actor,
2971 );
2972 let has_pending_bind = pending_binds
2973 .values()
2974 .any(|pending| pending.bind_root_id == completion.bind_root_id);
2975 if !root_channels
2976 .get(&completion.bind_root_id)
2977 .is_some_and(|channels| !channels.is_empty())
2978 && !has_pending_bind
2979 {
2980 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
2981 }
2982 log::debug!(
2983 "subc attach: discarded completed RouteBind for cancelled route {} root {}",
2984 completion.route,
2985 completion.bind_root_id.as_path().display()
2986 );
2987 remove_installed_route(installed_route_epochs, route_id);
2988 return Ok(());
2989 }
2990
2991 let failure = if !completion.configure_response.success {
2992 Some((
2993 &completion.configure_response,
2994 "configure failed during route bind",
2995 ))
2996 } else {
2997 None
2998 };
2999
3000 if let Some((response, fallback)) = failure {
3001 rollback_pending_bind_actor(
3002 executor,
3003 live_roots,
3004 pending_binds,
3005 &completion.bind_root_id,
3006 inserted_new_actor,
3007 );
3008 let has_pending_bind = pending_binds
3009 .values()
3010 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3011 if !root_channels
3012 .get(&completion.bind_root_id)
3013 .is_some_and(|channels| !channels.is_empty())
3014 && !has_pending_bind
3015 {
3016 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3017 }
3018 let message = response_message(response, fallback);
3019 let fatal = response_is_fatal_panic(response);
3020 let error_code = route_bind_error_code_for_configure_response(response);
3021 send_route_bind_error_parts(
3022 tx,
3023 completion.ver,
3024 completion.corr,
3025 completion.flags,
3026 error_code,
3027 &message,
3028 metrics,
3029 )
3030 .await?;
3031 remove_installed_route(installed_route_epochs, route_id);
3032 if fatal {
3033 signal_fatal_teardown(
3034 tx,
3035 Some(completion.route),
3036 completion.ver,
3037 completion.corr,
3038 shutdown,
3039 metrics,
3040 )
3041 .await;
3042 }
3043 return Ok(());
3044 }
3045
3046 remember_session_identity(session_identity, &completion.identity);
3047 let replay_key = push::ReplayKey::from_identity(&completion.identity);
3048 let bind_trust = completion.identity.trust;
3049 insert_route_channel(routes, root_channels, route_id, completion.identity);
3050 let restore_watcher = live_roots
3051 .get(&completion.bind_root_id)
3052 .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3053 live_roots
3054 .entry(completion.bind_root_id.clone())
3055 .and_modify(|meta| {
3056 meta.reactivate_bound();
3057 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3058 meta.maintenance_poisoned = false;
3059 })
3060 .or_insert_with(|| RootMeta::new(Instant::now()));
3061 if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3062 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3063 meta.maintenance_poisoned = false;
3064 }
3065 if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3066 ctx.mark_subc_bound();
3067 if restore_watcher {
3068 crate::commands::configure::ensure_project_watcher(&ctx);
3069 }
3070 }
3071
3072 let ack =
3073 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3074 let response = Frame::build_with_version(
3075 completion.ver,
3076 FrameType::Response,
3077 control_flags(),
3078 0,
3079 0,
3080 completion.corr,
3081 ack,
3082 )
3083 .map_err(SubcError::FrameBuild)?;
3084 send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3085 queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3086 let replayed = push::replay_buffered_push_frames(
3087 tx,
3088 metrics,
3089 route_id,
3090 push_buffer,
3091 &replay_key,
3092 bind_trust,
3093 );
3094 if replayed > 0 {
3095 log::debug!(
3096 "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3097 replayed,
3098 completion.route,
3099 replay_key.root.as_path().display(),
3100 replay_key.harness,
3101 replay_key.session
3102 );
3103 }
3104 log::info!(
3105 "subc attach: route {} bound to root {}",
3106 completion.route,
3107 completion.bind_root_id.as_path().display()
3108 );
3109 Ok(())
3110}
3111
3112async fn expire_overdue_route_binds(
3113 tx: &WriterSender,
3114 executor: &Arc<Executor>,
3115 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3116 installed_route_epochs: &mut HashMap<u16, u32>,
3117 metrics: &DispatchPathMetrics,
3118) -> Result<(), SubcError> {
3119 let now = Instant::now();
3120 let expired: Vec<_> = pending_binds
3121 .iter()
3122 .filter_map(|(route, pending)| {
3123 let age = now.saturating_duration_since(pending.started_at);
3124 (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3125 (
3126 *route,
3127 pending.corr,
3128 pending.ver,
3129 pending.flags,
3130 pending.bind_root_id.clone(),
3131 pending.configure_request_id.clone(),
3132 age,
3133 )
3134 })
3135 })
3136 .collect();
3137
3138 for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3139 if let Some(pending) = pending_binds.get_mut(&route) {
3140 pending.cancelled = true;
3141 pending.deadline_reported = true;
3142 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3143 log::debug!(
3144 "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3145 );
3146 }
3147 remove_installed_route(installed_route_epochs, route);
3148 let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3149 let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3150 send_route_bind_error_parts(
3151 tx,
3152 ver,
3153 corr,
3154 flags,
3155 "actor_not_ready",
3156 &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3157 metrics,
3158 )
3159 .await?;
3160 log::warn!(
3161 "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3162 route,
3163 root_id.as_path().display(),
3164 deadline_ms,
3165 configure_request_id
3166 );
3167 }
3168
3169 Ok(())
3170}
3171
3172#[allow(clippy::too_many_arguments)]
3177async fn handle_control_request(
3178 tx: &WriterSender,
3179 frame: &Frame,
3180 shared_app: &Arc<App>,
3181 executor: &Arc<Executor>,
3182 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3183 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3184 installed_route_epochs: &mut HashMap<u16, u32>,
3185 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3186 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3187 bg_subs: &mut HashMap<RouteChannel, BgSub>,
3188 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3189 bg_wake_pending: &mut HashSet<RouteChannel>,
3190 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3191 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3192 retry_buffer: &mut RetryBuffer,
3193 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3194 shutdown: &Arc<Notify>,
3195 control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3196 metrics: &Arc<DispatchPathMetrics>,
3197 push_senders: &PushSenders,
3198 dispatch: DispatchFn,
3199 user_config_path: Option<&Path>,
3200) -> Result<(), SubcError> {
3201 let request =
3202 serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3203 match request {
3204 ModuleControlRequest::RouteBind {
3205 route_channel,
3206 epoch,
3207 target: _,
3208 identity,
3209 principal,
3210 consumer_capabilities,
3211 } => {
3212 let route_id = route_key(route_channel, epoch);
3213 if epoch == 0 {
3214 return send_route_bind_error(
3215 tx,
3216 frame,
3217 "config_divergence",
3218 "route bind uses an invalid channel generation",
3219 metrics,
3220 )
3221 .await;
3222 }
3223 let mut bind_root_id = None;
3224 if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3225 if installed_epoch >= epoch {
3226 return send_route_bind_error(
3227 tx,
3228 frame,
3229 "config_divergence",
3230 "route bind generation is not newer than the installed generation",
3231 metrics,
3232 )
3233 .await;
3234 }
3235
3236 let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3237 Ok(root_id) => root_id,
3238 Err(error) => {
3239 return send_route_bind_error(
3240 tx,
3241 frame,
3242 "config_divergence",
3243 &format!("invalid route project root: {error}"),
3244 metrics,
3245 )
3246 .await;
3247 }
3248 };
3249 teardown_installed_route(
3250 tx,
3251 metrics,
3252 executor,
3253 route_key(route_channel, installed_epoch),
3254 "higher-epoch RouteBind",
3255 Some(&replacement_root),
3256 installed_route_epochs,
3257 routes,
3258 root_channels,
3259 bg_subs,
3260 bg_sub_by_session,
3261 bg_wake_pending,
3262 pending_bash_asks,
3263 live_roots,
3264 route_bash_cancels,
3265 pending_binds,
3266 retry_buffer,
3267 push_buffer,
3268 shutdown,
3269 )
3270 .await?;
3271 bind_root_id = Some(replacement_root);
3272 }
3273 if pending_binds.contains_key(&route_id) {
3274 return send_route_bind_error(
3275 tx,
3276 frame,
3277 "config_divergence",
3278 "route bind is already pending for channel",
3279 metrics,
3280 )
3281 .await;
3282 }
3283 let bind_root_id = match bind_root_id {
3284 Some(root_id) => root_id,
3285 None => match ProjectRootId::from_path(&identity.project_root) {
3286 Ok(root_id) => root_id,
3287 Err(error) => {
3288 return send_route_bind_error(
3289 tx,
3290 frame,
3291 "config_divergence",
3292 &format!("invalid route project root: {error}"),
3293 metrics,
3294 )
3295 .await;
3296 }
3297 },
3298 };
3299
3300 let request_id = format!("subc-bind-{route_channel}");
3303 let bind_project_root = identity.project_root.clone();
3304 let bind_harness = identity.harness.clone();
3305 let bind_session = identity.session.clone();
3306 let bind_trust = trust_for_bind(&bind_harness, &principal);
3307 let consumer_elicitation_capable = consumer_capabilities
3312 .as_ref()
3313 .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
3314 log::info!(
3315 "subc attach: route {} harness={} principal={} trust={} elicitation={}",
3316 route_channel,
3317 bind_harness,
3318 principal_label(&principal),
3319 bind_trust.label(),
3320 consumer_elicitation_capable
3321 );
3322
3323 let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
3335 user_config_path,
3336 Path::new(&bind_project_root),
3337 );
3338 let config_tiers: Vec<Value> = local_tiers
3339 .iter()
3340 .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
3341 .collect();
3342 let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
3343 let configure_json = json!({
3344 "id": request_id,
3345 "command": "configure",
3346 "project_root": bind_project_root,
3347 "harness": bind_harness,
3348 "session_id": bind_session.clone(),
3349 "config": config_tiers,
3350 });
3351 let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
3352 Ok(req) => req,
3353 Err(error) => {
3354 return send_route_bind_error(
3355 tx,
3356 frame,
3357 "config_divergence",
3358 &format!("failed to build configure request: {error}"),
3359 metrics,
3360 )
3361 .await;
3362 }
3363 };
3364
3365 let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
3366 root: bind_root_id.clone(),
3367 project_root: PathBuf::from(&bind_project_root),
3368 harness: bind_harness.clone(),
3369 session: bind_session.clone(),
3370 trust: bind_trust,
3371 consumer_elicitation_capable,
3372 }));
3373 let configure_session = route_identity.session.clone();
3374 let root_was_live = live_roots.contains_key(&bind_root_id);
3375 let inserted_new_actor = register_actor_for_bind(
3376 shared_app,
3377 executor,
3378 push_senders,
3379 &bind_root_id,
3380 route_channel,
3381 root_was_live,
3382 );
3383
3384 let configure_request_id = configure_req.id.clone();
3385 installed_route_epochs.insert(route_channel, epoch);
3386 if let Some(meta) = live_roots.get_mut(&bind_root_id) {
3387 meta.maintenance_queued_kinds.clear();
3388 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
3389 }
3390 let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
3391 bind_root_id.clone(),
3392 Lane::Mutating,
3393 configure_request_id.clone(),
3394 Box::new(move |ctx| {
3395 log_ctx::with_session(Some(configure_session.clone()), || {
3396 dispatch(configure_req, ctx)
3397 })
3398 }),
3399 );
3400 pending_binds.insert(
3401 route_id,
3402 PendingBind {
3403 bind_root_id: bind_root_id.clone(),
3404 inserted_new_actor,
3405 cancelled: false,
3406 configure_request_id: configure_request_id.clone(),
3407 started_at: Instant::now(),
3408 warned_half_deadline: false,
3409 deadline_reported: false,
3410 corr: frame.header.corr,
3411 ver: frame.header.ver,
3412 flags: frame.header.flags,
3413 cancellation: configure_cancellation,
3414 },
3415 );
3416
3417 let completion_tx = control_completion_tx.clone();
3418 let completion_identity = route_identity;
3419 let completion_root = bind_root_id.clone();
3420 let completion_route_channel = route_channel;
3421 let completion_ver = frame.header.ver;
3422 let completion_corr = frame.header.corr;
3423 let completion_flags = frame.header.flags;
3424 let completion_metrics = Arc::clone(metrics);
3425 tokio::spawn(async move {
3426 let _response_task = ResponseTaskGuard::new(&completion_metrics);
3427 let configure_response =
3428 await_executor_response(configure_rx, configure_request_id.clone()).await;
3429 let completion = RouteBindCompletion {
3434 route: route_key(completion_route_channel, epoch),
3435 identity: completion_identity,
3436 bind_root_id: completion_root,
3437 inserted_new_actor,
3438 configure_response,
3439 diagnostics_on_edit,
3440 ver: completion_ver,
3441 corr: completion_corr,
3442 flags: completion_flags,
3443 };
3444 if send_counted_channel(
3445 &completion_tx,
3446 &completion_metrics.control_completion_queued,
3447 completion,
3448 )
3449 .await
3450 .is_err()
3451 {
3452 log::debug!(
3453 "subc attach: dropped RouteBind completion for route {} after loop exit",
3454 completion_route_channel
3455 );
3456 }
3457 });
3458
3459 Ok(())
3460 }
3461 ModuleControlRequest::HealthCheck {} => {
3462 let report = build_health_report(executor, pending_binds, metrics);
3463 let body = serde_json::to_vec(&ModuleControlResponse::from(report))
3464 .map_err(SubcError::Json)?;
3465 let response = Frame::build_with_version(
3466 frame.header.ver,
3467 FrameType::Response,
3468 frame.header.flags,
3469 0,
3470 0,
3471 frame.header.corr,
3472 body,
3473 )
3474 .map_err(SubcError::FrameBuild)?;
3475 send_frame(tx, metrics, response).await
3476 }
3477 }
3478}
3479
3480fn install_bash_compressor(ctx: &AppContext) {
3481 let filter_registry_handle = ctx.shared_filter_registry();
3483 let compress_flag = ctx.bash_compress_flag();
3484 ctx.bash_background().set_compressor_with_exit_code(
3485 move |command: &str, output: String, exit_code: Option<i32>| {
3486 if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
3487 return crate::compress::CompressionResult::new(output);
3488 }
3489 let registry_guard = match filter_registry_handle.read() {
3490 Ok(g) => g,
3491 Err(poisoned) => poisoned.into_inner(),
3492 };
3493 crate::compress::compress_with_registry_exit_code(
3494 command,
3495 &output,
3496 exit_code,
3497 ®istry_guard,
3498 )
3499 },
3500 );
3501}
3502
3503fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
3504 let mut diagnostics_on_edit = false;
3505 for tier in tiers {
3506 if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
3507 diagnostics_on_edit = value;
3508 }
3509 }
3510 diagnostics_on_edit
3511}
3512
3513fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
3514 let stripped = strip_jsonc(doc);
3515 let value = serde_json::from_str::<Value>(&stripped).ok()?;
3516 value
3517 .get("lsp")
3518 .and_then(Value::as_object)?
3519 .get("diagnostics_on_edit")
3520 .and_then(Value::as_bool)
3521}
3522
3523async fn send_route_bind_error(
3524 tx: &WriterSender,
3525 frame: &Frame,
3526 code: &str,
3527 message: &str,
3528 metrics: &DispatchPathMetrics,
3529) -> Result<(), SubcError> {
3530 send_route_bind_error_parts(
3531 tx,
3532 frame.header.ver,
3533 frame.header.corr,
3534 frame.header.flags,
3535 code,
3536 message,
3537 metrics,
3538 )
3539 .await
3540}
3541
3542async fn send_route_bind_error_parts(
3543 tx: &WriterSender,
3544 ver: u8,
3545 corr: u64,
3546 flags: Flags,
3547 code: &str,
3548 message: &str,
3549 metrics: &DispatchPathMetrics,
3550) -> Result<(), SubcError> {
3551 let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
3552 send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
3553 log::warn!("subc attach: route bind rejected ({code}): {message}");
3554 Ok(())
3555}
3556
3557async fn handle_tool_call(
3562 tx: &WriterSender,
3563 frame: &Frame,
3564 mut phase_trace: PhaseTrace,
3565 routes: &HashMap<RouteChannel, RouteIdentity>,
3566 pending_binds: &HashMap<RouteChannel, PendingBind>,
3567 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3568 executor: &Arc<Executor>,
3569 shutdown: &Arc<Notify>,
3570 connection_cancel: &PersistentCancelSignal,
3571 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
3572 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
3573 metrics: &Arc<DispatchPathMetrics>,
3574 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3575 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3576 next_bash_ask_corr: &mut u64,
3577 bg_subs: &mut HashMap<RouteChannel, BgSub>,
3578 bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3579 bg_wake_pending: &mut HashSet<RouteChannel>,
3580 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
3581 dispatch: DispatchFn,
3582 allow_native_passthrough: bool,
3583) -> Result<(), SubcError> {
3584 let route_id = route_key(frame.header.channel, frame.header.epoch);
3585 if pending_binds.contains_key(&route_id) {
3586 let error = build_error_frame(
3587 frame.header.ver,
3588 frame.header.channel,
3589 frame.header.epoch,
3590 frame.header.corr,
3591 frame.header.flags,
3592 "route_not_bound",
3593 "route is not bound before tool call",
3594 )?;
3595 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3596 }
3597
3598 let Some(identity) = routes.get(&route_id).cloned() else {
3599 let error = build_error_frame(
3600 frame.header.ver,
3601 frame.header.channel,
3602 frame.header.epoch,
3603 frame.header.corr,
3604 frame.header.flags,
3605 "route_not_bound",
3606 "route is not bound before tool call",
3607 )?;
3608 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3609 };
3610 let restore_watcher = live_roots
3611 .get(&identity.root)
3612 .is_some_and(|meta| meta.idle_artifacts_evicted);
3613 if let Some(meta) = live_roots.get_mut(&identity.root) {
3614 meta.reactivate_bound();
3615 }
3616 if restore_watcher {
3617 if let Some(ctx) = executor.actor_context(&identity.root) {
3618 crate::commands::configure::ensure_project_watcher(&ctx);
3619 }
3620 }
3621
3622 let route_request =
3623 serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
3624 if matches!(
3625 route_request,
3626 RouteRequest::BgEvents(BgEventsRequest {
3627 op: BgEventsOp::BgEvents
3628 })
3629 ) {
3630 if let Some(old_sub) = bg_subs.get(&route_id).copied() {
3631 push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
3632 }
3633 if !identity.trust.allows_bash_observation() {
3634 bg_subs.remove(&route_id);
3635 bg_wake_pending.remove(&route_id);
3636 remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
3637 let denied_sub = BgSub {
3638 corr: frame.header.corr,
3639 ver: frame.header.ver,
3640 flags: frame.header.flags,
3641 };
3642 push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
3643 return Ok(());
3644 }
3645 bg_subs.insert(
3646 route_id,
3647 BgSub {
3648 corr: frame.header.corr,
3649 ver: frame.header.ver,
3650 flags: frame.header.flags,
3651 },
3652 );
3653 bg_sub_by_session.insert((identity.root.clone(), identity.session.clone()), route_id);
3654 push::arm_bg_wake(
3655 identity.root.clone(),
3656 identity.session.clone(),
3657 route_id,
3658 bg_wake_pending,
3659 bg_wake_epoch,
3660 );
3661 return Ok(());
3662 }
3663
3664 let RouteRequest::ToolCall(call) = route_request else {
3665 unreachable!("background event subscription returned above")
3666 };
3667 let bare_name = call.name;
3668 let arguments = strip_agent_preview_arg_owned(call.arguments);
3669 let format_context = crate::subc_format::FormatContext::from_tool_call(
3670 &bare_name,
3671 &arguments,
3672 identity.project_root.as_path(),
3673 );
3674
3675 let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
3676 let bind_trust = identity.trust;
3677 let diagnostics_on_edit = live_roots
3678 .get(&identity.root)
3679 .map(|meta| meta.diagnostics_on_edit)
3680 .unwrap_or(false);
3681
3682 if matches!(bind_trust, BindTrust::Untrusted)
3683 && is_bash_family_tool(&bare_name)
3684 && (bare_name != "bash" || !identity.consumer_elicitation_capable)
3685 {
3686 let response = bash::bash_denied_untrusted_response(request_id.clone());
3687 let text = crate::subc_format::format_response_with_context(
3688 &bare_name,
3689 &response,
3690 &format_context,
3691 );
3692 let result = ToolCallResult { text, response };
3693 let response_frame = build_tool_response_frame(
3694 frame.header.ver,
3695 route_id,
3696 frame.header.corr,
3697 frame.header.flags,
3698 &result,
3699 bind_trust,
3700 )?;
3701 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3702 }
3703
3704 if !is_subc_agent_core_tool(&bare_name)
3712 && !is_subc_native_plumbing_tool(&bare_name)
3713 && !allow_native_passthrough
3714 {
3715 log::warn!(
3716 "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
3717 bare_name,
3718 frame.header.channel
3719 );
3720 let response = Response::error(
3721 request_id.clone(),
3722 "unknown_tool",
3723 format!("tool {:?} is not in the AFT tool manifest", bare_name),
3724 );
3725 let text = crate::subc_format::format_response_with_context(
3726 &bare_name,
3727 &response,
3728 &format_context,
3729 );
3730 let result = ToolCallResult { text, response };
3731 let response_frame = build_tool_response_frame(
3732 frame.header.ver,
3733 route_id,
3734 frame.header.corr,
3735 frame.header.flags,
3736 &result,
3737 bind_trust,
3738 )?;
3739 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3740 }
3741
3742 if bare_name == "bash" {
3743 if matches!(bind_trust, BindTrust::Untrusted) {
3744 let plan = match bash::prepare_bash_elicitation_plan(
3745 &arguments,
3746 identity.project_root.as_path(),
3747 ) {
3748 Ok(plan) => plan,
3749 Err(error) => {
3750 let response = Response::error(request_id.clone(), error.code, error.message);
3751 let text = crate::subc_format::format_response_with_context(
3752 &bare_name,
3753 &response,
3754 &format_context,
3755 );
3756 let result = ToolCallResult { text, response };
3757 let response_frame = build_tool_response_frame(
3758 frame.header.ver,
3759 route_id,
3760 frame.header.corr,
3761 frame.header.flags,
3762 &result,
3763 bind_trust,
3764 )?;
3765 return send_reliable_writer_frame(
3766 tx,
3767 metrics,
3768 response_frame,
3769 "tool response",
3770 )
3771 .await;
3772 }
3773 };
3774
3775 let meta = live_roots
3776 .entry(identity.root.clone())
3777 .or_insert_with(|| RootMeta::new(Instant::now()));
3778 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
3779 meta.reactivate_bound();
3780
3781 let route_cancel =
3782 route_bash_cancels
3783 .entry(route_id)
3784 .or_insert_with(|| bash::RouteBashCancel {
3785 token: PersistentCancelSignal::new(),
3786 active_waits: 0,
3787 });
3788 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
3789 let cancel = bash::BashWaitCancel {
3790 connection: connection_cancel.clone(),
3791 route: route_cancel.token.clone(),
3792 };
3793
3794 let reverse_corr =
3795 allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
3796 let ask_frame = build_bash_elicitation_request_frame(
3797 frame.header.ver,
3798 route_id,
3799 reverse_corr,
3800 frame.header.flags,
3801 &plan.command,
3802 &plan.asks,
3803 )?;
3804 pending_bash_asks.insert(
3805 ReverseCorrKey {
3806 route: route_id,
3807 corr: reverse_corr,
3808 },
3809 PendingBashAsk {
3810 route: route_id,
3811 tool_corr: frame.header.corr,
3812 tool_flags: frame.header.flags,
3813 tool_ver: frame.header.ver,
3814 root: identity.root.clone(),
3815 project_root: identity.project_root.clone(),
3816 session_id: identity.session.clone(),
3817 request_id,
3818 arguments,
3819 format_context,
3820 cancel,
3821 grants: plan.grants,
3822 expires_at: Instant::now() + bash_elicitation_timeout(),
3823 },
3824 );
3825 return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
3826 .await;
3827 }
3828
3829 let meta = live_roots
3830 .entry(identity.root.clone())
3831 .or_insert_with(|| RootMeta::new(Instant::now()));
3832 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
3833 meta.reactivate_bound();
3834
3835 let route_cancel =
3836 route_bash_cancels
3837 .entry(route_id)
3838 .or_insert_with(|| bash::RouteBashCancel {
3839 token: PersistentCancelSignal::new(),
3840 active_waits: 0,
3841 });
3842 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
3843 let cancel = bash::BashWaitCancel {
3844 connection: connection_cancel.clone(),
3845 route: route_cancel.token.clone(),
3846 };
3847
3848 bash::submit_deferred_bash(
3849 executor,
3850 bash_deferred_tx,
3851 bash_poll_touch_tx,
3852 metrics,
3853 dispatch,
3854 identity.root.clone(),
3855 identity.project_root.clone(),
3856 identity.session.clone(),
3857 request_id,
3858 route_id,
3859 frame.header.corr,
3860 frame.header.flags,
3861 frame.header.ver,
3862 arguments,
3863 format_context,
3864 cancel,
3865 bind_trust,
3866 None,
3867 );
3868 return Ok(());
3869 }
3870
3871 let lane = command_lane(&bare_name);
3872 let tool_call_context = ToolCallContext {
3873 project_root: identity.project_root.clone(),
3874 session_id: Some(identity.session.clone()),
3875 request_id: request_id.clone(),
3876 diagnostics_on_edit,
3877 preview: call.preview,
3878 };
3879 let bare_name_for_frame = bare_name.clone();
3880 let identity_for_run = identity.clone();
3881 let completion_session = identity.session.clone();
3882 let completion_root = identity.project_root.clone();
3883 let request_id_for_force = request_id.clone();
3884 let format_context_for_frame = format_context.clone();
3885 let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
3886 phase_trace.mark_executor_submitted();
3887 let rx = executor.submit_async(
3888 identity.root.clone(),
3889 lane,
3890 request_id.clone(),
3891 Box::new(move |ctx| {
3892 phase_trace.mark_job_admitted();
3893 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
3894 let run = || {
3895 let finalizer = |response: &mut Response| {
3896 crate::response_finalize::finalize_response_with_bg_completions(
3897 response,
3898 ctx,
3899 &identity_for_run.session,
3900 &bare_name,
3901 bind_trust.allows_bash_observation(),
3902 );
3903 };
3904 match run_tool_call(
3905 &bare_name,
3906 &arguments,
3907 &format_context,
3908 &tool_call_context,
3909 ctx,
3910 &dispatch,
3911 Some(&finalizer),
3912 Some(&mut phase_trace),
3913 ) {
3914 ToolCallOutcome::Unary(result) => {
3915 let response = result.response;
3916 let _ = tool_call_tx.send(ToolCallCompletion {
3917 text: result.text,
3918 phase_trace,
3919 });
3920 response
3921 }
3922 }
3923 };
3924 if matches!(bind_trust, BindTrust::Untrusted) {
3925 ctx.with_force_restrict(&request_id_for_force, run)
3926 } else {
3927 run()
3928 }
3929 })
3930 }),
3931 );
3932 let completion_tx = tx.clone();
3933 let completion_shutdown = Arc::clone(shutdown);
3934 let route = route_id;
3935 let corr = frame.header.corr;
3936 let flags = frame.header.flags;
3937 let ver = frame.header.ver;
3938 let completion_metrics = Arc::clone(metrics);
3939 tokio::spawn(async move {
3940 let _response_task = ResponseTaskGuard::new(&completion_metrics);
3941 let response = await_executor_response(rx, request_id.clone()).await;
3942 let (text, phase_trace) = match tool_call_rx.await {
3943 Ok(completion) => (completion.text, Some(completion.phase_trace)),
3944 Err(_) => (
3945 crate::subc_format::format_response_with_context(
3946 &bare_name_for_frame,
3947 &response,
3948 &format_context_for_frame,
3949 ),
3950 None,
3951 ),
3952 };
3953 let result = ToolCallResult { text, response };
3954 let fatal = response_is_fatal_panic(&result.response);
3955 match build_tool_response_frame(ver, route, corr, flags, &result, bind_trust) {
3956 Ok(response_frame) => {
3957 let send_result = if let Some(phase_trace) = phase_trace {
3958 let trace = ToolResponseWriteTrace::new(
3959 phase_trace,
3960 bare_name_for_frame,
3961 completion_root,
3962 completion_session,
3963 route.channel,
3964 corr,
3965 );
3966 send_traced_tool_response_frame(
3967 &completion_tx,
3968 &completion_metrics,
3969 response_frame,
3970 trace,
3971 )
3972 .await
3973 } else {
3974 send_reliable_writer_frame(
3975 &completion_tx,
3976 &completion_metrics,
3977 response_frame,
3978 "tool response",
3979 )
3980 .await
3981 };
3982 if let Err(error) = send_result {
3983 log::warn!("subc attach: failed to queue tool response frame: {error}");
3984 }
3985 }
3986 Err(error) => {
3987 log::error!("subc attach: failed to build tool response frame: {error}");
3988 }
3989 }
3990 if fatal {
3991 signal_fatal_teardown(
3992 &completion_tx,
3993 Some(route),
3994 ver,
3995 corr,
3996 &completion_shutdown,
3997 &completion_metrics,
3998 )
3999 .await;
4000 }
4001 });
4002 Ok(())
4003}
4004
4005fn submit_maintenance_job(
4006 executor: &Arc<Executor>,
4007 root_id: ProjectRootId,
4008 kind: MaintenanceDrainKind,
4009 bg_sessions_to_check: Vec<(String, u64)>,
4010 completion_tx: &mpsc::Sender<MaintenanceCompletion>,
4011 metrics: &Arc<DispatchPathMetrics>,
4012) {
4013 let request_id = format!(
4014 "subc-maintenance-drain-{}-{}",
4015 kind.label(),
4016 root_id.as_path().to_string_lossy()
4017 );
4018 let response_id = request_id.clone();
4019 let completion_root_id = root_id.clone();
4020 let maintenance_generation = executor
4021 .actor_context(&root_id)
4022 .map(|ctx| ctx.configure_generation())
4023 .unwrap_or(0);
4024 let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
4025 let lane = match kind {
4030 MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
4031 MaintenanceDrainKind::Watcher
4032 | MaintenanceDrainKind::Lsp
4033 | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
4034 };
4035 let rx = executor.submit_maintenance_async(
4036 root_id,
4037 lane,
4038 request_id.clone(),
4039 Box::new(move |ctx| {
4040 let outcome = match kind {
4041 MaintenanceDrainKind::Watcher => {
4042 let drained = runtime_drain::drain_watcher_events_bounded(
4043 ctx,
4044 runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
4045 );
4046 MaintenanceJobOutcome {
4047 empty_bg_sessions: Vec::new(),
4048 requeue_kind: drained.has_more.then_some(kind),
4049 }
4050 }
4051 MaintenanceDrainKind::Lsp => {
4052 let drained = runtime_drain::drain_lsp_events_bounded(
4053 ctx,
4054 runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
4055 );
4056 MaintenanceJobOutcome {
4057 empty_bg_sessions: Vec::new(),
4058 requeue_kind: drained.has_more.then_some(kind),
4059 }
4060 }
4061 MaintenanceDrainKind::ConfigureTail => {
4062 runtime_drain::drain_deferred_configure_maintenance(ctx);
4063 runtime_drain::drain_configure_warning_events(ctx);
4064 MaintenanceJobOutcome::default()
4065 }
4066 MaintenanceDrainKind::CompletionDrains => {
4067 runtime_drain::drain_search_index_events(ctx);
4068 runtime_drain::drain_callgraph_store_events(ctx);
4069 runtime_drain::drain_semantic_index_events(ctx);
4070 runtime_drain::drain_semantic_refresh_events(ctx);
4071 runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
4072 let empty_bg_sessions = bg_sessions_to_check
4073 .into_iter()
4074 .filter(|(session, _)| {
4075 !ctx.bash_background()
4076 .has_completions_for_session(Some(session.as_str()))
4077 })
4078 .collect();
4079 MaintenanceJobOutcome {
4080 empty_bg_sessions,
4081 requeue_kind: None,
4082 }
4083 }
4084 };
4085 let requeued = outcome.requeue_kind.is_some();
4086 let _ = outcome_tx.send(outcome);
4087 Response::success(
4088 response_id,
4089 json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
4090 )
4091 }),
4092 );
4093 let completion_tx = completion_tx.clone();
4094 let completion_metrics = Arc::clone(metrics);
4095 tokio::spawn(async move {
4096 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4097 let response = await_executor_response(rx, request_id).await;
4098 let outcome = outcome_rx.await.unwrap_or_default();
4099 let _ = send_counted_channel(
4100 &completion_tx,
4101 &completion_metrics.maintenance_queued,
4102 MaintenanceCompletion {
4103 root_id: completion_root_id,
4104 kind,
4105 response,
4106 empty_bg_sessions: outcome.empty_bg_sessions,
4107 requeue_kind: outcome.requeue_kind,
4108 },
4109 )
4110 .await;
4111 });
4112}
4113
4114async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
4115 rx.await
4116 .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
4117}
4118async fn signal_fatal_teardown(
4119 tx: &WriterSender,
4120 route: Option<RouteChannel>,
4121 ver: u8,
4122 corr: u64,
4123 shutdown: &Arc<Notify>,
4124 metrics: &DispatchPathMetrics,
4125) {
4126 if let Some(route) = route {
4127 if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
4128 if let Err(error) = send_frame(tx, metrics, frame).await {
4129 log::warn!(
4130 "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
4131 );
4132 }
4133 }
4134 }
4135 if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
4136 if let Err(error) = send_frame(tx, metrics, frame).await {
4137 log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
4138 }
4139 }
4140 shutdown.notify_one();
4141}
4142#[derive(Debug, Deserialize)]
4143#[serde(untagged)]
4144enum RouteRequest {
4145 BgEvents(BgEventsRequest),
4146 ToolCall(ToolCallRequest),
4147}
4148
4149#[derive(Debug, Deserialize)]
4150struct BgEventsRequest {
4151 op: BgEventsOp,
4152}
4153
4154#[derive(Debug, Deserialize)]
4155#[serde(rename_all = "snake_case")]
4156enum BgEventsOp {
4157 BgEvents,
4158}
4159
4160#[derive(Debug, Deserialize)]
4161struct ToolCallRequest {
4162 name: String,
4163 #[serde(default)]
4164 arguments: Value,
4165 #[serde(default)]
4170 preview: bool,
4171}
4172
4173#[cfg(test)]
4174pub(crate) mod test_support {
4175 use super::*;
4176 use crate::bash_background::BgTaskStatus;
4177 use crate::protocol::{
4178 BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
4179 ProgressFrame, StatusChangedFrame,
4180 };
4181 use serde_json::json;
4182
4183 pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
4184 let dir = tempfile::Builder::new()
4185 .prefix(name)
4186 .tempdir()
4187 .expect("temp root");
4188 let root = ProjectRootId::from_path(dir.path()).expect("project root id");
4189 (dir, root)
4190 }
4191
4192 pub(super) fn test_ctx() -> Arc<AppContext> {
4193 Arc::new(AppContext::new(
4194 Box::new(crate::parser::TreeSitterProvider::new()),
4195 crate::config::Config::default(),
4196 ))
4197 }
4198
4199 pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
4200 let deadline = Instant::now() + Duration::from_secs(30);
4201 loop {
4202 let observed = ctx.watcher_registry_count();
4203 if observed == expected {
4204 return;
4205 }
4206 assert!(
4207 Instant::now() < deadline,
4208 "watcher count did not settle before deadline: expected={expected}, observed={observed}"
4209 );
4210 std::thread::sleep(Duration::from_millis(50));
4211 }
4212 }
4213
4214 pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
4215 let deadline = Instant::now() + Duration::from_secs(30);
4216 loop {
4217 let observed = app.actor_root_count();
4218 if observed == expected {
4219 return;
4220 }
4221 assert!(
4222 Instant::now() < deadline,
4223 "actor root count did not settle before deadline: expected={expected}, observed={observed}"
4224 );
4225 std::thread::sleep(Duration::from_millis(50));
4226 }
4227 }
4228
4229 pub(super) fn status_frame(seq: u64) -> PushFrame {
4230 status_frame_with_session(seq, None)
4231 }
4232
4233 pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
4234 PushFrame::StatusChanged(StatusChangedFrame {
4235 frame_type: "status_changed",
4236 session_id: session_id.map(str::to_string),
4237 snapshot: json!({ "seq": seq }),
4238 })
4239 }
4240
4241 pub(super) fn completion_frame(task_id: &str) -> PushFrame {
4242 completion_frame_with_session(task_id, "session-1")
4243 }
4244
4245 pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
4246 PushFrame::BashCompleted(BashCompletedFrame {
4247 frame_type: "bash_completed",
4248 task_id: task_id.to_string(),
4249 session_id: session_id.to_string(),
4250 status: BgTaskStatus::Completed,
4251 exit_code: Some(0),
4252 command: format!("echo {task_id}"),
4253 output_preview: String::new(),
4254 output_truncated: false,
4255 original_tokens: None,
4256 compressed_tokens: None,
4257 tokens_skipped: false,
4258 })
4259 }
4260
4261 pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
4262 long_running_frame_with_session(task_id, "session-1", elapsed_ms)
4263 }
4264
4265 pub(super) fn long_running_frame_with_session(
4266 task_id: &str,
4267 session_id: &str,
4268 elapsed_ms: u64,
4269 ) -> PushFrame {
4270 PushFrame::BashLongRunning(BashLongRunningFrame {
4271 frame_type: "bash_long_running",
4272 task_id: task_id.to_string(),
4273 session_id: session_id.to_string(),
4274 command: format!("sleep {elapsed_ms}"),
4275 elapsed_ms,
4276 })
4277 }
4278
4279 pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
4280 PushFrame::BashPatternMatch(BashPatternMatchFrame {
4281 frame_type: "bash_pattern_match",
4282 task_id: "task-pattern".to_string(),
4283 session_id: session_id.to_string(),
4284 watch_id: "watch-1".to_string(),
4285 match_text: "needle".to_string(),
4286 match_offset: 7,
4287 context: "haystack needle".to_string(),
4288 once: true,
4289 reason: "pattern_match",
4290 })
4291 }
4292
4293 pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
4294 PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
4295 frame_type: "configure_warnings",
4296 session_id: session_id.map(str::to_string),
4297 project_root: "/tmp/subc-test".to_string(),
4298 warnings: Vec::new(),
4299 })
4300 }
4301
4302 pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
4303 route_identity_with_trust(root, session_id, BindTrust::FirstParty)
4304 }
4305
4306 pub(super) fn route_identity_with_trust(
4307 root: &ProjectRootId,
4308 session_id: &str,
4309 trust: BindTrust,
4310 ) -> RouteIdentity {
4311 RouteIdentity(Arc::new(RouteIdentityData {
4312 root: root.clone(),
4313 project_root: root.as_path().to_path_buf(),
4314 harness: "opencode".to_string(),
4315 session: session_id.to_string(),
4316 trust,
4317 consumer_elicitation_capable: false,
4318 }))
4319 }
4320
4321 pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
4322 PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
4323 }
4324
4325 pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
4326 match frame {
4327 PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
4328 _ => None,
4329 }
4330 }
4331
4332 pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
4333 match frame {
4334 PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
4335 _ => None,
4336 }
4337 }
4338
4339 pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
4340 let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
4341 body.get("task_id")
4342 .and_then(serde_json::Value::as_str)
4343 .map(str::to_string)
4344 }
4345}
4346
4347#[cfg(test)]
4348mod tests {
4349 use super::test_support::{
4350 completion_frame, test_ctx, test_root, wait_for_actor_root_count, wait_for_watcher_count,
4351 };
4352 use super::*;
4353
4354 fn attach_error(kind: io::ErrorKind) -> SubcError {
4355 SubcError::Connect {
4356 endpoint: "127.0.0.1:1".to_string(),
4357 source: io::Error::new(kind, "constructed attach failure"),
4358 }
4359 }
4360
4361 fn auth_io_error(kind: io::ErrorKind) -> SubcError {
4362 SubcError::Auth {
4363 endpoint: "127.0.0.1:1".to_string(),
4364 source: subc_transport::AuthError::Io {
4365 stage: subc_transport::AuthStage::ServerProof,
4366 source: io::Error::new(kind, "constructed auth failure"),
4367 },
4368 }
4369 }
4370
4371 #[test]
4372 fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
4373 let transient_errors = vec![
4374 attach_error(io::ErrorKind::ConnectionRefused),
4375 attach_error(io::ErrorKind::TimedOut),
4376 attach_error(io::ErrorKind::ConnectionReset),
4377 auth_io_error(io::ErrorKind::ConnectionAborted),
4378 auth_io_error(io::ErrorKind::BrokenPipe),
4379 SubcError::Auth {
4380 endpoint: "127.0.0.1:1".to_string(),
4381 source: subc_transport::AuthError::UnexpectedEof {
4382 stage: subc_transport::AuthStage::ServerProof,
4383 expected: 4,
4384 actual: 0,
4385 },
4386 },
4387 SubcError::Auth {
4388 endpoint: "127.0.0.1:1".to_string(),
4389 source: subc_transport::AuthError::Timeout {
4390 stage: subc_transport::AuthStage::ServerProof,
4391 deadline: AUTH_DEADLINE,
4392 },
4393 },
4394 ];
4395 for error in &transient_errors {
4396 assert_eq!(
4397 classify_attach_error(error),
4398 AttachErrorClass::Transient,
4399 "expected transient: {error}"
4400 );
4401 }
4402
4403 let permanent_errors = vec![
4404 attach_error(io::ErrorKind::PermissionDenied),
4405 auth_io_error(io::ErrorKind::InvalidData),
4406 SubcError::Auth {
4407 endpoint: "127.0.0.1:1".to_string(),
4408 source: subc_transport::AuthError::InvalidServerProof,
4409 },
4410 SubcError::Auth {
4411 endpoint: "127.0.0.1:1".to_string(),
4412 source: subc_transport::AuthError::DaemonIdMismatch,
4413 },
4414 SubcError::ConnectionFile {
4415 path: PathBuf::from("subc-connection.json"),
4416 source: subc_transport::ConnectionFileError::Invalid {
4417 reason: "constructed invalid file".to_string(),
4418 },
4419 },
4420 SubcError::NoEndpoint {
4421 path: PathBuf::from("subc-connection.json"),
4422 },
4423 SubcError::InvalidEndpoint {
4424 path: PathBuf::from("subc-connection.json"),
4425 endpoint: "not-an-ip:1234".to_string(),
4426 },
4427 ];
4428 for error in &permanent_errors {
4429 assert_eq!(
4430 classify_attach_error(error),
4431 AttachErrorClass::Permanent,
4432 "expected permanent: {error}"
4433 );
4434 }
4435 }
4436
4437 #[test]
4438 fn incompatible_wire_version_is_rejected_before_tcp_connect() {
4439 let conn_dir = tempfile::tempdir().expect("connection tempdir");
4440 let conn_path = conn_dir.path().join("subc-connection.json");
4441 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
4442 listener
4443 .set_nonblocking(true)
4444 .expect("set listener nonblocking");
4445 let port = listener.local_addr().expect("listener addr").port();
4446 connection_file::write_atomic(
4447 &conn_path,
4448 &connection_file::ConnectionInfo {
4449 schema: connection_file::SCHEMA_VERSION,
4450 wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
4451 endpoints: vec![connection_file::Endpoint {
4452 host: "127.0.0.1".to_string(),
4453 port,
4454 }],
4455 key: vec![0x42; subc_transport::KEY_LEN],
4456 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4457 pid: std::process::id(),
4458 daemon_ver: "subc-test".to_string(),
4459 },
4460 )
4461 .expect("write connection file");
4462
4463 let runtime = tokio::runtime::Builder::new_current_thread()
4464 .enable_all()
4465 .build()
4466 .expect("test runtime");
4467 let result = runtime.block_on(connect_and_authenticate_with_policy(
4468 &conn_path,
4469 AttachRetryPolicy {
4470 budget: Duration::from_secs(1),
4471 initial_backoff: Duration::from_millis(5),
4472 max_backoff: Duration::from_millis(10),
4473 jitter_percent: 0,
4474 },
4475 ));
4476 assert!(matches!(
4477 result,
4478 Err(SubcError::ConnectionFile {
4479 source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
4480 ..
4481 })
4482 ));
4483 assert!(matches!(
4484 listener.accept(),
4485 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
4486 ));
4487 }
4488
4489 #[test]
4490 fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
4491 let conn_dir = tempfile::tempdir().expect("connection tempdir");
4492 let conn_path = conn_dir.path().join("subc-connection.json");
4493 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
4494 let port = listener.local_addr().expect("reserved addr").port();
4495 drop(listener);
4496 connection_file::write_atomic(
4497 &conn_path,
4498 &connection_file::ConnectionInfo {
4499 schema: connection_file::SCHEMA_VERSION,
4500 wire_version: Some(PROTOCOL_VERSION),
4501 endpoints: vec![connection_file::Endpoint {
4502 host: "127.0.0.1".to_string(),
4503 port,
4504 }],
4505 key: vec![0x42; subc_transport::KEY_LEN],
4506 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4507 pid: std::process::id(),
4508 daemon_ver: "subc-test".to_string(),
4509 },
4510 )
4511 .expect("write connection file");
4512
4513 let policy = AttachRetryPolicy {
4514 budget: Duration::from_millis(40),
4515 initial_backoff: Duration::from_millis(5),
4516 max_backoff: Duration::from_millis(10),
4517 jitter_percent: 0,
4518 };
4519 let runtime = tokio::runtime::Builder::new_current_thread()
4520 .enable_all()
4521 .build()
4522 .expect("test runtime");
4523 let started_at = Instant::now();
4524 let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
4525 let elapsed = started_at.elapsed();
4526 let error = match result {
4527 Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
4528 Err(error) => error,
4529 };
4530
4531 assert!(matches!(error, SubcError::Connect { .. }), "{error}");
4532 assert!(
4533 elapsed >= Duration::from_millis(35),
4534 "retry budget ended too early: {elapsed:?}"
4535 );
4536 assert!(
4537 elapsed < Duration::from_secs(1),
4538 "retry budget was not bounded: {elapsed:?}"
4539 );
4540 }
4541
4542 fn due_maintenance_jobs_without_actor_context(
4543 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4544 budget: usize,
4545 pending_bind_roots: &HashSet<ProjectRootId>,
4546 ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
4547 due_maintenance_jobs(
4548 live_roots,
4549 None,
4550 &HashMap::new(),
4551 &HashSet::new(),
4552 budget,
4553 pending_bind_roots,
4554 )
4555 }
4556
4557 fn actor_ctx_with_dirty_search_index(
4558 root: &Path,
4559 storage: &Path,
4560 file_name: &str,
4561 old_contents: &str,
4562 new_contents: &str,
4563 ) -> (Arc<AppContext>, PathBuf, PathBuf) {
4564 let file = root.join(file_name);
4565 std::fs::write(&file, old_contents).expect("write source");
4566 let canonical_root = std::fs::canonicalize(root).expect("canonical root");
4567 let ctx = Arc::new(AppContext::new(
4568 Box::new(crate::parser::TreeSitterProvider::new()),
4569 Config {
4570 project_root: Some(root.to_path_buf()),
4571 storage_dir: Some(storage.to_path_buf()),
4572 ..Config::default()
4573 },
4574 ));
4575 ctx.set_canonical_cache_root(canonical_root.clone());
4576
4577 let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
4578 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
4579 let git_head = index.stored_git_head().map(str::to_owned);
4580 index.write_to_disk(&cache_dir, git_head.as_deref());
4581
4582 std::fs::write(&file, new_contents).expect("edit source");
4583 index.update_file(&file);
4584 *ctx.search_index()
4585 .write()
4586 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
4587 (ctx, canonical_root, cache_dir)
4588 }
4589
4590 #[test]
4591 fn graceful_shutdown_flushes_every_actor_search_index() {
4592 let storage = tempfile::tempdir().expect("storage tempdir");
4593 let (root1_dir, root1) = test_root("shutdown-flush-root-1");
4594 let (root2_dir, root2) = test_root("shutdown-flush-root-2");
4595 let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
4596 root1_dir.path(),
4597 storage.path(),
4598 "alpha.txt",
4599 "old actor one token\n",
4600 "new actor one token\n",
4601 );
4602 let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
4603 root2_dir.path(),
4604 storage.path(),
4605 "beta.txt",
4606 "old actor two token\n",
4607 "new actor two token\n",
4608 );
4609
4610 let executor = Executor::new();
4611 assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
4612 assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
4613
4614 flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
4615
4616 let mut restored1 =
4617 crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
4618 .expect("load flushed root one index");
4619 restored1.ready = true;
4620 assert_eq!(
4621 restored1
4622 .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
4623 .matches
4624 .len(),
4625 1,
4626 "graceful subc shutdown should flush the first root's trigram delta"
4627 );
4628
4629 let mut restored2 =
4630 crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
4631 .expect("load flushed root two index");
4632 restored2.ready = true;
4633 assert_eq!(
4634 restored2
4635 .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
4636 .matches
4637 .len(),
4638 1,
4639 "graceful subc shutdown should flush every registered root"
4640 );
4641 }
4642
4643 #[test]
4644 fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
4645 let _ = env_logger::builder().is_test(true).try_init();
4646 let (root_dir, root) = test_root("idle-root-reaper");
4647 let storage = tempfile::tempdir().expect("storage tempdir");
4648 std::fs::write(
4649 root_dir.path().join("main.rs"),
4650 "fn entry() { leaf(); }\nfn leaf() {}\n",
4651 )
4652 .expect("source file");
4653 let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
4654 let app = App::default_shared();
4655 let ctx = Arc::new(AppContext::from_app(
4656 Arc::clone(&app),
4657 Config {
4658 project_root: Some(canonical_root.clone()),
4659 storage_dir: Some(storage.path().to_path_buf()),
4660 callgraph_store: true,
4661 search_index: true,
4662 ..Config::default()
4663 },
4664 ));
4665 ctx.set_canonical_cache_root(canonical_root.clone());
4666 assert!(ctx
4667 .ensure_callgraph_store()
4668 .expect("build callgraph store")
4669 .is_some());
4670
4671 let cache_dir =
4672 crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
4673 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
4674 let git_head = index.stored_git_head().map(str::to_owned);
4675 index.write_to_disk(&cache_dir, git_head.as_deref());
4676 *ctx.search_index()
4677 .write()
4678 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
4679 let seeded_generation =
4682 crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
4683 .expect("seeded artifact generation");
4684 crate::cache_freshness::record_verify_completed(
4685 &canonical_root,
4686 crate::cache_freshness::VerifyArtifact::Search,
4687 Some(seeded_generation),
4688 );
4689 assert!(
4690 matches!(
4691 crate::cache_freshness::warm_verify_plan(
4692 canonical_root.as_path(),
4693 crate::cache_freshness::VerifyArtifact::Search,
4694 Some(seeded_generation),
4695 ),
4696 crate::cache_freshness::WarmVerifyPlan::Skip
4697 ),
4698 "memo must be warm before eviction for the downgrade assertion to bite"
4699 );
4700
4701 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
4702 let _dispatch_tx = dispatch_tx;
4703 let shutdown = Arc::new(AtomicBool::new(false));
4704 let thread_shutdown = Arc::clone(&shutdown);
4705 let join = std::thread::spawn(move || {
4706 while !thread_shutdown.load(Ordering::SeqCst) {
4707 std::thread::yield_now();
4708 }
4709 });
4710 ctx.install_watcher_runtime(
4711 dispatch_rx,
4712 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
4713 );
4714 wait_for_watcher_count(&ctx, 1);
4715
4716 let executor = Arc::new(Executor::new());
4717 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4718 ctx.mark_subc_unbound();
4719 let mut live_roots = HashMap::new();
4720 let mut meta = RootMeta::new(Instant::now());
4721 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
4722 meta.unbound_quiesced = true;
4723 live_roots.insert(root.clone(), meta);
4724
4725 let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
4726 assert!(message.contains("evicted idle root"));
4727 assert!(message.contains("freed ~"));
4728 assert!(message.contains("semantic"));
4729 assert!(!message.contains("semantic not estimated retained"));
4730 assert!(message.contains("trigram"));
4731 assert!(message.contains("retained: bash"));
4732 assert!(message.contains("parser_pool"));
4733
4734 assert_eq!(
4735 reap_idle_roots(
4736 Instant::now(),
4737 &mut live_roots,
4738 &HashMap::new(),
4739 &HashMap::new(),
4740 &executor,
4741 )
4742 .evicted,
4743 1
4744 );
4745 assert!(ctx.search_index().read().unwrap().is_none());
4746 wait_for_watcher_count(&ctx, 0);
4747 assert!(
4752 matches!(
4753 crate::cache_freshness::warm_verify_plan(
4754 canonical_root.as_path(),
4755 crate::cache_freshness::VerifyArtifact::Search,
4756 Some(seeded_generation),
4757 ),
4758 crate::cache_freshness::WarmVerifyPlan::Strict
4759 ),
4760 "idle eviction must force strict re-verification"
4761 );
4762 assert!(
4763 crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
4764 );
4765 ctx.mark_subc_bound();
4766 assert!(ctx
4767 .ensure_callgraph_store()
4768 .expect("reopen callgraph store")
4769 .is_some());
4770 assert!(live_roots[&root].idle_artifacts_evicted);
4771 }
4772
4773 #[test]
4774 fn idle_root_reaper_applies_ttl_to_unbound_roots() {
4775 let (_root_dir, root) = test_root("idle-root-ttl-gate");
4776 let ctx = test_ctx();
4777 let executor = Arc::new(Executor::new());
4778 assert!(executor.register_actor(root.clone(), ctx));
4779 let ctx = executor.actor_context(&root).expect("actor context");
4780 ctx.mark_subc_unbound();
4781 let now = Instant::now();
4782 let mut meta = RootMeta::new(now);
4783 meta.unbound_quiesced = true;
4784 let mut live_roots = HashMap::from([(root.clone(), meta)]);
4785
4786 assert_eq!(
4790 reap_idle_roots(
4791 now,
4792 &mut live_roots,
4793 &HashMap::new(),
4794 &HashMap::new(),
4795 &executor,
4796 )
4797 .evicted,
4798 0
4799 );
4800 assert!(!live_roots[&root].idle_artifacts_evicted);
4801
4802 ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
4808 assert_eq!(
4809 reap_idle_roots(
4810 now + IDLE_ROOT_TTL,
4811 &mut live_roots,
4812 &HashMap::new(),
4813 &HashMap::new(),
4814 &executor,
4815 )
4816 .evicted,
4817 1
4818 );
4819 assert!(live_roots[&root].idle_artifacts_evicted);
4820 assert!(
4821 ctx.take_pending_search_index_paths().is_empty(),
4822 "TTL eviction must dispose retained pending reconciliation paths"
4823 );
4824 }
4825
4826 #[test]
4827 fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
4828 let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
4829 let ctx = test_ctx();
4830 let executor = Arc::new(Executor::new());
4831 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4832 ctx.mark_subc_unbound();
4833
4834 let pending = root.as_path().join("edited-while-unbound.rs");
4840 ctx.add_pending_search_index_paths([pending.clone()]);
4841 let dirty_source = root.as_path().join("dirty.rs");
4842 std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
4843 let mut dirty = crate::search_index::SearchIndex::new();
4844 dirty.ready = true;
4845 dirty.update_file(&dirty_source);
4846 *ctx.search_index()
4847 .write()
4848 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
4849 assert!(ctx.artifact_eviction_blocked());
4850
4851 let mut live_roots = HashMap::new();
4852 let mut meta = RootMeta::new(Instant::now());
4853 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
4854 meta.unbound_quiesced = true;
4855 live_roots.insert(root.clone(), meta);
4856
4857 assert_eq!(
4858 reap_idle_roots(
4859 Instant::now(),
4860 &mut live_roots,
4861 &HashMap::new(),
4862 &HashMap::new(),
4863 &executor,
4864 )
4865 .evicted,
4866 0,
4867 "the dirty index must still block this eviction"
4868 );
4869 assert_eq!(
4870 ctx.take_pending_search_index_paths(),
4871 vec![pending],
4872 "a blocked eviction must restore the taken pending paths"
4873 );
4874 }
4875
4876 #[test]
4877 fn idle_reap_with_bound_route_keeps_watcher_running() {
4878 let (_root_dir, root) = test_root("bound-root-reap-gate");
4879 let ctx = test_ctx();
4880 let executor = Arc::new(Executor::new());
4881 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4882
4883 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
4884 let _dispatch_tx = dispatch_tx;
4885 let shutdown = Arc::new(AtomicBool::new(false));
4886 let thread_shutdown = Arc::clone(&shutdown);
4887 let join = std::thread::spawn(move || {
4888 while !thread_shutdown.load(Ordering::SeqCst) {
4889 std::thread::yield_now();
4890 }
4891 });
4892 ctx.install_watcher_runtime(
4893 dispatch_rx,
4894 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
4895 );
4896
4897 let mut meta = RootMeta::new(Instant::now());
4898 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
4899 let mut live_roots = HashMap::from([(root.clone(), meta)]);
4900 let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
4901 assert_eq!(
4902 reap_idle_roots(
4903 Instant::now(),
4904 &mut live_roots,
4905 &HashMap::new(),
4906 &bound,
4907 &executor,
4908 )
4909 .evicted,
4910 0
4911 );
4912 wait_for_watcher_count(&ctx, 1);
4913 ctx.stop_watcher_runtime_in_background();
4914 wait_for_watcher_count(&ctx, 0);
4915 }
4916
4917 #[test]
4918 fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
4919 let (root_dir, root) = test_root("deleted-root-reap");
4920 let app = App::default_shared();
4921 let ctx = Arc::new(AppContext::from_app(
4922 Arc::clone(&app),
4923 Config {
4924 project_root: Some(root.as_path().to_path_buf()),
4925 ..Config::default()
4926 },
4927 ));
4928 ctx.set_canonical_cache_root(root.as_path().to_path_buf());
4929 ctx.mark_subc_unbound();
4930 let executor = Arc::new(Executor::new());
4931 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4932 assert_eq!(app.actor_root_count(), 1);
4933 drop(ctx);
4934 root_dir.close().expect("delete project root");
4935
4936 let mut meta = RootMeta::new(Instant::now());
4937 meta.unbound_quiesced = true;
4938 let mut live_roots = HashMap::from([(root.clone(), meta)]);
4939 let outcome = reap_idle_roots(
4940 Instant::now(),
4941 &mut live_roots,
4942 &HashMap::new(),
4943 &HashMap::new(),
4944 &executor,
4945 );
4946 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
4947 assert!(!executor.actor_registered(&root));
4948 assert!(!live_roots.contains_key(&root));
4949 wait_for_actor_root_count(&app, 0);
4950
4951 let status_ctx = AppContext::from_app(app, Config::default());
4952 let status = status_ctx.build_status_snapshot();
4953 assert_eq!(status["runtime"]["live_actor_roots"], 0);
4954 assert_eq!(status["runtime"]["open_routes"], 0);
4955 }
4956
4957 #[test]
4958 fn unbound_root_quiesces_maintenance_without_removing_actor() {
4959 let (_root_dir, root) = test_root("unbound-root-quiesce");
4960 let ctx = test_ctx();
4961 let executor = Arc::new(Executor::new());
4962 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4963 let mut meta = RootMeta::new(Instant::now());
4964 meta.maintenance_pending = true;
4965 meta.maintenance_jobs_in_flight = 1;
4966 meta.maintenance_queued_kinds
4967 .push_back(MaintenanceDrainKind::ConfigureTail);
4968 let mut live_roots = HashMap::from([(root.clone(), meta)]);
4969 *ctx.search_index()
4971 .write()
4972 .unwrap_or_else(std::sync::PoisonError::into_inner) =
4973 Some(crate::search_index::SearchIndex::new());
4974 ctx.set_cache_writer_capabilities(true, true);
4975 let pending = root.as_path().join("pending.rs");
4976 ctx.add_pending_search_index_paths([pending.clone()]);
4977 let canonical_root = root.as_path().to_path_buf();
4981 let artifact = canonical_root.join("cache.bin");
4982 std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
4983 let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
4984 crate::cache_freshness::record_verify_completed(
4985 &canonical_root,
4986 crate::cache_freshness::VerifyArtifact::Search,
4987 seeded_generation,
4988 );
4989 assert!(matches!(
4990 crate::cache_freshness::warm_verify_plan(
4991 &canonical_root,
4992 crate::cache_freshness::VerifyArtifact::Search,
4993 seeded_generation,
4994 ),
4995 crate::cache_freshness::WarmVerifyPlan::Skip
4996 ));
4997 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5000 let _dispatch_tx = dispatch_tx;
5001 let shutdown = Arc::new(AtomicBool::new(false));
5002 let thread_shutdown = Arc::clone(&shutdown);
5003 let join = std::thread::spawn(move || {
5004 while !thread_shutdown.load(Ordering::SeqCst) {
5005 std::thread::yield_now();
5006 }
5007 });
5008 ctx.install_watcher_runtime(
5009 dispatch_rx,
5010 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5011 );
5012 assert!(ctx.watcher_runtime_active());
5013
5014 quiesce_unbound_root(&root, &mut live_roots, &executor);
5015 let meta = &live_roots[&root];
5016 assert!(meta.unbound_quiesced);
5017 assert!(ctx.subc_unbound_quiesced());
5018 assert!(meta.maintenance_pending);
5019 assert!(meta.maintenance_queued_kinds.is_empty());
5020 assert!(executor.actor_registered(&root));
5021 assert!(
5025 ctx.search_index()
5026 .read()
5027 .unwrap_or_else(std::sync::PoisonError::into_inner)
5028 .is_some(),
5029 "quiesce must not evict resident artifacts"
5030 );
5031 assert_eq!(
5032 ctx.pending_callgraph_store_force_token(),
5033 None,
5034 "quiesce must not force a callgraph rebuild"
5035 );
5036 assert_eq!(
5037 ctx.take_pending_search_index_paths(),
5038 vec![pending],
5039 "quiesce must retain pending watcher-derived paths"
5040 );
5041 assert!(
5042 matches!(
5043 crate::cache_freshness::warm_verify_plan(
5044 &canonical_root,
5045 crate::cache_freshness::VerifyArtifact::Search,
5046 seeded_generation,
5047 ),
5048 crate::cache_freshness::WarmVerifyPlan::Skip
5049 ),
5050 "quiesce must not invalidate the warm verify memo"
5051 );
5052 assert!(
5053 ctx.watcher_runtime_active(),
5054 "quiesce must not stop a running watcher"
5055 );
5056 ctx.stop_watcher_runtime();
5057
5058 let meta = live_roots.get_mut(&root).expect("root metadata");
5059 note_maintenance_completion(
5060 meta,
5061 Some(MaintenanceDrainKind::ConfigureTail),
5062 false,
5063 meta.unbound_quiesced,
5064 );
5065 assert!(!meta.maintenance_pending);
5066 assert!(meta.maintenance_queued_kinds.is_empty());
5067 }
5068
5069 #[test]
5070 fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
5071 let (_dir, root) = test_root("same-root-replacement");
5072 let route = route_key(7, 1);
5073 let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5074 let root_channels = HashMap::new();
5075
5076 assert!(!route_removal_will_quiesce_root(
5077 &root,
5078 route,
5079 &installed_channels,
5080 false,
5081 Some(&root),
5082 ));
5083 assert!(route_removal_will_quiesce_root(
5084 &root,
5085 route,
5086 &installed_channels,
5087 false,
5088 None,
5089 ));
5090 assert!(!should_quiesce_removed_root(
5091 &root,
5092 &root_channels,
5093 false,
5094 Some(&root),
5095 ));
5096 assert!(should_quiesce_removed_root(
5097 &root,
5098 &root_channels,
5099 false,
5100 None,
5101 ));
5102 assert!(!should_quiesce_removed_root(
5103 &root,
5104 &root_channels,
5105 true,
5106 None,
5107 ));
5108 }
5109
5110 #[test]
5111 fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
5112 let (_root_dir, root) = test_root("unbound-root-route-count");
5113 let executor = Arc::new(Executor::new());
5114 assert!(executor.register_actor(root.clone(), test_ctx()));
5115 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5116 let mut root_channels = HashMap::from([(
5117 root.clone(),
5118 HashSet::from([route_key(7, 1), route_key(8, 1)]),
5119 )]);
5120
5121 remove_root_channel(&mut root_channels, &root, route_key(7, 1));
5122 if !root_channels.contains_key(&root) {
5123 quiesce_unbound_root(&root, &mut live_roots, &executor);
5124 }
5125 assert!(!live_roots[&root].unbound_quiesced);
5126
5127 remove_root_channel(&mut root_channels, &root, route_key(8, 1));
5128 if !root_channels.contains_key(&root) {
5129 quiesce_unbound_root(&root, &mut live_roots, &executor);
5130 }
5131 assert!(live_roots[&root].unbound_quiesced);
5132
5133 live_roots
5134 .get_mut(&root)
5135 .expect("root metadata")
5136 .note_activity();
5137 assert!(
5138 live_roots[&root].unbound_quiesced,
5139 "late asynchronous activity must not reactivate an unbound root"
5140 );
5141
5142 live_roots
5143 .get_mut(&root)
5144 .expect("root metadata")
5145 .reactivate_bound();
5146 assert!(!live_roots[&root].unbound_quiesced);
5147 }
5148
5149 #[test]
5150 fn allocator_pressure_relief_requires_every_root_to_be_idle() {
5151 let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
5152 let (_active_dir, active_root) = test_root("allocator-relief-active");
5153 let now = Instant::now();
5154 let mut live_roots = HashMap::new();
5155 let mut idle = RootMeta::new(now);
5156 idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
5157 live_roots.insert(idle_root, idle);
5158 assert!(process_has_been_idle(now, &live_roots));
5159
5160 live_roots.insert(active_root.clone(), RootMeta::new(now));
5161 assert!(!process_has_been_idle(now, &live_roots));
5162
5163 let active = live_roots
5164 .get_mut(&active_root)
5165 .expect("active root metadata");
5166 active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
5167 active.active_bash_waits = 1;
5168 assert!(!process_has_been_idle(now, &live_roots));
5169 }
5170
5171 #[test]
5172 fn pressure_relief_log_reports_before_and_after_measurements() {
5173 let allocator = crate::memory::AllocatorMemorySnapshot {
5174 status: "measured",
5175 bytes_in_use: Some(8 * 1024 * 1024),
5176 size_allocated: Some(12 * 1024 * 1024),
5177 retained_slack_bytes: Some(4 * 1024 * 1024),
5178 not_estimated: None,
5179 };
5180 let relief = crate::memory::AllocatorPressureRelief {
5181 bytes_released: 3 * 1024 * 1024,
5182 rss_before_bytes: Some(20 * 1024 * 1024),
5183 rss_after_bytes: Some(17 * 1024 * 1024),
5184 allocator_before: allocator.clone(),
5185 allocator_after: crate::memory::AllocatorMemorySnapshot {
5186 size_allocated: Some(9 * 1024 * 1024),
5187 retained_slack_bytes: Some(1024 * 1024),
5188 ..allocator
5189 },
5190 };
5191 let message = pressure_relief_label(&relief);
5192 assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
5193 assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
5194 assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
5195 assert!(message.contains("reported 3.0 MB released"));
5196 }
5197
5198 #[test]
5199 fn due_maintenance_jobs_skip_poisoned_roots() {
5200 let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
5201 let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
5202 let mut live_roots = HashMap::new();
5203 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
5204 let mut poisoned_meta = RootMeta::new(Instant::now());
5205 poisoned_meta.maintenance_poisoned = true;
5206 live_roots.insert(poisoned_root.clone(), poisoned_meta);
5207
5208 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5209 &mut live_roots,
5210 MAINTENANCE_SUBMIT_BUDGET,
5211 &HashSet::new(),
5212 );
5213
5214 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5215 assert!(due.iter().all(|(root, _)| root == &healthy_root));
5216 assert!(!deferred);
5217 assert!(live_roots[&healthy_root].maintenance_pending);
5218 assert_eq!(
5219 live_roots[&healthy_root].maintenance_jobs_in_flight,
5220 INITIAL_MAINTENANCE_JOB_COUNT
5221 );
5222 assert!(!live_roots[&poisoned_root].maintenance_pending);
5223 }
5224
5225 #[test]
5226 fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
5227 let (_dir, root) = test_root("maintenance-unbound");
5228 let mut meta = RootMeta::new(Instant::now());
5229 meta.unbound_quiesced = true;
5230 let mut live_roots = HashMap::from([(root.clone(), meta)]);
5231
5232 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5233 &mut live_roots,
5234 MAINTENANCE_SUBMIT_BUDGET,
5235 &HashSet::new(),
5236 );
5237
5238 assert!(due.is_empty());
5239 assert!(!deferred);
5240 assert!(!live_roots[&root].maintenance_pending);
5241 }
5242
5243 #[test]
5244 fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
5245 let (_dir, root) = test_root("maintenance-idle-bg-subscription");
5246 let ctx = test_ctx();
5247 assert!(!ctx.completion_drains_have_work());
5248
5249 let executor = Executor::new();
5250 assert!(executor.register_actor(root.clone(), ctx));
5251 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5252 let session = "idle-session".to_string();
5253 let channel = route_key(17, 1);
5254 let bg_sub_by_session = HashMap::from([((root.clone(), session.clone()), channel)]);
5255 let mut bg_wake_pending = HashSet::new();
5256
5257 let (idle_tick_jobs, deferred) = due_maintenance_jobs(
5258 &mut live_roots,
5259 Some(&executor),
5260 &bg_sub_by_session,
5261 &bg_wake_pending,
5262 MAINTENANCE_SUBMIT_BUDGET,
5263 &HashSet::new(),
5264 );
5265 assert!(idle_tick_jobs.is_empty());
5266 assert!(!deferred);
5267 assert!(!live_roots[&root].maintenance_pending);
5268
5269 let mut bg_wake_epoch = HashMap::new();
5272 push::arm_bg_wake(
5273 root.clone(),
5274 session,
5275 channel,
5276 &mut bg_wake_pending,
5277 &mut bg_wake_epoch,
5278 );
5279 let (next_tick_jobs, deferred) = due_maintenance_jobs(
5280 &mut live_roots,
5281 Some(&executor),
5282 &bg_sub_by_session,
5283 &bg_wake_pending,
5284 MAINTENANCE_SUBMIT_BUDGET,
5285 &HashSet::new(),
5286 );
5287 assert_eq!(
5288 next_tick_jobs,
5289 vec![(root, MaintenanceDrainKind::CompletionDrains)]
5290 );
5291 assert!(!deferred);
5292 }
5293
5294 #[tokio::test]
5295 async fn subc_configure_tail_precedes_completed_search_install() {
5296 let root_dir = tempfile::tempdir().unwrap();
5297 let storage = tempfile::tempdir().unwrap();
5298 let root = ProjectRootId::from_path(root_dir.path()).unwrap();
5299 let (ctx, ignored_path) =
5300 runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
5301 let ctx = Arc::new(ctx);
5302 assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
5303 &ctx,
5304 &ignored_path
5305 ));
5306
5307 let executor = Arc::new(Executor::new());
5308 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5309 let metrics = Arc::new(DispatchPathMetrics::new());
5310 let (completion_tx, mut completion_rx) = mpsc::channel(4);
5311 submit_maintenance_job(
5312 &executor,
5313 root.clone(),
5314 MaintenanceDrainKind::ConfigureTail,
5315 Vec::new(),
5316 &completion_tx,
5317 &metrics,
5318 );
5319 submit_maintenance_job(
5320 &executor,
5321 root,
5322 MaintenanceDrainKind::CompletionDrains,
5323 Vec::new(),
5324 &completion_tx,
5325 &metrics,
5326 );
5327
5328 let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
5329 .await
5330 .expect("configure-tail completion timed out")
5331 .expect("configure-tail completion channel closed");
5332 let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
5333 .await
5334 .expect("completion-drains completion timed out")
5335 .expect("completion-drains completion channel closed");
5336 assert!(first.response.id.contains("configure-tail"));
5337 assert!(second.response.id.contains("completion-drains"));
5338 assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
5339 &ctx,
5340 &ignored_path
5341 ));
5342 assert_eq!(
5343 ctx.search_index()
5344 .read()
5345 .unwrap_or_else(std::sync::PoisonError::into_inner)
5346 .as_ref()
5347 .expect("completed search index installed")
5348 .file_count(),
5349 0,
5350 "configure must install the ignore matcher before pending paths replay"
5351 );
5352 ctx.stop_watcher_runtime();
5353 }
5354
5355 #[test]
5356 fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
5357 let (_dir, root) = test_root("maintenance-post-bind");
5358 let mut live_roots = HashMap::new();
5359 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5360
5361 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
5362 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
5363
5364 let meta = live_roots.get(&root).expect("root metadata");
5365 assert!(meta.maintenance_pending);
5366 assert_eq!(meta.maintenance_jobs_in_flight, 0);
5367 assert_eq!(
5368 meta.maintenance_queued_kinds
5369 .iter()
5370 .copied()
5371 .collect::<Vec<_>>(),
5372 vec![
5373 MaintenanceDrainKind::ConfigureTail,
5374 MaintenanceDrainKind::CompletionDrains,
5375 ]
5376 );
5377
5378 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5379 &mut live_roots,
5380 MAINTENANCE_SUBMIT_BUDGET,
5381 &HashSet::new(),
5382 );
5383
5384 assert_eq!(
5385 due,
5386 vec![
5387 (root.clone(), MaintenanceDrainKind::ConfigureTail),
5388 (root.clone(), MaintenanceDrainKind::CompletionDrains),
5389 ]
5390 );
5391 assert!(!deferred);
5392 assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
5393 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
5394 }
5395
5396 #[test]
5397 fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
5398 let mut live_roots = HashMap::new();
5399 let mut root_ids = Vec::new();
5400 let mut _dirs = Vec::new();
5401 for index in 0..4 {
5402 let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
5403 live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
5404 root_ids.push(root_id);
5405 _dirs.push(dir);
5406 }
5407
5408 let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
5409 let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
5410 &mut live_roots,
5411 small_budget,
5412 &HashSet::new(),
5413 );
5414
5415 assert_eq!(first_due.len(), small_budget);
5416 assert!(first_deferred);
5417 let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
5418 assert!(first_due_set
5419 .iter()
5420 .all(|root| live_roots[root].maintenance_pending));
5421 assert!(first_due_set
5422 .iter()
5423 .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
5424
5425 let all_roots: HashSet<_> = root_ids.into_iter().collect();
5426 let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
5427 assert!(deferred_roots
5428 .iter()
5429 .all(|root| !live_roots[root].maintenance_pending));
5430 }
5431
5432 #[test]
5433 fn due_maintenance_jobs_defers_pending_bind_roots() {
5434 let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
5435 let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
5436 let mut live_roots = HashMap::new();
5437 live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
5438 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
5439 let pending_bind_roots = HashSet::from([bind_root.clone()]);
5440
5441 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5442 &mut live_roots,
5443 usize::MAX,
5444 &pending_bind_roots,
5445 );
5446
5447 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5448 assert!(due.iter().all(|(root, _)| root == &healthy_root));
5449 assert!(!deferred);
5450 assert!(!live_roots[&bind_root].maintenance_pending);
5451 assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
5452 }
5453
5454 #[test]
5455 fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
5456 let (_dir, root) = test_root("maintenance-requeue");
5457 let mut live_roots = HashMap::new();
5458 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5459 let (due, deferred) = due_maintenance_jobs_without_actor_context(
5460 &mut live_roots,
5461 usize::MAX,
5462 &HashSet::new(),
5463 );
5464 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5465 assert!(due.iter().all(|(due_root, _)| due_root == &root));
5466 assert!(!deferred);
5467
5468 let meta = live_roots.get_mut(&root).unwrap();
5469 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
5470 assert!(meta.maintenance_pending);
5471 assert_eq!(
5472 meta.maintenance_jobs_in_flight,
5473 INITIAL_MAINTENANCE_JOB_COUNT - 1
5474 );
5475 assert_eq!(meta.maintenance_queued_kinds.len(), 1);
5476
5477 let (requeued, deferred) =
5478 due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
5479 assert_eq!(
5480 requeued,
5481 vec![(root.clone(), MaintenanceDrainKind::Watcher)]
5482 );
5483 assert!(!deferred);
5484 let meta = live_roots.get_mut(&root).unwrap();
5485 assert_eq!(
5486 meta.maintenance_jobs_in_flight,
5487 INITIAL_MAINTENANCE_JOB_COUNT
5488 );
5489 assert!(meta.maintenance_queued_kinds.is_empty());
5490
5491 for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
5492 note_maintenance_completion(meta, None, false, false);
5493 }
5494 assert!(!meta.maintenance_pending);
5495 assert_eq!(meta.maintenance_jobs_in_flight, 0);
5496 }
5497
5498 #[test]
5499 fn maintenance_requeue_drops_while_bind_is_pending() {
5500 let (_dir, root) = test_root("maintenance-bind-requeue");
5501 let mut live_roots = HashMap::new();
5502 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5503 let (due, _) = due_maintenance_jobs_without_actor_context(
5504 &mut live_roots,
5505 usize::MAX,
5506 &HashSet::new(),
5507 );
5508 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5509
5510 let meta = live_roots.get_mut(&root).unwrap();
5511 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
5512
5513 assert_eq!(
5514 meta.maintenance_jobs_in_flight,
5515 INITIAL_MAINTENANCE_JOB_COUNT - 1
5516 );
5517 assert!(meta.maintenance_queued_kinds.is_empty());
5518 assert!(meta.maintenance_pending);
5519 }
5520
5521 #[test]
5522 fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
5523 let mut meta = RootMeta::new(Instant::now());
5524 meta.unbound_quiesced = true;
5525
5526 assert!(!should_requiesce_after_maintenance(
5527 &meta,
5528 MaintenanceDrainKind::Lsp,
5529 false,
5530 ));
5531 assert!(!should_requiesce_after_maintenance(
5532 &meta,
5533 MaintenanceDrainKind::ConfigureTail,
5534 true,
5535 ));
5536 assert!(should_requiesce_after_maintenance(
5537 &meta,
5538 MaintenanceDrainKind::ConfigureTail,
5539 false,
5540 ));
5541 }
5542
5543 #[test]
5544 fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
5545 let (_dir, root) = test_root("maintenance-fatal");
5546 let mut live_roots = HashMap::new();
5547 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5548 let (due, _) = due_maintenance_jobs_without_actor_context(
5549 &mut live_roots,
5550 usize::MAX,
5551 &HashSet::new(),
5552 );
5553 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5554
5555 let meta = live_roots.get_mut(&root).unwrap();
5556 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
5557 assert!(meta.maintenance_poisoned);
5558 assert!(meta.maintenance_queued_kinds.is_empty());
5559
5560 for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
5561 note_maintenance_completion(meta, None, false, false);
5562 }
5563 assert!(!meta.maintenance_pending);
5564 assert_eq!(meta.maintenance_jobs_in_flight, 0);
5565 }
5566
5567 #[test]
5568 fn trust_for_principal_matrix() {
5569 assert_eq!(
5570 trust_for_principal(&Some(Principal::Direct)),
5571 BindTrust::FirstParty
5572 );
5573 assert_eq!(
5574 trust_for_principal(&Some(Principal::Reserved {
5575 module_id: "llm-runner".to_string(),
5576 })),
5577 BindTrust::FirstParty
5578 );
5579 assert_eq!(
5580 trust_for_principal(&Some(Principal::Reserved {
5581 module_id: "aft".to_string(),
5582 })),
5583 BindTrust::FirstParty
5584 );
5585 assert_eq!(
5586 trust_for_principal(&Some(Principal::Reserved {
5587 module_id: "subc-mcp".to_string(),
5588 })),
5589 BindTrust::Untrusted
5590 );
5591 assert_eq!(
5592 trust_for_principal(&Some(Principal::Reserved {
5593 module_id: "anything-unknown".to_string(),
5594 })),
5595 BindTrust::Untrusted
5596 );
5597 assert_eq!(
5598 trust_for_principal(&Some(Principal::Unverified)),
5599 BindTrust::Untrusted
5600 );
5601 assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
5602 }
5603
5604 #[test]
5605 fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
5606 let principal = Some(Principal::Direct);
5607 let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
5608 let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
5609
5610 assert_eq!(
5611 trust_for_bind(fingerprint_a, &principal),
5612 BindTrust::Untrusted
5613 );
5614 assert_eq!(
5615 trust_for_bind(fingerprint_b, &principal),
5616 BindTrust::Untrusted
5617 );
5618 }
5619
5620 #[tokio::test]
5621 async fn persistent_cancel_resolves_when_fired_before_await() {
5622 let signal = PersistentCancelSignal::new();
5626 signal.cancel();
5627 tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
5629 .await
5630 .expect("cancelled() must resolve when cancel fired beforehand");
5631
5632 let racing = PersistentCancelSignal::new();
5634 let racing_for_task = racing.clone();
5635 let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
5636 racing.cancel();
5637 tokio::time::timeout(Duration::from_secs(1), waiter)
5638 .await
5639 .expect("cancelled() must resolve when cancel races the await")
5640 .expect("waiter task panicked");
5641 }
5642
5643 #[test]
5644 fn ingress_epoch_validation_silently_drops_every_stale_route_frame() {
5645 let installed = HashMap::from([(7, 9)]);
5646 for ty in [
5647 FrameType::Request,
5648 FrameType::Response,
5649 FrameType::Error,
5650 FrameType::Push,
5651 FrameType::Cancel,
5652 FrameType::Goodbye,
5653 ] {
5654 let body = if ty.is_pure_header() {
5655 Vec::new()
5656 } else {
5657 br#"{}"#.to_vec()
5658 };
5659 let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
5660 assert!(!ingress_route_is_current(&installed, &stale), "{ty:?}");
5661 }
5662
5663 let uninstalled = Frame::build(
5664 FrameType::Request,
5665 control_flags(),
5666 8,
5667 1,
5668 42,
5669 br#"{}"#.to_vec(),
5670 )
5671 .unwrap();
5672 assert!(!ingress_route_is_current(&installed, &uninstalled));
5673
5674 let current = Frame::build(
5675 FrameType::Request,
5676 control_flags(),
5677 7,
5678 9,
5679 43,
5680 br#"{}"#.to_vec(),
5681 )
5682 .unwrap();
5683 let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
5684 assert!(ingress_route_is_current(&installed, ¤t));
5685 assert!(ingress_route_is_current(&installed, &control));
5686 assert_eq!(installed, HashMap::from([(7, 9)]));
5687 }
5688
5689 #[tokio::test]
5690 async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
5691 let (_dir, root) = test_root("route-bind-b2-ordering");
5692 let route = route_key(7, 3);
5693 let identity = RouteIdentity(Arc::new(RouteIdentityData {
5694 root: root.clone(),
5695 project_root: root.as_path().to_path_buf(),
5696 harness: "opencode".to_string(),
5697 session: "b2-session".to_string(),
5698 trust: BindTrust::FirstParty,
5699 consumer_elicitation_capable: false,
5700 }));
5701 let replay_key = push::ReplayKey::from_identity(&identity);
5702 let completion = RouteBindCompletion {
5703 route,
5704 identity,
5705 bind_root_id: root.clone(),
5706 inserted_new_actor: false,
5707 configure_response: Response::success("subc-bind-7", json!({})),
5708 diagnostics_on_edit: false,
5709 ver: PROTOCOL_VERSION,
5710 corr: 91,
5711 flags: control_flags(),
5712 };
5713 let mut pending_binds = HashMap::from([(
5714 route,
5715 PendingBind {
5716 bind_root_id: root,
5717 inserted_new_actor: false,
5718 cancelled: false,
5719 configure_request_id: "subc-bind-7".to_string(),
5720 started_at: Instant::now(),
5721 warned_half_deadline: false,
5722 deadline_reported: false,
5723 corr: 91,
5724 ver: PROTOCOL_VERSION,
5725 flags: control_flags(),
5726 cancellation: crate::executor::JobCancellation::new(),
5727 },
5728 )]);
5729 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5730 let mut push_buffer =
5731 HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
5732 let (writer_tx, mut writer_rx) = mpsc::channel(8);
5733 let metrics = Arc::new(DispatchPathMetrics::new());
5734
5735 handle_route_bind_completion(
5736 &writer_tx,
5737 completion,
5738 &mut HashMap::new(),
5739 &mut HashMap::new(),
5740 &mut HashMap::new(),
5741 &mut push_buffer,
5742 &mut HashMap::new(),
5743 &mut pending_binds,
5744 &mut installed_route_epochs,
5745 &Arc::new(Executor::new()),
5746 &Arc::new(Notify::new()),
5747 &metrics,
5748 )
5749 .await
5750 .unwrap();
5751
5752 let ack = writer_rx.try_recv().expect("RouteBindAck");
5753 assert_eq!(ack.header.ty, FrameType::Response);
5754 assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
5755 let route_frame = writer_rx.try_recv().expect("post-ack route frame");
5756 assert_eq!(route_frame.header.ty, FrameType::Push);
5757 assert_eq!(
5758 (route_frame.header.channel, route_frame.header.epoch),
5759 (route.channel, route.epoch)
5760 );
5761 }
5762}