1use crate::protocol::{
2 C2ErrorCategory, C2NodeEvent, C2NodeFailure, C2NodeResponse, C2RelayFailure, C2RelayFailureCode, GapKind, HealthResponse, NodeCursor, NodeFreshness, NodeGap, NodeId,
3 NodeIncarnationId, NodeRequest, ResolvedSpawnReceipt, RoutedNodeEvent, RoutedNodeResponse,
4 ManagedWorktreeLeaseState, ManagedWorktreeSpawnRequest, ManagedWorktreeSpawnRequestV2,
5 SpawnOverride, SpawnSpec,
6 NodeTransportState, ObservedNode, ProviderAdapterContractSupport, ProviderContractSupport,
7 ReadyResponse, SanitizedError, SlimNodeInventory, StatusResponse,
8 C2_API_VERSION, C2_PROVIDER_CONTRACT_MANIFEST_CAPABILITY,
9 MAX_C2_GAPS_PER_NODE, MAX_C2_NODES,
10};
11#[cfg(windows)]
12use crate::protocol::MAX_C2_ENDPOINT_BYTES;
13use gate4agent_node_protocol::{
14 ClientRole, FrameError, NegotiatedNodeCompatibility, NodeEvent, NodeEventEnvelope, NodeFailureCode,
15 NodeResponse, NodeSnapshot, ServerFrame,
16};
17use gate4agent_node_wire::{read_call_home_announce, LocalNodeClient, NodeClientError};
18use std::collections::{BTreeMap, BTreeSet};
19use std::io;
20use std::net::SocketAddr;
21use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, SystemTime, UNIX_EPOCH};
24#[cfg(unix)]
25use std::path::{Path, PathBuf};
26#[cfg(unix)]
27use std::os::unix::fs::PermissionsExt;
28use thiserror::Error;
29use tokio::io::{AsyncReadExt, AsyncWriteExt};
30use tokio::net::{TcpListener, TcpStream};
31use tokio::sync::{mpsc, oneshot, watch, Semaphore};
32use tokio::task::{JoinHandle, JoinSet};
33use tokio::time::{sleep, timeout, Instant};
34
35const MANAGED_RESUME_SETTLE_DEADLINE: Duration = Duration::from_secs(30);
36const NODE_REQUEST_IO_HEADROOM: Duration = Duration::from_secs(5);
37const NATIVE_SESSION_REQUEST_DEADLINE: Duration = Duration::from_secs(35);
38const WORKSPACE_ENTRY_CREATE_NODE_SEMANTIC_DEADLINE: Duration = Duration::from_secs(5);
39const WORKSPACE_ENTRY_CREATE_RELAY_DEADLINE: Duration =
40 WORKSPACE_ENTRY_CREATE_NODE_SEMANTIC_DEADLINE.saturating_add(NODE_REQUEST_IO_HEADROOM);
41
42const HEADER_LIMIT_BYTES: usize = 16 * 1024;
43const MAX_HTTP_CONNECTIONS: usize = 16;
44const RESPONSE_BODY_LIMIT_BYTES: usize = 8 * 1024 * 1024;
45
46mod control;
47
48#[cfg(windows)]
49pub const DEFAULT_C2_CONTROL_ENDPOINT: &str = r"\\.\pipe\gate4agent-c2";
50#[cfg(unix)]
51pub const DEFAULT_C2_CONTROL_ENDPOINT: &str = "gate4agent-c2.sock";
52
53#[cfg(windows)]
54pub fn default_c2_control_endpoint() -> Result<String, C2ConfigError> {
55 Ok(DEFAULT_C2_CONTROL_ENDPOINT.to_owned())
56}
57
58#[cfg(unix)]
59pub fn default_c2_control_endpoint() -> Result<String, C2ConfigError> {
60 let root = unix_runtime_root()?;
61 let directory = root.join("gate4agent");
62 let endpoint = directory.join(DEFAULT_C2_CONTROL_ENDPOINT);
63 validate_unix_endpoint(&endpoint).map_err(|_| C2ConfigError::InvalidControlEndpoint)?;
64 Ok(endpoint.to_string_lossy().into_owned())
65}
66
67#[cfg(unix)]
68fn unix_runtime_root() -> Result<PathBuf, C2ConfigError> {
69 if let Some(root) = std::env::var_os("XDG_RUNTIME_DIR").filter(|value| !value.is_empty()) {
70 let root = PathBuf::from(root);
71 if root.is_absolute() { return Ok(root); }
72 }
73 let home = std::env::var_os("HOME").filter(|value| !value.is_empty())
74 .ok_or_else(|| C2ConfigError::RuntimeEndpoint(
75 "neither absolute XDG_RUNTIME_DIR nor HOME is available".to_owned(),
76 ))?;
77 let home = PathBuf::from(home);
78 if !home.is_absolute() {
79 return Err(C2ConfigError::RuntimeEndpoint("HOME is not absolute".to_owned()));
80 }
81 Ok(home.join(".gate4agent").join("run"))
82}
83
84#[derive(Clone)]
85pub struct C2NodeConfig {
86 pub node_id: NodeId,
87 pub endpoint: String,
88 route: C2NodeRoute,
89 token: String,
90}
91
92#[derive(Clone, Copy, Debug, Eq, PartialEq)]
93enum C2NodeRoute {
94 Local,
95 SshForwardedLoopback(SocketAddr),
96 CallHome,
106}
107
108impl C2NodeConfig {
109 pub fn new(node_id: NodeId, endpoint: impl Into<String>, token: impl Into<String>) -> Result<Self, C2ConfigError> {
110 let endpoint = endpoint.into();
111 let token = token.into();
112 let (endpoint, route) = parse_node_endpoint(&endpoint)
113 .ok_or_else(|| C2ConfigError::InvalidEndpoint(node_id.clone()))?;
114 validate_token(&token)?;
115 Ok(Self { node_id, endpoint, route, token })
116 }
117
118 fn transport_label(&self) -> &'static str {
119 match self.route {
120 C2NodeRoute::Local => local_transport_label(),
121 C2NodeRoute::SshForwardedLoopback(_) => "ssh-forwarded-loopback",
122 C2NodeRoute::CallHome => "call-home",
123 }
124 }
125}
126
127#[derive(Clone, Copy, Debug)]
128pub struct C2Timings {
129 pub poll_interval: Duration,
130 pub fresh_for: Duration,
131 pub attempt_deadline: Duration,
132 pub transient_backoffs: [Duration; 5],
133 pub parked_backoff: Duration,
134 pub http_io_deadline: Duration,
135}
136
137impl Default for C2Timings {
138 fn default() -> Self {
139 Self {
140 poll_interval: Duration::from_millis(250),
141 fresh_for: Duration::from_secs(10),
142 attempt_deadline: Duration::from_secs(5),
143 transient_backoffs: [
144 Duration::from_millis(500), Duration::from_secs(1), Duration::from_secs(2),
145 Duration::from_secs(4), Duration::from_secs(8),
146 ],
147 parked_backoff: Duration::from_secs(30),
148 http_io_deadline: Duration::from_secs(3),
149 }
150 }
151}
152
153#[derive(Clone)]
154pub struct C2Config {
155 pub api_listen: SocketAddr,
156 pub control_endpoint: String,
157 pub control_tunnel_listen: Option<SocketAddr>,
160 api_token: String,
161 pub nodes: Vec<C2NodeConfig>,
162 pub node_listen: Option<SocketAddr>,
165 pub timings: C2Timings,
166}
167
168impl C2Config {
169 pub fn new(api_listen: SocketAddr, api_token: impl Into<String>, nodes: Vec<C2NodeConfig>) -> Result<Self, C2ConfigError> {
170 let api_token = api_token.into();
171 if !api_listen.ip().is_loopback() { return Err(C2ConfigError::NonLoopback(api_listen)); }
172 validate_token(&api_token)?;
173 if nodes.is_empty() || nodes.len() > MAX_C2_NODES { return Err(C2ConfigError::NodeCount(nodes.len())); }
174 let mut ids = BTreeSet::new();
175 let mut endpoints = BTreeSet::new();
176 if nodes.iter().any(|node| !ids.insert(node.node_id.clone())) { return Err(C2ConfigError::DuplicateNode); }
177 if nodes
183 .iter()
184 .filter(|node| node.route != C2NodeRoute::CallHome)
185 .any(|node| !endpoints.insert(endpoint_key(&node.endpoint)))
186 {
187 return Err(C2ConfigError::DuplicateEndpoint);
188 }
189 let control_endpoint = default_c2_control_endpoint()?;
190 if nodes.iter().any(|node| endpoints_equal(&node.endpoint, &control_endpoint)) {
191 return Err(C2ConfigError::ControlEndpointConflict);
192 }
193 Ok(Self {
194 api_listen,
195 control_endpoint,
196 control_tunnel_listen: None,
197 api_token,
198 nodes,
199 node_listen: None,
200 timings: C2Timings::default(),
201 })
202 }
203
204 pub fn with_node_listen(mut self, node_listen: SocketAddr) -> Result<Self, C2ConfigError> {
213 if node_listen.port() == 0 || node_listen.ip().is_unspecified() {
216 return Err(C2ConfigError::NonLoopbackNodeListen(node_listen));
217 }
218 if node_listen == self.api_listen {
219 return Err(C2ConfigError::NodeListenConflict);
220 }
221 if self.control_tunnel_listen == Some(node_listen) {
222 return Err(C2ConfigError::NonLoopbackNodeListen(node_listen));
223 }
224 self.node_listen = Some(node_listen);
225 Ok(self)
226 }
227
228 pub fn validate_call_home(&self) -> Result<(), C2ConfigError> {
234 let waiting = self.nodes.iter().find(|node| node.route == C2NodeRoute::CallHome);
235 match (waiting, self.node_listen) {
236 (Some(node), None) => Err(C2ConfigError::CallHomeWithoutListener(node.node_id.clone())),
237 _ => Ok(()),
238 }
239 }
240
241 pub fn with_timings(mut self, timings: C2Timings) -> Self {
242 self.timings = timings;
243 self
244 }
245
246 pub fn with_control_endpoint(mut self, endpoint: impl Into<String>) -> Result<Self, C2ConfigError> {
247 let endpoint = endpoint.into();
248 validate_control_endpoint(&endpoint)?;
249 if self.nodes.iter().any(|node| endpoints_equal(&node.endpoint, &endpoint)) {
250 return Err(C2ConfigError::ControlEndpointConflict);
251 }
252 self.control_endpoint = endpoint;
253 Ok(self)
254 }
255
256 pub fn with_control_tunnel_listen(mut self, listen: SocketAddr) -> Result<Self, C2ConfigError> {
260 if listen.port() == 0 || listen.ip().is_unspecified() {
261 return Err(C2ConfigError::InvalidControlTunnel(listen));
262 }
263 if listen == self.api_listen || self.node_listen == Some(listen) {
264 return Err(C2ConfigError::InvalidControlTunnel(listen));
265 }
266 self.control_tunnel_listen = Some(listen);
267 Ok(self)
268 }
269}
270
271fn validate_control_endpoint(endpoint: &str) -> Result<(), C2ConfigError> {
272 if !valid_local_endpoint(endpoint) {
273 return Err(C2ConfigError::InvalidControlEndpoint);
274 }
275 Ok(())
276}
277
278fn parse_node_endpoint(endpoint: &str) -> Option<(String, C2NodeRoute)> {
279 if endpoint == "accept" {
285 return Some((endpoint.to_owned(), C2NodeRoute::CallHome));
286 }
287 if let Some(authority) = endpoint.strip_prefix("tcp://") {
288 let address = authority.parse::<SocketAddr>().ok()?;
289 let is_exact_loopback = match address.ip() {
290 std::net::IpAddr::V4(ip) => ip == std::net::Ipv4Addr::LOCALHOST,
291 std::net::IpAddr::V6(ip) => ip == std::net::Ipv6Addr::LOCALHOST,
292 };
293 if !is_exact_loopback || address.port() == 0 {
294 return None;
295 }
296 return Some((format!("tcp://{address}"), C2NodeRoute::SshForwardedLoopback(address)));
297 }
298 if endpoint.contains("://") || !valid_local_endpoint(endpoint) {
299 return None;
300 }
301 Some((endpoint.to_owned(), C2NodeRoute::Local))
302}
303
304#[cfg(windows)]
305fn valid_local_endpoint(endpoint: &str) -> bool {
306 endpoint.starts_with(r"\\.\pipe\") && endpoint.len() > r"\\.\pipe\".len()
307 && endpoint.len() <= MAX_C2_ENDPOINT_BYTES
308}
309
310#[cfg(unix)]
311fn valid_local_endpoint(endpoint: &str) -> bool {
312 validate_unix_endpoint(Path::new(endpoint)).is_ok()
313}
314
315#[cfg(unix)]
316fn validate_unix_endpoint(endpoint: &Path) -> Result<(), ()> {
317 const MAX_UNIX_ENDPOINT_BYTES: usize = 103;
318 use std::os::unix::ffi::OsStrExt;
319
320 if !endpoint.is_absolute() || endpoint.file_name().is_none()
321 || endpoint.as_os_str().as_bytes().len() > MAX_UNIX_ENDPOINT_BYTES
322 {
323 return Err(());
324 }
325 Ok(())
326}
327
328#[cfg(windows)]
329fn endpoint_key(endpoint: &str) -> String {
330 if endpoint.starts_with("tcp://") { endpoint.to_owned() } else { endpoint.to_ascii_lowercase() }
331}
332
333#[cfg(unix)]
334fn endpoint_key(endpoint: &str) -> String { endpoint.to_owned() }
335
336fn endpoints_equal(left: &str, right: &str) -> bool { endpoint_key(left) == endpoint_key(right) }
337
338fn validate_token(token: &str) -> Result<(), C2ConfigError> {
339 if token.is_empty() || token.len() > 4096 || !token.bytes().all(|byte| matches!(byte, 0x21..=0x7e)) {
340 return Err(C2ConfigError::InvalidToken);
341 }
342 Ok(())
343}
344
345#[derive(Debug, Error)]
346pub enum C2ConfigError {
347 #[error("C2 tokens must contain 1..=4096 visible ASCII bytes without whitespace")]
348 InvalidToken,
349 #[cfg_attr(windows, error("node '{0}' requires a bounded Windows named pipe or exact loopback TCP endpoint"))]
350 #[cfg_attr(unix, error("node '{0}' requires a bounded local socket or exact loopback TCP endpoint"))]
351 InvalidEndpoint(NodeId),
352 #[error("C2 API listen address must be loopback: {0}")]
353 NonLoopback(SocketAddr),
354 #[error("C2 requires 1..=64 configured nodes; received {0}")]
355 NodeCount(usize),
356 #[error("C2 node IDs must be unique")]
357 DuplicateNode,
358 #[error("C2 node endpoints must be unique")]
359 DuplicateEndpoint,
360 #[cfg_attr(windows, error("C2 control endpoint must be a bounded local Windows named pipe"))]
361 #[cfg_attr(unix, error("C2 control endpoint must be a bounded local endpoint"))]
362 InvalidControlEndpoint,
363 #[error("C2 control endpoint must not equal a configured node endpoint")]
364 ControlEndpointConflict,
365 #[error("C2 control tunnel listen must be a concrete non-unspecified address with a nonzero port, not shared with another listener: {0}")]
366 InvalidControlTunnel(SocketAddr),
367 #[error("C2 node call-home listen must be loopback or a concrete C2 tunnel address with a nonzero port (not 0.0.0.0, not shared with another listener): {0}")]
368 NonLoopbackNodeListen(SocketAddr),
369 #[error("C2 node call-home listen address must not equal the API listen address")]
370 NodeListenConflict,
371 #[error("node '{0}' waits to be called but no --node-listen address was configured")]
372 CallHomeWithoutListener(NodeId),
373 #[cfg(unix)]
374 #[error("C2 default runtime endpoint is unavailable: {0}")]
375 RuntimeEndpoint(String),
376}
377
378type RelayResult = Result<RoutedNodeResponse, C2RelayFailure>;
379
380enum RelayCommand {
381 Request {
382 operator_connection_id: u64,
383 expected_incarnation_id: NodeIncarnationId,
384 request: NodeRequest,
385 reply: oneshot::Sender<RelayResult>,
386 },
387}
388
389#[derive(Clone)]
390struct RelayEndpoint {
391 commands: mpsc::Sender<RelayCommand>,
392 releases: mpsc::Sender<oneshot::Sender<()>>,
393 force_disconnect: watch::Sender<u64>,
394}
395
396#[derive(Clone)]
397struct OperatorHub {
398 sink: Arc<Mutex<Option<OperatorEventSink>>>,
399}
400
401#[derive(Clone)]
402struct OperatorEventSink {
403 connection_id: u64,
404 outbound: mpsc::Sender<control::QueuedFrame>,
405 budget: Arc<AtomicUsize>,
406 disconnect: watch::Sender<bool>,
407}
408
409impl OperatorHub {
410 fn new() -> Self { Self { sink: Arc::new(Mutex::new(None)) } }
411
412 fn attach(&self, sink: OperatorEventSink) {
413 *self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(sink);
414 }
415
416 fn detach(&self, connection_id: u64) {
417 let mut sink = self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
418 if sink.as_ref().is_some_and(|current| current.connection_id == connection_id) { *sink = None; }
419 }
420
421 fn is_active(&self, connection_id: u64) -> bool {
422 self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
423 .as_ref().is_some_and(|current| current.connection_id == connection_id)
424 }
425
426 fn has_active_operator(&self) -> bool {
427 self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner()).is_some()
428 }
429
430 fn publish(&self, event: RoutedNodeEvent) {
431 let sink = self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
432 if let Some(sink) = sink.as_ref() {
433 if control::queue_operator_event(&sink.outbound, &sink.budget, event).is_err() {
434 let _ = sink.disconnect.send(true);
435 }
436 }
437 }
438}
439
440fn relay_failure(
441 code: C2RelayFailureCode,
442 message: &'static str,
443 current_incarnation_id: Option<NodeIncarnationId>,
444) -> C2RelayFailure {
445 C2RelayFailure { code, message: message.to_owned(), current_incarnation_id }
446}
447
448#[derive(Debug, Error)]
449pub enum C2Error {
450 #[error("C2 API failed: {0}")]
451 Api(#[from] io::Error),
452 #[error("C2 task failed: {0}")]
453 Task(#[from] tokio::task::JoinError),
454}
455
456#[derive(Clone)]
457pub struct C2ShutdownHandle {
458 shutdown: watch::Sender<bool>,
459}
460
461impl C2ShutdownHandle {
462 pub fn shutdown(&self) { let _ = self.shutdown.send(true); }
463}
464
465pub struct C2Running {
466 api_addr: SocketAddr,
467 shutdown: C2ShutdownHandle,
468 task: Option<JoinHandle<Result<(), C2Error>>>,
469}
470
471impl C2Running {
472 pub async fn start(config: C2Config) -> Result<Self, C2Error> {
473 prepare_default_control_parent(&config.control_endpoint)?;
474 let listener = TcpListener::bind(config.api_listen).await?;
475 let api_addr = listener.local_addr()?;
476 let (shutdown_tx, shutdown_rx) = watch::channel(false);
477 let shutdown = C2ShutdownHandle { shutdown: shutdown_tx };
478 let task = tokio::spawn(run_bound(config, listener, shutdown_rx));
479 Ok(Self { api_addr, shutdown, task: Some(task) })
480 }
481
482 pub fn api_addr(&self) -> SocketAddr { self.api_addr }
483 pub fn shutdown_handle(&self) -> C2ShutdownHandle { self.shutdown.clone() }
484 pub async fn wait(mut self) -> Result<(), C2Error> {
485 self.task.take().expect("C2 task is present").await?
486 }
487}
488
489#[cfg(windows)]
490fn prepare_default_control_parent(_endpoint: &str) -> io::Result<()> { Ok(()) }
491
492#[cfg(unix)]
493fn prepare_default_control_parent(endpoint: &str) -> io::Result<()> {
494 let default = default_c2_control_endpoint()
495 .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error.to_string()))?;
496 if endpoint != default { return Ok(()); }
497 let parent = Path::new(endpoint).parent().ok_or_else(|| {
498 io::Error::new(io::ErrorKind::InvalidInput, "default C2 endpoint has no parent")
499 })?;
500 std::fs::create_dir_all(parent)?;
501 std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700))
502}
503
504impl Drop for C2Running {
505 fn drop(&mut self) {
506 self.shutdown.shutdown();
507 if let Some(task) = self.task.take() { task.abort(); }
508 }
509}
510
511async fn run_bound(config: C2Config, listener: TcpListener, mut shutdown: watch::Receiver<bool>) -> Result<(), C2Error> {
512 let now = unix_ms();
513 let nodes = config.nodes.iter().map(|node| {
514 (node.node_id.clone(), initial_observed_node(node))
515 }).collect();
516 let initial = Arc::new(StatusResponse { api_version: C2_API_VERSION, ready: false, observed_at_unix_ms: now, nodes });
517 let (status_tx, status_rx) = watch::channel(initial);
518 let (ingress_tx, ingress_rx) = mpsc::channel(config.nodes.len().saturating_mul(2).max(2));
519 let mut tasks = JoinSet::new();
520 let hub = OperatorHub::new();
521 let mut relay_senders = BTreeMap::new();
522 let mut relay_receivers = Vec::new();
523 let mut call_home_routes: BTreeMap<NodeId, mpsc::Sender<TcpStream>> = BTreeMap::new();
529 for node in config.nodes.clone() {
530 let (commands_tx, commands_rx) = mpsc::channel(8);
531 let (releases_tx, releases_rx) = mpsc::channel(1);
532 let (force_tx, force_rx) = watch::channel(0_u64);
533 relay_senders.insert(node.node_id.clone(), RelayEndpoint {
534 commands: commands_tx,
535 releases: releases_tx,
536 force_disconnect: force_tx,
537 });
538 let call_home_rx = if node.route == C2NodeRoute::CallHome {
539 let (call_home_tx, call_home_rx) = mpsc::channel(1);
540 call_home_routes.insert(node.node_id.clone(), call_home_tx);
541 Some(call_home_rx)
542 } else {
543 None
544 };
545 relay_receivers.push((node, commands_rx, releases_rx, force_rx, call_home_rx));
546 }
547 let relay_senders = Arc::new(relay_senders);
548 if let Some(node_listen) = config.node_listen {
549 tasks.spawn(accept_call_home(node_listen, call_home_routes, shutdown.clone()));
550 }
551 tasks.spawn(inventory_owner(config.nodes.len(), config.timings.fresh_for, ingress_rx, status_tx, shutdown.clone()));
552 for (node, commands, releases, force_disconnect, call_home) in relay_receivers {
553 tasks.spawn(node_relay_worker(node, config.timings, commands, releases, force_disconnect, call_home, ingress_tx.clone(), status_rx.clone(), hub.clone(), shutdown.clone()));
554 }
555 drop(ingress_tx);
556 tasks.spawn(http_server(listener, config.api_token.clone(), config.timings.http_io_deadline, status_rx.clone(), shutdown.clone()));
557 let slots = control::ControlSlots::new();
558 let tunnel_listen = config.control_tunnel_listen;
559 let tunnel_token = config.api_token.clone();
560 tasks.spawn(control::run(
561 config.control_endpoint,
562 config.api_token,
563 Arc::clone(&relay_senders),
564 status_rx.clone(),
565 hub.clone(),
566 shutdown.clone(),
567 slots.clone(),
568 ));
569 if let Some(listen) = tunnel_listen {
570 tasks.spawn(control::run_tcp(
571 listen,
572 tunnel_token,
573 relay_senders,
574 status_rx,
575 hub,
576 shutdown.clone(),
577 slots,
578 ));
579 }
580 loop {
581 tokio::select! {
582 changed = shutdown.changed() => {
583 if changed.is_err() || *shutdown.borrow() { break; }
584 }
585 result = tasks.join_next() => {
586 match result {
587 Some(Ok(Ok(()))) if *shutdown.borrow() => break,
588 Some(Ok(Ok(()))) => return Err(C2Error::Api(io::Error::new(io::ErrorKind::Other, "C2 task exited unexpectedly"))),
589 Some(Ok(Err(error))) => return Err(C2Error::Api(error)),
590 Some(Err(error)) => return Err(C2Error::Task(error)),
591 None => break,
592 }
593 }
594 }
595 }
596 tasks.shutdown().await;
597 Ok(())
598}
599
600fn initial_observed_node(node: &C2NodeConfig) -> ObservedNode {
601 ObservedNode {
602 endpoint: node.endpoint.clone(), transport_label: node.transport_label().to_owned(),
603 transport: NodeTransportState::Offline, freshness: NodeFreshness::Unavailable,
604 cursor: None, inventory: None, last_attempt_unix_ms: None, last_success_unix_ms: None,
605 consecutive_failures: 0, last_error: None, gaps: Vec::new(), gaps_truncated: 0,
606 }
607}
608
609#[cfg(windows)]
610fn local_transport_label() -> &'static str { "windows-named-pipe" }
611
612#[cfg(unix)]
613fn local_transport_label() -> &'static str { "unix-domain-socket" }
614
615#[derive(Clone, Default)]
616struct ProviderContractManifest {
617 provider_contracts: Vec<ProviderContractSupport>,
618 provider_adapter_contracts: Vec<ProviderAdapterContractSupport>,
619}
620
621impl ProviderContractManifest {
622 fn from_compatibility(compatibility: Option<&NegotiatedNodeCompatibility>) -> Self {
623 let Some(compatibility) = compatibility.filter(|compatibility| {
624 compatibility.capabilities.iter().any(|capability| {
625 capability.as_str() == C2_PROVIDER_CONTRACT_MANIFEST_CAPABILITY
626 })
627 }) else {
628 return Self::default();
629 };
630 Self {
631 provider_contracts: compatibility.provider_contracts.clone(),
632 provider_adapter_contracts: compatibility.provider_adapter_contracts.clone(),
633 }
634 }
635}
636
637enum AttemptResult {
638 Connected {
639 cursor: NodeCursor,
640 snapshot: NodeSnapshot,
641 gaps: Vec<GapKind>,
642 provider_contract_manifest: ProviderContractManifest,
643 },
644 Success { cursor: NodeCursor, snapshot: NodeSnapshot, gaps: Vec<GapKind> },
645 Cursor {
646 cursor: NodeCursor,
647 gaps: Vec<GapKind>,
648 managed_worktree_events: Vec<NodeEvent>,
649 },
650 Failure { error: SanitizedError, hard: bool },
651}
652
653struct Attempt { node_id: NodeId, at_unix_ms: u64, result: AttemptResult }
654
655async fn accept_call_home(
671 listen: SocketAddr,
672 routes: BTreeMap<NodeId, mpsc::Sender<TcpStream>>,
673 mut shutdown: watch::Receiver<bool>,
674) -> io::Result<()> {
675 let listener = TcpListener::bind(listen).await?;
676 tracing::info!(%listen, waiting_nodes = routes.len(), "call-home listener bound");
677 loop {
678 if *shutdown.borrow() {
679 return Ok(());
680 }
681 let (stream, peer) = tokio::select! {
682 accepted = listener.accept() => accepted?,
683 changed = shutdown.changed() => {
684 if changed.is_err() || *shutdown.borrow() { return Ok(()); }
685 continue;
686 }
687 };
688 let routes = routes.clone();
689 tokio::spawn(async move {
693 let mut stream = stream;
694 let _ = stream.set_nodelay(true);
695 let node_id = match read_call_home_announce(&mut stream).await {
696 Ok(node_id) => node_id,
697 Err(error) => {
698 tracing::warn!(%peer, cause = %error, "call-home connection rejected");
699 return;
700 }
701 };
702 let Some(sender) = routes.get(&node_id) else {
703 tracing::warn!(
704 %peer,
705 %node_id,
706 "call-home connection named a node this relay does not wait for",
707 );
708 return;
709 };
710 match sender.try_send(stream) {
711 Ok(()) => tracing::info!(%peer, %node_id, "node called home"),
712 Err(mpsc::error::TrySendError::Full(_)) => tracing::warn!(
713 %peer,
714 %node_id,
715 "node called home while its previous connection is still being taken up",
716 ),
717 Err(mpsc::error::TrySendError::Closed(_)) => tracing::warn!(
718 %peer,
719 %node_id,
720 "node called home but its relay worker is gone",
721 ),
722 }
723 });
724 }
725}
726
727async fn await_call_home(
739 receiver: Option<&mut mpsc::Receiver<TcpStream>>,
740 shutdown: &mut watch::Receiver<bool>,
741) -> Option<TcpStream> {
742 let receiver = receiver?;
743 loop {
744 if *shutdown.borrow() {
745 return None;
746 }
747 tokio::select! {
748 stream = receiver.recv() => return stream,
749 changed = shutdown.changed() => {
750 if changed.is_err() || *shutdown.borrow() { return None; }
751 }
752 }
753 }
754}
755
756async fn node_relay_worker(
757 node: C2NodeConfig,
758 timings: C2Timings,
759 mut commands: mpsc::Receiver<RelayCommand>,
760 mut releases: mpsc::Receiver<oneshot::Sender<()>>,
761 mut force_disconnect: watch::Receiver<u64>,
762 mut call_home: Option<mpsc::Receiver<TcpStream>>,
763 ingress: mpsc::Sender<Attempt>,
764 status: watch::Receiver<Arc<StatusResponse>>,
765 hub: OperatorHub,
766 mut shutdown: watch::Receiver<bool>,
767) -> io::Result<()> {
768 let mut failures = 0_usize;
769 loop {
770 if *shutdown.borrow() { return Ok(()); }
771 let previous = status.borrow().nodes.get(&node.node_id).and_then(|item| item.cursor);
772 let adopted = if node.route == C2NodeRoute::CallHome {
777 match await_call_home(call_home.as_mut(), &mut shutdown).await {
778 Some(stream) => Some(stream),
779 None => return Ok(()),
780 }
781 } else {
782 None
783 };
784 let connected = timeout(timings.attempt_deadline, connect_operator(&node, adopted)).await;
785 let mut client = match connected {
786 Ok(Ok(client)) => client,
787 Ok(Err(error)) => {
788 failures = failures.saturating_add(1);
789 let (error, hard) = sanitize_node_error(&error);
790 tracing::warn!(
791 node_id = %node.node_id,
792 cause = %error.message,
793 category = ?error.category,
794 hard,
795 "node connection attempt failed",
796 );
797 ingress_attempt(&ingress, &node.node_id, AttemptResult::Failure { error, hard }).await?;
798 reject_disconnected_commands(&mut commands, previous);
799 acknowledge_disconnected_releases(&mut releases);
800 relay_backoff(&mut shutdown, timings, failures, hard).await?;
801 continue;
802 }
803 Err(_) => {
804 failures = failures.saturating_add(1);
805 tracing::warn!(
806 node_id = %node.node_id,
807 cause = "node connection deadline exceeded",
808 "node connection attempt failed",
809 );
810 ingress_attempt(&ingress, &node.node_id, AttemptResult::Failure {
811 error: SanitizedError { category: C2ErrorCategory::Timeout, message: "node connection deadline exceeded".to_owned() },
812 hard: false,
813 }).await?;
814 reject_disconnected_commands(&mut commands, previous);
815 acknowledge_disconnected_releases(&mut releases);
816 relay_backoff(&mut shutdown, timings, failures, false).await?;
817 continue;
818 }
819 };
820 failures = 0;
821 let hello = client.hello().clone();
822 let provider_contract_manifest =
823 ProviderContractManifest::from_compatibility(hello.compatibility.as_ref());
824 let incarnation_id = hello.incarnation_id;
825 let connection_id = hello.connection_id;
826 tracing::info!(
827 node_id = %node.node_id,
828 connection_id,
829 incarnation_id = ?incarnation_id,
830 "node attached to relay",
831 );
832 let mut controller_owned = hello.controller.as_ref().is_some_and(|controller| controller.connection_id == connection_id);
833 let mut cursor = NodeCursor { incarnation_id, sequence: hello.event_sequence };
834 let mut snapshot = hello.snapshot;
835 let mut gaps = Vec::new();
836 let mut did_resync = false;
837 if let Some(previous) = previous {
838 if previous.incarnation_id != incarnation_id {
839 gaps.push(GapKind::IncarnationChanged);
840 } else if cursor.sequence < previous.sequence {
841 gaps.push(GapKind::CursorRegression);
842 } else if cursor.sequence > previous.sequence {
843 match bounded_node_request(&mut client, NodeRequest::Resync { after_sequence: previous.sequence }).await {
844 Ok(NodeResponse::Resync {
845 event_sequence,
846 oldest_available_sequence,
847 snapshot: resync_snapshot,
848 events,
849 }) => {
850 let resync_gaps = validate_resync(
851 previous.sequence,
852 cursor.sequence,
853 event_sequence,
854 oldest_available_sequence,
855 &events,
856 );
857 if resync_gaps.is_empty() {
858 publish_recovered_events(&node.node_id, incarnation_id, &events, &hub);
859 } else {
860 hub.publish(RoutedNodeEvent {
861 node_id: node.node_id.clone(),
862 cursor: NodeCursor { incarnation_id, sequence: event_sequence },
863 event: C2NodeEvent::ResyncRequired {
864 oldest_available_sequence,
865 },
866 });
867 }
868 gaps.extend(resync_gaps);
869 cursor.sequence = event_sequence;
870 snapshot = resync_snapshot;
871 did_resync = true;
872 }
873 Ok(_) => gaps.push(GapKind::NonContiguousEvents),
874 Err(error) => {
875 ingress_attempt(&ingress, &node.node_id, relay_failure_attempt(&error)).await?;
876 reject_disconnected_commands(&mut commands, Some(cursor));
877 continue;
878 }
879 }
880 }
881 }
882 ingress_attempt(&ingress, &node.node_id, AttemptResult::Connected {
883 cursor,
884 snapshot,
885 gaps,
886 provider_contract_manifest,
887 }).await?;
888 if let Err(error) = drain_pending_events(&mut client, &node.node_id, &mut cursor, &hub, &ingress, did_resync, None).await {
889 ingress_attempt(&ingress, &node.node_id, relay_failure_attempt(&error)).await?;
890 reject_disconnected_commands(&mut commands, Some(cursor));
891 continue;
892 }
893
894 let cadence = timings.poll_interval.max(Duration::from_millis(1)).min(Duration::from_millis(250));
895 let mut snapshot_tick = tokio::time::interval(cadence);
896 snapshot_tick.reset();
897 let mut lease_tick = tokio::time::interval(Duration::from_secs(30));
898 lease_tick.reset();
899 let disconnect_error = loop {
900 tokio::select! {
901 command = commands.recv() => {
902 let Some(command) = command else { return Ok(()); };
903 tokio::select! {
904 result = handle_relay_command(
905 &mut client, command, &node.node_id, incarnation_id, connection_id,
906 &mut controller_owned, &hub, &mut cursor, &ingress,
907 ) => match result {
908 Ok(()) => {}
909 Err(error) => break error,
910 },
911 changed = force_disconnect.changed() => {
912 let _ = changed;
913 break NodeClientError::Io(io::Error::new(io::ErrorKind::ConnectionAborted, "node relay cleanup forced reconnect"));
914 }
915 }
916 }
917 release = releases.recv() => {
918 let Some(reply) = release else { return Ok(()); };
919 let result = release_controller(&mut client, &mut controller_owned).await;
920 let _ = reply.send(());
921 if let Err(error) = result { break error; }
922 }
923 changed = force_disconnect.changed() => {
924 let _ = changed;
925 break NodeClientError::Io(io::Error::new(io::ErrorKind::ConnectionAborted, "node relay cleanup forced reconnect"));
926 }
927 frame = client.recv() => {
928 match frame {
929 Ok(ServerFrame::Event(envelope)) => {
930 if let Err(error) = handle_live_node_event(
931 &mut client,
932 &node.node_id,
933 envelope,
934 &mut cursor,
935 &hub,
936 &ingress,
937 ).await {
938 break error;
939 }
940 }
941 Ok(ServerFrame::Reply(_)
942 | ServerFrame::Challenge(_)
943 | ServerFrame::Hello(_)) => {
944 break NodeClientError::Protocol(
945 "node sent an unexpected idle frame".to_owned(),
946 );
947 }
948 Err(error) => break error,
949 }
950 }
951 _ = snapshot_tick.tick() => {
952 match bounded_node_request(&mut client, NodeRequest::Snapshot).await {
953 Ok(NodeResponse::Snapshot { event_sequence, snapshot, .. }) => {
954 if let Err(error) = drain_pending_events(&mut client, &node.node_id, &mut cursor, &hub, &ingress, false, None).await { break error; }
955 if event_sequence >= cursor.sequence {
956 cursor.sequence = event_sequence;
957 ingress_attempt(&ingress, &node.node_id, AttemptResult::Success { cursor, snapshot, gaps: Vec::new() }).await?;
958 }
959 }
960 Ok(_) => break NodeClientError::Protocol("snapshot returned a different response".to_owned()),
961 Err(error) => break error,
962 }
963 }
964 _ = lease_tick.tick() => {
965 if controller_owned {
966 if hub.has_active_operator() {
967 match acquire_controller(&mut client, connection_id).await {
968 Ok(owned) => controller_owned = owned,
969 Err(error) => break error,
970 }
971 } else if let Err(error) = release_controller(&mut client, &mut controller_owned).await {
972 break error;
973 }
974 if let Err(error) = drain_pending_events(&mut client, &node.node_id, &mut cursor, &hub, &ingress, false, None).await { break error; }
975 }
976 }
977 changed = shutdown.changed() => {
978 if changed.is_err() || *shutdown.borrow() {
979 let _ = release_controller(&mut client, &mut controller_owned).await;
980 return Ok(());
981 }
982 }
983 }
984 };
985 let (error, hard) = sanitize_node_error(&disconnect_error);
986 tracing::warn!(
987 node_id = %node.node_id,
988 connection_id,
989 cause = %error.message,
990 category = ?error.category,
991 hard,
992 "node dropped from relay",
993 );
994 ingress_attempt(&ingress, &node.node_id, AttemptResult::Failure { error, hard }).await?;
995 reject_disconnected_commands(&mut commands, Some(cursor));
996 failures = failures.saturating_add(1);
997 relay_backoff(&mut shutdown, timings, failures, hard).await?;
998 }
999}
1000
1001async fn connect_operator(
1012 node: &C2NodeConfig,
1013 adopted: Option<TcpStream>,
1014) -> Result<LocalNodeClient, NodeClientError> {
1015 match node.route {
1016 C2NodeRoute::Local => {
1017 LocalNodeClient::connect(&node.endpoint, &node.node_id, ClientRole::Operator, &node.token).await
1018 }
1019 C2NodeRoute::SshForwardedLoopback(endpoint) => {
1020 LocalNodeClient::connect_loopback(endpoint, &node.node_id, ClientRole::Operator, &node.token).await
1021 }
1022 C2NodeRoute::CallHome => {
1023 let stream = adopted.ok_or_else(|| {
1029 NodeClientError::Protocol(
1030 "call-home node reached the connect step with no adopted stream".to_owned(),
1031 )
1032 })?;
1033 LocalNodeClient::adopt(stream, &node.node_id, ClientRole::Operator, &node.token).await
1034 }
1035 }
1036}
1037
1038async fn bounded_node_request(
1039 client: &mut LocalNodeClient,
1040 request: NodeRequest,
1041) -> Result<NodeResponse, NodeClientError> {
1042 bounded_node_request_with_deadline(client, request, None).await
1043}
1044
1045async fn bounded_node_request_with_deadline(
1046 client: &mut LocalNodeClient,
1047 request: NodeRequest,
1048 relay_deadline: Option<Instant>,
1049) -> Result<NodeResponse, NodeClientError> {
1050 let deadline = request_budget(&request, relay_deadline, Instant::now());
1051 if deadline.is_zero() {
1052 return Err(NodeClientError::Frame(FrameError::PrefixTimedOut));
1053 }
1054 match timeout(deadline, client.request(request)).await {
1055 Ok(result) => result,
1056 Err(_) => Err(NodeClientError::Frame(FrameError::PrefixTimedOut)),
1057 }
1058}
1059
1060fn relay_request_deadline(request: &NodeRequest, now: Instant) -> Option<Instant> {
1061 matches!(request, NodeRequest::ResumeSessionRecord { .. })
1062 .then(|| now + node_request_deadline(request))
1063}
1064
1065fn request_budget(request: &NodeRequest, relay_deadline: Option<Instant>, now: Instant) -> Duration {
1066 relay_deadline
1067 .map(|deadline| deadline.saturating_duration_since(now))
1068 .unwrap_or_else(|| node_request_deadline(request))
1069}
1070
1071const WORKSPACE_INSPECTION_RELAY_DEADLINE: Duration = Duration::from_secs(15);
1077
1078fn node_request_deadline(request: &NodeRequest) -> Duration {
1079 match request {
1080 NodeRequest::Snapshot
1081 | NodeRequest::Resync { .. }
1082 | NodeRequest::ArmHarnessMcpReservation { .. }
1083 | NodeRequest::ActivateHarnessMcpReservation { .. }
1084 | NodeRequest::AbortHarnessMcpReservation { .. }
1085 | NodeRequest::PutHarnessMcpReplyChunk { .. }
1086 | NodeRequest::RejectHarnessMcpCall { .. }
1087 | NodeRequest::BrowseHostDirectories { .. }
1088 | NodeRequest::ReadWorkspaceFile { .. }
1089 | NodeRequest::WriteWorkspaceFile { .. }
1090 | NodeRequest::ReadGitHistory { .. }
1091 | NodeRequest::ReadGitDiff { .. }
1092 | NodeRequest::AcquireController { .. }
1093 | NodeRequest::ReleaseController
1094 | NodeRequest::RenameSessionRecord { .. }
1095 | NodeRequest::SetSessionTask { .. }
1096 | NodeRequest::ForgetSessionRecord { .. } => Duration::from_secs(5),
1097 NodeRequest::InspectWorkspace { .. } => WORKSPACE_INSPECTION_RELAY_DEADLINE,
1115 NodeRequest::CreateWorkspaceFile { .. }
1116 | NodeRequest::CreateWorkspaceDirectory { .. } => {
1117 WORKSPACE_ENTRY_CREATE_RELAY_DEADLINE
1118 }
1119 NodeRequest::CatalogNativeSessions { .. }
1120 | NodeRequest::PageNativeSessions { .. }
1121 | NodeRequest::PreviewNativeSession { .. }
1122 | NodeRequest::IndexNativeSession { .. }
1123 | NodeRequest::PreviewSessionRecord { .. } => NATIVE_SESSION_REQUEST_DEADLINE,
1124 NodeRequest::CreateStandaloneWorkspace { .. }
1125 | NodeRequest::CreateWorktree { .. }
1126 | NodeRequest::RemoveWorktree { .. }
1127 | NodeRequest::CleanupManagedWorktree { .. } => Duration::from_secs(240),
1128 NodeRequest::Spawn { .. }
1129 | NodeRequest::Resume { .. }
1130 | NodeRequest::Stop { .. } => Duration::from_secs(15),
1131 NodeRequest::SpawnSpec { spec } =>
1132 Duration::from_millis(spec.deadline_ms.get()) + NODE_REQUEST_IO_HEADROOM,
1133 NodeRequest::SpawnSpecWithHarnessMcp { deadline_unix_ms, .. } =>
1134 Duration::from_millis(deadline_unix_ms.saturating_sub(unix_ms()))
1135 + NODE_REQUEST_IO_HEADROOM,
1136 NodeRequest::SpawnManagedWorktree { request } =>
1137 Duration::from_millis(request.spawn_spec.deadline_ms.get())
1138 + NODE_REQUEST_IO_HEADROOM,
1139 NodeRequest::ResumeSessionRecord { .. } =>
1140 MANAGED_RESUME_SETTLE_DEADLINE + NODE_REQUEST_IO_HEADROOM,
1141 _ => Duration::from_secs(10),
1142 }
1143}
1144
1145fn validate_resync(
1146 previous: u64,
1147 hello: u64,
1148 current: u64,
1149 oldest_available_sequence: u64,
1150 events: &[NodeEventEnvelope],
1151) -> Vec<GapKind> {
1152 let mut gaps = validate_events(
1153 previous,
1154 current,
1155 oldest_available_sequence,
1156 events,
1157 );
1158 if current < hello && !gaps.contains(&GapKind::CursorRegression) {
1159 gaps.push(GapKind::CursorRegression);
1160 }
1161 gaps
1162}
1163
1164async fn ingress_attempt(
1165 ingress: &mpsc::Sender<Attempt>,
1166 node_id: &NodeId,
1167 result: AttemptResult,
1168) -> io::Result<()> {
1169 ingress.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result })
1170 .await.map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "inventory owner closed"))
1171}
1172
1173async fn relay_backoff(
1174 shutdown: &mut watch::Receiver<bool>,
1175 timings: C2Timings,
1176 failures: usize,
1177 hard: bool,
1178) -> io::Result<()> {
1179 let delay = if hard || failures >= timings.transient_backoffs.len() {
1180 timings.parked_backoff
1181 } else {
1182 timings.transient_backoffs[failures.saturating_sub(1)]
1183 };
1184 tokio::select! {
1185 _ = sleep(delay) => Ok(()),
1186 _ = shutdown.changed() => {
1187 Ok(())
1188 }
1189 }
1190}
1191
1192fn relay_failure_attempt(error: &NodeClientError) -> AttemptResult {
1193 let (error, hard) = sanitize_node_error(error);
1194 AttemptResult::Failure { error, hard }
1195}
1196
1197async fn handle_relay_command(
1198 client: &mut LocalNodeClient,
1199 command: RelayCommand,
1200 node_id: &NodeId,
1201 incarnation_id: NodeIncarnationId,
1202 connection_id: u64,
1203 controller_owned: &mut bool,
1204 hub: &OperatorHub,
1205 cursor: &mut NodeCursor,
1206 ingress: &mpsc::Sender<Attempt>,
1207) -> Result<(), NodeClientError> {
1208 match command {
1209 RelayCommand::Request { operator_connection_id, expected_incarnation_id, request, reply } => {
1210 let relay_deadline = relay_request_deadline(&request, Instant::now());
1211 let expected_spawn = expected_spawn_request(&request);
1212 let expected_provider_session_index = matches!(
1213 &request,
1214 NodeRequest::IndexProviderSession { .. }
1215 ).then(|| request.clone());
1216 let expected_native_session = matches!(
1217 &request,
1218 NodeRequest::CatalogNativeSessions { .. }
1219 | NodeRequest::PageNativeSessions { .. }
1220 | NodeRequest::PreviewNativeSession { .. }
1221 | NodeRequest::IndexNativeSession { .. }
1222 ).then(|| request.clone());
1223 let expected_workspace_content = matches!(
1224 &request,
1225 NodeRequest::ReadWorkspaceFile { .. }
1226 | NodeRequest::WriteWorkspaceFile { .. }
1227 | NodeRequest::CreateWorkspaceFile { .. }
1228 | NodeRequest::CreateWorkspaceDirectory { .. }
1229 | NodeRequest::ReadGitHistory { .. }
1230 | NodeRequest::ReadGitDiff { .. }
1231 ).then(|| request.clone());
1232 let expected_session_task = matches!(
1233 &request,
1234 NodeRequest::SetSessionTask { .. }
1235 ).then(|| request.clone());
1236 let expected_harness_mcp = (request.required_capability()
1237 == Some(gate4agent_node_protocol::NODE_HARNESS_MCP_READ_PROXY_CAPABILITY))
1238 .then(|| request.clone());
1239 if !hub.is_active(operator_connection_id) {
1240 let _ = reply.send(Err(relay_failure(C2RelayFailureCode::ClientLagged, "C2 operator connection is no longer active", Some(incarnation_id))));
1241 return Ok(());
1242 }
1243 if expected_incarnation_id != incarnation_id {
1244 let _ = reply.send(Err(relay_failure(C2RelayFailureCode::StaleNodeIncarnation, "node incarnation changed", Some(incarnation_id))));
1245 return Ok(());
1246 }
1247 if matches!(&request, NodeRequest::IndexNativeSession { selection, .. }
1248 if selection.route.scope
1249 != gate4agent_node_protocol::NativeSessionCatalogScope::Workspace)
1250 {
1251 let _ = reply.send(Err(relay_failure(
1252 C2RelayFailureCode::RequestForbidden,
1253 "external native sessions must be registered as workspaces before indexing",
1254 Some(incarnation_id),
1255 )));
1256 return Ok(());
1257 }
1258 if !is_read_only_request(&request) && !*controller_owned {
1259 match acquire_controller_with_deadline(client, connection_id, relay_deadline).await {
1260 Ok(owned) if owned => *controller_owned = true,
1261 Ok(_) => {
1262 let _ = reply.send(Err(relay_failure(C2RelayFailureCode::RelayBusy, "node controller lease is unavailable", Some(incarnation_id))));
1263 return Ok(());
1264 }
1265 Err(error) if relay_node_failure(&error).is_some() => {
1266 let failure = relay_node_failure(&error)
1267 .expect("guarded node request failure");
1268 let _ = reply.send(Ok(RoutedNodeResponse {
1269 node_id: node_id.clone(),
1270 incarnation_id,
1271 response: Err(failure),
1272 }));
1273 return Ok(());
1274 }
1275 Err(error) => {
1276 let _ = reply.send(Err(relay_failure(C2RelayFailureCode::NodeOffline, "node relay disconnected", Some(incarnation_id))));
1277 return Err(error);
1278 }
1279 }
1280 }
1281 let response = match bounded_node_request_with_deadline(client, request, relay_deadline).await {
1282 Ok(response) => Ok(response),
1283 Err(NodeClientError::Node(failure)) => Err(failure),
1284 Err(error @ NodeClientError::UnsupportedCapability(_)) => {
1285 let failure = relay_node_failure(&error)
1286 .expect("unsupported capability is a routed node failure");
1287 let _ = reply.send(Ok(RoutedNodeResponse {
1288 node_id: node_id.clone(),
1289 incarnation_id,
1290 response: Err(failure),
1291 }));
1292 return Ok(());
1293 }
1294 Err(error) => {
1295 let _ = reply.send(Err(relay_failure(C2RelayFailureCode::NodeOffline, "node relay disconnected", Some(incarnation_id))));
1296 return Err(error);
1297 }
1298 };
1299 if let Err(message) = validate_spawn_spec_response(
1300 expected_spawn.as_ref(),
1301 &response,
1302 incarnation_id,
1303 ) {
1304 let _ = reply.send(Err(relay_failure(
1305 C2RelayFailureCode::NodeOffline,
1306 "node relay returned an invalid spawn receipt",
1307 Some(incarnation_id),
1308 )));
1309 return Err(NodeClientError::Protocol(message.to_owned()));
1310 }
1311 if let Err(message) = validate_provider_session_index_response(
1312 expected_provider_session_index.as_ref(),
1313 &response,
1314 ) {
1315 let _ = reply.send(Err(relay_failure(
1316 C2RelayFailureCode::NodeOffline,
1317 "node relay returned an invalid provider session index response",
1318 Some(incarnation_id),
1319 )));
1320 return Err(NodeClientError::Protocol(message.to_owned()));
1321 }
1322 if let Err(message) = validate_native_session_response(
1323 expected_native_session.as_ref(),
1324 &response,
1325 ) {
1326 let _ = reply.send(Err(relay_failure(
1327 C2RelayFailureCode::NodeOffline,
1328 "node relay returned an invalid native session response",
1329 Some(incarnation_id),
1330 )));
1331 return Err(NodeClientError::Protocol(message.to_owned()));
1332 }
1333 if let Err(message) = validate_workspace_content_response(
1334 expected_workspace_content.as_ref(),
1335 &response,
1336 ) {
1337 let _ = reply.send(Err(relay_failure(
1338 C2RelayFailureCode::NodeOffline,
1339 "node relay returned an invalid workspace content response",
1340 Some(incarnation_id),
1341 )));
1342 return Err(NodeClientError::Protocol(message.to_owned()));
1343 }
1344 if let Err(message) = validate_session_task_response(
1345 expected_session_task.as_ref(),
1346 &response,
1347 ) {
1348 let _ = reply.send(Err(relay_failure(
1349 C2RelayFailureCode::NodeOffline,
1350 "node relay returned an invalid session task response",
1351 Some(incarnation_id),
1352 )));
1353 return Err(NodeClientError::Protocol(message.to_owned()));
1354 }
1355 if let Err(message) = validate_harness_mcp_response(
1356 expected_harness_mcp.as_ref(),
1357 &response,
1358 ) {
1359 let _ = reply.send(Err(relay_failure(
1360 C2RelayFailureCode::NodeOffline,
1361 "node relay returned an invalid harness MCP response",
1362 Some(incarnation_id),
1363 )));
1364 return Err(NodeClientError::Protocol(message.to_owned()));
1365 }
1366 drain_pending_events(client, node_id, cursor, hub, ingress, false, relay_deadline).await?;
1367 update_inventory_from_response(node_id, cursor, &response, ingress).await
1368 .map_err(NodeClientError::Io)?;
1369 let response = response
1370 .map(|response| C2NodeResponse::from_node_response_with_control_detail(&response))
1371 .map_err(|failure| C2NodeFailure::from(&failure));
1372 let _ = reply.send(Ok(RoutedNodeResponse { node_id: node_id.clone(), incarnation_id, response }));
1373 }
1374 }
1375 Ok(())
1376}
1377
1378fn validate_session_task_response(
1379 expected: Option<&NodeRequest>,
1380 response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1381) -> Result<(), &'static str> {
1382 match (expected, response) {
1383 (Some(_), Err(_)) | (None, Err(_)) => Ok(()),
1384 (
1385 Some(NodeRequest::SetSessionTask {
1386 record_id,
1387 expected_revision,
1388 target,
1389 }),
1390 Ok(NodeResponse::SessionRecordUpdated { record }),
1391 ) if session_task_record_matches(record, record_id, *expected_revision, target) => Ok(()),
1392 (Some(NodeRequest::SetSessionTask { .. }), Ok(_)) => {
1393 Err("session task response does not match the routed request")
1394 }
1395 (None, Ok(_)) => Ok(()),
1396 (Some(_), Ok(_)) => Ok(()),
1397 }
1398}
1399
1400fn validate_harness_mcp_response(
1401 expected: Option<&NodeRequest>,
1402 response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1403) -> Result<(), &'static str> {
1404 use NodeRequest as Request;
1405 use NodeResponse as Response;
1406 let valid = match (expected, response) {
1407 (Some(Request::ArmHarnessMcpReservation { reservation_id, activation_digest, expires_at_unix_ms, .. }),
1408 Ok(Response::Armed { reservation_id: echoed_id, activation_digest: echoed_digest, expires_at_unix_ms: echoed_expiry })) =>
1409 reservation_id == echoed_id && activation_digest == echoed_digest
1410 && expires_at_unix_ms == echoed_expiry,
1411 (Some(Request::SpawnSpecWithHarnessMcp { reservation_id, activation_digest, .. }),
1412 Ok(Response::Spawned { reservation_id: echoed_id, activation_digest: echoed_digest, receipt })) =>
1413 reservation_id == echoed_id && activation_digest == echoed_digest
1414 && receipt.harness_mcp_proxy.as_ref().is_some_and(|proxy| {
1415 &proxy.reservation_id == reservation_id
1416 && &proxy.activation_digest == activation_digest
1417 }),
1418 (Some(Request::ActivateHarnessMcpReservation { reservation_id, activation_digest, record_id, session }),
1419 Ok(Response::Activated { reservation_id: echoed_id, activation_digest: echoed_digest, record_id: echoed_record, session: echoed_session })) =>
1420 reservation_id == echoed_id && activation_digest == echoed_digest
1421 && record_id == echoed_record && session == echoed_session,
1422 (Some(Request::AbortHarnessMcpReservation { reservation_id, activation_digest }),
1423 Ok(Response::Aborted { reservation_id: echoed_id, activation_digest: echoed_digest })) =>
1424 reservation_id == echoed_id && activation_digest == echoed_digest,
1425 (Some(Request::PutHarnessMcpReplyChunk { reservation_id, activation_digest, record_id, session, call_id, offset, final_chunk, chunk_hex }),
1426 Ok(Response::ReplyChunkAccepted { reservation_id: echoed_id, activation_digest: echoed_digest, record_id: echoed_record, session: echoed_session, call_id: echoed_call, next_offset, completed })) =>
1427 reservation_id == echoed_id && activation_digest == echoed_digest
1428 && record_id == echoed_record && session == echoed_session && call_id == echoed_call
1429 && offset.checked_add(u32::try_from(chunk_hex.raw_len()).unwrap_or(u32::MAX))
1430 == Some(*next_offset) && completed == final_chunk,
1431 (Some(Request::RejectHarnessMcpCall { reservation_id, activation_digest, record_id, session, call_id, .. }),
1432 Ok(Response::CallRejected { reservation_id: echoed_id, activation_digest: echoed_digest, record_id: echoed_record, session: echoed_session, call_id: echoed_call })) =>
1433 reservation_id == echoed_id && activation_digest == echoed_digest
1434 && record_id == echoed_record && session == echoed_session && call_id == echoed_call,
1435 (Some(_), Err(_)) | (None, Err(_)) => true,
1436 (Some(_), Ok(_)) => false,
1437 (None, Ok(response)) => !response.requires_harness_mcp_proxy_capability(),
1438 };
1439 if valid { Ok(()) } else { Err("harness MCP response does not match the routed request") }
1440}
1441
1442fn session_task_record_matches(
1443 record: &gate4agent_node_protocol::ManagedSessionRecord,
1444 record_id: &gate4agent_node_protocol::SessionRecordId,
1445 expected_revision: u64,
1446 target: &gate4agent_node_protocol::SessionTaskTargetV1,
1447) -> bool {
1448 if &record.record_id != record_id { return false; }
1449 let next_revision = expected_revision.checked_add(1);
1450 match target {
1451 gate4agent_node_protocol::SessionTaskTargetV1::New => record.task_binding.as_ref()
1452 .is_some_and(|binding| Some(binding.revision) == next_revision && binding.task_id.is_some()),
1453 gate4agent_node_protocol::SessionTaskTargetV1::Existing { task_id } => record.task_binding.as_ref()
1454 .is_some_and(|binding| (binding.revision == expected_revision || Some(binding.revision) == next_revision)
1455 && binding.task_id.as_ref() == Some(task_id)),
1456 gate4agent_node_protocol::SessionTaskTargetV1::Clear => match &record.task_binding {
1457 None => expected_revision == 0,
1458 Some(binding) => binding.task_id.is_none()
1459 && (binding.revision == expected_revision || Some(binding.revision) == next_revision),
1460 },
1461 }
1462}
1463
1464fn validate_workspace_content_response(
1465 expected: Option<&NodeRequest>,
1466 response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1467) -> Result<(), &'static str> {
1468 match (expected, response) {
1469 (Some(_), Err(_)) | (None, Err(_)) => Ok(()),
1470 (
1471 Some(NodeRequest::ReadWorkspaceFile { workspace_id, path }),
1472 Ok(NodeResponse::WorkspaceFileRead { file }),
1473 ) if &file.workspace_id == workspace_id && &file.path == path => Ok(()),
1474 (
1475 Some(NodeRequest::WriteWorkspaceFile { workspace_id, path, text, .. }),
1476 Ok(NodeResponse::WorkspaceFileWritten { file }),
1477 ) if &file.workspace_id == workspace_id
1478 && &file.path == path
1479 && matches!(
1480 &file.content,
1481 gate4agent_node_protocol::WorkspaceFileContent::Utf8 {
1482 text: written,
1483 byte_len,
1484 } if written == text
1485 && u32::try_from(text.len()).ok() == Some(*byte_len)
1486 ) => Ok(()),
1487 (
1488 Some(NodeRequest::CreateWorkspaceFile { workspace_id, path }),
1489 Ok(NodeResponse::WorkspaceFileCreated { file }),
1490 ) if &file.workspace_id == workspace_id
1491 && &file.path == path
1492 && file.revision.is_some()
1493 && matches!(
1494 &file.content,
1495 gate4agent_node_protocol::WorkspaceFileContent::Utf8 {
1496 text,
1497 byte_len: 0,
1498 } if text.is_empty()
1499 ) => Ok(()),
1500 (
1501 Some(NodeRequest::CreateWorkspaceDirectory { workspace_id, path }),
1502 Ok(NodeResponse::WorkspaceDirectoryCreated {
1503 workspace_id: actual_workspace_id,
1504 entry,
1505 }),
1506 ) if actual_workspace_id == workspace_id
1507 && &entry.relative_path == path
1508 && entry.kind == gate4agent_node_protocol::WorkspaceEntryKind::Directory => Ok(()),
1509 (
1510 Some(NodeRequest::ReadGitHistory { workspace_id, .. }),
1511 Ok(NodeResponse::GitHistoryRead { workspace_id: actual, .. }),
1512 ) if actual == workspace_id => Ok(()),
1513 (
1514 Some(NodeRequest::ReadGitDiff { workspace_id, request }),
1515 Ok(NodeResponse::GitDiffRead { workspace_id: actual, diff }),
1516 ) if actual == workspace_id && diff.mode == request.mode && diff.path == request.path => Ok(()),
1517 (Some(_), Ok(_)) => Err("workspace content response does not match routed request"),
1518 (
1519 None,
1520 Ok(NodeResponse::WorkspaceFileRead { .. }
1521 | NodeResponse::WorkspaceFileWritten { .. }
1522 | NodeResponse::WorkspaceFileCreated { .. }
1523 | NodeResponse::WorkspaceDirectoryCreated { .. }
1524 | NodeResponse::GitHistoryRead { .. }
1525 | NodeResponse::GitDiffRead { .. }),
1526 ) => Err("unexpected workspace content response"),
1527 (None, Ok(_)) => Ok(()),
1528 }
1529}
1530
1531enum ExpectedSpawnRequest {
1532 Spec(SpawnSpec),
1533 ManagedV1(ManagedWorktreeSpawnRequest),
1534 ManagedV2(ManagedWorktreeSpawnRequestV2),
1535}
1536
1537fn expected_spawn_request(request: &NodeRequest) -> Option<ExpectedSpawnRequest> {
1538 match request {
1539 NodeRequest::SpawnSpec { spec } => Some(ExpectedSpawnRequest::Spec(spec.clone())),
1540 NodeRequest::SpawnManagedWorktree { request } => {
1541 Some(ExpectedSpawnRequest::ManagedV1(request.clone()))
1542 }
1543 NodeRequest::SpawnManagedWorktreeV2 { request } => {
1544 Some(ExpectedSpawnRequest::ManagedV2(request.clone()))
1545 }
1546 _ => None,
1547 }
1548}
1549
1550fn validate_provider_session_index_response(
1551 expected: Option<&NodeRequest>,
1552 response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1553) -> Result<(), &'static str> {
1554 match (expected, response) {
1555 (
1556 Some(NodeRequest::IndexProviderSession {
1557 workspace_id,
1558 provider,
1559 identity,
1560 ..
1561 }),
1562 Ok(NodeResponse::ProviderSessionIndexed { record }),
1563 ) if &record.workspace_id == workspace_id
1564 && &record.provider == provider
1565 && record.provider_session.as_ref() == Some(identity) => Ok(()),
1566 (Some(_), Ok(NodeResponse::ProviderSessionIndexed { .. })) => {
1567 Err("provider session index response does not match routed request")
1568 }
1569 (Some(_), Ok(_)) => Err("provider session index request returned a different response"),
1570 (None, Ok(NodeResponse::ProviderSessionIndexed { .. })) => {
1571 Err("unexpected provider session index response for a different node request")
1572 }
1573 (Some(_), Err(_)) | (None, _) => Ok(()),
1574 }
1575}
1576
1577fn validate_native_session_response(
1578 expected: Option<&NodeRequest>,
1579 response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1580) -> Result<(), &'static str> {
1581 match (expected, response) {
1582 (
1583 Some(NodeRequest::CatalogNativeSessions { route, .. }),
1584 Ok(NodeResponse::NativeSessionsCataloged {
1585 route: echoed_route,
1586 ..
1587 }),
1588 ) if echoed_route == route => Ok(()),
1589 (
1590 Some(NodeRequest::PageNativeSessions {
1591 route,
1592 window,
1593 catalog_revision,
1594 ..
1595 }),
1596 Ok(NodeResponse::NativeSessionsPaged {
1597 route: echoed_route,
1598 page,
1599 }),
1600 ) if echoed_route == route
1601 && page.window == *window
1602 && page.revision == *catalog_revision => Ok(()),
1603 (
1604 Some(NodeRequest::PreviewNativeSession { selection, .. }),
1605 Ok(NodeResponse::NativeSessionPreviewed {
1606 selection: echoed_selection,
1607 ..
1608 }),
1609 ) if echoed_selection == selection => Ok(()),
1610 (
1611 Some(NodeRequest::IndexNativeSession { selection, .. }),
1612 Ok(NodeResponse::NativeSessionIndexed {
1613 selection: echoed_selection,
1614 record,
1615 }),
1616 ) if echoed_selection == selection
1617 && selection.route.scope
1618 == gate4agent_node_protocol::NativeSessionCatalogScope::Workspace
1619 && selection.route.workspace_id.as_ref() == Some(&record.workspace_id)
1620 && selection.route.provider == record.provider => Ok(()),
1621 (Some(_), Err(_)) => Ok(()),
1622 (Some(_), Ok(_)) => Err("native session response does not match routed request"),
1623 (
1624 None,
1625 Ok(
1626 NodeResponse::NativeSessionsCataloged { .. }
1627 | NodeResponse::NativeSessionsPaged { .. }
1628 | NodeResponse::NativeSessionPreviewed { .. }
1629 | NodeResponse::NativeSessionIndexed { .. },
1630 ),
1631 ) => Err("unexpected native session response for a different node request"),
1632 (None, _) => Ok(()),
1633 }
1634}
1635
1636fn validate_spawn_spec_response(
1637 expected: Option<&ExpectedSpawnRequest>,
1638 response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1639 relay_incarnation_id: NodeIncarnationId,
1640) -> Result<(), &'static str> {
1641 match (expected, response) {
1642 (Some(ExpectedSpawnRequest::Spec(spec)), Ok(NodeResponse::SpawnSpecAccepted { receipt })) => {
1643 validate_spawn_receipt(spec, receipt, relay_incarnation_id)
1644 }
1645 (
1646 Some(ExpectedSpawnRequest::ManagedV1(request)),
1647 Ok(NodeResponse::ManagedWorktreeSpawnAccepted { receipt }),
1648 ) => validate_managed_spawn_receipt(
1649 &request.spawn_spec,
1650 &request.worktree_profile_id,
1651 receipt,
1652 relay_incarnation_id,
1653 ),
1654 (
1655 Some(ExpectedSpawnRequest::ManagedV2(request)),
1656 Ok(NodeResponse::ManagedWorktreeSpawnAccepted { receipt }),
1657 ) if receipt.lease.profile_revision == request.expected_profile_revision => {
1658 validate_managed_spawn_receipt(
1659 &request.spawn_spec,
1660 &request.worktree_profile_id,
1661 receipt,
1662 relay_incarnation_id,
1663 )
1664 }
1665 (
1666 Some(ExpectedSpawnRequest::ManagedV2(_)),
1667 Ok(NodeResponse::ManagedWorktreeSpawnAccepted { .. }),
1668 ) => Err("managed spawn receipt profile revision does not match routed request"),
1669 (Some(_), Ok(_)) => return Err("spawn spec request returned a different response"),
1670 (Some(_), Err(_)) => return Ok(()),
1671 (None, Ok(NodeResponse::SpawnSpecAccepted { .. }
1672 | NodeResponse::ManagedWorktreeSpawnAccepted { .. })) => {
1673 return Err("unexpected spawn receipt for a different node request");
1674 }
1675 (None, _) => return Ok(()),
1676 }
1677}
1678
1679fn validate_managed_spawn_receipt(
1680 spawn_spec: &SpawnSpec,
1681 worktree_profile_id: &gate4agent_node_protocol::WorktreeProfileId,
1682 receipt: &gate4agent_node_protocol::ManagedWorktreeSpawnReceipt,
1683 relay_incarnation_id: NodeIncarnationId,
1684) -> Result<(), &'static str> {
1685 if receipt.lease.source_workspace_id != spawn_spec.target.workspace_id
1686 || &receipt.lease.profile_id != worktree_profile_id
1687 || receipt.lease.state != ManagedWorktreeLeaseState::InUse
1688 || receipt.lease.cleanup_failure.is_some()
1689 || receipt.lease.active_session_count != 1
1690 || receipt.spawn.target.node_id != spawn_spec.target.node_id
1691 || receipt.spawn.target.workspace_id != spawn_spec.target.workspace_id
1692 || receipt.spawn.target.worktree_id.as_ref() != Some(&receipt.lease.workspace_id)
1693 || receipt.spawn.session.workspace_id != receipt.lease.workspace_id
1694 {
1695 return Err("managed spawn receipt does not match routed request");
1696 }
1697 let mut resolved_spec = spawn_spec.clone();
1698 resolved_spec.target.worktree_id = Some(receipt.lease.workspace_id.clone());
1699 validate_spawn_receipt(&resolved_spec, &receipt.spawn, relay_incarnation_id)
1700}
1701
1702fn validate_spawn_receipt(
1703 spec: &SpawnSpec,
1704 receipt: &ResolvedSpawnReceipt,
1705 relay_incarnation_id: NodeIncarnationId,
1706) -> Result<(), &'static str> {
1707 if receipt.incarnation_id != relay_incarnation_id
1708 || !receipt.context_binding_is_valid()
1709 || receipt.target != spec.target
1710 || receipt.profile_id != spec.profile_id
1711 || receipt.idempotency_key != spec.idempotency_key
1712 || receipt.deadline_ms != spec.deadline_ms
1713 || receipt.required_capabilities != spec.required_capabilities
1714 || receipt.bundle.as_ref().is_some_and(|bundle| {
1715 receipt.bundle_id.as_ref() != Some(&bundle.id)
1716 })
1717 || &receipt.session.workspace_id
1718 != spec
1719 .target
1720 .worktree_id
1721 .as_ref()
1722 .unwrap_or(&spec.target.workspace_id)
1723 {
1724 return Err("spawn receipt does not match routed request");
1725 }
1726 if !required_override_matches(&spec.overrides.provider, &receipt.provider)
1727 || !required_override_matches(&spec.overrides.mode, &receipt.mode)
1728 || !required_override_matches(&spec.overrides.terminal_size, &receipt.terminal_size)
1729 || !optional_override_matches(&spec.overrides.bundle_id, &receipt.bundle_id)
1730 || !optional_override_matches(&spec.overrides.context_id, &receipt.context_id)
1731 || !environment_profile_override_matches(
1732 &spec.overrides.environment_profile_id,
1733 receipt.environment_profile.as_ref(),
1734 )
1735 {
1736 return Err("spawn receipt contradicts explicit overrides");
1737 }
1738 match &spec.overrides.prompt {
1739 SpawnOverride::Inherit => {}
1740 SpawnOverride::Set { value }
1741 if receipt.prompt.present
1742 && receipt.prompt.byte_len == u32::try_from(value.byte_len()).unwrap_or(0) => {}
1743 SpawnOverride::Clear if !receipt.prompt.present && receipt.prompt.byte_len == 0 => {}
1744 SpawnOverride::Set { .. } | SpawnOverride::Clear => {
1745 return Err("spawn receipt contradicts explicit prompt override");
1746 }
1747 }
1748 Ok(())
1749}
1750
1751fn environment_profile_override_matches(
1752 expected: &SpawnOverride<gate4agent_node_protocol::SpawnEnvironmentProfileId>,
1753 actual: Option<&gate4agent_node_protocol::ResolvedEnvironmentProfileReceipt>,
1754) -> bool {
1755 match expected {
1756 SpawnOverride::Inherit => true,
1757 SpawnOverride::Set { value } => {
1758 actual.is_some_and(|receipt| &receipt.profile_id == value)
1759 }
1760 SpawnOverride::Clear => actual.is_none(),
1761 }
1762}
1763
1764fn required_override_matches<T: Eq>(override_value: &SpawnOverride<T>, actual: &T) -> bool {
1765 match override_value {
1766 SpawnOverride::Inherit => true,
1767 SpawnOverride::Set { value } => value == actual,
1768 SpawnOverride::Clear => false,
1769 }
1770}
1771
1772fn optional_override_matches<T: Eq>(
1773 override_value: &SpawnOverride<T>,
1774 actual: &Option<T>,
1775) -> bool {
1776 match override_value {
1777 SpawnOverride::Inherit => true,
1778 SpawnOverride::Set { value } => actual.as_ref() == Some(value),
1779 SpawnOverride::Clear => actual.is_none(),
1780 }
1781}
1782
1783fn relay_node_failure(error: &NodeClientError) -> Option<C2NodeFailure> {
1784 match error {
1785 NodeClientError::Node(failure) => Some(C2NodeFailure::from(failure)),
1786 NodeClientError::UnsupportedCapability(_) => Some(C2NodeFailure {
1787 code: NodeFailureCode::UnsupportedCapability,
1788 message: "required capability unavailable".to_owned(),
1789 }),
1790 NodeClientError::Io(_)
1791 | NodeClientError::Frame(_)
1792 | NodeClientError::Protocol(_)
1793 | NodeClientError::BuildStampMismatch { .. }
1794 | NodeClientError::AuthenticationTimedOut
1795 | NodeClientError::Authentication(_)
1796 | NodeClientError::RequestIdExhausted => None,
1797 }
1798}
1799
1800fn is_read_only_request(request: &NodeRequest) -> bool {
1801 matches!(request,
1802 NodeRequest::Snapshot
1803 | NodeRequest::Resync { .. }
1804 | NodeRequest::BrowseHostDirectories { .. }
1805 | NodeRequest::InspectWorkspace { .. }
1806 | NodeRequest::ReadWorkspaceFile { .. }
1807 | NodeRequest::ReadGitHistory { .. }
1808 | NodeRequest::ReadGitDiff { .. }
1809 | NodeRequest::CatalogNativeSessions { .. }
1810 | NodeRequest::PageNativeSessions { .. }
1811 | NodeRequest::PreviewNativeSession { .. }
1812 | NodeRequest::PreviewSessionRecord { .. }
1813 )
1814}
1815
1816async fn acquire_controller(
1817 client: &mut LocalNodeClient,
1818 connection_id: u64,
1819) -> Result<bool, NodeClientError> {
1820 acquire_controller_with_deadline(client, connection_id, None).await
1821}
1822
1823async fn acquire_controller_with_deadline(
1824 client: &mut LocalNodeClient,
1825 connection_id: u64,
1826 relay_deadline: Option<Instant>,
1827) -> Result<bool, NodeClientError> {
1828 match bounded_node_request_with_deadline(
1829 client,
1830 NodeRequest::AcquireController { lease_ms: gate4agent_node_protocol::MAX_CONTROLLER_LEASE_MS },
1831 relay_deadline,
1832 ).await? {
1833 NodeResponse::Controller { controller } => Ok(controller.as_ref().is_some_and(|state| state.connection_id == connection_id)),
1834 _ => Err(NodeClientError::Protocol("controller acquisition returned a different response".to_owned())),
1835 }
1836}
1837
1838async fn release_controller(
1839 client: &mut LocalNodeClient,
1840 controller_owned: &mut bool,
1841) -> Result<(), NodeClientError> {
1842 if !*controller_owned { return Ok(()); }
1843 match bounded_node_request(client, NodeRequest::ReleaseController).await? {
1844 NodeResponse::Controller { .. } => { *controller_owned = false; Ok(()) }
1845 _ => Err(NodeClientError::Protocol("controller release returned a different response".to_owned())),
1846 }
1847}
1848
1849async fn update_inventory_from_response(
1850 node_id: &NodeId,
1851 cursor: &mut NodeCursor,
1852 response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1853 ingress: &mpsc::Sender<Attempt>,
1854) -> io::Result<()> {
1855 match response {
1856 Ok(NodeResponse::Snapshot { event_sequence, snapshot, .. })
1857 | Ok(NodeResponse::Resync { event_sequence, snapshot, .. }) => {
1858 if *event_sequence < cursor.sequence { return Ok(()); }
1859 cursor.sequence = *event_sequence;
1860 ingress_attempt(ingress, node_id, AttemptResult::Success { cursor: *cursor, snapshot: snapshot.clone(), gaps: Vec::new() }).await
1861 }
1862 _ => Ok(()),
1863 }
1864}
1865
1866fn publish_recovered_events(
1867 node_id: &NodeId,
1868 incarnation_id: NodeIncarnationId,
1869 events: &[NodeEventEnvelope],
1870 hub: &OperatorHub,
1871) {
1872 for envelope in events {
1873 if let Some(event) = routed_recovered_node_event(node_id, incarnation_id, envelope) {
1874 hub.publish(event);
1875 }
1876 }
1877}
1878
1879fn routed_recovered_node_event(
1880 node_id: &NodeId,
1881 incarnation_id: NodeIncarnationId,
1882 envelope: &NodeEventEnvelope,
1883) -> Option<RoutedNodeEvent> {
1884 (!matches!(&envelope.event, NodeEvent::HarnessMcpReadCall { .. })).then(|| {
1885 RoutedNodeEvent {
1886 node_id: node_id.clone(),
1887 cursor: NodeCursor { incarnation_id, sequence: envelope.sequence },
1888 event: C2NodeEvent::from_node_event_with_control_detail(&envelope.event),
1889 }
1890 })
1891}
1892
1893fn routed_transient_node_event(
1894 node_id: &NodeId,
1895 cursor: NodeCursor,
1896 envelope: &NodeEventEnvelope,
1897) -> Option<RoutedNodeEvent> {
1898 matches!(&envelope.event, NodeEvent::HarnessMcpReadCall { .. }).then(|| RoutedNodeEvent {
1899 node_id: node_id.clone(),
1900 cursor,
1901 event: C2NodeEvent::from(&envelope.event),
1902 })
1903}
1904
1905fn route_agent_stream_event(
1942 node_id: &NodeId,
1943 cursor: &mut NodeCursor,
1944 envelope: &NodeEventEnvelope,
1945) -> Option<RoutedNodeEvent> {
1946 if !matches!(&envelope.event, NodeEvent::AgentStream { .. }) {
1947 return None;
1948 }
1949 let routed = RoutedNodeEvent {
1950 node_id: node_id.clone(),
1951 cursor: *cursor,
1952 event: C2NodeEvent::from(&envelope.event),
1953 };
1954 cursor.sequence = cursor.sequence.max(envelope.sequence);
1955 Some(routed)
1956}
1957
1958async fn drain_pending_events(
1959 client: &mut LocalNodeClient,
1960 node_id: &NodeId,
1961 cursor: &mut NodeCursor,
1962 hub: &OperatorHub,
1963 ingress: &mpsc::Sender<Attempt>,
1964 skip_replayed: bool,
1965 relay_deadline: Option<Instant>,
1966) -> Result<(), NodeClientError> {
1967 let mut skip_replayed = skip_replayed;
1968 for repair_pass in 0..=1 {
1969 let mut gaps = Vec::new();
1970 let mut managed_worktree_events = Vec::new();
1971 let mut changed = false;
1972 let mut repair = false;
1973 while let Some(envelope) = client.take_event() {
1974 if let Some(event) = routed_transient_node_event(node_id, *cursor, &envelope) {
1975 hub.publish(event);
1976 continue;
1977 }
1978 if let Some(event) = route_agent_stream_event(node_id, cursor, &envelope) {
1979 hub.publish(event);
1980 continue;
1981 }
1982 if skip_replayed && envelope.sequence <= cursor.sequence { continue; }
1983 let resync_required = matches!(&envelope.event, gate4agent_node_protocol::NodeEvent::ResyncRequired { .. });
1984 if resync_required || envelope.sequence != cursor.sequence.saturating_add(1) {
1985 gaps.push(if envelope.sequence <= cursor.sequence && !resync_required {
1986 GapKind::CursorRegression
1987 } else if resync_required {
1988 GapKind::HistoryEvicted
1989 } else {
1990 GapKind::NonContiguousEvents
1991 });
1992 repair = true;
1993 continue;
1994 }
1995 let event_cursor = NodeCursor { incarnation_id: cursor.incarnation_id, sequence: envelope.sequence };
1996 if matches!(
1997 &envelope.event,
1998 NodeEvent::ManagedWorktreeUpserted { .. }
1999 | NodeEvent::ManagedWorktreeRemoved { .. }
2000 ) {
2001 managed_worktree_events.push(envelope.event.clone());
2002 }
2003 hub.publish(RoutedNodeEvent {
2004 node_id: node_id.clone(),
2005 cursor: event_cursor,
2006 event: C2NodeEvent::from_node_event_with_control_detail(&envelope.event),
2007 });
2008 cursor.sequence = envelope.sequence;
2009 changed = true;
2010 }
2011 if !repair {
2012 if changed || !gaps.is_empty() {
2013 ingress_attempt(ingress, node_id, AttemptResult::Cursor {
2014 cursor: *cursor,
2015 gaps,
2016 managed_worktree_events,
2017 }).await
2018 .map_err(NodeClientError::Io)?;
2019 }
2020 return Ok(());
2021 }
2022 if repair_pass == 1 {
2023 ingress_attempt(ingress, node_id, AttemptResult::Cursor {
2024 cursor: *cursor,
2025 gaps,
2026 managed_worktree_events,
2027 }).await
2028 .map_err(NodeClientError::Io)?;
2029 return Err(NodeClientError::Protocol("node event stream remained noncontiguous after resync".to_owned()));
2030 }
2031 let after_sequence = cursor.sequence;
2032 let response = bounded_node_request_with_deadline(
2033 client,
2034 NodeRequest::Resync { after_sequence },
2035 relay_deadline,
2036 ).await?;
2037 let NodeResponse::Resync {
2038 event_sequence,
2039 oldest_available_sequence,
2040 snapshot,
2041 events,
2042 } = response else {
2043 return Err(NodeClientError::Protocol("event repair resync returned a different response".to_owned()));
2044 };
2045 let repair_gaps = validate_events(
2046 after_sequence,
2047 event_sequence,
2048 oldest_available_sequence,
2049 &events,
2050 );
2051 let contiguous = repair_gaps.is_empty();
2052 gaps = repair_gaps;
2053 if contiguous {
2054 for envelope in events.iter().filter(|event| event.sequence > after_sequence) {
2055 if let Some(event) = routed_recovered_node_event(
2056 node_id,
2057 cursor.incarnation_id,
2058 envelope,
2059 ) {
2060 hub.publish(event);
2061 }
2062 }
2063 } else {
2064 hub.publish(RoutedNodeEvent {
2065 node_id: node_id.clone(),
2066 cursor: NodeCursor { incarnation_id: cursor.incarnation_id, sequence: event_sequence },
2067 event: C2NodeEvent::ResyncRequired {
2068 oldest_available_sequence,
2069 },
2070 });
2071 }
2072 cursor.sequence = event_sequence;
2073 ingress_attempt(ingress, node_id, AttemptResult::Success { cursor: *cursor, snapshot, gaps }).await
2074 .map_err(NodeClientError::Io)?;
2075 skip_replayed = true;
2076 }
2077 Ok(())
2078}
2079
2080async fn handle_live_node_event(
2081 client: &mut LocalNodeClient,
2082 node_id: &NodeId,
2083 envelope: NodeEventEnvelope,
2084 cursor: &mut NodeCursor,
2085 hub: &OperatorHub,
2086 ingress: &mpsc::Sender<Attempt>,
2087) -> Result<(), NodeClientError> {
2088 if let Some(event) = routed_transient_node_event(node_id, *cursor, &envelope) {
2089 hub.publish(event);
2090 return Ok(());
2091 }
2092 if let Some(event) = route_agent_stream_event(node_id, cursor, &envelope) {
2093 hub.publish(event);
2094 return Ok(());
2095 }
2096 if live_event_gap(cursor.sequence, &envelope).is_some() {
2097 let after_sequence = cursor.sequence;
2098 let response = bounded_node_request(
2099 client,
2100 NodeRequest::Resync { after_sequence },
2101 ).await?;
2102 let NodeResponse::Resync {
2103 event_sequence,
2104 oldest_available_sequence,
2105 snapshot,
2106 events,
2107 } = response else {
2108 return Err(NodeClientError::Protocol(
2109 "live event repair resync returned a different response".to_owned(),
2110 ));
2111 };
2112 let repair_gaps = validate_events(
2113 after_sequence,
2114 event_sequence,
2115 oldest_available_sequence,
2116 &events,
2117 );
2118 if repair_gaps.is_empty() {
2119 publish_recovered_events(node_id, cursor.incarnation_id, &events, hub);
2120 } else {
2121 hub.publish(RoutedNodeEvent {
2122 node_id: node_id.clone(),
2123 cursor: NodeCursor {
2124 incarnation_id: cursor.incarnation_id,
2125 sequence: event_sequence,
2126 },
2127 event: C2NodeEvent::ResyncRequired {
2128 oldest_available_sequence,
2129 },
2130 });
2131 }
2132 cursor.sequence = event_sequence;
2133 ingress_attempt(
2134 ingress,
2135 node_id,
2136 AttemptResult::Success {
2137 cursor: *cursor,
2138 snapshot,
2139 gaps: repair_gaps,
2140 },
2141 ).await.map_err(NodeClientError::Io)?;
2142 return drain_pending_events(
2143 client,
2144 node_id,
2145 cursor,
2146 hub,
2147 ingress,
2148 true,
2149 None,
2150 ).await;
2151 }
2152
2153 cursor.sequence = envelope.sequence;
2154 let managed_worktree_events = matches!(
2155 &envelope.event,
2156 NodeEvent::ManagedWorktreeUpserted { .. }
2157 | NodeEvent::ManagedWorktreeRemoved { .. }
2158 )
2159 .then(|| vec![envelope.event.clone()])
2160 .unwrap_or_default();
2161 hub.publish(RoutedNodeEvent {
2162 node_id: node_id.clone(),
2163 cursor: *cursor,
2164 event: C2NodeEvent::from_node_event_with_control_detail(&envelope.event),
2165 });
2166 ingress_attempt(
2167 ingress,
2168 node_id,
2169 AttemptResult::Cursor {
2170 cursor: *cursor,
2171 gaps: Vec::new(),
2172 managed_worktree_events,
2173 },
2174 ).await.map_err(NodeClientError::Io)
2175}
2176
2177fn live_event_gap(previous: u64, envelope: &NodeEventEnvelope) -> Option<GapKind> {
2178 if matches!(
2179 &envelope.event,
2180 gate4agent_node_protocol::NodeEvent::ResyncRequired { .. }
2181 ) {
2182 return Some(GapKind::HistoryEvicted);
2183 }
2184 if envelope.sequence <= previous {
2185 return Some(GapKind::CursorRegression);
2186 }
2187 (envelope.sequence != previous.saturating_add(1))
2188 .then_some(GapKind::NonContiguousEvents)
2189}
2190
2191fn reject_disconnected_commands(commands: &mut mpsc::Receiver<RelayCommand>, cursor: Option<NodeCursor>) {
2192 while let Ok(command) = commands.try_recv() {
2193 match command {
2194 RelayCommand::Request { reply, .. } => {
2195 let _ = reply.send(Err(relay_failure(
2196 C2RelayFailureCode::NodeOffline,
2197 "node relay disconnected before request dispatch",
2198 cursor.map(|value| value.incarnation_id),
2199 )));
2200 }
2201 }
2202 }
2203}
2204
2205fn acknowledge_disconnected_releases(releases: &mut mpsc::Receiver<oneshot::Sender<()>>) {
2206 while let Ok(reply) = releases.try_recv() { let _ = reply.send(()); }
2207}
2208
2209fn validate_events(
2210 previous: u64,
2211 current: u64,
2212 oldest_available_sequence: u64,
2213 events: &[NodeEventEnvelope],
2214) -> Vec<GapKind> {
2215 if current < previous { return vec![GapKind::CursorRegression]; }
2216 let max_floor = current.checked_add(1).unwrap_or(u64::MAX);
2217 let minimum_event_sequence = previous
2218 .checked_add(1)
2219 .unwrap_or(u64::MAX)
2220 .max(oldest_available_sequence);
2221 if oldest_available_sequence == 0
2222 || oldest_available_sequence > max_floor
2223 || (current == previous && !events.is_empty())
2224 || events.iter().any(|event| {
2225 event.sequence < minimum_event_sequence || event.sequence > current
2226 })
2227 || events.windows(2).any(|pair| pair[0].sequence >= pair[1].sequence)
2228 {
2229 return vec![GapKind::NonContiguousEvents];
2230 }
2231 if previous.saturating_add(1) < oldest_available_sequence {
2232 return vec![GapKind::HistoryEvicted];
2233 }
2234 Vec::new()
2235}
2236
2237fn sanitize_node_error(error: &NodeClientError) -> (SanitizedError, bool) {
2238 let (category, message, hard) = match error {
2239 NodeClientError::Protocol(message) if message.contains("identity mismatch") =>
2240 (C2ErrorCategory::Identity, "node identity mismatch", true),
2241 NodeClientError::Protocol(message) if message.contains("access-token proof") || message.contains("access denied") =>
2242 (C2ErrorCategory::Authentication, "node authentication failed", true),
2243 NodeClientError::Protocol(_) | NodeClientError::Frame(FrameError::Json(_) | FrameError::InvalidLength { .. }) =>
2244 (C2ErrorCategory::Protocol, "node protocol failed", true),
2245 NodeClientError::BuildStampMismatch { .. } =>
2246 (C2ErrorCategory::Protocol, "node build stamp mismatch", true),
2247 NodeClientError::UnsupportedCapability(_) =>
2248 (C2ErrorCategory::Protocol, "node capability unavailable", true),
2249 NodeClientError::Node(failure) if failure.code == NodeFailureCode::Unauthorized =>
2250 (C2ErrorCategory::Authentication, "node request authentication failed", true),
2251 NodeClientError::Node(_) =>
2252 (C2ErrorCategory::Protocol, "node rejected observer request", true),
2253 NodeClientError::Frame(FrameError::BodyTimedOut { .. } | FrameError::PrefixTimedOut)
2254 | NodeClientError::AuthenticationTimedOut =>
2255 (C2ErrorCategory::Timeout, "node observation deadline exceeded", false),
2256 NodeClientError::Authentication(_) | NodeClientError::RequestIdExhausted =>
2257 (C2ErrorCategory::Internal, "node client failed internally", true),
2258 NodeClientError::Io(_) | NodeClientError::Frame(FrameError::Io(_)) =>
2259 (C2ErrorCategory::Transport, "node transport unavailable", false),
2260 };
2261 (SanitizedError { category, message: message.to_owned() }, hard)
2262}
2263
2264async fn inventory_owner(
2265 configured: usize, fresh_for: Duration, mut ingress: mpsc::Receiver<Attempt>,
2266 status: watch::Sender<Arc<StatusResponse>>, mut shutdown: watch::Receiver<bool>,
2267) -> io::Result<()> {
2268 let mut current = (**status.borrow()).clone();
2269 let mut attempted = BTreeSet::new();
2270 loop {
2271 tokio::select! {
2272 attempt = ingress.recv() => {
2273 let Some(attempt) = attempt else { return Ok(()); };
2274 attempted.insert(attempt.node_id.clone());
2275 let node = current.nodes.get_mut(&attempt.node_id).expect("configured poller node exists");
2276 node.last_attempt_unix_ms = Some(attempt.at_unix_ms);
2277 match attempt.result {
2278 AttemptResult::Connected {
2279 cursor,
2280 snapshot,
2281 gaps,
2282 provider_contract_manifest,
2283 } => {
2284 let previous = node.cursor;
2285 let mut inventory = SlimNodeInventory::from_snapshot(&snapshot);
2286 inventory.provider_contracts =
2287 provider_contract_manifest.provider_contracts;
2288 inventory.provider_adapter_contracts =
2289 provider_contract_manifest.provider_adapter_contracts;
2290 node.transport = NodeTransportState::Online;
2291 node.freshness = NodeFreshness::Fresh;
2292 node.cursor = Some(cursor);
2293 node.inventory = Some(inventory);
2294 node.last_success_unix_ms = Some(attempt.at_unix_ms);
2295 node.consecutive_failures = 0;
2296 node.last_error = None;
2297 for kind in gaps {
2298 if node.gaps.len() == MAX_C2_GAPS_PER_NODE { node.gaps.remove(0); node.gaps_truncated += 1; }
2299 node.gaps.push(NodeGap { kind, detected_at_unix_ms: attempt.at_unix_ms, previous, observed: cursor });
2300 }
2301 }
2302 AttemptResult::Success { cursor, snapshot, gaps } => {
2303 let previous = node.cursor;
2304 let provider_contract_manifest = node.inventory.as_ref().map(|inventory| {
2305 ProviderContractManifest {
2306 provider_contracts: inventory.provider_contracts.clone(),
2307 provider_adapter_contracts: inventory.provider_adapter_contracts.clone(),
2308 }
2309 }).unwrap_or_default();
2310 let mut inventory = SlimNodeInventory::from_snapshot(&snapshot);
2311 inventory.provider_contracts =
2312 provider_contract_manifest.provider_contracts;
2313 inventory.provider_adapter_contracts =
2314 provider_contract_manifest.provider_adapter_contracts;
2315 node.transport = NodeTransportState::Online;
2316 node.freshness = NodeFreshness::Fresh;
2317 node.cursor = Some(cursor);
2318 node.inventory = Some(inventory);
2319 node.last_success_unix_ms = Some(attempt.at_unix_ms);
2320 node.consecutive_failures = 0;
2321 node.last_error = None;
2322 for kind in gaps {
2323 if node.gaps.len() == MAX_C2_GAPS_PER_NODE { node.gaps.remove(0); node.gaps_truncated += 1; }
2324 node.gaps.push(NodeGap { kind, detected_at_unix_ms: attempt.at_unix_ms, previous, observed: cursor });
2325 }
2326 }
2327 AttemptResult::Cursor {
2328 cursor,
2329 gaps,
2330 managed_worktree_events,
2331 } => {
2332 let previous = node.cursor;
2333 let incarnation_changed = previous.is_some_and(|previous| {
2334 previous.incarnation_id != cursor.incarnation_id
2335 });
2336 apply_managed_worktree_cursor(
2337 node.inventory.as_mut(),
2338 incarnation_changed,
2339 &managed_worktree_events,
2340 );
2341 node.transport = NodeTransportState::Online;
2342 node.freshness = NodeFreshness::Fresh;
2343 node.cursor = Some(cursor);
2344 node.last_success_unix_ms = Some(attempt.at_unix_ms);
2345 node.consecutive_failures = 0;
2346 node.last_error = None;
2347 for kind in gaps {
2348 if node.gaps.len() == MAX_C2_GAPS_PER_NODE { node.gaps.remove(0); node.gaps_truncated += 1; }
2349 node.gaps.push(NodeGap { kind, detected_at_unix_ms: attempt.at_unix_ms, previous, observed: cursor });
2350 }
2351 }
2352 AttemptResult::Failure { error, hard } => {
2353 node.consecutive_failures = node.consecutive_failures.saturating_add(1);
2354 node.transport = if hard || node.consecutive_failures >= 5 { NodeTransportState::Parked } else { NodeTransportState::Offline };
2355 node.last_error = Some(error);
2356 }
2357 }
2358 current.ready = attempted.len() == configured;
2359 refresh_freshness(&mut current, fresh_for);
2360 current.observed_at_unix_ms = unix_ms();
2361 status.send_replace(Arc::new(current.clone()));
2362 }
2363 _ = sleep(Duration::from_millis(250)) => {
2364 refresh_freshness(&mut current, fresh_for);
2365 current.observed_at_unix_ms = unix_ms();
2366 status.send_replace(Arc::new(current.clone()));
2367 }
2368 changed = shutdown.changed() => if changed.is_err() || *shutdown.borrow() { return Ok(()); },
2369 }
2370 }
2371}
2372
2373fn apply_managed_worktree_cursor(
2374 inventory: Option<&mut SlimNodeInventory>,
2375 incarnation_changed: bool,
2376 events: &[NodeEvent],
2377) {
2378 let Some(inventory) = inventory else { return; };
2379 if incarnation_changed {
2380 inventory.provider_runtime_statuses.clear();
2381 inventory.managed_worktrees.clear();
2382 inventory.managed_worktree_count = 0;
2383 inventory.managed_worktrees_truncated = false;
2384 return;
2385 }
2386 for event in events {
2387 match event {
2388 NodeEvent::ManagedWorktreeUpserted { lease } => {
2389 inventory.managed_worktrees.retain(|existing| {
2390 existing.lease_id != lease.lease_id
2391 && existing.workspace_id != lease.workspace_id
2392 });
2393 inventory.managed_worktrees.push(lease.clone());
2394 inventory.managed_worktrees.sort_by(|left, right| {
2395 left.lease_id.cmp(&right.lease_id)
2396 });
2397 inventory.managed_worktree_count = inventory.managed_worktrees.len();
2398 inventory.managed_worktrees.truncate(
2399 crate::protocol::MAX_C2_MANAGED_WORKTREES_PER_NODE,
2400 );
2401 inventory.managed_worktrees_truncated =
2402 inventory.managed_worktrees.len() < inventory.managed_worktree_count;
2403 }
2404 NodeEvent::ManagedWorktreeRemoved { lease_id } => {
2405 let before = inventory.managed_worktrees.len();
2406 inventory
2407 .managed_worktrees
2408 .retain(|lease| &lease.lease_id != lease_id);
2409 if inventory.managed_worktrees.len() < before
2410 || inventory.managed_worktrees_truncated
2411 {
2412 inventory.managed_worktree_count =
2413 inventory.managed_worktree_count.saturating_sub(1);
2414 }
2415 inventory.managed_worktrees_truncated =
2416 inventory.managed_worktrees.len() < inventory.managed_worktree_count;
2417 }
2418 _ => {}
2419 }
2420 }
2421}
2422
2423fn refresh_freshness(status: &mut StatusResponse, fresh_for: Duration) {
2424 let now = unix_ms();
2425 let fresh_ms = fresh_for.as_millis().min(u64::MAX as u128) as u64;
2426 for node in status.nodes.values_mut() {
2427 node.freshness = match node.last_success_unix_ms {
2428 None => NodeFreshness::Unavailable,
2429 Some(last) if now.saturating_sub(last) <= fresh_ms => NodeFreshness::Fresh,
2430 Some(_) => NodeFreshness::Stale,
2431 };
2432 }
2433}
2434
2435async fn http_server(
2436 listener: TcpListener, token: String, io_deadline: Duration,
2437 status: watch::Receiver<Arc<StatusResponse>>, mut shutdown: watch::Receiver<bool>,
2438) -> io::Result<()> {
2439 let permits = Arc::new(Semaphore::new(MAX_HTTP_CONNECTIONS));
2440 let mut connections = JoinSet::new();
2441 loop {
2442 tokio::select! {
2443 accepted = listener.accept() => {
2444 let (stream, _) = accepted?;
2445 let Ok(permit) = Arc::clone(&permits).try_acquire_owned() else { drop(stream); continue; };
2446 let token = token.clone();
2447 let status = status.clone();
2448 connections.spawn(async move { let _permit = permit; let _ = serve_http(stream, &token, io_deadline, &status).await; });
2449 }
2450 changed = shutdown.changed() => if changed.is_err() || *shutdown.borrow() { break; },
2451 }
2452 while let Some(result) = connections.try_join_next() { result.map_err(io::Error::other)?; }
2453 }
2454 connections.shutdown().await;
2455 Ok(())
2456}
2457
2458async fn serve_http(mut stream: TcpStream, token: &str, deadline: Duration, status: &watch::Receiver<Arc<StatusResponse>>) -> io::Result<()> {
2459 let request = match timeout(deadline, read_request(&mut stream)).await {
2460 Ok(Ok(request)) => request,
2461 Ok(Err(ReadError::TooLarge)) => return write_response(&mut stream, deadline, Response::plain(413, "Payload Too Large")).await,
2462 Ok(Err(ReadError::Io(error))) => return Err(error),
2463 _ => return Ok(()),
2464 };
2465 let response = route(request, token, status.borrow().as_ref());
2466 write_response(&mut stream, deadline, response).await
2467}
2468
2469struct Request { method: String, path: String, authorization: Option<String> }
2470enum ReadError { Closed, Invalid, TooLarge, Io(io::Error) }
2471
2472async fn read_request(stream: &mut TcpStream) -> Result<Request, ReadError> {
2473 let mut bytes = Vec::with_capacity(1024);
2474 let mut chunk = [0_u8; 1024];
2475 loop {
2476 let count = stream.read(&mut chunk).await.map_err(ReadError::Io)?;
2477 if count == 0 { return Err(ReadError::Closed); }
2478 if bytes.len().saturating_add(count) > HEADER_LIMIT_BYTES { return Err(ReadError::TooLarge); }
2479 bytes.extend_from_slice(&chunk[..count]);
2480 if bytes.windows(4).any(|window| window == b"\r\n\r\n") { break; }
2481 }
2482 let text = std::str::from_utf8(&bytes).map_err(|_| ReadError::Invalid)?;
2483 let mut lines = text.split("\r\n");
2484 let mut first = lines.next().ok_or(ReadError::Invalid)?.split_whitespace();
2485 let method = first.next().ok_or(ReadError::Invalid)?;
2486 let path = first.next().ok_or(ReadError::Invalid)?;
2487 let version = first.next().ok_or(ReadError::Invalid)?;
2488 if first.next().is_some() || !version.starts_with("HTTP/1.") || !path.starts_with('/') { return Err(ReadError::Invalid); }
2489 let mut authorization = None;
2490 for line in lines {
2491 if line.is_empty() { break; }
2492 let (name, value) = line.split_once(':').ok_or(ReadError::Invalid)?;
2493 if name.eq_ignore_ascii_case("authorization") {
2494 if authorization.is_some() { return Err(ReadError::Invalid); }
2495 authorization = Some(value.trim().to_owned());
2496 }
2497 }
2498 Ok(Request { method: method.to_owned(), path: path.to_owned(), authorization })
2499}
2500
2501fn route(request: Request, token: &str, status: &StatusResponse) -> Response {
2502 if request.method != "GET" { return Response::plain(405, "Method Not Allowed").allow_get(); }
2503 let path = request.path.split_once('?').map_or(request.path.as_str(), |pair| pair.0);
2504 match path {
2505 "/health" => Response::json(200, &HealthResponse { ok: true, service: "gate4agent-c2".to_owned(), api_version: C2_API_VERSION, pid: std::process::id(), version: env!("CARGO_PKG_VERSION").to_owned() }),
2506 "/ready" => {
2507 let online_nodes = status.nodes.values().filter(|node| node.transport == NodeTransportState::Online).count();
2508 let offline_nodes = status.nodes.values().filter(|node| node.transport == NodeTransportState::Offline).count();
2509 let parked_nodes = status.nodes.values().filter(|node| node.transport == NodeTransportState::Parked).count();
2510 let body = ReadyResponse { ready: status.ready, api_version: C2_API_VERSION, configured_nodes: status.nodes.len(), attempted_nodes: status.nodes.values().filter(|node| node.last_attempt_unix_ms.is_some()).count(), online_nodes, offline_nodes, parked_nodes };
2511 Response::json(if status.ready { 200 } else { 503 }, &body)
2512 }
2513 "/status" => {
2514 if !authorized(request.authorization.as_deref(), token) { Response::plain(401, "Unauthorized").authenticate() }
2515 else { Response::json(200, status) }
2516 }
2517 _ => Response::plain(404, "Not Found"),
2518 }
2519}
2520
2521fn authorized(header: Option<&str>, token: &str) -> bool {
2522 let Some((scheme, candidate)) = header.and_then(|value| value.split_once(' ')) else { return false; };
2523 scheme.eq_ignore_ascii_case("bearer") && constant_time_eq(candidate.as_bytes(), token.as_bytes())
2524}
2525
2526fn constant_time_eq(left: &[u8], right: &[u8]) -> bool {
2527 if left.len() != right.len() { return false; }
2528 left.iter().zip(right).fold(0_u8, |difference, (left, right)| difference | (left ^ right)) == 0
2529}
2530
2531struct Response { status: u16, reason: &'static str, content_type: &'static str, body: Vec<u8>, headers: Vec<(&'static str, &'static str)> }
2532impl Response {
2533 fn plain(status: u16, reason: &'static str) -> Self { Self { status, reason, content_type: "text/plain; charset=utf-8", body: reason.as_bytes().to_vec(), headers: Vec::new() } }
2534 fn json<T: serde::Serialize>(status: u16, value: &T) -> Self {
2535 let body = serde_json::to_vec(value).expect("C2 DTO must serialize");
2536 if body.len() > RESPONSE_BODY_LIMIT_BYTES { return Self::plain(503, "Service Unavailable"); }
2537 let reason = if status == 200 { "OK" } else { "Service Unavailable" };
2538 Self { status, reason, content_type: "application/json", body, headers: Vec::new() }
2539 }
2540 fn allow_get(mut self) -> Self { self.headers.push(("Allow", "GET")); self }
2541 fn authenticate(mut self) -> Self { self.headers.push(("WWW-Authenticate", "Bearer")); self }
2542}
2543
2544async fn write_response(stream: &mut TcpStream, deadline: Duration, response: Response) -> io::Result<()> {
2545 let mut headers = format!("HTTP/1.1 {} {}\r\nContent-Type: {}\r\nContent-Length: {}\r\nConnection: close\r\n", response.status, response.reason, response.content_type, response.body.len());
2546 for (name, value) in response.headers { headers.push_str(name); headers.push_str(": "); headers.push_str(value); headers.push_str("\r\n"); }
2547 headers.push_str("\r\n");
2548 timeout(deadline, async { stream.write_all(headers.as_bytes()).await?; stream.write_all(&response.body).await?; stream.shutdown().await }).await
2549 .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "C2 HTTP write timed out"))?
2550}
2551
2552fn unix_ms() -> u64 {
2553 SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis().min(u64::MAX as u128) as u64
2554}
2555
2556#[cfg(test)]
2557mod endpoint_tests {
2558 use super::*;
2559 use crate::protocol::{C2RelayRoute, C2Topology};
2560
2561 fn node(endpoint: &str) -> Result<C2NodeConfig, C2ConfigError> {
2562 C2NodeConfig::new(NodeId::new("remote-node").unwrap(), endpoint, "safe-token")
2563 }
2564
2565 #[cfg(windows)]
2566 const LOCAL_ENDPOINT: &str = r"\\.\pipe\relay-route-fact";
2567 #[cfg(unix)]
2568 const LOCAL_ENDPOINT: &str = "/tmp/gate4agent-relay-route-fact.sock";
2569
2570 fn projected_route(node: &C2NodeConfig, transport: NodeTransportState) -> C2RelayRoute {
2571 let mut observed = initial_observed_node(node);
2572 observed.transport = transport;
2573 let status = StatusResponse {
2574 api_version: C2_API_VERSION,
2575 ready: false,
2576 observed_at_unix_ms: 1,
2577 nodes: BTreeMap::from([(node.node_id.clone(), observed)]),
2578 };
2579 C2Topology::from_status(&status).nodes[0].relay_route
2580 }
2581
2582 #[test]
2583 fn relay_route_fact_projects_exactly_and_survives_offline_and_parked() {
2584 let local = node(LOCAL_ENDPOINT).unwrap();
2585 let ssh = node("tcp://127.0.0.1:48100").unwrap();
2586
2587 for transport in [
2588 NodeTransportState::Online,
2589 NodeTransportState::Offline,
2590 NodeTransportState::Parked,
2591 ] {
2592 assert_eq!(projected_route(&local, transport), C2RelayRoute::LocalIpc);
2593 assert_eq!(
2594 projected_route(&ssh, transport),
2595 C2RelayRoute::SshForwardedLoopback,
2596 );
2597 }
2598 }
2599
2600 #[test]
2605 fn a_node_that_waits_to_be_called_is_declared_by_name_not_by_address() {
2606 let waiting = node("accept").unwrap();
2607 assert_eq!(waiting.route, C2NodeRoute::CallHome);
2608 assert_eq!(waiting.transport_label(), "call-home");
2609
2610 let second = C2NodeConfig::new(
2615 NodeId::new("second-waiting-node").unwrap(),
2616 "accept",
2617 "safe-token",
2618 )
2619 .unwrap();
2620 let config = C2Config::new(
2621 "127.0.0.1:0".parse().unwrap(),
2622 "safe-token",
2623 vec![waiting.clone(), second],
2624 )
2625 .expect("two waiting nodes are not a duplicate endpoint");
2626 assert_eq!(config.nodes.len(), 2);
2627 }
2628
2629 #[test]
2633 fn waiting_for_a_call_requires_somewhere_to_be_called() {
2634 let waiting = node("accept").unwrap();
2635 let config =
2636 C2Config::new("127.0.0.1:0".parse().unwrap(), "safe-token", vec![waiting]).unwrap();
2637 assert!(matches!(
2638 config.validate_call_home(),
2639 Err(C2ConfigError::CallHomeWithoutListener(_)),
2640 ));
2641 let config = config.with_node_listen("127.0.0.1:48200".parse().unwrap()).unwrap();
2642 assert!(config.validate_call_home().is_ok());
2643 }
2644
2645 #[test]
2649 fn the_call_home_listener_refuses_unspecified_and_accepts_tunnel() {
2650 let waiting = node("accept").unwrap();
2651 let config =
2652 C2Config::new("127.0.0.1:9000".parse().unwrap(), "safe-token", vec![waiting]).unwrap();
2653 for refused in ["0.0.0.0:48200", "127.0.0.1:0", "[::]:48200"] {
2654 assert!(
2655 matches!(
2656 config.clone().with_node_listen(refused.parse().unwrap()),
2657 Err(C2ConfigError::NonLoopbackNodeListen(_)),
2658 ),
2659 "accepted {refused}",
2660 );
2661 }
2662 let tunnel: SocketAddr = "10.88.1.2:27442".parse().unwrap();
2663 let accepted = config.clone().with_node_listen(tunnel).unwrap();
2664 assert_eq!(accepted.node_listen, Some(tunnel));
2665 assert!(matches!(
2668 config.with_node_listen("127.0.0.1:9000".parse().unwrap()),
2669 Err(C2ConfigError::NodeListenConflict),
2670 ));
2671 }
2672
2673 #[test]
2674 fn ssh_forwarded_loopback_route_is_strict_canonical_and_control_stays_local() {
2675 let ipv4 = node("tcp://127.0.0.1:48100").unwrap();
2676 assert_eq!(ipv4.endpoint, "tcp://127.0.0.1:48100");
2677 assert_eq!(ipv4.route, C2NodeRoute::SshForwardedLoopback("127.0.0.1:48100".parse().unwrap()));
2678 assert_eq!(ipv4.transport_label(), "ssh-forwarded-loopback");
2679
2680 let ipv6 = node("tcp://[0:0:0:0:0:0:0:1]:48100").unwrap();
2681 assert_eq!(ipv6.endpoint, "tcp://[::1]:48100");
2682 assert_eq!(ipv6.transport_label(), "ssh-forwarded-loopback");
2683
2684 for invalid in [
2685 "tcp://localhost:48100",
2686 "tcp://127.0.0.2:48100",
2687 "tcp://0.0.0.0:48100",
2688 "tcp://127.0.0.1:0",
2689 "tcp://user@127.0.0.1:48100",
2690 "tcp://127.0.0.1:48100/path",
2691 "TCP://127.0.0.1:48100",
2692 ] {
2693 assert!(matches!(node(invalid), Err(C2ConfigError::InvalidEndpoint(_))), "accepted {invalid}");
2694 }
2695
2696 assert!(matches!(
2697 C2Config::new(
2698 "127.0.0.1:0".parse().unwrap(),
2699 "safe-token",
2700 vec![ipv4.clone(), C2NodeConfig::new(
2701 NodeId::new("duplicate-route").unwrap(),
2702 "tcp://127.0.0.1:48100",
2703 "safe-token",
2704 ).unwrap()],
2705 ),
2706 Err(C2ConfigError::DuplicateEndpoint)
2707 ));
2708 assert!(matches!(
2709 C2Config::new("127.0.0.1:0".parse().unwrap(), "safe-token", vec![ipv4])
2710 .unwrap()
2711 .with_control_endpoint("tcp://127.0.0.1:48101"),
2712 Err(C2ConfigError::InvalidControlEndpoint)
2713 ));
2714 }
2715}
2716
2717#[cfg(all(test, windows))]
2718mod tests {
2719 use super::*;
2720 use gate4agent_node_protocol::{
2721 AgentId, AgentStreamChunkKindV1, AgentStreamChunkV1, CapabilityId, SessionAddress,
2722 SessionKey, SessionMode, SessionRecordId,
2723 HarnessMcpActivationDigest, HarnessMcpCallId, HarnessMcpContentTypeV1,
2724 HarnessMcpOpaquePayloadV1, HarnessMcpReservationId,
2725 SpawnDeadlineMs, SpawnFieldProvenance, SpawnIdempotencyKey, SpawnOverrides,
2726 SpawnProfileId, SpawnProfileRevision, SpawnPrompt, SpawnPromptMetadata,
2727 SpawnRequiredCapabilities, SpawnResolutionProvenance, SpawnTarget, WorkspaceId,
2728 ManagedWorktreeCleanupFailure, ManagedWorktreeLeaseId,
2729 ManagedWorktreeLeaseSnapshot, ManagedWorktreeRetention, ManagedWorktreeSpawnReceipt,
2730 WorktreeProfileId, WorktreeProfileRevision,
2731 };
2732 use gate4agent_types::{AgentInstanceId, SessionGeneration, TerminalSize};
2733 use std::collections::BTreeMap;
2734
2735 fn agent(value: &str) -> gate4agent_node_protocol::AgentId {
2736 gate4agent_node_protocol::AgentId::new(value).unwrap()
2737 }
2738
2739 fn managed_lease(
2740 lease_id: &str,
2741 workspace_id: &str,
2742 state: ManagedWorktreeLeaseState,
2743 ) -> ManagedWorktreeLeaseSnapshot {
2744 let in_use = state == ManagedWorktreeLeaseState::InUse;
2745 ManagedWorktreeLeaseSnapshot {
2746 lease_id: ManagedWorktreeLeaseId::new(lease_id).unwrap(),
2747 source_workspace_id: WorkspaceId::new("repo").unwrap(),
2748 workspace_id: WorkspaceId::new(workspace_id).unwrap(),
2749 profile_id: WorktreeProfileId::new("review").unwrap(),
2750 profile_revision: WorktreeProfileRevision::new("review.r1").unwrap(),
2751 retention: ManagedWorktreeRetention::RemoveWhenReleased,
2752 state,
2753 active_session_count: u16::from(in_use),
2754 managed_record_count: u16::from(in_use),
2755 cleanup_failure: None::<ManagedWorktreeCleanupFailure>,
2756 created_at_unix_ms: 1,
2757 updated_at_unix_ms: 2,
2758 }
2759 }
2760
2761 #[test]
2762 fn spawn_spec_receipt_correlation_rejects_mismatches_before_forwarding() {
2763 let incarnation_id = NodeIncarnationId::from_bytes([7; 16]);
2764 let terminal_size = TerminalSize { rows: 24, columns: 80 };
2765 let prompt = SpawnPrompt::new("hi").unwrap();
2766 let required_capabilities = SpawnRequiredCapabilities::new([
2767 CapabilityId::new("raw-pty-lifecycle").unwrap(),
2768 ]).unwrap();
2769 let spec = SpawnSpec {
2770 target: SpawnTarget {
2771 node_id: NodeId::new("node-a").unwrap(),
2772 workspace_id: WorkspaceId::new("repo").unwrap(),
2773 worktree_id: None,
2774 },
2775 profile_id: SpawnProfileId::new("default").unwrap(),
2776 expected_profile_revision: SpawnProfileRevision::new("r1").unwrap(),
2777 overrides: SpawnOverrides {
2778 provider: SpawnOverride::Set { value: AgentId::new("codex").unwrap() },
2779 mode: SpawnOverride::Set { value: SessionMode::Pty },
2780 terminal_size: SpawnOverride::Set { value: terminal_size },
2781 prompt: SpawnOverride::Set { value: prompt.clone() },
2782 bundle_id: SpawnOverride::Clear,
2783 context_id: SpawnOverride::Clear,
2784 environment_profile_id: SpawnOverride::Clear,
2785 approval_level: None,
2786 network_allowlist: None,
2787 browser_profile_id: None,
2788 },
2789 deadline_ms: SpawnDeadlineMs::new(5_000).unwrap(),
2790 idempotency_key: SpawnIdempotencyKey::new("spawn-1").unwrap(),
2791 required_capabilities: required_capabilities.clone(),
2792 };
2793 let receipt = ResolvedSpawnReceipt {
2794 incarnation_id,
2795 session: SessionAddress {
2796 workspace_id: spec.target.workspace_id.clone(),
2797 session: SessionKey {
2798 instance_id: AgentInstanceId(7),
2799 generation: SessionGeneration(1),
2800 },
2801 },
2802 target: spec.target.clone(),
2803 profile_id: spec.profile_id.clone(),
2804 profile_revision: SpawnProfileRevision::new("r1").unwrap(),
2805 provider: AgentId::new("codex").unwrap(),
2806 mode: SessionMode::Pty,
2807 terminal_size,
2808 prompt: SpawnPromptMetadata::from_prompt(Some(&prompt)),
2809 bundle_id: None,
2810 bundle: None,
2811 context_id: None,
2812 context: None,
2813 environment_profile: None,
2814 deadline_ms: spec.deadline_ms,
2815 idempotency_key: spec.idempotency_key.clone(),
2816 required_capabilities,
2817 provenance: SpawnResolutionProvenance {
2818 provider: SpawnFieldProvenance::Override,
2819 mode: SpawnFieldProvenance::Override,
2820 terminal_size: SpawnFieldProvenance::Override,
2821 prompt: SpawnFieldProvenance::Override,
2822 bundle_id: SpawnFieldProvenance::Cleared,
2823 context_id: SpawnFieldProvenance::Cleared,
2824 environment_profile_id: SpawnFieldProvenance::Cleared,
2825 },
2826 harness_mcp_proxy: None,
2827 };
2828 let response = |receipt| Ok(NodeResponse::SpawnSpecAccepted { receipt });
2829 let expected = ExpectedSpawnRequest::Spec(spec.clone());
2830 assert!(validate_spawn_spec_response(
2831 Some(&expected),
2832 &response(receipt.clone()),
2833 incarnation_id,
2834 ).is_ok());
2835
2836 let mut mismatches = Vec::new();
2837 let mut changed = receipt.clone();
2838 changed.incarnation_id = NodeIncarnationId::from_bytes([8; 16]);
2839 mismatches.push(changed);
2840 let mut changed = receipt.clone();
2841 changed.target.node_id = NodeId::new("node-b").unwrap();
2842 mismatches.push(changed);
2843 let mut changed = receipt.clone();
2844 changed.profile_id = SpawnProfileId::new("other").unwrap();
2845 mismatches.push(changed);
2846 let mut changed = receipt.clone();
2847 changed.idempotency_key = SpawnIdempotencyKey::new("spawn-2").unwrap();
2848 mismatches.push(changed);
2849 let mut changed = receipt.clone();
2850 changed.deadline_ms = SpawnDeadlineMs::new(4_999).unwrap();
2851 mismatches.push(changed);
2852 let mut changed = receipt.clone();
2853 changed.required_capabilities = SpawnRequiredCapabilities::default();
2854 mismatches.push(changed);
2855 let mut changed = receipt.clone();
2856 changed.session.workspace_id = WorkspaceId::new("other").unwrap();
2857 mismatches.push(changed);
2858 let mut changed = receipt.clone();
2859 changed.provider = AgentId::new("claude").unwrap();
2860 mismatches.push(changed);
2861 let mut changed = receipt.clone();
2862 changed.prompt = SpawnPromptMetadata { present: false, byte_len: 0 };
2863 mismatches.push(changed);
2864 let mut changed = receipt.clone();
2865 changed.environment_profile = Some(
2866 gate4agent_node_protocol::ResolvedEnvironmentProfileReceipt {
2867 profile_id:
2868 gate4agent_node_protocol::SpawnEnvironmentProfileId::new(
2869 "local-default",
2870 )
2871 .unwrap(),
2872 profile_revision:
2873 gate4agent_node_protocol::SpawnEnvironmentProfileRevision::new(
2874 "local-default.r1",
2875 )
2876 .unwrap(),
2877 network_allowlist: None,
2878 browser_profile_id: None,
2879 },
2880 );
2881 mismatches.push(changed);
2882 let mut changed = receipt.clone();
2883 changed.bundle = Some(gate4agent_node_protocol::ResolvedBundleReceipt {
2884 id: gate4agent_node_protocol::SpawnBundleId::new("unexpected-bundle")
2885 .unwrap(),
2886 revision: gate4agent_node_protocol::SpawnBundleRevision::new(
2887 "unexpected-bundle.r1",
2888 )
2889 .unwrap(),
2890 digest: gate4agent_node_protocol::SpawnBundleDigest::new(format!(
2891 "sha256:{}",
2892 "a".repeat(64),
2893 ))
2894 .unwrap(),
2895 });
2896 mismatches.push(changed);
2897 let mut changed = receipt.clone();
2898 changed.context = Some(gate4agent_node_protocol::ResolvedContextPackReceipt {
2899 id: gate4agent_node_protocol::SpawnContextId::new("unexpected-context")
2900 .unwrap(),
2901 digest: gate4agent_node_protocol::SpawnContextDigest::new(format!(
2902 "sha256:{}",
2903 "b".repeat(64),
2904 ))
2905 .unwrap(),
2906 lineage: gate4agent_node_protocol::ContextPackLineageReceipt {
2907 source_node_id: NodeId::new("node-a").unwrap(),
2908 source_session: receipt.session.clone(),
2909 source_provider: AgentId::new("codex").unwrap(),
2910 },
2911 source_message_count: 1,
2912 retained_message_count: 1,
2913 byte_len: 16,
2914 truncated: false,
2915 });
2916 mismatches.push(changed);
2917
2918 for mismatch in mismatches {
2919 assert!(validate_spawn_spec_response(
2920 Some(&expected),
2921 &response(mismatch),
2922 incarnation_id,
2923 ).is_err());
2924 }
2925
2926 let mut environment_spec = spec;
2927 environment_spec.overrides.environment_profile_id = SpawnOverride::Set {
2928 value: gate4agent_node_protocol::SpawnEnvironmentProfileId::new(
2929 "local-default",
2930 )
2931 .unwrap(),
2932 };
2933 let environment_expected = ExpectedSpawnRequest::Spec(environment_spec);
2934 let mut environment_receipt = receipt;
2935 environment_receipt.environment_profile = Some(
2936 gate4agent_node_protocol::ResolvedEnvironmentProfileReceipt {
2937 profile_id:
2938 gate4agent_node_protocol::SpawnEnvironmentProfileId::new(
2939 "local-default",
2940 )
2941 .unwrap(),
2942 profile_revision:
2943 gate4agent_node_protocol::SpawnEnvironmentProfileRevision::new(
2944 "local-default.r1",
2945 )
2946 .unwrap(),
2947 network_allowlist: None,
2948 browser_profile_id: None,
2949 },
2950 );
2951 assert!(validate_spawn_spec_response(
2952 Some(&environment_expected),
2953 &response(environment_receipt.clone()),
2954 incarnation_id,
2955 )
2956 .is_ok());
2957 environment_receipt.environment_profile.as_mut().unwrap().profile_id =
2958 gate4agent_node_protocol::SpawnEnvironmentProfileId::new("other")
2959 .unwrap();
2960 assert!(validate_spawn_spec_response(
2961 Some(&environment_expected),
2962 &response(environment_receipt),
2963 incarnation_id,
2964 )
2965 .is_err());
2966 }
2967
2968 #[test]
2969 fn provider_session_index_correlation_accepts_exact_identity_and_rejects_mismatches() {
2970 let identity = gate4agent_types::ProviderSessionIdentity {
2971 key: gate4agent_types::ProviderSessionKey::SessionId,
2972 id: "native-session-1".to_owned(),
2973 transcript_path: Some(r"C:\provider\sessions\native-session-1.jsonl".to_owned()),
2974 };
2975 let expected = NodeRequest::IndexProviderSession {
2976 workspace_id: WorkspaceId::new("primary").unwrap(),
2977 provider: AgentId::new("codex").unwrap(),
2978 identity: identity.clone(),
2979 display_name: "release shepherd".to_owned(),
2980 };
2981 let NodeRequest::IndexProviderSession { workspace_id, provider, .. } = &expected else {
2982 unreachable!();
2983 };
2984 let record = gate4agent_node_protocol::ManagedSessionRecord {
2985 record_id: SessionRecordId::new("session-001").unwrap(),
2986 display_name: "release shepherd".to_owned(),
2987 provider: provider.clone(),
2988 mode: SessionMode::Pty,
2989 state: gate4agent_node_protocol::ManagedSessionState::Dormant,
2990 workspace_id: workspace_id.clone(),
2991 canonical_root: gate4agent_node_protocol::OpaqueHostPath::utf8(
2992 r"C:\repo".to_owned(),
2993 ).unwrap(),
2994 provider_session: Some(identity),
2995 active_session: None,
2996 environment_profile: None,
2997 bundle: None,
2998 context_id: None,
2999 context: None,
3000 exported_context: None,
3001 task_binding: None,
3002 created_at_unix_ms: 1,
3003 updated_at_unix_ms: 2,
3004 last_error: None,
3005 };
3006 let response = |record| Ok(NodeResponse::ProviderSessionIndexed { record });
3007
3008 assert!(validate_provider_session_index_response(
3009 Some(&expected),
3010 &response(record.clone()),
3011 ).is_ok());
3012 assert!(validate_provider_session_index_response(
3013 None,
3014 &response(record.clone()),
3015 ).is_err());
3016
3017 let mut identity_mismatch = record.clone();
3018 identity_mismatch.provider_session.as_mut().unwrap().id =
3019 "native-session-2".to_owned();
3020 assert!(validate_provider_session_index_response(
3021 Some(&expected),
3022 &response(identity_mismatch),
3023 ).is_err());
3024
3025 let mut workspace_mismatch = record.clone();
3026 workspace_mismatch.workspace_id = WorkspaceId::new("other").unwrap();
3027 assert!(validate_provider_session_index_response(
3028 Some(&expected),
3029 &response(workspace_mismatch),
3030 ).is_err());
3031
3032 let mut provider_mismatch = record;
3033 provider_mismatch.provider = AgentId::new("claude").unwrap();
3034 assert!(validate_provider_session_index_response(
3035 Some(&expected),
3036 &response(provider_mismatch),
3037 ).is_err());
3038 }
3039
3040 #[test]
3041 fn native_session_route_correlation_rejects_mismatches_before_projection() {
3042 let route = gate4agent_node_protocol::NativeSessionCatalogRoute::workspace(
3043 WorkspaceId::new("primary").unwrap(),
3044 AgentId::new("codex").unwrap(),
3045 );
3046 let selection = gate4agent_node_protocol::NativeSessionSelection {
3047 route: route.clone(),
3048 catalog_revision: 7,
3049 recent_cutoff_unix_ms: 70,
3050 selection_id: "selection-7".to_owned(),
3051 };
3052 let catalog = NodeRequest::CatalogNativeSessions {
3053 route: route.clone(),
3054 limit: 10,
3055 };
3056 let catalog_response = Ok(NodeResponse::NativeSessionsCataloged {
3057 route: route.clone(),
3058 entries: Vec::new(),
3059 summary: None,
3060 });
3061 assert!(validate_native_session_response(Some(&catalog), &catalog_response).is_ok());
3062 let mismatched_route = gate4agent_node_protocol::NativeSessionCatalogRoute::workspace(
3063 WorkspaceId::new("other").unwrap(),
3064 AgentId::new("codex").unwrap(),
3065 );
3066 assert!(validate_native_session_response(
3067 Some(&catalog),
3068 &Ok(NodeResponse::NativeSessionsCataloged {
3069 route: mismatched_route,
3070 entries: Vec::new(),
3071 summary: None,
3072 }),
3073 )
3074 .is_err());
3075
3076 let page = NodeRequest::PageNativeSessions {
3077 route: route.clone(),
3078 window: gate4agent_node_protocol::NativeSessionCatalogWindow::Recent,
3079 catalog_revision: 7,
3080 recent_cutoff_unix_ms: 70,
3081 after_selection_id: None,
3082 limit: 10,
3083 };
3084 let page_response = |revision| {
3085 Ok(NodeResponse::NativeSessionsPaged {
3086 route: route.clone(),
3087 page: gate4agent_node_protocol::NativeSessionCatalogPage {
3088 window: gate4agent_node_protocol::NativeSessionCatalogWindow::Recent,
3089 revision,
3090 entries: Vec::new(),
3091 next_after_selection_id: None,
3092 remaining_count: 0,
3093 has_more: false,
3094 },
3095 })
3096 };
3097 assert!(validate_native_session_response(Some(&page), &page_response(7)).is_ok());
3098 assert!(validate_native_session_response(Some(&page), &page_response(8)).is_err());
3099
3100 let preview = NodeRequest::PreviewNativeSession {
3101 selection: selection.clone(),
3102 message_limit: 10,
3103 };
3104 let preview_response = |selection| {
3105 Ok(NodeResponse::NativeSessionPreviewed {
3106 selection,
3107 preview: gate4agent_node_protocol::SessionRecordPreview {
3108 title: None,
3109 modified_at_unix_ms: None,
3110 model: None,
3111 message_count: 0,
3112 message_count_exact: true,
3113 completed_turn_count: None,
3114 total_tokens: None,
3115 truncated: false,
3116 messages: Vec::new(),
3117 },
3118 })
3119 };
3120 assert!(validate_native_session_response(
3121 Some(&preview),
3122 &preview_response(selection.clone()),
3123 )
3124 .is_ok());
3125 let mut mismatched_selection = selection.clone();
3126 mismatched_selection.catalog_revision = 8;
3127 assert!(validate_native_session_response(
3128 Some(&preview),
3129 &preview_response(mismatched_selection),
3130 )
3131 .is_err());
3132
3133 let index = NodeRequest::IndexNativeSession {
3134 selection: selection.clone(),
3135 display_name: "Indexed".to_owned(),
3136 };
3137 let record = gate4agent_node_protocol::ManagedSessionRecord {
3138 record_id: SessionRecordId::new("session-007").unwrap(),
3139 display_name: "Indexed".to_owned(),
3140 provider: route.provider.clone(),
3141 mode: SessionMode::Pty,
3142 state: gate4agent_node_protocol::ManagedSessionState::Dormant,
3143 workspace_id: route.workspace_id.clone().unwrap(),
3144 canonical_root: gate4agent_node_protocol::OpaqueHostPath::utf8(
3145 r"C:\repo".to_owned(),
3146 )
3147 .unwrap(),
3148 provider_session: None,
3149 active_session: None,
3150 environment_profile: None,
3151 bundle: None,
3152 context_id: None,
3153 context: None,
3154 exported_context: None,
3155 task_binding: None,
3156 created_at_unix_ms: 1,
3157 updated_at_unix_ms: 2,
3158 last_error: None,
3159 };
3160 assert!(validate_native_session_response(
3161 Some(&index),
3162 &Ok(NodeResponse::NativeSessionIndexed {
3163 selection: selection.clone(),
3164 record: record.clone(),
3165 }),
3166 )
3167 .is_ok());
3168 assert!(validate_native_session_response(
3169 Some(&index),
3170 &Ok(NodeResponse::ProviderSessionIndexed {
3171 record: record.clone(),
3172 }),
3173 )
3174 .is_err());
3175 let mut wrong_echo = selection.clone();
3176 wrong_echo.catalog_revision = 8;
3177 assert!(validate_native_session_response(
3178 Some(&index),
3179 &Ok(NodeResponse::NativeSessionIndexed {
3180 selection: wrong_echo,
3181 record: record.clone(),
3182 }),
3183 )
3184 .is_err());
3185 let mut wrong_provider = record.clone();
3186 wrong_provider.provider = AgentId::new("claude").unwrap();
3187 assert!(validate_native_session_response(
3188 Some(&index),
3189 &Ok(NodeResponse::NativeSessionIndexed {
3190 selection: selection.clone(),
3191 record: wrong_provider.clone(),
3192 }),
3193 )
3194 .is_err());
3195 let mut wrong_workspace = record.clone();
3196 wrong_workspace.workspace_id = WorkspaceId::new("other").unwrap();
3197 assert!(validate_native_session_response(
3198 Some(&index),
3199 &Ok(NodeResponse::NativeSessionIndexed {
3200 selection: selection.clone(),
3201 record: wrong_workspace,
3202 }),
3203 )
3204 .is_err());
3205
3206 let external = NodeRequest::IndexNativeSession {
3207 selection: gate4agent_node_protocol::NativeSessionSelection {
3208 route: gate4agent_node_protocol::NativeSessionCatalogRoute::unregistered(
3209 AgentId::new("codex").unwrap(),
3210 ),
3211 ..selection
3212 },
3213 display_name: "External".to_owned(),
3214 };
3215 let NodeRequest::IndexNativeSession {
3216 selection: external_selection,
3217 ..
3218 } = &external else {
3219 unreachable!();
3220 };
3221 assert!(validate_native_session_response(
3222 Some(&external),
3223 &Ok(NodeResponse::NativeSessionIndexed {
3224 selection: external_selection.clone(),
3225 record: gate4agent_node_protocol::ManagedSessionRecord {
3226 provider: AgentId::new("codex").unwrap(),
3227 ..wrong_provider
3228 },
3229 }),
3230 )
3231 .is_err());
3232 }
3233
3234 #[test]
3235 fn managed_spawn_receipt_correlation_covers_legacy_and_v2_requests() {
3236 let incarnation_id = NodeIncarnationId::from_bytes([7; 16]);
3237 let spec = SpawnSpec {
3238 target: SpawnTarget {
3239 node_id: NodeId::new("node-a").unwrap(),
3240 workspace_id: WorkspaceId::new("repo").unwrap(),
3241 worktree_id: None,
3242 },
3243 profile_id: SpawnProfileId::new("default").unwrap(),
3244 expected_profile_revision:
3245 SpawnProfileRevision::new("default.r1").unwrap(),
3246 overrides: SpawnOverrides::default(),
3247 deadline_ms: SpawnDeadlineMs::new(5_000).unwrap(),
3248 idempotency_key: SpawnIdempotencyKey::new("managed-1").unwrap(),
3249 required_capabilities: SpawnRequiredCapabilities::default(),
3250 };
3251 let managed = ManagedWorktreeSpawnRequest {
3252 spawn_spec: spec.clone(),
3253 worktree_profile_id: WorktreeProfileId::new("review").unwrap(),
3254 };
3255 let workspace_id = WorkspaceId::new("managed-a").unwrap();
3256 let spawn = ResolvedSpawnReceipt {
3257 incarnation_id,
3258 session: SessionAddress {
3259 workspace_id: workspace_id.clone(),
3260 session: SessionKey {
3261 instance_id: AgentInstanceId(8),
3262 generation: SessionGeneration(1),
3263 },
3264 },
3265 target: SpawnTarget {
3266 node_id: spec.target.node_id.clone(),
3267 workspace_id: spec.target.workspace_id.clone(),
3268 worktree_id: Some(workspace_id.clone()),
3269 },
3270 profile_id: spec.profile_id.clone(),
3271 profile_revision: SpawnProfileRevision::new("default.r1").unwrap(),
3272 provider: AgentId::new("claude").unwrap(),
3273 mode: SessionMode::Pty,
3274 terminal_size: TerminalSize { rows: 24, columns: 80 },
3275 prompt: SpawnPromptMetadata { present: false, byte_len: 0 },
3276 bundle_id: None,
3277 bundle: None,
3278 context_id: None,
3279 context: None,
3280 environment_profile: None,
3281 deadline_ms: spec.deadline_ms,
3282 idempotency_key: spec.idempotency_key.clone(),
3283 required_capabilities: SpawnRequiredCapabilities::default(),
3284 provenance: SpawnResolutionProvenance {
3285 provider: SpawnFieldProvenance::Profile,
3286 mode: SpawnFieldProvenance::Profile,
3287 terminal_size: SpawnFieldProvenance::Profile,
3288 prompt: SpawnFieldProvenance::Profile,
3289 bundle_id: SpawnFieldProvenance::Profile,
3290 context_id: SpawnFieldProvenance::Profile,
3291 environment_profile_id: SpawnFieldProvenance::Profile,
3292 },
3293 harness_mcp_proxy: None,
3294 };
3295 let receipt = ManagedWorktreeSpawnReceipt {
3296 spawn,
3297 lease: managed_lease(
3298 "lease-a",
3299 "managed-a",
3300 ManagedWorktreeLeaseState::InUse,
3301 ),
3302 };
3303 let expected = ExpectedSpawnRequest::ManagedV1(managed.clone());
3304 let response = |receipt| Ok(NodeResponse::ManagedWorktreeSpawnAccepted { receipt });
3305 assert!(validate_spawn_spec_response(
3306 Some(&expected),
3307 &response(receipt.clone()),
3308 incarnation_id,
3309 )
3310 .is_ok());
3311
3312 let mut mismatches = Vec::new();
3313 let mut changed = receipt.clone();
3314 changed.lease.state = ManagedWorktreeLeaseState::Ready;
3315 mismatches.push(changed);
3316 let mut changed = receipt.clone();
3317 changed.lease.cleanup_failure = Some(ManagedWorktreeCleanupFailure::Busy);
3318 mismatches.push(changed);
3319 let mut changed = receipt.clone();
3320 changed.lease.active_session_count = 0;
3321 mismatches.push(changed);
3322 let mut changed = receipt.clone();
3323 changed.lease.profile_id = WorktreeProfileId::new("other").unwrap();
3324 mismatches.push(changed);
3325 let mut changed = receipt.clone();
3326 changed.lease.source_workspace_id = WorkspaceId::new("other").unwrap();
3327 mismatches.push(changed);
3328
3329 for mismatch in mismatches {
3330 assert!(validate_spawn_spec_response(
3331 Some(&expected),
3332 &response(mismatch),
3333 incarnation_id,
3334 )
3335 .is_err());
3336 }
3337
3338 let mut legacy_revision = receipt.clone();
3339 legacy_revision.lease.profile_revision =
3340 WorktreeProfileRevision::new("review.r2").unwrap();
3341 assert!(validate_spawn_spec_response(
3342 Some(&expected),
3343 &response(legacy_revision),
3344 incarnation_id,
3345 )
3346 .is_ok());
3347
3348 let v2_request = ManagedWorktreeSpawnRequestV2 {
3349 spawn_spec: managed.spawn_spec,
3350 worktree_profile_id: managed.worktree_profile_id,
3351 expected_profile_revision: WorktreeProfileRevision::new("review.r1").unwrap(),
3352 };
3353 let routed = NodeRequest::SpawnManagedWorktreeV2 {
3354 request: v2_request.clone(),
3355 };
3356 let captured_expected = expected_spawn_request(&routed).unwrap();
3357 let ExpectedSpawnRequest::ManagedV2(captured_request) = &captured_expected else {
3358 panic!("V2 managed spawn request was not captured separately");
3359 };
3360 assert_eq!(captured_request, &v2_request);
3361 assert!(validate_spawn_spec_response(
3362 Some(&captured_expected),
3363 &response(receipt.clone()),
3364 incarnation_id,
3365 )
3366 .is_ok());
3367
3368 let mut wrong_revision = receipt.clone();
3369 wrong_revision.lease.profile_revision =
3370 WorktreeProfileRevision::new("review.r2").unwrap();
3371 assert!(validate_spawn_spec_response(
3372 Some(&ExpectedSpawnRequest::ManagedV2(v2_request.clone())),
3373 &response(wrong_revision),
3374 incarnation_id,
3375 )
3376 .is_err());
3377 assert!(validate_spawn_spec_response(
3378 Some(&ExpectedSpawnRequest::ManagedV2(v2_request)),
3379 &Ok(NodeResponse::SpawnSpecAccepted {
3380 receipt: receipt.spawn,
3381 }),
3382 incarnation_id,
3383 )
3384 .is_err());
3385 }
3386
3387 #[test]
3388 fn windows_runtime_default_control_endpoint_is_exact_valid_and_distinct_from_a_node() {
3389 assert_eq!(DEFAULT_C2_CONTROL_ENDPOINT, r"\\.\pipe\gate4agent-c2");
3390 validate_control_endpoint(DEFAULT_C2_CONTROL_ENDPOINT).unwrap();
3391 let node = C2NodeConfig::new(
3392 NodeId::new("node-a").unwrap(),
3393 r"\\.\pipe\gate4agent-node",
3394 "safe-token",
3395 )
3396 .unwrap();
3397 let config = C2Config::new(
3398 "127.0.0.1:0".parse().unwrap(),
3399 "safe-token",
3400 vec![node],
3401 )
3402 .unwrap();
3403 assert_eq!(config.control_endpoint, DEFAULT_C2_CONTROL_ENDPOINT);
3404 assert!(!config.nodes[0]
3405 .endpoint
3406 .eq_ignore_ascii_case(&config.control_endpoint));
3407
3408 let conflicting_node = C2NodeConfig::new(
3409 NodeId::new("node-b").unwrap(),
3410 DEFAULT_C2_CONTROL_ENDPOINT,
3411 "safe-token",
3412 )
3413 .unwrap();
3414 assert!(matches!(
3415 C2Config::new(
3416 "127.0.0.1:0".parse().unwrap(),
3417 "safe-token",
3418 vec![conflicting_node],
3419 ),
3420 Err(C2ConfigError::ControlEndpointConflict)
3421 ));
3422 }
3423
3424 #[test]
3425 fn terminal_only_sequence_holes_do_not_mark_c2_partial() {
3426 use gate4agent_node_protocol::{NodeEvent, WorkspaceId};
3427 let event = |sequence| NodeEventEnvelope { sequence, event: NodeEvent::WorkspaceRemoved { workspace_id: WorkspaceId::new("work").unwrap() } };
3428 assert_eq!(validate_events(4, 7, 1, &[event(6)]), Vec::<GapKind>::new());
3429 assert_eq!(validate_events(4, 7, 1, &[]), Vec::<GapKind>::new());
3430 assert_eq!(validate_events(4, 7, 1, &[event(7), event(6)]), vec![GapKind::NonContiguousEvents]);
3431 assert_eq!(validate_events(7, 6, 1, &[]), vec![GapKind::CursorRegression]);
3432 assert_eq!(validate_resync(4, 6, 4, 1, &[]), vec![GapKind::CursorRegression]);
3433 }
3434
3435 #[test]
3436 fn real_durable_eviction_marks_c2_history_evicted() {
3437 use gate4agent_node_protocol::{NodeEvent, WorkspaceId};
3438 let event = |sequence| NodeEventEnvelope {
3439 sequence,
3440 event: NodeEvent::WorkspaceRemoved {
3441 workspace_id: WorkspaceId::new("work").unwrap(),
3442 },
3443 };
3444 assert_eq!(
3445 validate_events(4, 7, 6, &[event(6)]),
3446 vec![GapKind::HistoryEvicted],
3447 );
3448 assert_eq!(validate_events(5, 7, 6, &[event(6)]), Vec::<GapKind>::new());
3449 }
3450
3451 #[test]
3452 fn live_event_gap_preserves_cursor_and_resync_rules() {
3453 use gate4agent_node_protocol::{NodeEvent, WorkspaceId};
3454 let event = |sequence, event| NodeEventEnvelope { sequence, event };
3455 let removed = || NodeEvent::WorkspaceRemoved {
3456 workspace_id: WorkspaceId::new("work").unwrap(),
3457 };
3458
3459 assert_eq!(live_event_gap(4, &event(5, removed())), None);
3460 assert_eq!(
3461 live_event_gap(4, &event(4, removed())),
3462 Some(GapKind::CursorRegression),
3463 );
3464 assert_eq!(
3465 live_event_gap(4, &event(7, removed())),
3466 Some(GapKind::NonContiguousEvents),
3467 );
3468 assert_eq!(
3469 live_event_gap(4, &event(5, NodeEvent::ResyncRequired {
3470 oldest_available_sequence: 3,
3471 })),
3472 Some(GapKind::HistoryEvicted),
3473 );
3474 }
3475
3476 #[test]
3477 fn harness_mcp_transient_live_and_pending_projection_preserves_cursor_without_replay() {
3478 let node_id = NodeId::new("node-a").unwrap();
3479 let cursor = NodeCursor {
3480 incarnation_id: NodeIncarnationId::from_bytes([7; 16]),
3481 sequence: 41,
3482 };
3483 let envelope = NodeEventEnvelope {
3484 sequence: 0,
3485 event: NodeEvent::HarnessMcpReadCall {
3486 reservation_id: HarnessMcpReservationId::new(
3487 format!("hmcpres_{}", "a".repeat(24)),
3488 ).unwrap(),
3489 activation_digest: HarnessMcpActivationDigest::new(
3490 format!("sha256:{}", "b".repeat(64)),
3491 ).unwrap(),
3492 record_id: SessionRecordId::new("session-001").unwrap(),
3493 session: SessionAddress {
3494 workspace_id: WorkspaceId::new("repo").unwrap(),
3495 session: SessionKey {
3496 instance_id: AgentInstanceId(8),
3497 generation: SessionGeneration(1),
3498 },
3499 },
3500 call_id: HarnessMcpCallId::new(
3501 format!("hmcpcall_{}", "c".repeat(24)),
3502 ).unwrap(),
3503 request: HarnessMcpOpaquePayloadV1 {
3504 content_type: HarnessMcpContentTypeV1::HarnessReadRequestJsonV1,
3505 body: br#"{"kind":"context-get"}"#.to_vec(),
3506 },
3507 deadline_unix_ms: u64::MAX,
3508 },
3509 };
3510
3511 let live = routed_transient_node_event(&node_id, cursor, &envelope).unwrap();
3512 let pending = routed_transient_node_event(&node_id, cursor, &envelope).unwrap();
3513 assert_eq!(live, pending);
3514 assert_eq!(live.cursor, cursor);
3515 assert!(matches!(live.event, C2NodeEvent::HarnessMcpReadCall { .. }));
3516 assert!(routed_recovered_node_event(
3517 &node_id,
3518 cursor.incarnation_id,
3519 &envelope,
3520 ).is_none());
3521 assert_eq!(cursor.sequence, 41);
3522 }
3523
3524 fn agent_stream_test_address() -> SessionAddress {
3525 SessionAddress {
3526 workspace_id: WorkspaceId::new("repo").unwrap(),
3527 session: SessionKey {
3528 instance_id: AgentInstanceId(8),
3529 generation: SessionGeneration(1),
3530 },
3531 }
3532 }
3533
3534 fn agent_stream_test_envelope(sequence: u64, source_sequence: u64) -> NodeEventEnvelope {
3535 NodeEventEnvelope {
3536 sequence,
3537 event: NodeEvent::AgentStream {
3538 address: agent_stream_test_address(),
3539 chunk: AgentStreamChunkV1 {
3540 source_sequence,
3541 kind: AgentStreamChunkKindV1::Text {
3542 text: format!("chunk-{source_sequence}"),
3543 is_delta: true,
3544 },
3545 },
3546 },
3547 }
3548 }
3549
3550 #[test]
3551 fn agent_stream_chunk_publishes_unconditionally_and_only_advances_cursor_forward() {
3552 let node_id = NodeId::new("node-a").unwrap();
3553 let mut cursor = NodeCursor {
3554 incarnation_id: NodeIncarnationId::from_bytes([7; 16]),
3555 sequence: 674,
3556 };
3557
3558 let control = NodeEventEnvelope {
3560 sequence: 675,
3561 event: NodeEvent::WorkspaceRemoved {
3562 workspace_id: WorkspaceId::new("work").unwrap(),
3563 },
3564 };
3565 assert!(route_agent_stream_event(&node_id, &mut cursor, &control).is_none());
3566 assert_eq!(cursor.sequence, 674);
3567
3568 let forward = agent_stream_test_envelope(675, 22);
3573 let routed = route_agent_stream_event(&node_id, &mut cursor, &forward).unwrap();
3574 assert_eq!(routed.cursor.sequence, 674);
3575 assert!(matches!(
3576 &routed.event,
3577 C2NodeEvent::AgentStream { chunk, .. } if chunk.source_sequence == 22
3578 ));
3579 assert_eq!(cursor.sequence, 675);
3580
3581 cursor.sequence = 711;
3588 let late = agent_stream_test_envelope(677, 23);
3589 let routed = route_agent_stream_event(&node_id, &mut cursor, &late).unwrap();
3590 assert_eq!(routed.cursor.sequence, 711);
3591 assert!(matches!(
3592 &routed.event,
3593 C2NodeEvent::AgentStream { chunk, .. } if chunk.source_sequence == 23
3594 ));
3595 assert_eq!(cursor.sequence, 711, "a late chunk must never rewind the cursor");
3596 }
3597
3598 #[test]
3599 fn agent_stream_burst_survives_a_durable_cursor_that_already_ran_past_it() {
3600 let node_id = NodeId::new("node-a").unwrap();
3609 let mut cursor = NodeCursor {
3610 incarnation_id: NodeIncarnationId::from_bytes([9; 16]),
3611 sequence: 0,
3612 };
3613 let chunk_count = 200_u64;
3614
3615 for turn in 1..=chunk_count {
3616 cursor.sequence = turn * 2;
3622 }
3623
3624 let mut recovered = Vec::new();
3625 for turn in 1..=chunk_count {
3626 let envelope = agent_stream_test_envelope(turn * 2 - 1, turn);
3627 let routed = route_agent_stream_event(&node_id, &mut cursor, &envelope)
3628 .expect("every chunk in the burst must be published, however far the durable cursor already ran past its sequence");
3629 recovered.push(routed);
3630 }
3631
3632 assert_eq!(recovered.len(), chunk_count as usize);
3633 for (turn, routed) in (1..=chunk_count).zip(recovered.iter()) {
3634 assert!(matches!(
3635 &routed.event,
3636 C2NodeEvent::AgentStream { chunk, .. } if chunk.source_sequence == turn
3637 ));
3638 }
3639 assert_eq!(
3640 cursor.sequence,
3641 chunk_count * 2,
3642 "a whole burst of stale chunks must never rewind the cursor the durable side already advanced",
3643 );
3644 }
3645
3646 #[test]
3647 fn config_rejects_header_injection_duplicate_nodes_and_non_loopback() {
3648 let id = NodeId::new("node-a").unwrap();
3649 assert!(matches!(C2NodeConfig::new(id.clone(), r"\\.\pipe\a", "bad\r\ntoken"), Err(C2ConfigError::InvalidToken)));
3650 let node = C2NodeConfig::new(id, r"\\.\pipe\a", "safe-token").unwrap();
3651 assert!(matches!(C2Config::new("0.0.0.0:0".parse().unwrap(), "safe", vec![node.clone()]), Err(C2ConfigError::NonLoopback(_))));
3652 assert!(matches!(C2Config::new("127.0.0.1:0".parse().unwrap(), "safe", vec![node.clone(), node]), Err(C2ConfigError::DuplicateNode)));
3653 let first = C2NodeConfig::new(NodeId::new("node-a").unwrap(), r"\\.\pipe\same", "safe").unwrap();
3654 let second = C2NodeConfig::new(NodeId::new("node-b").unwrap(), r"\\.\pipe\same", "safe").unwrap();
3655 assert!(matches!(C2Config::new("127.0.0.1:0".parse().unwrap(), "safe", vec![first, second]), Err(C2ConfigError::DuplicateEndpoint)));
3656 let oversized = format!(r"\\.\pipe\{}", "x".repeat(MAX_C2_ENDPOINT_BYTES));
3657 assert!(matches!(C2NodeConfig::new(NodeId::new("node-c").unwrap(), oversized, "safe"), Err(C2ConfigError::InvalidEndpoint(_))));
3658 }
3659
3660 #[test]
3661 fn durable_session_mutations_require_controller_and_use_bounded_deadlines() {
3662 let record_id = SessionRecordId::new("session-001").unwrap();
3663 let rename = NodeRequest::RenameSessionRecord {
3664 record_id: record_id.clone(),
3665 display_name: "release shepherd".to_owned(),
3666 };
3667 let resume = NodeRequest::ResumeSessionRecord {
3668 record_id: record_id.clone(),
3669 terminal_size: TerminalSize { rows: 40, columns: 120 },
3670 initial_prompt: None,
3671 };
3672 let forget = NodeRequest::ForgetSessionRecord { record_id };
3673
3674 assert!(!is_read_only_request(&rename));
3675 assert!(!is_read_only_request(&resume));
3676 assert!(!is_read_only_request(&forget));
3677 assert_eq!(node_request_deadline(&rename), Duration::from_secs(5));
3678 assert_eq!(node_request_deadline(&resume), Duration::from_secs(35));
3679 assert!(node_request_deadline(&resume) > MANAGED_RESUME_SETTLE_DEADLINE);
3680 assert_eq!(
3681 node_request_deadline(&resume) - MANAGED_RESUME_SETTLE_DEADLINE,
3682 NODE_REQUEST_IO_HEADROOM,
3683 );
3684 let started = Instant::now();
3685 let relay_deadline = relay_request_deadline(&resume, started).unwrap();
3686 assert_eq!(
3687 request_budget(&resume, Some(relay_deadline), started),
3688 Duration::from_secs(35),
3689 );
3690 assert_eq!(
3691 request_budget(
3692 &resume,
3693 Some(relay_deadline),
3694 started + Duration::from_secs(5),
3695 ),
3696 MANAGED_RESUME_SETTLE_DEADLINE,
3697 );
3698 assert_eq!(
3699 request_budget(
3700 &resume,
3701 Some(relay_deadline),
3702 started + Duration::from_secs(35),
3703 ),
3704 Duration::ZERO,
3705 );
3706 assert_eq!(node_request_deadline(&forget), Duration::from_secs(5));
3707 }
3708
3709 #[test]
3710 fn workspace_file_reads_are_read_only_with_five_second_deadline() {
3711 let request = NodeRequest::ReadWorkspaceFile {
3712 workspace_id: gate4agent_node_protocol::WorkspaceId::new("primary").unwrap(),
3713 path: gate4agent_node_protocol::RepositoryPath::utf8(
3714 "src/lib.rs".to_owned(),
3715 ).unwrap(),
3716 };
3717
3718 assert!(is_read_only_request(&request));
3719 assert_eq!(node_request_deadline(&request), Duration::from_secs(5));
3720 assert!(relay_request_deadline(&request, Instant::now()).is_none());
3721
3722 }
3723
3724 #[test]
3725 fn workspace_entry_create_relay_has_headroom_and_preserves_semantic_timeout() {
3726 let workspace_id = gate4agent_node_protocol::WorkspaceId::new("primary").unwrap();
3727 let file_path = gate4agent_node_protocol::RepositoryPath::utf8(
3728 "src/new.rs".to_owned(),
3729 ).unwrap();
3730 let create_file = NodeRequest::CreateWorkspaceFile {
3731 workspace_id: workspace_id.clone(),
3732 path: file_path.clone(),
3733 };
3734 assert!(!is_read_only_request(&create_file));
3735 assert_eq!(
3736 node_request_deadline(&create_file),
3737 WORKSPACE_ENTRY_CREATE_RELAY_DEADLINE,
3738 );
3739 assert!(
3740 node_request_deadline(&create_file)
3741 > WORKSPACE_ENTRY_CREATE_NODE_SEMANTIC_DEADLINE,
3742 );
3743 assert!(relay_request_deadline(&create_file, Instant::now()).is_none());
3744
3745 let propagated = relay_node_failure(&NodeClientError::Node(
3746 gate4agent_node_protocol::NodeFailure {
3747 code: NodeFailureCode::RepositoryEntryCreateTimedOut,
3748 message: "private node timeout detail".to_owned(),
3749 },
3750 )).unwrap();
3751 assert_eq!(
3752 propagated.code,
3753 NodeFailureCode::RepositoryEntryCreateTimedOut,
3754 );
3755 assert_eq!(propagated.message, "repository entry creation timed out");
3756
3757 let created_file = gate4agent_node_protocol::WorkspaceFileRead {
3758 workspace_id: workspace_id.clone(),
3759 path: file_path,
3760 content: gate4agent_node_protocol::WorkspaceFileContent::Utf8 {
3761 text: String::new(),
3762 byte_len: 0,
3763 },
3764 revision: Some(
3765 gate4agent_node_protocol::WorkspaceFileRevision::new(
3766 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3767 .to_owned(),
3768 )
3769 .unwrap(),
3770 ),
3771 };
3772 assert!(validate_workspace_content_response(
3773 Some(&create_file),
3774 &Ok(NodeResponse::WorkspaceFileCreated {
3775 file: created_file.clone(),
3776 }),
3777 ).is_ok());
3778 let mut wrong_content = created_file;
3779 wrong_content.content = gate4agent_node_protocol::WorkspaceFileContent::Utf8 {
3780 text: "unexpected".to_owned(),
3781 byte_len: 10,
3782 };
3783 assert!(validate_workspace_content_response(
3784 Some(&create_file),
3785 &Ok(NodeResponse::WorkspaceFileCreated { file: wrong_content }),
3786 ).is_err());
3787
3788 let directory_path = gate4agent_node_protocol::RepositoryPath::utf8(
3789 "src/new".to_owned(),
3790 ).unwrap();
3791 let create_directory = NodeRequest::CreateWorkspaceDirectory {
3792 workspace_id: workspace_id.clone(),
3793 path: directory_path.clone(),
3794 };
3795 assert!(!is_read_only_request(&create_directory));
3796 assert_eq!(
3797 node_request_deadline(&create_directory),
3798 WORKSPACE_ENTRY_CREATE_RELAY_DEADLINE,
3799 );
3800 assert!(validate_workspace_content_response(
3801 Some(&create_directory),
3802 &Ok(NodeResponse::WorkspaceDirectoryCreated {
3803 workspace_id,
3804 entry: gate4agent_node_protocol::WorkspaceEntry {
3805 relative_path: directory_path,
3806 kind: gate4agent_node_protocol::WorkspaceEntryKind::Directory,
3807 },
3808 }),
3809 ).is_ok());
3810 assert!(validate_workspace_content_response(
3811 Some(&create_directory),
3812 &Ok(NodeResponse::Accepted),
3813 ).is_err());
3814 }
3815
3816 #[test]
3817 fn native_session_catalog_is_lease_free_read_only() {
3818 let route = gate4agent_node_protocol::NativeSessionCatalogRoute::workspace(
3819 gate4agent_node_protocol::WorkspaceId::new("primary").unwrap(),
3820 gate4agent_types::AgentId::new("codex").unwrap(),
3821 );
3822 let request = NodeRequest::CatalogNativeSessions {
3823 route: route.clone(),
3824 limit: 8,
3825 };
3826 assert!(is_read_only_request(&request));
3827 assert_eq!(
3828 node_request_deadline(&request),
3829 NATIVE_SESSION_REQUEST_DEADLINE,
3830 );
3831 assert!(relay_request_deadline(&request, Instant::now()).is_none());
3832
3833 let page = NodeRequest::PageNativeSessions {
3834 route: route.clone(),
3835 window: gate4agent_types::NativeSessionCatalogWindow::Older,
3836 catalog_revision: 7,
3837 recent_cutoff_unix_ms: 8,
3838 after_selection_id: Some("hist_selection_1".to_owned()),
3839 limit: 8,
3840 };
3841 assert!(is_read_only_request(&page));
3842 assert_eq!(node_request_deadline(&page), NATIVE_SESSION_REQUEST_DEADLINE);
3843 assert!(relay_request_deadline(&page, Instant::now()).is_none());
3844
3845 let preview = NodeRequest::PreviewNativeSession {
3846 selection: gate4agent_node_protocol::NativeSessionSelection {
3847 route,
3848 catalog_revision: 7,
3849 recent_cutoff_unix_ms: 8,
3850 selection_id: "hist_selection_1".to_owned(),
3851 },
3852 message_limit: 12,
3853 };
3854 assert!(is_read_only_request(&preview));
3855 assert_eq!(
3856 node_request_deadline(&preview),
3857 NATIVE_SESSION_REQUEST_DEADLINE,
3858 );
3859 assert!(relay_request_deadline(&preview, Instant::now()).is_none());
3860
3861 let record_preview = NodeRequest::PreviewSessionRecord {
3862 record_id: gate4agent_node_protocol::SessionRecordId::new("record-1").unwrap(),
3863 message_limit: 12,
3864 };
3865 assert!(is_read_only_request(&record_preview));
3866 assert_eq!(
3867 node_request_deadline(&record_preview),
3868 NATIVE_SESSION_REQUEST_DEADLINE,
3869 );
3870 assert!(relay_request_deadline(&record_preview, Instant::now()).is_none());
3871 }
3872
3873 #[test]
3874 fn standalone_workspace_creation_is_controller_mutation_with_worktree_deadline() {
3875 let request = NodeRequest::CreateStandaloneWorkspace {
3876 workspace_id: gate4agent_node_protocol::WorkspaceId::new("standalone").unwrap(),
3877 root: gate4agent_node_protocol::OpaqueHostPath::utf8(
3878 r"C:\standalone".to_owned(),
3879 ).unwrap(),
3880 initial_branch: Some("main".to_owned()),
3881 };
3882
3883 assert!(!is_read_only_request(&request));
3884 assert_eq!(node_request_deadline(&request), Duration::from_secs(240));
3885 assert!(relay_request_deadline(&request, Instant::now()).is_none());
3886 }
3887
3888 #[test]
3889 fn unsupported_node_capability_is_correlated_without_offline_classification() {
3890 let error = NodeClientError::UnsupportedCapability(
3891 "workspace-file-read-v1-private-detail".to_owned(),
3892 );
3893 let failure = relay_node_failure(&error)
3894 .expect("unsupported node capability must remain an in-band node failure");
3895
3896 assert_eq!(failure.code, NodeFailureCode::UnsupportedCapability);
3897 assert_eq!(failure.message, "required capability unavailable");
3898 assert!(!failure.message.contains("private-detail"));
3899 assert!(relay_node_failure(&NodeClientError::Io(io::Error::new(
3900 io::ErrorKind::BrokenPipe,
3901 "transport closed",
3902 ))).is_none());
3903 }
3904
3905 async fn raw_request(request: Vec<u8>, status: StatusResponse) -> Vec<u8> {
3906 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3907 let address = listener.local_addr().unwrap();
3908 let (_status_tx, status_rx) = watch::channel(Arc::new(status));
3909 let server = tokio::spawn(async move {
3910 let (stream, _) = listener.accept().await.unwrap();
3911 serve_http(stream, "api-token", Duration::from_secs(1), &status_rx).await.unwrap();
3912 });
3913 let mut stream = TcpStream::connect(address).await.unwrap();
3914 stream.write_all(&request).await.unwrap();
3915 let mut response = Vec::new();
3916 stream.read_to_end(&mut response).await.unwrap();
3917 server.await.unwrap();
3918 response
3919 }
3920
3921 fn empty_status(ready: bool) -> StatusResponse {
3922 StatusResponse { api_version: C2_API_VERSION, ready, observed_at_unix_ms: 0, nodes: BTreeMap::new() }
3923 }
3924
3925 #[tokio::test]
3926 async fn http_api_enforces_initializing_auth_method_path_and_header_bounds() {
3927 let initializing = raw_request(b"GET /ready HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), empty_status(false)).await;
3928 assert!(initializing.starts_with(b"HTTP/1.1 503 Service Unavailable\r\n"));
3929 assert!(String::from_utf8_lossy(&initializing).contains("\"ready\":false"));
3930
3931 let unauthorized = raw_request(b"GET /status HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), empty_status(true)).await;
3932 assert!(unauthorized.starts_with(b"HTTP/1.1 401 Unauthorized\r\n"));
3933 let method = raw_request(b"POST /health HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), empty_status(true)).await;
3934 assert!(method.starts_with(b"HTTP/1.1 405 Method Not Allowed\r\n"));
3935 let missing = raw_request(b"GET /missing HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), empty_status(true)).await;
3936 assert!(missing.starts_with(b"HTTP/1.1 404 Not Found\r\n"));
3937
3938 let mut oversized = b"GET /health HTTP/1.1\r\nX-Fill: ".to_vec();
3939 oversized.extend(std::iter::repeat(b'x').take(HEADER_LIMIT_BYTES));
3940 oversized.extend_from_slice(b"\r\n\r\n");
3941 let rejected = raw_request(oversized, empty_status(true)).await;
3942 assert!(rejected.starts_with(b"HTTP/1.1 413 Payload Too Large\r\n"));
3943 }
3944
3945 #[tokio::test]
3946 async fn inventory_state_transitions_offline_to_stale_parked_and_recovers() {
3947 let node_id = NodeId::new("node-a").unwrap();
3948 let mut nodes = BTreeMap::new();
3949 nodes.insert(node_id.clone(), ObservedNode {
3950 endpoint: r"\\.\pipe\a".to_owned(), transport_label: "windows-named-pipe".to_owned(),
3951 transport: NodeTransportState::Offline, freshness: NodeFreshness::Unavailable,
3952 cursor: None, inventory: None, last_attempt_unix_ms: None, last_success_unix_ms: None,
3953 consecutive_failures: 0, last_error: None, gaps: Vec::new(), gaps_truncated: 0,
3954 });
3955 let initial = Arc::new(StatusResponse { api_version: C2_API_VERSION, ready: false, observed_at_unix_ms: unix_ms(), nodes });
3956 let (status_tx, mut status_rx) = watch::channel(initial);
3957 let (ingress_tx, ingress_rx) = mpsc::channel(4);
3958 let (shutdown_tx, shutdown_rx) = watch::channel(false);
3959 let owner = tokio::spawn(inventory_owner(1, Duration::from_millis(10), ingress_rx, status_tx, shutdown_rx));
3960 let snapshot = NodeSnapshot {
3961 node_id: node_id.clone(),
3962 enabled_providers: Vec::new(),
3963 provider_runtime_statuses: crate::protocol::ProviderRuntimeStatuses::default(),
3964 workspaces: Vec::new(),
3965 session_records: Vec::new(),
3966 managed_worktrees: Vec::new(),
3967 launch_inventory: None,
3968 agent_progress: Vec::new(),
3969 };
3970 let cursor = NodeCursor { incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([1; 16]), sequence: 0 };
3971 let old_manifest = ProviderContractManifest {
3972 provider_contracts: vec![crate::protocol::ProviderContractSupport {
3973 provider: agent("codex"),
3974 revision: crate::protocol::ProviderContractRevision::new("old-contract").unwrap(),
3975 }],
3976 provider_adapter_contracts: vec![crate::protocol::ProviderAdapterContractSupport {
3977 provider: agent("codex"),
3978 family: crate::protocol::AdapterFamily::PtySemantic,
3979 adapter_id: crate::protocol::AdapterId::new("codex").unwrap(),
3980 revision: crate::protocol::AdapterContractRevision::new("old-adapter").unwrap(),
3981 }],
3982 };
3983 ingress_tx.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result: AttemptResult::Connected {
3984 cursor,
3985 snapshot: snapshot.clone(),
3986 gaps: Vec::new(),
3987 provider_contract_manifest: old_manifest,
3988 } }).await.unwrap();
3989 status_rx.changed().await.unwrap();
3990 assert_eq!(status_rx.borrow().nodes[&node_id].freshness, NodeFreshness::Fresh);
3991 assert_eq!(
3992 status_rx.borrow().nodes[&node_id].inventory.as_ref().unwrap()
3993 .provider_contracts[0].revision.as_str(),
3994 "old-contract",
3995 );
3996
3997 let failure = || AttemptResult::Failure { error: SanitizedError { category: C2ErrorCategory::Transport, message: "node transport unavailable".to_owned() }, hard: false };
3998 ingress_tx.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result: failure() }).await.unwrap();
3999 status_rx.changed().await.unwrap();
4000 assert_eq!(status_rx.borrow().nodes[&node_id].transport, NodeTransportState::Offline);
4001 timeout(Duration::from_secs(1), async {
4002 loop {
4003 status_rx.changed().await.unwrap();
4004 if status_rx.borrow().nodes[&node_id].freshness == NodeFreshness::Stale { break; }
4005 }
4006 }).await.unwrap();
4007 for _ in 0..4 {
4008 ingress_tx.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result: failure() }).await.unwrap();
4009 status_rx.changed().await.unwrap();
4010 }
4011 assert_eq!(status_rx.borrow().nodes[&node_id].transport, NodeTransportState::Parked);
4012 let replacement_manifest = ProviderContractManifest {
4013 provider_contracts: vec![crate::protocol::ProviderContractSupport {
4014 provider: agent("claude"),
4015 revision: crate::protocol::ProviderContractRevision::new("new-contract").unwrap(),
4016 }],
4017 provider_adapter_contracts: Vec::new(),
4018 };
4019 let replacement_cursor = NodeCursor {
4020 incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([2; 16]),
4021 sequence: 0,
4022 };
4023 ingress_tx.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result: AttemptResult::Connected {
4024 cursor: replacement_cursor,
4025 snapshot,
4026 gaps: Vec::new(),
4027 provider_contract_manifest: replacement_manifest,
4028 } }).await.unwrap();
4029 status_rx.changed().await.unwrap();
4030 assert_eq!(status_rx.borrow().nodes[&node_id].transport, NodeTransportState::Online);
4031 assert_eq!(status_rx.borrow().nodes[&node_id].freshness, NodeFreshness::Fresh);
4032 let recovered_inventory = status_rx.borrow().nodes[&node_id].inventory.as_ref().unwrap().clone();
4033 assert_eq!(recovered_inventory.provider_contracts.len(), 1);
4034 assert_eq!(recovered_inventory.provider_contracts[0].provider, agent("claude"));
4035 assert_eq!(recovered_inventory.provider_contracts[0].revision.as_str(), "new-contract");
4036 assert!(recovered_inventory.provider_adapter_contracts.is_empty());
4037 ingress_tx.send(Attempt {
4038 node_id: node_id.clone(),
4039 at_unix_ms: unix_ms(),
4040 result: AttemptResult::Connected {
4041 cursor: NodeCursor {
4042 incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([3; 16]),
4043 sequence: 0,
4044 },
4045 snapshot: NodeSnapshot {
4046 node_id: node_id.clone(),
4047 enabled_providers: Vec::new(),
4048 provider_runtime_statuses: crate::protocol::ProviderRuntimeStatuses::default(),
4049 workspaces: Vec::new(),
4050 session_records: Vec::new(),
4051 managed_worktrees: Vec::new(),
4052 launch_inventory: None,
4053 agent_progress: Vec::new(),
4054 },
4055 gaps: Vec::new(),
4056 provider_contract_manifest: ProviderContractManifest::default(),
4057 },
4058 }).await.unwrap();
4059 status_rx.changed().await.unwrap();
4060 let unpublished = status_rx.borrow().nodes[&node_id].inventory.as_ref().unwrap().clone();
4061 assert!(unpublished.provider_contracts.is_empty());
4062 assert!(unpublished.provider_adapter_contracts.is_empty());
4063 shutdown_tx.send(true).unwrap();
4064 owner.await.unwrap().unwrap();
4065 }
4066
4067 fn runtime_statuses(
4068 provider: gate4agent_node_protocol::AgentId,
4069 version: &str,
4070 ) -> crate::protocol::ProviderRuntimeStatuses {
4071 crate::protocol::ProviderRuntimeStatuses::new([
4072 crate::protocol::ProviderRuntimeStatus::raw_passthrough(
4073 provider,
4074 Some(crate::protocol::ProviderRuntimeVersion::new(version).unwrap()),
4075 ),
4076 ])
4077 .unwrap()
4078 }
4079
4080 async fn runtime_inventory_owner() -> (
4081 NodeId,
4082 mpsc::Sender<Attempt>,
4083 watch::Receiver<Arc<StatusResponse>>,
4084 watch::Sender<bool>,
4085 tokio::task::JoinHandle<io::Result<()>>,
4086 ) {
4087 let node_id = NodeId::new("runtime-node").unwrap();
4088 let nodes = BTreeMap::from([(
4089 node_id.clone(),
4090 ObservedNode {
4091 endpoint: r"\\.\pipe\runtime-node".to_owned(),
4092 transport_label: "windows-named-pipe".to_owned(),
4093 transport: NodeTransportState::Offline,
4094 freshness: NodeFreshness::Unavailable,
4095 cursor: None,
4096 inventory: None,
4097 last_attempt_unix_ms: None,
4098 last_success_unix_ms: None,
4099 consecutive_failures: 0,
4100 last_error: None,
4101 gaps: Vec::new(),
4102 gaps_truncated: 0,
4103 },
4104 )]);
4105 let initial = Arc::new(StatusResponse {
4106 api_version: C2_API_VERSION,
4107 ready: false,
4108 observed_at_unix_ms: unix_ms(),
4109 nodes,
4110 });
4111 let (status_tx, status_rx) = watch::channel(initial);
4112 let (ingress_tx, ingress_rx) = mpsc::channel(4);
4113 let (shutdown_tx, shutdown_rx) = watch::channel(false);
4114 let owner = tokio::spawn(inventory_owner(
4115 1,
4116 Duration::from_secs(1),
4117 ingress_rx,
4118 status_tx,
4119 shutdown_rx,
4120 ));
4121 (node_id, ingress_tx, status_rx, shutdown_tx, owner)
4122 }
4123
4124 #[tokio::test]
4125 async fn incarnation_change_replaces_runtime_status() {
4126 let (node_id, ingress, mut status, shutdown, owner) = runtime_inventory_owner().await;
4127 for (incarnation, provider, version) in [
4128 (1, agent("claude"), "1.0.0"),
4129 (2, agent("codex"), "2.0.0"),
4130 ] {
4131 ingress
4132 .send(Attempt {
4133 node_id: node_id.clone(),
4134 at_unix_ms: unix_ms(),
4135 result: AttemptResult::Connected {
4136 cursor: NodeCursor {
4137 incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([
4138 incarnation;
4139 16
4140 ]),
4141 sequence: 0,
4142 },
4143 snapshot: NodeSnapshot {
4144 node_id: node_id.clone(),
4145 enabled_providers: vec![provider.clone()],
4146 provider_runtime_statuses: runtime_statuses(provider, version),
4147 workspaces: Vec::new(),
4148 session_records: Vec::new(),
4149 managed_worktrees: Vec::new(),
4150 launch_inventory: None,
4151 agent_progress: Vec::new(),
4152 },
4153 gaps: Vec::new(),
4154 provider_contract_manifest: ProviderContractManifest::default(),
4155 },
4156 })
4157 .await
4158 .unwrap();
4159 status.changed().await.unwrap();
4160 }
4161 let current_status = status.borrow();
4162 let statuses = ¤t_status.nodes[&node_id]
4163 .inventory
4164 .as_ref()
4165 .unwrap()
4166 .provider_runtime_statuses;
4167 assert_eq!(statuses.as_slice().len(), 1);
4168 assert_eq!(
4169 statuses.as_slice()[0].provider(),
4170 &agent("codex"),
4171 );
4172 assert_eq!(statuses.as_slice()[0].version().unwrap().as_str(), "2.0.0");
4173 shutdown.send(true).unwrap();
4174 owner.await.unwrap().unwrap();
4175 }
4176
4177 #[tokio::test]
4178 async fn incarnation_change_without_snapshot_clears_dynamic_inventory() {
4179 let (node_id, ingress, mut status, shutdown, owner) = runtime_inventory_owner().await;
4180 ingress
4181 .send(Attempt {
4182 node_id: node_id.clone(),
4183 at_unix_ms: unix_ms(),
4184 result: AttemptResult::Connected {
4185 cursor: NodeCursor {
4186 incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([
4187 3; 16
4188 ]),
4189 sequence: 0,
4190 },
4191 snapshot: NodeSnapshot {
4192 node_id: node_id.clone(),
4193 enabled_providers: vec![agent("claude")],
4194 provider_runtime_statuses: runtime_statuses(
4195 agent("claude"),
4196 "3.0.0",
4197 ),
4198 workspaces: Vec::new(),
4199 session_records: Vec::new(),
4200 managed_worktrees: Vec::new(),
4201 launch_inventory: None,
4202 agent_progress: Vec::new(),
4203 },
4204 gaps: Vec::new(),
4205 provider_contract_manifest: ProviderContractManifest::default(),
4206 },
4207 })
4208 .await
4209 .unwrap();
4210 status.changed().await.unwrap();
4211 ingress
4212 .send(Attempt {
4213 node_id: node_id.clone(),
4214 at_unix_ms: unix_ms(),
4215 result: AttemptResult::Cursor {
4216 cursor: NodeCursor {
4217 incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([
4218 4; 16
4219 ]),
4220 sequence: 0,
4221 },
4222 gaps: vec![GapKind::IncarnationChanged],
4223 managed_worktree_events: Vec::new(),
4224 },
4225 })
4226 .await
4227 .unwrap();
4228 status.changed().await.unwrap();
4229 assert!(status.borrow().nodes[&node_id]
4230 .inventory
4231 .as_ref()
4232 .unwrap()
4233 .provider_runtime_statuses
4234 .is_empty());
4235 shutdown.send(true).unwrap();
4236 owner.await.unwrap().unwrap();
4237 }
4238
4239 #[test]
4240 fn managed_worktree_inventory_events_are_exact_bounded_and_incarnation_fenced() {
4241 let mut inventory = SlimNodeInventory::from_snapshot(&NodeSnapshot {
4242 node_id: NodeId::new("node-a").unwrap(),
4243 enabled_providers: Vec::new(),
4244 provider_runtime_statuses: crate::protocol::ProviderRuntimeStatuses::default(),
4245 workspaces: Vec::new(),
4246 session_records: Vec::new(),
4247 managed_worktrees: vec![managed_lease(
4248 "lease-a",
4249 "managed-a",
4250 ManagedWorktreeLeaseState::Ready,
4251 )],
4252 launch_inventory: None,
4253 agent_progress: Vec::new(),
4254 });
4255 apply_managed_worktree_cursor(
4256 Some(&mut inventory),
4257 false,
4258 &[
4259 NodeEvent::ManagedWorktreeUpserted {
4260 lease: managed_lease(
4261 "lease-b",
4262 "managed-b",
4263 ManagedWorktreeLeaseState::Ready,
4264 ),
4265 },
4266 NodeEvent::ManagedWorktreeUpserted {
4267 lease: managed_lease(
4268 "lease-a",
4269 "managed-a",
4270 ManagedWorktreeLeaseState::InUse,
4271 ),
4272 },
4273 ],
4274 );
4275 assert_eq!(inventory.managed_worktree_count, 2);
4276 assert_eq!(inventory.managed_worktrees[0].lease_id.as_str(), "lease-a");
4277 assert_eq!(
4278 inventory.managed_worktrees[0].state,
4279 ManagedWorktreeLeaseState::InUse,
4280 );
4281
4282 apply_managed_worktree_cursor(
4283 Some(&mut inventory),
4284 false,
4285 &[NodeEvent::ManagedWorktreeRemoved {
4286 lease_id: ManagedWorktreeLeaseId::new("lease-a").unwrap(),
4287 }],
4288 );
4289 assert_eq!(inventory.managed_worktree_count, 1);
4290 assert_eq!(inventory.managed_worktrees[0].lease_id.as_str(), "lease-b");
4291
4292 apply_managed_worktree_cursor(
4293 Some(&mut inventory),
4294 true,
4295 &[NodeEvent::ManagedWorktreeUpserted {
4296 lease: managed_lease(
4297 "lease-c",
4298 "managed-c",
4299 ManagedWorktreeLeaseState::Ready,
4300 ),
4301 }],
4302 );
4303 assert!(inventory.managed_worktrees.is_empty());
4304 assert_eq!(inventory.managed_worktree_count, 0);
4305 assert!(!inventory.managed_worktrees_truncated);
4306 }
4307}
4308
4309#[cfg(test)]
4310mod relay_deadline_tests {
4311 use super::*;
4312
4313 #[test]
4321 fn workspace_inspection_relay_bound_clears_the_nodes_own_maximum() {
4322 const NODE_INSPECTION_MAX: Duration = Duration::from_millis(11_000);
4327 assert!(
4328 WORKSPACE_INSPECTION_RELAY_DEADLINE > NODE_INSPECTION_MAX,
4329 "relay bound {:?} must exceed the node's own inspection maximum {:?}",
4330 WORKSPACE_INSPECTION_RELAY_DEADLINE,
4331 NODE_INSPECTION_MAX,
4332 );
4333 }
4334}
4335
4336#[cfg(test)]
4337mod control_tunnel_listen_tests {
4338 use super::*;
4339
4340 fn config() -> C2Config {
4341 let node = C2NodeConfig::new(
4342 NodeId::new("node-a").unwrap(),
4343 if cfg!(unix) { "/tmp/gate4agent-node-a.sock" } else { r"\\.\pipe\gate4agent-node-a" },
4344 "token-token",
4345 ).unwrap();
4346 C2Config::new("127.0.0.1:18321".parse().unwrap(), "api-token", vec![node]).unwrap()
4347 }
4348
4349 #[test]
4350 fn control_tunnel_rejects_unspecified_and_accepts_tunnel_address() {
4351 let bad: SocketAddr = "0.0.0.0:27441".parse().unwrap();
4352 assert!(matches!(
4353 config().with_control_tunnel_listen(bad),
4354 Err(C2ConfigError::InvalidControlTunnel(_))
4355 ));
4356 let ok: SocketAddr = "10.88.1.2:27441".parse().unwrap();
4357 let config = config().with_control_tunnel_listen(ok).unwrap();
4358 assert_eq!(config.control_tunnel_listen, Some(ok));
4359 assert!(!config.control_endpoint.is_empty());
4360 }
4361}