1use crate::protocol::{
2 C2ErrorCategory, C2NodeEvent, C2NodeFailure, C2NodeResponse, C2RelayFailure, C2RelayFailureCode, GapKind, HealthResponse, NodeCursor, NodeFreshness, NodeGap, NodeId,
3 NodeIncarnationId, NodeRequest, ResolvedSpawnReceipt, RoutedNodeEvent, RoutedNodeResponse,
4 ManagedWorktreeLeaseState, ManagedWorktreeSpawnRequest, ManagedWorktreeSpawnRequestV2,
5 SpawnOverride, SpawnSpec,
6 NodeTransportState, ObservedNode, ProviderAdapterContractSupport, ProviderContractSupport,
7 ReadyResponse, SanitizedError, SlimNodeInventory, StatusResponse,
8 C2_API_VERSION, C2_PROVIDER_CONTRACT_MANIFEST_CAPABILITY,
9 MAX_C2_GAPS_PER_NODE, MAX_C2_NODES,
10};
11#[cfg(windows)]
12use crate::protocol::MAX_C2_ENDPOINT_BYTES;
13use gate4agent_node_protocol::{
14 ClientRole, FrameError, NegotiatedNodeCompatibility, NodeEvent, NodeEventEnvelope, NodeFailureCode,
15 NodeResponse, NodeSnapshot, ServerFrame,
16};
17use gate4agent_node_wire::{read_call_home_announce, LocalNodeClient, NodeClientError};
18use std::collections::{BTreeMap, BTreeSet};
19use std::io;
20use std::net::SocketAddr;
21use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, SystemTime, UNIX_EPOCH};
24#[cfg(unix)]
25use std::path::{Path, PathBuf};
26#[cfg(unix)]
27use std::os::unix::fs::PermissionsExt;
28use thiserror::Error;
29use tokio::io::{AsyncReadExt, AsyncWriteExt};
30use tokio::net::{TcpListener, TcpStream};
31use tokio::sync::{mpsc, oneshot, watch, Semaphore};
32use tokio::task::{JoinHandle, JoinSet};
33use tokio::time::{sleep, timeout, Instant};
34
35const MANAGED_RESUME_SETTLE_DEADLINE: Duration = Duration::from_secs(30);
36const NODE_REQUEST_IO_HEADROOM: Duration = Duration::from_secs(5);
37const NATIVE_SESSION_REQUEST_DEADLINE: Duration = Duration::from_secs(35);
38const WORKSPACE_ENTRY_CREATE_NODE_SEMANTIC_DEADLINE: Duration = Duration::from_secs(5);
39const WORKSPACE_ENTRY_CREATE_RELAY_DEADLINE: Duration =
40 WORKSPACE_ENTRY_CREATE_NODE_SEMANTIC_DEADLINE.saturating_add(NODE_REQUEST_IO_HEADROOM);
41
42const HEADER_LIMIT_BYTES: usize = 16 * 1024;
43const OPERATOR_BODY_LIMIT_BYTES: usize = 16 * 1024;
44const MAX_HTTP_CONNECTIONS: usize = 16;
45const RESPONSE_BODY_LIMIT_BYTES: usize = 8 * 1024 * 1024;
46
47mod control;
48
49#[cfg(windows)]
50pub const DEFAULT_C2_CONTROL_ENDPOINT: &str = r"\\.\pipe\gate4agent-c2";
51#[cfg(unix)]
52pub const DEFAULT_C2_CONTROL_ENDPOINT: &str = "gate4agent-c2.sock";
53
54#[cfg(windows)]
55pub fn default_c2_control_endpoint() -> Result<String, C2ConfigError> {
56 Ok(DEFAULT_C2_CONTROL_ENDPOINT.to_owned())
57}
58
59#[cfg(unix)]
60pub fn default_c2_control_endpoint() -> Result<String, C2ConfigError> {
61 let root = unix_runtime_root()?;
62 let directory = root.join("gate4agent");
63 let endpoint = directory.join(DEFAULT_C2_CONTROL_ENDPOINT);
64 validate_unix_endpoint(&endpoint).map_err(|_| C2ConfigError::InvalidControlEndpoint)?;
65 Ok(endpoint.to_string_lossy().into_owned())
66}
67
68#[cfg(unix)]
69fn unix_runtime_root() -> Result<PathBuf, C2ConfigError> {
70 if let Some(root) = std::env::var_os("XDG_RUNTIME_DIR").filter(|value| !value.is_empty()) {
71 let root = PathBuf::from(root);
72 if root.is_absolute() { return Ok(root); }
73 }
74 let home = std::env::var_os("HOME").filter(|value| !value.is_empty())
75 .ok_or_else(|| C2ConfigError::RuntimeEndpoint(
76 "neither absolute XDG_RUNTIME_DIR nor HOME is available".to_owned(),
77 ))?;
78 let home = PathBuf::from(home);
79 if !home.is_absolute() {
80 return Err(C2ConfigError::RuntimeEndpoint("HOME is not absolute".to_owned()));
81 }
82 Ok(home.join(".gate4agent").join("run"))
83}
84
85#[derive(Clone)]
86pub struct C2NodeConfig {
87 pub node_id: NodeId,
88 pub endpoint: String,
89 route: C2NodeRoute,
90 token: String,
91}
92
93#[derive(Clone, Copy, Debug, Eq, PartialEq)]
94enum C2NodeRoute {
95 Local,
96 SshForwardedLoopback(SocketAddr),
97 CallHome,
107}
108
109impl C2NodeConfig {
110 pub fn new(node_id: NodeId, endpoint: impl Into<String>, token: impl Into<String>) -> Result<Self, C2ConfigError> {
111 let endpoint = endpoint.into();
112 let token = token.into();
113 let (endpoint, route) = parse_node_endpoint(&endpoint)
114 .ok_or_else(|| C2ConfigError::InvalidEndpoint(node_id.clone()))?;
115 validate_token(&token)?;
116 Ok(Self { node_id, endpoint, route, token })
117 }
118
119 fn transport_label(&self) -> &'static str {
120 match self.route {
121 C2NodeRoute::Local => local_transport_label(),
122 C2NodeRoute::SshForwardedLoopback(_) => "ssh-forwarded-loopback",
123 C2NodeRoute::CallHome => "call-home",
124 }
125 }
126}
127
128#[derive(Clone, Copy, Debug)]
129pub struct C2Timings {
130 pub poll_interval: Duration,
131 pub fresh_for: Duration,
132 pub attempt_deadline: Duration,
133 pub transient_backoffs: [Duration; 5],
134 pub parked_backoff: Duration,
135 pub http_io_deadline: Duration,
136}
137
138impl Default for C2Timings {
139 fn default() -> Self {
140 Self {
141 poll_interval: Duration::from_millis(250),
142 fresh_for: Duration::from_secs(10),
143 attempt_deadline: Duration::from_secs(5),
144 transient_backoffs: [
145 Duration::from_millis(500), Duration::from_secs(1), Duration::from_secs(2),
146 Duration::from_secs(4), Duration::from_secs(8),
147 ],
148 parked_backoff: Duration::from_secs(30),
149 http_io_deadline: Duration::from_secs(3),
150 }
151 }
152}
153
154#[derive(Clone)]
155pub struct C2Config {
156 pub api_listen: SocketAddr,
157 pub control_endpoint: String,
158 pub control_tunnel_listen: Option<SocketAddr>,
161 api_token: String,
162 pub nodes: Vec<C2NodeConfig>,
163 pub node_listen: Option<SocketAddr>,
166 pub timings: C2Timings,
167 pub enrollment: Option<crate::enroll_service::EnrollConfig>,
170}
171
172impl C2Config {
173 pub fn with_enrollment(mut self, enrollment: crate::enroll_service::EnrollConfig) -> Self {
175 self.enrollment = Some(enrollment);
176 self
177 }
178
179 pub fn new(api_listen: SocketAddr, api_token: impl Into<String>, nodes: Vec<C2NodeConfig>) -> Result<Self, C2ConfigError> {
180 let api_token = api_token.into();
181 if !api_listen.ip().is_loopback() { return Err(C2ConfigError::NonLoopback(api_listen)); }
182 validate_token(&api_token)?;
183 if nodes.len() > MAX_C2_NODES { return Err(C2ConfigError::NodeCount(nodes.len())); }
184 let mut ids = BTreeSet::new();
185 let mut endpoints = BTreeSet::new();
186 if nodes.iter().any(|node| !ids.insert(node.node_id.clone())) { return Err(C2ConfigError::DuplicateNode); }
187 if nodes
193 .iter()
194 .filter(|node| node.route != C2NodeRoute::CallHome)
195 .any(|node| !endpoints.insert(endpoint_key(&node.endpoint)))
196 {
197 return Err(C2ConfigError::DuplicateEndpoint);
198 }
199 let control_endpoint = default_c2_control_endpoint()?;
200 if nodes.iter().any(|node| endpoints_equal(&node.endpoint, &control_endpoint)) {
201 return Err(C2ConfigError::ControlEndpointConflict);
202 }
203 Ok(Self {
204 api_listen,
205 control_endpoint,
206 control_tunnel_listen: None,
207 api_token,
208 nodes,
209 node_listen: None,
210 timings: C2Timings::default(),
211 enrollment: None,
212 })
213 }
214
215 pub fn with_node_listen(mut self, node_listen: SocketAddr) -> Result<Self, C2ConfigError> {
224 if node_listen.port() == 0 || node_listen.ip().is_unspecified() {
227 return Err(C2ConfigError::NonLoopbackNodeListen(node_listen));
228 }
229 if node_listen == self.api_listen {
230 return Err(C2ConfigError::NodeListenConflict);
231 }
232 if self.control_tunnel_listen == Some(node_listen) {
233 return Err(C2ConfigError::NonLoopbackNodeListen(node_listen));
234 }
235 self.node_listen = Some(node_listen);
236 Ok(self)
237 }
238
239 pub fn validate_call_home(&self) -> Result<(), C2ConfigError> {
245 if self.nodes.is_empty() && self.enrollment.is_none() {
246 return Err(C2ConfigError::NodeCount(0));
247 }
248 let waiting = self.nodes.iter().find(|node| node.route == C2NodeRoute::CallHome);
249 match (waiting, self.node_listen) {
250 (Some(node), None) => Err(C2ConfigError::CallHomeWithoutListener(node.node_id.clone())),
251 (None, None) if self.enrollment.is_some() => Err(C2ConfigError::EnrollmentWithoutListener),
252 _ => Ok(()),
253 }
254 }
255
256 pub fn with_timings(mut self, timings: C2Timings) -> Self {
257 self.timings = timings;
258 self
259 }
260
261 pub fn with_control_endpoint(mut self, endpoint: impl Into<String>) -> Result<Self, C2ConfigError> {
262 let endpoint = endpoint.into();
263 validate_control_endpoint(&endpoint)?;
264 if self.nodes.iter().any(|node| endpoints_equal(&node.endpoint, &endpoint)) {
265 return Err(C2ConfigError::ControlEndpointConflict);
266 }
267 self.control_endpoint = endpoint;
268 Ok(self)
269 }
270
271 pub fn with_control_tunnel_listen(mut self, listen: SocketAddr) -> Result<Self, C2ConfigError> {
275 if listen.port() == 0 || listen.ip().is_unspecified() {
276 return Err(C2ConfigError::InvalidControlTunnel(listen));
277 }
278 if listen == self.api_listen || self.node_listen == Some(listen) {
279 return Err(C2ConfigError::InvalidControlTunnel(listen));
280 }
281 self.control_tunnel_listen = Some(listen);
282 Ok(self)
283 }
284}
285
286fn validate_control_endpoint(endpoint: &str) -> Result<(), C2ConfigError> {
287 if !valid_local_endpoint(endpoint) {
288 return Err(C2ConfigError::InvalidControlEndpoint);
289 }
290 Ok(())
291}
292
293fn parse_node_endpoint(endpoint: &str) -> Option<(String, C2NodeRoute)> {
294 if endpoint == "accept" {
300 return Some((endpoint.to_owned(), C2NodeRoute::CallHome));
301 }
302 if let Some(authority) = endpoint.strip_prefix("tcp://") {
303 let address = authority.parse::<SocketAddr>().ok()?;
304 let is_exact_loopback = match address.ip() {
305 std::net::IpAddr::V4(ip) => ip == std::net::Ipv4Addr::LOCALHOST,
306 std::net::IpAddr::V6(ip) => ip == std::net::Ipv6Addr::LOCALHOST,
307 };
308 if !is_exact_loopback || address.port() == 0 {
309 return None;
310 }
311 return Some((format!("tcp://{address}"), C2NodeRoute::SshForwardedLoopback(address)));
312 }
313 if endpoint.contains("://") || !valid_local_endpoint(endpoint) {
314 return None;
315 }
316 Some((endpoint.to_owned(), C2NodeRoute::Local))
317}
318
319#[cfg(windows)]
320fn valid_local_endpoint(endpoint: &str) -> bool {
321 endpoint.starts_with(r"\\.\pipe\") && endpoint.len() > r"\\.\pipe\".len()
322 && endpoint.len() <= MAX_C2_ENDPOINT_BYTES
323}
324
325#[cfg(unix)]
326fn valid_local_endpoint(endpoint: &str) -> bool {
327 validate_unix_endpoint(Path::new(endpoint)).is_ok()
328}
329
330#[cfg(unix)]
331fn validate_unix_endpoint(endpoint: &Path) -> Result<(), ()> {
332 const MAX_UNIX_ENDPOINT_BYTES: usize = 103;
333 use std::os::unix::ffi::OsStrExt;
334
335 if !endpoint.is_absolute() || endpoint.file_name().is_none()
336 || endpoint.as_os_str().as_bytes().len() > MAX_UNIX_ENDPOINT_BYTES
337 {
338 return Err(());
339 }
340 Ok(())
341}
342
343#[cfg(windows)]
344fn endpoint_key(endpoint: &str) -> String {
345 if endpoint.starts_with("tcp://") { endpoint.to_owned() } else { endpoint.to_ascii_lowercase() }
346}
347
348#[cfg(unix)]
349fn endpoint_key(endpoint: &str) -> String { endpoint.to_owned() }
350
351fn endpoints_equal(left: &str, right: &str) -> bool { endpoint_key(left) == endpoint_key(right) }
352
353fn validate_token(token: &str) -> Result<(), C2ConfigError> {
354 if token.is_empty() || token.len() > 4096 || !token.bytes().all(|byte| matches!(byte, 0x21..=0x7e)) {
355 return Err(C2ConfigError::InvalidToken);
356 }
357 Ok(())
358}
359
360#[derive(Debug, Error)]
361pub enum C2ConfigError {
362 #[error("C2 tokens must contain 1..=4096 visible ASCII bytes without whitespace")]
363 InvalidToken,
364 #[cfg_attr(windows, error("node '{0}' requires a bounded Windows named pipe or exact loopback TCP endpoint"))]
365 #[cfg_attr(unix, error("node '{0}' requires a bounded local socket or exact loopback TCP endpoint"))]
366 InvalidEndpoint(NodeId),
367 #[error("C2 API listen address must be loopback: {0}")]
368 NonLoopback(SocketAddr),
369 #[error("C2 requires 1..=64 configured nodes; received {0}")]
370 NodeCount(usize),
371 #[error("C2 node IDs must be unique")]
372 DuplicateNode,
373 #[error("C2 node endpoints must be unique")]
374 DuplicateEndpoint,
375 #[cfg_attr(windows, error("C2 control endpoint must be a bounded local Windows named pipe"))]
376 #[cfg_attr(unix, error("C2 control endpoint must be a bounded local endpoint"))]
377 InvalidControlEndpoint,
378 #[error("enrollment needs a --node-listen address for the nodes to call home on")]
379 EnrollmentWithoutListener,
380 #[error("C2 control endpoint must not equal a configured node endpoint")]
381 ControlEndpointConflict,
382 #[error("C2 control tunnel listen must be a concrete non-unspecified address with a nonzero port, not shared with another listener: {0}")]
383 InvalidControlTunnel(SocketAddr),
384 #[error("C2 node call-home listen must be loopback or a concrete C2 tunnel address with a nonzero port (not 0.0.0.0, not shared with another listener): {0}")]
385 NonLoopbackNodeListen(SocketAddr),
386 #[error("C2 node call-home listen address must not equal the API listen address")]
387 NodeListenConflict,
388 #[error("node '{0}' waits to be called but no --node-listen address was configured")]
389 CallHomeWithoutListener(NodeId),
390 #[cfg(unix)]
391 #[error("C2 default runtime endpoint is unavailable: {0}")]
392 RuntimeEndpoint(String),
393}
394
395type RelayResult = Result<RoutedNodeResponse, C2RelayFailure>;
396
397enum RelayCommand {
398 Request {
399 operator_connection_id: u64,
400 expected_incarnation_id: NodeIncarnationId,
401 request: NodeRequest,
402 reply: oneshot::Sender<RelayResult>,
403 },
404}
405
406#[derive(Clone)]
409struct RelayRegistry {
410 map: Arc<std::sync::RwLock<Arc<BTreeMap<NodeId, RelayEndpoint>>>>,
411 gate: Option<Arc<dyn crate::enroll_service::CapabilityGate>>,
412}
413
414impl RelayRegistry {
415 fn new(initial: BTreeMap<NodeId, RelayEndpoint>) -> Self {
416 Self { map: Arc::new(std::sync::RwLock::new(Arc::new(initial))), gate: None }
417 }
418 fn allows(&self, node_id: &NodeId, capability: &str) -> bool {
420 self.gate.as_ref().and_then(|gate| gate.allows(node_id.as_str(), capability)).unwrap_or(true)
421 }
422 fn redact(&self, status: &mut StatusResponse) {
424 for (id, node) in status.nodes.iter_mut() {
425 if !self.allows(id, "attach") {
426 node.inventory = None;
427 }
428 }
429 }
430 fn snapshot(&self) -> Arc<BTreeMap<NodeId, RelayEndpoint>> {
431 Arc::clone(&self.map.read().expect("relay registry lock"))
432 }
433 fn insert(&self, node_id: NodeId, endpoint: RelayEndpoint) {
434 let mut guard = self.map.write().expect("relay registry lock");
435 let mut next = (**guard).clone();
436 next.insert(node_id, endpoint);
437 *guard = Arc::new(next);
438 }
439}
440
441#[derive(Clone)]
442struct RelayEndpoint {
443 commands: mpsc::Sender<RelayCommand>,
444 releases: mpsc::Sender<oneshot::Sender<()>>,
445 force_disconnect: watch::Sender<u64>,
446}
447
448#[derive(Clone)]
449struct OperatorHub {
450 sink: Arc<Mutex<Option<OperatorEventSink>>>,
451 gate: Arc<Mutex<Option<Arc<dyn crate::enroll_service::CapabilityGate>>>>,
452}
453
454#[derive(Clone)]
455struct OperatorEventSink {
456 connection_id: u64,
457 outbound: mpsc::Sender<control::QueuedFrame>,
458 budget: Arc<AtomicUsize>,
459 disconnect: watch::Sender<bool>,
460}
461
462impl OperatorHub {
463 fn new() -> Self { Self { sink: Arc::new(Mutex::new(None)), gate: Arc::new(Mutex::new(None)) } }
464
465 fn set_gate(&self, gate: Arc<dyn crate::enroll_service::CapabilityGate>) {
466 *self.gate.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(gate);
467 }
468
469 fn attach(&self, sink: OperatorEventSink) {
470 *self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(sink);
471 }
472
473 fn detach(&self, connection_id: u64) {
474 let mut sink = self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
475 if sink.as_ref().is_some_and(|current| current.connection_id == connection_id) { *sink = None; }
476 }
477
478 fn is_active(&self, connection_id: u64) -> bool {
479 self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
480 .as_ref().is_some_and(|current| current.connection_id == connection_id)
481 }
482
483 fn has_active_operator(&self) -> bool {
484 self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner()).is_some()
485 }
486
487 fn publish(&self, event: RoutedNodeEvent) {
488 let gate = self.gate.lock().unwrap_or_else(|poisoned| poisoned.into_inner()).clone();
490 if gate.is_some_and(|gate| gate.allows(event.node_id.as_str(), "attach") == Some(false)) {
491 return;
492 }
493 let sink = self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
494 if let Some(sink) = sink.as_ref() {
495 if control::queue_operator_event(&sink.outbound, &sink.budget, event).is_err() {
496 let _ = sink.disconnect.send(true);
497 }
498 }
499 }
500}
501
502fn relay_failure(
503 code: C2RelayFailureCode,
504 message: &'static str,
505 current_incarnation_id: Option<NodeIncarnationId>,
506) -> C2RelayFailure {
507 C2RelayFailure { code, message: message.to_owned(), current_incarnation_id }
508}
509
510#[derive(Debug, Error)]
511pub enum C2Error {
512 #[error("C2 API failed: {0}")]
513 Api(#[from] io::Error),
514 #[error("C2 task failed: {0}")]
515 Task(#[from] tokio::task::JoinError),
516}
517
518#[derive(Clone)]
519pub struct C2ShutdownHandle {
520 shutdown: watch::Sender<bool>,
521}
522
523impl C2ShutdownHandle {
524 pub fn shutdown(&self) { let _ = self.shutdown.send(true); }
525}
526
527pub struct C2Running {
528 api_addr: SocketAddr,
529 shutdown: C2ShutdownHandle,
530 task: Option<JoinHandle<Result<(), C2Error>>>,
531}
532
533impl C2Running {
534 pub async fn start(config: C2Config) -> Result<Self, C2Error> {
535 prepare_default_control_parent(&config.control_endpoint)?;
536 let listener = TcpListener::bind(config.api_listen).await?;
537 let api_addr = listener.local_addr()?;
538 let (shutdown_tx, shutdown_rx) = watch::channel(false);
539 let shutdown = C2ShutdownHandle { shutdown: shutdown_tx };
540 let task = tokio::spawn(run_bound(config, listener, shutdown_rx));
541 Ok(Self { api_addr, shutdown, task: Some(task) })
542 }
543
544 pub fn api_addr(&self) -> SocketAddr { self.api_addr }
545 pub fn shutdown_handle(&self) -> C2ShutdownHandle { self.shutdown.clone() }
546 pub async fn wait(mut self) -> Result<(), C2Error> {
547 self.task.take().expect("C2 task is present").await?
548 }
549}
550
551#[cfg(windows)]
552fn prepare_default_control_parent(_endpoint: &str) -> io::Result<()> { Ok(()) }
553
554#[cfg(unix)]
555fn prepare_default_control_parent(endpoint: &str) -> io::Result<()> {
556 let default = default_c2_control_endpoint()
557 .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error.to_string()))?;
558 if endpoint != default { return Ok(()); }
559 let parent = Path::new(endpoint).parent().ok_or_else(|| {
560 io::Error::new(io::ErrorKind::InvalidInput, "default C2 endpoint has no parent")
561 })?;
562 std::fs::create_dir_all(parent)?;
563 std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700))
564}
565
566impl Drop for C2Running {
567 fn drop(&mut self) {
568 self.shutdown.shutdown();
569 if let Some(task) = self.task.take() { task.abort(); }
570 }
571}
572
573type CallHomeSocket = (TcpStream, Option<String>);
576type CallHomeRoutes = Arc<Mutex<BTreeMap<NodeId, mpsc::Sender<CallHomeSocket>>>>;
577
578struct Spawner {
579 timings: C2Timings,
580 relay_senders: RelayRegistry,
581 call_home_routes: CallHomeRoutes,
582 ingress: mpsc::Sender<Attempt>,
583 status: watch::Receiver<Arc<StatusResponse>>,
584 hub: OperatorHub,
585}
586
587impl Spawner {
588 fn spawn_enrolled(&self, node_id: &NodeId, shutdown: &watch::Receiver<bool>, tasks: &mut JoinSet<io::Result<()>>) {
590 if self.relay_senders.snapshot().contains_key(node_id) {
591 return;
592 }
593 let placeholder = "0".repeat(64);
596 let Ok(node) = C2NodeConfig::new(node_id.clone(), "accept", placeholder) else { return };
597 let (commands_tx, commands_rx) = mpsc::channel(8);
598 let (releases_tx, releases_rx) = mpsc::channel(1);
599 let (force_tx, force_rx) = watch::channel(0_u64);
600 let (call_home_tx, call_home_rx) = mpsc::channel(1);
601 self.call_home_routes.lock().expect("routes lock").insert(node_id.clone(), call_home_tx);
602 self.relay_senders.insert(node_id.clone(), RelayEndpoint { commands: commands_tx, releases: releases_tx, force_disconnect: force_tx });
603 tasks.spawn(node_relay_worker(
604 node,
605 self.timings,
606 commands_rx,
607 releases_rx,
608 force_rx,
609 Some(call_home_rx),
610 self.ingress.clone(),
611 self.status.clone(),
612 self.hub.clone(),
613 shutdown.clone(),
614 ));
615 tracing::info!(%node_id, "relay worker created for enrolled node");
616 }
617}
618
619async fn run_bound(config: C2Config, listener: TcpListener, mut shutdown: watch::Receiver<bool>) -> Result<(), C2Error> {
620 let now = unix_ms();
621 let nodes = config.nodes.iter().map(|node| {
622 (node.node_id.clone(), initial_observed_node(node))
623 }).collect();
624 let initial = Arc::new(StatusResponse { api_version: C2_API_VERSION, ready: config.nodes.is_empty(), observed_at_unix_ms: now, nodes });
625 let (status_tx, status_rx) = watch::channel(initial);
626 let (ingress_tx, ingress_rx) = mpsc::channel(config.nodes.len().saturating_mul(2).max(16));
627 let mut tasks = JoinSet::new();
628 let hub = OperatorHub::new();
629 let mut relay_senders = BTreeMap::new();
630 let mut relay_receivers = Vec::new();
631 let mut static_routes: BTreeMap<NodeId, mpsc::Sender<CallHomeSocket>> = BTreeMap::new();
637 for node in config.nodes.clone() {
638 let (commands_tx, commands_rx) = mpsc::channel(8);
639 let (releases_tx, releases_rx) = mpsc::channel(1);
640 let (force_tx, force_rx) = watch::channel(0_u64);
641 relay_senders.insert(node.node_id.clone(), RelayEndpoint {
642 commands: commands_tx,
643 releases: releases_tx,
644 force_disconnect: force_tx,
645 });
646 let call_home_rx = if node.route == C2NodeRoute::CallHome {
647 let (call_home_tx, call_home_rx) = mpsc::channel(1);
648 static_routes.insert(node.node_id.clone(), call_home_tx);
649 Some(call_home_rx)
650 } else {
651 None
652 };
653 relay_receivers.push((node, commands_rx, releases_rx, force_rx, call_home_rx));
654 }
655 let static_ids: BTreeSet<NodeId> = static_routes.keys().cloned().collect();
656 let call_home_routes: CallHomeRoutes = Arc::new(Mutex::new(static_routes));
657 let mut relay_senders = RelayRegistry::new(relay_senders);
658 let mut enroll_events = None;
660 let mut enroll_service = None;
661 if let Some(enrollment) = &config.enrollment {
662 let (service, events) = crate::enroll_service::EnrollService::start(enrollment)
663 .map_err(|error| C2Error::Api(io::Error::other(error.to_string())))?;
664 let enroll_listener = TcpListener::bind(enrollment.listen).await?;
665 tasks.spawn(crate::enroll_service::serve_enroll(enroll_listener, Arc::clone(&service), shutdown.clone()));
666 relay_senders.gate = Some(Arc::clone(&service) as Arc<dyn crate::enroll_service::CapabilityGate>);
667 hub.set_gate(Arc::clone(&service) as Arc<dyn crate::enroll_service::CapabilityGate>);
668 enroll_events = Some(events);
669 enroll_service = Some(service);
670 }
671 if let Some(node_listen) = config.node_listen {
672 tasks.spawn(accept_call_home(node_listen, Arc::clone(&call_home_routes), static_ids, enroll_service.clone(), shutdown.clone()));
673 }
674 tasks.spawn(inventory_owner(config.timings.fresh_for, ingress_rx, status_tx, shutdown.clone()));
675 for (node, commands, releases, force_disconnect, call_home) in relay_receivers {
676 tasks.spawn(node_relay_worker(node, config.timings, commands, releases, force_disconnect, call_home, ingress_tx.clone(), status_rx.clone(), hub.clone(), shutdown.clone()));
677 }
678 let spawner = Spawner {
679 timings: config.timings,
680 relay_senders: relay_senders.clone(),
681 call_home_routes: Arc::clone(&call_home_routes),
682 ingress: ingress_tx.clone(),
683 status: status_rx.clone(),
684 hub: hub.clone(),
685 };
686 drop(ingress_tx);
687 tasks.spawn(http_server(listener, config.api_token.clone(), config.timings.http_io_deadline, status_rx.clone(), enroll_service.clone(), shutdown.clone()));
688 let slots = control::ControlSlots::new();
689 let tunnel_listen = config.control_tunnel_listen;
690 let tunnel_token = config.api_token.clone();
691 tasks.spawn(control::run(
692 config.control_endpoint,
693 config.api_token,
694 relay_senders.clone(),
695 status_rx.clone(),
696 hub.clone(),
697 shutdown.clone(),
698 slots.clone(),
699 ));
700 if let Some(listen) = tunnel_listen {
701 tasks.spawn(control::run_tcp(
702 listen,
703 tunnel_token,
704 relay_senders.clone(),
705 status_rx,
706 hub,
707 shutdown.clone(),
708 slots,
709 ));
710 }
711 loop {
712 tokio::select! {
713 changed = shutdown.changed() => {
714 if changed.is_err() || *shutdown.borrow() { break; }
715 }
716 event = async {
717 match enroll_events.as_mut() {
718 Some(events) => events.recv().await,
719 None => std::future::pending().await,
720 }
721 } => {
722 match event {
723 Some(crate::enroll_service::EnrollEvent::Enrolled(id)) => {
724 if let Ok(node_id) = NodeId::new(id) { spawner.spawn_enrolled(&node_id, &shutdown, &mut tasks); }
725 }
726 Some(crate::enroll_service::EnrollEvent::StatusChanged(id, _)) => {
727 if let Ok(node_id) = NodeId::new(id) {
729 if !relay_senders.allows(&node_id, "heartbeat") {
730 if let Some(endpoint) = relay_senders.snapshot().get(&node_id) {
731 endpoint.force_disconnect.send_modify(|generation| *generation = generation.wrapping_add(1));
732 }
733 }
734 }
735 }
736 Some(crate::enroll_service::EnrollEvent::PolicyChanged) => {
737 if let Some(service) = &enroll_service {
738 for id in service.nodes_without("heartbeat") {
739 if let Ok(node_id) = NodeId::new(id) {
740 if let Some(endpoint) = relay_senders.snapshot().get(&node_id) {
741 endpoint.force_disconnect.send_modify(|generation| *generation = generation.wrapping_add(1));
742 }
743 }
744 }
745 }
746 }
747 None => enroll_events = None,
748 }
749 }
750 result = tasks.join_next() => {
751 match result {
752 Some(Ok(Ok(()))) if *shutdown.borrow() => break,
753 Some(Ok(Ok(()))) => return Err(C2Error::Api(io::Error::new(io::ErrorKind::Other, "C2 task exited unexpectedly"))),
754 Some(Ok(Err(error))) => return Err(C2Error::Api(error)),
755 Some(Err(error)) => return Err(C2Error::Task(error)),
756 None => break,
757 }
758 }
759 }
760 }
761 tasks.shutdown().await;
762 Ok(())
763}
764
765fn initial_enrolled_node() -> ObservedNode {
766 ObservedNode {
767 endpoint: "call-home".to_owned(), transport_label: "call-home".to_owned(),
768 transport: NodeTransportState::Offline, freshness: NodeFreshness::Unavailable,
769 cursor: None, inventory: None, last_attempt_unix_ms: None, last_success_unix_ms: None,
770 consecutive_failures: 0, last_error: None, gaps: Vec::new(), gaps_truncated: 0,
771 }
772}
773
774fn initial_observed_node(node: &C2NodeConfig) -> ObservedNode {
775 ObservedNode {
776 endpoint: node.endpoint.clone(), transport_label: node.transport_label().to_owned(),
777 transport: NodeTransportState::Offline, freshness: NodeFreshness::Unavailable,
778 cursor: None, inventory: None, last_attempt_unix_ms: None, last_success_unix_ms: None,
779 consecutive_failures: 0, last_error: None, gaps: Vec::new(), gaps_truncated: 0,
780 }
781}
782
783#[cfg(windows)]
784fn local_transport_label() -> &'static str { "windows-named-pipe" }
785
786#[cfg(unix)]
787fn local_transport_label() -> &'static str { "unix-domain-socket" }
788
789#[derive(Clone, Default)]
790struct ProviderContractManifest {
791 provider_contracts: Vec<ProviderContractSupport>,
792 provider_adapter_contracts: Vec<ProviderAdapterContractSupport>,
793}
794
795impl ProviderContractManifest {
796 fn from_compatibility(compatibility: Option<&NegotiatedNodeCompatibility>) -> Self {
797 let Some(compatibility) = compatibility.filter(|compatibility| {
798 compatibility.capabilities.iter().any(|capability| {
799 capability.as_str() == C2_PROVIDER_CONTRACT_MANIFEST_CAPABILITY
800 })
801 }) else {
802 return Self::default();
803 };
804 Self {
805 provider_contracts: compatibility.provider_contracts.clone(),
806 provider_adapter_contracts: compatibility.provider_adapter_contracts.clone(),
807 }
808 }
809}
810
811enum AttemptResult {
812 Connected {
813 cursor: NodeCursor,
814 snapshot: NodeSnapshot,
815 gaps: Vec<GapKind>,
816 provider_contract_manifest: ProviderContractManifest,
817 },
818 Success { cursor: NodeCursor, snapshot: NodeSnapshot, gaps: Vec<GapKind> },
819 Cursor {
820 cursor: NodeCursor,
821 gaps: Vec<GapKind>,
822 managed_worktree_events: Vec<NodeEvent>,
823 },
824 Failure { error: SanitizedError, hard: bool },
825}
826
827struct Attempt { node_id: NodeId, at_unix_ms: u64, result: AttemptResult }
828
829async fn accept_call_home(
845 listen: SocketAddr,
846 routes: CallHomeRoutes,
847 static_ids: BTreeSet<NodeId>,
848 enroll: Option<Arc<crate::enroll_service::EnrollService>>,
849 mut shutdown: watch::Receiver<bool>,
850) -> io::Result<()> {
851 let listener = TcpListener::bind(listen).await?;
852 tracing::info!(%listen, waiting_nodes = routes.lock().expect("routes lock").len(), "call-home listener bound");
853 loop {
854 if *shutdown.borrow() {
855 return Ok(());
856 }
857 let (stream, peer) = tokio::select! {
858 accepted = listener.accept() => accepted?,
859 changed = shutdown.changed() => {
860 if changed.is_err() || *shutdown.borrow() { return Ok(()); }
861 continue;
862 }
863 };
864 let routes = Arc::clone(&routes);
865 let static_ids = static_ids.clone();
866 let enroll = enroll.clone();
867 tokio::spawn(async move {
871 let mut stream = stream;
872 let _ = stream.set_nodelay(true);
873 let node_id = match read_call_home_announce(&mut stream).await {
874 Ok(node_id) => node_id,
875 Err(error) => {
876 tracing::warn!(%peer, cause = %error, "call-home connection rejected");
877 return;
878 }
879 };
880 let mut session_token = None;
884 if !static_ids.contains(&node_id) {
885 let Some(enroll) = enroll.as_ref() else {
886 tracing::warn!(%peer, %node_id, "call-home connection named a node this relay does not wait for");
887 return;
888 };
889 match enroll.signed_call_home(&mut stream, node_id.as_str()).await {
890 Ok(token) => session_token = Some(token),
891 Err(cause) => {
892 tracing::warn!(%peer, %node_id, %cause, "signed call-home refused");
893 return;
894 }
895 }
896 }
897 let sender = routes.lock().expect("routes lock").get(&node_id).cloned();
898 let Some(sender) = sender else {
899 tracing::warn!(
900 %peer,
901 %node_id,
902 "call-home connection named a node this relay does not wait for",
903 );
904 return;
905 };
906 match sender.try_send((stream, session_token)) {
907 Ok(()) => tracing::info!(%peer, %node_id, "node called home"),
908 Err(mpsc::error::TrySendError::Full(_)) => tracing::warn!(
909 %peer,
910 %node_id,
911 "node called home while its previous connection is still being taken up",
912 ),
913 Err(mpsc::error::TrySendError::Closed(_)) => tracing::warn!(
914 %peer,
915 %node_id,
916 "node called home but its relay worker is gone",
917 ),
918 }
919 });
920 }
921}
922
923async fn await_call_home(
935 receiver: Option<&mut mpsc::Receiver<CallHomeSocket>>,
936 shutdown: &mut watch::Receiver<bool>,
937) -> Option<CallHomeSocket> {
938 let receiver = receiver?;
939 loop {
940 if *shutdown.borrow() {
941 return None;
942 }
943 tokio::select! {
944 stream = receiver.recv() => return stream,
945 changed = shutdown.changed() => {
946 if changed.is_err() || *shutdown.borrow() { return None; }
947 }
948 }
949 }
950}
951
952async fn node_relay_worker(
953 node: C2NodeConfig,
954 timings: C2Timings,
955 mut commands: mpsc::Receiver<RelayCommand>,
956 mut releases: mpsc::Receiver<oneshot::Sender<()>>,
957 mut force_disconnect: watch::Receiver<u64>,
958 mut call_home: Option<mpsc::Receiver<CallHomeSocket>>,
959 ingress: mpsc::Sender<Attempt>,
960 status: watch::Receiver<Arc<StatusResponse>>,
961 hub: OperatorHub,
962 mut shutdown: watch::Receiver<bool>,
963) -> io::Result<()> {
964 let mut failures = 0_usize;
965 loop {
966 if *shutdown.borrow() { return Ok(()); }
967 let previous = status.borrow().nodes.get(&node.node_id).and_then(|item| item.cursor);
968 let adopted = if node.route == C2NodeRoute::CallHome {
973 match await_call_home(call_home.as_mut(), &mut shutdown).await {
974 Some((stream, token)) => Some((stream, token)),
975 None => return Ok(()),
976 }
977 } else {
978 None
979 };
980 let (adopted, session_token) = match adopted {
981 Some((stream, token)) => (Some(stream), token),
982 None => (None, None),
983 };
984 let attempt_node = match session_token {
985 Some(token) => C2NodeConfig { token, ..node.clone() },
986 None => node.clone(),
987 };
988 let connected = timeout(timings.attempt_deadline, connect_operator(&attempt_node, adopted)).await;
989 let mut client = match connected {
990 Ok(Ok(client)) => client,
991 Ok(Err(error)) => {
992 failures = failures.saturating_add(1);
993 let (error, hard) = sanitize_node_error(&error);
994 tracing::warn!(
995 node_id = %node.node_id,
996 cause = %error.message,
997 category = ?error.category,
998 hard,
999 "node connection attempt failed",
1000 );
1001 ingress_attempt(&ingress, &node.node_id, AttemptResult::Failure { error, hard }).await?;
1002 reject_disconnected_commands(&mut commands, previous);
1003 acknowledge_disconnected_releases(&mut releases);
1004 relay_backoff(&mut shutdown, timings, failures, hard).await?;
1005 continue;
1006 }
1007 Err(_) => {
1008 failures = failures.saturating_add(1);
1009 tracing::warn!(
1010 node_id = %node.node_id,
1011 cause = "node connection deadline exceeded",
1012 "node connection attempt failed",
1013 );
1014 ingress_attempt(&ingress, &node.node_id, AttemptResult::Failure {
1015 error: SanitizedError { category: C2ErrorCategory::Timeout, message: "node connection deadline exceeded".to_owned() },
1016 hard: false,
1017 }).await?;
1018 reject_disconnected_commands(&mut commands, previous);
1019 acknowledge_disconnected_releases(&mut releases);
1020 relay_backoff(&mut shutdown, timings, failures, false).await?;
1021 continue;
1022 }
1023 };
1024 failures = 0;
1025 let hello = client.hello().clone();
1026 let provider_contract_manifest =
1027 ProviderContractManifest::from_compatibility(hello.compatibility.as_ref());
1028 let incarnation_id = hello.incarnation_id;
1029 let connection_id = hello.connection_id;
1030 tracing::info!(
1031 node_id = %node.node_id,
1032 connection_id,
1033 incarnation_id = ?incarnation_id,
1034 "node attached to relay",
1035 );
1036 let mut controller_owned = hello.controller.as_ref().is_some_and(|controller| controller.connection_id == connection_id);
1037 let mut cursor = NodeCursor { incarnation_id, sequence: hello.event_sequence };
1038 let mut snapshot = hello.snapshot;
1039 let mut gaps = Vec::new();
1040 let mut did_resync = false;
1041 if let Some(previous) = previous {
1042 if previous.incarnation_id != incarnation_id {
1043 gaps.push(GapKind::IncarnationChanged);
1044 } else if cursor.sequence < previous.sequence {
1045 gaps.push(GapKind::CursorRegression);
1046 } else if cursor.sequence > previous.sequence {
1047 match bounded_node_request(&mut client, NodeRequest::Resync { after_sequence: previous.sequence }).await {
1048 Ok(NodeResponse::Resync {
1049 event_sequence,
1050 oldest_available_sequence,
1051 snapshot: resync_snapshot,
1052 events,
1053 }) => {
1054 let resync_gaps = validate_resync(
1055 previous.sequence,
1056 cursor.sequence,
1057 event_sequence,
1058 oldest_available_sequence,
1059 &events,
1060 );
1061 if resync_gaps.is_empty() {
1062 publish_recovered_events(&node.node_id, incarnation_id, &events, &hub);
1063 } else {
1064 hub.publish(RoutedNodeEvent {
1065 node_id: node.node_id.clone(),
1066 cursor: NodeCursor { incarnation_id, sequence: event_sequence },
1067 event: C2NodeEvent::ResyncRequired {
1068 oldest_available_sequence,
1069 },
1070 });
1071 }
1072 gaps.extend(resync_gaps);
1073 cursor.sequence = event_sequence;
1074 snapshot = resync_snapshot;
1075 did_resync = true;
1076 }
1077 Ok(_) => gaps.push(GapKind::NonContiguousEvents),
1078 Err(error) => {
1079 ingress_attempt(&ingress, &node.node_id, relay_failure_attempt(&error)).await?;
1080 reject_disconnected_commands(&mut commands, Some(cursor));
1081 continue;
1082 }
1083 }
1084 }
1085 }
1086 ingress_attempt(&ingress, &node.node_id, AttemptResult::Connected {
1087 cursor,
1088 snapshot,
1089 gaps,
1090 provider_contract_manifest,
1091 }).await?;
1092 if let Err(error) = drain_pending_events(&mut client, &node.node_id, &mut cursor, &hub, &ingress, did_resync, None).await {
1093 ingress_attempt(&ingress, &node.node_id, relay_failure_attempt(&error)).await?;
1094 reject_disconnected_commands(&mut commands, Some(cursor));
1095 continue;
1096 }
1097
1098 let cadence = timings.poll_interval.max(Duration::from_millis(1)).min(Duration::from_millis(250));
1099 let mut snapshot_tick = tokio::time::interval(cadence);
1100 snapshot_tick.reset();
1101 let mut lease_tick = tokio::time::interval(Duration::from_secs(30));
1102 lease_tick.reset();
1103 let disconnect_error = loop {
1104 tokio::select! {
1105 command = commands.recv() => {
1106 let Some(command) = command else { return Ok(()); };
1107 tokio::select! {
1108 result = handle_relay_command(
1109 &mut client, command, &node.node_id, incarnation_id, connection_id,
1110 &mut controller_owned, &hub, &mut cursor, &ingress,
1111 ) => match result {
1112 Ok(()) => {}
1113 Err(error) => break error,
1114 },
1115 changed = force_disconnect.changed() => {
1116 let _ = changed;
1117 break NodeClientError::Io(io::Error::new(io::ErrorKind::ConnectionAborted, "node relay cleanup forced reconnect"));
1118 }
1119 }
1120 }
1121 release = releases.recv() => {
1122 let Some(reply) = release else { return Ok(()); };
1123 let result = release_controller(&mut client, &mut controller_owned).await;
1124 let _ = reply.send(());
1125 if let Err(error) = result { break error; }
1126 }
1127 changed = force_disconnect.changed() => {
1128 let _ = changed;
1129 break NodeClientError::Io(io::Error::new(io::ErrorKind::ConnectionAborted, "node relay cleanup forced reconnect"));
1130 }
1131 frame = client.recv() => {
1132 match frame {
1133 Ok(ServerFrame::Event(envelope)) => {
1134 if let Err(error) = handle_live_node_event(
1135 &mut client,
1136 &node.node_id,
1137 envelope,
1138 &mut cursor,
1139 &hub,
1140 &ingress,
1141 ).await {
1142 break error;
1143 }
1144 }
1145 Ok(ServerFrame::Reply(_)
1146 | ServerFrame::Challenge(_)
1147 | ServerFrame::Hello(_)) => {
1148 break NodeClientError::Protocol(
1149 "node sent an unexpected idle frame".to_owned(),
1150 );
1151 }
1152 Err(error) => break error,
1153 }
1154 }
1155 _ = snapshot_tick.tick() => {
1156 match bounded_node_request(&mut client, NodeRequest::Snapshot).await {
1157 Ok(NodeResponse::Snapshot { event_sequence, snapshot, .. }) => {
1158 if let Err(error) = drain_pending_events(&mut client, &node.node_id, &mut cursor, &hub, &ingress, false, None).await { break error; }
1159 if event_sequence >= cursor.sequence {
1160 cursor.sequence = event_sequence;
1161 ingress_attempt(&ingress, &node.node_id, AttemptResult::Success { cursor, snapshot, gaps: Vec::new() }).await?;
1162 }
1163 }
1164 Ok(_) => break NodeClientError::Protocol("snapshot returned a different response".to_owned()),
1165 Err(error) => break error,
1166 }
1167 }
1168 _ = lease_tick.tick() => {
1169 if controller_owned {
1170 if hub.has_active_operator() {
1171 match acquire_controller(&mut client, connection_id).await {
1172 Ok(owned) => controller_owned = owned,
1173 Err(error) => break error,
1174 }
1175 } else if let Err(error) = release_controller(&mut client, &mut controller_owned).await {
1176 break error;
1177 }
1178 if let Err(error) = drain_pending_events(&mut client, &node.node_id, &mut cursor, &hub, &ingress, false, None).await { break error; }
1179 }
1180 }
1181 changed = shutdown.changed() => {
1182 if changed.is_err() || *shutdown.borrow() {
1183 let _ = release_controller(&mut client, &mut controller_owned).await;
1184 return Ok(());
1185 }
1186 }
1187 }
1188 };
1189 let (error, hard) = sanitize_node_error(&disconnect_error);
1190 tracing::warn!(
1191 node_id = %node.node_id,
1192 connection_id,
1193 cause = %error.message,
1194 category = ?error.category,
1195 hard,
1196 "node dropped from relay",
1197 );
1198 ingress_attempt(&ingress, &node.node_id, AttemptResult::Failure { error, hard }).await?;
1199 reject_disconnected_commands(&mut commands, Some(cursor));
1200 failures = failures.saturating_add(1);
1201 relay_backoff(&mut shutdown, timings, failures, hard).await?;
1202 }
1203}
1204
1205async fn connect_operator(
1216 node: &C2NodeConfig,
1217 adopted: Option<TcpStream>,
1218) -> Result<LocalNodeClient, NodeClientError> {
1219 match node.route {
1220 C2NodeRoute::Local => {
1221 LocalNodeClient::connect(&node.endpoint, &node.node_id, ClientRole::Operator, &node.token).await
1222 }
1223 C2NodeRoute::SshForwardedLoopback(endpoint) => {
1224 LocalNodeClient::connect_loopback(endpoint, &node.node_id, ClientRole::Operator, &node.token).await
1225 }
1226 C2NodeRoute::CallHome => {
1227 let stream = adopted.ok_or_else(|| {
1233 NodeClientError::Protocol(
1234 "call-home node reached the connect step with no adopted stream".to_owned(),
1235 )
1236 })?;
1237 LocalNodeClient::adopt(stream, &node.node_id, ClientRole::Operator, &node.token).await
1238 }
1239 }
1240}
1241
1242async fn bounded_node_request(
1243 client: &mut LocalNodeClient,
1244 request: NodeRequest,
1245) -> Result<NodeResponse, NodeClientError> {
1246 bounded_node_request_with_deadline(client, request, None).await
1247}
1248
1249async fn bounded_node_request_with_deadline(
1250 client: &mut LocalNodeClient,
1251 request: NodeRequest,
1252 relay_deadline: Option<Instant>,
1253) -> Result<NodeResponse, NodeClientError> {
1254 let deadline = request_budget(&request, relay_deadline, Instant::now());
1255 if deadline.is_zero() {
1256 return Err(NodeClientError::Frame(FrameError::PrefixTimedOut));
1257 }
1258 match timeout(deadline, client.request(request)).await {
1259 Ok(result) => result,
1260 Err(_) => Err(NodeClientError::Frame(FrameError::PrefixTimedOut)),
1261 }
1262}
1263
1264fn relay_request_deadline(request: &NodeRequest, now: Instant) -> Option<Instant> {
1265 matches!(request, NodeRequest::ResumeSessionRecord { .. })
1266 .then(|| now + node_request_deadline(request))
1267}
1268
1269fn request_budget(request: &NodeRequest, relay_deadline: Option<Instant>, now: Instant) -> Duration {
1270 relay_deadline
1271 .map(|deadline| deadline.saturating_duration_since(now))
1272 .unwrap_or_else(|| node_request_deadline(request))
1273}
1274
1275const WORKSPACE_INSPECTION_RELAY_DEADLINE: Duration = Duration::from_secs(15);
1281
1282fn node_request_deadline(request: &NodeRequest) -> Duration {
1283 match request {
1284 NodeRequest::Snapshot
1285 | NodeRequest::Resync { .. }
1286 | NodeRequest::ArmHarnessMcpReservation { .. }
1287 | NodeRequest::ActivateHarnessMcpReservation { .. }
1288 | NodeRequest::AbortHarnessMcpReservation { .. }
1289 | NodeRequest::PutHarnessMcpReplyChunk { .. }
1290 | NodeRequest::RejectHarnessMcpCall { .. }
1291 | NodeRequest::BrowseHostDirectories { .. }
1292 | NodeRequest::ReadWorkspaceFile { .. }
1293 | NodeRequest::WriteWorkspaceFile { .. }
1294 | NodeRequest::ReadGitHistory { .. }
1295 | NodeRequest::ReadGitDiff { .. }
1296 | NodeRequest::AcquireController { .. }
1297 | NodeRequest::ReleaseController
1298 | NodeRequest::RenameSessionRecord { .. }
1299 | NodeRequest::SetSessionTask { .. }
1300 | NodeRequest::ForgetSessionRecord { .. } => Duration::from_secs(5),
1301 NodeRequest::InspectWorkspace { .. } => WORKSPACE_INSPECTION_RELAY_DEADLINE,
1319 NodeRequest::CreateWorkspaceFile { .. }
1320 | NodeRequest::CreateWorkspaceDirectory { .. } => {
1321 WORKSPACE_ENTRY_CREATE_RELAY_DEADLINE
1322 }
1323 NodeRequest::CatalogNativeSessions { .. }
1324 | NodeRequest::PageNativeSessions { .. }
1325 | NodeRequest::PreviewNativeSession { .. }
1326 | NodeRequest::IndexNativeSession { .. }
1327 | NodeRequest::PreviewSessionRecord { .. } => NATIVE_SESSION_REQUEST_DEADLINE,
1328 NodeRequest::CreateStandaloneWorkspace { .. }
1329 | NodeRequest::CreateWorktree { .. }
1330 | NodeRequest::RemoveWorktree { .. }
1331 | NodeRequest::CleanupManagedWorktree { .. } => Duration::from_secs(240),
1332 NodeRequest::Spawn { .. }
1333 | NodeRequest::Resume { .. }
1334 | NodeRequest::Stop { .. } => Duration::from_secs(15),
1335 NodeRequest::SpawnSpec { spec } =>
1336 Duration::from_millis(spec.deadline_ms.get()) + NODE_REQUEST_IO_HEADROOM,
1337 NodeRequest::SpawnSpecWithHarnessMcp { deadline_unix_ms, .. } =>
1338 Duration::from_millis(deadline_unix_ms.saturating_sub(unix_ms()))
1339 + NODE_REQUEST_IO_HEADROOM,
1340 NodeRequest::SpawnManagedWorktree { request } =>
1341 Duration::from_millis(request.spawn_spec.deadline_ms.get())
1342 + NODE_REQUEST_IO_HEADROOM,
1343 NodeRequest::ResumeSessionRecord { .. } =>
1344 MANAGED_RESUME_SETTLE_DEADLINE + NODE_REQUEST_IO_HEADROOM,
1345 _ => Duration::from_secs(10),
1346 }
1347}
1348
1349fn validate_resync(
1350 previous: u64,
1351 hello: u64,
1352 current: u64,
1353 oldest_available_sequence: u64,
1354 events: &[NodeEventEnvelope],
1355) -> Vec<GapKind> {
1356 let mut gaps = validate_events(
1357 previous,
1358 current,
1359 oldest_available_sequence,
1360 events,
1361 );
1362 if current < hello && !gaps.contains(&GapKind::CursorRegression) {
1363 gaps.push(GapKind::CursorRegression);
1364 }
1365 gaps
1366}
1367
1368async fn ingress_attempt(
1369 ingress: &mpsc::Sender<Attempt>,
1370 node_id: &NodeId,
1371 result: AttemptResult,
1372) -> io::Result<()> {
1373 ingress.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result })
1374 .await.map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "inventory owner closed"))
1375}
1376
1377async fn relay_backoff(
1378 shutdown: &mut watch::Receiver<bool>,
1379 timings: C2Timings,
1380 failures: usize,
1381 hard: bool,
1382) -> io::Result<()> {
1383 let delay = if hard || failures >= timings.transient_backoffs.len() {
1384 timings.parked_backoff
1385 } else {
1386 timings.transient_backoffs[failures.saturating_sub(1)]
1387 };
1388 tokio::select! {
1389 _ = sleep(delay) => Ok(()),
1390 _ = shutdown.changed() => {
1391 Ok(())
1392 }
1393 }
1394}
1395
1396fn relay_failure_attempt(error: &NodeClientError) -> AttemptResult {
1397 let (error, hard) = sanitize_node_error(error);
1398 AttemptResult::Failure { error, hard }
1399}
1400
1401async fn handle_relay_command(
1402 client: &mut LocalNodeClient,
1403 command: RelayCommand,
1404 node_id: &NodeId,
1405 incarnation_id: NodeIncarnationId,
1406 connection_id: u64,
1407 controller_owned: &mut bool,
1408 hub: &OperatorHub,
1409 cursor: &mut NodeCursor,
1410 ingress: &mpsc::Sender<Attempt>,
1411) -> Result<(), NodeClientError> {
1412 match command {
1413 RelayCommand::Request { operator_connection_id, expected_incarnation_id, request, reply } => {
1414 let relay_deadline = relay_request_deadline(&request, Instant::now());
1415 let expected_spawn = expected_spawn_request(&request);
1416 let expected_provider_session_index = matches!(
1417 &request,
1418 NodeRequest::IndexProviderSession { .. }
1419 ).then(|| request.clone());
1420 let expected_native_session = matches!(
1421 &request,
1422 NodeRequest::CatalogNativeSessions { .. }
1423 | NodeRequest::PageNativeSessions { .. }
1424 | NodeRequest::PreviewNativeSession { .. }
1425 | NodeRequest::IndexNativeSession { .. }
1426 ).then(|| request.clone());
1427 let expected_workspace_content = matches!(
1428 &request,
1429 NodeRequest::ReadWorkspaceFile { .. }
1430 | NodeRequest::WriteWorkspaceFile { .. }
1431 | NodeRequest::CreateWorkspaceFile { .. }
1432 | NodeRequest::CreateWorkspaceDirectory { .. }
1433 | NodeRequest::ReadGitHistory { .. }
1434 | NodeRequest::ReadGitDiff { .. }
1435 ).then(|| request.clone());
1436 let expected_session_task = matches!(
1437 &request,
1438 NodeRequest::SetSessionTask { .. }
1439 ).then(|| request.clone());
1440 let expected_harness_mcp = (request.required_capability()
1441 == Some(gate4agent_node_protocol::NODE_HARNESS_MCP_READ_PROXY_CAPABILITY))
1442 .then(|| request.clone());
1443 if !hub.is_active(operator_connection_id) {
1444 let _ = reply.send(Err(relay_failure(C2RelayFailureCode::ClientLagged, "C2 operator connection is no longer active", Some(incarnation_id))));
1445 return Ok(());
1446 }
1447 if expected_incarnation_id != incarnation_id {
1448 let _ = reply.send(Err(relay_failure(C2RelayFailureCode::StaleNodeIncarnation, "node incarnation changed", Some(incarnation_id))));
1449 return Ok(());
1450 }
1451 if matches!(&request, NodeRequest::IndexNativeSession { selection, .. }
1452 if selection.route.scope
1453 != gate4agent_node_protocol::NativeSessionCatalogScope::Workspace)
1454 {
1455 let _ = reply.send(Err(relay_failure(
1456 C2RelayFailureCode::RequestForbidden,
1457 "external native sessions must be registered as workspaces before indexing",
1458 Some(incarnation_id),
1459 )));
1460 return Ok(());
1461 }
1462 if !is_read_only_request(&request) && !*controller_owned {
1463 match acquire_controller_with_deadline(client, connection_id, relay_deadline).await {
1464 Ok(owned) if owned => *controller_owned = true,
1465 Ok(_) => {
1466 let _ = reply.send(Err(relay_failure(C2RelayFailureCode::RelayBusy, "node controller lease is unavailable", Some(incarnation_id))));
1467 return Ok(());
1468 }
1469 Err(error) if relay_node_failure(&error).is_some() => {
1470 let failure = relay_node_failure(&error)
1471 .expect("guarded node request failure");
1472 let _ = reply.send(Ok(RoutedNodeResponse {
1473 node_id: node_id.clone(),
1474 incarnation_id,
1475 response: Err(failure),
1476 }));
1477 return Ok(());
1478 }
1479 Err(error) => {
1480 let _ = reply.send(Err(relay_failure(C2RelayFailureCode::NodeOffline, "node relay disconnected", Some(incarnation_id))));
1481 return Err(error);
1482 }
1483 }
1484 }
1485 let response = match bounded_node_request_with_deadline(client, request, relay_deadline).await {
1486 Ok(response) => Ok(response),
1487 Err(NodeClientError::Node(failure)) => Err(failure),
1488 Err(error @ NodeClientError::UnsupportedCapability(_)) => {
1489 let failure = relay_node_failure(&error)
1490 .expect("unsupported capability is a routed node failure");
1491 let _ = reply.send(Ok(RoutedNodeResponse {
1492 node_id: node_id.clone(),
1493 incarnation_id,
1494 response: Err(failure),
1495 }));
1496 return Ok(());
1497 }
1498 Err(error) => {
1499 let _ = reply.send(Err(relay_failure(C2RelayFailureCode::NodeOffline, "node relay disconnected", Some(incarnation_id))));
1500 return Err(error);
1501 }
1502 };
1503 if let Err(message) = validate_spawn_spec_response(
1504 expected_spawn.as_ref(),
1505 &response,
1506 incarnation_id,
1507 ) {
1508 let _ = reply.send(Err(relay_failure(
1509 C2RelayFailureCode::NodeOffline,
1510 "node relay returned an invalid spawn receipt",
1511 Some(incarnation_id),
1512 )));
1513 return Err(NodeClientError::Protocol(message.to_owned()));
1514 }
1515 if let Err(message) = validate_provider_session_index_response(
1516 expected_provider_session_index.as_ref(),
1517 &response,
1518 ) {
1519 let _ = reply.send(Err(relay_failure(
1520 C2RelayFailureCode::NodeOffline,
1521 "node relay returned an invalid provider session index response",
1522 Some(incarnation_id),
1523 )));
1524 return Err(NodeClientError::Protocol(message.to_owned()));
1525 }
1526 if let Err(message) = validate_native_session_response(
1527 expected_native_session.as_ref(),
1528 &response,
1529 ) {
1530 let _ = reply.send(Err(relay_failure(
1531 C2RelayFailureCode::NodeOffline,
1532 "node relay returned an invalid native session response",
1533 Some(incarnation_id),
1534 )));
1535 return Err(NodeClientError::Protocol(message.to_owned()));
1536 }
1537 if let Err(message) = validate_workspace_content_response(
1538 expected_workspace_content.as_ref(),
1539 &response,
1540 ) {
1541 let _ = reply.send(Err(relay_failure(
1542 C2RelayFailureCode::NodeOffline,
1543 "node relay returned an invalid workspace content response",
1544 Some(incarnation_id),
1545 )));
1546 return Err(NodeClientError::Protocol(message.to_owned()));
1547 }
1548 if let Err(message) = validate_session_task_response(
1549 expected_session_task.as_ref(),
1550 &response,
1551 ) {
1552 let _ = reply.send(Err(relay_failure(
1553 C2RelayFailureCode::NodeOffline,
1554 "node relay returned an invalid session task response",
1555 Some(incarnation_id),
1556 )));
1557 return Err(NodeClientError::Protocol(message.to_owned()));
1558 }
1559 if let Err(message) = validate_harness_mcp_response(
1560 expected_harness_mcp.as_ref(),
1561 &response,
1562 ) {
1563 let _ = reply.send(Err(relay_failure(
1564 C2RelayFailureCode::NodeOffline,
1565 "node relay returned an invalid harness MCP response",
1566 Some(incarnation_id),
1567 )));
1568 return Err(NodeClientError::Protocol(message.to_owned()));
1569 }
1570 drain_pending_events(client, node_id, cursor, hub, ingress, false, relay_deadline).await?;
1571 update_inventory_from_response(node_id, cursor, &response, ingress).await
1572 .map_err(NodeClientError::Io)?;
1573 let response = response
1574 .map(|response| C2NodeResponse::from_node_response_with_control_detail(&response))
1575 .map_err(|failure| C2NodeFailure::from(&failure));
1576 let _ = reply.send(Ok(RoutedNodeResponse { node_id: node_id.clone(), incarnation_id, response }));
1577 }
1578 }
1579 Ok(())
1580}
1581
1582fn validate_session_task_response(
1583 expected: Option<&NodeRequest>,
1584 response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1585) -> Result<(), &'static str> {
1586 match (expected, response) {
1587 (Some(_), Err(_)) | (None, Err(_)) => Ok(()),
1588 (
1589 Some(NodeRequest::SetSessionTask {
1590 record_id,
1591 expected_revision,
1592 target,
1593 }),
1594 Ok(NodeResponse::SessionRecordUpdated { record }),
1595 ) if session_task_record_matches(record, record_id, *expected_revision, target) => Ok(()),
1596 (Some(NodeRequest::SetSessionTask { .. }), Ok(_)) => {
1597 Err("session task response does not match the routed request")
1598 }
1599 (None, Ok(_)) => Ok(()),
1600 (Some(_), Ok(_)) => Ok(()),
1601 }
1602}
1603
1604fn validate_harness_mcp_response(
1605 expected: Option<&NodeRequest>,
1606 response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1607) -> Result<(), &'static str> {
1608 use NodeRequest as Request;
1609 use NodeResponse as Response;
1610 let valid = match (expected, response) {
1611 (Some(Request::ArmHarnessMcpReservation { reservation_id, activation_digest, expires_at_unix_ms, .. }),
1612 Ok(Response::Armed { reservation_id: echoed_id, activation_digest: echoed_digest, expires_at_unix_ms: echoed_expiry })) =>
1613 reservation_id == echoed_id && activation_digest == echoed_digest
1614 && expires_at_unix_ms == echoed_expiry,
1615 (Some(Request::SpawnSpecWithHarnessMcp { reservation_id, activation_digest, .. }),
1616 Ok(Response::Spawned { reservation_id: echoed_id, activation_digest: echoed_digest, receipt })) =>
1617 reservation_id == echoed_id && activation_digest == echoed_digest
1618 && receipt.harness_mcp_proxy.as_ref().is_some_and(|proxy| {
1619 &proxy.reservation_id == reservation_id
1620 && &proxy.activation_digest == activation_digest
1621 }),
1622 (Some(Request::ActivateHarnessMcpReservation { reservation_id, activation_digest, record_id, session }),
1623 Ok(Response::Activated { reservation_id: echoed_id, activation_digest: echoed_digest, record_id: echoed_record, session: echoed_session })) =>
1624 reservation_id == echoed_id && activation_digest == echoed_digest
1625 && record_id == echoed_record && session == echoed_session,
1626 (Some(Request::AbortHarnessMcpReservation { reservation_id, activation_digest }),
1627 Ok(Response::Aborted { reservation_id: echoed_id, activation_digest: echoed_digest })) =>
1628 reservation_id == echoed_id && activation_digest == echoed_digest,
1629 (Some(Request::PutHarnessMcpReplyChunk { reservation_id, activation_digest, record_id, session, call_id, offset, final_chunk, chunk_hex }),
1630 Ok(Response::ReplyChunkAccepted { reservation_id: echoed_id, activation_digest: echoed_digest, record_id: echoed_record, session: echoed_session, call_id: echoed_call, next_offset, completed })) =>
1631 reservation_id == echoed_id && activation_digest == echoed_digest
1632 && record_id == echoed_record && session == echoed_session && call_id == echoed_call
1633 && offset.checked_add(u32::try_from(chunk_hex.raw_len()).unwrap_or(u32::MAX))
1634 == Some(*next_offset) && completed == final_chunk,
1635 (Some(Request::RejectHarnessMcpCall { reservation_id, activation_digest, record_id, session, call_id, .. }),
1636 Ok(Response::CallRejected { reservation_id: echoed_id, activation_digest: echoed_digest, record_id: echoed_record, session: echoed_session, call_id: echoed_call })) =>
1637 reservation_id == echoed_id && activation_digest == echoed_digest
1638 && record_id == echoed_record && session == echoed_session && call_id == echoed_call,
1639 (Some(_), Err(_)) | (None, Err(_)) => true,
1640 (Some(_), Ok(_)) => false,
1641 (None, Ok(response)) => !response.requires_harness_mcp_proxy_capability(),
1642 };
1643 if valid { Ok(()) } else { Err("harness MCP response does not match the routed request") }
1644}
1645
1646fn session_task_record_matches(
1647 record: &gate4agent_node_protocol::ManagedSessionRecord,
1648 record_id: &gate4agent_node_protocol::SessionRecordId,
1649 expected_revision: u64,
1650 target: &gate4agent_node_protocol::SessionTaskTargetV1,
1651) -> bool {
1652 if &record.record_id != record_id { return false; }
1653 let next_revision = expected_revision.checked_add(1);
1654 match target {
1655 gate4agent_node_protocol::SessionTaskTargetV1::New => record.task_binding.as_ref()
1656 .is_some_and(|binding| Some(binding.revision) == next_revision && binding.task_id.is_some()),
1657 gate4agent_node_protocol::SessionTaskTargetV1::Existing { task_id } => record.task_binding.as_ref()
1658 .is_some_and(|binding| (binding.revision == expected_revision || Some(binding.revision) == next_revision)
1659 && binding.task_id.as_ref() == Some(task_id)),
1660 gate4agent_node_protocol::SessionTaskTargetV1::Clear => match &record.task_binding {
1661 None => expected_revision == 0,
1662 Some(binding) => binding.task_id.is_none()
1663 && (binding.revision == expected_revision || Some(binding.revision) == next_revision),
1664 },
1665 }
1666}
1667
1668fn validate_workspace_content_response(
1669 expected: Option<&NodeRequest>,
1670 response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1671) -> Result<(), &'static str> {
1672 match (expected, response) {
1673 (Some(_), Err(_)) | (None, Err(_)) => Ok(()),
1674 (
1675 Some(NodeRequest::ReadWorkspaceFile { workspace_id, path }),
1676 Ok(NodeResponse::WorkspaceFileRead { file }),
1677 ) if &file.workspace_id == workspace_id && &file.path == path => Ok(()),
1678 (
1679 Some(NodeRequest::WriteWorkspaceFile { workspace_id, path, text, .. }),
1680 Ok(NodeResponse::WorkspaceFileWritten { file }),
1681 ) if &file.workspace_id == workspace_id
1682 && &file.path == path
1683 && matches!(
1684 &file.content,
1685 gate4agent_node_protocol::WorkspaceFileContent::Utf8 {
1686 text: written,
1687 byte_len,
1688 } if written == text
1689 && u32::try_from(text.len()).ok() == Some(*byte_len)
1690 ) => Ok(()),
1691 (
1692 Some(NodeRequest::CreateWorkspaceFile { workspace_id, path }),
1693 Ok(NodeResponse::WorkspaceFileCreated { file }),
1694 ) if &file.workspace_id == workspace_id
1695 && &file.path == path
1696 && file.revision.is_some()
1697 && matches!(
1698 &file.content,
1699 gate4agent_node_protocol::WorkspaceFileContent::Utf8 {
1700 text,
1701 byte_len: 0,
1702 } if text.is_empty()
1703 ) => Ok(()),
1704 (
1705 Some(NodeRequest::CreateWorkspaceDirectory { workspace_id, path }),
1706 Ok(NodeResponse::WorkspaceDirectoryCreated {
1707 workspace_id: actual_workspace_id,
1708 entry,
1709 }),
1710 ) if actual_workspace_id == workspace_id
1711 && &entry.relative_path == path
1712 && entry.kind == gate4agent_node_protocol::WorkspaceEntryKind::Directory => Ok(()),
1713 (
1714 Some(NodeRequest::ReadGitHistory { workspace_id, .. }),
1715 Ok(NodeResponse::GitHistoryRead { workspace_id: actual, .. }),
1716 ) if actual == workspace_id => Ok(()),
1717 (
1718 Some(NodeRequest::ReadGitDiff { workspace_id, request }),
1719 Ok(NodeResponse::GitDiffRead { workspace_id: actual, diff }),
1720 ) if actual == workspace_id && diff.mode == request.mode && diff.path == request.path => Ok(()),
1721 (Some(_), Ok(_)) => Err("workspace content response does not match routed request"),
1722 (
1723 None,
1724 Ok(NodeResponse::WorkspaceFileRead { .. }
1725 | NodeResponse::WorkspaceFileWritten { .. }
1726 | NodeResponse::WorkspaceFileCreated { .. }
1727 | NodeResponse::WorkspaceDirectoryCreated { .. }
1728 | NodeResponse::GitHistoryRead { .. }
1729 | NodeResponse::GitDiffRead { .. }),
1730 ) => Err("unexpected workspace content response"),
1731 (None, Ok(_)) => Ok(()),
1732 }
1733}
1734
1735enum ExpectedSpawnRequest {
1736 Spec(SpawnSpec),
1737 ManagedV1(ManagedWorktreeSpawnRequest),
1738 ManagedV2(ManagedWorktreeSpawnRequestV2),
1739}
1740
1741fn expected_spawn_request(request: &NodeRequest) -> Option<ExpectedSpawnRequest> {
1742 match request {
1743 NodeRequest::SpawnSpec { spec } => Some(ExpectedSpawnRequest::Spec(spec.clone())),
1744 NodeRequest::SpawnManagedWorktree { request } => {
1745 Some(ExpectedSpawnRequest::ManagedV1(request.clone()))
1746 }
1747 NodeRequest::SpawnManagedWorktreeV2 { request } => {
1748 Some(ExpectedSpawnRequest::ManagedV2(request.clone()))
1749 }
1750 _ => None,
1751 }
1752}
1753
1754fn validate_provider_session_index_response(
1755 expected: Option<&NodeRequest>,
1756 response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1757) -> Result<(), &'static str> {
1758 match (expected, response) {
1759 (
1760 Some(NodeRequest::IndexProviderSession {
1761 workspace_id,
1762 provider,
1763 identity,
1764 ..
1765 }),
1766 Ok(NodeResponse::ProviderSessionIndexed { record }),
1767 ) if &record.workspace_id == workspace_id
1768 && &record.provider == provider
1769 && record.provider_session.as_ref() == Some(identity) => Ok(()),
1770 (Some(_), Ok(NodeResponse::ProviderSessionIndexed { .. })) => {
1771 Err("provider session index response does not match routed request")
1772 }
1773 (Some(_), Ok(_)) => Err("provider session index request returned a different response"),
1774 (None, Ok(NodeResponse::ProviderSessionIndexed { .. })) => {
1775 Err("unexpected provider session index response for a different node request")
1776 }
1777 (Some(_), Err(_)) | (None, _) => Ok(()),
1778 }
1779}
1780
1781fn validate_native_session_response(
1782 expected: Option<&NodeRequest>,
1783 response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1784) -> Result<(), &'static str> {
1785 match (expected, response) {
1786 (
1787 Some(NodeRequest::CatalogNativeSessions { route, .. }),
1788 Ok(NodeResponse::NativeSessionsCataloged {
1789 route: echoed_route,
1790 ..
1791 }),
1792 ) if echoed_route == route => Ok(()),
1793 (
1794 Some(NodeRequest::PageNativeSessions {
1795 route,
1796 window,
1797 catalog_revision,
1798 ..
1799 }),
1800 Ok(NodeResponse::NativeSessionsPaged {
1801 route: echoed_route,
1802 page,
1803 }),
1804 ) if echoed_route == route
1805 && page.window == *window
1806 && page.revision == *catalog_revision => Ok(()),
1807 (
1808 Some(NodeRequest::PreviewNativeSession { selection, .. }),
1809 Ok(NodeResponse::NativeSessionPreviewed {
1810 selection: echoed_selection,
1811 ..
1812 }),
1813 ) if echoed_selection == selection => Ok(()),
1814 (
1815 Some(NodeRequest::IndexNativeSession { selection, .. }),
1816 Ok(NodeResponse::NativeSessionIndexed {
1817 selection: echoed_selection,
1818 record,
1819 }),
1820 ) if echoed_selection == selection
1821 && selection.route.scope
1822 == gate4agent_node_protocol::NativeSessionCatalogScope::Workspace
1823 && selection.route.workspace_id.as_ref() == Some(&record.workspace_id)
1824 && selection.route.provider == record.provider => Ok(()),
1825 (Some(_), Err(_)) => Ok(()),
1826 (Some(_), Ok(_)) => Err("native session response does not match routed request"),
1827 (
1828 None,
1829 Ok(
1830 NodeResponse::NativeSessionsCataloged { .. }
1831 | NodeResponse::NativeSessionsPaged { .. }
1832 | NodeResponse::NativeSessionPreviewed { .. }
1833 | NodeResponse::NativeSessionIndexed { .. },
1834 ),
1835 ) => Err("unexpected native session response for a different node request"),
1836 (None, _) => Ok(()),
1837 }
1838}
1839
1840fn validate_spawn_spec_response(
1841 expected: Option<&ExpectedSpawnRequest>,
1842 response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1843 relay_incarnation_id: NodeIncarnationId,
1844) -> Result<(), &'static str> {
1845 match (expected, response) {
1846 (Some(ExpectedSpawnRequest::Spec(spec)), Ok(NodeResponse::SpawnSpecAccepted { receipt })) => {
1847 validate_spawn_receipt(spec, receipt, relay_incarnation_id)
1848 }
1849 (
1850 Some(ExpectedSpawnRequest::ManagedV1(request)),
1851 Ok(NodeResponse::ManagedWorktreeSpawnAccepted { receipt }),
1852 ) => validate_managed_spawn_receipt(
1853 &request.spawn_spec,
1854 &request.worktree_profile_id,
1855 receipt,
1856 relay_incarnation_id,
1857 ),
1858 (
1859 Some(ExpectedSpawnRequest::ManagedV2(request)),
1860 Ok(NodeResponse::ManagedWorktreeSpawnAccepted { receipt }),
1861 ) if receipt.lease.profile_revision == request.expected_profile_revision => {
1862 validate_managed_spawn_receipt(
1863 &request.spawn_spec,
1864 &request.worktree_profile_id,
1865 receipt,
1866 relay_incarnation_id,
1867 )
1868 }
1869 (
1870 Some(ExpectedSpawnRequest::ManagedV2(_)),
1871 Ok(NodeResponse::ManagedWorktreeSpawnAccepted { .. }),
1872 ) => Err("managed spawn receipt profile revision does not match routed request"),
1873 (Some(_), Ok(_)) => return Err("spawn spec request returned a different response"),
1874 (Some(_), Err(_)) => return Ok(()),
1875 (None, Ok(NodeResponse::SpawnSpecAccepted { .. }
1876 | NodeResponse::ManagedWorktreeSpawnAccepted { .. })) => {
1877 return Err("unexpected spawn receipt for a different node request");
1878 }
1879 (None, _) => return Ok(()),
1880 }
1881}
1882
1883fn validate_managed_spawn_receipt(
1884 spawn_spec: &SpawnSpec,
1885 worktree_profile_id: &gate4agent_node_protocol::WorktreeProfileId,
1886 receipt: &gate4agent_node_protocol::ManagedWorktreeSpawnReceipt,
1887 relay_incarnation_id: NodeIncarnationId,
1888) -> Result<(), &'static str> {
1889 if receipt.lease.source_workspace_id != spawn_spec.target.workspace_id
1890 || &receipt.lease.profile_id != worktree_profile_id
1891 || receipt.lease.state != ManagedWorktreeLeaseState::InUse
1892 || receipt.lease.cleanup_failure.is_some()
1893 || receipt.lease.active_session_count != 1
1894 || receipt.spawn.target.node_id != spawn_spec.target.node_id
1895 || receipt.spawn.target.workspace_id != spawn_spec.target.workspace_id
1896 || receipt.spawn.target.worktree_id.as_ref() != Some(&receipt.lease.workspace_id)
1897 || receipt.spawn.session.workspace_id != receipt.lease.workspace_id
1898 {
1899 return Err("managed spawn receipt does not match routed request");
1900 }
1901 let mut resolved_spec = spawn_spec.clone();
1902 resolved_spec.target.worktree_id = Some(receipt.lease.workspace_id.clone());
1903 validate_spawn_receipt(&resolved_spec, &receipt.spawn, relay_incarnation_id)
1904}
1905
1906fn validate_spawn_receipt(
1907 spec: &SpawnSpec,
1908 receipt: &ResolvedSpawnReceipt,
1909 relay_incarnation_id: NodeIncarnationId,
1910) -> Result<(), &'static str> {
1911 if receipt.incarnation_id != relay_incarnation_id
1912 || !receipt.context_binding_is_valid()
1913 || receipt.target != spec.target
1914 || receipt.profile_id != spec.profile_id
1915 || receipt.idempotency_key != spec.idempotency_key
1916 || receipt.deadline_ms != spec.deadline_ms
1917 || receipt.required_capabilities != spec.required_capabilities
1918 || receipt.bundle.as_ref().is_some_and(|bundle| {
1919 receipt.bundle_id.as_ref() != Some(&bundle.id)
1920 })
1921 || &receipt.session.workspace_id
1922 != spec
1923 .target
1924 .worktree_id
1925 .as_ref()
1926 .unwrap_or(&spec.target.workspace_id)
1927 {
1928 return Err("spawn receipt does not match routed request");
1929 }
1930 if !required_override_matches(&spec.overrides.provider, &receipt.provider)
1931 || !required_override_matches(&spec.overrides.mode, &receipt.mode)
1932 || !required_override_matches(&spec.overrides.terminal_size, &receipt.terminal_size)
1933 || !optional_override_matches(&spec.overrides.bundle_id, &receipt.bundle_id)
1934 || !optional_override_matches(&spec.overrides.context_id, &receipt.context_id)
1935 || !environment_profile_override_matches(
1936 &spec.overrides.environment_profile_id,
1937 receipt.environment_profile.as_ref(),
1938 )
1939 {
1940 return Err("spawn receipt contradicts explicit overrides");
1941 }
1942 match &spec.overrides.prompt {
1943 SpawnOverride::Inherit => {}
1944 SpawnOverride::Set { value }
1945 if receipt.prompt.present
1946 && receipt.prompt.byte_len == u32::try_from(value.byte_len()).unwrap_or(0) => {}
1947 SpawnOverride::Clear if !receipt.prompt.present && receipt.prompt.byte_len == 0 => {}
1948 SpawnOverride::Set { .. } | SpawnOverride::Clear => {
1949 return Err("spawn receipt contradicts explicit prompt override");
1950 }
1951 }
1952 Ok(())
1953}
1954
1955fn environment_profile_override_matches(
1956 expected: &SpawnOverride<gate4agent_node_protocol::SpawnEnvironmentProfileId>,
1957 actual: Option<&gate4agent_node_protocol::ResolvedEnvironmentProfileReceipt>,
1958) -> bool {
1959 match expected {
1960 SpawnOverride::Inherit => true,
1961 SpawnOverride::Set { value } => {
1962 actual.is_some_and(|receipt| &receipt.profile_id == value)
1963 }
1964 SpawnOverride::Clear => actual.is_none(),
1965 }
1966}
1967
1968fn required_override_matches<T: Eq>(override_value: &SpawnOverride<T>, actual: &T) -> bool {
1969 match override_value {
1970 SpawnOverride::Inherit => true,
1971 SpawnOverride::Set { value } => value == actual,
1972 SpawnOverride::Clear => false,
1973 }
1974}
1975
1976fn optional_override_matches<T: Eq>(
1977 override_value: &SpawnOverride<T>,
1978 actual: &Option<T>,
1979) -> bool {
1980 match override_value {
1981 SpawnOverride::Inherit => true,
1982 SpawnOverride::Set { value } => actual.as_ref() == Some(value),
1983 SpawnOverride::Clear => actual.is_none(),
1984 }
1985}
1986
1987fn relay_node_failure(error: &NodeClientError) -> Option<C2NodeFailure> {
1988 match error {
1989 NodeClientError::Node(failure) => Some(C2NodeFailure::from(failure)),
1990 NodeClientError::UnsupportedCapability(_) => Some(C2NodeFailure {
1991 code: NodeFailureCode::UnsupportedCapability,
1992 message: "required capability unavailable".to_owned(),
1993 }),
1994 NodeClientError::Io(_)
1995 | NodeClientError::Frame(_)
1996 | NodeClientError::Protocol(_)
1997 | NodeClientError::BuildStampMismatch { .. }
1998 | NodeClientError::AuthenticationTimedOut
1999 | NodeClientError::Authentication(_)
2000 | NodeClientError::RequestIdExhausted => None,
2001 }
2002}
2003
2004fn is_read_only_request(request: &NodeRequest) -> bool {
2005 matches!(request,
2006 NodeRequest::Snapshot
2007 | NodeRequest::Resync { .. }
2008 | NodeRequest::BrowseHostDirectories { .. }
2009 | NodeRequest::InspectWorkspace { .. }
2010 | NodeRequest::ReadWorkspaceFile { .. }
2011 | NodeRequest::ReadGitHistory { .. }
2012 | NodeRequest::ReadGitDiff { .. }
2013 | NodeRequest::CatalogNativeSessions { .. }
2014 | NodeRequest::PageNativeSessions { .. }
2015 | NodeRequest::PreviewNativeSession { .. }
2016 | NodeRequest::PreviewSessionRecord { .. }
2017 )
2018}
2019
2020async fn acquire_controller(
2021 client: &mut LocalNodeClient,
2022 connection_id: u64,
2023) -> Result<bool, NodeClientError> {
2024 acquire_controller_with_deadline(client, connection_id, None).await
2025}
2026
2027async fn acquire_controller_with_deadline(
2028 client: &mut LocalNodeClient,
2029 connection_id: u64,
2030 relay_deadline: Option<Instant>,
2031) -> Result<bool, NodeClientError> {
2032 match bounded_node_request_with_deadline(
2033 client,
2034 NodeRequest::AcquireController { lease_ms: gate4agent_node_protocol::MAX_CONTROLLER_LEASE_MS },
2035 relay_deadline,
2036 ).await? {
2037 NodeResponse::Controller { controller } => Ok(controller.as_ref().is_some_and(|state| state.connection_id == connection_id)),
2038 _ => Err(NodeClientError::Protocol("controller acquisition returned a different response".to_owned())),
2039 }
2040}
2041
2042async fn release_controller(
2043 client: &mut LocalNodeClient,
2044 controller_owned: &mut bool,
2045) -> Result<(), NodeClientError> {
2046 if !*controller_owned { return Ok(()); }
2047 match bounded_node_request(client, NodeRequest::ReleaseController).await? {
2048 NodeResponse::Controller { .. } => { *controller_owned = false; Ok(()) }
2049 _ => Err(NodeClientError::Protocol("controller release returned a different response".to_owned())),
2050 }
2051}
2052
2053async fn update_inventory_from_response(
2054 node_id: &NodeId,
2055 cursor: &mut NodeCursor,
2056 response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
2057 ingress: &mpsc::Sender<Attempt>,
2058) -> io::Result<()> {
2059 match response {
2060 Ok(NodeResponse::Snapshot { event_sequence, snapshot, .. })
2061 | Ok(NodeResponse::Resync { event_sequence, snapshot, .. }) => {
2062 if *event_sequence < cursor.sequence { return Ok(()); }
2063 cursor.sequence = *event_sequence;
2064 ingress_attempt(ingress, node_id, AttemptResult::Success { cursor: *cursor, snapshot: snapshot.clone(), gaps: Vec::new() }).await
2065 }
2066 _ => Ok(()),
2067 }
2068}
2069
2070fn publish_recovered_events(
2071 node_id: &NodeId,
2072 incarnation_id: NodeIncarnationId,
2073 events: &[NodeEventEnvelope],
2074 hub: &OperatorHub,
2075) {
2076 for envelope in events {
2077 if let Some(event) = routed_recovered_node_event(node_id, incarnation_id, envelope) {
2078 hub.publish(event);
2079 }
2080 }
2081}
2082
2083fn routed_recovered_node_event(
2084 node_id: &NodeId,
2085 incarnation_id: NodeIncarnationId,
2086 envelope: &NodeEventEnvelope,
2087) -> Option<RoutedNodeEvent> {
2088 (!matches!(&envelope.event, NodeEvent::HarnessMcpReadCall { .. })).then(|| {
2089 RoutedNodeEvent {
2090 node_id: node_id.clone(),
2091 cursor: NodeCursor { incarnation_id, sequence: envelope.sequence },
2092 event: C2NodeEvent::from_node_event_with_control_detail(&envelope.event),
2093 }
2094 })
2095}
2096
2097fn routed_transient_node_event(
2098 node_id: &NodeId,
2099 cursor: NodeCursor,
2100 envelope: &NodeEventEnvelope,
2101) -> Option<RoutedNodeEvent> {
2102 matches!(&envelope.event, NodeEvent::HarnessMcpReadCall { .. }).then(|| RoutedNodeEvent {
2103 node_id: node_id.clone(),
2104 cursor,
2105 event: C2NodeEvent::from(&envelope.event),
2106 })
2107}
2108
2109fn route_agent_stream_event(
2146 node_id: &NodeId,
2147 cursor: &mut NodeCursor,
2148 envelope: &NodeEventEnvelope,
2149) -> Option<RoutedNodeEvent> {
2150 if !matches!(&envelope.event, NodeEvent::AgentStream { .. }) {
2151 return None;
2152 }
2153 let routed = RoutedNodeEvent {
2154 node_id: node_id.clone(),
2155 cursor: *cursor,
2156 event: C2NodeEvent::from(&envelope.event),
2157 };
2158 cursor.sequence = cursor.sequence.max(envelope.sequence);
2159 Some(routed)
2160}
2161
2162async fn drain_pending_events(
2163 client: &mut LocalNodeClient,
2164 node_id: &NodeId,
2165 cursor: &mut NodeCursor,
2166 hub: &OperatorHub,
2167 ingress: &mpsc::Sender<Attempt>,
2168 skip_replayed: bool,
2169 relay_deadline: Option<Instant>,
2170) -> Result<(), NodeClientError> {
2171 let mut skip_replayed = skip_replayed;
2172 for repair_pass in 0..=1 {
2173 let mut gaps = Vec::new();
2174 let mut managed_worktree_events = Vec::new();
2175 let mut changed = false;
2176 let mut repair = false;
2177 while let Some(envelope) = client.take_event() {
2178 if let Some(event) = routed_transient_node_event(node_id, *cursor, &envelope) {
2179 hub.publish(event);
2180 continue;
2181 }
2182 if let Some(event) = route_agent_stream_event(node_id, cursor, &envelope) {
2183 hub.publish(event);
2184 continue;
2185 }
2186 if skip_replayed && envelope.sequence <= cursor.sequence { continue; }
2187 let resync_required = matches!(&envelope.event, gate4agent_node_protocol::NodeEvent::ResyncRequired { .. });
2188 if resync_required || envelope.sequence != cursor.sequence.saturating_add(1) {
2189 gaps.push(if envelope.sequence <= cursor.sequence && !resync_required {
2190 GapKind::CursorRegression
2191 } else if resync_required {
2192 GapKind::HistoryEvicted
2193 } else {
2194 GapKind::NonContiguousEvents
2195 });
2196 repair = true;
2197 continue;
2198 }
2199 let event_cursor = NodeCursor { incarnation_id: cursor.incarnation_id, sequence: envelope.sequence };
2200 if matches!(
2201 &envelope.event,
2202 NodeEvent::ManagedWorktreeUpserted { .. }
2203 | NodeEvent::ManagedWorktreeRemoved { .. }
2204 ) {
2205 managed_worktree_events.push(envelope.event.clone());
2206 }
2207 hub.publish(RoutedNodeEvent {
2208 node_id: node_id.clone(),
2209 cursor: event_cursor,
2210 event: C2NodeEvent::from_node_event_with_control_detail(&envelope.event),
2211 });
2212 cursor.sequence = envelope.sequence;
2213 changed = true;
2214 }
2215 if !repair {
2216 if changed || !gaps.is_empty() {
2217 ingress_attempt(ingress, node_id, AttemptResult::Cursor {
2218 cursor: *cursor,
2219 gaps,
2220 managed_worktree_events,
2221 }).await
2222 .map_err(NodeClientError::Io)?;
2223 }
2224 return Ok(());
2225 }
2226 if repair_pass == 1 {
2227 ingress_attempt(ingress, node_id, AttemptResult::Cursor {
2228 cursor: *cursor,
2229 gaps,
2230 managed_worktree_events,
2231 }).await
2232 .map_err(NodeClientError::Io)?;
2233 return Err(NodeClientError::Protocol("node event stream remained noncontiguous after resync".to_owned()));
2234 }
2235 let after_sequence = cursor.sequence;
2236 let response = bounded_node_request_with_deadline(
2237 client,
2238 NodeRequest::Resync { after_sequence },
2239 relay_deadline,
2240 ).await?;
2241 let NodeResponse::Resync {
2242 event_sequence,
2243 oldest_available_sequence,
2244 snapshot,
2245 events,
2246 } = response else {
2247 return Err(NodeClientError::Protocol("event repair resync returned a different response".to_owned()));
2248 };
2249 let repair_gaps = validate_events(
2250 after_sequence,
2251 event_sequence,
2252 oldest_available_sequence,
2253 &events,
2254 );
2255 let contiguous = repair_gaps.is_empty();
2256 gaps = repair_gaps;
2257 if contiguous {
2258 for envelope in events.iter().filter(|event| event.sequence > after_sequence) {
2259 if let Some(event) = routed_recovered_node_event(
2260 node_id,
2261 cursor.incarnation_id,
2262 envelope,
2263 ) {
2264 hub.publish(event);
2265 }
2266 }
2267 } else {
2268 hub.publish(RoutedNodeEvent {
2269 node_id: node_id.clone(),
2270 cursor: NodeCursor { incarnation_id: cursor.incarnation_id, sequence: event_sequence },
2271 event: C2NodeEvent::ResyncRequired {
2272 oldest_available_sequence,
2273 },
2274 });
2275 }
2276 cursor.sequence = event_sequence;
2277 ingress_attempt(ingress, node_id, AttemptResult::Success { cursor: *cursor, snapshot, gaps }).await
2278 .map_err(NodeClientError::Io)?;
2279 skip_replayed = true;
2280 }
2281 Ok(())
2282}
2283
2284async fn handle_live_node_event(
2285 client: &mut LocalNodeClient,
2286 node_id: &NodeId,
2287 envelope: NodeEventEnvelope,
2288 cursor: &mut NodeCursor,
2289 hub: &OperatorHub,
2290 ingress: &mpsc::Sender<Attempt>,
2291) -> Result<(), NodeClientError> {
2292 if let Some(event) = routed_transient_node_event(node_id, *cursor, &envelope) {
2293 hub.publish(event);
2294 return Ok(());
2295 }
2296 if let Some(event) = route_agent_stream_event(node_id, cursor, &envelope) {
2297 hub.publish(event);
2298 return Ok(());
2299 }
2300 if live_event_gap(cursor.sequence, &envelope).is_some() {
2301 let after_sequence = cursor.sequence;
2302 let response = bounded_node_request(
2303 client,
2304 NodeRequest::Resync { after_sequence },
2305 ).await?;
2306 let NodeResponse::Resync {
2307 event_sequence,
2308 oldest_available_sequence,
2309 snapshot,
2310 events,
2311 } = response else {
2312 return Err(NodeClientError::Protocol(
2313 "live event repair resync returned a different response".to_owned(),
2314 ));
2315 };
2316 let repair_gaps = validate_events(
2317 after_sequence,
2318 event_sequence,
2319 oldest_available_sequence,
2320 &events,
2321 );
2322 if repair_gaps.is_empty() {
2323 publish_recovered_events(node_id, cursor.incarnation_id, &events, hub);
2324 } else {
2325 hub.publish(RoutedNodeEvent {
2326 node_id: node_id.clone(),
2327 cursor: NodeCursor {
2328 incarnation_id: cursor.incarnation_id,
2329 sequence: event_sequence,
2330 },
2331 event: C2NodeEvent::ResyncRequired {
2332 oldest_available_sequence,
2333 },
2334 });
2335 }
2336 cursor.sequence = event_sequence;
2337 ingress_attempt(
2338 ingress,
2339 node_id,
2340 AttemptResult::Success {
2341 cursor: *cursor,
2342 snapshot,
2343 gaps: repair_gaps,
2344 },
2345 ).await.map_err(NodeClientError::Io)?;
2346 return drain_pending_events(
2347 client,
2348 node_id,
2349 cursor,
2350 hub,
2351 ingress,
2352 true,
2353 None,
2354 ).await;
2355 }
2356
2357 cursor.sequence = envelope.sequence;
2358 let managed_worktree_events = matches!(
2359 &envelope.event,
2360 NodeEvent::ManagedWorktreeUpserted { .. }
2361 | NodeEvent::ManagedWorktreeRemoved { .. }
2362 )
2363 .then(|| vec![envelope.event.clone()])
2364 .unwrap_or_default();
2365 hub.publish(RoutedNodeEvent {
2366 node_id: node_id.clone(),
2367 cursor: *cursor,
2368 event: C2NodeEvent::from_node_event_with_control_detail(&envelope.event),
2369 });
2370 ingress_attempt(
2371 ingress,
2372 node_id,
2373 AttemptResult::Cursor {
2374 cursor: *cursor,
2375 gaps: Vec::new(),
2376 managed_worktree_events,
2377 },
2378 ).await.map_err(NodeClientError::Io)
2379}
2380
2381fn live_event_gap(previous: u64, envelope: &NodeEventEnvelope) -> Option<GapKind> {
2382 if matches!(
2383 &envelope.event,
2384 gate4agent_node_protocol::NodeEvent::ResyncRequired { .. }
2385 ) {
2386 return Some(GapKind::HistoryEvicted);
2387 }
2388 if envelope.sequence <= previous {
2389 return Some(GapKind::CursorRegression);
2390 }
2391 (envelope.sequence != previous.saturating_add(1))
2392 .then_some(GapKind::NonContiguousEvents)
2393}
2394
2395fn reject_disconnected_commands(commands: &mut mpsc::Receiver<RelayCommand>, cursor: Option<NodeCursor>) {
2396 while let Ok(command) = commands.try_recv() {
2397 match command {
2398 RelayCommand::Request { reply, .. } => {
2399 let _ = reply.send(Err(relay_failure(
2400 C2RelayFailureCode::NodeOffline,
2401 "node relay disconnected before request dispatch",
2402 cursor.map(|value| value.incarnation_id),
2403 )));
2404 }
2405 }
2406 }
2407}
2408
2409fn acknowledge_disconnected_releases(releases: &mut mpsc::Receiver<oneshot::Sender<()>>) {
2410 while let Ok(reply) = releases.try_recv() { let _ = reply.send(()); }
2411}
2412
2413fn validate_events(
2414 previous: u64,
2415 current: u64,
2416 oldest_available_sequence: u64,
2417 events: &[NodeEventEnvelope],
2418) -> Vec<GapKind> {
2419 if current < previous { return vec![GapKind::CursorRegression]; }
2420 let max_floor = current.checked_add(1).unwrap_or(u64::MAX);
2421 let minimum_event_sequence = previous
2422 .checked_add(1)
2423 .unwrap_or(u64::MAX)
2424 .max(oldest_available_sequence);
2425 if oldest_available_sequence == 0
2426 || oldest_available_sequence > max_floor
2427 || (current == previous && !events.is_empty())
2428 || events.iter().any(|event| {
2429 event.sequence < minimum_event_sequence || event.sequence > current
2430 })
2431 || events.windows(2).any(|pair| pair[0].sequence >= pair[1].sequence)
2432 {
2433 return vec![GapKind::NonContiguousEvents];
2434 }
2435 if previous.saturating_add(1) < oldest_available_sequence {
2436 return vec![GapKind::HistoryEvicted];
2437 }
2438 Vec::new()
2439}
2440
2441fn sanitize_node_error(error: &NodeClientError) -> (SanitizedError, bool) {
2442 let (category, message, hard) = match error {
2443 NodeClientError::Protocol(message) if message.contains("identity mismatch") =>
2444 (C2ErrorCategory::Identity, "node identity mismatch", true),
2445 NodeClientError::Protocol(message) if message.contains("access-token proof") || message.contains("access denied") =>
2446 (C2ErrorCategory::Authentication, "node authentication failed", true),
2447 NodeClientError::Protocol(_) | NodeClientError::Frame(FrameError::Json(_) | FrameError::InvalidLength { .. }) =>
2448 (C2ErrorCategory::Protocol, "node protocol failed", true),
2449 NodeClientError::BuildStampMismatch { .. } =>
2450 (C2ErrorCategory::Protocol, "node build stamp mismatch", true),
2451 NodeClientError::UnsupportedCapability(_) =>
2452 (C2ErrorCategory::Protocol, "node capability unavailable", true),
2453 NodeClientError::Node(failure) if failure.code == NodeFailureCode::Unauthorized =>
2454 (C2ErrorCategory::Authentication, "node request authentication failed", true),
2455 NodeClientError::Node(_) =>
2456 (C2ErrorCategory::Protocol, "node rejected observer request", true),
2457 NodeClientError::Frame(FrameError::BodyTimedOut { .. } | FrameError::PrefixTimedOut)
2458 | NodeClientError::AuthenticationTimedOut =>
2459 (C2ErrorCategory::Timeout, "node observation deadline exceeded", false),
2460 NodeClientError::Authentication(_) | NodeClientError::RequestIdExhausted =>
2461 (C2ErrorCategory::Internal, "node client failed internally", true),
2462 NodeClientError::Io(_) | NodeClientError::Frame(FrameError::Io(_)) =>
2463 (C2ErrorCategory::Transport, "node transport unavailable", false),
2464 };
2465 (SanitizedError { category, message: message.to_owned() }, hard)
2466}
2467
2468async fn inventory_owner(
2469 fresh_for: Duration, mut ingress: mpsc::Receiver<Attempt>,
2470 status: watch::Sender<Arc<StatusResponse>>, mut shutdown: watch::Receiver<bool>,
2471) -> io::Result<()> {
2472 let mut current = (**status.borrow()).clone();
2473 let mut attempted = BTreeSet::new();
2474 loop {
2475 tokio::select! {
2476 attempt = ingress.recv() => {
2477 let Some(attempt) = attempt else { return Ok(()); };
2478 attempted.insert(attempt.node_id.clone());
2479 let node = current.nodes.entry(attempt.node_id.clone()).or_insert_with(initial_enrolled_node);
2480 node.last_attempt_unix_ms = Some(attempt.at_unix_ms);
2481 match attempt.result {
2482 AttemptResult::Connected {
2483 cursor,
2484 snapshot,
2485 gaps,
2486 provider_contract_manifest,
2487 } => {
2488 let previous = node.cursor;
2489 let mut inventory = SlimNodeInventory::from_snapshot(&snapshot);
2490 inventory.provider_contracts =
2491 provider_contract_manifest.provider_contracts;
2492 inventory.provider_adapter_contracts =
2493 provider_contract_manifest.provider_adapter_contracts;
2494 node.transport = NodeTransportState::Online;
2495 node.freshness = NodeFreshness::Fresh;
2496 node.cursor = Some(cursor);
2497 node.inventory = Some(inventory);
2498 node.last_success_unix_ms = Some(attempt.at_unix_ms);
2499 node.consecutive_failures = 0;
2500 node.last_error = None;
2501 for kind in gaps {
2502 if node.gaps.len() == MAX_C2_GAPS_PER_NODE { node.gaps.remove(0); node.gaps_truncated += 1; }
2503 node.gaps.push(NodeGap { kind, detected_at_unix_ms: attempt.at_unix_ms, previous, observed: cursor });
2504 }
2505 }
2506 AttemptResult::Success { cursor, snapshot, gaps } => {
2507 let previous = node.cursor;
2508 let provider_contract_manifest = node.inventory.as_ref().map(|inventory| {
2509 ProviderContractManifest {
2510 provider_contracts: inventory.provider_contracts.clone(),
2511 provider_adapter_contracts: inventory.provider_adapter_contracts.clone(),
2512 }
2513 }).unwrap_or_default();
2514 let mut inventory = SlimNodeInventory::from_snapshot(&snapshot);
2515 inventory.provider_contracts =
2516 provider_contract_manifest.provider_contracts;
2517 inventory.provider_adapter_contracts =
2518 provider_contract_manifest.provider_adapter_contracts;
2519 node.transport = NodeTransportState::Online;
2520 node.freshness = NodeFreshness::Fresh;
2521 node.cursor = Some(cursor);
2522 node.inventory = Some(inventory);
2523 node.last_success_unix_ms = Some(attempt.at_unix_ms);
2524 node.consecutive_failures = 0;
2525 node.last_error = None;
2526 for kind in gaps {
2527 if node.gaps.len() == MAX_C2_GAPS_PER_NODE { node.gaps.remove(0); node.gaps_truncated += 1; }
2528 node.gaps.push(NodeGap { kind, detected_at_unix_ms: attempt.at_unix_ms, previous, observed: cursor });
2529 }
2530 }
2531 AttemptResult::Cursor {
2532 cursor,
2533 gaps,
2534 managed_worktree_events,
2535 } => {
2536 let previous = node.cursor;
2537 let incarnation_changed = previous.is_some_and(|previous| {
2538 previous.incarnation_id != cursor.incarnation_id
2539 });
2540 apply_managed_worktree_cursor(
2541 node.inventory.as_mut(),
2542 incarnation_changed,
2543 &managed_worktree_events,
2544 );
2545 node.transport = NodeTransportState::Online;
2546 node.freshness = NodeFreshness::Fresh;
2547 node.cursor = Some(cursor);
2548 node.last_success_unix_ms = Some(attempt.at_unix_ms);
2549 node.consecutive_failures = 0;
2550 node.last_error = None;
2551 for kind in gaps {
2552 if node.gaps.len() == MAX_C2_GAPS_PER_NODE { node.gaps.remove(0); node.gaps_truncated += 1; }
2553 node.gaps.push(NodeGap { kind, detected_at_unix_ms: attempt.at_unix_ms, previous, observed: cursor });
2554 }
2555 }
2556 AttemptResult::Failure { error, hard } => {
2557 node.consecutive_failures = node.consecutive_failures.saturating_add(1);
2558 node.transport = if hard || node.consecutive_failures >= 5 { NodeTransportState::Parked } else { NodeTransportState::Offline };
2559 node.last_error = Some(error);
2560 }
2561 }
2562 current.ready = current.nodes.keys().all(|id| attempted.contains(id));
2563 refresh_freshness(&mut current, fresh_for);
2564 current.observed_at_unix_ms = unix_ms();
2565 status.send_replace(Arc::new(current.clone()));
2566 }
2567 _ = sleep(Duration::from_millis(250)) => {
2568 refresh_freshness(&mut current, fresh_for);
2569 current.observed_at_unix_ms = unix_ms();
2570 status.send_replace(Arc::new(current.clone()));
2571 }
2572 changed = shutdown.changed() => if changed.is_err() || *shutdown.borrow() { return Ok(()); },
2573 }
2574 }
2575}
2576
2577fn apply_managed_worktree_cursor(
2578 inventory: Option<&mut SlimNodeInventory>,
2579 incarnation_changed: bool,
2580 events: &[NodeEvent],
2581) {
2582 let Some(inventory) = inventory else { return; };
2583 if incarnation_changed {
2584 inventory.provider_runtime_statuses.clear();
2585 inventory.managed_worktrees.clear();
2586 inventory.managed_worktree_count = 0;
2587 inventory.managed_worktrees_truncated = false;
2588 return;
2589 }
2590 for event in events {
2591 match event {
2592 NodeEvent::ManagedWorktreeUpserted { lease } => {
2593 inventory.managed_worktrees.retain(|existing| {
2594 existing.lease_id != lease.lease_id
2595 && existing.workspace_id != lease.workspace_id
2596 });
2597 inventory.managed_worktrees.push(lease.clone());
2598 inventory.managed_worktrees.sort_by(|left, right| {
2599 left.lease_id.cmp(&right.lease_id)
2600 });
2601 inventory.managed_worktree_count = inventory.managed_worktrees.len();
2602 inventory.managed_worktrees.truncate(
2603 crate::protocol::MAX_C2_MANAGED_WORKTREES_PER_NODE,
2604 );
2605 inventory.managed_worktrees_truncated =
2606 inventory.managed_worktrees.len() < inventory.managed_worktree_count;
2607 }
2608 NodeEvent::ManagedWorktreeRemoved { lease_id } => {
2609 let before = inventory.managed_worktrees.len();
2610 inventory
2611 .managed_worktrees
2612 .retain(|lease| &lease.lease_id != lease_id);
2613 if inventory.managed_worktrees.len() < before
2614 || inventory.managed_worktrees_truncated
2615 {
2616 inventory.managed_worktree_count =
2617 inventory.managed_worktree_count.saturating_sub(1);
2618 }
2619 inventory.managed_worktrees_truncated =
2620 inventory.managed_worktrees.len() < inventory.managed_worktree_count;
2621 }
2622 _ => {}
2623 }
2624 }
2625}
2626
2627fn refresh_freshness(status: &mut StatusResponse, fresh_for: Duration) {
2628 let now = unix_ms();
2629 let fresh_ms = fresh_for.as_millis().min(u64::MAX as u128) as u64;
2630 for node in status.nodes.values_mut() {
2631 node.freshness = match node.last_success_unix_ms {
2632 None => NodeFreshness::Unavailable,
2633 Some(last) if now.saturating_sub(last) <= fresh_ms => NodeFreshness::Fresh,
2634 Some(_) => NodeFreshness::Stale,
2635 };
2636 }
2637}
2638
2639async fn http_server(
2640 listener: TcpListener, token: String, io_deadline: Duration,
2641 status: watch::Receiver<Arc<StatusResponse>>,
2642 enroll: Option<Arc<crate::enroll_service::EnrollService>>,
2643 mut shutdown: watch::Receiver<bool>,
2644) -> io::Result<()> {
2645 let permits = Arc::new(Semaphore::new(MAX_HTTP_CONNECTIONS));
2646 let mut connections = JoinSet::new();
2647 loop {
2648 tokio::select! {
2649 accepted = listener.accept() => {
2650 let (stream, _) = accepted?;
2651 let Ok(permit) = Arc::clone(&permits).try_acquire_owned() else { drop(stream); continue; };
2652 let token = token.clone();
2653 let status = status.clone();
2654 let enroll = enroll.clone();
2655 connections.spawn(async move { let _permit = permit; let _ = serve_http(stream, &token, io_deadline, &status, enroll.as_deref()).await; });
2656 }
2657 changed = shutdown.changed() => if changed.is_err() || *shutdown.borrow() { break; },
2658 }
2659 while let Some(result) = connections.try_join_next() { result.map_err(io::Error::other)?; }
2660 }
2661 connections.shutdown().await;
2662 Ok(())
2663}
2664
2665async fn serve_http(mut stream: TcpStream, token: &str, deadline: Duration, status: &watch::Receiver<Arc<StatusResponse>>, enroll: Option<&crate::enroll_service::EnrollService>) -> io::Result<()> {
2666 let request = match timeout(deadline, read_request(&mut stream)).await {
2667 Ok(Ok(request)) => request,
2668 Ok(Err(ReadError::TooLarge)) => return write_response(&mut stream, deadline, Response::plain(413, "Payload Too Large")).await,
2669 Ok(Err(ReadError::Io(error))) => return Err(error),
2670 _ => return Ok(()),
2671 };
2672 let response = route(request, token, status.borrow().as_ref(), enroll);
2673 write_response(&mut stream, deadline, response).await
2674}
2675
2676struct Request { method: String, path: String, authorization: Option<String>, body: Vec<u8> }
2677enum ReadError { Closed, Invalid, TooLarge, Io(io::Error) }
2678
2679async fn read_request(stream: &mut TcpStream) -> Result<Request, ReadError> {
2680 let mut bytes = Vec::with_capacity(1024);
2681 let mut chunk = [0_u8; 1024];
2682 let header_end = loop {
2683 let count = stream.read(&mut chunk).await.map_err(ReadError::Io)?;
2684 if count == 0 { return Err(ReadError::Closed); }
2685 if bytes.len().saturating_add(count) > HEADER_LIMIT_BYTES + OPERATOR_BODY_LIMIT_BYTES { return Err(ReadError::TooLarge); }
2686 bytes.extend_from_slice(&chunk[..count]);
2687 if let Some(position) = bytes.windows(4).position(|window| window == b"\r\n\r\n") { break position + 4; }
2688 };
2689 let text = std::str::from_utf8(&bytes[..header_end]).map_err(|_| ReadError::Invalid)?;
2690 let mut lines = text.split("\r\n");
2691 let mut first = lines.next().ok_or(ReadError::Invalid)?.split_whitespace();
2692 let method = first.next().ok_or(ReadError::Invalid)?;
2693 let path = first.next().ok_or(ReadError::Invalid)?;
2694 let version = first.next().ok_or(ReadError::Invalid)?;
2695 if first.next().is_some() || !version.starts_with("HTTP/1.") || !path.starts_with('/') { return Err(ReadError::Invalid); }
2696 let mut authorization = None;
2697 let mut content_length = 0_usize;
2698 for line in lines {
2699 if line.is_empty() { break; }
2700 let (name, value) = line.split_once(':').ok_or(ReadError::Invalid)?;
2701 if name.eq_ignore_ascii_case("authorization") {
2702 if authorization.is_some() { return Err(ReadError::Invalid); }
2703 authorization = Some(value.trim().to_owned());
2704 } else if name.eq_ignore_ascii_case("content-length") {
2705 content_length = value.trim().parse().map_err(|_| ReadError::Invalid)?;
2706 }
2707 }
2708 if content_length > OPERATOR_BODY_LIMIT_BYTES { return Err(ReadError::TooLarge); }
2709 let mut body = bytes[header_end..].to_vec();
2710 while body.len() < content_length {
2711 let count = stream.read(&mut chunk).await.map_err(ReadError::Io)?;
2712 if count == 0 { return Err(ReadError::Closed); }
2713 body.extend_from_slice(&chunk[..count]);
2714 }
2715 body.truncate(content_length);
2716 Ok(Request { method: method.to_owned(), path: path.to_owned(), authorization, body })
2717}
2718
2719fn route(request: Request, token: &str, status: &StatusResponse, enroll: Option<&crate::enroll_service::EnrollService>) -> Response {
2720 let path = request.path.split_once('?').map_or(request.path.as_str(), |pair| pair.0);
2721 if path == "/v1/enroll" || path.starts_with("/v1/enroll/") {
2723 let Some(service) = enroll else { return Response::plain(404, "Not Found"); };
2724 if !authorized(request.authorization.as_deref(), token) { return Response::plain(401, "Unauthorized").authenticate(); }
2725 let reply = service.handle_operator(&request.method, path, &request.body);
2726 return Response::raw_json(reply.status, reply.body);
2727 }
2728 if request.method != "GET" { return Response::plain(405, "Method Not Allowed").allow_get(); }
2729 match path {
2730 "/health" => Response::json(200, &HealthResponse { ok: true, service: "gate4agent-c2".to_owned(), api_version: C2_API_VERSION, pid: std::process::id(), version: env!("CARGO_PKG_VERSION").to_owned() }),
2731 "/ready" => {
2732 let online_nodes = status.nodes.values().filter(|node| node.transport == NodeTransportState::Online).count();
2733 let offline_nodes = status.nodes.values().filter(|node| node.transport == NodeTransportState::Offline).count();
2734 let parked_nodes = status.nodes.values().filter(|node| node.transport == NodeTransportState::Parked).count();
2735 let body = ReadyResponse { ready: status.ready, api_version: C2_API_VERSION, configured_nodes: status.nodes.len(), attempted_nodes: status.nodes.values().filter(|node| node.last_attempt_unix_ms.is_some()).count(), online_nodes, offline_nodes, parked_nodes };
2736 Response::json(if status.ready { 200 } else { 503 }, &body)
2737 }
2738 "/status" => {
2739 if !authorized(request.authorization.as_deref(), token) { Response::plain(401, "Unauthorized").authenticate() }
2740 else {
2741 let mut shown = status.clone();
2743 if let Some(service) = enroll {
2744 for (id, node) in shown.nodes.iter_mut() {
2745 if service.allows(id.as_str(), "attach") == Some(false) { node.inventory = None; }
2746 }
2747 }
2748 Response::json(200, &shown)
2749 }
2750 }
2751 _ => Response::plain(404, "Not Found"),
2752 }
2753}
2754
2755fn authorized(header: Option<&str>, token: &str) -> bool {
2756 let Some((scheme, candidate)) = header.and_then(|value| value.split_once(' ')) else { return false; };
2757 scheme.eq_ignore_ascii_case("bearer") && constant_time_eq(candidate.as_bytes(), token.as_bytes())
2758}
2759
2760fn constant_time_eq(left: &[u8], right: &[u8]) -> bool {
2761 if left.len() != right.len() { return false; }
2762 left.iter().zip(right).fold(0_u8, |difference, (left, right)| difference | (left ^ right)) == 0
2763}
2764
2765struct Response { status: u16, reason: &'static str, content_type: &'static str, body: Vec<u8>, headers: Vec<(&'static str, &'static str)> }
2766impl Response {
2767 fn plain(status: u16, reason: &'static str) -> Self { Self { status, reason, content_type: "text/plain; charset=utf-8", body: reason.as_bytes().to_vec(), headers: Vec::new() } }
2768 fn json<T: serde::Serialize>(status: u16, value: &T) -> Self {
2769 let body = serde_json::to_vec(value).expect("C2 DTO must serialize");
2770 if body.len() > RESPONSE_BODY_LIMIT_BYTES { return Self::plain(503, "Service Unavailable"); }
2771 let reason = if status == 200 { "OK" } else { "Service Unavailable" };
2772 Self { status, reason, content_type: "application/json", body, headers: Vec::new() }
2773 }
2774 fn raw_json(status: u16, body: Vec<u8>) -> Self {
2775 let reason = match status { 200 => "OK", 400 => "Bad Request", 404 => "Not Found", 409 => "Conflict", 429 => "Too Many Requests", _ => "Service Unavailable" };
2776 Self { status, reason, content_type: "application/json", body, headers: Vec::new() }
2777 }
2778 fn allow_get(mut self) -> Self { self.headers.push(("Allow", "GET")); self }
2779 fn authenticate(mut self) -> Self { self.headers.push(("WWW-Authenticate", "Bearer")); self }
2780}
2781
2782async fn write_response(stream: &mut TcpStream, deadline: Duration, response: Response) -> io::Result<()> {
2783 let mut headers = format!("HTTP/1.1 {} {}\r\nContent-Type: {}\r\nContent-Length: {}\r\nConnection: close\r\n", response.status, response.reason, response.content_type, response.body.len());
2784 for (name, value) in response.headers { headers.push_str(name); headers.push_str(": "); headers.push_str(value); headers.push_str("\r\n"); }
2785 headers.push_str("\r\n");
2786 timeout(deadline, async { stream.write_all(headers.as_bytes()).await?; stream.write_all(&response.body).await?; stream.shutdown().await }).await
2787 .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "C2 HTTP write timed out"))?
2788}
2789
2790fn unix_ms() -> u64 {
2791 SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis().min(u64::MAX as u128) as u64
2792}
2793
2794#[cfg(test)]
2795mod endpoint_tests {
2796 use super::*;
2797 use crate::protocol::{C2RelayRoute, C2Topology};
2798
2799 fn node(endpoint: &str) -> Result<C2NodeConfig, C2ConfigError> {
2800 C2NodeConfig::new(NodeId::new("remote-node").unwrap(), endpoint, "safe-token")
2801 }
2802
2803 #[cfg(windows)]
2804 const LOCAL_ENDPOINT: &str = r"\\.\pipe\relay-route-fact";
2805 #[cfg(unix)]
2806 const LOCAL_ENDPOINT: &str = "/tmp/gate4agent-relay-route-fact.sock";
2807
2808 fn projected_route(node: &C2NodeConfig, transport: NodeTransportState) -> C2RelayRoute {
2809 let mut observed = initial_observed_node(node);
2810 observed.transport = transport;
2811 let status = StatusResponse {
2812 api_version: C2_API_VERSION,
2813 ready: false,
2814 observed_at_unix_ms: 1,
2815 nodes: BTreeMap::from([(node.node_id.clone(), observed)]),
2816 };
2817 C2Topology::from_status(&status).nodes[0].relay_route
2818 }
2819
2820 #[test]
2821 fn relay_route_fact_projects_exactly_and_survives_offline_and_parked() {
2822 let local = node(LOCAL_ENDPOINT).unwrap();
2823 let ssh = node("tcp://127.0.0.1:48100").unwrap();
2824
2825 for transport in [
2826 NodeTransportState::Online,
2827 NodeTransportState::Offline,
2828 NodeTransportState::Parked,
2829 ] {
2830 assert_eq!(projected_route(&local, transport), C2RelayRoute::LocalIpc);
2831 assert_eq!(
2832 projected_route(&ssh, transport),
2833 C2RelayRoute::SshForwardedLoopback,
2834 );
2835 }
2836 }
2837
2838 #[test]
2843 fn a_node_that_waits_to_be_called_is_declared_by_name_not_by_address() {
2844 let waiting = node("accept").unwrap();
2845 assert_eq!(waiting.route, C2NodeRoute::CallHome);
2846 assert_eq!(waiting.transport_label(), "call-home");
2847
2848 let second = C2NodeConfig::new(
2853 NodeId::new("second-waiting-node").unwrap(),
2854 "accept",
2855 "safe-token",
2856 )
2857 .unwrap();
2858 let config = C2Config::new(
2859 "127.0.0.1:0".parse().unwrap(),
2860 "safe-token",
2861 vec![waiting.clone(), second],
2862 )
2863 .expect("two waiting nodes are not a duplicate endpoint");
2864 assert_eq!(config.nodes.len(), 2);
2865 }
2866
2867 #[test]
2871 fn waiting_for_a_call_requires_somewhere_to_be_called() {
2872 let waiting = node("accept").unwrap();
2873 let config =
2874 C2Config::new("127.0.0.1:0".parse().unwrap(), "safe-token", vec![waiting]).unwrap();
2875 assert!(matches!(
2876 config.validate_call_home(),
2877 Err(C2ConfigError::CallHomeWithoutListener(_)),
2878 ));
2879 let config = config.with_node_listen("127.0.0.1:48200".parse().unwrap()).unwrap();
2880 assert!(config.validate_call_home().is_ok());
2881 }
2882
2883 #[test]
2887 fn the_call_home_listener_refuses_unspecified_and_accepts_tunnel() {
2888 let waiting = node("accept").unwrap();
2889 let config =
2890 C2Config::new("127.0.0.1:9000".parse().unwrap(), "safe-token", vec![waiting]).unwrap();
2891 for refused in ["0.0.0.0:48200", "127.0.0.1:0", "[::]:48200"] {
2892 assert!(
2893 matches!(
2894 config.clone().with_node_listen(refused.parse().unwrap()),
2895 Err(C2ConfigError::NonLoopbackNodeListen(_)),
2896 ),
2897 "accepted {refused}",
2898 );
2899 }
2900 let tunnel: SocketAddr = "10.88.1.2:27442".parse().unwrap();
2901 let accepted = config.clone().with_node_listen(tunnel).unwrap();
2902 assert_eq!(accepted.node_listen, Some(tunnel));
2903 assert!(matches!(
2906 config.with_node_listen("127.0.0.1:9000".parse().unwrap()),
2907 Err(C2ConfigError::NodeListenConflict),
2908 ));
2909 }
2910
2911 #[test]
2912 fn ssh_forwarded_loopback_route_is_strict_canonical_and_control_stays_local() {
2913 let ipv4 = node("tcp://127.0.0.1:48100").unwrap();
2914 assert_eq!(ipv4.endpoint, "tcp://127.0.0.1:48100");
2915 assert_eq!(ipv4.route, C2NodeRoute::SshForwardedLoopback("127.0.0.1:48100".parse().unwrap()));
2916 assert_eq!(ipv4.transport_label(), "ssh-forwarded-loopback");
2917
2918 let ipv6 = node("tcp://[0:0:0:0:0:0:0:1]:48100").unwrap();
2919 assert_eq!(ipv6.endpoint, "tcp://[::1]:48100");
2920 assert_eq!(ipv6.transport_label(), "ssh-forwarded-loopback");
2921
2922 for invalid in [
2923 "tcp://localhost:48100",
2924 "tcp://127.0.0.2:48100",
2925 "tcp://0.0.0.0:48100",
2926 "tcp://127.0.0.1:0",
2927 "tcp://user@127.0.0.1:48100",
2928 "tcp://127.0.0.1:48100/path",
2929 "TCP://127.0.0.1:48100",
2930 ] {
2931 assert!(matches!(node(invalid), Err(C2ConfigError::InvalidEndpoint(_))), "accepted {invalid}");
2932 }
2933
2934 assert!(matches!(
2935 C2Config::new(
2936 "127.0.0.1:0".parse().unwrap(),
2937 "safe-token",
2938 vec![ipv4.clone(), C2NodeConfig::new(
2939 NodeId::new("duplicate-route").unwrap(),
2940 "tcp://127.0.0.1:48100",
2941 "safe-token",
2942 ).unwrap()],
2943 ),
2944 Err(C2ConfigError::DuplicateEndpoint)
2945 ));
2946 assert!(matches!(
2947 C2Config::new("127.0.0.1:0".parse().unwrap(), "safe-token", vec![ipv4])
2948 .unwrap()
2949 .with_control_endpoint("tcp://127.0.0.1:48101"),
2950 Err(C2ConfigError::InvalidControlEndpoint)
2951 ));
2952 }
2953}
2954
2955#[cfg(all(test, windows))]
2956mod tests {
2957 use super::*;
2958 use gate4agent_node_protocol::{
2959 AgentId, AgentStreamChunkKindV1, AgentStreamChunkV1, CapabilityId, SessionAddress,
2960 SessionKey, SessionMode, SessionRecordId,
2961 HarnessMcpActivationDigest, HarnessMcpCallId, HarnessMcpContentTypeV1,
2962 HarnessMcpOpaquePayloadV1, HarnessMcpReservationId,
2963 SpawnDeadlineMs, SpawnFieldProvenance, SpawnIdempotencyKey, SpawnOverrides,
2964 SpawnProfileId, SpawnProfileRevision, SpawnPrompt, SpawnPromptMetadata,
2965 SpawnRequiredCapabilities, SpawnResolutionProvenance, SpawnTarget, WorkspaceId,
2966 ManagedWorktreeCleanupFailure, ManagedWorktreeLeaseId,
2967 ManagedWorktreeLeaseSnapshot, ManagedWorktreeRetention, ManagedWorktreeSpawnReceipt,
2968 WorktreeProfileId, WorktreeProfileRevision,
2969 };
2970 use gate4agent_types::{AgentInstanceId, SessionGeneration, TerminalSize};
2971 use std::collections::BTreeMap;
2972
2973 fn agent(value: &str) -> gate4agent_node_protocol::AgentId {
2974 gate4agent_node_protocol::AgentId::new(value).unwrap()
2975 }
2976
2977 fn managed_lease(
2978 lease_id: &str,
2979 workspace_id: &str,
2980 state: ManagedWorktreeLeaseState,
2981 ) -> ManagedWorktreeLeaseSnapshot {
2982 let in_use = state == ManagedWorktreeLeaseState::InUse;
2983 ManagedWorktreeLeaseSnapshot {
2984 lease_id: ManagedWorktreeLeaseId::new(lease_id).unwrap(),
2985 source_workspace_id: WorkspaceId::new("repo").unwrap(),
2986 workspace_id: WorkspaceId::new(workspace_id).unwrap(),
2987 profile_id: WorktreeProfileId::new("review").unwrap(),
2988 profile_revision: WorktreeProfileRevision::new("review.r1").unwrap(),
2989 retention: ManagedWorktreeRetention::RemoveWhenReleased,
2990 state,
2991 active_session_count: u16::from(in_use),
2992 managed_record_count: u16::from(in_use),
2993 cleanup_failure: None::<ManagedWorktreeCleanupFailure>,
2994 created_at_unix_ms: 1,
2995 updated_at_unix_ms: 2,
2996 }
2997 }
2998
2999 #[test]
3000 fn spawn_spec_receipt_correlation_rejects_mismatches_before_forwarding() {
3001 let incarnation_id = NodeIncarnationId::from_bytes([7; 16]);
3002 let terminal_size = TerminalSize { rows: 24, columns: 80 };
3003 let prompt = SpawnPrompt::new("hi").unwrap();
3004 let required_capabilities = SpawnRequiredCapabilities::new([
3005 CapabilityId::new("raw-pty-lifecycle").unwrap(),
3006 ]).unwrap();
3007 let spec = SpawnSpec {
3008 target: SpawnTarget {
3009 node_id: NodeId::new("node-a").unwrap(),
3010 workspace_id: WorkspaceId::new("repo").unwrap(),
3011 worktree_id: None,
3012 },
3013 profile_id: SpawnProfileId::new("default").unwrap(),
3014 expected_profile_revision: SpawnProfileRevision::new("r1").unwrap(),
3015 overrides: SpawnOverrides {
3016 provider: SpawnOverride::Set { value: AgentId::new("codex").unwrap() },
3017 mode: SpawnOverride::Set { value: SessionMode::Pty },
3018 terminal_size: SpawnOverride::Set { value: terminal_size },
3019 prompt: SpawnOverride::Set { value: prompt.clone() },
3020 bundle_id: SpawnOverride::Clear,
3021 context_id: SpawnOverride::Clear,
3022 environment_profile_id: SpawnOverride::Clear,
3023 approval_level: None,
3024 network_allowlist: None,
3025 browser_profile_id: None,
3026 },
3027 deadline_ms: SpawnDeadlineMs::new(5_000).unwrap(),
3028 idempotency_key: SpawnIdempotencyKey::new("spawn-1").unwrap(),
3029 required_capabilities: required_capabilities.clone(),
3030 };
3031 let receipt = ResolvedSpawnReceipt {
3032 incarnation_id,
3033 session: SessionAddress {
3034 workspace_id: spec.target.workspace_id.clone(),
3035 session: SessionKey {
3036 instance_id: AgentInstanceId(7),
3037 generation: SessionGeneration(1),
3038 },
3039 },
3040 target: spec.target.clone(),
3041 profile_id: spec.profile_id.clone(),
3042 profile_revision: SpawnProfileRevision::new("r1").unwrap(),
3043 provider: AgentId::new("codex").unwrap(),
3044 mode: SessionMode::Pty,
3045 terminal_size,
3046 prompt: SpawnPromptMetadata::from_prompt(Some(&prompt)),
3047 bundle_id: None,
3048 bundle: None,
3049 context_id: None,
3050 context: None,
3051 environment_profile: None,
3052 deadline_ms: spec.deadline_ms,
3053 idempotency_key: spec.idempotency_key.clone(),
3054 required_capabilities,
3055 provenance: SpawnResolutionProvenance {
3056 provider: SpawnFieldProvenance::Override,
3057 mode: SpawnFieldProvenance::Override,
3058 terminal_size: SpawnFieldProvenance::Override,
3059 prompt: SpawnFieldProvenance::Override,
3060 bundle_id: SpawnFieldProvenance::Cleared,
3061 context_id: SpawnFieldProvenance::Cleared,
3062 environment_profile_id: SpawnFieldProvenance::Cleared,
3063 },
3064 harness_mcp_proxy: None,
3065 };
3066 let response = |receipt| Ok(NodeResponse::SpawnSpecAccepted { receipt });
3067 let expected = ExpectedSpawnRequest::Spec(spec.clone());
3068 assert!(validate_spawn_spec_response(
3069 Some(&expected),
3070 &response(receipt.clone()),
3071 incarnation_id,
3072 ).is_ok());
3073
3074 let mut mismatches = Vec::new();
3075 let mut changed = receipt.clone();
3076 changed.incarnation_id = NodeIncarnationId::from_bytes([8; 16]);
3077 mismatches.push(changed);
3078 let mut changed = receipt.clone();
3079 changed.target.node_id = NodeId::new("node-b").unwrap();
3080 mismatches.push(changed);
3081 let mut changed = receipt.clone();
3082 changed.profile_id = SpawnProfileId::new("other").unwrap();
3083 mismatches.push(changed);
3084 let mut changed = receipt.clone();
3085 changed.idempotency_key = SpawnIdempotencyKey::new("spawn-2").unwrap();
3086 mismatches.push(changed);
3087 let mut changed = receipt.clone();
3088 changed.deadline_ms = SpawnDeadlineMs::new(4_999).unwrap();
3089 mismatches.push(changed);
3090 let mut changed = receipt.clone();
3091 changed.required_capabilities = SpawnRequiredCapabilities::default();
3092 mismatches.push(changed);
3093 let mut changed = receipt.clone();
3094 changed.session.workspace_id = WorkspaceId::new("other").unwrap();
3095 mismatches.push(changed);
3096 let mut changed = receipt.clone();
3097 changed.provider = AgentId::new("claude").unwrap();
3098 mismatches.push(changed);
3099 let mut changed = receipt.clone();
3100 changed.prompt = SpawnPromptMetadata { present: false, byte_len: 0 };
3101 mismatches.push(changed);
3102 let mut changed = receipt.clone();
3103 changed.environment_profile = Some(
3104 gate4agent_node_protocol::ResolvedEnvironmentProfileReceipt {
3105 profile_id:
3106 gate4agent_node_protocol::SpawnEnvironmentProfileId::new(
3107 "local-default",
3108 )
3109 .unwrap(),
3110 profile_revision:
3111 gate4agent_node_protocol::SpawnEnvironmentProfileRevision::new(
3112 "local-default.r1",
3113 )
3114 .unwrap(),
3115 network_allowlist: None,
3116 browser_profile_id: None,
3117 },
3118 );
3119 mismatches.push(changed);
3120 let mut changed = receipt.clone();
3121 changed.bundle = Some(gate4agent_node_protocol::ResolvedBundleReceipt {
3122 id: gate4agent_node_protocol::SpawnBundleId::new("unexpected-bundle")
3123 .unwrap(),
3124 revision: gate4agent_node_protocol::SpawnBundleRevision::new(
3125 "unexpected-bundle.r1",
3126 )
3127 .unwrap(),
3128 digest: gate4agent_node_protocol::SpawnBundleDigest::new(format!(
3129 "sha256:{}",
3130 "a".repeat(64),
3131 ))
3132 .unwrap(),
3133 });
3134 mismatches.push(changed);
3135 let mut changed = receipt.clone();
3136 changed.context = Some(gate4agent_node_protocol::ResolvedContextPackReceipt {
3137 id: gate4agent_node_protocol::SpawnContextId::new("unexpected-context")
3138 .unwrap(),
3139 digest: gate4agent_node_protocol::SpawnContextDigest::new(format!(
3140 "sha256:{}",
3141 "b".repeat(64),
3142 ))
3143 .unwrap(),
3144 lineage: gate4agent_node_protocol::ContextPackLineageReceipt {
3145 source_node_id: NodeId::new("node-a").unwrap(),
3146 source_session: receipt.session.clone(),
3147 source_provider: AgentId::new("codex").unwrap(),
3148 },
3149 source_message_count: 1,
3150 retained_message_count: 1,
3151 byte_len: 16,
3152 truncated: false,
3153 });
3154 mismatches.push(changed);
3155
3156 for mismatch in mismatches {
3157 assert!(validate_spawn_spec_response(
3158 Some(&expected),
3159 &response(mismatch),
3160 incarnation_id,
3161 ).is_err());
3162 }
3163
3164 let mut environment_spec = spec;
3165 environment_spec.overrides.environment_profile_id = SpawnOverride::Set {
3166 value: gate4agent_node_protocol::SpawnEnvironmentProfileId::new(
3167 "local-default",
3168 )
3169 .unwrap(),
3170 };
3171 let environment_expected = ExpectedSpawnRequest::Spec(environment_spec);
3172 let mut environment_receipt = receipt;
3173 environment_receipt.environment_profile = Some(
3174 gate4agent_node_protocol::ResolvedEnvironmentProfileReceipt {
3175 profile_id:
3176 gate4agent_node_protocol::SpawnEnvironmentProfileId::new(
3177 "local-default",
3178 )
3179 .unwrap(),
3180 profile_revision:
3181 gate4agent_node_protocol::SpawnEnvironmentProfileRevision::new(
3182 "local-default.r1",
3183 )
3184 .unwrap(),
3185 network_allowlist: None,
3186 browser_profile_id: None,
3187 },
3188 );
3189 assert!(validate_spawn_spec_response(
3190 Some(&environment_expected),
3191 &response(environment_receipt.clone()),
3192 incarnation_id,
3193 )
3194 .is_ok());
3195 environment_receipt.environment_profile.as_mut().unwrap().profile_id =
3196 gate4agent_node_protocol::SpawnEnvironmentProfileId::new("other")
3197 .unwrap();
3198 assert!(validate_spawn_spec_response(
3199 Some(&environment_expected),
3200 &response(environment_receipt),
3201 incarnation_id,
3202 )
3203 .is_err());
3204 }
3205
3206 #[test]
3207 fn provider_session_index_correlation_accepts_exact_identity_and_rejects_mismatches() {
3208 let identity = gate4agent_types::ProviderSessionIdentity {
3209 key: gate4agent_types::ProviderSessionKey::SessionId,
3210 id: "native-session-1".to_owned(),
3211 transcript_path: Some(r"C:\provider\sessions\native-session-1.jsonl".to_owned()),
3212 };
3213 let expected = NodeRequest::IndexProviderSession {
3214 workspace_id: WorkspaceId::new("primary").unwrap(),
3215 provider: AgentId::new("codex").unwrap(),
3216 identity: identity.clone(),
3217 display_name: "release shepherd".to_owned(),
3218 };
3219 let NodeRequest::IndexProviderSession { workspace_id, provider, .. } = &expected else {
3220 unreachable!();
3221 };
3222 let record = gate4agent_node_protocol::ManagedSessionRecord {
3223 record_id: SessionRecordId::new("session-001").unwrap(),
3224 display_name: "release shepherd".to_owned(),
3225 provider: provider.clone(),
3226 mode: SessionMode::Pty,
3227 state: gate4agent_node_protocol::ManagedSessionState::Dormant,
3228 workspace_id: workspace_id.clone(),
3229 canonical_root: gate4agent_node_protocol::OpaqueHostPath::utf8(
3230 r"C:\repo".to_owned(),
3231 ).unwrap(),
3232 provider_session: Some(identity),
3233 active_session: None,
3234 environment_profile: None,
3235 bundle: None,
3236 context_id: None,
3237 context: None,
3238 exported_context: None,
3239 task_binding: None,
3240 created_at_unix_ms: 1,
3241 updated_at_unix_ms: 2,
3242 last_error: None,
3243 };
3244 let response = |record| Ok(NodeResponse::ProviderSessionIndexed { record });
3245
3246 assert!(validate_provider_session_index_response(
3247 Some(&expected),
3248 &response(record.clone()),
3249 ).is_ok());
3250 assert!(validate_provider_session_index_response(
3251 None,
3252 &response(record.clone()),
3253 ).is_err());
3254
3255 let mut identity_mismatch = record.clone();
3256 identity_mismatch.provider_session.as_mut().unwrap().id =
3257 "native-session-2".to_owned();
3258 assert!(validate_provider_session_index_response(
3259 Some(&expected),
3260 &response(identity_mismatch),
3261 ).is_err());
3262
3263 let mut workspace_mismatch = record.clone();
3264 workspace_mismatch.workspace_id = WorkspaceId::new("other").unwrap();
3265 assert!(validate_provider_session_index_response(
3266 Some(&expected),
3267 &response(workspace_mismatch),
3268 ).is_err());
3269
3270 let mut provider_mismatch = record;
3271 provider_mismatch.provider = AgentId::new("claude").unwrap();
3272 assert!(validate_provider_session_index_response(
3273 Some(&expected),
3274 &response(provider_mismatch),
3275 ).is_err());
3276 }
3277
3278 #[test]
3279 fn native_session_route_correlation_rejects_mismatches_before_projection() {
3280 let route = gate4agent_node_protocol::NativeSessionCatalogRoute::workspace(
3281 WorkspaceId::new("primary").unwrap(),
3282 AgentId::new("codex").unwrap(),
3283 );
3284 let selection = gate4agent_node_protocol::NativeSessionSelection {
3285 route: route.clone(),
3286 catalog_revision: 7,
3287 recent_cutoff_unix_ms: 70,
3288 selection_id: "selection-7".to_owned(),
3289 };
3290 let catalog = NodeRequest::CatalogNativeSessions {
3291 route: route.clone(),
3292 limit: 10,
3293 };
3294 let catalog_response = Ok(NodeResponse::NativeSessionsCataloged {
3295 route: route.clone(),
3296 entries: Vec::new(),
3297 summary: None,
3298 });
3299 assert!(validate_native_session_response(Some(&catalog), &catalog_response).is_ok());
3300 let mismatched_route = gate4agent_node_protocol::NativeSessionCatalogRoute::workspace(
3301 WorkspaceId::new("other").unwrap(),
3302 AgentId::new("codex").unwrap(),
3303 );
3304 assert!(validate_native_session_response(
3305 Some(&catalog),
3306 &Ok(NodeResponse::NativeSessionsCataloged {
3307 route: mismatched_route,
3308 entries: Vec::new(),
3309 summary: None,
3310 }),
3311 )
3312 .is_err());
3313
3314 let page = NodeRequest::PageNativeSessions {
3315 route: route.clone(),
3316 window: gate4agent_node_protocol::NativeSessionCatalogWindow::Recent,
3317 catalog_revision: 7,
3318 recent_cutoff_unix_ms: 70,
3319 after_selection_id: None,
3320 limit: 10,
3321 };
3322 let page_response = |revision| {
3323 Ok(NodeResponse::NativeSessionsPaged {
3324 route: route.clone(),
3325 page: gate4agent_node_protocol::NativeSessionCatalogPage {
3326 window: gate4agent_node_protocol::NativeSessionCatalogWindow::Recent,
3327 revision,
3328 entries: Vec::new(),
3329 next_after_selection_id: None,
3330 remaining_count: 0,
3331 has_more: false,
3332 },
3333 })
3334 };
3335 assert!(validate_native_session_response(Some(&page), &page_response(7)).is_ok());
3336 assert!(validate_native_session_response(Some(&page), &page_response(8)).is_err());
3337
3338 let preview = NodeRequest::PreviewNativeSession {
3339 selection: selection.clone(),
3340 message_limit: 10,
3341 };
3342 let preview_response = |selection| {
3343 Ok(NodeResponse::NativeSessionPreviewed {
3344 selection,
3345 preview: gate4agent_node_protocol::SessionRecordPreview {
3346 title: None,
3347 modified_at_unix_ms: None,
3348 model: None,
3349 message_count: 0,
3350 message_count_exact: true,
3351 completed_turn_count: None,
3352 total_tokens: None,
3353 truncated: false,
3354 messages: Vec::new(),
3355 },
3356 })
3357 };
3358 assert!(validate_native_session_response(
3359 Some(&preview),
3360 &preview_response(selection.clone()),
3361 )
3362 .is_ok());
3363 let mut mismatched_selection = selection.clone();
3364 mismatched_selection.catalog_revision = 8;
3365 assert!(validate_native_session_response(
3366 Some(&preview),
3367 &preview_response(mismatched_selection),
3368 )
3369 .is_err());
3370
3371 let index = NodeRequest::IndexNativeSession {
3372 selection: selection.clone(),
3373 display_name: "Indexed".to_owned(),
3374 };
3375 let record = gate4agent_node_protocol::ManagedSessionRecord {
3376 record_id: SessionRecordId::new("session-007").unwrap(),
3377 display_name: "Indexed".to_owned(),
3378 provider: route.provider.clone(),
3379 mode: SessionMode::Pty,
3380 state: gate4agent_node_protocol::ManagedSessionState::Dormant,
3381 workspace_id: route.workspace_id.clone().unwrap(),
3382 canonical_root: gate4agent_node_protocol::OpaqueHostPath::utf8(
3383 r"C:\repo".to_owned(),
3384 )
3385 .unwrap(),
3386 provider_session: None,
3387 active_session: None,
3388 environment_profile: None,
3389 bundle: None,
3390 context_id: None,
3391 context: None,
3392 exported_context: None,
3393 task_binding: None,
3394 created_at_unix_ms: 1,
3395 updated_at_unix_ms: 2,
3396 last_error: None,
3397 };
3398 assert!(validate_native_session_response(
3399 Some(&index),
3400 &Ok(NodeResponse::NativeSessionIndexed {
3401 selection: selection.clone(),
3402 record: record.clone(),
3403 }),
3404 )
3405 .is_ok());
3406 assert!(validate_native_session_response(
3407 Some(&index),
3408 &Ok(NodeResponse::ProviderSessionIndexed {
3409 record: record.clone(),
3410 }),
3411 )
3412 .is_err());
3413 let mut wrong_echo = selection.clone();
3414 wrong_echo.catalog_revision = 8;
3415 assert!(validate_native_session_response(
3416 Some(&index),
3417 &Ok(NodeResponse::NativeSessionIndexed {
3418 selection: wrong_echo,
3419 record: record.clone(),
3420 }),
3421 )
3422 .is_err());
3423 let mut wrong_provider = record.clone();
3424 wrong_provider.provider = AgentId::new("claude").unwrap();
3425 assert!(validate_native_session_response(
3426 Some(&index),
3427 &Ok(NodeResponse::NativeSessionIndexed {
3428 selection: selection.clone(),
3429 record: wrong_provider.clone(),
3430 }),
3431 )
3432 .is_err());
3433 let mut wrong_workspace = record.clone();
3434 wrong_workspace.workspace_id = WorkspaceId::new("other").unwrap();
3435 assert!(validate_native_session_response(
3436 Some(&index),
3437 &Ok(NodeResponse::NativeSessionIndexed {
3438 selection: selection.clone(),
3439 record: wrong_workspace,
3440 }),
3441 )
3442 .is_err());
3443
3444 let external = NodeRequest::IndexNativeSession {
3445 selection: gate4agent_node_protocol::NativeSessionSelection {
3446 route: gate4agent_node_protocol::NativeSessionCatalogRoute::unregistered(
3447 AgentId::new("codex").unwrap(),
3448 ),
3449 ..selection
3450 },
3451 display_name: "External".to_owned(),
3452 };
3453 let NodeRequest::IndexNativeSession {
3454 selection: external_selection,
3455 ..
3456 } = &external else {
3457 unreachable!();
3458 };
3459 assert!(validate_native_session_response(
3460 Some(&external),
3461 &Ok(NodeResponse::NativeSessionIndexed {
3462 selection: external_selection.clone(),
3463 record: gate4agent_node_protocol::ManagedSessionRecord {
3464 provider: AgentId::new("codex").unwrap(),
3465 ..wrong_provider
3466 },
3467 }),
3468 )
3469 .is_err());
3470 }
3471
3472 #[test]
3473 fn managed_spawn_receipt_correlation_covers_legacy_and_v2_requests() {
3474 let incarnation_id = NodeIncarnationId::from_bytes([7; 16]);
3475 let spec = SpawnSpec {
3476 target: SpawnTarget {
3477 node_id: NodeId::new("node-a").unwrap(),
3478 workspace_id: WorkspaceId::new("repo").unwrap(),
3479 worktree_id: None,
3480 },
3481 profile_id: SpawnProfileId::new("default").unwrap(),
3482 expected_profile_revision:
3483 SpawnProfileRevision::new("default.r1").unwrap(),
3484 overrides: SpawnOverrides::default(),
3485 deadline_ms: SpawnDeadlineMs::new(5_000).unwrap(),
3486 idempotency_key: SpawnIdempotencyKey::new("managed-1").unwrap(),
3487 required_capabilities: SpawnRequiredCapabilities::default(),
3488 };
3489 let managed = ManagedWorktreeSpawnRequest {
3490 spawn_spec: spec.clone(),
3491 worktree_profile_id: WorktreeProfileId::new("review").unwrap(),
3492 };
3493 let workspace_id = WorkspaceId::new("managed-a").unwrap();
3494 let spawn = ResolvedSpawnReceipt {
3495 incarnation_id,
3496 session: SessionAddress {
3497 workspace_id: workspace_id.clone(),
3498 session: SessionKey {
3499 instance_id: AgentInstanceId(8),
3500 generation: SessionGeneration(1),
3501 },
3502 },
3503 target: SpawnTarget {
3504 node_id: spec.target.node_id.clone(),
3505 workspace_id: spec.target.workspace_id.clone(),
3506 worktree_id: Some(workspace_id.clone()),
3507 },
3508 profile_id: spec.profile_id.clone(),
3509 profile_revision: SpawnProfileRevision::new("default.r1").unwrap(),
3510 provider: AgentId::new("claude").unwrap(),
3511 mode: SessionMode::Pty,
3512 terminal_size: TerminalSize { rows: 24, columns: 80 },
3513 prompt: SpawnPromptMetadata { present: false, byte_len: 0 },
3514 bundle_id: None,
3515 bundle: None,
3516 context_id: None,
3517 context: None,
3518 environment_profile: None,
3519 deadline_ms: spec.deadline_ms,
3520 idempotency_key: spec.idempotency_key.clone(),
3521 required_capabilities: SpawnRequiredCapabilities::default(),
3522 provenance: SpawnResolutionProvenance {
3523 provider: SpawnFieldProvenance::Profile,
3524 mode: SpawnFieldProvenance::Profile,
3525 terminal_size: SpawnFieldProvenance::Profile,
3526 prompt: SpawnFieldProvenance::Profile,
3527 bundle_id: SpawnFieldProvenance::Profile,
3528 context_id: SpawnFieldProvenance::Profile,
3529 environment_profile_id: SpawnFieldProvenance::Profile,
3530 },
3531 harness_mcp_proxy: None,
3532 };
3533 let receipt = ManagedWorktreeSpawnReceipt {
3534 spawn,
3535 lease: managed_lease(
3536 "lease-a",
3537 "managed-a",
3538 ManagedWorktreeLeaseState::InUse,
3539 ),
3540 };
3541 let expected = ExpectedSpawnRequest::ManagedV1(managed.clone());
3542 let response = |receipt| Ok(NodeResponse::ManagedWorktreeSpawnAccepted { receipt });
3543 assert!(validate_spawn_spec_response(
3544 Some(&expected),
3545 &response(receipt.clone()),
3546 incarnation_id,
3547 )
3548 .is_ok());
3549
3550 let mut mismatches = Vec::new();
3551 let mut changed = receipt.clone();
3552 changed.lease.state = ManagedWorktreeLeaseState::Ready;
3553 mismatches.push(changed);
3554 let mut changed = receipt.clone();
3555 changed.lease.cleanup_failure = Some(ManagedWorktreeCleanupFailure::Busy);
3556 mismatches.push(changed);
3557 let mut changed = receipt.clone();
3558 changed.lease.active_session_count = 0;
3559 mismatches.push(changed);
3560 let mut changed = receipt.clone();
3561 changed.lease.profile_id = WorktreeProfileId::new("other").unwrap();
3562 mismatches.push(changed);
3563 let mut changed = receipt.clone();
3564 changed.lease.source_workspace_id = WorkspaceId::new("other").unwrap();
3565 mismatches.push(changed);
3566
3567 for mismatch in mismatches {
3568 assert!(validate_spawn_spec_response(
3569 Some(&expected),
3570 &response(mismatch),
3571 incarnation_id,
3572 )
3573 .is_err());
3574 }
3575
3576 let mut legacy_revision = receipt.clone();
3577 legacy_revision.lease.profile_revision =
3578 WorktreeProfileRevision::new("review.r2").unwrap();
3579 assert!(validate_spawn_spec_response(
3580 Some(&expected),
3581 &response(legacy_revision),
3582 incarnation_id,
3583 )
3584 .is_ok());
3585
3586 let v2_request = ManagedWorktreeSpawnRequestV2 {
3587 spawn_spec: managed.spawn_spec,
3588 worktree_profile_id: managed.worktree_profile_id,
3589 expected_profile_revision: WorktreeProfileRevision::new("review.r1").unwrap(),
3590 };
3591 let routed = NodeRequest::SpawnManagedWorktreeV2 {
3592 request: v2_request.clone(),
3593 };
3594 let captured_expected = expected_spawn_request(&routed).unwrap();
3595 let ExpectedSpawnRequest::ManagedV2(captured_request) = &captured_expected else {
3596 panic!("V2 managed spawn request was not captured separately");
3597 };
3598 assert_eq!(captured_request, &v2_request);
3599 assert!(validate_spawn_spec_response(
3600 Some(&captured_expected),
3601 &response(receipt.clone()),
3602 incarnation_id,
3603 )
3604 .is_ok());
3605
3606 let mut wrong_revision = receipt.clone();
3607 wrong_revision.lease.profile_revision =
3608 WorktreeProfileRevision::new("review.r2").unwrap();
3609 assert!(validate_spawn_spec_response(
3610 Some(&ExpectedSpawnRequest::ManagedV2(v2_request.clone())),
3611 &response(wrong_revision),
3612 incarnation_id,
3613 )
3614 .is_err());
3615 assert!(validate_spawn_spec_response(
3616 Some(&ExpectedSpawnRequest::ManagedV2(v2_request)),
3617 &Ok(NodeResponse::SpawnSpecAccepted {
3618 receipt: receipt.spawn,
3619 }),
3620 incarnation_id,
3621 )
3622 .is_err());
3623 }
3624
3625 #[test]
3626 fn windows_runtime_default_control_endpoint_is_exact_valid_and_distinct_from_a_node() {
3627 assert_eq!(DEFAULT_C2_CONTROL_ENDPOINT, r"\\.\pipe\gate4agent-c2");
3628 validate_control_endpoint(DEFAULT_C2_CONTROL_ENDPOINT).unwrap();
3629 let node = C2NodeConfig::new(
3630 NodeId::new("node-a").unwrap(),
3631 r"\\.\pipe\gate4agent-node",
3632 "safe-token",
3633 )
3634 .unwrap();
3635 let config = C2Config::new(
3636 "127.0.0.1:0".parse().unwrap(),
3637 "safe-token",
3638 vec![node],
3639 )
3640 .unwrap();
3641 assert_eq!(config.control_endpoint, DEFAULT_C2_CONTROL_ENDPOINT);
3642 assert!(!config.nodes[0]
3643 .endpoint
3644 .eq_ignore_ascii_case(&config.control_endpoint));
3645
3646 let conflicting_node = C2NodeConfig::new(
3647 NodeId::new("node-b").unwrap(),
3648 DEFAULT_C2_CONTROL_ENDPOINT,
3649 "safe-token",
3650 )
3651 .unwrap();
3652 assert!(matches!(
3653 C2Config::new(
3654 "127.0.0.1:0".parse().unwrap(),
3655 "safe-token",
3656 vec![conflicting_node],
3657 ),
3658 Err(C2ConfigError::ControlEndpointConflict)
3659 ));
3660 }
3661
3662 #[test]
3663 fn terminal_only_sequence_holes_do_not_mark_c2_partial() {
3664 use gate4agent_node_protocol::{NodeEvent, WorkspaceId};
3665 let event = |sequence| NodeEventEnvelope { sequence, event: NodeEvent::WorkspaceRemoved { workspace_id: WorkspaceId::new("work").unwrap() } };
3666 assert_eq!(validate_events(4, 7, 1, &[event(6)]), Vec::<GapKind>::new());
3667 assert_eq!(validate_events(4, 7, 1, &[]), Vec::<GapKind>::new());
3668 assert_eq!(validate_events(4, 7, 1, &[event(7), event(6)]), vec![GapKind::NonContiguousEvents]);
3669 assert_eq!(validate_events(7, 6, 1, &[]), vec![GapKind::CursorRegression]);
3670 assert_eq!(validate_resync(4, 6, 4, 1, &[]), vec![GapKind::CursorRegression]);
3671 }
3672
3673 #[test]
3674 fn real_durable_eviction_marks_c2_history_evicted() {
3675 use gate4agent_node_protocol::{NodeEvent, WorkspaceId};
3676 let event = |sequence| NodeEventEnvelope {
3677 sequence,
3678 event: NodeEvent::WorkspaceRemoved {
3679 workspace_id: WorkspaceId::new("work").unwrap(),
3680 },
3681 };
3682 assert_eq!(
3683 validate_events(4, 7, 6, &[event(6)]),
3684 vec![GapKind::HistoryEvicted],
3685 );
3686 assert_eq!(validate_events(5, 7, 6, &[event(6)]), Vec::<GapKind>::new());
3687 }
3688
3689 #[test]
3690 fn live_event_gap_preserves_cursor_and_resync_rules() {
3691 use gate4agent_node_protocol::{NodeEvent, WorkspaceId};
3692 let event = |sequence, event| NodeEventEnvelope { sequence, event };
3693 let removed = || NodeEvent::WorkspaceRemoved {
3694 workspace_id: WorkspaceId::new("work").unwrap(),
3695 };
3696
3697 assert_eq!(live_event_gap(4, &event(5, removed())), None);
3698 assert_eq!(
3699 live_event_gap(4, &event(4, removed())),
3700 Some(GapKind::CursorRegression),
3701 );
3702 assert_eq!(
3703 live_event_gap(4, &event(7, removed())),
3704 Some(GapKind::NonContiguousEvents),
3705 );
3706 assert_eq!(
3707 live_event_gap(4, &event(5, NodeEvent::ResyncRequired {
3708 oldest_available_sequence: 3,
3709 })),
3710 Some(GapKind::HistoryEvicted),
3711 );
3712 }
3713
3714 #[test]
3715 fn harness_mcp_transient_live_and_pending_projection_preserves_cursor_without_replay() {
3716 let node_id = NodeId::new("node-a").unwrap();
3717 let cursor = NodeCursor {
3718 incarnation_id: NodeIncarnationId::from_bytes([7; 16]),
3719 sequence: 41,
3720 };
3721 let envelope = NodeEventEnvelope {
3722 sequence: 0,
3723 event: NodeEvent::HarnessMcpReadCall {
3724 reservation_id: HarnessMcpReservationId::new(
3725 format!("hmcpres_{}", "a".repeat(24)),
3726 ).unwrap(),
3727 activation_digest: HarnessMcpActivationDigest::new(
3728 format!("sha256:{}", "b".repeat(64)),
3729 ).unwrap(),
3730 record_id: SessionRecordId::new("session-001").unwrap(),
3731 session: SessionAddress {
3732 workspace_id: WorkspaceId::new("repo").unwrap(),
3733 session: SessionKey {
3734 instance_id: AgentInstanceId(8),
3735 generation: SessionGeneration(1),
3736 },
3737 },
3738 call_id: HarnessMcpCallId::new(
3739 format!("hmcpcall_{}", "c".repeat(24)),
3740 ).unwrap(),
3741 request: HarnessMcpOpaquePayloadV1 {
3742 content_type: HarnessMcpContentTypeV1::HarnessReadRequestJsonV1,
3743 body: br#"{"kind":"context-get"}"#.to_vec(),
3744 },
3745 deadline_unix_ms: u64::MAX,
3746 },
3747 };
3748
3749 let live = routed_transient_node_event(&node_id, cursor, &envelope).unwrap();
3750 let pending = routed_transient_node_event(&node_id, cursor, &envelope).unwrap();
3751 assert_eq!(live, pending);
3752 assert_eq!(live.cursor, cursor);
3753 assert!(matches!(live.event, C2NodeEvent::HarnessMcpReadCall { .. }));
3754 assert!(routed_recovered_node_event(
3755 &node_id,
3756 cursor.incarnation_id,
3757 &envelope,
3758 ).is_none());
3759 assert_eq!(cursor.sequence, 41);
3760 }
3761
3762 fn agent_stream_test_address() -> SessionAddress {
3763 SessionAddress {
3764 workspace_id: WorkspaceId::new("repo").unwrap(),
3765 session: SessionKey {
3766 instance_id: AgentInstanceId(8),
3767 generation: SessionGeneration(1),
3768 },
3769 }
3770 }
3771
3772 fn agent_stream_test_envelope(sequence: u64, source_sequence: u64) -> NodeEventEnvelope {
3773 NodeEventEnvelope {
3774 sequence,
3775 event: NodeEvent::AgentStream {
3776 address: agent_stream_test_address(),
3777 chunk: AgentStreamChunkV1 {
3778 source_sequence,
3779 kind: AgentStreamChunkKindV1::Text {
3780 text: format!("chunk-{source_sequence}"),
3781 is_delta: true,
3782 },
3783 },
3784 },
3785 }
3786 }
3787
3788 #[test]
3789 fn agent_stream_chunk_publishes_unconditionally_and_only_advances_cursor_forward() {
3790 let node_id = NodeId::new("node-a").unwrap();
3791 let mut cursor = NodeCursor {
3792 incarnation_id: NodeIncarnationId::from_bytes([7; 16]),
3793 sequence: 674,
3794 };
3795
3796 let control = NodeEventEnvelope {
3798 sequence: 675,
3799 event: NodeEvent::WorkspaceRemoved {
3800 workspace_id: WorkspaceId::new("work").unwrap(),
3801 },
3802 };
3803 assert!(route_agent_stream_event(&node_id, &mut cursor, &control).is_none());
3804 assert_eq!(cursor.sequence, 674);
3805
3806 let forward = agent_stream_test_envelope(675, 22);
3811 let routed = route_agent_stream_event(&node_id, &mut cursor, &forward).unwrap();
3812 assert_eq!(routed.cursor.sequence, 674);
3813 assert!(matches!(
3814 &routed.event,
3815 C2NodeEvent::AgentStream { chunk, .. } if chunk.source_sequence == 22
3816 ));
3817 assert_eq!(cursor.sequence, 675);
3818
3819 cursor.sequence = 711;
3826 let late = agent_stream_test_envelope(677, 23);
3827 let routed = route_agent_stream_event(&node_id, &mut cursor, &late).unwrap();
3828 assert_eq!(routed.cursor.sequence, 711);
3829 assert!(matches!(
3830 &routed.event,
3831 C2NodeEvent::AgentStream { chunk, .. } if chunk.source_sequence == 23
3832 ));
3833 assert_eq!(cursor.sequence, 711, "a late chunk must never rewind the cursor");
3834 }
3835
3836 #[test]
3837 fn agent_stream_burst_survives_a_durable_cursor_that_already_ran_past_it() {
3838 let node_id = NodeId::new("node-a").unwrap();
3847 let mut cursor = NodeCursor {
3848 incarnation_id: NodeIncarnationId::from_bytes([9; 16]),
3849 sequence: 0,
3850 };
3851 let chunk_count = 200_u64;
3852
3853 for turn in 1..=chunk_count {
3854 cursor.sequence = turn * 2;
3860 }
3861
3862 let mut recovered = Vec::new();
3863 for turn in 1..=chunk_count {
3864 let envelope = agent_stream_test_envelope(turn * 2 - 1, turn);
3865 let routed = route_agent_stream_event(&node_id, &mut cursor, &envelope)
3866 .expect("every chunk in the burst must be published, however far the durable cursor already ran past its sequence");
3867 recovered.push(routed);
3868 }
3869
3870 assert_eq!(recovered.len(), chunk_count as usize);
3871 for (turn, routed) in (1..=chunk_count).zip(recovered.iter()) {
3872 assert!(matches!(
3873 &routed.event,
3874 C2NodeEvent::AgentStream { chunk, .. } if chunk.source_sequence == turn
3875 ));
3876 }
3877 assert_eq!(
3878 cursor.sequence,
3879 chunk_count * 2,
3880 "a whole burst of stale chunks must never rewind the cursor the durable side already advanced",
3881 );
3882 }
3883
3884 #[test]
3885 fn config_rejects_header_injection_duplicate_nodes_and_non_loopback() {
3886 let id = NodeId::new("node-a").unwrap();
3887 assert!(matches!(C2NodeConfig::new(id.clone(), r"\\.\pipe\a", "bad\r\ntoken"), Err(C2ConfigError::InvalidToken)));
3888 let node = C2NodeConfig::new(id, r"\\.\pipe\a", "safe-token").unwrap();
3889 assert!(matches!(C2Config::new("0.0.0.0:0".parse().unwrap(), "safe", vec![node.clone()]), Err(C2ConfigError::NonLoopback(_))));
3890 assert!(matches!(C2Config::new("127.0.0.1:0".parse().unwrap(), "safe", vec![node.clone(), node]), Err(C2ConfigError::DuplicateNode)));
3891 let first = C2NodeConfig::new(NodeId::new("node-a").unwrap(), r"\\.\pipe\same", "safe").unwrap();
3892 let second = C2NodeConfig::new(NodeId::new("node-b").unwrap(), r"\\.\pipe\same", "safe").unwrap();
3893 assert!(matches!(C2Config::new("127.0.0.1:0".parse().unwrap(), "safe", vec![first, second]), Err(C2ConfigError::DuplicateEndpoint)));
3894 let oversized = format!(r"\\.\pipe\{}", "x".repeat(MAX_C2_ENDPOINT_BYTES));
3895 assert!(matches!(C2NodeConfig::new(NodeId::new("node-c").unwrap(), oversized, "safe"), Err(C2ConfigError::InvalidEndpoint(_))));
3896 }
3897
3898 #[test]
3899 fn durable_session_mutations_require_controller_and_use_bounded_deadlines() {
3900 let record_id = SessionRecordId::new("session-001").unwrap();
3901 let rename = NodeRequest::RenameSessionRecord {
3902 record_id: record_id.clone(),
3903 display_name: "release shepherd".to_owned(),
3904 };
3905 let resume = NodeRequest::ResumeSessionRecord {
3906 record_id: record_id.clone(),
3907 terminal_size: TerminalSize { rows: 40, columns: 120 },
3908 initial_prompt: None,
3909 };
3910 let forget = NodeRequest::ForgetSessionRecord { record_id };
3911
3912 assert!(!is_read_only_request(&rename));
3913 assert!(!is_read_only_request(&resume));
3914 assert!(!is_read_only_request(&forget));
3915 assert_eq!(node_request_deadline(&rename), Duration::from_secs(5));
3916 assert_eq!(node_request_deadline(&resume), Duration::from_secs(35));
3917 assert!(node_request_deadline(&resume) > MANAGED_RESUME_SETTLE_DEADLINE);
3918 assert_eq!(
3919 node_request_deadline(&resume) - MANAGED_RESUME_SETTLE_DEADLINE,
3920 NODE_REQUEST_IO_HEADROOM,
3921 );
3922 let started = Instant::now();
3923 let relay_deadline = relay_request_deadline(&resume, started).unwrap();
3924 assert_eq!(
3925 request_budget(&resume, Some(relay_deadline), started),
3926 Duration::from_secs(35),
3927 );
3928 assert_eq!(
3929 request_budget(
3930 &resume,
3931 Some(relay_deadline),
3932 started + Duration::from_secs(5),
3933 ),
3934 MANAGED_RESUME_SETTLE_DEADLINE,
3935 );
3936 assert_eq!(
3937 request_budget(
3938 &resume,
3939 Some(relay_deadline),
3940 started + Duration::from_secs(35),
3941 ),
3942 Duration::ZERO,
3943 );
3944 assert_eq!(node_request_deadline(&forget), Duration::from_secs(5));
3945 }
3946
3947 #[test]
3948 fn workspace_file_reads_are_read_only_with_five_second_deadline() {
3949 let request = NodeRequest::ReadWorkspaceFile {
3950 workspace_id: gate4agent_node_protocol::WorkspaceId::new("primary").unwrap(),
3951 path: gate4agent_node_protocol::RepositoryPath::utf8(
3952 "src/lib.rs".to_owned(),
3953 ).unwrap(),
3954 };
3955
3956 assert!(is_read_only_request(&request));
3957 assert_eq!(node_request_deadline(&request), Duration::from_secs(5));
3958 assert!(relay_request_deadline(&request, Instant::now()).is_none());
3959
3960 }
3961
3962 #[test]
3963 fn workspace_entry_create_relay_has_headroom_and_preserves_semantic_timeout() {
3964 let workspace_id = gate4agent_node_protocol::WorkspaceId::new("primary").unwrap();
3965 let file_path = gate4agent_node_protocol::RepositoryPath::utf8(
3966 "src/new.rs".to_owned(),
3967 ).unwrap();
3968 let create_file = NodeRequest::CreateWorkspaceFile {
3969 workspace_id: workspace_id.clone(),
3970 path: file_path.clone(),
3971 };
3972 assert!(!is_read_only_request(&create_file));
3973 assert_eq!(
3974 node_request_deadline(&create_file),
3975 WORKSPACE_ENTRY_CREATE_RELAY_DEADLINE,
3976 );
3977 assert!(
3978 node_request_deadline(&create_file)
3979 > WORKSPACE_ENTRY_CREATE_NODE_SEMANTIC_DEADLINE,
3980 );
3981 assert!(relay_request_deadline(&create_file, Instant::now()).is_none());
3982
3983 let propagated = relay_node_failure(&NodeClientError::Node(
3984 gate4agent_node_protocol::NodeFailure {
3985 code: NodeFailureCode::RepositoryEntryCreateTimedOut,
3986 message: "private node timeout detail".to_owned(),
3987 },
3988 )).unwrap();
3989 assert_eq!(
3990 propagated.code,
3991 NodeFailureCode::RepositoryEntryCreateTimedOut,
3992 );
3993 assert_eq!(propagated.message, "repository entry creation timed out");
3994
3995 let created_file = gate4agent_node_protocol::WorkspaceFileRead {
3996 workspace_id: workspace_id.clone(),
3997 path: file_path,
3998 content: gate4agent_node_protocol::WorkspaceFileContent::Utf8 {
3999 text: String::new(),
4000 byte_len: 0,
4001 },
4002 revision: Some(
4003 gate4agent_node_protocol::WorkspaceFileRevision::new(
4004 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4005 .to_owned(),
4006 )
4007 .unwrap(),
4008 ),
4009 };
4010 assert!(validate_workspace_content_response(
4011 Some(&create_file),
4012 &Ok(NodeResponse::WorkspaceFileCreated {
4013 file: created_file.clone(),
4014 }),
4015 ).is_ok());
4016 let mut wrong_content = created_file;
4017 wrong_content.content = gate4agent_node_protocol::WorkspaceFileContent::Utf8 {
4018 text: "unexpected".to_owned(),
4019 byte_len: 10,
4020 };
4021 assert!(validate_workspace_content_response(
4022 Some(&create_file),
4023 &Ok(NodeResponse::WorkspaceFileCreated { file: wrong_content }),
4024 ).is_err());
4025
4026 let directory_path = gate4agent_node_protocol::RepositoryPath::utf8(
4027 "src/new".to_owned(),
4028 ).unwrap();
4029 let create_directory = NodeRequest::CreateWorkspaceDirectory {
4030 workspace_id: workspace_id.clone(),
4031 path: directory_path.clone(),
4032 };
4033 assert!(!is_read_only_request(&create_directory));
4034 assert_eq!(
4035 node_request_deadline(&create_directory),
4036 WORKSPACE_ENTRY_CREATE_RELAY_DEADLINE,
4037 );
4038 assert!(validate_workspace_content_response(
4039 Some(&create_directory),
4040 &Ok(NodeResponse::WorkspaceDirectoryCreated {
4041 workspace_id,
4042 entry: gate4agent_node_protocol::WorkspaceEntry {
4043 relative_path: directory_path,
4044 kind: gate4agent_node_protocol::WorkspaceEntryKind::Directory,
4045 },
4046 }),
4047 ).is_ok());
4048 assert!(validate_workspace_content_response(
4049 Some(&create_directory),
4050 &Ok(NodeResponse::Accepted),
4051 ).is_err());
4052 }
4053
4054 #[test]
4055 fn native_session_catalog_is_lease_free_read_only() {
4056 let route = gate4agent_node_protocol::NativeSessionCatalogRoute::workspace(
4057 gate4agent_node_protocol::WorkspaceId::new("primary").unwrap(),
4058 gate4agent_types::AgentId::new("codex").unwrap(),
4059 );
4060 let request = NodeRequest::CatalogNativeSessions {
4061 route: route.clone(),
4062 limit: 8,
4063 };
4064 assert!(is_read_only_request(&request));
4065 assert_eq!(
4066 node_request_deadline(&request),
4067 NATIVE_SESSION_REQUEST_DEADLINE,
4068 );
4069 assert!(relay_request_deadline(&request, Instant::now()).is_none());
4070
4071 let page = NodeRequest::PageNativeSessions {
4072 route: route.clone(),
4073 window: gate4agent_types::NativeSessionCatalogWindow::Older,
4074 catalog_revision: 7,
4075 recent_cutoff_unix_ms: 8,
4076 after_selection_id: Some("hist_selection_1".to_owned()),
4077 limit: 8,
4078 };
4079 assert!(is_read_only_request(&page));
4080 assert_eq!(node_request_deadline(&page), NATIVE_SESSION_REQUEST_DEADLINE);
4081 assert!(relay_request_deadline(&page, Instant::now()).is_none());
4082
4083 let preview = NodeRequest::PreviewNativeSession {
4084 selection: gate4agent_node_protocol::NativeSessionSelection {
4085 route,
4086 catalog_revision: 7,
4087 recent_cutoff_unix_ms: 8,
4088 selection_id: "hist_selection_1".to_owned(),
4089 },
4090 message_limit: 12,
4091 };
4092 assert!(is_read_only_request(&preview));
4093 assert_eq!(
4094 node_request_deadline(&preview),
4095 NATIVE_SESSION_REQUEST_DEADLINE,
4096 );
4097 assert!(relay_request_deadline(&preview, Instant::now()).is_none());
4098
4099 let record_preview = NodeRequest::PreviewSessionRecord {
4100 record_id: gate4agent_node_protocol::SessionRecordId::new("record-1").unwrap(),
4101 message_limit: 12,
4102 };
4103 assert!(is_read_only_request(&record_preview));
4104 assert_eq!(
4105 node_request_deadline(&record_preview),
4106 NATIVE_SESSION_REQUEST_DEADLINE,
4107 );
4108 assert!(relay_request_deadline(&record_preview, Instant::now()).is_none());
4109 }
4110
4111 #[test]
4112 fn standalone_workspace_creation_is_controller_mutation_with_worktree_deadline() {
4113 let request = NodeRequest::CreateStandaloneWorkspace {
4114 workspace_id: gate4agent_node_protocol::WorkspaceId::new("standalone").unwrap(),
4115 root: gate4agent_node_protocol::OpaqueHostPath::utf8(
4116 r"C:\standalone".to_owned(),
4117 ).unwrap(),
4118 initial_branch: Some("main".to_owned()),
4119 };
4120
4121 assert!(!is_read_only_request(&request));
4122 assert_eq!(node_request_deadline(&request), Duration::from_secs(240));
4123 assert!(relay_request_deadline(&request, Instant::now()).is_none());
4124 }
4125
4126 #[test]
4127 fn unsupported_node_capability_is_correlated_without_offline_classification() {
4128 let error = NodeClientError::UnsupportedCapability(
4129 "workspace-file-read-v1-private-detail".to_owned(),
4130 );
4131 let failure = relay_node_failure(&error)
4132 .expect("unsupported node capability must remain an in-band node failure");
4133
4134 assert_eq!(failure.code, NodeFailureCode::UnsupportedCapability);
4135 assert_eq!(failure.message, "required capability unavailable");
4136 assert!(!failure.message.contains("private-detail"));
4137 assert!(relay_node_failure(&NodeClientError::Io(io::Error::new(
4138 io::ErrorKind::BrokenPipe,
4139 "transport closed",
4140 ))).is_none());
4141 }
4142
4143 async fn raw_request(request: Vec<u8>, status: StatusResponse) -> Vec<u8> {
4144 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4145 let address = listener.local_addr().unwrap();
4146 let (_status_tx, status_rx) = watch::channel(Arc::new(status));
4147 let server = tokio::spawn(async move {
4148 let (stream, _) = listener.accept().await.unwrap();
4149 serve_http(stream, "api-token", Duration::from_secs(1), &status_rx).await.unwrap();
4150 });
4151 let mut stream = TcpStream::connect(address).await.unwrap();
4152 stream.write_all(&request).await.unwrap();
4153 let mut response = Vec::new();
4154 stream.read_to_end(&mut response).await.unwrap();
4155 server.await.unwrap();
4156 response
4157 }
4158
4159 fn empty_status(ready: bool) -> StatusResponse {
4160 StatusResponse { api_version: C2_API_VERSION, ready, observed_at_unix_ms: 0, nodes: BTreeMap::new() }
4161 }
4162
4163 #[tokio::test]
4164 async fn http_api_enforces_initializing_auth_method_path_and_header_bounds() {
4165 let initializing = raw_request(b"GET /ready HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), empty_status(false)).await;
4166 assert!(initializing.starts_with(b"HTTP/1.1 503 Service Unavailable\r\n"));
4167 assert!(String::from_utf8_lossy(&initializing).contains("\"ready\":false"));
4168
4169 let unauthorized = raw_request(b"GET /status HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), empty_status(true)).await;
4170 assert!(unauthorized.starts_with(b"HTTP/1.1 401 Unauthorized\r\n"));
4171 let method = raw_request(b"POST /health HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), empty_status(true)).await;
4172 assert!(method.starts_with(b"HTTP/1.1 405 Method Not Allowed\r\n"));
4173 let missing = raw_request(b"GET /missing HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), empty_status(true)).await;
4174 assert!(missing.starts_with(b"HTTP/1.1 404 Not Found\r\n"));
4175
4176 let mut oversized = b"GET /health HTTP/1.1\r\nX-Fill: ".to_vec();
4177 oversized.extend(std::iter::repeat(b'x').take(HEADER_LIMIT_BYTES));
4178 oversized.extend_from_slice(b"\r\n\r\n");
4179 let rejected = raw_request(oversized, empty_status(true)).await;
4180 assert!(rejected.starts_with(b"HTTP/1.1 413 Payload Too Large\r\n"));
4181 }
4182
4183 #[tokio::test]
4184 async fn inventory_state_transitions_offline_to_stale_parked_and_recovers() {
4185 let node_id = NodeId::new("node-a").unwrap();
4186 let mut nodes = BTreeMap::new();
4187 nodes.insert(node_id.clone(), ObservedNode {
4188 endpoint: r"\\.\pipe\a".to_owned(), transport_label: "windows-named-pipe".to_owned(),
4189 transport: NodeTransportState::Offline, freshness: NodeFreshness::Unavailable,
4190 cursor: None, inventory: None, last_attempt_unix_ms: None, last_success_unix_ms: None,
4191 consecutive_failures: 0, last_error: None, gaps: Vec::new(), gaps_truncated: 0,
4192 });
4193 let initial = Arc::new(StatusResponse { api_version: C2_API_VERSION, ready: false, observed_at_unix_ms: unix_ms(), nodes });
4194 let (status_tx, mut status_rx) = watch::channel(initial);
4195 let (ingress_tx, ingress_rx) = mpsc::channel(4);
4196 let (shutdown_tx, shutdown_rx) = watch::channel(false);
4197 let owner = tokio::spawn(inventory_owner(1, Duration::from_millis(10), ingress_rx, status_tx, shutdown_rx));
4198 let snapshot = NodeSnapshot {
4199 node_id: node_id.clone(),
4200 enabled_providers: Vec::new(),
4201 provider_runtime_statuses: crate::protocol::ProviderRuntimeStatuses::default(),
4202 workspaces: Vec::new(),
4203 session_records: Vec::new(),
4204 managed_worktrees: Vec::new(),
4205 launch_inventory: None,
4206 agent_progress: Vec::new(),
4207 };
4208 let cursor = NodeCursor { incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([1; 16]), sequence: 0 };
4209 let old_manifest = ProviderContractManifest {
4210 provider_contracts: vec![crate::protocol::ProviderContractSupport {
4211 provider: agent("codex"),
4212 revision: crate::protocol::ProviderContractRevision::new("old-contract").unwrap(),
4213 }],
4214 provider_adapter_contracts: vec![crate::protocol::ProviderAdapterContractSupport {
4215 provider: agent("codex"),
4216 family: crate::protocol::AdapterFamily::PtySemantic,
4217 adapter_id: crate::protocol::AdapterId::new("codex").unwrap(),
4218 revision: crate::protocol::AdapterContractRevision::new("old-adapter").unwrap(),
4219 }],
4220 };
4221 ingress_tx.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result: AttemptResult::Connected {
4222 cursor,
4223 snapshot: snapshot.clone(),
4224 gaps: Vec::new(),
4225 provider_contract_manifest: old_manifest,
4226 } }).await.unwrap();
4227 status_rx.changed().await.unwrap();
4228 assert_eq!(status_rx.borrow().nodes[&node_id].freshness, NodeFreshness::Fresh);
4229 assert_eq!(
4230 status_rx.borrow().nodes[&node_id].inventory.as_ref().unwrap()
4231 .provider_contracts[0].revision.as_str(),
4232 "old-contract",
4233 );
4234
4235 let failure = || AttemptResult::Failure { error: SanitizedError { category: C2ErrorCategory::Transport, message: "node transport unavailable".to_owned() }, hard: false };
4236 ingress_tx.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result: failure() }).await.unwrap();
4237 status_rx.changed().await.unwrap();
4238 assert_eq!(status_rx.borrow().nodes[&node_id].transport, NodeTransportState::Offline);
4239 timeout(Duration::from_secs(1), async {
4240 loop {
4241 status_rx.changed().await.unwrap();
4242 if status_rx.borrow().nodes[&node_id].freshness == NodeFreshness::Stale { break; }
4243 }
4244 }).await.unwrap();
4245 for _ in 0..4 {
4246 ingress_tx.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result: failure() }).await.unwrap();
4247 status_rx.changed().await.unwrap();
4248 }
4249 assert_eq!(status_rx.borrow().nodes[&node_id].transport, NodeTransportState::Parked);
4250 let replacement_manifest = ProviderContractManifest {
4251 provider_contracts: vec![crate::protocol::ProviderContractSupport {
4252 provider: agent("claude"),
4253 revision: crate::protocol::ProviderContractRevision::new("new-contract").unwrap(),
4254 }],
4255 provider_adapter_contracts: Vec::new(),
4256 };
4257 let replacement_cursor = NodeCursor {
4258 incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([2; 16]),
4259 sequence: 0,
4260 };
4261 ingress_tx.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result: AttemptResult::Connected {
4262 cursor: replacement_cursor,
4263 snapshot,
4264 gaps: Vec::new(),
4265 provider_contract_manifest: replacement_manifest,
4266 } }).await.unwrap();
4267 status_rx.changed().await.unwrap();
4268 assert_eq!(status_rx.borrow().nodes[&node_id].transport, NodeTransportState::Online);
4269 assert_eq!(status_rx.borrow().nodes[&node_id].freshness, NodeFreshness::Fresh);
4270 let recovered_inventory = status_rx.borrow().nodes[&node_id].inventory.as_ref().unwrap().clone();
4271 assert_eq!(recovered_inventory.provider_contracts.len(), 1);
4272 assert_eq!(recovered_inventory.provider_contracts[0].provider, agent("claude"));
4273 assert_eq!(recovered_inventory.provider_contracts[0].revision.as_str(), "new-contract");
4274 assert!(recovered_inventory.provider_adapter_contracts.is_empty());
4275 ingress_tx.send(Attempt {
4276 node_id: node_id.clone(),
4277 at_unix_ms: unix_ms(),
4278 result: AttemptResult::Connected {
4279 cursor: NodeCursor {
4280 incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([3; 16]),
4281 sequence: 0,
4282 },
4283 snapshot: NodeSnapshot {
4284 node_id: node_id.clone(),
4285 enabled_providers: Vec::new(),
4286 provider_runtime_statuses: crate::protocol::ProviderRuntimeStatuses::default(),
4287 workspaces: Vec::new(),
4288 session_records: Vec::new(),
4289 managed_worktrees: Vec::new(),
4290 launch_inventory: None,
4291 agent_progress: Vec::new(),
4292 },
4293 gaps: Vec::new(),
4294 provider_contract_manifest: ProviderContractManifest::default(),
4295 },
4296 }).await.unwrap();
4297 status_rx.changed().await.unwrap();
4298 let unpublished = status_rx.borrow().nodes[&node_id].inventory.as_ref().unwrap().clone();
4299 assert!(unpublished.provider_contracts.is_empty());
4300 assert!(unpublished.provider_adapter_contracts.is_empty());
4301 shutdown_tx.send(true).unwrap();
4302 owner.await.unwrap().unwrap();
4303 }
4304
4305 fn runtime_statuses(
4306 provider: gate4agent_node_protocol::AgentId,
4307 version: &str,
4308 ) -> crate::protocol::ProviderRuntimeStatuses {
4309 crate::protocol::ProviderRuntimeStatuses::new([
4310 crate::protocol::ProviderRuntimeStatus::raw_passthrough(
4311 provider,
4312 Some(crate::protocol::ProviderRuntimeVersion::new(version).unwrap()),
4313 ),
4314 ])
4315 .unwrap()
4316 }
4317
4318 async fn runtime_inventory_owner() -> (
4319 NodeId,
4320 mpsc::Sender<Attempt>,
4321 watch::Receiver<Arc<StatusResponse>>,
4322 watch::Sender<bool>,
4323 tokio::task::JoinHandle<io::Result<()>>,
4324 ) {
4325 let node_id = NodeId::new("runtime-node").unwrap();
4326 let nodes = BTreeMap::from([(
4327 node_id.clone(),
4328 ObservedNode {
4329 endpoint: r"\\.\pipe\runtime-node".to_owned(),
4330 transport_label: "windows-named-pipe".to_owned(),
4331 transport: NodeTransportState::Offline,
4332 freshness: NodeFreshness::Unavailable,
4333 cursor: None,
4334 inventory: None,
4335 last_attempt_unix_ms: None,
4336 last_success_unix_ms: None,
4337 consecutive_failures: 0,
4338 last_error: None,
4339 gaps: Vec::new(),
4340 gaps_truncated: 0,
4341 },
4342 )]);
4343 let initial = Arc::new(StatusResponse {
4344 api_version: C2_API_VERSION,
4345 ready: false,
4346 observed_at_unix_ms: unix_ms(),
4347 nodes,
4348 });
4349 let (status_tx, status_rx) = watch::channel(initial);
4350 let (ingress_tx, ingress_rx) = mpsc::channel(4);
4351 let (shutdown_tx, shutdown_rx) = watch::channel(false);
4352 let owner = tokio::spawn(inventory_owner(
4353 1,
4354 Duration::from_secs(1),
4355 ingress_rx,
4356 status_tx,
4357 shutdown_rx,
4358 ));
4359 (node_id, ingress_tx, status_rx, shutdown_tx, owner)
4360 }
4361
4362 #[tokio::test]
4363 async fn incarnation_change_replaces_runtime_status() {
4364 let (node_id, ingress, mut status, shutdown, owner) = runtime_inventory_owner().await;
4365 for (incarnation, provider, version) in [
4366 (1, agent("claude"), "1.0.0"),
4367 (2, agent("codex"), "2.0.0"),
4368 ] {
4369 ingress
4370 .send(Attempt {
4371 node_id: node_id.clone(),
4372 at_unix_ms: unix_ms(),
4373 result: AttemptResult::Connected {
4374 cursor: NodeCursor {
4375 incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([
4376 incarnation;
4377 16
4378 ]),
4379 sequence: 0,
4380 },
4381 snapshot: NodeSnapshot {
4382 node_id: node_id.clone(),
4383 enabled_providers: vec![provider.clone()],
4384 provider_runtime_statuses: runtime_statuses(provider, version),
4385 workspaces: Vec::new(),
4386 session_records: Vec::new(),
4387 managed_worktrees: Vec::new(),
4388 launch_inventory: None,
4389 agent_progress: Vec::new(),
4390 },
4391 gaps: Vec::new(),
4392 provider_contract_manifest: ProviderContractManifest::default(),
4393 },
4394 })
4395 .await
4396 .unwrap();
4397 status.changed().await.unwrap();
4398 }
4399 let current_status = status.borrow();
4400 let statuses = ¤t_status.nodes[&node_id]
4401 .inventory
4402 .as_ref()
4403 .unwrap()
4404 .provider_runtime_statuses;
4405 assert_eq!(statuses.as_slice().len(), 1);
4406 assert_eq!(
4407 statuses.as_slice()[0].provider(),
4408 &agent("codex"),
4409 );
4410 assert_eq!(statuses.as_slice()[0].version().unwrap().as_str(), "2.0.0");
4411 shutdown.send(true).unwrap();
4412 owner.await.unwrap().unwrap();
4413 }
4414
4415 #[tokio::test]
4416 async fn incarnation_change_without_snapshot_clears_dynamic_inventory() {
4417 let (node_id, ingress, mut status, shutdown, owner) = runtime_inventory_owner().await;
4418 ingress
4419 .send(Attempt {
4420 node_id: node_id.clone(),
4421 at_unix_ms: unix_ms(),
4422 result: AttemptResult::Connected {
4423 cursor: NodeCursor {
4424 incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([
4425 3; 16
4426 ]),
4427 sequence: 0,
4428 },
4429 snapshot: NodeSnapshot {
4430 node_id: node_id.clone(),
4431 enabled_providers: vec![agent("claude")],
4432 provider_runtime_statuses: runtime_statuses(
4433 agent("claude"),
4434 "3.0.0",
4435 ),
4436 workspaces: Vec::new(),
4437 session_records: Vec::new(),
4438 managed_worktrees: Vec::new(),
4439 launch_inventory: None,
4440 agent_progress: Vec::new(),
4441 },
4442 gaps: Vec::new(),
4443 provider_contract_manifest: ProviderContractManifest::default(),
4444 },
4445 })
4446 .await
4447 .unwrap();
4448 status.changed().await.unwrap();
4449 ingress
4450 .send(Attempt {
4451 node_id: node_id.clone(),
4452 at_unix_ms: unix_ms(),
4453 result: AttemptResult::Cursor {
4454 cursor: NodeCursor {
4455 incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([
4456 4; 16
4457 ]),
4458 sequence: 0,
4459 },
4460 gaps: vec![GapKind::IncarnationChanged],
4461 managed_worktree_events: Vec::new(),
4462 },
4463 })
4464 .await
4465 .unwrap();
4466 status.changed().await.unwrap();
4467 assert!(status.borrow().nodes[&node_id]
4468 .inventory
4469 .as_ref()
4470 .unwrap()
4471 .provider_runtime_statuses
4472 .is_empty());
4473 shutdown.send(true).unwrap();
4474 owner.await.unwrap().unwrap();
4475 }
4476
4477 #[test]
4478 fn managed_worktree_inventory_events_are_exact_bounded_and_incarnation_fenced() {
4479 let mut inventory = SlimNodeInventory::from_snapshot(&NodeSnapshot {
4480 node_id: NodeId::new("node-a").unwrap(),
4481 enabled_providers: Vec::new(),
4482 provider_runtime_statuses: crate::protocol::ProviderRuntimeStatuses::default(),
4483 workspaces: Vec::new(),
4484 session_records: Vec::new(),
4485 managed_worktrees: vec![managed_lease(
4486 "lease-a",
4487 "managed-a",
4488 ManagedWorktreeLeaseState::Ready,
4489 )],
4490 launch_inventory: None,
4491 agent_progress: Vec::new(),
4492 });
4493 apply_managed_worktree_cursor(
4494 Some(&mut inventory),
4495 false,
4496 &[
4497 NodeEvent::ManagedWorktreeUpserted {
4498 lease: managed_lease(
4499 "lease-b",
4500 "managed-b",
4501 ManagedWorktreeLeaseState::Ready,
4502 ),
4503 },
4504 NodeEvent::ManagedWorktreeUpserted {
4505 lease: managed_lease(
4506 "lease-a",
4507 "managed-a",
4508 ManagedWorktreeLeaseState::InUse,
4509 ),
4510 },
4511 ],
4512 );
4513 assert_eq!(inventory.managed_worktree_count, 2);
4514 assert_eq!(inventory.managed_worktrees[0].lease_id.as_str(), "lease-a");
4515 assert_eq!(
4516 inventory.managed_worktrees[0].state,
4517 ManagedWorktreeLeaseState::InUse,
4518 );
4519
4520 apply_managed_worktree_cursor(
4521 Some(&mut inventory),
4522 false,
4523 &[NodeEvent::ManagedWorktreeRemoved {
4524 lease_id: ManagedWorktreeLeaseId::new("lease-a").unwrap(),
4525 }],
4526 );
4527 assert_eq!(inventory.managed_worktree_count, 1);
4528 assert_eq!(inventory.managed_worktrees[0].lease_id.as_str(), "lease-b");
4529
4530 apply_managed_worktree_cursor(
4531 Some(&mut inventory),
4532 true,
4533 &[NodeEvent::ManagedWorktreeUpserted {
4534 lease: managed_lease(
4535 "lease-c",
4536 "managed-c",
4537 ManagedWorktreeLeaseState::Ready,
4538 ),
4539 }],
4540 );
4541 assert!(inventory.managed_worktrees.is_empty());
4542 assert_eq!(inventory.managed_worktree_count, 0);
4543 assert!(!inventory.managed_worktrees_truncated);
4544 }
4545}
4546
4547#[cfg(test)]
4548mod relay_deadline_tests {
4549 use super::*;
4550
4551 #[test]
4559 fn workspace_inspection_relay_bound_clears_the_nodes_own_maximum() {
4560 const NODE_INSPECTION_MAX: Duration = Duration::from_millis(11_000);
4565 assert!(
4566 WORKSPACE_INSPECTION_RELAY_DEADLINE > NODE_INSPECTION_MAX,
4567 "relay bound {:?} must exceed the node's own inspection maximum {:?}",
4568 WORKSPACE_INSPECTION_RELAY_DEADLINE,
4569 NODE_INSPECTION_MAX,
4570 );
4571 }
4572}
4573
4574#[cfg(test)]
4575mod control_tunnel_listen_tests {
4576 use super::*;
4577
4578 fn config() -> C2Config {
4579 let node = C2NodeConfig::new(
4580 NodeId::new("node-a").unwrap(),
4581 if cfg!(unix) { "/tmp/gate4agent-node-a.sock" } else { r"\\.\pipe\gate4agent-node-a" },
4582 "token-token",
4583 ).unwrap();
4584 C2Config::new("127.0.0.1:18321".parse().unwrap(), "api-token", vec![node]).unwrap()
4585 }
4586
4587 #[test]
4588 fn control_tunnel_rejects_unspecified_and_accepts_tunnel_address() {
4589 let bad: SocketAddr = "0.0.0.0:27441".parse().unwrap();
4590 assert!(matches!(
4591 config().with_control_tunnel_listen(bad),
4592 Err(C2ConfigError::InvalidControlTunnel(_))
4593 ));
4594 let ok: SocketAddr = "10.88.1.2:27441".parse().unwrap();
4595 let config = config().with_control_tunnel_listen(ok).unwrap();
4596 assert_eq!(config.control_tunnel_listen, Some(ok));
4597 assert!(!config.control_endpoint.is_empty());
4598 }
4599}