Skip to main content

hatchery_c2/
runtime.rs

1use crate::protocol::{
2    C2ErrorCategory, C2NodeEvent, C2NodeFailure, C2NodeResponse, C2RelayFailure, C2RelayFailureCode, GapKind, HealthResponse, NodeCursor, NodeFreshness, NodeGap, NodeId,
3    NodeIncarnationId, NodeRequest, ResolvedSpawnReceipt, RoutedNodeEvent, RoutedNodeResponse,
4    ManagedWorktreeLeaseState, ManagedWorktreeSpawnRequest, ManagedWorktreeSpawnRequestV2,
5    SpawnOverride, SpawnSpec,
6    NodeTransportState, ObservedNode, ProviderAdapterContractSupport, ProviderContractSupport,
7    ReadyResponse, SanitizedError, SlimNodeInventory, StatusResponse,
8    C2_API_VERSION, C2_PROVIDER_CONTRACT_MANIFEST_CAPABILITY,
9    MAX_C2_GAPS_PER_NODE, MAX_C2_NODES,
10};
11#[cfg(windows)]
12use crate::protocol::MAX_C2_ENDPOINT_BYTES;
13use gate4agent_node_protocol::{
14    ClientRole, FrameError, NegotiatedNodeCompatibility, NodeEvent, NodeEventEnvelope, NodeFailureCode,
15    NodeResponse, NodeSnapshot, ServerFrame,
16};
17use gate4agent_node_wire::{read_call_home_announce, LocalNodeClient, NodeClientError};
18use std::collections::{BTreeMap, BTreeSet};
19use std::io;
20use std::net::SocketAddr;
21use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, SystemTime, UNIX_EPOCH};
24#[cfg(unix)]
25use std::path::{Path, PathBuf};
26#[cfg(unix)]
27use std::os::unix::fs::PermissionsExt;
28use thiserror::Error;
29use tokio::io::{AsyncReadExt, AsyncWriteExt};
30use tokio::net::{TcpListener, TcpStream};
31use tokio::sync::{mpsc, oneshot, watch, Semaphore};
32use tokio::task::{JoinHandle, JoinSet};
33use tokio::time::{sleep, timeout, Instant};
34
35const MANAGED_RESUME_SETTLE_DEADLINE: Duration = Duration::from_secs(30);
36const NODE_REQUEST_IO_HEADROOM: Duration = Duration::from_secs(5);
37const NATIVE_SESSION_REQUEST_DEADLINE: Duration = Duration::from_secs(35);
38const WORKSPACE_ENTRY_CREATE_NODE_SEMANTIC_DEADLINE: Duration = Duration::from_secs(5);
39const WORKSPACE_ENTRY_CREATE_RELAY_DEADLINE: Duration =
40    WORKSPACE_ENTRY_CREATE_NODE_SEMANTIC_DEADLINE.saturating_add(NODE_REQUEST_IO_HEADROOM);
41
42const HEADER_LIMIT_BYTES: usize = 16 * 1024;
43const MAX_HTTP_CONNECTIONS: usize = 16;
44const RESPONSE_BODY_LIMIT_BYTES: usize = 8 * 1024 * 1024;
45
46mod control;
47
48#[cfg(windows)]
49pub const DEFAULT_C2_CONTROL_ENDPOINT: &str = r"\\.\pipe\gate4agent-c2";
50#[cfg(unix)]
51pub const DEFAULT_C2_CONTROL_ENDPOINT: &str = "gate4agent-c2.sock";
52
53#[cfg(windows)]
54pub fn default_c2_control_endpoint() -> Result<String, C2ConfigError> {
55    Ok(DEFAULT_C2_CONTROL_ENDPOINT.to_owned())
56}
57
58#[cfg(unix)]
59pub fn default_c2_control_endpoint() -> Result<String, C2ConfigError> {
60    let root = unix_runtime_root()?;
61    let directory = root.join("gate4agent");
62    let endpoint = directory.join(DEFAULT_C2_CONTROL_ENDPOINT);
63    validate_unix_endpoint(&endpoint).map_err(|_| C2ConfigError::InvalidControlEndpoint)?;
64    Ok(endpoint.to_string_lossy().into_owned())
65}
66
67#[cfg(unix)]
68fn unix_runtime_root() -> Result<PathBuf, C2ConfigError> {
69    if let Some(root) = std::env::var_os("XDG_RUNTIME_DIR").filter(|value| !value.is_empty()) {
70        let root = PathBuf::from(root);
71        if root.is_absolute() { return Ok(root); }
72    }
73    let home = std::env::var_os("HOME").filter(|value| !value.is_empty())
74        .ok_or_else(|| C2ConfigError::RuntimeEndpoint(
75            "neither absolute XDG_RUNTIME_DIR nor HOME is available".to_owned(),
76        ))?;
77    let home = PathBuf::from(home);
78    if !home.is_absolute() {
79        return Err(C2ConfigError::RuntimeEndpoint("HOME is not absolute".to_owned()));
80    }
81    Ok(home.join(".gate4agent").join("run"))
82}
83
84#[derive(Clone)]
85pub struct C2NodeConfig {
86    pub node_id: NodeId,
87    pub endpoint: String,
88    route: C2NodeRoute,
89    token: String,
90}
91
92#[derive(Clone, Copy, Debug, Eq, PartialEq)]
93enum C2NodeRoute {
94    Local,
95    SshForwardedLoopback(SocketAddr),
96    /// This relay does not dial the node; the node dials the relay.
97    ///
98    /// For a node the relay has no way to reach -- behind NAT, in a
99    /// container with no published port, on a machine with no listener the
100    /// relay is allowed to open. The protocol roles are unchanged: the
101    /// node is still the server, this relay is still the client asking it
102    /// for things. Only the direction of the TCP connection moves, which
103    /// is why the whole of this variant's implementation is "wait for a
104    /// socket instead of opening one" and then the identical handshake.
105    CallHome,
106}
107
108impl C2NodeConfig {
109    pub fn new(node_id: NodeId, endpoint: impl Into<String>, token: impl Into<String>) -> Result<Self, C2ConfigError> {
110        let endpoint = endpoint.into();
111        let token = token.into();
112        let (endpoint, route) = parse_node_endpoint(&endpoint)
113            .ok_or_else(|| C2ConfigError::InvalidEndpoint(node_id.clone()))?;
114        validate_token(&token)?;
115        Ok(Self { node_id, endpoint, route, token })
116    }
117
118    fn transport_label(&self) -> &'static str {
119        match self.route {
120            C2NodeRoute::Local => local_transport_label(),
121            C2NodeRoute::SshForwardedLoopback(_) => "ssh-forwarded-loopback",
122            C2NodeRoute::CallHome => "call-home",
123        }
124    }
125}
126
127#[derive(Clone, Copy, Debug)]
128pub struct C2Timings {
129    pub poll_interval: Duration,
130    pub fresh_for: Duration,
131    pub attempt_deadline: Duration,
132    pub transient_backoffs: [Duration; 5],
133    pub parked_backoff: Duration,
134    pub http_io_deadline: Duration,
135}
136
137impl Default for C2Timings {
138    fn default() -> Self {
139        Self {
140            poll_interval: Duration::from_millis(250),
141            fresh_for: Duration::from_secs(10),
142            attempt_deadline: Duration::from_secs(5),
143            transient_backoffs: [
144                Duration::from_millis(500), Duration::from_secs(1), Duration::from_secs(2),
145                Duration::from_secs(4), Duration::from_secs(8),
146            ],
147            parked_backoff: Duration::from_secs(30),
148            http_io_deadline: Duration::from_secs(3),
149        }
150    }
151}
152
153#[derive(Clone)]
154pub struct C2Config {
155    pub api_listen: SocketAddr,
156    pub control_endpoint: String,
157    /// Optional TCP control accept bound to the C2 tunnel address.
158    /// The unix `control_endpoint` listener is always started as well.
159    pub control_tunnel_listen: Option<SocketAddr>,
160    api_token: String,
161    pub nodes: Vec<C2NodeConfig>,
162    /// Where nodes that cannot be dialled come to announce themselves.
163    /// `None` unless at least one node is configured `accept`.
164    pub node_listen: Option<SocketAddr>,
165    pub timings: C2Timings,
166}
167
168impl C2Config {
169    pub fn new(api_listen: SocketAddr, api_token: impl Into<String>, nodes: Vec<C2NodeConfig>) -> Result<Self, C2ConfigError> {
170        let api_token = api_token.into();
171        if !api_listen.ip().is_loopback() { return Err(C2ConfigError::NonLoopback(api_listen)); }
172        validate_token(&api_token)?;
173        if nodes.is_empty() || nodes.len() > MAX_C2_NODES { return Err(C2ConfigError::NodeCount(nodes.len())); }
174        let mut ids = BTreeSet::new();
175        let mut endpoints = BTreeSet::new();
176        if nodes.iter().any(|node| !ids.insert(node.node_id.clone())) { return Err(C2ConfigError::DuplicateNode); }
177        // Call-home nodes are exempt: they name no endpoint, so every one
178        // of them carries the same `accept` and the uniqueness this check
179        // enforces is `node_id`'s job for them (already done above). The
180        // check exists to stop two dialled nodes pointing at one socket,
181        // which cannot happen to a node that is never dialled.
182        if nodes
183            .iter()
184            .filter(|node| node.route != C2NodeRoute::CallHome)
185            .any(|node| !endpoints.insert(endpoint_key(&node.endpoint)))
186        {
187            return Err(C2ConfigError::DuplicateEndpoint);
188        }
189        let control_endpoint = default_c2_control_endpoint()?;
190        if nodes.iter().any(|node| endpoints_equal(&node.endpoint, &control_endpoint)) {
191            return Err(C2ConfigError::ControlEndpointConflict);
192        }
193        Ok(Self {
194            api_listen,
195            control_endpoint,
196            control_tunnel_listen: None,
197            api_token,
198            nodes,
199            node_listen: None,
200            timings: C2Timings::default(),
201        })
202    }
203
204    /// Binds the address nodes call in on.
205    ///
206    /// Loopback, or one concrete C2 WireGuard tunnel address (same rule as
207    /// [`Self::with_control_tunnel_listen`]). The node wire authenticates both
208    /// ends but encrypts nothing, so `0.0.0.0` and other unspecified binds are
209    /// refused: off-tunnel plaintext would expose terminal contents. When the
210    /// listen address is the C2 tunnel IP, kernel WireGuard encrypts the path
211    /// and either side may dial.
212    pub fn with_node_listen(mut self, node_listen: SocketAddr) -> Result<Self, C2ConfigError> {
213        // Concrete non-unspecified + nonzero port. Loopback stays valid; WG
214        // tunnel addresses (e.g. 10.88.1.2:PORT) are valid for kit peers.
215        if node_listen.port() == 0 || node_listen.ip().is_unspecified() {
216            return Err(C2ConfigError::NonLoopbackNodeListen(node_listen));
217        }
218        if node_listen == self.api_listen {
219            return Err(C2ConfigError::NodeListenConflict);
220        }
221        if self.control_tunnel_listen == Some(node_listen) {
222            return Err(C2ConfigError::NonLoopbackNodeListen(node_listen));
223        }
224        self.node_listen = Some(node_listen);
225        Ok(self)
226    }
227
228    /// Every node that waits to be called needs somewhere to call, so a
229    /// config that asks for one without the other is refused rather than
230    /// started into a relay that can never reach that node. Checked at
231    /// startup, not at connect time, because the failure is total and
232    /// permanent -- there is nothing to retry.
233    pub fn validate_call_home(&self) -> Result<(), C2ConfigError> {
234        let waiting = self.nodes.iter().find(|node| node.route == C2NodeRoute::CallHome);
235        match (waiting, self.node_listen) {
236            (Some(node), None) => Err(C2ConfigError::CallHomeWithoutListener(node.node_id.clone())),
237            _ => Ok(()),
238        }
239    }
240
241    pub fn with_timings(mut self, timings: C2Timings) -> Self {
242        self.timings = timings;
243        self
244    }
245
246    pub fn with_control_endpoint(mut self, endpoint: impl Into<String>) -> Result<Self, C2ConfigError> {
247        let endpoint = endpoint.into();
248        validate_control_endpoint(&endpoint)?;
249        if self.nodes.iter().any(|node| endpoints_equal(&node.endpoint, &endpoint)) {
250            return Err(C2ConfigError::ControlEndpointConflict);
251        }
252        self.control_endpoint = endpoint;
253        Ok(self)
254    }
255
256    /// TCP door for an HQ WireGuard client. Bound to one concrete address
257    /// (the address on the C2 tunnel), never `0.0.0.0` or port 0. Does not
258    /// replace the unix control listener.
259    pub fn with_control_tunnel_listen(mut self, listen: SocketAddr) -> Result<Self, C2ConfigError> {
260        if listen.port() == 0 || listen.ip().is_unspecified() {
261            return Err(C2ConfigError::InvalidControlTunnel(listen));
262        }
263        if listen == self.api_listen || self.node_listen == Some(listen) {
264            return Err(C2ConfigError::InvalidControlTunnel(listen));
265        }
266        self.control_tunnel_listen = Some(listen);
267        Ok(self)
268    }
269}
270
271fn validate_control_endpoint(endpoint: &str) -> Result<(), C2ConfigError> {
272    if !valid_local_endpoint(endpoint) {
273        return Err(C2ConfigError::InvalidControlEndpoint);
274    }
275    Ok(())
276}
277
278fn parse_node_endpoint(endpoint: &str) -> Option<(String, C2NodeRoute)> {
279    // The one assignment that names no address, because there is nothing
280    // to address: this node will arrive on the call-home listener under
281    // its own name. Spelled as a word rather than an empty value so a
282    // config reader can tell "waits to be called" from "somebody forgot to
283    // fill this in".
284    if endpoint == "accept" {
285        return Some((endpoint.to_owned(), C2NodeRoute::CallHome));
286    }
287    if let Some(authority) = endpoint.strip_prefix("tcp://") {
288        let address = authority.parse::<SocketAddr>().ok()?;
289        let is_exact_loopback = match address.ip() {
290            std::net::IpAddr::V4(ip) => ip == std::net::Ipv4Addr::LOCALHOST,
291            std::net::IpAddr::V6(ip) => ip == std::net::Ipv6Addr::LOCALHOST,
292        };
293        if !is_exact_loopback || address.port() == 0 {
294            return None;
295        }
296        return Some((format!("tcp://{address}"), C2NodeRoute::SshForwardedLoopback(address)));
297    }
298    if endpoint.contains("://") || !valid_local_endpoint(endpoint) {
299        return None;
300    }
301    Some((endpoint.to_owned(), C2NodeRoute::Local))
302}
303
304#[cfg(windows)]
305fn valid_local_endpoint(endpoint: &str) -> bool {
306    endpoint.starts_with(r"\\.\pipe\") && endpoint.len() > r"\\.\pipe\".len()
307        && endpoint.len() <= MAX_C2_ENDPOINT_BYTES
308}
309
310#[cfg(unix)]
311fn valid_local_endpoint(endpoint: &str) -> bool {
312    validate_unix_endpoint(Path::new(endpoint)).is_ok()
313}
314
315#[cfg(unix)]
316fn validate_unix_endpoint(endpoint: &Path) -> Result<(), ()> {
317    const MAX_UNIX_ENDPOINT_BYTES: usize = 103;
318    use std::os::unix::ffi::OsStrExt;
319
320    if !endpoint.is_absolute() || endpoint.file_name().is_none()
321        || endpoint.as_os_str().as_bytes().len() > MAX_UNIX_ENDPOINT_BYTES
322    {
323        return Err(());
324    }
325    Ok(())
326}
327
328#[cfg(windows)]
329fn endpoint_key(endpoint: &str) -> String {
330    if endpoint.starts_with("tcp://") { endpoint.to_owned() } else { endpoint.to_ascii_lowercase() }
331}
332
333#[cfg(unix)]
334fn endpoint_key(endpoint: &str) -> String { endpoint.to_owned() }
335
336fn endpoints_equal(left: &str, right: &str) -> bool { endpoint_key(left) == endpoint_key(right) }
337
338fn validate_token(token: &str) -> Result<(), C2ConfigError> {
339    if token.is_empty() || token.len() > 4096 || !token.bytes().all(|byte| matches!(byte, 0x21..=0x7e)) {
340        return Err(C2ConfigError::InvalidToken);
341    }
342    Ok(())
343}
344
345#[derive(Debug, Error)]
346pub enum C2ConfigError {
347    #[error("C2 tokens must contain 1..=4096 visible ASCII bytes without whitespace")]
348    InvalidToken,
349    #[cfg_attr(windows, error("node '{0}' requires a bounded Windows named pipe or exact loopback TCP endpoint"))]
350    #[cfg_attr(unix, error("node '{0}' requires a bounded local socket or exact loopback TCP endpoint"))]
351    InvalidEndpoint(NodeId),
352    #[error("C2 API listen address must be loopback: {0}")]
353    NonLoopback(SocketAddr),
354    #[error("C2 requires 1..=64 configured nodes; received {0}")]
355    NodeCount(usize),
356    #[error("C2 node IDs must be unique")]
357    DuplicateNode,
358    #[error("C2 node endpoints must be unique")]
359    DuplicateEndpoint,
360    #[cfg_attr(windows, error("C2 control endpoint must be a bounded local Windows named pipe"))]
361    #[cfg_attr(unix, error("C2 control endpoint must be a bounded local endpoint"))]
362    InvalidControlEndpoint,
363    #[error("C2 control endpoint must not equal a configured node endpoint")]
364    ControlEndpointConflict,
365    #[error("C2 control tunnel listen must be a concrete non-unspecified address with a nonzero port, not shared with another listener: {0}")]
366    InvalidControlTunnel(SocketAddr),
367    #[error("C2 node call-home listen must be loopback or a concrete C2 tunnel address with a nonzero port (not 0.0.0.0, not shared with another listener): {0}")]
368    NonLoopbackNodeListen(SocketAddr),
369    #[error("C2 node call-home listen address must not equal the API listen address")]
370    NodeListenConflict,
371    #[error("node '{0}' waits to be called but no --node-listen address was configured")]
372    CallHomeWithoutListener(NodeId),
373    #[cfg(unix)]
374    #[error("C2 default runtime endpoint is unavailable: {0}")]
375    RuntimeEndpoint(String),
376}
377
378type RelayResult = Result<RoutedNodeResponse, C2RelayFailure>;
379
380enum RelayCommand {
381    Request {
382        operator_connection_id: u64,
383        expected_incarnation_id: NodeIncarnationId,
384        request: NodeRequest,
385        reply: oneshot::Sender<RelayResult>,
386    },
387}
388
389#[derive(Clone)]
390struct RelayEndpoint {
391    commands: mpsc::Sender<RelayCommand>,
392    releases: mpsc::Sender<oneshot::Sender<()>>,
393    force_disconnect: watch::Sender<u64>,
394}
395
396#[derive(Clone)]
397struct OperatorHub {
398    sink: Arc<Mutex<Option<OperatorEventSink>>>,
399}
400
401#[derive(Clone)]
402struct OperatorEventSink {
403    connection_id: u64,
404    outbound: mpsc::Sender<control::QueuedFrame>,
405    budget: Arc<AtomicUsize>,
406    disconnect: watch::Sender<bool>,
407}
408
409impl OperatorHub {
410    fn new() -> Self { Self { sink: Arc::new(Mutex::new(None)) } }
411
412    fn attach(&self, sink: OperatorEventSink) {
413        *self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(sink);
414    }
415
416    fn detach(&self, connection_id: u64) {
417        let mut sink = self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
418        if sink.as_ref().is_some_and(|current| current.connection_id == connection_id) { *sink = None; }
419    }
420
421    fn is_active(&self, connection_id: u64) -> bool {
422        self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
423            .as_ref().is_some_and(|current| current.connection_id == connection_id)
424    }
425
426    fn has_active_operator(&self) -> bool {
427        self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner()).is_some()
428    }
429
430    fn publish(&self, event: RoutedNodeEvent) {
431        let sink = self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
432        if let Some(sink) = sink.as_ref() {
433            if control::queue_operator_event(&sink.outbound, &sink.budget, event).is_err() {
434                let _ = sink.disconnect.send(true);
435            }
436        }
437    }
438}
439
440fn relay_failure(
441    code: C2RelayFailureCode,
442    message: &'static str,
443    current_incarnation_id: Option<NodeIncarnationId>,
444) -> C2RelayFailure {
445    C2RelayFailure { code, message: message.to_owned(), current_incarnation_id }
446}
447
448#[derive(Debug, Error)]
449pub enum C2Error {
450    #[error("C2 API failed: {0}")]
451    Api(#[from] io::Error),
452    #[error("C2 task failed: {0}")]
453    Task(#[from] tokio::task::JoinError),
454}
455
456#[derive(Clone)]
457pub struct C2ShutdownHandle {
458    shutdown: watch::Sender<bool>,
459}
460
461impl C2ShutdownHandle {
462    pub fn shutdown(&self) { let _ = self.shutdown.send(true); }
463}
464
465pub struct C2Running {
466    api_addr: SocketAddr,
467    shutdown: C2ShutdownHandle,
468    task: Option<JoinHandle<Result<(), C2Error>>>,
469}
470
471impl C2Running {
472    pub async fn start(config: C2Config) -> Result<Self, C2Error> {
473        prepare_default_control_parent(&config.control_endpoint)?;
474        let listener = TcpListener::bind(config.api_listen).await?;
475        let api_addr = listener.local_addr()?;
476        let (shutdown_tx, shutdown_rx) = watch::channel(false);
477        let shutdown = C2ShutdownHandle { shutdown: shutdown_tx };
478        let task = tokio::spawn(run_bound(config, listener, shutdown_rx));
479        Ok(Self { api_addr, shutdown, task: Some(task) })
480    }
481
482    pub fn api_addr(&self) -> SocketAddr { self.api_addr }
483    pub fn shutdown_handle(&self) -> C2ShutdownHandle { self.shutdown.clone() }
484    pub async fn wait(mut self) -> Result<(), C2Error> {
485        self.task.take().expect("C2 task is present").await?
486    }
487}
488
489#[cfg(windows)]
490fn prepare_default_control_parent(_endpoint: &str) -> io::Result<()> { Ok(()) }
491
492#[cfg(unix)]
493fn prepare_default_control_parent(endpoint: &str) -> io::Result<()> {
494    let default = default_c2_control_endpoint()
495        .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error.to_string()))?;
496    if endpoint != default { return Ok(()); }
497    let parent = Path::new(endpoint).parent().ok_or_else(|| {
498        io::Error::new(io::ErrorKind::InvalidInput, "default C2 endpoint has no parent")
499    })?;
500    std::fs::create_dir_all(parent)?;
501    std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700))
502}
503
504impl Drop for C2Running {
505    fn drop(&mut self) {
506        self.shutdown.shutdown();
507        if let Some(task) = self.task.take() { task.abort(); }
508    }
509}
510
511async fn run_bound(config: C2Config, listener: TcpListener, mut shutdown: watch::Receiver<bool>) -> Result<(), C2Error> {
512    let now = unix_ms();
513    let nodes = config.nodes.iter().map(|node| {
514        (node.node_id.clone(), initial_observed_node(node))
515    }).collect();
516    let initial = Arc::new(StatusResponse { api_version: C2_API_VERSION, ready: false, observed_at_unix_ms: now, nodes });
517    let (status_tx, status_rx) = watch::channel(initial);
518    let (ingress_tx, ingress_rx) = mpsc::channel(config.nodes.len().saturating_mul(2).max(2));
519    let mut tasks = JoinSet::new();
520    let hub = OperatorHub::new();
521    let mut relay_senders = BTreeMap::new();
522    let mut relay_receivers = Vec::new();
523    // Where the call-home listener hands a freshly announced socket. One
524    // slot per waiting node and no more: a node has one live connection to
525    // this relay, so a second socket arriving while the first is still
526    // being served means the node reconnected and the relay has not
527    // noticed yet -- queueing more of those would only serve them stale.
528    let mut call_home_routes: BTreeMap<NodeId, mpsc::Sender<TcpStream>> = BTreeMap::new();
529    for node in config.nodes.clone() {
530        let (commands_tx, commands_rx) = mpsc::channel(8);
531        let (releases_tx, releases_rx) = mpsc::channel(1);
532        let (force_tx, force_rx) = watch::channel(0_u64);
533        relay_senders.insert(node.node_id.clone(), RelayEndpoint {
534            commands: commands_tx,
535            releases: releases_tx,
536            force_disconnect: force_tx,
537        });
538        let call_home_rx = if node.route == C2NodeRoute::CallHome {
539            let (call_home_tx, call_home_rx) = mpsc::channel(1);
540            call_home_routes.insert(node.node_id.clone(), call_home_tx);
541            Some(call_home_rx)
542        } else {
543            None
544        };
545        relay_receivers.push((node, commands_rx, releases_rx, force_rx, call_home_rx));
546    }
547    let relay_senders = Arc::new(relay_senders);
548    if let Some(node_listen) = config.node_listen {
549        tasks.spawn(accept_call_home(node_listen, call_home_routes, shutdown.clone()));
550    }
551    tasks.spawn(inventory_owner(config.nodes.len(), config.timings.fresh_for, ingress_rx, status_tx, shutdown.clone()));
552    for (node, commands, releases, force_disconnect, call_home) in relay_receivers {
553        tasks.spawn(node_relay_worker(node, config.timings, commands, releases, force_disconnect, call_home, ingress_tx.clone(), status_rx.clone(), hub.clone(), shutdown.clone()));
554    }
555    drop(ingress_tx);
556    tasks.spawn(http_server(listener, config.api_token.clone(), config.timings.http_io_deadline, status_rx.clone(), shutdown.clone()));
557    let slots = control::ControlSlots::new();
558    let tunnel_listen = config.control_tunnel_listen;
559    let tunnel_token = config.api_token.clone();
560    tasks.spawn(control::run(
561        config.control_endpoint,
562        config.api_token,
563        Arc::clone(&relay_senders),
564        status_rx.clone(),
565        hub.clone(),
566        shutdown.clone(),
567        slots.clone(),
568    ));
569    if let Some(listen) = tunnel_listen {
570        tasks.spawn(control::run_tcp(
571            listen,
572            tunnel_token,
573            relay_senders,
574            status_rx,
575            hub,
576            shutdown.clone(),
577            slots,
578        ));
579    }
580    loop {
581        tokio::select! {
582            changed = shutdown.changed() => {
583                if changed.is_err() || *shutdown.borrow() { break; }
584            }
585            result = tasks.join_next() => {
586                match result {
587                    Some(Ok(Ok(()))) if *shutdown.borrow() => break,
588                    Some(Ok(Ok(()))) => return Err(C2Error::Api(io::Error::new(io::ErrorKind::Other, "C2 task exited unexpectedly"))),
589                    Some(Ok(Err(error))) => return Err(C2Error::Api(error)),
590                    Some(Err(error)) => return Err(C2Error::Task(error)),
591                    None => break,
592                }
593            }
594        }
595    }
596    tasks.shutdown().await;
597    Ok(())
598}
599
600fn initial_observed_node(node: &C2NodeConfig) -> ObservedNode {
601    ObservedNode {
602        endpoint: node.endpoint.clone(), transport_label: node.transport_label().to_owned(),
603        transport: NodeTransportState::Offline, freshness: NodeFreshness::Unavailable,
604        cursor: None, inventory: None, last_attempt_unix_ms: None, last_success_unix_ms: None,
605        consecutive_failures: 0, last_error: None, gaps: Vec::new(), gaps_truncated: 0,
606    }
607}
608
609#[cfg(windows)]
610fn local_transport_label() -> &'static str { "windows-named-pipe" }
611
612#[cfg(unix)]
613fn local_transport_label() -> &'static str { "unix-domain-socket" }
614
615#[derive(Clone, Default)]
616struct ProviderContractManifest {
617    provider_contracts: Vec<ProviderContractSupport>,
618    provider_adapter_contracts: Vec<ProviderAdapterContractSupport>,
619}
620
621impl ProviderContractManifest {
622    fn from_compatibility(compatibility: Option<&NegotiatedNodeCompatibility>) -> Self {
623        let Some(compatibility) = compatibility.filter(|compatibility| {
624            compatibility.capabilities.iter().any(|capability| {
625                capability.as_str() == C2_PROVIDER_CONTRACT_MANIFEST_CAPABILITY
626            })
627        }) else {
628            return Self::default();
629        };
630        Self {
631            provider_contracts: compatibility.provider_contracts.clone(),
632            provider_adapter_contracts: compatibility.provider_adapter_contracts.clone(),
633        }
634    }
635}
636
637enum AttemptResult {
638    Connected {
639        cursor: NodeCursor,
640        snapshot: NodeSnapshot,
641        gaps: Vec<GapKind>,
642        provider_contract_manifest: ProviderContractManifest,
643    },
644    Success { cursor: NodeCursor, snapshot: NodeSnapshot, gaps: Vec<GapKind> },
645    Cursor {
646        cursor: NodeCursor,
647        gaps: Vec<GapKind>,
648        managed_worktree_events: Vec<NodeEvent>,
649    },
650    Failure { error: SanitizedError, hard: bool },
651}
652
653struct Attempt { node_id: NodeId, at_unix_ms: u64, result: AttemptResult }
654
655/// Accepts node connections and hands each one to the relay worker for
656/// whichever node it says it is.
657///
658/// This is the only place in the relay that learns a node's identity from
659/// the wire rather than from configuration, and it is careful about what
660/// that means: the announced name SELECTS a worker and nothing more. The
661/// worker then runs the same mutual challenge-response it runs on a
662/// dialled connection, against the token configured for that node, so a
663/// caller that announced a name it cannot prove gets no further than the
664/// next frame.
665///
666/// A socket for an unknown node, an unreadable preface, or a node whose
667/// slot is already full is dropped with a line saying which -- never
668/// silently, because "the node is calling but nothing happens" is exactly
669/// the failure that is impossible to diagnose from the node's end.
670async fn accept_call_home(
671    listen: SocketAddr,
672    routes: BTreeMap<NodeId, mpsc::Sender<TcpStream>>,
673    mut shutdown: watch::Receiver<bool>,
674) -> io::Result<()> {
675    let listener = TcpListener::bind(listen).await?;
676    tracing::info!(%listen, waiting_nodes = routes.len(), "call-home listener bound");
677    loop {
678        if *shutdown.borrow() {
679            return Ok(());
680        }
681        let (stream, peer) = tokio::select! {
682            accepted = listener.accept() => accepted?,
683            changed = shutdown.changed() => {
684                if changed.is_err() || *shutdown.borrow() { return Ok(()); }
685                continue;
686            }
687        };
688        let routes = routes.clone();
689        // Off the accept loop: the preface has its own deadline, and one
690        // peer that connects and then says nothing must not stop every
691        // other node from being accepted meanwhile.
692        tokio::spawn(async move {
693            let mut stream = stream;
694            let _ = stream.set_nodelay(true);
695            let node_id = match read_call_home_announce(&mut stream).await {
696                Ok(node_id) => node_id,
697                Err(error) => {
698                    tracing::warn!(%peer, cause = %error, "call-home connection rejected");
699                    return;
700                }
701            };
702            let Some(sender) = routes.get(&node_id) else {
703                tracing::warn!(
704                    %peer,
705                    %node_id,
706                    "call-home connection named a node this relay does not wait for",
707                );
708                return;
709            };
710            match sender.try_send(stream) {
711                Ok(()) => tracing::info!(%peer, %node_id, "node called home"),
712                Err(mpsc::error::TrySendError::Full(_)) => tracing::warn!(
713                    %peer,
714                    %node_id,
715                    "node called home while its previous connection is still being taken up",
716                ),
717                Err(mpsc::error::TrySendError::Closed(_)) => tracing::warn!(
718                    %peer,
719                    %node_id,
720                    "node called home but its relay worker is gone",
721                ),
722            }
723        });
724    }
725}
726
727/// Blocks until this node calls in, or shutdown -- with no deadline of its
728/// own on purpose.
729///
730/// A dialled connection either answers within `attempt_deadline` or has
731/// failed, and treating a slow answer as a failure is right there. Waiting
732/// for a node to call is the opposite: a node that has not called yet has
733/// not failed at anything, and it may be minutes from starting. Putting
734/// that wait under the attempt deadline would turn an idle relay into a
735/// stream of "connection deadline exceeded" warnings about nodes that are
736/// merely not running. The handshake that FOLLOWS is deadlined normally --
737/// a peer that connects and then stalls is a real failure.
738async fn await_call_home(
739    receiver: Option<&mut mpsc::Receiver<TcpStream>>,
740    shutdown: &mut watch::Receiver<bool>,
741) -> Option<TcpStream> {
742    let receiver = receiver?;
743    loop {
744        if *shutdown.borrow() {
745            return None;
746        }
747        tokio::select! {
748            stream = receiver.recv() => return stream,
749            changed = shutdown.changed() => {
750                if changed.is_err() || *shutdown.borrow() { return None; }
751            }
752        }
753    }
754}
755
756async fn node_relay_worker(
757    node: C2NodeConfig,
758    timings: C2Timings,
759    mut commands: mpsc::Receiver<RelayCommand>,
760    mut releases: mpsc::Receiver<oneshot::Sender<()>>,
761    mut force_disconnect: watch::Receiver<u64>,
762    mut call_home: Option<mpsc::Receiver<TcpStream>>,
763    ingress: mpsc::Sender<Attempt>,
764    status: watch::Receiver<Arc<StatusResponse>>,
765    hub: OperatorHub,
766    mut shutdown: watch::Receiver<bool>,
767) -> io::Result<()> {
768    let mut failures = 0_usize;
769    loop {
770        if *shutdown.borrow() { return Ok(()); }
771        let previous = status.borrow().nodes.get(&node.node_id).and_then(|item| item.cursor);
772        // A waiting node's socket is collected BEFORE the deadline starts,
773        // because waiting for a node to call is not an attempt that can
774        // time out -- see `await_call_home`. Once it is here, the
775        // handshake over it is deadlined exactly like a dialled one.
776        let adopted = if node.route == C2NodeRoute::CallHome {
777            match await_call_home(call_home.as_mut(), &mut shutdown).await {
778                Some(stream) => Some(stream),
779                None => return Ok(()),
780            }
781        } else {
782            None
783        };
784        let connected = timeout(timings.attempt_deadline, connect_operator(&node, adopted)).await;
785        let mut client = match connected {
786            Ok(Ok(client)) => client,
787            Ok(Err(error)) => {
788                failures = failures.saturating_add(1);
789                let (error, hard) = sanitize_node_error(&error);
790                tracing::warn!(
791                    node_id = %node.node_id,
792                    cause = %error.message,
793                    category = ?error.category,
794                    hard,
795                    "node connection attempt failed",
796                );
797                ingress_attempt(&ingress, &node.node_id, AttemptResult::Failure { error, hard }).await?;
798                reject_disconnected_commands(&mut commands, previous);
799                acknowledge_disconnected_releases(&mut releases);
800                relay_backoff(&mut shutdown, timings, failures, hard).await?;
801                continue;
802            }
803            Err(_) => {
804                failures = failures.saturating_add(1);
805                tracing::warn!(
806                    node_id = %node.node_id,
807                    cause = "node connection deadline exceeded",
808                    "node connection attempt failed",
809                );
810                ingress_attempt(&ingress, &node.node_id, AttemptResult::Failure {
811                    error: SanitizedError { category: C2ErrorCategory::Timeout, message: "node connection deadline exceeded".to_owned() },
812                    hard: false,
813                }).await?;
814                reject_disconnected_commands(&mut commands, previous);
815                acknowledge_disconnected_releases(&mut releases);
816                relay_backoff(&mut shutdown, timings, failures, false).await?;
817                continue;
818            }
819        };
820        failures = 0;
821        let hello = client.hello().clone();
822        let provider_contract_manifest =
823            ProviderContractManifest::from_compatibility(hello.compatibility.as_ref());
824        let incarnation_id = hello.incarnation_id;
825        let connection_id = hello.connection_id;
826        tracing::info!(
827            node_id = %node.node_id,
828            connection_id,
829            incarnation_id = ?incarnation_id,
830            "node attached to relay",
831        );
832        let mut controller_owned = hello.controller.as_ref().is_some_and(|controller| controller.connection_id == connection_id);
833        let mut cursor = NodeCursor { incarnation_id, sequence: hello.event_sequence };
834        let mut snapshot = hello.snapshot;
835        let mut gaps = Vec::new();
836        let mut did_resync = false;
837        if let Some(previous) = previous {
838            if previous.incarnation_id != incarnation_id {
839                gaps.push(GapKind::IncarnationChanged);
840            } else if cursor.sequence < previous.sequence {
841                gaps.push(GapKind::CursorRegression);
842            } else if cursor.sequence > previous.sequence {
843                match bounded_node_request(&mut client, NodeRequest::Resync { after_sequence: previous.sequence }).await {
844                    Ok(NodeResponse::Resync {
845                        event_sequence,
846                        oldest_available_sequence,
847                        snapshot: resync_snapshot,
848                        events,
849                    }) => {
850                        let resync_gaps = validate_resync(
851                            previous.sequence,
852                            cursor.sequence,
853                            event_sequence,
854                            oldest_available_sequence,
855                            &events,
856                        );
857                        if resync_gaps.is_empty() {
858                            publish_recovered_events(&node.node_id, incarnation_id, &events, &hub);
859                        } else {
860                            hub.publish(RoutedNodeEvent {
861                                node_id: node.node_id.clone(),
862                                cursor: NodeCursor { incarnation_id, sequence: event_sequence },
863                                event: C2NodeEvent::ResyncRequired {
864                                    oldest_available_sequence,
865                                },
866                            });
867                        }
868                        gaps.extend(resync_gaps);
869                        cursor.sequence = event_sequence;
870                        snapshot = resync_snapshot;
871                        did_resync = true;
872                    }
873                    Ok(_) => gaps.push(GapKind::NonContiguousEvents),
874                    Err(error) => {
875                        ingress_attempt(&ingress, &node.node_id, relay_failure_attempt(&error)).await?;
876                        reject_disconnected_commands(&mut commands, Some(cursor));
877                        continue;
878                    }
879                }
880            }
881        }
882        ingress_attempt(&ingress, &node.node_id, AttemptResult::Connected {
883            cursor,
884            snapshot,
885            gaps,
886            provider_contract_manifest,
887        }).await?;
888        if let Err(error) = drain_pending_events(&mut client, &node.node_id, &mut cursor, &hub, &ingress, did_resync, None).await {
889            ingress_attempt(&ingress, &node.node_id, relay_failure_attempt(&error)).await?;
890            reject_disconnected_commands(&mut commands, Some(cursor));
891            continue;
892        }
893
894        let cadence = timings.poll_interval.max(Duration::from_millis(1)).min(Duration::from_millis(250));
895        let mut snapshot_tick = tokio::time::interval(cadence);
896        snapshot_tick.reset();
897        let mut lease_tick = tokio::time::interval(Duration::from_secs(30));
898        lease_tick.reset();
899        let disconnect_error = loop {
900            tokio::select! {
901                command = commands.recv() => {
902                    let Some(command) = command else { return Ok(()); };
903                    tokio::select! {
904                        result = handle_relay_command(
905                            &mut client, command, &node.node_id, incarnation_id, connection_id,
906                            &mut controller_owned, &hub, &mut cursor, &ingress,
907                        ) => match result {
908                            Ok(()) => {}
909                            Err(error) => break error,
910                        },
911                        changed = force_disconnect.changed() => {
912                            let _ = changed;
913                            break NodeClientError::Io(io::Error::new(io::ErrorKind::ConnectionAborted, "node relay cleanup forced reconnect"));
914                        }
915                    }
916                }
917                release = releases.recv() => {
918                    let Some(reply) = release else { return Ok(()); };
919                    let result = release_controller(&mut client, &mut controller_owned).await;
920                    let _ = reply.send(());
921                    if let Err(error) = result { break error; }
922                }
923                changed = force_disconnect.changed() => {
924                    let _ = changed;
925                    break NodeClientError::Io(io::Error::new(io::ErrorKind::ConnectionAborted, "node relay cleanup forced reconnect"));
926                }
927                frame = client.recv() => {
928                    match frame {
929                        Ok(ServerFrame::Event(envelope)) => {
930                            if let Err(error) = handle_live_node_event(
931                                &mut client,
932                                &node.node_id,
933                                envelope,
934                                &mut cursor,
935                                &hub,
936                                &ingress,
937                            ).await {
938                                break error;
939                            }
940                        }
941                        Ok(ServerFrame::Reply(_)
942                            | ServerFrame::Challenge(_)
943                            | ServerFrame::Hello(_)) => {
944                            break NodeClientError::Protocol(
945                                "node sent an unexpected idle frame".to_owned(),
946                            );
947                        }
948                        Err(error) => break error,
949                    }
950                }
951                _ = snapshot_tick.tick() => {
952                    match bounded_node_request(&mut client, NodeRequest::Snapshot).await {
953                        Ok(NodeResponse::Snapshot { event_sequence, snapshot, .. }) => {
954                            if let Err(error) = drain_pending_events(&mut client, &node.node_id, &mut cursor, &hub, &ingress, false, None).await { break error; }
955                            if event_sequence >= cursor.sequence {
956                                cursor.sequence = event_sequence;
957                                ingress_attempt(&ingress, &node.node_id, AttemptResult::Success { cursor, snapshot, gaps: Vec::new() }).await?;
958                            }
959                        }
960                        Ok(_) => break NodeClientError::Protocol("snapshot returned a different response".to_owned()),
961                        Err(error) => break error,
962                    }
963                }
964                _ = lease_tick.tick() => {
965                    if controller_owned {
966                        if hub.has_active_operator() {
967                            match acquire_controller(&mut client, connection_id).await {
968                                Ok(owned) => controller_owned = owned,
969                                Err(error) => break error,
970                            }
971                        } else if let Err(error) = release_controller(&mut client, &mut controller_owned).await {
972                            break error;
973                        }
974                        if let Err(error) = drain_pending_events(&mut client, &node.node_id, &mut cursor, &hub, &ingress, false, None).await { break error; }
975                    }
976                }
977                changed = shutdown.changed() => {
978                    if changed.is_err() || *shutdown.borrow() {
979                        let _ = release_controller(&mut client, &mut controller_owned).await;
980                        return Ok(());
981                    }
982                }
983            }
984        };
985        let (error, hard) = sanitize_node_error(&disconnect_error);
986        tracing::warn!(
987            node_id = %node.node_id,
988            connection_id,
989            cause = %error.message,
990            category = ?error.category,
991            hard,
992            "node dropped from relay",
993        );
994        ingress_attempt(&ingress, &node.node_id, AttemptResult::Failure { error, hard }).await?;
995        reject_disconnected_commands(&mut commands, Some(cursor));
996        failures = failures.saturating_add(1);
997        relay_backoff(&mut shutdown, timings, failures, hard).await?;
998    }
999}
1000
1001/// Produces an authenticated operator client for `node`, whichever end
1002/// opened the socket.
1003///
1004/// All three arms end in the same handshake. The first two open a
1005/// connection and hand it straight to it; the third is handed one that a
1006/// node opened and the listener already matched to this node's name. That
1007/// symmetry is the point of the change: `LocalNodeClient::adopt` is what
1008/// `connect`/`connect_loopback` both do after their own `connect` call, so
1009/// a call-home node is authenticated by exactly the same code, against
1010/// exactly the same per-node token, as one this relay dialled.
1011async fn connect_operator(
1012    node: &C2NodeConfig,
1013    adopted: Option<TcpStream>,
1014) -> Result<LocalNodeClient, NodeClientError> {
1015    match node.route {
1016        C2NodeRoute::Local => {
1017            LocalNodeClient::connect(&node.endpoint, &node.node_id, ClientRole::Operator, &node.token).await
1018        }
1019        C2NodeRoute::SshForwardedLoopback(endpoint) => {
1020            LocalNodeClient::connect_loopback(endpoint, &node.node_id, ClientRole::Operator, &node.token).await
1021        }
1022        C2NodeRoute::CallHome => {
1023            // `node_relay_worker` collects the socket before calling this
1024            // and returns rather than calling without one, so `None` here
1025            // is a caller bug, not a runtime condition -- reported instead
1026            // of panicking because a relay worker is not worth aborting a
1027            // whole C2 over.
1028            let stream = adopted.ok_or_else(|| {
1029                NodeClientError::Protocol(
1030                    "call-home node reached the connect step with no adopted stream".to_owned(),
1031                )
1032            })?;
1033            LocalNodeClient::adopt(stream, &node.node_id, ClientRole::Operator, &node.token).await
1034        }
1035    }
1036}
1037
1038async fn bounded_node_request(
1039    client: &mut LocalNodeClient,
1040    request: NodeRequest,
1041) -> Result<NodeResponse, NodeClientError> {
1042    bounded_node_request_with_deadline(client, request, None).await
1043}
1044
1045async fn bounded_node_request_with_deadline(
1046    client: &mut LocalNodeClient,
1047    request: NodeRequest,
1048    relay_deadline: Option<Instant>,
1049) -> Result<NodeResponse, NodeClientError> {
1050    let deadline = request_budget(&request, relay_deadline, Instant::now());
1051    if deadline.is_zero() {
1052        return Err(NodeClientError::Frame(FrameError::PrefixTimedOut));
1053    }
1054    match timeout(deadline, client.request(request)).await {
1055        Ok(result) => result,
1056        Err(_) => Err(NodeClientError::Frame(FrameError::PrefixTimedOut)),
1057    }
1058}
1059
1060fn relay_request_deadline(request: &NodeRequest, now: Instant) -> Option<Instant> {
1061    matches!(request, NodeRequest::ResumeSessionRecord { .. })
1062        .then(|| now + node_request_deadline(request))
1063}
1064
1065fn request_budget(request: &NodeRequest, relay_deadline: Option<Instant>, now: Instant) -> Duration {
1066    relay_deadline
1067        .map(|deadline| deadline.saturating_duration_since(now))
1068        .unwrap_or_else(|| node_request_deadline(request))
1069}
1070
1071/// Relay bound for a workspace inspection. The node's own inspection
1072/// budget is 8s by default and 11s at most; this must clear that maximum
1073/// plus the round trip, or the relay kills a request the node was still
1074/// legitimately working on -- and killing it drops the node, not just the
1075/// request. See `node_request_deadline`'s own `InspectWorkspace` arm.
1076const WORKSPACE_INSPECTION_RELAY_DEADLINE: Duration = Duration::from_secs(15);
1077
1078fn node_request_deadline(request: &NodeRequest) -> Duration {
1079    match request {
1080        NodeRequest::Snapshot
1081        | NodeRequest::Resync { .. }
1082        | NodeRequest::ArmHarnessMcpReservation { .. }
1083        | NodeRequest::ActivateHarnessMcpReservation { .. }
1084        | NodeRequest::AbortHarnessMcpReservation { .. }
1085        | NodeRequest::PutHarnessMcpReplyChunk { .. }
1086        | NodeRequest::RejectHarnessMcpCall { .. }
1087        | NodeRequest::BrowseHostDirectories { .. }
1088        | NodeRequest::ReadWorkspaceFile { .. }
1089        | NodeRequest::WriteWorkspaceFile { .. }
1090        | NodeRequest::ReadGitHistory { .. }
1091        | NodeRequest::ReadGitDiff { .. }
1092        | NodeRequest::AcquireController { .. }
1093        | NodeRequest::ReleaseController
1094        | NodeRequest::RenameSessionRecord { .. }
1095        | NodeRequest::SetSessionTask { .. }
1096        | NodeRequest::ForgetSessionRecord { .. } => Duration::from_secs(5),
1097        // The node gives its own workspace inspection 8s by default and up
1098        // to 11s, and says so when it uses them
1099        // (`git_time_budget_exceeded`, elapsed over eight seconds on a
1100        // real repository here). Bounding it from out here at five made
1101        // that inner budget unreachable: any workspace big enough to spend
1102        // its own allowance timed out at the relay every single time.
1103        //
1104        // And a relay timeout is not a slow answer, it is a dead node --
1105        // the node is dropped from the relay, its controller lease is
1106        // released, and every read behind it starts answering
1107        // "unavailable" until it reattaches, which it then does, and the
1108        // cycle repeats. One number two seconds too small took the whole
1109        // stack down in a loop: no inventory, so a spawned session could
1110        // never be shown, while its process ran perfectly well.
1111        //
1112        // The outer bound must therefore clear the node's own MAXIMUM, not
1113        // its default, with room for the round trip on top.
1114        NodeRequest::InspectWorkspace { .. } => WORKSPACE_INSPECTION_RELAY_DEADLINE,
1115        NodeRequest::CreateWorkspaceFile { .. }
1116        | NodeRequest::CreateWorkspaceDirectory { .. } => {
1117            WORKSPACE_ENTRY_CREATE_RELAY_DEADLINE
1118        }
1119        NodeRequest::CatalogNativeSessions { .. }
1120        | NodeRequest::PageNativeSessions { .. }
1121        | NodeRequest::PreviewNativeSession { .. }
1122        | NodeRequest::IndexNativeSession { .. }
1123        | NodeRequest::PreviewSessionRecord { .. } => NATIVE_SESSION_REQUEST_DEADLINE,
1124        NodeRequest::CreateStandaloneWorkspace { .. }
1125        | NodeRequest::CreateWorktree { .. }
1126        | NodeRequest::RemoveWorktree { .. }
1127        | NodeRequest::CleanupManagedWorktree { .. } => Duration::from_secs(240),
1128        NodeRequest::Spawn { .. }
1129        | NodeRequest::Resume { .. }
1130        | NodeRequest::Stop { .. } => Duration::from_secs(15),
1131        NodeRequest::SpawnSpec { spec } =>
1132            Duration::from_millis(spec.deadline_ms.get()) + NODE_REQUEST_IO_HEADROOM,
1133        NodeRequest::SpawnSpecWithHarnessMcp { deadline_unix_ms, .. } =>
1134            Duration::from_millis(deadline_unix_ms.saturating_sub(unix_ms()))
1135                + NODE_REQUEST_IO_HEADROOM,
1136        NodeRequest::SpawnManagedWorktree { request } =>
1137            Duration::from_millis(request.spawn_spec.deadline_ms.get())
1138                + NODE_REQUEST_IO_HEADROOM,
1139        NodeRequest::ResumeSessionRecord { .. } =>
1140            MANAGED_RESUME_SETTLE_DEADLINE + NODE_REQUEST_IO_HEADROOM,
1141        _ => Duration::from_secs(10),
1142    }
1143}
1144
1145fn validate_resync(
1146    previous: u64,
1147    hello: u64,
1148    current: u64,
1149    oldest_available_sequence: u64,
1150    events: &[NodeEventEnvelope],
1151) -> Vec<GapKind> {
1152    let mut gaps = validate_events(
1153        previous,
1154        current,
1155        oldest_available_sequence,
1156        events,
1157    );
1158    if current < hello && !gaps.contains(&GapKind::CursorRegression) {
1159        gaps.push(GapKind::CursorRegression);
1160    }
1161    gaps
1162}
1163
1164async fn ingress_attempt(
1165    ingress: &mpsc::Sender<Attempt>,
1166    node_id: &NodeId,
1167    result: AttemptResult,
1168) -> io::Result<()> {
1169    ingress.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result })
1170        .await.map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "inventory owner closed"))
1171}
1172
1173async fn relay_backoff(
1174    shutdown: &mut watch::Receiver<bool>,
1175    timings: C2Timings,
1176    failures: usize,
1177    hard: bool,
1178) -> io::Result<()> {
1179    let delay = if hard || failures >= timings.transient_backoffs.len() {
1180        timings.parked_backoff
1181    } else {
1182        timings.transient_backoffs[failures.saturating_sub(1)]
1183    };
1184    tokio::select! {
1185        _ = sleep(delay) => Ok(()),
1186        _ = shutdown.changed() => {
1187            Ok(())
1188        }
1189    }
1190}
1191
1192fn relay_failure_attempt(error: &NodeClientError) -> AttemptResult {
1193    let (error, hard) = sanitize_node_error(error);
1194    AttemptResult::Failure { error, hard }
1195}
1196
1197async fn handle_relay_command(
1198    client: &mut LocalNodeClient,
1199    command: RelayCommand,
1200    node_id: &NodeId,
1201    incarnation_id: NodeIncarnationId,
1202    connection_id: u64,
1203    controller_owned: &mut bool,
1204    hub: &OperatorHub,
1205    cursor: &mut NodeCursor,
1206    ingress: &mpsc::Sender<Attempt>,
1207) -> Result<(), NodeClientError> {
1208    match command {
1209        RelayCommand::Request { operator_connection_id, expected_incarnation_id, request, reply } => {
1210            let relay_deadline = relay_request_deadline(&request, Instant::now());
1211            let expected_spawn = expected_spawn_request(&request);
1212            let expected_provider_session_index = matches!(
1213                &request,
1214                NodeRequest::IndexProviderSession { .. }
1215            ).then(|| request.clone());
1216            let expected_native_session = matches!(
1217                &request,
1218                NodeRequest::CatalogNativeSessions { .. }
1219                    | NodeRequest::PageNativeSessions { .. }
1220                    | NodeRequest::PreviewNativeSession { .. }
1221                    | NodeRequest::IndexNativeSession { .. }
1222            ).then(|| request.clone());
1223            let expected_workspace_content = matches!(
1224                &request,
1225                NodeRequest::ReadWorkspaceFile { .. }
1226                    | NodeRequest::WriteWorkspaceFile { .. }
1227                    | NodeRequest::CreateWorkspaceFile { .. }
1228                    | NodeRequest::CreateWorkspaceDirectory { .. }
1229                    | NodeRequest::ReadGitHistory { .. }
1230                    | NodeRequest::ReadGitDiff { .. }
1231            ).then(|| request.clone());
1232            let expected_session_task = matches!(
1233                &request,
1234                NodeRequest::SetSessionTask { .. }
1235            ).then(|| request.clone());
1236            let expected_harness_mcp = (request.required_capability()
1237                == Some(gate4agent_node_protocol::NODE_HARNESS_MCP_READ_PROXY_CAPABILITY))
1238                .then(|| request.clone());
1239            if !hub.is_active(operator_connection_id) {
1240                let _ = reply.send(Err(relay_failure(C2RelayFailureCode::ClientLagged, "C2 operator connection is no longer active", Some(incarnation_id))));
1241                return Ok(());
1242            }
1243            if expected_incarnation_id != incarnation_id {
1244                let _ = reply.send(Err(relay_failure(C2RelayFailureCode::StaleNodeIncarnation, "node incarnation changed", Some(incarnation_id))));
1245                return Ok(());
1246            }
1247            if matches!(&request, NodeRequest::IndexNativeSession { selection, .. }
1248                if selection.route.scope
1249                    != gate4agent_node_protocol::NativeSessionCatalogScope::Workspace)
1250            {
1251                let _ = reply.send(Err(relay_failure(
1252                    C2RelayFailureCode::RequestForbidden,
1253                    "external native sessions must be registered as workspaces before indexing",
1254                    Some(incarnation_id),
1255                )));
1256                return Ok(());
1257            }
1258            if !is_read_only_request(&request) && !*controller_owned {
1259                match acquire_controller_with_deadline(client, connection_id, relay_deadline).await {
1260                    Ok(owned) if owned => *controller_owned = true,
1261                    Ok(_) => {
1262                        let _ = reply.send(Err(relay_failure(C2RelayFailureCode::RelayBusy, "node controller lease is unavailable", Some(incarnation_id))));
1263                        return Ok(());
1264                    }
1265                    Err(error) if relay_node_failure(&error).is_some() => {
1266                        let failure = relay_node_failure(&error)
1267                            .expect("guarded node request failure");
1268                        let _ = reply.send(Ok(RoutedNodeResponse {
1269                            node_id: node_id.clone(),
1270                            incarnation_id,
1271                            response: Err(failure),
1272                        }));
1273                        return Ok(());
1274                    }
1275                    Err(error) => {
1276                        let _ = reply.send(Err(relay_failure(C2RelayFailureCode::NodeOffline, "node relay disconnected", Some(incarnation_id))));
1277                        return Err(error);
1278                    }
1279                }
1280            }
1281            let response = match bounded_node_request_with_deadline(client, request, relay_deadline).await {
1282                Ok(response) => Ok(response),
1283                Err(NodeClientError::Node(failure)) => Err(failure),
1284                Err(error @ NodeClientError::UnsupportedCapability(_)) => {
1285                    let failure = relay_node_failure(&error)
1286                        .expect("unsupported capability is a routed node failure");
1287                    let _ = reply.send(Ok(RoutedNodeResponse {
1288                        node_id: node_id.clone(),
1289                        incarnation_id,
1290                        response: Err(failure),
1291                    }));
1292                    return Ok(());
1293                }
1294                Err(error) => {
1295                    let _ = reply.send(Err(relay_failure(C2RelayFailureCode::NodeOffline, "node relay disconnected", Some(incarnation_id))));
1296                    return Err(error);
1297                }
1298            };
1299            if let Err(message) = validate_spawn_spec_response(
1300                expected_spawn.as_ref(),
1301                &response,
1302                incarnation_id,
1303            ) {
1304                let _ = reply.send(Err(relay_failure(
1305                    C2RelayFailureCode::NodeOffline,
1306                    "node relay returned an invalid spawn receipt",
1307                    Some(incarnation_id),
1308                )));
1309                return Err(NodeClientError::Protocol(message.to_owned()));
1310            }
1311            if let Err(message) = validate_provider_session_index_response(
1312                expected_provider_session_index.as_ref(),
1313                &response,
1314            ) {
1315                let _ = reply.send(Err(relay_failure(
1316                    C2RelayFailureCode::NodeOffline,
1317                    "node relay returned an invalid provider session index response",
1318                    Some(incarnation_id),
1319                )));
1320                return Err(NodeClientError::Protocol(message.to_owned()));
1321            }
1322            if let Err(message) = validate_native_session_response(
1323                expected_native_session.as_ref(),
1324                &response,
1325            ) {
1326                let _ = reply.send(Err(relay_failure(
1327                    C2RelayFailureCode::NodeOffline,
1328                    "node relay returned an invalid native session response",
1329                    Some(incarnation_id),
1330                )));
1331                return Err(NodeClientError::Protocol(message.to_owned()));
1332            }
1333            if let Err(message) = validate_workspace_content_response(
1334                expected_workspace_content.as_ref(),
1335                &response,
1336            ) {
1337                let _ = reply.send(Err(relay_failure(
1338                    C2RelayFailureCode::NodeOffline,
1339                    "node relay returned an invalid workspace content response",
1340                    Some(incarnation_id),
1341                )));
1342                return Err(NodeClientError::Protocol(message.to_owned()));
1343            }
1344            if let Err(message) = validate_session_task_response(
1345                expected_session_task.as_ref(),
1346                &response,
1347            ) {
1348                let _ = reply.send(Err(relay_failure(
1349                    C2RelayFailureCode::NodeOffline,
1350                    "node relay returned an invalid session task response",
1351                    Some(incarnation_id),
1352                )));
1353                return Err(NodeClientError::Protocol(message.to_owned()));
1354            }
1355            if let Err(message) = validate_harness_mcp_response(
1356                expected_harness_mcp.as_ref(),
1357                &response,
1358            ) {
1359                let _ = reply.send(Err(relay_failure(
1360                    C2RelayFailureCode::NodeOffline,
1361                    "node relay returned an invalid harness MCP response",
1362                    Some(incarnation_id),
1363                )));
1364                return Err(NodeClientError::Protocol(message.to_owned()));
1365            }
1366            drain_pending_events(client, node_id, cursor, hub, ingress, false, relay_deadline).await?;
1367            update_inventory_from_response(node_id, cursor, &response, ingress).await
1368                .map_err(NodeClientError::Io)?;
1369            let response = response
1370                .map(|response| C2NodeResponse::from_node_response_with_control_detail(&response))
1371                .map_err(|failure| C2NodeFailure::from(&failure));
1372            let _ = reply.send(Ok(RoutedNodeResponse { node_id: node_id.clone(), incarnation_id, response }));
1373        }
1374    }
1375    Ok(())
1376}
1377
1378fn validate_session_task_response(
1379    expected: Option<&NodeRequest>,
1380    response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1381) -> Result<(), &'static str> {
1382    match (expected, response) {
1383        (Some(_), Err(_)) | (None, Err(_)) => Ok(()),
1384        (
1385            Some(NodeRequest::SetSessionTask {
1386                record_id,
1387                expected_revision,
1388                target,
1389            }),
1390            Ok(NodeResponse::SessionRecordUpdated { record }),
1391        ) if session_task_record_matches(record, record_id, *expected_revision, target) => Ok(()),
1392        (Some(NodeRequest::SetSessionTask { .. }), Ok(_)) => {
1393            Err("session task response does not match the routed request")
1394        }
1395        (None, Ok(_)) => Ok(()),
1396        (Some(_), Ok(_)) => Ok(()),
1397    }
1398}
1399
1400fn validate_harness_mcp_response(
1401    expected: Option<&NodeRequest>,
1402    response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1403) -> Result<(), &'static str> {
1404    use NodeRequest as Request;
1405    use NodeResponse as Response;
1406    let valid = match (expected, response) {
1407        (Some(Request::ArmHarnessMcpReservation { reservation_id, activation_digest, expires_at_unix_ms, .. }),
1408            Ok(Response::Armed { reservation_id: echoed_id, activation_digest: echoed_digest, expires_at_unix_ms: echoed_expiry })) =>
1409            reservation_id == echoed_id && activation_digest == echoed_digest
1410                && expires_at_unix_ms == echoed_expiry,
1411        (Some(Request::SpawnSpecWithHarnessMcp { reservation_id, activation_digest, .. }),
1412            Ok(Response::Spawned { reservation_id: echoed_id, activation_digest: echoed_digest, receipt })) =>
1413            reservation_id == echoed_id && activation_digest == echoed_digest
1414                && receipt.harness_mcp_proxy.as_ref().is_some_and(|proxy| {
1415                    &proxy.reservation_id == reservation_id
1416                        && &proxy.activation_digest == activation_digest
1417                }),
1418        (Some(Request::ActivateHarnessMcpReservation { reservation_id, activation_digest, record_id, session }),
1419            Ok(Response::Activated { reservation_id: echoed_id, activation_digest: echoed_digest, record_id: echoed_record, session: echoed_session })) =>
1420            reservation_id == echoed_id && activation_digest == echoed_digest
1421                && record_id == echoed_record && session == echoed_session,
1422        (Some(Request::AbortHarnessMcpReservation { reservation_id, activation_digest }),
1423            Ok(Response::Aborted { reservation_id: echoed_id, activation_digest: echoed_digest })) =>
1424            reservation_id == echoed_id && activation_digest == echoed_digest,
1425        (Some(Request::PutHarnessMcpReplyChunk { reservation_id, activation_digest, record_id, session, call_id, offset, final_chunk, chunk_hex }),
1426            Ok(Response::ReplyChunkAccepted { reservation_id: echoed_id, activation_digest: echoed_digest, record_id: echoed_record, session: echoed_session, call_id: echoed_call, next_offset, completed })) =>
1427            reservation_id == echoed_id && activation_digest == echoed_digest
1428                && record_id == echoed_record && session == echoed_session && call_id == echoed_call
1429                && offset.checked_add(u32::try_from(chunk_hex.raw_len()).unwrap_or(u32::MAX))
1430                    == Some(*next_offset) && completed == final_chunk,
1431        (Some(Request::RejectHarnessMcpCall { reservation_id, activation_digest, record_id, session, call_id, .. }),
1432            Ok(Response::CallRejected { reservation_id: echoed_id, activation_digest: echoed_digest, record_id: echoed_record, session: echoed_session, call_id: echoed_call })) =>
1433            reservation_id == echoed_id && activation_digest == echoed_digest
1434                && record_id == echoed_record && session == echoed_session && call_id == echoed_call,
1435        (Some(_), Err(_)) | (None, Err(_)) => true,
1436        (Some(_), Ok(_)) => false,
1437        (None, Ok(response)) => !response.requires_harness_mcp_proxy_capability(),
1438    };
1439    if valid { Ok(()) } else { Err("harness MCP response does not match the routed request") }
1440}
1441
1442fn session_task_record_matches(
1443    record: &gate4agent_node_protocol::ManagedSessionRecord,
1444    record_id: &gate4agent_node_protocol::SessionRecordId,
1445    expected_revision: u64,
1446    target: &gate4agent_node_protocol::SessionTaskTargetV1,
1447) -> bool {
1448    if &record.record_id != record_id { return false; }
1449    let next_revision = expected_revision.checked_add(1);
1450    match target {
1451        gate4agent_node_protocol::SessionTaskTargetV1::New => record.task_binding.as_ref()
1452            .is_some_and(|binding| Some(binding.revision) == next_revision && binding.task_id.is_some()),
1453        gate4agent_node_protocol::SessionTaskTargetV1::Existing { task_id } => record.task_binding.as_ref()
1454            .is_some_and(|binding| (binding.revision == expected_revision || Some(binding.revision) == next_revision)
1455                && binding.task_id.as_ref() == Some(task_id)),
1456        gate4agent_node_protocol::SessionTaskTargetV1::Clear => match &record.task_binding {
1457            None => expected_revision == 0,
1458            Some(binding) => binding.task_id.is_none()
1459                && (binding.revision == expected_revision || Some(binding.revision) == next_revision),
1460        },
1461    }
1462}
1463
1464fn validate_workspace_content_response(
1465    expected: Option<&NodeRequest>,
1466    response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1467) -> Result<(), &'static str> {
1468    match (expected, response) {
1469        (Some(_), Err(_)) | (None, Err(_)) => Ok(()),
1470        (
1471            Some(NodeRequest::ReadWorkspaceFile { workspace_id, path }),
1472            Ok(NodeResponse::WorkspaceFileRead { file }),
1473        ) if &file.workspace_id == workspace_id && &file.path == path => Ok(()),
1474        (
1475            Some(NodeRequest::WriteWorkspaceFile { workspace_id, path, text, .. }),
1476            Ok(NodeResponse::WorkspaceFileWritten { file }),
1477        ) if &file.workspace_id == workspace_id
1478            && &file.path == path
1479            && matches!(
1480                &file.content,
1481                gate4agent_node_protocol::WorkspaceFileContent::Utf8 {
1482                    text: written,
1483                    byte_len,
1484                } if written == text
1485                    && u32::try_from(text.len()).ok() == Some(*byte_len)
1486            ) => Ok(()),
1487        (
1488            Some(NodeRequest::CreateWorkspaceFile { workspace_id, path }),
1489            Ok(NodeResponse::WorkspaceFileCreated { file }),
1490        ) if &file.workspace_id == workspace_id
1491            && &file.path == path
1492            && file.revision.is_some()
1493            && matches!(
1494                &file.content,
1495                gate4agent_node_protocol::WorkspaceFileContent::Utf8 {
1496                    text,
1497                    byte_len: 0,
1498                } if text.is_empty()
1499            ) => Ok(()),
1500        (
1501            Some(NodeRequest::CreateWorkspaceDirectory { workspace_id, path }),
1502            Ok(NodeResponse::WorkspaceDirectoryCreated {
1503                workspace_id: actual_workspace_id,
1504                entry,
1505            }),
1506        ) if actual_workspace_id == workspace_id
1507            && &entry.relative_path == path
1508            && entry.kind == gate4agent_node_protocol::WorkspaceEntryKind::Directory => Ok(()),
1509        (
1510            Some(NodeRequest::ReadGitHistory { workspace_id, .. }),
1511            Ok(NodeResponse::GitHistoryRead { workspace_id: actual, .. }),
1512        ) if actual == workspace_id => Ok(()),
1513        (
1514            Some(NodeRequest::ReadGitDiff { workspace_id, request }),
1515            Ok(NodeResponse::GitDiffRead { workspace_id: actual, diff }),
1516        ) if actual == workspace_id && diff.mode == request.mode && diff.path == request.path => Ok(()),
1517        (Some(_), Ok(_)) => Err("workspace content response does not match routed request"),
1518        (
1519            None,
1520            Ok(NodeResponse::WorkspaceFileRead { .. }
1521                | NodeResponse::WorkspaceFileWritten { .. }
1522                | NodeResponse::WorkspaceFileCreated { .. }
1523                | NodeResponse::WorkspaceDirectoryCreated { .. }
1524                | NodeResponse::GitHistoryRead { .. }
1525                | NodeResponse::GitDiffRead { .. }),
1526        ) => Err("unexpected workspace content response"),
1527        (None, Ok(_)) => Ok(()),
1528    }
1529}
1530
1531enum ExpectedSpawnRequest {
1532    Spec(SpawnSpec),
1533    ManagedV1(ManagedWorktreeSpawnRequest),
1534    ManagedV2(ManagedWorktreeSpawnRequestV2),
1535}
1536
1537fn expected_spawn_request(request: &NodeRequest) -> Option<ExpectedSpawnRequest> {
1538    match request {
1539        NodeRequest::SpawnSpec { spec } => Some(ExpectedSpawnRequest::Spec(spec.clone())),
1540        NodeRequest::SpawnManagedWorktree { request } => {
1541            Some(ExpectedSpawnRequest::ManagedV1(request.clone()))
1542        }
1543        NodeRequest::SpawnManagedWorktreeV2 { request } => {
1544            Some(ExpectedSpawnRequest::ManagedV2(request.clone()))
1545        }
1546        _ => None,
1547    }
1548}
1549
1550fn validate_provider_session_index_response(
1551    expected: Option<&NodeRequest>,
1552    response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1553) -> Result<(), &'static str> {
1554    match (expected, response) {
1555        (
1556            Some(NodeRequest::IndexProviderSession {
1557                workspace_id,
1558                provider,
1559                identity,
1560                ..
1561            }),
1562            Ok(NodeResponse::ProviderSessionIndexed { record }),
1563        ) if &record.workspace_id == workspace_id
1564            && &record.provider == provider
1565            && record.provider_session.as_ref() == Some(identity) => Ok(()),
1566        (Some(_), Ok(NodeResponse::ProviderSessionIndexed { .. })) => {
1567            Err("provider session index response does not match routed request")
1568        }
1569        (Some(_), Ok(_)) => Err("provider session index request returned a different response"),
1570        (None, Ok(NodeResponse::ProviderSessionIndexed { .. })) => {
1571            Err("unexpected provider session index response for a different node request")
1572        }
1573        (Some(_), Err(_)) | (None, _) => Ok(()),
1574    }
1575}
1576
1577fn validate_native_session_response(
1578    expected: Option<&NodeRequest>,
1579    response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1580) -> Result<(), &'static str> {
1581    match (expected, response) {
1582        (
1583            Some(NodeRequest::CatalogNativeSessions { route, .. }),
1584            Ok(NodeResponse::NativeSessionsCataloged {
1585                route: echoed_route,
1586                ..
1587            }),
1588        ) if echoed_route == route => Ok(()),
1589        (
1590            Some(NodeRequest::PageNativeSessions {
1591                route,
1592                window,
1593                catalog_revision,
1594                ..
1595            }),
1596            Ok(NodeResponse::NativeSessionsPaged {
1597                route: echoed_route,
1598                page,
1599            }),
1600        ) if echoed_route == route
1601            && page.window == *window
1602            && page.revision == *catalog_revision => Ok(()),
1603        (
1604            Some(NodeRequest::PreviewNativeSession { selection, .. }),
1605            Ok(NodeResponse::NativeSessionPreviewed {
1606                selection: echoed_selection,
1607                ..
1608            }),
1609        ) if echoed_selection == selection => Ok(()),
1610        (
1611            Some(NodeRequest::IndexNativeSession { selection, .. }),
1612            Ok(NodeResponse::NativeSessionIndexed {
1613                selection: echoed_selection,
1614                record,
1615            }),
1616        ) if echoed_selection == selection
1617            && selection.route.scope
1618                == gate4agent_node_protocol::NativeSessionCatalogScope::Workspace
1619            && selection.route.workspace_id.as_ref() == Some(&record.workspace_id)
1620            && selection.route.provider == record.provider => Ok(()),
1621        (Some(_), Err(_)) => Ok(()),
1622        (Some(_), Ok(_)) => Err("native session response does not match routed request"),
1623        (
1624            None,
1625            Ok(
1626                NodeResponse::NativeSessionsCataloged { .. }
1627                | NodeResponse::NativeSessionsPaged { .. }
1628                | NodeResponse::NativeSessionPreviewed { .. }
1629                | NodeResponse::NativeSessionIndexed { .. },
1630            ),
1631        ) => Err("unexpected native session response for a different node request"),
1632        (None, _) => Ok(()),
1633    }
1634}
1635
1636fn validate_spawn_spec_response(
1637    expected: Option<&ExpectedSpawnRequest>,
1638    response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1639    relay_incarnation_id: NodeIncarnationId,
1640) -> Result<(), &'static str> {
1641    match (expected, response) {
1642        (Some(ExpectedSpawnRequest::Spec(spec)), Ok(NodeResponse::SpawnSpecAccepted { receipt })) => {
1643            validate_spawn_receipt(spec, receipt, relay_incarnation_id)
1644        }
1645        (
1646            Some(ExpectedSpawnRequest::ManagedV1(request)),
1647            Ok(NodeResponse::ManagedWorktreeSpawnAccepted { receipt }),
1648        ) => validate_managed_spawn_receipt(
1649            &request.spawn_spec,
1650            &request.worktree_profile_id,
1651            receipt,
1652            relay_incarnation_id,
1653        ),
1654        (
1655            Some(ExpectedSpawnRequest::ManagedV2(request)),
1656            Ok(NodeResponse::ManagedWorktreeSpawnAccepted { receipt }),
1657        ) if receipt.lease.profile_revision == request.expected_profile_revision => {
1658            validate_managed_spawn_receipt(
1659                &request.spawn_spec,
1660                &request.worktree_profile_id,
1661                receipt,
1662                relay_incarnation_id,
1663            )
1664        }
1665        (
1666            Some(ExpectedSpawnRequest::ManagedV2(_)),
1667            Ok(NodeResponse::ManagedWorktreeSpawnAccepted { .. }),
1668        ) => Err("managed spawn receipt profile revision does not match routed request"),
1669        (Some(_), Ok(_)) => return Err("spawn spec request returned a different response"),
1670        (Some(_), Err(_)) => return Ok(()),
1671        (None, Ok(NodeResponse::SpawnSpecAccepted { .. }
1672            | NodeResponse::ManagedWorktreeSpawnAccepted { .. })) => {
1673            return Err("unexpected spawn receipt for a different node request");
1674        }
1675        (None, _) => return Ok(()),
1676    }
1677}
1678
1679fn validate_managed_spawn_receipt(
1680    spawn_spec: &SpawnSpec,
1681    worktree_profile_id: &gate4agent_node_protocol::WorktreeProfileId,
1682    receipt: &gate4agent_node_protocol::ManagedWorktreeSpawnReceipt,
1683    relay_incarnation_id: NodeIncarnationId,
1684) -> Result<(), &'static str> {
1685    if receipt.lease.source_workspace_id != spawn_spec.target.workspace_id
1686        || &receipt.lease.profile_id != worktree_profile_id
1687        || receipt.lease.state != ManagedWorktreeLeaseState::InUse
1688        || receipt.lease.cleanup_failure.is_some()
1689        || receipt.lease.active_session_count != 1
1690        || receipt.spawn.target.node_id != spawn_spec.target.node_id
1691        || receipt.spawn.target.workspace_id != spawn_spec.target.workspace_id
1692        || receipt.spawn.target.worktree_id.as_ref() != Some(&receipt.lease.workspace_id)
1693        || receipt.spawn.session.workspace_id != receipt.lease.workspace_id
1694    {
1695        return Err("managed spawn receipt does not match routed request");
1696    }
1697    let mut resolved_spec = spawn_spec.clone();
1698    resolved_spec.target.worktree_id = Some(receipt.lease.workspace_id.clone());
1699    validate_spawn_receipt(&resolved_spec, &receipt.spawn, relay_incarnation_id)
1700}
1701
1702fn validate_spawn_receipt(
1703    spec: &SpawnSpec,
1704    receipt: &ResolvedSpawnReceipt,
1705    relay_incarnation_id: NodeIncarnationId,
1706) -> Result<(), &'static str> {
1707    if receipt.incarnation_id != relay_incarnation_id
1708        || !receipt.context_binding_is_valid()
1709        || receipt.target != spec.target
1710        || receipt.profile_id != spec.profile_id
1711        || receipt.idempotency_key != spec.idempotency_key
1712        || receipt.deadline_ms != spec.deadline_ms
1713        || receipt.required_capabilities != spec.required_capabilities
1714        || receipt.bundle.as_ref().is_some_and(|bundle| {
1715            receipt.bundle_id.as_ref() != Some(&bundle.id)
1716        })
1717        || &receipt.session.workspace_id
1718            != spec
1719                .target
1720                .worktree_id
1721                .as_ref()
1722                .unwrap_or(&spec.target.workspace_id)
1723    {
1724        return Err("spawn receipt does not match routed request");
1725    }
1726    if !required_override_matches(&spec.overrides.provider, &receipt.provider)
1727        || !required_override_matches(&spec.overrides.mode, &receipt.mode)
1728        || !required_override_matches(&spec.overrides.terminal_size, &receipt.terminal_size)
1729        || !optional_override_matches(&spec.overrides.bundle_id, &receipt.bundle_id)
1730        || !optional_override_matches(&spec.overrides.context_id, &receipt.context_id)
1731        || !environment_profile_override_matches(
1732            &spec.overrides.environment_profile_id,
1733            receipt.environment_profile.as_ref(),
1734        )
1735    {
1736        return Err("spawn receipt contradicts explicit overrides");
1737    }
1738    match &spec.overrides.prompt {
1739        SpawnOverride::Inherit => {}
1740        SpawnOverride::Set { value }
1741            if receipt.prompt.present
1742                && receipt.prompt.byte_len == u32::try_from(value.byte_len()).unwrap_or(0) => {}
1743        SpawnOverride::Clear if !receipt.prompt.present && receipt.prompt.byte_len == 0 => {}
1744        SpawnOverride::Set { .. } | SpawnOverride::Clear => {
1745            return Err("spawn receipt contradicts explicit prompt override");
1746        }
1747    }
1748    Ok(())
1749}
1750
1751fn environment_profile_override_matches(
1752    expected: &SpawnOverride<gate4agent_node_protocol::SpawnEnvironmentProfileId>,
1753    actual: Option<&gate4agent_node_protocol::ResolvedEnvironmentProfileReceipt>,
1754) -> bool {
1755    match expected {
1756        SpawnOverride::Inherit => true,
1757        SpawnOverride::Set { value } => {
1758            actual.is_some_and(|receipt| &receipt.profile_id == value)
1759        }
1760        SpawnOverride::Clear => actual.is_none(),
1761    }
1762}
1763
1764fn required_override_matches<T: Eq>(override_value: &SpawnOverride<T>, actual: &T) -> bool {
1765    match override_value {
1766        SpawnOverride::Inherit => true,
1767        SpawnOverride::Set { value } => value == actual,
1768        SpawnOverride::Clear => false,
1769    }
1770}
1771
1772fn optional_override_matches<T: Eq>(
1773    override_value: &SpawnOverride<T>,
1774    actual: &Option<T>,
1775) -> bool {
1776    match override_value {
1777        SpawnOverride::Inherit => true,
1778        SpawnOverride::Set { value } => actual.as_ref() == Some(value),
1779        SpawnOverride::Clear => actual.is_none(),
1780    }
1781}
1782
1783fn relay_node_failure(error: &NodeClientError) -> Option<C2NodeFailure> {
1784    match error {
1785        NodeClientError::Node(failure) => Some(C2NodeFailure::from(failure)),
1786        NodeClientError::UnsupportedCapability(_) => Some(C2NodeFailure {
1787            code: NodeFailureCode::UnsupportedCapability,
1788            message: "required capability unavailable".to_owned(),
1789        }),
1790        NodeClientError::Io(_)
1791        | NodeClientError::Frame(_)
1792        | NodeClientError::Protocol(_)
1793        | NodeClientError::BuildStampMismatch { .. }
1794        | NodeClientError::AuthenticationTimedOut
1795        | NodeClientError::Authentication(_)
1796        | NodeClientError::RequestIdExhausted => None,
1797    }
1798}
1799
1800fn is_read_only_request(request: &NodeRequest) -> bool {
1801    matches!(request,
1802        NodeRequest::Snapshot
1803        | NodeRequest::Resync { .. }
1804        | NodeRequest::BrowseHostDirectories { .. }
1805        | NodeRequest::InspectWorkspace { .. }
1806        | NodeRequest::ReadWorkspaceFile { .. }
1807        | NodeRequest::ReadGitHistory { .. }
1808        | NodeRequest::ReadGitDiff { .. }
1809        | NodeRequest::CatalogNativeSessions { .. }
1810        | NodeRequest::PageNativeSessions { .. }
1811        | NodeRequest::PreviewNativeSession { .. }
1812        | NodeRequest::PreviewSessionRecord { .. }
1813    )
1814}
1815
1816async fn acquire_controller(
1817    client: &mut LocalNodeClient,
1818    connection_id: u64,
1819) -> Result<bool, NodeClientError> {
1820    acquire_controller_with_deadline(client, connection_id, None).await
1821}
1822
1823async fn acquire_controller_with_deadline(
1824    client: &mut LocalNodeClient,
1825    connection_id: u64,
1826    relay_deadline: Option<Instant>,
1827) -> Result<bool, NodeClientError> {
1828    match bounded_node_request_with_deadline(
1829        client,
1830        NodeRequest::AcquireController { lease_ms: gate4agent_node_protocol::MAX_CONTROLLER_LEASE_MS },
1831        relay_deadline,
1832    ).await? {
1833        NodeResponse::Controller { controller } => Ok(controller.as_ref().is_some_and(|state| state.connection_id == connection_id)),
1834        _ => Err(NodeClientError::Protocol("controller acquisition returned a different response".to_owned())),
1835    }
1836}
1837
1838async fn release_controller(
1839    client: &mut LocalNodeClient,
1840    controller_owned: &mut bool,
1841) -> Result<(), NodeClientError> {
1842    if !*controller_owned { return Ok(()); }
1843    match bounded_node_request(client, NodeRequest::ReleaseController).await? {
1844        NodeResponse::Controller { .. } => { *controller_owned = false; Ok(()) }
1845        _ => Err(NodeClientError::Protocol("controller release returned a different response".to_owned())),
1846    }
1847}
1848
1849async fn update_inventory_from_response(
1850    node_id: &NodeId,
1851    cursor: &mut NodeCursor,
1852    response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1853    ingress: &mpsc::Sender<Attempt>,
1854) -> io::Result<()> {
1855    match response {
1856        Ok(NodeResponse::Snapshot { event_sequence, snapshot, .. })
1857        | Ok(NodeResponse::Resync { event_sequence, snapshot, .. }) => {
1858            if *event_sequence < cursor.sequence { return Ok(()); }
1859            cursor.sequence = *event_sequence;
1860            ingress_attempt(ingress, node_id, AttemptResult::Success { cursor: *cursor, snapshot: snapshot.clone(), gaps: Vec::new() }).await
1861        }
1862        _ => Ok(()),
1863    }
1864}
1865
1866fn publish_recovered_events(
1867    node_id: &NodeId,
1868    incarnation_id: NodeIncarnationId,
1869    events: &[NodeEventEnvelope],
1870    hub: &OperatorHub,
1871) {
1872    for envelope in events {
1873        if let Some(event) = routed_recovered_node_event(node_id, incarnation_id, envelope) {
1874            hub.publish(event);
1875        }
1876    }
1877}
1878
1879fn routed_recovered_node_event(
1880    node_id: &NodeId,
1881    incarnation_id: NodeIncarnationId,
1882    envelope: &NodeEventEnvelope,
1883) -> Option<RoutedNodeEvent> {
1884    (!matches!(&envelope.event, NodeEvent::HarnessMcpReadCall { .. })).then(|| {
1885        RoutedNodeEvent {
1886            node_id: node_id.clone(),
1887            cursor: NodeCursor { incarnation_id, sequence: envelope.sequence },
1888            event: C2NodeEvent::from_node_event_with_control_detail(&envelope.event),
1889        }
1890    })
1891}
1892
1893fn routed_transient_node_event(
1894    node_id: &NodeId,
1895    cursor: NodeCursor,
1896    envelope: &NodeEventEnvelope,
1897) -> Option<RoutedNodeEvent> {
1898    matches!(&envelope.event, NodeEvent::HarnessMcpReadCall { .. }).then(|| RoutedNodeEvent {
1899        node_id: node_id.clone(),
1900        cursor,
1901        event: C2NodeEvent::from(&envelope.event),
1902    })
1903}
1904
1905/// `NodeEvent::AgentStream` chunks are published unconditionally -- never
1906/// gated on `cursor` contiguity, and never the cause of a
1907/// `CursorRegression`/`NonContiguousEvents` gap -- but, unlike
1908/// `HarnessMcpReadCall` (always `sequence: 0`, see
1909/// `gate4agent-node/src/server.rs::publish_transient`), an
1910/// `AgentStreamChunkV1` envelope draws its `sequence` from the
1911/// SAME counter every durable `NodeEvent` shares
1912/// (`gate4agent-node/src/server.rs::publish_agent_stream_chunk`'s own doc,
1913/// there purely so the node's per-connection discard watermark can compare
1914/// it against everything else). So a chunk really does consume a slot in
1915/// the durable sequence space, and `routed_transient_node_event`'s
1916/// "leave `cursor` untouched" rule -- correct for a call that never had a
1917/// slot to begin with -- would otherwise make the very next durable event
1918/// look one short of contiguous, forcing a resync round trip on every
1919/// single provider turn that streams so much as one chunk.
1920///
1921/// This folds the chunk's sequence into `cursor` with `max` instead: on the
1922/// ordinary, non-bursty path (`Control(N)` then its own `AgentStream(N+1)`,
1923/// each one delivered as they are produced) that keeps `cursor` moving
1924/// exactly as it did before this function existed, so the very next
1925/// `Control(N+2)` still finds `cursor == N+1` and stays contiguous. On the
1926/// bursty path -- the one actually measured live: 19 chunks published
1927/// `source_sequence` 22-40 in a 17ms window, `envelope_sequence` interleaved
1928/// 1:1 with `Control`, only 2 of the 19 ever reached the harness -- the
1929/// node's connection loop can drain its durable channel far enough ahead of
1930/// this dedicated one that a lower-sequence chunk arrives at this relay
1931/// AFTER `cursor` has already been carried past it by a resync recovering
1932/// the higher-sequence `Control` envelopes around it. `max` refuses to let
1933/// that late chunk rewind `cursor` -- it is still published (chunks promise
1934/// no resync, so there is nothing to recover if it were dropped instead:
1935/// `gate4agent-node-protocol::AgentStreamChunkV1`'s own doc, "no
1936/// `ObservationV1` resync promise") -- it simply stops mattering to the
1937/// durable cursor's own contiguity bookkeeping once something newer has
1938/// already passed it by. Ordering and true loss within the agent-stream
1939/// channel itself remain the node's own broadcast `Lagged` warn to name,
1940/// not this cursor's to adjudicate.
1941fn route_agent_stream_event(
1942    node_id: &NodeId,
1943    cursor: &mut NodeCursor,
1944    envelope: &NodeEventEnvelope,
1945) -> Option<RoutedNodeEvent> {
1946    if !matches!(&envelope.event, NodeEvent::AgentStream { .. }) {
1947        return None;
1948    }
1949    let routed = RoutedNodeEvent {
1950        node_id: node_id.clone(),
1951        cursor: *cursor,
1952        event: C2NodeEvent::from(&envelope.event),
1953    };
1954    cursor.sequence = cursor.sequence.max(envelope.sequence);
1955    Some(routed)
1956}
1957
1958async fn drain_pending_events(
1959    client: &mut LocalNodeClient,
1960    node_id: &NodeId,
1961    cursor: &mut NodeCursor,
1962    hub: &OperatorHub,
1963    ingress: &mpsc::Sender<Attempt>,
1964    skip_replayed: bool,
1965    relay_deadline: Option<Instant>,
1966) -> Result<(), NodeClientError> {
1967    let mut skip_replayed = skip_replayed;
1968    for repair_pass in 0..=1 {
1969        let mut gaps = Vec::new();
1970        let mut managed_worktree_events = Vec::new();
1971        let mut changed = false;
1972        let mut repair = false;
1973        while let Some(envelope) = client.take_event() {
1974            if let Some(event) = routed_transient_node_event(node_id, *cursor, &envelope) {
1975                hub.publish(event);
1976                continue;
1977            }
1978            if let Some(event) = route_agent_stream_event(node_id, cursor, &envelope) {
1979                hub.publish(event);
1980                continue;
1981            }
1982            if skip_replayed && envelope.sequence <= cursor.sequence { continue; }
1983            let resync_required = matches!(&envelope.event, gate4agent_node_protocol::NodeEvent::ResyncRequired { .. });
1984            if resync_required || envelope.sequence != cursor.sequence.saturating_add(1) {
1985                gaps.push(if envelope.sequence <= cursor.sequence && !resync_required {
1986                    GapKind::CursorRegression
1987                } else if resync_required {
1988                    GapKind::HistoryEvicted
1989                } else {
1990                    GapKind::NonContiguousEvents
1991                });
1992                repair = true;
1993                continue;
1994            }
1995            let event_cursor = NodeCursor { incarnation_id: cursor.incarnation_id, sequence: envelope.sequence };
1996            if matches!(
1997                &envelope.event,
1998                NodeEvent::ManagedWorktreeUpserted { .. }
1999                    | NodeEvent::ManagedWorktreeRemoved { .. }
2000            ) {
2001                managed_worktree_events.push(envelope.event.clone());
2002            }
2003            hub.publish(RoutedNodeEvent {
2004                node_id: node_id.clone(),
2005                cursor: event_cursor,
2006                event: C2NodeEvent::from_node_event_with_control_detail(&envelope.event),
2007            });
2008            cursor.sequence = envelope.sequence;
2009            changed = true;
2010        }
2011        if !repair {
2012            if changed || !gaps.is_empty() {
2013                ingress_attempt(ingress, node_id, AttemptResult::Cursor {
2014                    cursor: *cursor,
2015                    gaps,
2016                    managed_worktree_events,
2017                }).await
2018                    .map_err(NodeClientError::Io)?;
2019            }
2020            return Ok(());
2021        }
2022        if repair_pass == 1 {
2023            ingress_attempt(ingress, node_id, AttemptResult::Cursor {
2024                cursor: *cursor,
2025                gaps,
2026                managed_worktree_events,
2027            }).await
2028                .map_err(NodeClientError::Io)?;
2029            return Err(NodeClientError::Protocol("node event stream remained noncontiguous after resync".to_owned()));
2030        }
2031        let after_sequence = cursor.sequence;
2032        let response = bounded_node_request_with_deadline(
2033            client,
2034            NodeRequest::Resync { after_sequence },
2035            relay_deadline,
2036        ).await?;
2037        let NodeResponse::Resync {
2038            event_sequence,
2039            oldest_available_sequence,
2040            snapshot,
2041            events,
2042        } = response else {
2043            return Err(NodeClientError::Protocol("event repair resync returned a different response".to_owned()));
2044        };
2045        let repair_gaps = validate_events(
2046            after_sequence,
2047            event_sequence,
2048            oldest_available_sequence,
2049            &events,
2050        );
2051        let contiguous = repair_gaps.is_empty();
2052        gaps = repair_gaps;
2053        if contiguous {
2054            for envelope in events.iter().filter(|event| event.sequence > after_sequence) {
2055                if let Some(event) = routed_recovered_node_event(
2056                    node_id,
2057                    cursor.incarnation_id,
2058                    envelope,
2059                ) {
2060                    hub.publish(event);
2061                }
2062            }
2063        } else {
2064            hub.publish(RoutedNodeEvent {
2065                node_id: node_id.clone(),
2066                cursor: NodeCursor { incarnation_id: cursor.incarnation_id, sequence: event_sequence },
2067                event: C2NodeEvent::ResyncRequired {
2068                    oldest_available_sequence,
2069                },
2070            });
2071        }
2072        cursor.sequence = event_sequence;
2073        ingress_attempt(ingress, node_id, AttemptResult::Success { cursor: *cursor, snapshot, gaps }).await
2074            .map_err(NodeClientError::Io)?;
2075        skip_replayed = true;
2076    }
2077    Ok(())
2078}
2079
2080async fn handle_live_node_event(
2081    client: &mut LocalNodeClient,
2082    node_id: &NodeId,
2083    envelope: NodeEventEnvelope,
2084    cursor: &mut NodeCursor,
2085    hub: &OperatorHub,
2086    ingress: &mpsc::Sender<Attempt>,
2087) -> Result<(), NodeClientError> {
2088    if let Some(event) = routed_transient_node_event(node_id, *cursor, &envelope) {
2089        hub.publish(event);
2090        return Ok(());
2091    }
2092    if let Some(event) = route_agent_stream_event(node_id, cursor, &envelope) {
2093        hub.publish(event);
2094        return Ok(());
2095    }
2096    if live_event_gap(cursor.sequence, &envelope).is_some() {
2097        let after_sequence = cursor.sequence;
2098        let response = bounded_node_request(
2099            client,
2100            NodeRequest::Resync { after_sequence },
2101        ).await?;
2102        let NodeResponse::Resync {
2103            event_sequence,
2104            oldest_available_sequence,
2105            snapshot,
2106            events,
2107        } = response else {
2108            return Err(NodeClientError::Protocol(
2109                "live event repair resync returned a different response".to_owned(),
2110            ));
2111        };
2112        let repair_gaps = validate_events(
2113            after_sequence,
2114            event_sequence,
2115            oldest_available_sequence,
2116            &events,
2117        );
2118        if repair_gaps.is_empty() {
2119            publish_recovered_events(node_id, cursor.incarnation_id, &events, hub);
2120        } else {
2121            hub.publish(RoutedNodeEvent {
2122                node_id: node_id.clone(),
2123                cursor: NodeCursor {
2124                    incarnation_id: cursor.incarnation_id,
2125                    sequence: event_sequence,
2126                },
2127                event: C2NodeEvent::ResyncRequired {
2128                    oldest_available_sequence,
2129                },
2130            });
2131        }
2132        cursor.sequence = event_sequence;
2133        ingress_attempt(
2134            ingress,
2135            node_id,
2136            AttemptResult::Success {
2137                cursor: *cursor,
2138                snapshot,
2139                gaps: repair_gaps,
2140            },
2141        ).await.map_err(NodeClientError::Io)?;
2142        return drain_pending_events(
2143            client,
2144            node_id,
2145            cursor,
2146            hub,
2147            ingress,
2148            true,
2149            None,
2150        ).await;
2151    }
2152
2153    cursor.sequence = envelope.sequence;
2154    let managed_worktree_events = matches!(
2155        &envelope.event,
2156        NodeEvent::ManagedWorktreeUpserted { .. }
2157            | NodeEvent::ManagedWorktreeRemoved { .. }
2158    )
2159    .then(|| vec![envelope.event.clone()])
2160    .unwrap_or_default();
2161    hub.publish(RoutedNodeEvent {
2162        node_id: node_id.clone(),
2163        cursor: *cursor,
2164        event: C2NodeEvent::from_node_event_with_control_detail(&envelope.event),
2165    });
2166    ingress_attempt(
2167        ingress,
2168        node_id,
2169        AttemptResult::Cursor {
2170            cursor: *cursor,
2171            gaps: Vec::new(),
2172            managed_worktree_events,
2173        },
2174    ).await.map_err(NodeClientError::Io)
2175}
2176
2177fn live_event_gap(previous: u64, envelope: &NodeEventEnvelope) -> Option<GapKind> {
2178    if matches!(
2179        &envelope.event,
2180        gate4agent_node_protocol::NodeEvent::ResyncRequired { .. }
2181    ) {
2182        return Some(GapKind::HistoryEvicted);
2183    }
2184    if envelope.sequence <= previous {
2185        return Some(GapKind::CursorRegression);
2186    }
2187    (envelope.sequence != previous.saturating_add(1))
2188        .then_some(GapKind::NonContiguousEvents)
2189}
2190
2191fn reject_disconnected_commands(commands: &mut mpsc::Receiver<RelayCommand>, cursor: Option<NodeCursor>) {
2192    while let Ok(command) = commands.try_recv() {
2193        match command {
2194            RelayCommand::Request { reply, .. } => {
2195                let _ = reply.send(Err(relay_failure(
2196                    C2RelayFailureCode::NodeOffline,
2197                    "node relay disconnected before request dispatch",
2198                    cursor.map(|value| value.incarnation_id),
2199                )));
2200            }
2201        }
2202    }
2203}
2204
2205fn acknowledge_disconnected_releases(releases: &mut mpsc::Receiver<oneshot::Sender<()>>) {
2206    while let Ok(reply) = releases.try_recv() { let _ = reply.send(()); }
2207}
2208
2209fn validate_events(
2210    previous: u64,
2211    current: u64,
2212    oldest_available_sequence: u64,
2213    events: &[NodeEventEnvelope],
2214) -> Vec<GapKind> {
2215    if current < previous { return vec![GapKind::CursorRegression]; }
2216    let max_floor = current.checked_add(1).unwrap_or(u64::MAX);
2217    let minimum_event_sequence = previous
2218        .checked_add(1)
2219        .unwrap_or(u64::MAX)
2220        .max(oldest_available_sequence);
2221    if oldest_available_sequence == 0
2222        || oldest_available_sequence > max_floor
2223        || (current == previous && !events.is_empty())
2224        || events.iter().any(|event| {
2225            event.sequence < minimum_event_sequence || event.sequence > current
2226        })
2227        || events.windows(2).any(|pair| pair[0].sequence >= pair[1].sequence)
2228    {
2229        return vec![GapKind::NonContiguousEvents];
2230    }
2231    if previous.saturating_add(1) < oldest_available_sequence {
2232        return vec![GapKind::HistoryEvicted];
2233    }
2234    Vec::new()
2235}
2236
2237fn sanitize_node_error(error: &NodeClientError) -> (SanitizedError, bool) {
2238    let (category, message, hard) = match error {
2239        NodeClientError::Protocol(message) if message.contains("identity mismatch") =>
2240            (C2ErrorCategory::Identity, "node identity mismatch", true),
2241        NodeClientError::Protocol(message) if message.contains("access-token proof") || message.contains("access denied") =>
2242            (C2ErrorCategory::Authentication, "node authentication failed", true),
2243        NodeClientError::Protocol(_) | NodeClientError::Frame(FrameError::Json(_) | FrameError::InvalidLength { .. }) =>
2244            (C2ErrorCategory::Protocol, "node protocol failed", true),
2245        NodeClientError::BuildStampMismatch { .. } =>
2246            (C2ErrorCategory::Protocol, "node build stamp mismatch", true),
2247        NodeClientError::UnsupportedCapability(_) =>
2248            (C2ErrorCategory::Protocol, "node capability unavailable", true),
2249        NodeClientError::Node(failure) if failure.code == NodeFailureCode::Unauthorized =>
2250            (C2ErrorCategory::Authentication, "node request authentication failed", true),
2251        NodeClientError::Node(_) =>
2252            (C2ErrorCategory::Protocol, "node rejected observer request", true),
2253        NodeClientError::Frame(FrameError::BodyTimedOut { .. } | FrameError::PrefixTimedOut)
2254            | NodeClientError::AuthenticationTimedOut =>
2255            (C2ErrorCategory::Timeout, "node observation deadline exceeded", false),
2256        NodeClientError::Authentication(_) | NodeClientError::RequestIdExhausted =>
2257            (C2ErrorCategory::Internal, "node client failed internally", true),
2258        NodeClientError::Io(_) | NodeClientError::Frame(FrameError::Io(_)) =>
2259            (C2ErrorCategory::Transport, "node transport unavailable", false),
2260    };
2261    (SanitizedError { category, message: message.to_owned() }, hard)
2262}
2263
2264async fn inventory_owner(
2265    configured: usize, fresh_for: Duration, mut ingress: mpsc::Receiver<Attempt>,
2266    status: watch::Sender<Arc<StatusResponse>>, mut shutdown: watch::Receiver<bool>,
2267) -> io::Result<()> {
2268    let mut current = (**status.borrow()).clone();
2269    let mut attempted = BTreeSet::new();
2270    loop {
2271        tokio::select! {
2272            attempt = ingress.recv() => {
2273                let Some(attempt) = attempt else { return Ok(()); };
2274                attempted.insert(attempt.node_id.clone());
2275                let node = current.nodes.get_mut(&attempt.node_id).expect("configured poller node exists");
2276                node.last_attempt_unix_ms = Some(attempt.at_unix_ms);
2277                match attempt.result {
2278                    AttemptResult::Connected {
2279                        cursor,
2280                        snapshot,
2281                        gaps,
2282                        provider_contract_manifest,
2283                    } => {
2284                        let previous = node.cursor;
2285                        let mut inventory = SlimNodeInventory::from_snapshot(&snapshot);
2286                        inventory.provider_contracts =
2287                            provider_contract_manifest.provider_contracts;
2288                        inventory.provider_adapter_contracts =
2289                            provider_contract_manifest.provider_adapter_contracts;
2290                        node.transport = NodeTransportState::Online;
2291                        node.freshness = NodeFreshness::Fresh;
2292                        node.cursor = Some(cursor);
2293                        node.inventory = Some(inventory);
2294                        node.last_success_unix_ms = Some(attempt.at_unix_ms);
2295                        node.consecutive_failures = 0;
2296                        node.last_error = None;
2297                        for kind in gaps {
2298                            if node.gaps.len() == MAX_C2_GAPS_PER_NODE { node.gaps.remove(0); node.gaps_truncated += 1; }
2299                            node.gaps.push(NodeGap { kind, detected_at_unix_ms: attempt.at_unix_ms, previous, observed: cursor });
2300                        }
2301                    }
2302                    AttemptResult::Success { cursor, snapshot, gaps } => {
2303                        let previous = node.cursor;
2304                        let provider_contract_manifest = node.inventory.as_ref().map(|inventory| {
2305                            ProviderContractManifest {
2306                                provider_contracts: inventory.provider_contracts.clone(),
2307                                provider_adapter_contracts: inventory.provider_adapter_contracts.clone(),
2308                            }
2309                        }).unwrap_or_default();
2310                        let mut inventory = SlimNodeInventory::from_snapshot(&snapshot);
2311                        inventory.provider_contracts =
2312                            provider_contract_manifest.provider_contracts;
2313                        inventory.provider_adapter_contracts =
2314                            provider_contract_manifest.provider_adapter_contracts;
2315                        node.transport = NodeTransportState::Online;
2316                        node.freshness = NodeFreshness::Fresh;
2317                        node.cursor = Some(cursor);
2318                        node.inventory = Some(inventory);
2319                        node.last_success_unix_ms = Some(attempt.at_unix_ms);
2320                        node.consecutive_failures = 0;
2321                        node.last_error = None;
2322                        for kind in gaps {
2323                            if node.gaps.len() == MAX_C2_GAPS_PER_NODE { node.gaps.remove(0); node.gaps_truncated += 1; }
2324                            node.gaps.push(NodeGap { kind, detected_at_unix_ms: attempt.at_unix_ms, previous, observed: cursor });
2325                        }
2326                    }
2327                    AttemptResult::Cursor {
2328                        cursor,
2329                        gaps,
2330                        managed_worktree_events,
2331                    } => {
2332                        let previous = node.cursor;
2333                        let incarnation_changed = previous.is_some_and(|previous| {
2334                            previous.incarnation_id != cursor.incarnation_id
2335                        });
2336                        apply_managed_worktree_cursor(
2337                            node.inventory.as_mut(),
2338                            incarnation_changed,
2339                            &managed_worktree_events,
2340                        );
2341                        node.transport = NodeTransportState::Online;
2342                        node.freshness = NodeFreshness::Fresh;
2343                        node.cursor = Some(cursor);
2344                        node.last_success_unix_ms = Some(attempt.at_unix_ms);
2345                        node.consecutive_failures = 0;
2346                        node.last_error = None;
2347                        for kind in gaps {
2348                            if node.gaps.len() == MAX_C2_GAPS_PER_NODE { node.gaps.remove(0); node.gaps_truncated += 1; }
2349                            node.gaps.push(NodeGap { kind, detected_at_unix_ms: attempt.at_unix_ms, previous, observed: cursor });
2350                        }
2351                    }
2352                    AttemptResult::Failure { error, hard } => {
2353                        node.consecutive_failures = node.consecutive_failures.saturating_add(1);
2354                        node.transport = if hard || node.consecutive_failures >= 5 { NodeTransportState::Parked } else { NodeTransportState::Offline };
2355                        node.last_error = Some(error);
2356                    }
2357                }
2358                current.ready = attempted.len() == configured;
2359                refresh_freshness(&mut current, fresh_for);
2360                current.observed_at_unix_ms = unix_ms();
2361                status.send_replace(Arc::new(current.clone()));
2362            }
2363            _ = sleep(Duration::from_millis(250)) => {
2364                refresh_freshness(&mut current, fresh_for);
2365                current.observed_at_unix_ms = unix_ms();
2366                status.send_replace(Arc::new(current.clone()));
2367            }
2368            changed = shutdown.changed() => if changed.is_err() || *shutdown.borrow() { return Ok(()); },
2369        }
2370    }
2371}
2372
2373fn apply_managed_worktree_cursor(
2374    inventory: Option<&mut SlimNodeInventory>,
2375    incarnation_changed: bool,
2376    events: &[NodeEvent],
2377) {
2378    let Some(inventory) = inventory else { return; };
2379    if incarnation_changed {
2380        inventory.provider_runtime_statuses.clear();
2381        inventory.managed_worktrees.clear();
2382        inventory.managed_worktree_count = 0;
2383        inventory.managed_worktrees_truncated = false;
2384        return;
2385    }
2386    for event in events {
2387        match event {
2388            NodeEvent::ManagedWorktreeUpserted { lease } => {
2389                inventory.managed_worktrees.retain(|existing| {
2390                    existing.lease_id != lease.lease_id
2391                        && existing.workspace_id != lease.workspace_id
2392                });
2393                inventory.managed_worktrees.push(lease.clone());
2394                inventory.managed_worktrees.sort_by(|left, right| {
2395                    left.lease_id.cmp(&right.lease_id)
2396                });
2397                inventory.managed_worktree_count = inventory.managed_worktrees.len();
2398                inventory.managed_worktrees.truncate(
2399                    crate::protocol::MAX_C2_MANAGED_WORKTREES_PER_NODE,
2400                );
2401                inventory.managed_worktrees_truncated =
2402                    inventory.managed_worktrees.len() < inventory.managed_worktree_count;
2403            }
2404            NodeEvent::ManagedWorktreeRemoved { lease_id } => {
2405                let before = inventory.managed_worktrees.len();
2406                inventory
2407                    .managed_worktrees
2408                    .retain(|lease| &lease.lease_id != lease_id);
2409                if inventory.managed_worktrees.len() < before
2410                    || inventory.managed_worktrees_truncated
2411                {
2412                    inventory.managed_worktree_count =
2413                        inventory.managed_worktree_count.saturating_sub(1);
2414                }
2415                inventory.managed_worktrees_truncated =
2416                    inventory.managed_worktrees.len() < inventory.managed_worktree_count;
2417            }
2418            _ => {}
2419        }
2420    }
2421}
2422
2423fn refresh_freshness(status: &mut StatusResponse, fresh_for: Duration) {
2424    let now = unix_ms();
2425    let fresh_ms = fresh_for.as_millis().min(u64::MAX as u128) as u64;
2426    for node in status.nodes.values_mut() {
2427        node.freshness = match node.last_success_unix_ms {
2428            None => NodeFreshness::Unavailable,
2429            Some(last) if now.saturating_sub(last) <= fresh_ms => NodeFreshness::Fresh,
2430            Some(_) => NodeFreshness::Stale,
2431        };
2432    }
2433}
2434
2435async fn http_server(
2436    listener: TcpListener, token: String, io_deadline: Duration,
2437    status: watch::Receiver<Arc<StatusResponse>>, mut shutdown: watch::Receiver<bool>,
2438) -> io::Result<()> {
2439    let permits = Arc::new(Semaphore::new(MAX_HTTP_CONNECTIONS));
2440    let mut connections = JoinSet::new();
2441    loop {
2442        tokio::select! {
2443            accepted = listener.accept() => {
2444                let (stream, _) = accepted?;
2445                let Ok(permit) = Arc::clone(&permits).try_acquire_owned() else { drop(stream); continue; };
2446                let token = token.clone();
2447                let status = status.clone();
2448                connections.spawn(async move { let _permit = permit; let _ = serve_http(stream, &token, io_deadline, &status).await; });
2449            }
2450            changed = shutdown.changed() => if changed.is_err() || *shutdown.borrow() { break; },
2451        }
2452        while let Some(result) = connections.try_join_next() { result.map_err(io::Error::other)?; }
2453    }
2454    connections.shutdown().await;
2455    Ok(())
2456}
2457
2458async fn serve_http(mut stream: TcpStream, token: &str, deadline: Duration, status: &watch::Receiver<Arc<StatusResponse>>) -> io::Result<()> {
2459    let request = match timeout(deadline, read_request(&mut stream)).await {
2460        Ok(Ok(request)) => request,
2461        Ok(Err(ReadError::TooLarge)) => return write_response(&mut stream, deadline, Response::plain(413, "Payload Too Large")).await,
2462        Ok(Err(ReadError::Io(error))) => return Err(error),
2463        _ => return Ok(()),
2464    };
2465    let response = route(request, token, status.borrow().as_ref());
2466    write_response(&mut stream, deadline, response).await
2467}
2468
2469struct Request { method: String, path: String, authorization: Option<String> }
2470enum ReadError { Closed, Invalid, TooLarge, Io(io::Error) }
2471
2472async fn read_request(stream: &mut TcpStream) -> Result<Request, ReadError> {
2473    let mut bytes = Vec::with_capacity(1024);
2474    let mut chunk = [0_u8; 1024];
2475    loop {
2476        let count = stream.read(&mut chunk).await.map_err(ReadError::Io)?;
2477        if count == 0 { return Err(ReadError::Closed); }
2478        if bytes.len().saturating_add(count) > HEADER_LIMIT_BYTES { return Err(ReadError::TooLarge); }
2479        bytes.extend_from_slice(&chunk[..count]);
2480        if bytes.windows(4).any(|window| window == b"\r\n\r\n") { break; }
2481    }
2482    let text = std::str::from_utf8(&bytes).map_err(|_| ReadError::Invalid)?;
2483    let mut lines = text.split("\r\n");
2484    let mut first = lines.next().ok_or(ReadError::Invalid)?.split_whitespace();
2485    let method = first.next().ok_or(ReadError::Invalid)?;
2486    let path = first.next().ok_or(ReadError::Invalid)?;
2487    let version = first.next().ok_or(ReadError::Invalid)?;
2488    if first.next().is_some() || !version.starts_with("HTTP/1.") || !path.starts_with('/') { return Err(ReadError::Invalid); }
2489    let mut authorization = None;
2490    for line in lines {
2491        if line.is_empty() { break; }
2492        let (name, value) = line.split_once(':').ok_or(ReadError::Invalid)?;
2493        if name.eq_ignore_ascii_case("authorization") {
2494            if authorization.is_some() { return Err(ReadError::Invalid); }
2495            authorization = Some(value.trim().to_owned());
2496        }
2497    }
2498    Ok(Request { method: method.to_owned(), path: path.to_owned(), authorization })
2499}
2500
2501fn route(request: Request, token: &str, status: &StatusResponse) -> Response {
2502    if request.method != "GET" { return Response::plain(405, "Method Not Allowed").allow_get(); }
2503    let path = request.path.split_once('?').map_or(request.path.as_str(), |pair| pair.0);
2504    match path {
2505        "/health" => Response::json(200, &HealthResponse { ok: true, service: "gate4agent-c2".to_owned(), api_version: C2_API_VERSION, pid: std::process::id(), version: env!("CARGO_PKG_VERSION").to_owned() }),
2506        "/ready" => {
2507            let online_nodes = status.nodes.values().filter(|node| node.transport == NodeTransportState::Online).count();
2508            let offline_nodes = status.nodes.values().filter(|node| node.transport == NodeTransportState::Offline).count();
2509            let parked_nodes = status.nodes.values().filter(|node| node.transport == NodeTransportState::Parked).count();
2510            let body = ReadyResponse { ready: status.ready, api_version: C2_API_VERSION, configured_nodes: status.nodes.len(), attempted_nodes: status.nodes.values().filter(|node| node.last_attempt_unix_ms.is_some()).count(), online_nodes, offline_nodes, parked_nodes };
2511            Response::json(if status.ready { 200 } else { 503 }, &body)
2512        }
2513        "/status" => {
2514            if !authorized(request.authorization.as_deref(), token) { Response::plain(401, "Unauthorized").authenticate() }
2515            else { Response::json(200, status) }
2516        }
2517        _ => Response::plain(404, "Not Found"),
2518    }
2519}
2520
2521fn authorized(header: Option<&str>, token: &str) -> bool {
2522    let Some((scheme, candidate)) = header.and_then(|value| value.split_once(' ')) else { return false; };
2523    scheme.eq_ignore_ascii_case("bearer") && constant_time_eq(candidate.as_bytes(), token.as_bytes())
2524}
2525
2526fn constant_time_eq(left: &[u8], right: &[u8]) -> bool {
2527    if left.len() != right.len() { return false; }
2528    left.iter().zip(right).fold(0_u8, |difference, (left, right)| difference | (left ^ right)) == 0
2529}
2530
2531struct Response { status: u16, reason: &'static str, content_type: &'static str, body: Vec<u8>, headers: Vec<(&'static str, &'static str)> }
2532impl Response {
2533    fn plain(status: u16, reason: &'static str) -> Self { Self { status, reason, content_type: "text/plain; charset=utf-8", body: reason.as_bytes().to_vec(), headers: Vec::new() } }
2534    fn json<T: serde::Serialize>(status: u16, value: &T) -> Self {
2535        let body = serde_json::to_vec(value).expect("C2 DTO must serialize");
2536        if body.len() > RESPONSE_BODY_LIMIT_BYTES { return Self::plain(503, "Service Unavailable"); }
2537        let reason = if status == 200 { "OK" } else { "Service Unavailable" };
2538        Self { status, reason, content_type: "application/json", body, headers: Vec::new() }
2539    }
2540    fn allow_get(mut self) -> Self { self.headers.push(("Allow", "GET")); self }
2541    fn authenticate(mut self) -> Self { self.headers.push(("WWW-Authenticate", "Bearer")); self }
2542}
2543
2544async fn write_response(stream: &mut TcpStream, deadline: Duration, response: Response) -> io::Result<()> {
2545    let mut headers = format!("HTTP/1.1 {} {}\r\nContent-Type: {}\r\nContent-Length: {}\r\nConnection: close\r\n", response.status, response.reason, response.content_type, response.body.len());
2546    for (name, value) in response.headers { headers.push_str(name); headers.push_str(": "); headers.push_str(value); headers.push_str("\r\n"); }
2547    headers.push_str("\r\n");
2548    timeout(deadline, async { stream.write_all(headers.as_bytes()).await?; stream.write_all(&response.body).await?; stream.shutdown().await }).await
2549        .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "C2 HTTP write timed out"))?
2550}
2551
2552fn unix_ms() -> u64 {
2553    SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis().min(u64::MAX as u128) as u64
2554}
2555
2556#[cfg(test)]
2557mod endpoint_tests {
2558    use super::*;
2559    use crate::protocol::{C2RelayRoute, C2Topology};
2560
2561    fn node(endpoint: &str) -> Result<C2NodeConfig, C2ConfigError> {
2562        C2NodeConfig::new(NodeId::new("remote-node").unwrap(), endpoint, "safe-token")
2563    }
2564
2565    #[cfg(windows)]
2566    const LOCAL_ENDPOINT: &str = r"\\.\pipe\relay-route-fact";
2567    #[cfg(unix)]
2568    const LOCAL_ENDPOINT: &str = "/tmp/gate4agent-relay-route-fact.sock";
2569
2570    fn projected_route(node: &C2NodeConfig, transport: NodeTransportState) -> C2RelayRoute {
2571        let mut observed = initial_observed_node(node);
2572        observed.transport = transport;
2573        let status = StatusResponse {
2574            api_version: C2_API_VERSION,
2575            ready: false,
2576            observed_at_unix_ms: 1,
2577            nodes: BTreeMap::from([(node.node_id.clone(), observed)]),
2578        };
2579        C2Topology::from_status(&status).nodes[0].relay_route
2580    }
2581
2582    #[test]
2583    fn relay_route_fact_projects_exactly_and_survives_offline_and_parked() {
2584        let local = node(LOCAL_ENDPOINT).unwrap();
2585        let ssh = node("tcp://127.0.0.1:48100").unwrap();
2586
2587        for transport in [
2588            NodeTransportState::Online,
2589            NodeTransportState::Offline,
2590            NodeTransportState::Parked,
2591        ] {
2592            assert_eq!(projected_route(&local, transport), C2RelayRoute::LocalIpc);
2593            assert_eq!(
2594                projected_route(&ssh, transport),
2595                C2RelayRoute::SshForwardedLoopback,
2596            );
2597        }
2598    }
2599
2600    /// `accept` is the assignment that names no address, and the config
2601    /// has to treat it as a different KIND of route rather than as a
2602    /// malformed endpoint: it is how a node that cannot be dialled is
2603    /// declared.
2604    #[test]
2605    fn a_node_that_waits_to_be_called_is_declared_by_name_not_by_address() {
2606        let waiting = node("accept").unwrap();
2607        assert_eq!(waiting.route, C2NodeRoute::CallHome);
2608        assert_eq!(waiting.transport_label(), "call-home");
2609
2610        // Every waiting node carries the same `accept`, so the
2611        // duplicate-endpoint check must not see them as two nodes fighting
2612        // over one socket. Their uniqueness is `node_id`'s job, and that
2613        // check still applies.
2614        let second = C2NodeConfig::new(
2615            NodeId::new("second-waiting-node").unwrap(),
2616            "accept",
2617            "safe-token",
2618        )
2619        .unwrap();
2620        let config = C2Config::new(
2621            "127.0.0.1:0".parse().unwrap(),
2622            "safe-token",
2623            vec![waiting.clone(), second],
2624        )
2625        .expect("two waiting nodes are not a duplicate endpoint");
2626        assert_eq!(config.nodes.len(), 2);
2627    }
2628
2629    /// A node that waits for a call the relay never listens for is a
2630    /// deployment that can never work, so it is refused at startup instead
2631    /// of becoming a node that is silently offline forever.
2632    #[test]
2633    fn waiting_for_a_call_requires_somewhere_to_be_called() {
2634        let waiting = node("accept").unwrap();
2635        let config =
2636            C2Config::new("127.0.0.1:0".parse().unwrap(), "safe-token", vec![waiting]).unwrap();
2637        assert!(matches!(
2638            config.validate_call_home(),
2639            Err(C2ConfigError::CallHomeWithoutListener(_)),
2640        ));
2641        let config = config.with_node_listen("127.0.0.1:48200".parse().unwrap()).unwrap();
2642        assert!(config.validate_call_home().is_ok());
2643    }
2644
2645    /// Call-home may bind loopback or a concrete C2 WireGuard tunnel address.
2646    /// Unspecified (`0.0.0.0`) and port 0 stay refused so plaintext frames do
2647    /// not land on every interface.
2648    #[test]
2649    fn the_call_home_listener_refuses_unspecified_and_accepts_tunnel() {
2650        let waiting = node("accept").unwrap();
2651        let config =
2652            C2Config::new("127.0.0.1:9000".parse().unwrap(), "safe-token", vec![waiting]).unwrap();
2653        for refused in ["0.0.0.0:48200", "127.0.0.1:0", "[::]:48200"] {
2654            assert!(
2655                matches!(
2656                    config.clone().with_node_listen(refused.parse().unwrap()),
2657                    Err(C2ConfigError::NonLoopbackNodeListen(_)),
2658                ),
2659                "accepted {refused}",
2660            );
2661        }
2662        let tunnel: SocketAddr = "10.88.1.2:27442".parse().unwrap();
2663        let accepted = config.clone().with_node_listen(tunnel).unwrap();
2664        assert_eq!(accepted.node_listen, Some(tunnel));
2665        // Sharing the API's own address would mean HTTP and node frames
2666        // arriving on one socket; neither parser would survive the other.
2667        assert!(matches!(
2668            config.with_node_listen("127.0.0.1:9000".parse().unwrap()),
2669            Err(C2ConfigError::NodeListenConflict),
2670        ));
2671    }
2672
2673    #[test]
2674    fn ssh_forwarded_loopback_route_is_strict_canonical_and_control_stays_local() {
2675        let ipv4 = node("tcp://127.0.0.1:48100").unwrap();
2676        assert_eq!(ipv4.endpoint, "tcp://127.0.0.1:48100");
2677        assert_eq!(ipv4.route, C2NodeRoute::SshForwardedLoopback("127.0.0.1:48100".parse().unwrap()));
2678        assert_eq!(ipv4.transport_label(), "ssh-forwarded-loopback");
2679
2680        let ipv6 = node("tcp://[0:0:0:0:0:0:0:1]:48100").unwrap();
2681        assert_eq!(ipv6.endpoint, "tcp://[::1]:48100");
2682        assert_eq!(ipv6.transport_label(), "ssh-forwarded-loopback");
2683
2684        for invalid in [
2685            "tcp://localhost:48100",
2686            "tcp://127.0.0.2:48100",
2687            "tcp://0.0.0.0:48100",
2688            "tcp://127.0.0.1:0",
2689            "tcp://user@127.0.0.1:48100",
2690            "tcp://127.0.0.1:48100/path",
2691            "TCP://127.0.0.1:48100",
2692        ] {
2693            assert!(matches!(node(invalid), Err(C2ConfigError::InvalidEndpoint(_))), "accepted {invalid}");
2694        }
2695
2696        assert!(matches!(
2697            C2Config::new(
2698                "127.0.0.1:0".parse().unwrap(),
2699                "safe-token",
2700                vec![ipv4.clone(), C2NodeConfig::new(
2701                    NodeId::new("duplicate-route").unwrap(),
2702                    "tcp://127.0.0.1:48100",
2703                    "safe-token",
2704                ).unwrap()],
2705            ),
2706            Err(C2ConfigError::DuplicateEndpoint)
2707        ));
2708        assert!(matches!(
2709            C2Config::new("127.0.0.1:0".parse().unwrap(), "safe-token", vec![ipv4])
2710                .unwrap()
2711                .with_control_endpoint("tcp://127.0.0.1:48101"),
2712            Err(C2ConfigError::InvalidControlEndpoint)
2713        ));
2714    }
2715}
2716
2717#[cfg(all(test, windows))]
2718mod tests {
2719    use super::*;
2720    use gate4agent_node_protocol::{
2721        AgentId, AgentStreamChunkKindV1, AgentStreamChunkV1, CapabilityId, SessionAddress,
2722        SessionKey, SessionMode, SessionRecordId,
2723        HarnessMcpActivationDigest, HarnessMcpCallId, HarnessMcpContentTypeV1,
2724        HarnessMcpOpaquePayloadV1, HarnessMcpReservationId,
2725        SpawnDeadlineMs, SpawnFieldProvenance, SpawnIdempotencyKey, SpawnOverrides,
2726        SpawnProfileId, SpawnProfileRevision, SpawnPrompt, SpawnPromptMetadata,
2727        SpawnRequiredCapabilities, SpawnResolutionProvenance, SpawnTarget, WorkspaceId,
2728        ManagedWorktreeCleanupFailure, ManagedWorktreeLeaseId,
2729        ManagedWorktreeLeaseSnapshot, ManagedWorktreeRetention, ManagedWorktreeSpawnReceipt,
2730        WorktreeProfileId, WorktreeProfileRevision,
2731    };
2732    use gate4agent_types::{AgentInstanceId, SessionGeneration, TerminalSize};
2733    use std::collections::BTreeMap;
2734
2735    fn agent(value: &str) -> gate4agent_node_protocol::AgentId {
2736        gate4agent_node_protocol::AgentId::new(value).unwrap()
2737    }
2738
2739    fn managed_lease(
2740        lease_id: &str,
2741        workspace_id: &str,
2742        state: ManagedWorktreeLeaseState,
2743    ) -> ManagedWorktreeLeaseSnapshot {
2744        let in_use = state == ManagedWorktreeLeaseState::InUse;
2745        ManagedWorktreeLeaseSnapshot {
2746            lease_id: ManagedWorktreeLeaseId::new(lease_id).unwrap(),
2747            source_workspace_id: WorkspaceId::new("repo").unwrap(),
2748            workspace_id: WorkspaceId::new(workspace_id).unwrap(),
2749            profile_id: WorktreeProfileId::new("review").unwrap(),
2750            profile_revision: WorktreeProfileRevision::new("review.r1").unwrap(),
2751            retention: ManagedWorktreeRetention::RemoveWhenReleased,
2752            state,
2753            active_session_count: u16::from(in_use),
2754            managed_record_count: u16::from(in_use),
2755            cleanup_failure: None::<ManagedWorktreeCleanupFailure>,
2756            created_at_unix_ms: 1,
2757            updated_at_unix_ms: 2,
2758        }
2759    }
2760
2761    #[test]
2762    fn spawn_spec_receipt_correlation_rejects_mismatches_before_forwarding() {
2763        let incarnation_id = NodeIncarnationId::from_bytes([7; 16]);
2764        let terminal_size = TerminalSize { rows: 24, columns: 80 };
2765        let prompt = SpawnPrompt::new("hi").unwrap();
2766        let required_capabilities = SpawnRequiredCapabilities::new([
2767            CapabilityId::new("raw-pty-lifecycle").unwrap(),
2768        ]).unwrap();
2769        let spec = SpawnSpec {
2770            target: SpawnTarget {
2771                node_id: NodeId::new("node-a").unwrap(),
2772                workspace_id: WorkspaceId::new("repo").unwrap(),
2773                worktree_id: None,
2774            },
2775            profile_id: SpawnProfileId::new("default").unwrap(),
2776            expected_profile_revision: SpawnProfileRevision::new("r1").unwrap(),
2777            overrides: SpawnOverrides {
2778                provider: SpawnOverride::Set { value: AgentId::new("codex").unwrap() },
2779                mode: SpawnOverride::Set { value: SessionMode::Pty },
2780                terminal_size: SpawnOverride::Set { value: terminal_size },
2781                prompt: SpawnOverride::Set { value: prompt.clone() },
2782                bundle_id: SpawnOverride::Clear,
2783                context_id: SpawnOverride::Clear,
2784                environment_profile_id: SpawnOverride::Clear,
2785                approval_level: None,
2786                network_allowlist: None,
2787                browser_profile_id: None,
2788            },
2789            deadline_ms: SpawnDeadlineMs::new(5_000).unwrap(),
2790            idempotency_key: SpawnIdempotencyKey::new("spawn-1").unwrap(),
2791            required_capabilities: required_capabilities.clone(),
2792        };
2793        let receipt = ResolvedSpawnReceipt {
2794            incarnation_id,
2795            session: SessionAddress {
2796                workspace_id: spec.target.workspace_id.clone(),
2797                session: SessionKey {
2798                    instance_id: AgentInstanceId(7),
2799                    generation: SessionGeneration(1),
2800                },
2801            },
2802            target: spec.target.clone(),
2803            profile_id: spec.profile_id.clone(),
2804            profile_revision: SpawnProfileRevision::new("r1").unwrap(),
2805            provider: AgentId::new("codex").unwrap(),
2806            mode: SessionMode::Pty,
2807            terminal_size,
2808            prompt: SpawnPromptMetadata::from_prompt(Some(&prompt)),
2809            bundle_id: None,
2810            bundle: None,
2811            context_id: None,
2812            context: None,
2813            environment_profile: None,
2814            deadline_ms: spec.deadline_ms,
2815            idempotency_key: spec.idempotency_key.clone(),
2816            required_capabilities,
2817            provenance: SpawnResolutionProvenance {
2818                provider: SpawnFieldProvenance::Override,
2819                mode: SpawnFieldProvenance::Override,
2820                terminal_size: SpawnFieldProvenance::Override,
2821                prompt: SpawnFieldProvenance::Override,
2822                bundle_id: SpawnFieldProvenance::Cleared,
2823                context_id: SpawnFieldProvenance::Cleared,
2824                environment_profile_id: SpawnFieldProvenance::Cleared,
2825            },
2826            harness_mcp_proxy: None,
2827        };
2828        let response = |receipt| Ok(NodeResponse::SpawnSpecAccepted { receipt });
2829        let expected = ExpectedSpawnRequest::Spec(spec.clone());
2830        assert!(validate_spawn_spec_response(
2831            Some(&expected),
2832            &response(receipt.clone()),
2833            incarnation_id,
2834        ).is_ok());
2835
2836        let mut mismatches = Vec::new();
2837        let mut changed = receipt.clone();
2838        changed.incarnation_id = NodeIncarnationId::from_bytes([8; 16]);
2839        mismatches.push(changed);
2840        let mut changed = receipt.clone();
2841        changed.target.node_id = NodeId::new("node-b").unwrap();
2842        mismatches.push(changed);
2843        let mut changed = receipt.clone();
2844        changed.profile_id = SpawnProfileId::new("other").unwrap();
2845        mismatches.push(changed);
2846        let mut changed = receipt.clone();
2847        changed.idempotency_key = SpawnIdempotencyKey::new("spawn-2").unwrap();
2848        mismatches.push(changed);
2849        let mut changed = receipt.clone();
2850        changed.deadline_ms = SpawnDeadlineMs::new(4_999).unwrap();
2851        mismatches.push(changed);
2852        let mut changed = receipt.clone();
2853        changed.required_capabilities = SpawnRequiredCapabilities::default();
2854        mismatches.push(changed);
2855        let mut changed = receipt.clone();
2856        changed.session.workspace_id = WorkspaceId::new("other").unwrap();
2857        mismatches.push(changed);
2858        let mut changed = receipt.clone();
2859        changed.provider = AgentId::new("claude").unwrap();
2860        mismatches.push(changed);
2861        let mut changed = receipt.clone();
2862        changed.prompt = SpawnPromptMetadata { present: false, byte_len: 0 };
2863        mismatches.push(changed);
2864        let mut changed = receipt.clone();
2865        changed.environment_profile = Some(
2866            gate4agent_node_protocol::ResolvedEnvironmentProfileReceipt {
2867                profile_id:
2868                    gate4agent_node_protocol::SpawnEnvironmentProfileId::new(
2869                        "local-default",
2870                    )
2871                    .unwrap(),
2872                profile_revision:
2873                    gate4agent_node_protocol::SpawnEnvironmentProfileRevision::new(
2874                        "local-default.r1",
2875                    )
2876                    .unwrap(),
2877                    network_allowlist: None,
2878                    browser_profile_id: None,
2879            },
2880        );
2881        mismatches.push(changed);
2882        let mut changed = receipt.clone();
2883        changed.bundle = Some(gate4agent_node_protocol::ResolvedBundleReceipt {
2884            id: gate4agent_node_protocol::SpawnBundleId::new("unexpected-bundle")
2885                .unwrap(),
2886            revision: gate4agent_node_protocol::SpawnBundleRevision::new(
2887                "unexpected-bundle.r1",
2888            )
2889            .unwrap(),
2890            digest: gate4agent_node_protocol::SpawnBundleDigest::new(format!(
2891                "sha256:{}",
2892                "a".repeat(64),
2893            ))
2894            .unwrap(),
2895        });
2896        mismatches.push(changed);
2897        let mut changed = receipt.clone();
2898        changed.context = Some(gate4agent_node_protocol::ResolvedContextPackReceipt {
2899            id: gate4agent_node_protocol::SpawnContextId::new("unexpected-context")
2900                .unwrap(),
2901            digest: gate4agent_node_protocol::SpawnContextDigest::new(format!(
2902                "sha256:{}",
2903                "b".repeat(64),
2904            ))
2905            .unwrap(),
2906            lineage: gate4agent_node_protocol::ContextPackLineageReceipt {
2907                source_node_id: NodeId::new("node-a").unwrap(),
2908                source_session: receipt.session.clone(),
2909                source_provider: AgentId::new("codex").unwrap(),
2910            },
2911            source_message_count: 1,
2912            retained_message_count: 1,
2913            byte_len: 16,
2914            truncated: false,
2915        });
2916        mismatches.push(changed);
2917
2918        for mismatch in mismatches {
2919            assert!(validate_spawn_spec_response(
2920                Some(&expected),
2921                &response(mismatch),
2922                incarnation_id,
2923            ).is_err());
2924        }
2925
2926        let mut environment_spec = spec;
2927        environment_spec.overrides.environment_profile_id = SpawnOverride::Set {
2928            value: gate4agent_node_protocol::SpawnEnvironmentProfileId::new(
2929                "local-default",
2930            )
2931            .unwrap(),
2932        };
2933        let environment_expected = ExpectedSpawnRequest::Spec(environment_spec);
2934        let mut environment_receipt = receipt;
2935        environment_receipt.environment_profile = Some(
2936            gate4agent_node_protocol::ResolvedEnvironmentProfileReceipt {
2937                profile_id:
2938                    gate4agent_node_protocol::SpawnEnvironmentProfileId::new(
2939                        "local-default",
2940                    )
2941                    .unwrap(),
2942                profile_revision:
2943                    gate4agent_node_protocol::SpawnEnvironmentProfileRevision::new(
2944                        "local-default.r1",
2945                    )
2946                    .unwrap(),
2947                    network_allowlist: None,
2948                    browser_profile_id: None,
2949            },
2950        );
2951        assert!(validate_spawn_spec_response(
2952            Some(&environment_expected),
2953            &response(environment_receipt.clone()),
2954            incarnation_id,
2955        )
2956        .is_ok());
2957        environment_receipt.environment_profile.as_mut().unwrap().profile_id =
2958            gate4agent_node_protocol::SpawnEnvironmentProfileId::new("other")
2959                .unwrap();
2960        assert!(validate_spawn_spec_response(
2961            Some(&environment_expected),
2962            &response(environment_receipt),
2963            incarnation_id,
2964        )
2965        .is_err());
2966    }
2967
2968    #[test]
2969    fn provider_session_index_correlation_accepts_exact_identity_and_rejects_mismatches() {
2970        let identity = gate4agent_types::ProviderSessionIdentity {
2971            key: gate4agent_types::ProviderSessionKey::SessionId,
2972            id: "native-session-1".to_owned(),
2973            transcript_path: Some(r"C:\provider\sessions\native-session-1.jsonl".to_owned()),
2974        };
2975        let expected = NodeRequest::IndexProviderSession {
2976            workspace_id: WorkspaceId::new("primary").unwrap(),
2977            provider: AgentId::new("codex").unwrap(),
2978            identity: identity.clone(),
2979            display_name: "release shepherd".to_owned(),
2980        };
2981        let NodeRequest::IndexProviderSession { workspace_id, provider, .. } = &expected else {
2982            unreachable!();
2983        };
2984        let record = gate4agent_node_protocol::ManagedSessionRecord {
2985            record_id: SessionRecordId::new("session-001").unwrap(),
2986            display_name: "release shepherd".to_owned(),
2987            provider: provider.clone(),
2988            mode: SessionMode::Pty,
2989            state: gate4agent_node_protocol::ManagedSessionState::Dormant,
2990            workspace_id: workspace_id.clone(),
2991            canonical_root: gate4agent_node_protocol::OpaqueHostPath::utf8(
2992                r"C:\repo".to_owned(),
2993            ).unwrap(),
2994            provider_session: Some(identity),
2995            active_session: None,
2996            environment_profile: None,
2997            bundle: None,
2998            context_id: None,
2999            context: None,
3000            exported_context: None,
3001            task_binding: None,
3002            created_at_unix_ms: 1,
3003            updated_at_unix_ms: 2,
3004            last_error: None,
3005        };
3006        let response = |record| Ok(NodeResponse::ProviderSessionIndexed { record });
3007
3008        assert!(validate_provider_session_index_response(
3009            Some(&expected),
3010            &response(record.clone()),
3011        ).is_ok());
3012        assert!(validate_provider_session_index_response(
3013            None,
3014            &response(record.clone()),
3015        ).is_err());
3016
3017        let mut identity_mismatch = record.clone();
3018        identity_mismatch.provider_session.as_mut().unwrap().id =
3019            "native-session-2".to_owned();
3020        assert!(validate_provider_session_index_response(
3021            Some(&expected),
3022            &response(identity_mismatch),
3023        ).is_err());
3024
3025        let mut workspace_mismatch = record.clone();
3026        workspace_mismatch.workspace_id = WorkspaceId::new("other").unwrap();
3027        assert!(validate_provider_session_index_response(
3028            Some(&expected),
3029            &response(workspace_mismatch),
3030        ).is_err());
3031
3032        let mut provider_mismatch = record;
3033        provider_mismatch.provider = AgentId::new("claude").unwrap();
3034        assert!(validate_provider_session_index_response(
3035            Some(&expected),
3036            &response(provider_mismatch),
3037        ).is_err());
3038    }
3039
3040    #[test]
3041    fn native_session_route_correlation_rejects_mismatches_before_projection() {
3042        let route = gate4agent_node_protocol::NativeSessionCatalogRoute::workspace(
3043            WorkspaceId::new("primary").unwrap(),
3044            AgentId::new("codex").unwrap(),
3045        );
3046        let selection = gate4agent_node_protocol::NativeSessionSelection {
3047            route: route.clone(),
3048            catalog_revision: 7,
3049            recent_cutoff_unix_ms: 70,
3050            selection_id: "selection-7".to_owned(),
3051        };
3052        let catalog = NodeRequest::CatalogNativeSessions {
3053            route: route.clone(),
3054            limit: 10,
3055        };
3056        let catalog_response = Ok(NodeResponse::NativeSessionsCataloged {
3057            route: route.clone(),
3058            entries: Vec::new(),
3059            summary: None,
3060        });
3061        assert!(validate_native_session_response(Some(&catalog), &catalog_response).is_ok());
3062        let mismatched_route = gate4agent_node_protocol::NativeSessionCatalogRoute::workspace(
3063            WorkspaceId::new("other").unwrap(),
3064            AgentId::new("codex").unwrap(),
3065        );
3066        assert!(validate_native_session_response(
3067            Some(&catalog),
3068            &Ok(NodeResponse::NativeSessionsCataloged {
3069                route: mismatched_route,
3070                entries: Vec::new(),
3071                summary: None,
3072            }),
3073        )
3074        .is_err());
3075
3076        let page = NodeRequest::PageNativeSessions {
3077            route: route.clone(),
3078            window: gate4agent_node_protocol::NativeSessionCatalogWindow::Recent,
3079            catalog_revision: 7,
3080            recent_cutoff_unix_ms: 70,
3081            after_selection_id: None,
3082            limit: 10,
3083        };
3084        let page_response = |revision| {
3085            Ok(NodeResponse::NativeSessionsPaged {
3086                route: route.clone(),
3087                page: gate4agent_node_protocol::NativeSessionCatalogPage {
3088                    window: gate4agent_node_protocol::NativeSessionCatalogWindow::Recent,
3089                    revision,
3090                    entries: Vec::new(),
3091                    next_after_selection_id: None,
3092                    remaining_count: 0,
3093                    has_more: false,
3094                },
3095            })
3096        };
3097        assert!(validate_native_session_response(Some(&page), &page_response(7)).is_ok());
3098        assert!(validate_native_session_response(Some(&page), &page_response(8)).is_err());
3099
3100        let preview = NodeRequest::PreviewNativeSession {
3101            selection: selection.clone(),
3102            message_limit: 10,
3103        };
3104        let preview_response = |selection| {
3105            Ok(NodeResponse::NativeSessionPreviewed {
3106                selection,
3107                preview: gate4agent_node_protocol::SessionRecordPreview {
3108                    title: None,
3109                    modified_at_unix_ms: None,
3110                    model: None,
3111                    message_count: 0,
3112                    message_count_exact: true,
3113                    completed_turn_count: None,
3114                    total_tokens: None,
3115                    truncated: false,
3116                    messages: Vec::new(),
3117                },
3118            })
3119        };
3120        assert!(validate_native_session_response(
3121            Some(&preview),
3122            &preview_response(selection.clone()),
3123        )
3124        .is_ok());
3125        let mut mismatched_selection = selection.clone();
3126        mismatched_selection.catalog_revision = 8;
3127        assert!(validate_native_session_response(
3128            Some(&preview),
3129            &preview_response(mismatched_selection),
3130        )
3131        .is_err());
3132
3133        let index = NodeRequest::IndexNativeSession {
3134            selection: selection.clone(),
3135            display_name: "Indexed".to_owned(),
3136        };
3137        let record = gate4agent_node_protocol::ManagedSessionRecord {
3138            record_id: SessionRecordId::new("session-007").unwrap(),
3139            display_name: "Indexed".to_owned(),
3140            provider: route.provider.clone(),
3141            mode: SessionMode::Pty,
3142            state: gate4agent_node_protocol::ManagedSessionState::Dormant,
3143            workspace_id: route.workspace_id.clone().unwrap(),
3144            canonical_root: gate4agent_node_protocol::OpaqueHostPath::utf8(
3145                r"C:\repo".to_owned(),
3146            )
3147            .unwrap(),
3148            provider_session: None,
3149            active_session: None,
3150            environment_profile: None,
3151            bundle: None,
3152            context_id: None,
3153            context: None,
3154            exported_context: None,
3155            task_binding: None,
3156            created_at_unix_ms: 1,
3157            updated_at_unix_ms: 2,
3158            last_error: None,
3159        };
3160        assert!(validate_native_session_response(
3161            Some(&index),
3162            &Ok(NodeResponse::NativeSessionIndexed {
3163                selection: selection.clone(),
3164                record: record.clone(),
3165            }),
3166        )
3167        .is_ok());
3168        assert!(validate_native_session_response(
3169            Some(&index),
3170            &Ok(NodeResponse::ProviderSessionIndexed {
3171                record: record.clone(),
3172            }),
3173        )
3174        .is_err());
3175        let mut wrong_echo = selection.clone();
3176        wrong_echo.catalog_revision = 8;
3177        assert!(validate_native_session_response(
3178            Some(&index),
3179            &Ok(NodeResponse::NativeSessionIndexed {
3180                selection: wrong_echo,
3181                record: record.clone(),
3182            }),
3183        )
3184        .is_err());
3185        let mut wrong_provider = record.clone();
3186        wrong_provider.provider = AgentId::new("claude").unwrap();
3187        assert!(validate_native_session_response(
3188            Some(&index),
3189            &Ok(NodeResponse::NativeSessionIndexed {
3190                selection: selection.clone(),
3191                record: wrong_provider.clone(),
3192            }),
3193        )
3194        .is_err());
3195        let mut wrong_workspace = record.clone();
3196        wrong_workspace.workspace_id = WorkspaceId::new("other").unwrap();
3197        assert!(validate_native_session_response(
3198            Some(&index),
3199            &Ok(NodeResponse::NativeSessionIndexed {
3200                selection: selection.clone(),
3201                record: wrong_workspace,
3202            }),
3203        )
3204        .is_err());
3205
3206        let external = NodeRequest::IndexNativeSession {
3207            selection: gate4agent_node_protocol::NativeSessionSelection {
3208                route: gate4agent_node_protocol::NativeSessionCatalogRoute::unregistered(
3209                    AgentId::new("codex").unwrap(),
3210                ),
3211                ..selection
3212            },
3213            display_name: "External".to_owned(),
3214        };
3215        let NodeRequest::IndexNativeSession {
3216            selection: external_selection,
3217            ..
3218        } = &external else {
3219            unreachable!();
3220        };
3221        assert!(validate_native_session_response(
3222            Some(&external),
3223            &Ok(NodeResponse::NativeSessionIndexed {
3224                selection: external_selection.clone(),
3225                record: gate4agent_node_protocol::ManagedSessionRecord {
3226                    provider: AgentId::new("codex").unwrap(),
3227                    ..wrong_provider
3228                },
3229            }),
3230        )
3231        .is_err());
3232    }
3233
3234    #[test]
3235    fn managed_spawn_receipt_correlation_covers_legacy_and_v2_requests() {
3236        let incarnation_id = NodeIncarnationId::from_bytes([7; 16]);
3237        let spec = SpawnSpec {
3238            target: SpawnTarget {
3239                node_id: NodeId::new("node-a").unwrap(),
3240                workspace_id: WorkspaceId::new("repo").unwrap(),
3241                worktree_id: None,
3242            },
3243            profile_id: SpawnProfileId::new("default").unwrap(),
3244            expected_profile_revision:
3245                SpawnProfileRevision::new("default.r1").unwrap(),
3246            overrides: SpawnOverrides::default(),
3247            deadline_ms: SpawnDeadlineMs::new(5_000).unwrap(),
3248            idempotency_key: SpawnIdempotencyKey::new("managed-1").unwrap(),
3249            required_capabilities: SpawnRequiredCapabilities::default(),
3250        };
3251        let managed = ManagedWorktreeSpawnRequest {
3252            spawn_spec: spec.clone(),
3253            worktree_profile_id: WorktreeProfileId::new("review").unwrap(),
3254        };
3255        let workspace_id = WorkspaceId::new("managed-a").unwrap();
3256        let spawn = ResolvedSpawnReceipt {
3257            incarnation_id,
3258            session: SessionAddress {
3259                workspace_id: workspace_id.clone(),
3260                session: SessionKey {
3261                    instance_id: AgentInstanceId(8),
3262                    generation: SessionGeneration(1),
3263                },
3264            },
3265            target: SpawnTarget {
3266                node_id: spec.target.node_id.clone(),
3267                workspace_id: spec.target.workspace_id.clone(),
3268                worktree_id: Some(workspace_id.clone()),
3269            },
3270            profile_id: spec.profile_id.clone(),
3271            profile_revision: SpawnProfileRevision::new("default.r1").unwrap(),
3272            provider: AgentId::new("claude").unwrap(),
3273            mode: SessionMode::Pty,
3274            terminal_size: TerminalSize { rows: 24, columns: 80 },
3275            prompt: SpawnPromptMetadata { present: false, byte_len: 0 },
3276            bundle_id: None,
3277            bundle: None,
3278            context_id: None,
3279            context: None,
3280            environment_profile: None,
3281            deadline_ms: spec.deadline_ms,
3282            idempotency_key: spec.idempotency_key.clone(),
3283            required_capabilities: SpawnRequiredCapabilities::default(),
3284            provenance: SpawnResolutionProvenance {
3285                provider: SpawnFieldProvenance::Profile,
3286                mode: SpawnFieldProvenance::Profile,
3287                terminal_size: SpawnFieldProvenance::Profile,
3288                prompt: SpawnFieldProvenance::Profile,
3289                bundle_id: SpawnFieldProvenance::Profile,
3290                context_id: SpawnFieldProvenance::Profile,
3291                environment_profile_id: SpawnFieldProvenance::Profile,
3292            },
3293            harness_mcp_proxy: None,
3294        };
3295        let receipt = ManagedWorktreeSpawnReceipt {
3296            spawn,
3297            lease: managed_lease(
3298                "lease-a",
3299                "managed-a",
3300                ManagedWorktreeLeaseState::InUse,
3301            ),
3302        };
3303        let expected = ExpectedSpawnRequest::ManagedV1(managed.clone());
3304        let response = |receipt| Ok(NodeResponse::ManagedWorktreeSpawnAccepted { receipt });
3305        assert!(validate_spawn_spec_response(
3306            Some(&expected),
3307            &response(receipt.clone()),
3308            incarnation_id,
3309        )
3310        .is_ok());
3311
3312        let mut mismatches = Vec::new();
3313        let mut changed = receipt.clone();
3314        changed.lease.state = ManagedWorktreeLeaseState::Ready;
3315        mismatches.push(changed);
3316        let mut changed = receipt.clone();
3317        changed.lease.cleanup_failure = Some(ManagedWorktreeCleanupFailure::Busy);
3318        mismatches.push(changed);
3319        let mut changed = receipt.clone();
3320        changed.lease.active_session_count = 0;
3321        mismatches.push(changed);
3322        let mut changed = receipt.clone();
3323        changed.lease.profile_id = WorktreeProfileId::new("other").unwrap();
3324        mismatches.push(changed);
3325        let mut changed = receipt.clone();
3326        changed.lease.source_workspace_id = WorkspaceId::new("other").unwrap();
3327        mismatches.push(changed);
3328
3329        for mismatch in mismatches {
3330            assert!(validate_spawn_spec_response(
3331                Some(&expected),
3332                &response(mismatch),
3333                incarnation_id,
3334            )
3335            .is_err());
3336        }
3337
3338        let mut legacy_revision = receipt.clone();
3339        legacy_revision.lease.profile_revision =
3340            WorktreeProfileRevision::new("review.r2").unwrap();
3341        assert!(validate_spawn_spec_response(
3342            Some(&expected),
3343            &response(legacy_revision),
3344            incarnation_id,
3345        )
3346        .is_ok());
3347
3348        let v2_request = ManagedWorktreeSpawnRequestV2 {
3349            spawn_spec: managed.spawn_spec,
3350            worktree_profile_id: managed.worktree_profile_id,
3351            expected_profile_revision: WorktreeProfileRevision::new("review.r1").unwrap(),
3352        };
3353        let routed = NodeRequest::SpawnManagedWorktreeV2 {
3354            request: v2_request.clone(),
3355        };
3356        let captured_expected = expected_spawn_request(&routed).unwrap();
3357        let ExpectedSpawnRequest::ManagedV2(captured_request) = &captured_expected else {
3358            panic!("V2 managed spawn request was not captured separately");
3359        };
3360        assert_eq!(captured_request, &v2_request);
3361        assert!(validate_spawn_spec_response(
3362            Some(&captured_expected),
3363            &response(receipt.clone()),
3364            incarnation_id,
3365        )
3366        .is_ok());
3367
3368        let mut wrong_revision = receipt.clone();
3369        wrong_revision.lease.profile_revision =
3370            WorktreeProfileRevision::new("review.r2").unwrap();
3371        assert!(validate_spawn_spec_response(
3372            Some(&ExpectedSpawnRequest::ManagedV2(v2_request.clone())),
3373            &response(wrong_revision),
3374            incarnation_id,
3375        )
3376        .is_err());
3377        assert!(validate_spawn_spec_response(
3378            Some(&ExpectedSpawnRequest::ManagedV2(v2_request)),
3379            &Ok(NodeResponse::SpawnSpecAccepted {
3380                receipt: receipt.spawn,
3381            }),
3382            incarnation_id,
3383        )
3384        .is_err());
3385    }
3386
3387    #[test]
3388    fn windows_runtime_default_control_endpoint_is_exact_valid_and_distinct_from_a_node() {
3389        assert_eq!(DEFAULT_C2_CONTROL_ENDPOINT, r"\\.\pipe\gate4agent-c2");
3390        validate_control_endpoint(DEFAULT_C2_CONTROL_ENDPOINT).unwrap();
3391        let node = C2NodeConfig::new(
3392            NodeId::new("node-a").unwrap(),
3393            r"\\.\pipe\gate4agent-node",
3394            "safe-token",
3395        )
3396        .unwrap();
3397        let config = C2Config::new(
3398            "127.0.0.1:0".parse().unwrap(),
3399            "safe-token",
3400            vec![node],
3401        )
3402        .unwrap();
3403        assert_eq!(config.control_endpoint, DEFAULT_C2_CONTROL_ENDPOINT);
3404        assert!(!config.nodes[0]
3405            .endpoint
3406            .eq_ignore_ascii_case(&config.control_endpoint));
3407
3408        let conflicting_node = C2NodeConfig::new(
3409            NodeId::new("node-b").unwrap(),
3410            DEFAULT_C2_CONTROL_ENDPOINT,
3411            "safe-token",
3412        )
3413        .unwrap();
3414        assert!(matches!(
3415            C2Config::new(
3416                "127.0.0.1:0".parse().unwrap(),
3417                "safe-token",
3418                vec![conflicting_node],
3419            ),
3420            Err(C2ConfigError::ControlEndpointConflict)
3421        ));
3422    }
3423
3424    #[test]
3425    fn terminal_only_sequence_holes_do_not_mark_c2_partial() {
3426        use gate4agent_node_protocol::{NodeEvent, WorkspaceId};
3427        let event = |sequence| NodeEventEnvelope { sequence, event: NodeEvent::WorkspaceRemoved { workspace_id: WorkspaceId::new("work").unwrap() } };
3428        assert_eq!(validate_events(4, 7, 1, &[event(6)]), Vec::<GapKind>::new());
3429        assert_eq!(validate_events(4, 7, 1, &[]), Vec::<GapKind>::new());
3430        assert_eq!(validate_events(4, 7, 1, &[event(7), event(6)]), vec![GapKind::NonContiguousEvents]);
3431        assert_eq!(validate_events(7, 6, 1, &[]), vec![GapKind::CursorRegression]);
3432        assert_eq!(validate_resync(4, 6, 4, 1, &[]), vec![GapKind::CursorRegression]);
3433    }
3434
3435    #[test]
3436    fn real_durable_eviction_marks_c2_history_evicted() {
3437        use gate4agent_node_protocol::{NodeEvent, WorkspaceId};
3438        let event = |sequence| NodeEventEnvelope {
3439            sequence,
3440            event: NodeEvent::WorkspaceRemoved {
3441                workspace_id: WorkspaceId::new("work").unwrap(),
3442            },
3443        };
3444        assert_eq!(
3445            validate_events(4, 7, 6, &[event(6)]),
3446            vec![GapKind::HistoryEvicted],
3447        );
3448        assert_eq!(validate_events(5, 7, 6, &[event(6)]), Vec::<GapKind>::new());
3449    }
3450
3451    #[test]
3452    fn live_event_gap_preserves_cursor_and_resync_rules() {
3453        use gate4agent_node_protocol::{NodeEvent, WorkspaceId};
3454        let event = |sequence, event| NodeEventEnvelope { sequence, event };
3455        let removed = || NodeEvent::WorkspaceRemoved {
3456            workspace_id: WorkspaceId::new("work").unwrap(),
3457        };
3458
3459        assert_eq!(live_event_gap(4, &event(5, removed())), None);
3460        assert_eq!(
3461            live_event_gap(4, &event(4, removed())),
3462            Some(GapKind::CursorRegression),
3463        );
3464        assert_eq!(
3465            live_event_gap(4, &event(7, removed())),
3466            Some(GapKind::NonContiguousEvents),
3467        );
3468        assert_eq!(
3469            live_event_gap(4, &event(5, NodeEvent::ResyncRequired {
3470                oldest_available_sequence: 3,
3471            })),
3472            Some(GapKind::HistoryEvicted),
3473        );
3474    }
3475
3476    #[test]
3477    fn harness_mcp_transient_live_and_pending_projection_preserves_cursor_without_replay() {
3478        let node_id = NodeId::new("node-a").unwrap();
3479        let cursor = NodeCursor {
3480            incarnation_id: NodeIncarnationId::from_bytes([7; 16]),
3481            sequence: 41,
3482        };
3483        let envelope = NodeEventEnvelope {
3484            sequence: 0,
3485            event: NodeEvent::HarnessMcpReadCall {
3486                reservation_id: HarnessMcpReservationId::new(
3487                    format!("hmcpres_{}", "a".repeat(24)),
3488                ).unwrap(),
3489                activation_digest: HarnessMcpActivationDigest::new(
3490                    format!("sha256:{}", "b".repeat(64)),
3491                ).unwrap(),
3492                record_id: SessionRecordId::new("session-001").unwrap(),
3493                session: SessionAddress {
3494                    workspace_id: WorkspaceId::new("repo").unwrap(),
3495                    session: SessionKey {
3496                        instance_id: AgentInstanceId(8),
3497                        generation: SessionGeneration(1),
3498                    },
3499                },
3500                call_id: HarnessMcpCallId::new(
3501                    format!("hmcpcall_{}", "c".repeat(24)),
3502                ).unwrap(),
3503                request: HarnessMcpOpaquePayloadV1 {
3504                    content_type: HarnessMcpContentTypeV1::HarnessReadRequestJsonV1,
3505                    body: br#"{"kind":"context-get"}"#.to_vec(),
3506                },
3507                deadline_unix_ms: u64::MAX,
3508            },
3509        };
3510
3511        let live = routed_transient_node_event(&node_id, cursor, &envelope).unwrap();
3512        let pending = routed_transient_node_event(&node_id, cursor, &envelope).unwrap();
3513        assert_eq!(live, pending);
3514        assert_eq!(live.cursor, cursor);
3515        assert!(matches!(live.event, C2NodeEvent::HarnessMcpReadCall { .. }));
3516        assert!(routed_recovered_node_event(
3517            &node_id,
3518            cursor.incarnation_id,
3519            &envelope,
3520        ).is_none());
3521        assert_eq!(cursor.sequence, 41);
3522    }
3523
3524    fn agent_stream_test_address() -> SessionAddress {
3525        SessionAddress {
3526            workspace_id: WorkspaceId::new("repo").unwrap(),
3527            session: SessionKey {
3528                instance_id: AgentInstanceId(8),
3529                generation: SessionGeneration(1),
3530            },
3531        }
3532    }
3533
3534    fn agent_stream_test_envelope(sequence: u64, source_sequence: u64) -> NodeEventEnvelope {
3535        NodeEventEnvelope {
3536            sequence,
3537            event: NodeEvent::AgentStream {
3538                address: agent_stream_test_address(),
3539                chunk: AgentStreamChunkV1 {
3540                    source_sequence,
3541                    kind: AgentStreamChunkKindV1::Text {
3542                        text: format!("chunk-{source_sequence}"),
3543                        is_delta: true,
3544                    },
3545                },
3546            },
3547        }
3548    }
3549
3550    #[test]
3551    fn agent_stream_chunk_publishes_unconditionally_and_only_advances_cursor_forward() {
3552        let node_id = NodeId::new("node-a").unwrap();
3553        let mut cursor = NodeCursor {
3554            incarnation_id: NodeIncarnationId::from_bytes([7; 16]),
3555            sequence: 674,
3556        };
3557
3558        // A non-`AgentStream` envelope is not this function's to route.
3559        let control = NodeEventEnvelope {
3560            sequence: 675,
3561            event: NodeEvent::WorkspaceRemoved {
3562                workspace_id: WorkspaceId::new("work").unwrap(),
3563            },
3564        };
3565        assert!(route_agent_stream_event(&node_id, &mut cursor, &control).is_none());
3566        assert_eq!(cursor.sequence, 674);
3567
3568        // A chunk one past the cursor is published, and its OWN cursor
3569        // field on the wire reads the cursor as it stood before this
3570        // chunk (mirroring `HarnessMcpReadCall`'s convention exactly) --
3571        // then the live cursor moves forward to make room for it.
3572        let forward = agent_stream_test_envelope(675, 22);
3573        let routed = route_agent_stream_event(&node_id, &mut cursor, &forward).unwrap();
3574        assert_eq!(routed.cursor.sequence, 674);
3575        assert!(matches!(
3576            &routed.event,
3577            C2NodeEvent::AgentStream { chunk, .. } if chunk.source_sequence == 22
3578        ));
3579        assert_eq!(cursor.sequence, 675);
3580
3581        // A chunk that arrives (or, after a durable resync already carried
3582        // the cursor past it, is only now examined) BEHIND the cursor is
3583        // still published -- chunks promise no resync, so there is nothing
3584        // to recover if this one were dropped instead -- but it must never
3585        // rewind the cursor a genuinely later `Control` envelope already
3586        // advanced past.
3587        cursor.sequence = 711;
3588        let late = agent_stream_test_envelope(677, 23);
3589        let routed = route_agent_stream_event(&node_id, &mut cursor, &late).unwrap();
3590        assert_eq!(routed.cursor.sequence, 711);
3591        assert!(matches!(
3592            &routed.event,
3593            C2NodeEvent::AgentStream { chunk, .. } if chunk.source_sequence == 23
3594        ));
3595        assert_eq!(cursor.sequence, 711, "a late chunk must never rewind the cursor");
3596    }
3597
3598    #[test]
3599    fn agent_stream_burst_survives_a_durable_cursor_that_already_ran_past_it() {
3600        // Reproduces the measured live defect's mechanism at its worst: the
3601        // node's connection loop can drain its durable channel to
3602        // exhaustion across several ticks before this dedicated channel
3603        // gets any budget at all, so a whole burst's `Control` envelopes
3604        // can reach this relay, and carry `cursor` forward, before a
3605        // single one of the SAME burst's interleaved `AgentStream` chunks
3606        // is even looked at. `route_agent_stream_event` must publish every
3607        // one of them regardless.
3608        let node_id = NodeId::new("node-a").unwrap();
3609        let mut cursor = NodeCursor {
3610            incarnation_id: NodeIncarnationId::from_bytes([9; 16]),
3611            sequence: 0,
3612        };
3613        let chunk_count = 200_u64;
3614
3615        for turn in 1..=chunk_count {
3616            // Durable (`Control`) processing advancing the cursor is the
3617            // pre-existing, unchanged path (`live_event_gap` and its
3618            // resync repair, exercised by this module's other tests) --
3619            // simulated here only by its end effect on `cursor`, since
3620            // this test is about the `AgentStream` side's own contract.
3621            cursor.sequence = turn * 2;
3622        }
3623
3624        let mut recovered = Vec::new();
3625        for turn in 1..=chunk_count {
3626            let envelope = agent_stream_test_envelope(turn * 2 - 1, turn);
3627            let routed = route_agent_stream_event(&node_id, &mut cursor, &envelope)
3628                .expect("every chunk in the burst must be published, however far the durable cursor already ran past its sequence");
3629            recovered.push(routed);
3630        }
3631
3632        assert_eq!(recovered.len(), chunk_count as usize);
3633        for (turn, routed) in (1..=chunk_count).zip(recovered.iter()) {
3634            assert!(matches!(
3635                &routed.event,
3636                C2NodeEvent::AgentStream { chunk, .. } if chunk.source_sequence == turn
3637            ));
3638        }
3639        assert_eq!(
3640            cursor.sequence,
3641            chunk_count * 2,
3642            "a whole burst of stale chunks must never rewind the cursor the durable side already advanced",
3643        );
3644    }
3645
3646    #[test]
3647    fn config_rejects_header_injection_duplicate_nodes_and_non_loopback() {
3648        let id = NodeId::new("node-a").unwrap();
3649        assert!(matches!(C2NodeConfig::new(id.clone(), r"\\.\pipe\a", "bad\r\ntoken"), Err(C2ConfigError::InvalidToken)));
3650        let node = C2NodeConfig::new(id, r"\\.\pipe\a", "safe-token").unwrap();
3651        assert!(matches!(C2Config::new("0.0.0.0:0".parse().unwrap(), "safe", vec![node.clone()]), Err(C2ConfigError::NonLoopback(_))));
3652        assert!(matches!(C2Config::new("127.0.0.1:0".parse().unwrap(), "safe", vec![node.clone(), node]), Err(C2ConfigError::DuplicateNode)));
3653        let first = C2NodeConfig::new(NodeId::new("node-a").unwrap(), r"\\.\pipe\same", "safe").unwrap();
3654        let second = C2NodeConfig::new(NodeId::new("node-b").unwrap(), r"\\.\pipe\same", "safe").unwrap();
3655        assert!(matches!(C2Config::new("127.0.0.1:0".parse().unwrap(), "safe", vec![first, second]), Err(C2ConfigError::DuplicateEndpoint)));
3656        let oversized = format!(r"\\.\pipe\{}", "x".repeat(MAX_C2_ENDPOINT_BYTES));
3657        assert!(matches!(C2NodeConfig::new(NodeId::new("node-c").unwrap(), oversized, "safe"), Err(C2ConfigError::InvalidEndpoint(_))));
3658    }
3659
3660    #[test]
3661    fn durable_session_mutations_require_controller_and_use_bounded_deadlines() {
3662        let record_id = SessionRecordId::new("session-001").unwrap();
3663        let rename = NodeRequest::RenameSessionRecord {
3664            record_id: record_id.clone(),
3665            display_name: "release shepherd".to_owned(),
3666        };
3667        let resume = NodeRequest::ResumeSessionRecord {
3668            record_id: record_id.clone(),
3669            terminal_size: TerminalSize { rows: 40, columns: 120 },
3670            initial_prompt: None,
3671        };
3672        let forget = NodeRequest::ForgetSessionRecord { record_id };
3673
3674        assert!(!is_read_only_request(&rename));
3675        assert!(!is_read_only_request(&resume));
3676        assert!(!is_read_only_request(&forget));
3677        assert_eq!(node_request_deadline(&rename), Duration::from_secs(5));
3678        assert_eq!(node_request_deadline(&resume), Duration::from_secs(35));
3679        assert!(node_request_deadline(&resume) > MANAGED_RESUME_SETTLE_DEADLINE);
3680        assert_eq!(
3681            node_request_deadline(&resume) - MANAGED_RESUME_SETTLE_DEADLINE,
3682            NODE_REQUEST_IO_HEADROOM,
3683        );
3684        let started = Instant::now();
3685        let relay_deadline = relay_request_deadline(&resume, started).unwrap();
3686        assert_eq!(
3687            request_budget(&resume, Some(relay_deadline), started),
3688            Duration::from_secs(35),
3689        );
3690        assert_eq!(
3691            request_budget(
3692                &resume,
3693                Some(relay_deadline),
3694                started + Duration::from_secs(5),
3695            ),
3696            MANAGED_RESUME_SETTLE_DEADLINE,
3697        );
3698        assert_eq!(
3699            request_budget(
3700                &resume,
3701                Some(relay_deadline),
3702                started + Duration::from_secs(35),
3703            ),
3704            Duration::ZERO,
3705        );
3706        assert_eq!(node_request_deadline(&forget), Duration::from_secs(5));
3707    }
3708
3709    #[test]
3710    fn workspace_file_reads_are_read_only_with_five_second_deadline() {
3711        let request = NodeRequest::ReadWorkspaceFile {
3712            workspace_id: gate4agent_node_protocol::WorkspaceId::new("primary").unwrap(),
3713            path: gate4agent_node_protocol::RepositoryPath::utf8(
3714                "src/lib.rs".to_owned(),
3715            ).unwrap(),
3716        };
3717
3718        assert!(is_read_only_request(&request));
3719        assert_eq!(node_request_deadline(&request), Duration::from_secs(5));
3720        assert!(relay_request_deadline(&request, Instant::now()).is_none());
3721
3722    }
3723
3724    #[test]
3725    fn workspace_entry_create_relay_has_headroom_and_preserves_semantic_timeout() {
3726        let workspace_id = gate4agent_node_protocol::WorkspaceId::new("primary").unwrap();
3727        let file_path = gate4agent_node_protocol::RepositoryPath::utf8(
3728            "src/new.rs".to_owned(),
3729        ).unwrap();
3730        let create_file = NodeRequest::CreateWorkspaceFile {
3731            workspace_id: workspace_id.clone(),
3732            path: file_path.clone(),
3733        };
3734        assert!(!is_read_only_request(&create_file));
3735        assert_eq!(
3736            node_request_deadline(&create_file),
3737            WORKSPACE_ENTRY_CREATE_RELAY_DEADLINE,
3738        );
3739        assert!(
3740            node_request_deadline(&create_file)
3741                > WORKSPACE_ENTRY_CREATE_NODE_SEMANTIC_DEADLINE,
3742        );
3743        assert!(relay_request_deadline(&create_file, Instant::now()).is_none());
3744
3745        let propagated = relay_node_failure(&NodeClientError::Node(
3746            gate4agent_node_protocol::NodeFailure {
3747                code: NodeFailureCode::RepositoryEntryCreateTimedOut,
3748                message: "private node timeout detail".to_owned(),
3749            },
3750        )).unwrap();
3751        assert_eq!(
3752            propagated.code,
3753            NodeFailureCode::RepositoryEntryCreateTimedOut,
3754        );
3755        assert_eq!(propagated.message, "repository entry creation timed out");
3756
3757        let created_file = gate4agent_node_protocol::WorkspaceFileRead {
3758            workspace_id: workspace_id.clone(),
3759            path: file_path,
3760            content: gate4agent_node_protocol::WorkspaceFileContent::Utf8 {
3761                text: String::new(),
3762                byte_len: 0,
3763            },
3764            revision: Some(
3765                gate4agent_node_protocol::WorkspaceFileRevision::new(
3766                    "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3767                        .to_owned(),
3768                )
3769                .unwrap(),
3770            ),
3771        };
3772        assert!(validate_workspace_content_response(
3773            Some(&create_file),
3774            &Ok(NodeResponse::WorkspaceFileCreated {
3775                file: created_file.clone(),
3776            }),
3777        ).is_ok());
3778        let mut wrong_content = created_file;
3779        wrong_content.content = gate4agent_node_protocol::WorkspaceFileContent::Utf8 {
3780            text: "unexpected".to_owned(),
3781            byte_len: 10,
3782        };
3783        assert!(validate_workspace_content_response(
3784            Some(&create_file),
3785            &Ok(NodeResponse::WorkspaceFileCreated { file: wrong_content }),
3786        ).is_err());
3787
3788        let directory_path = gate4agent_node_protocol::RepositoryPath::utf8(
3789            "src/new".to_owned(),
3790        ).unwrap();
3791        let create_directory = NodeRequest::CreateWorkspaceDirectory {
3792            workspace_id: workspace_id.clone(),
3793            path: directory_path.clone(),
3794        };
3795        assert!(!is_read_only_request(&create_directory));
3796        assert_eq!(
3797            node_request_deadline(&create_directory),
3798            WORKSPACE_ENTRY_CREATE_RELAY_DEADLINE,
3799        );
3800        assert!(validate_workspace_content_response(
3801            Some(&create_directory),
3802            &Ok(NodeResponse::WorkspaceDirectoryCreated {
3803                workspace_id,
3804                entry: gate4agent_node_protocol::WorkspaceEntry {
3805                    relative_path: directory_path,
3806                    kind: gate4agent_node_protocol::WorkspaceEntryKind::Directory,
3807                },
3808            }),
3809        ).is_ok());
3810        assert!(validate_workspace_content_response(
3811            Some(&create_directory),
3812            &Ok(NodeResponse::Accepted),
3813        ).is_err());
3814    }
3815
3816    #[test]
3817    fn native_session_catalog_is_lease_free_read_only() {
3818        let route = gate4agent_node_protocol::NativeSessionCatalogRoute::workspace(
3819            gate4agent_node_protocol::WorkspaceId::new("primary").unwrap(),
3820            gate4agent_types::AgentId::new("codex").unwrap(),
3821        );
3822        let request = NodeRequest::CatalogNativeSessions {
3823            route: route.clone(),
3824            limit: 8,
3825        };
3826        assert!(is_read_only_request(&request));
3827        assert_eq!(
3828            node_request_deadline(&request),
3829            NATIVE_SESSION_REQUEST_DEADLINE,
3830        );
3831        assert!(relay_request_deadline(&request, Instant::now()).is_none());
3832
3833        let page = NodeRequest::PageNativeSessions {
3834            route: route.clone(),
3835            window: gate4agent_types::NativeSessionCatalogWindow::Older,
3836            catalog_revision: 7,
3837            recent_cutoff_unix_ms: 8,
3838            after_selection_id: Some("hist_selection_1".to_owned()),
3839            limit: 8,
3840        };
3841        assert!(is_read_only_request(&page));
3842        assert_eq!(node_request_deadline(&page), NATIVE_SESSION_REQUEST_DEADLINE);
3843        assert!(relay_request_deadline(&page, Instant::now()).is_none());
3844
3845        let preview = NodeRequest::PreviewNativeSession {
3846            selection: gate4agent_node_protocol::NativeSessionSelection {
3847                route,
3848                catalog_revision: 7,
3849                recent_cutoff_unix_ms: 8,
3850                selection_id: "hist_selection_1".to_owned(),
3851            },
3852            message_limit: 12,
3853        };
3854        assert!(is_read_only_request(&preview));
3855        assert_eq!(
3856            node_request_deadline(&preview),
3857            NATIVE_SESSION_REQUEST_DEADLINE,
3858        );
3859        assert!(relay_request_deadline(&preview, Instant::now()).is_none());
3860
3861        let record_preview = NodeRequest::PreviewSessionRecord {
3862            record_id: gate4agent_node_protocol::SessionRecordId::new("record-1").unwrap(),
3863            message_limit: 12,
3864        };
3865        assert!(is_read_only_request(&record_preview));
3866        assert_eq!(
3867            node_request_deadline(&record_preview),
3868            NATIVE_SESSION_REQUEST_DEADLINE,
3869        );
3870        assert!(relay_request_deadline(&record_preview, Instant::now()).is_none());
3871    }
3872
3873    #[test]
3874    fn standalone_workspace_creation_is_controller_mutation_with_worktree_deadline() {
3875        let request = NodeRequest::CreateStandaloneWorkspace {
3876            workspace_id: gate4agent_node_protocol::WorkspaceId::new("standalone").unwrap(),
3877            root: gate4agent_node_protocol::OpaqueHostPath::utf8(
3878                r"C:\standalone".to_owned(),
3879            ).unwrap(),
3880            initial_branch: Some("main".to_owned()),
3881        };
3882
3883        assert!(!is_read_only_request(&request));
3884        assert_eq!(node_request_deadline(&request), Duration::from_secs(240));
3885        assert!(relay_request_deadline(&request, Instant::now()).is_none());
3886    }
3887
3888    #[test]
3889    fn unsupported_node_capability_is_correlated_without_offline_classification() {
3890        let error = NodeClientError::UnsupportedCapability(
3891            "workspace-file-read-v1-private-detail".to_owned(),
3892        );
3893        let failure = relay_node_failure(&error)
3894            .expect("unsupported node capability must remain an in-band node failure");
3895
3896        assert_eq!(failure.code, NodeFailureCode::UnsupportedCapability);
3897        assert_eq!(failure.message, "required capability unavailable");
3898        assert!(!failure.message.contains("private-detail"));
3899        assert!(relay_node_failure(&NodeClientError::Io(io::Error::new(
3900            io::ErrorKind::BrokenPipe,
3901            "transport closed",
3902        ))).is_none());
3903    }
3904
3905    async fn raw_request(request: Vec<u8>, status: StatusResponse) -> Vec<u8> {
3906        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3907        let address = listener.local_addr().unwrap();
3908        let (_status_tx, status_rx) = watch::channel(Arc::new(status));
3909        let server = tokio::spawn(async move {
3910            let (stream, _) = listener.accept().await.unwrap();
3911            serve_http(stream, "api-token", Duration::from_secs(1), &status_rx).await.unwrap();
3912        });
3913        let mut stream = TcpStream::connect(address).await.unwrap();
3914        stream.write_all(&request).await.unwrap();
3915        let mut response = Vec::new();
3916        stream.read_to_end(&mut response).await.unwrap();
3917        server.await.unwrap();
3918        response
3919    }
3920
3921    fn empty_status(ready: bool) -> StatusResponse {
3922        StatusResponse { api_version: C2_API_VERSION, ready, observed_at_unix_ms: 0, nodes: BTreeMap::new() }
3923    }
3924
3925    #[tokio::test]
3926    async fn http_api_enforces_initializing_auth_method_path_and_header_bounds() {
3927        let initializing = raw_request(b"GET /ready HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), empty_status(false)).await;
3928        assert!(initializing.starts_with(b"HTTP/1.1 503 Service Unavailable\r\n"));
3929        assert!(String::from_utf8_lossy(&initializing).contains("\"ready\":false"));
3930
3931        let unauthorized = raw_request(b"GET /status HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), empty_status(true)).await;
3932        assert!(unauthorized.starts_with(b"HTTP/1.1 401 Unauthorized\r\n"));
3933        let method = raw_request(b"POST /health HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), empty_status(true)).await;
3934        assert!(method.starts_with(b"HTTP/1.1 405 Method Not Allowed\r\n"));
3935        let missing = raw_request(b"GET /missing HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), empty_status(true)).await;
3936        assert!(missing.starts_with(b"HTTP/1.1 404 Not Found\r\n"));
3937
3938        let mut oversized = b"GET /health HTTP/1.1\r\nX-Fill: ".to_vec();
3939        oversized.extend(std::iter::repeat(b'x').take(HEADER_LIMIT_BYTES));
3940        oversized.extend_from_slice(b"\r\n\r\n");
3941        let rejected = raw_request(oversized, empty_status(true)).await;
3942        assert!(rejected.starts_with(b"HTTP/1.1 413 Payload Too Large\r\n"));
3943    }
3944
3945    #[tokio::test]
3946    async fn inventory_state_transitions_offline_to_stale_parked_and_recovers() {
3947        let node_id = NodeId::new("node-a").unwrap();
3948        let mut nodes = BTreeMap::new();
3949        nodes.insert(node_id.clone(), ObservedNode {
3950            endpoint: r"\\.\pipe\a".to_owned(), transport_label: "windows-named-pipe".to_owned(),
3951            transport: NodeTransportState::Offline, freshness: NodeFreshness::Unavailable,
3952            cursor: None, inventory: None, last_attempt_unix_ms: None, last_success_unix_ms: None,
3953            consecutive_failures: 0, last_error: None, gaps: Vec::new(), gaps_truncated: 0,
3954        });
3955        let initial = Arc::new(StatusResponse { api_version: C2_API_VERSION, ready: false, observed_at_unix_ms: unix_ms(), nodes });
3956        let (status_tx, mut status_rx) = watch::channel(initial);
3957        let (ingress_tx, ingress_rx) = mpsc::channel(4);
3958        let (shutdown_tx, shutdown_rx) = watch::channel(false);
3959        let owner = tokio::spawn(inventory_owner(1, Duration::from_millis(10), ingress_rx, status_tx, shutdown_rx));
3960        let snapshot = NodeSnapshot {
3961            node_id: node_id.clone(),
3962            enabled_providers: Vec::new(),
3963            provider_runtime_statuses: crate::protocol::ProviderRuntimeStatuses::default(),
3964            workspaces: Vec::new(),
3965            session_records: Vec::new(),
3966            managed_worktrees: Vec::new(),
3967            launch_inventory: None,
3968            agent_progress: Vec::new(),
3969        };
3970        let cursor = NodeCursor { incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([1; 16]), sequence: 0 };
3971        let old_manifest = ProviderContractManifest {
3972            provider_contracts: vec![crate::protocol::ProviderContractSupport {
3973                provider: agent("codex"),
3974                revision: crate::protocol::ProviderContractRevision::new("old-contract").unwrap(),
3975            }],
3976            provider_adapter_contracts: vec![crate::protocol::ProviderAdapterContractSupport {
3977                provider: agent("codex"),
3978                family: crate::protocol::AdapterFamily::PtySemantic,
3979                adapter_id: crate::protocol::AdapterId::new("codex").unwrap(),
3980                revision: crate::protocol::AdapterContractRevision::new("old-adapter").unwrap(),
3981            }],
3982        };
3983        ingress_tx.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result: AttemptResult::Connected {
3984            cursor,
3985            snapshot: snapshot.clone(),
3986            gaps: Vec::new(),
3987            provider_contract_manifest: old_manifest,
3988        } }).await.unwrap();
3989        status_rx.changed().await.unwrap();
3990        assert_eq!(status_rx.borrow().nodes[&node_id].freshness, NodeFreshness::Fresh);
3991        assert_eq!(
3992            status_rx.borrow().nodes[&node_id].inventory.as_ref().unwrap()
3993                .provider_contracts[0].revision.as_str(),
3994            "old-contract",
3995        );
3996
3997        let failure = || AttemptResult::Failure { error: SanitizedError { category: C2ErrorCategory::Transport, message: "node transport unavailable".to_owned() }, hard: false };
3998        ingress_tx.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result: failure() }).await.unwrap();
3999        status_rx.changed().await.unwrap();
4000        assert_eq!(status_rx.borrow().nodes[&node_id].transport, NodeTransportState::Offline);
4001        timeout(Duration::from_secs(1), async {
4002            loop {
4003                status_rx.changed().await.unwrap();
4004                if status_rx.borrow().nodes[&node_id].freshness == NodeFreshness::Stale { break; }
4005            }
4006        }).await.unwrap();
4007        for _ in 0..4 {
4008            ingress_tx.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result: failure() }).await.unwrap();
4009            status_rx.changed().await.unwrap();
4010        }
4011        assert_eq!(status_rx.borrow().nodes[&node_id].transport, NodeTransportState::Parked);
4012        let replacement_manifest = ProviderContractManifest {
4013            provider_contracts: vec![crate::protocol::ProviderContractSupport {
4014                provider: agent("claude"),
4015                revision: crate::protocol::ProviderContractRevision::new("new-contract").unwrap(),
4016            }],
4017            provider_adapter_contracts: Vec::new(),
4018        };
4019        let replacement_cursor = NodeCursor {
4020            incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([2; 16]),
4021            sequence: 0,
4022        };
4023        ingress_tx.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result: AttemptResult::Connected {
4024            cursor: replacement_cursor,
4025            snapshot,
4026            gaps: Vec::new(),
4027            provider_contract_manifest: replacement_manifest,
4028        } }).await.unwrap();
4029        status_rx.changed().await.unwrap();
4030        assert_eq!(status_rx.borrow().nodes[&node_id].transport, NodeTransportState::Online);
4031        assert_eq!(status_rx.borrow().nodes[&node_id].freshness, NodeFreshness::Fresh);
4032        let recovered_inventory = status_rx.borrow().nodes[&node_id].inventory.as_ref().unwrap().clone();
4033        assert_eq!(recovered_inventory.provider_contracts.len(), 1);
4034        assert_eq!(recovered_inventory.provider_contracts[0].provider, agent("claude"));
4035        assert_eq!(recovered_inventory.provider_contracts[0].revision.as_str(), "new-contract");
4036        assert!(recovered_inventory.provider_adapter_contracts.is_empty());
4037        ingress_tx.send(Attempt {
4038            node_id: node_id.clone(),
4039            at_unix_ms: unix_ms(),
4040            result: AttemptResult::Connected {
4041                cursor: NodeCursor {
4042                    incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([3; 16]),
4043                    sequence: 0,
4044                },
4045                snapshot: NodeSnapshot {
4046                    node_id: node_id.clone(),
4047                    enabled_providers: Vec::new(),
4048                    provider_runtime_statuses: crate::protocol::ProviderRuntimeStatuses::default(),
4049                    workspaces: Vec::new(),
4050                    session_records: Vec::new(),
4051                    managed_worktrees: Vec::new(),
4052                    launch_inventory: None,
4053                    agent_progress: Vec::new(),
4054                },
4055                gaps: Vec::new(),
4056                provider_contract_manifest: ProviderContractManifest::default(),
4057            },
4058        }).await.unwrap();
4059        status_rx.changed().await.unwrap();
4060        let unpublished = status_rx.borrow().nodes[&node_id].inventory.as_ref().unwrap().clone();
4061        assert!(unpublished.provider_contracts.is_empty());
4062        assert!(unpublished.provider_adapter_contracts.is_empty());
4063        shutdown_tx.send(true).unwrap();
4064        owner.await.unwrap().unwrap();
4065    }
4066
4067    fn runtime_statuses(
4068        provider: gate4agent_node_protocol::AgentId,
4069        version: &str,
4070    ) -> crate::protocol::ProviderRuntimeStatuses {
4071        crate::protocol::ProviderRuntimeStatuses::new([
4072            crate::protocol::ProviderRuntimeStatus::raw_passthrough(
4073                provider,
4074                Some(crate::protocol::ProviderRuntimeVersion::new(version).unwrap()),
4075            ),
4076        ])
4077        .unwrap()
4078    }
4079
4080    async fn runtime_inventory_owner() -> (
4081        NodeId,
4082        mpsc::Sender<Attempt>,
4083        watch::Receiver<Arc<StatusResponse>>,
4084        watch::Sender<bool>,
4085        tokio::task::JoinHandle<io::Result<()>>,
4086    ) {
4087        let node_id = NodeId::new("runtime-node").unwrap();
4088        let nodes = BTreeMap::from([(
4089            node_id.clone(),
4090            ObservedNode {
4091                endpoint: r"\\.\pipe\runtime-node".to_owned(),
4092                transport_label: "windows-named-pipe".to_owned(),
4093                transport: NodeTransportState::Offline,
4094                freshness: NodeFreshness::Unavailable,
4095                cursor: None,
4096                inventory: None,
4097                last_attempt_unix_ms: None,
4098                last_success_unix_ms: None,
4099                consecutive_failures: 0,
4100                last_error: None,
4101                gaps: Vec::new(),
4102                gaps_truncated: 0,
4103            },
4104        )]);
4105        let initial = Arc::new(StatusResponse {
4106            api_version: C2_API_VERSION,
4107            ready: false,
4108            observed_at_unix_ms: unix_ms(),
4109            nodes,
4110        });
4111        let (status_tx, status_rx) = watch::channel(initial);
4112        let (ingress_tx, ingress_rx) = mpsc::channel(4);
4113        let (shutdown_tx, shutdown_rx) = watch::channel(false);
4114        let owner = tokio::spawn(inventory_owner(
4115            1,
4116            Duration::from_secs(1),
4117            ingress_rx,
4118            status_tx,
4119            shutdown_rx,
4120        ));
4121        (node_id, ingress_tx, status_rx, shutdown_tx, owner)
4122    }
4123
4124    #[tokio::test]
4125    async fn incarnation_change_replaces_runtime_status() {
4126        let (node_id, ingress, mut status, shutdown, owner) = runtime_inventory_owner().await;
4127        for (incarnation, provider, version) in [
4128            (1, agent("claude"), "1.0.0"),
4129            (2, agent("codex"), "2.0.0"),
4130        ] {
4131            ingress
4132                .send(Attempt {
4133                    node_id: node_id.clone(),
4134                    at_unix_ms: unix_ms(),
4135                    result: AttemptResult::Connected {
4136                        cursor: NodeCursor {
4137                            incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([
4138                                incarnation;
4139                                16
4140                            ]),
4141                            sequence: 0,
4142                        },
4143                        snapshot: NodeSnapshot {
4144                            node_id: node_id.clone(),
4145                            enabled_providers: vec![provider.clone()],
4146                            provider_runtime_statuses: runtime_statuses(provider, version),
4147                            workspaces: Vec::new(),
4148                            session_records: Vec::new(),
4149                            managed_worktrees: Vec::new(),
4150                            launch_inventory: None,
4151                            agent_progress: Vec::new(),
4152                        },
4153                        gaps: Vec::new(),
4154                        provider_contract_manifest: ProviderContractManifest::default(),
4155                    },
4156                })
4157                .await
4158                .unwrap();
4159            status.changed().await.unwrap();
4160        }
4161        let current_status = status.borrow();
4162        let statuses = &current_status.nodes[&node_id]
4163            .inventory
4164            .as_ref()
4165            .unwrap()
4166            .provider_runtime_statuses;
4167        assert_eq!(statuses.as_slice().len(), 1);
4168        assert_eq!(
4169            statuses.as_slice()[0].provider(),
4170            &agent("codex"),
4171        );
4172        assert_eq!(statuses.as_slice()[0].version().unwrap().as_str(), "2.0.0");
4173        shutdown.send(true).unwrap();
4174        owner.await.unwrap().unwrap();
4175    }
4176
4177    #[tokio::test]
4178    async fn incarnation_change_without_snapshot_clears_dynamic_inventory() {
4179        let (node_id, ingress, mut status, shutdown, owner) = runtime_inventory_owner().await;
4180        ingress
4181            .send(Attempt {
4182                node_id: node_id.clone(),
4183                at_unix_ms: unix_ms(),
4184                result: AttemptResult::Connected {
4185                    cursor: NodeCursor {
4186                        incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([
4187                            3; 16
4188                        ]),
4189                        sequence: 0,
4190                    },
4191                    snapshot: NodeSnapshot {
4192                        node_id: node_id.clone(),
4193                        enabled_providers: vec![agent("claude")],
4194                        provider_runtime_statuses: runtime_statuses(
4195                            agent("claude"),
4196                            "3.0.0",
4197                        ),
4198                        workspaces: Vec::new(),
4199                        session_records: Vec::new(),
4200                        managed_worktrees: Vec::new(),
4201                        launch_inventory: None,
4202                        agent_progress: Vec::new(),
4203                    },
4204                    gaps: Vec::new(),
4205                    provider_contract_manifest: ProviderContractManifest::default(),
4206                },
4207            })
4208            .await
4209            .unwrap();
4210        status.changed().await.unwrap();
4211        ingress
4212            .send(Attempt {
4213                node_id: node_id.clone(),
4214                at_unix_ms: unix_ms(),
4215                result: AttemptResult::Cursor {
4216                    cursor: NodeCursor {
4217                        incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([
4218                            4; 16
4219                        ]),
4220                        sequence: 0,
4221                    },
4222                    gaps: vec![GapKind::IncarnationChanged],
4223                    managed_worktree_events: Vec::new(),
4224                },
4225            })
4226            .await
4227            .unwrap();
4228        status.changed().await.unwrap();
4229        assert!(status.borrow().nodes[&node_id]
4230            .inventory
4231            .as_ref()
4232            .unwrap()
4233            .provider_runtime_statuses
4234            .is_empty());
4235        shutdown.send(true).unwrap();
4236        owner.await.unwrap().unwrap();
4237    }
4238
4239    #[test]
4240    fn managed_worktree_inventory_events_are_exact_bounded_and_incarnation_fenced() {
4241        let mut inventory = SlimNodeInventory::from_snapshot(&NodeSnapshot {
4242            node_id: NodeId::new("node-a").unwrap(),
4243            enabled_providers: Vec::new(),
4244            provider_runtime_statuses: crate::protocol::ProviderRuntimeStatuses::default(),
4245            workspaces: Vec::new(),
4246            session_records: Vec::new(),
4247            managed_worktrees: vec![managed_lease(
4248                "lease-a",
4249                "managed-a",
4250                ManagedWorktreeLeaseState::Ready,
4251            )],
4252            launch_inventory: None,
4253            agent_progress: Vec::new(),
4254        });
4255        apply_managed_worktree_cursor(
4256            Some(&mut inventory),
4257            false,
4258            &[
4259                NodeEvent::ManagedWorktreeUpserted {
4260                    lease: managed_lease(
4261                        "lease-b",
4262                        "managed-b",
4263                        ManagedWorktreeLeaseState::Ready,
4264                    ),
4265                },
4266                NodeEvent::ManagedWorktreeUpserted {
4267                    lease: managed_lease(
4268                        "lease-a",
4269                        "managed-a",
4270                        ManagedWorktreeLeaseState::InUse,
4271                    ),
4272                },
4273            ],
4274        );
4275        assert_eq!(inventory.managed_worktree_count, 2);
4276        assert_eq!(inventory.managed_worktrees[0].lease_id.as_str(), "lease-a");
4277        assert_eq!(
4278            inventory.managed_worktrees[0].state,
4279            ManagedWorktreeLeaseState::InUse,
4280        );
4281
4282        apply_managed_worktree_cursor(
4283            Some(&mut inventory),
4284            false,
4285            &[NodeEvent::ManagedWorktreeRemoved {
4286                lease_id: ManagedWorktreeLeaseId::new("lease-a").unwrap(),
4287            }],
4288        );
4289        assert_eq!(inventory.managed_worktree_count, 1);
4290        assert_eq!(inventory.managed_worktrees[0].lease_id.as_str(), "lease-b");
4291
4292        apply_managed_worktree_cursor(
4293            Some(&mut inventory),
4294            true,
4295            &[NodeEvent::ManagedWorktreeUpserted {
4296                lease: managed_lease(
4297                    "lease-c",
4298                    "managed-c",
4299                    ManagedWorktreeLeaseState::Ready,
4300                ),
4301            }],
4302        );
4303        assert!(inventory.managed_worktrees.is_empty());
4304        assert_eq!(inventory.managed_worktree_count, 0);
4305        assert!(!inventory.managed_worktrees_truncated);
4306    }
4307}
4308
4309#[cfg(test)]
4310mod relay_deadline_tests {
4311    use super::*;
4312
4313    /// The defect this pins, stated as an invariant rather than a number:
4314    /// the relay's bound on a request must never sit below the budget the
4315    /// node itself is allowed to spend answering it. It sat at five
4316    /// seconds against the node's eight-to-eleven, so every inspection of
4317    /// a workspace large enough to use its own allowance timed out at the
4318    /// relay -- and a relay timeout drops the NODE, not the request, which
4319    /// took every read behind it down in a loop.
4320    #[test]
4321    fn workspace_inspection_relay_bound_clears_the_nodes_own_maximum() {
4322        // The node's own ceiling (`WORKSPACE_INSPECTION_TIME_BUDGET_MS_MAX`
4323        // in the node crate) restated here rather than imported: these are
4324        // separate crates by design, and the point of the test is that the
4325        // two numbers must be compared by a human when either moves.
4326        const NODE_INSPECTION_MAX: Duration = Duration::from_millis(11_000);
4327        assert!(
4328            WORKSPACE_INSPECTION_RELAY_DEADLINE > NODE_INSPECTION_MAX,
4329            "relay bound {:?} must exceed the node's own inspection maximum {:?}",
4330            WORKSPACE_INSPECTION_RELAY_DEADLINE,
4331            NODE_INSPECTION_MAX,
4332        );
4333    }
4334}
4335
4336#[cfg(test)]
4337mod control_tunnel_listen_tests {
4338    use super::*;
4339
4340    fn config() -> C2Config {
4341        let node = C2NodeConfig::new(
4342            NodeId::new("node-a").unwrap(),
4343            if cfg!(unix) { "/tmp/gate4agent-node-a.sock" } else { r"\\.\pipe\gate4agent-node-a" },
4344            "token-token",
4345        ).unwrap();
4346        C2Config::new("127.0.0.1:18321".parse().unwrap(), "api-token", vec![node]).unwrap()
4347    }
4348
4349    #[test]
4350    fn control_tunnel_rejects_unspecified_and_accepts_tunnel_address() {
4351        let bad: SocketAddr = "0.0.0.0:27441".parse().unwrap();
4352        assert!(matches!(
4353            config().with_control_tunnel_listen(bad),
4354            Err(C2ConfigError::InvalidControlTunnel(_))
4355        ));
4356        let ok: SocketAddr = "10.88.1.2:27441".parse().unwrap();
4357        let config = config().with_control_tunnel_listen(ok).unwrap();
4358        assert_eq!(config.control_tunnel_listen, Some(ok));
4359        assert!(!config.control_endpoint.is_empty());
4360    }
4361}