Skip to main content

hatchery_c2/
runtime.rs

1use crate::protocol::{
2    C2ErrorCategory, C2NodeEvent, C2NodeFailure, C2NodeResponse, C2RelayFailure, C2RelayFailureCode, GapKind, HealthResponse, NodeCursor, NodeFreshness, NodeGap, NodeId,
3    NodeIncarnationId, NodeRequest, ResolvedSpawnReceipt, RoutedNodeEvent, RoutedNodeResponse,
4    ManagedWorktreeLeaseState, ManagedWorktreeSpawnRequest, ManagedWorktreeSpawnRequestV2,
5    SpawnOverride, SpawnSpec,
6    NodeTransportState, ObservedNode, ProviderAdapterContractSupport, ProviderContractSupport,
7    ReadyResponse, SanitizedError, SlimNodeInventory, StatusResponse,
8    C2_API_VERSION, C2_PROVIDER_CONTRACT_MANIFEST_CAPABILITY,
9    MAX_C2_GAPS_PER_NODE, MAX_C2_NODES,
10};
11#[cfg(windows)]
12use crate::protocol::MAX_C2_ENDPOINT_BYTES;
13use gate4agent_node_protocol::{
14    ClientRole, FrameError, NegotiatedNodeCompatibility, NodeEvent, NodeEventEnvelope, NodeFailureCode,
15    NodeResponse, NodeSnapshot, ServerFrame,
16};
17use gate4agent_node_wire::{read_call_home_announce, LocalNodeClient, NodeClientError};
18use std::collections::{BTreeMap, BTreeSet};
19use std::io;
20use std::net::SocketAddr;
21use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, SystemTime, UNIX_EPOCH};
24#[cfg(unix)]
25use std::path::{Path, PathBuf};
26#[cfg(unix)]
27use std::os::unix::fs::PermissionsExt;
28use thiserror::Error;
29use tokio::io::{AsyncReadExt, AsyncWriteExt};
30use tokio::net::{TcpListener, TcpStream};
31use tokio::sync::{mpsc, oneshot, watch, Semaphore};
32use tokio::task::{JoinHandle, JoinSet};
33use tokio::time::{sleep, timeout, Instant};
34
35const MANAGED_RESUME_SETTLE_DEADLINE: Duration = Duration::from_secs(30);
36const NODE_REQUEST_IO_HEADROOM: Duration = Duration::from_secs(5);
37const NATIVE_SESSION_REQUEST_DEADLINE: Duration = Duration::from_secs(35);
38const WORKSPACE_ENTRY_CREATE_NODE_SEMANTIC_DEADLINE: Duration = Duration::from_secs(5);
39const WORKSPACE_ENTRY_CREATE_RELAY_DEADLINE: Duration =
40    WORKSPACE_ENTRY_CREATE_NODE_SEMANTIC_DEADLINE.saturating_add(NODE_REQUEST_IO_HEADROOM);
41
42const HEADER_LIMIT_BYTES: usize = 16 * 1024;
43const OPERATOR_BODY_LIMIT_BYTES: usize = 16 * 1024;
44const MAX_HTTP_CONNECTIONS: usize = 16;
45const RESPONSE_BODY_LIMIT_BYTES: usize = 8 * 1024 * 1024;
46
47mod control;
48
49#[cfg(windows)]
50pub const DEFAULT_C2_CONTROL_ENDPOINT: &str = r"\\.\pipe\gate4agent-c2";
51#[cfg(unix)]
52pub const DEFAULT_C2_CONTROL_ENDPOINT: &str = "gate4agent-c2.sock";
53
54#[cfg(windows)]
55pub fn default_c2_control_endpoint() -> Result<String, C2ConfigError> {
56    Ok(DEFAULT_C2_CONTROL_ENDPOINT.to_owned())
57}
58
59#[cfg(unix)]
60pub fn default_c2_control_endpoint() -> Result<String, C2ConfigError> {
61    let root = unix_runtime_root()?;
62    let directory = root.join("gate4agent");
63    let endpoint = directory.join(DEFAULT_C2_CONTROL_ENDPOINT);
64    validate_unix_endpoint(&endpoint).map_err(|_| C2ConfigError::InvalidControlEndpoint)?;
65    Ok(endpoint.to_string_lossy().into_owned())
66}
67
68#[cfg(unix)]
69fn unix_runtime_root() -> Result<PathBuf, C2ConfigError> {
70    if let Some(root) = std::env::var_os("XDG_RUNTIME_DIR").filter(|value| !value.is_empty()) {
71        let root = PathBuf::from(root);
72        if root.is_absolute() { return Ok(root); }
73    }
74    let home = std::env::var_os("HOME").filter(|value| !value.is_empty())
75        .ok_or_else(|| C2ConfigError::RuntimeEndpoint(
76            "neither absolute XDG_RUNTIME_DIR nor HOME is available".to_owned(),
77        ))?;
78    let home = PathBuf::from(home);
79    if !home.is_absolute() {
80        return Err(C2ConfigError::RuntimeEndpoint("HOME is not absolute".to_owned()));
81    }
82    Ok(home.join(".gate4agent").join("run"))
83}
84
85#[derive(Clone)]
86pub struct C2NodeConfig {
87    pub node_id: NodeId,
88    pub endpoint: String,
89    route: C2NodeRoute,
90    token: String,
91}
92
93#[derive(Clone, Copy, Debug, Eq, PartialEq)]
94enum C2NodeRoute {
95    Local,
96    SshForwardedLoopback(SocketAddr),
97    /// This relay does not dial the node; the node dials the relay.
98    ///
99    /// For a node the relay has no way to reach -- behind NAT, in a
100    /// container with no published port, on a machine with no listener the
101    /// relay is allowed to open. The protocol roles are unchanged: the
102    /// node is still the server, this relay is still the client asking it
103    /// for things. Only the direction of the TCP connection moves, which
104    /// is why the whole of this variant's implementation is "wait for a
105    /// socket instead of opening one" and then the identical handshake.
106    CallHome,
107}
108
109impl C2NodeConfig {
110    pub fn new(node_id: NodeId, endpoint: impl Into<String>, token: impl Into<String>) -> Result<Self, C2ConfigError> {
111        let endpoint = endpoint.into();
112        let token = token.into();
113        let (endpoint, route) = parse_node_endpoint(&endpoint)
114            .ok_or_else(|| C2ConfigError::InvalidEndpoint(node_id.clone()))?;
115        validate_token(&token)?;
116        Ok(Self { node_id, endpoint, route, token })
117    }
118
119    fn transport_label(&self) -> &'static str {
120        match self.route {
121            C2NodeRoute::Local => local_transport_label(),
122            C2NodeRoute::SshForwardedLoopback(_) => "ssh-forwarded-loopback",
123            C2NodeRoute::CallHome => "call-home",
124        }
125    }
126}
127
128#[derive(Clone, Copy, Debug)]
129pub struct C2Timings {
130    pub poll_interval: Duration,
131    pub fresh_for: Duration,
132    pub attempt_deadline: Duration,
133    pub transient_backoffs: [Duration; 5],
134    pub parked_backoff: Duration,
135    pub http_io_deadline: Duration,
136}
137
138impl Default for C2Timings {
139    fn default() -> Self {
140        Self {
141            poll_interval: Duration::from_millis(250),
142            fresh_for: Duration::from_secs(10),
143            attempt_deadline: Duration::from_secs(5),
144            transient_backoffs: [
145                Duration::from_millis(500), Duration::from_secs(1), Duration::from_secs(2),
146                Duration::from_secs(4), Duration::from_secs(8),
147            ],
148            parked_backoff: Duration::from_secs(30),
149            http_io_deadline: Duration::from_secs(3),
150        }
151    }
152}
153
154#[derive(Clone)]
155pub struct C2Config {
156    pub api_listen: SocketAddr,
157    pub control_endpoint: String,
158    /// Optional TCP control accept bound to the C2 tunnel address.
159    /// The unix `control_endpoint` listener is always started as well.
160    pub control_tunnel_listen: Option<SocketAddr>,
161    api_token: String,
162    pub nodes: Vec<C2NodeConfig>,
163    /// Where nodes that cannot be dialled come to announce themselves.
164    /// `None` unless at least one node is configured `accept`.
165    pub node_listen: Option<SocketAddr>,
166    pub timings: C2Timings,
167    /// Node enrollment (one-time code, neutral status, runtime node add). `None` keeps the
168    /// fixed `--node` set and token authentication only.
169    pub enrollment: Option<crate::enroll_service::EnrollConfig>,
170}
171
172impl C2Config {
173    /// Turn on enrollment. Requires a node-listen address (checked by `validate_call_home`).
174    pub fn with_enrollment(mut self, enrollment: crate::enroll_service::EnrollConfig) -> Self {
175        self.enrollment = Some(enrollment);
176        self
177    }
178
179    pub fn new(api_listen: SocketAddr, api_token: impl Into<String>, nodes: Vec<C2NodeConfig>) -> Result<Self, C2ConfigError> {
180        let api_token = api_token.into();
181        if !api_listen.ip().is_loopback() { return Err(C2ConfigError::NonLoopback(api_listen)); }
182        validate_token(&api_token)?;
183        if nodes.len() > MAX_C2_NODES { return Err(C2ConfigError::NodeCount(nodes.len())); }
184        let mut ids = BTreeSet::new();
185        let mut endpoints = BTreeSet::new();
186        if nodes.iter().any(|node| !ids.insert(node.node_id.clone())) { return Err(C2ConfigError::DuplicateNode); }
187        // Call-home nodes are exempt: they name no endpoint, so every one
188        // of them carries the same `accept` and the uniqueness this check
189        // enforces is `node_id`'s job for them (already done above). The
190        // check exists to stop two dialled nodes pointing at one socket,
191        // which cannot happen to a node that is never dialled.
192        if nodes
193            .iter()
194            .filter(|node| node.route != C2NodeRoute::CallHome)
195            .any(|node| !endpoints.insert(endpoint_key(&node.endpoint)))
196        {
197            return Err(C2ConfigError::DuplicateEndpoint);
198        }
199        let control_endpoint = default_c2_control_endpoint()?;
200        if nodes.iter().any(|node| endpoints_equal(&node.endpoint, &control_endpoint)) {
201            return Err(C2ConfigError::ControlEndpointConflict);
202        }
203        Ok(Self {
204            api_listen,
205            control_endpoint,
206            control_tunnel_listen: None,
207            api_token,
208            nodes,
209            node_listen: None,
210            timings: C2Timings::default(),
211            enrollment: None,
212        })
213    }
214
215    /// Binds the address nodes call in on.
216    ///
217    /// Loopback, or one concrete C2 WireGuard tunnel address (same rule as
218    /// [`Self::with_control_tunnel_listen`]). The node wire authenticates both
219    /// ends but encrypts nothing, so `0.0.0.0` and other unspecified binds are
220    /// refused: off-tunnel plaintext would expose terminal contents. When the
221    /// listen address is the C2 tunnel IP, kernel WireGuard encrypts the path
222    /// and either side may dial.
223    pub fn with_node_listen(mut self, node_listen: SocketAddr) -> Result<Self, C2ConfigError> {
224        // Concrete non-unspecified + nonzero port. Loopback stays valid; WG
225        // tunnel addresses (e.g. 10.88.1.2:PORT) are valid for kit peers.
226        if node_listen.port() == 0 || node_listen.ip().is_unspecified() {
227            return Err(C2ConfigError::NonLoopbackNodeListen(node_listen));
228        }
229        if node_listen == self.api_listen {
230            return Err(C2ConfigError::NodeListenConflict);
231        }
232        if self.control_tunnel_listen == Some(node_listen) {
233            return Err(C2ConfigError::NonLoopbackNodeListen(node_listen));
234        }
235        self.node_listen = Some(node_listen);
236        Ok(self)
237    }
238
239    /// Every node that waits to be called needs somewhere to call, so a
240    /// config that asks for one without the other is refused rather than
241    /// started into a relay that can never reach that node. Checked at
242    /// startup, not at connect time, because the failure is total and
243    /// permanent -- there is nothing to retry.
244    pub fn validate_call_home(&self) -> Result<(), C2ConfigError> {
245        if self.nodes.is_empty() && self.enrollment.is_none() {
246            return Err(C2ConfigError::NodeCount(0));
247        }
248        let waiting = self.nodes.iter().find(|node| node.route == C2NodeRoute::CallHome);
249        match (waiting, self.node_listen) {
250            (Some(node), None) => Err(C2ConfigError::CallHomeWithoutListener(node.node_id.clone())),
251            (None, None) if self.enrollment.is_some() => Err(C2ConfigError::EnrollmentWithoutListener),
252            _ => Ok(()),
253        }
254    }
255
256    pub fn with_timings(mut self, timings: C2Timings) -> Self {
257        self.timings = timings;
258        self
259    }
260
261    pub fn with_control_endpoint(mut self, endpoint: impl Into<String>) -> Result<Self, C2ConfigError> {
262        let endpoint = endpoint.into();
263        validate_control_endpoint(&endpoint)?;
264        if self.nodes.iter().any(|node| endpoints_equal(&node.endpoint, &endpoint)) {
265            return Err(C2ConfigError::ControlEndpointConflict);
266        }
267        self.control_endpoint = endpoint;
268        Ok(self)
269    }
270
271    /// TCP door for an HQ WireGuard client. Bound to one concrete address
272    /// (the address on the C2 tunnel), never `0.0.0.0` or port 0. Does not
273    /// replace the unix control listener.
274    pub fn with_control_tunnel_listen(mut self, listen: SocketAddr) -> Result<Self, C2ConfigError> {
275        if listen.port() == 0 || listen.ip().is_unspecified() {
276            return Err(C2ConfigError::InvalidControlTunnel(listen));
277        }
278        if listen == self.api_listen || self.node_listen == Some(listen) {
279            return Err(C2ConfigError::InvalidControlTunnel(listen));
280        }
281        self.control_tunnel_listen = Some(listen);
282        Ok(self)
283    }
284}
285
286fn validate_control_endpoint(endpoint: &str) -> Result<(), C2ConfigError> {
287    if !valid_local_endpoint(endpoint) {
288        return Err(C2ConfigError::InvalidControlEndpoint);
289    }
290    Ok(())
291}
292
293fn parse_node_endpoint(endpoint: &str) -> Option<(String, C2NodeRoute)> {
294    // The one assignment that names no address, because there is nothing
295    // to address: this node will arrive on the call-home listener under
296    // its own name. Spelled as a word rather than an empty value so a
297    // config reader can tell "waits to be called" from "somebody forgot to
298    // fill this in".
299    if endpoint == "accept" {
300        return Some((endpoint.to_owned(), C2NodeRoute::CallHome));
301    }
302    if let Some(authority) = endpoint.strip_prefix("tcp://") {
303        let address = authority.parse::<SocketAddr>().ok()?;
304        let is_exact_loopback = match address.ip() {
305            std::net::IpAddr::V4(ip) => ip == std::net::Ipv4Addr::LOCALHOST,
306            std::net::IpAddr::V6(ip) => ip == std::net::Ipv6Addr::LOCALHOST,
307        };
308        if !is_exact_loopback || address.port() == 0 {
309            return None;
310        }
311        return Some((format!("tcp://{address}"), C2NodeRoute::SshForwardedLoopback(address)));
312    }
313    if endpoint.contains("://") || !valid_local_endpoint(endpoint) {
314        return None;
315    }
316    Some((endpoint.to_owned(), C2NodeRoute::Local))
317}
318
319#[cfg(windows)]
320fn valid_local_endpoint(endpoint: &str) -> bool {
321    endpoint.starts_with(r"\\.\pipe\") && endpoint.len() > r"\\.\pipe\".len()
322        && endpoint.len() <= MAX_C2_ENDPOINT_BYTES
323}
324
325#[cfg(unix)]
326fn valid_local_endpoint(endpoint: &str) -> bool {
327    validate_unix_endpoint(Path::new(endpoint)).is_ok()
328}
329
330#[cfg(unix)]
331fn validate_unix_endpoint(endpoint: &Path) -> Result<(), ()> {
332    const MAX_UNIX_ENDPOINT_BYTES: usize = 103;
333    use std::os::unix::ffi::OsStrExt;
334
335    if !endpoint.is_absolute() || endpoint.file_name().is_none()
336        || endpoint.as_os_str().as_bytes().len() > MAX_UNIX_ENDPOINT_BYTES
337    {
338        return Err(());
339    }
340    Ok(())
341}
342
343#[cfg(windows)]
344fn endpoint_key(endpoint: &str) -> String {
345    if endpoint.starts_with("tcp://") { endpoint.to_owned() } else { endpoint.to_ascii_lowercase() }
346}
347
348#[cfg(unix)]
349fn endpoint_key(endpoint: &str) -> String { endpoint.to_owned() }
350
351fn endpoints_equal(left: &str, right: &str) -> bool { endpoint_key(left) == endpoint_key(right) }
352
353fn validate_token(token: &str) -> Result<(), C2ConfigError> {
354    if token.is_empty() || token.len() > 4096 || !token.bytes().all(|byte| matches!(byte, 0x21..=0x7e)) {
355        return Err(C2ConfigError::InvalidToken);
356    }
357    Ok(())
358}
359
360#[derive(Debug, Error)]
361pub enum C2ConfigError {
362    #[error("C2 tokens must contain 1..=4096 visible ASCII bytes without whitespace")]
363    InvalidToken,
364    #[cfg_attr(windows, error("node '{0}' requires a bounded Windows named pipe or exact loopback TCP endpoint"))]
365    #[cfg_attr(unix, error("node '{0}' requires a bounded local socket or exact loopback TCP endpoint"))]
366    InvalidEndpoint(NodeId),
367    #[error("C2 API listen address must be loopback: {0}")]
368    NonLoopback(SocketAddr),
369    #[error("C2 requires 1..=64 configured nodes; received {0}")]
370    NodeCount(usize),
371    #[error("C2 node IDs must be unique")]
372    DuplicateNode,
373    #[error("C2 node endpoints must be unique")]
374    DuplicateEndpoint,
375    #[cfg_attr(windows, error("C2 control endpoint must be a bounded local Windows named pipe"))]
376    #[cfg_attr(unix, error("C2 control endpoint must be a bounded local endpoint"))]
377    InvalidControlEndpoint,
378    #[error("enrollment needs a --node-listen address for the nodes to call home on")]
379    EnrollmentWithoutListener,
380    #[error("C2 control endpoint must not equal a configured node endpoint")]
381    ControlEndpointConflict,
382    #[error("C2 control tunnel listen must be a concrete non-unspecified address with a nonzero port, not shared with another listener: {0}")]
383    InvalidControlTunnel(SocketAddr),
384    #[error("C2 node call-home listen must be loopback or a concrete C2 tunnel address with a nonzero port (not 0.0.0.0, not shared with another listener): {0}")]
385    NonLoopbackNodeListen(SocketAddr),
386    #[error("C2 node call-home listen address must not equal the API listen address")]
387    NodeListenConflict,
388    #[error("node '{0}' waits to be called but no --node-listen address was configured")]
389    CallHomeWithoutListener(NodeId),
390    #[cfg(unix)]
391    #[error("C2 default runtime endpoint is unavailable: {0}")]
392    RuntimeEndpoint(String),
393}
394
395type RelayResult = Result<RoutedNodeResponse, C2RelayFailure>;
396
397enum RelayCommand {
398    Request {
399        operator_connection_id: u64,
400        expected_incarnation_id: NodeIncarnationId,
401        request: NodeRequest,
402        reply: oneshot::Sender<RelayResult>,
403    },
404}
405
406/// Relay endpoints by node. Nodes can be added while C2 runs (enrollment), so readers take a
407/// snapshot per operation instead of holding a fixed map.
408#[derive(Clone)]
409struct RelayRegistry {
410    map: Arc<std::sync::RwLock<Arc<BTreeMap<NodeId, RelayEndpoint>>>>,
411    gate: Option<Arc<dyn crate::enroll_service::CapabilityGate>>,
412}
413
414impl RelayRegistry {
415    fn new(initial: BTreeMap<NodeId, RelayEndpoint>) -> Self {
416        Self { map: Arc::new(std::sync::RwLock::new(Arc::new(initial))), gate: None }
417    }
418    /// May the operator drive and observe this node? Token nodes are not governed by policy.
419    fn allows(&self, node_id: &NodeId, capability: &str) -> bool {
420        self.gate.as_ref().and_then(|gate| gate.allows(node_id.as_str(), capability)).unwrap_or(true)
421    }
422    /// Heartbeat-only nodes show liveness but no inventory.
423    fn redact(&self, status: &mut StatusResponse) {
424        for (id, node) in status.nodes.iter_mut() {
425            if !self.allows(id, "attach") {
426                node.inventory = None;
427            }
428        }
429    }
430    fn snapshot(&self) -> Arc<BTreeMap<NodeId, RelayEndpoint>> {
431        Arc::clone(&self.map.read().expect("relay registry lock"))
432    }
433    fn insert(&self, node_id: NodeId, endpoint: RelayEndpoint) {
434        let mut guard = self.map.write().expect("relay registry lock");
435        let mut next = (**guard).clone();
436        next.insert(node_id, endpoint);
437        *guard = Arc::new(next);
438    }
439}
440
441#[derive(Clone)]
442struct RelayEndpoint {
443    commands: mpsc::Sender<RelayCommand>,
444    releases: mpsc::Sender<oneshot::Sender<()>>,
445    force_disconnect: watch::Sender<u64>,
446}
447
448#[derive(Clone)]
449struct OperatorHub {
450    sink: Arc<Mutex<Option<OperatorEventSink>>>,
451    gate: Arc<Mutex<Option<Arc<dyn crate::enroll_service::CapabilityGate>>>>,
452}
453
454#[derive(Clone)]
455struct OperatorEventSink {
456    connection_id: u64,
457    outbound: mpsc::Sender<control::QueuedFrame>,
458    budget: Arc<AtomicUsize>,
459    disconnect: watch::Sender<bool>,
460}
461
462impl OperatorHub {
463    fn new() -> Self { Self { sink: Arc::new(Mutex::new(None)), gate: Arc::new(Mutex::new(None)) } }
464
465    fn set_gate(&self, gate: Arc<dyn crate::enroll_service::CapabilityGate>) {
466        *self.gate.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(gate);
467    }
468
469    fn attach(&self, sink: OperatorEventSink) {
470        *self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(sink);
471    }
472
473    fn detach(&self, connection_id: u64) {
474        let mut sink = self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
475        if sink.as_ref().is_some_and(|current| current.connection_id == connection_id) { *sink = None; }
476    }
477
478    fn is_active(&self, connection_id: u64) -> bool {
479        self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
480            .as_ref().is_some_and(|current| current.connection_id == connection_id)
481    }
482
483    fn has_active_operator(&self) -> bool {
484        self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner()).is_some()
485    }
486
487    fn publish(&self, event: RoutedNodeEvent) {
488        // Events of a node whose status does not hold `attach` never reach the operator.
489        let gate = self.gate.lock().unwrap_or_else(|poisoned| poisoned.into_inner()).clone();
490        if gate.is_some_and(|gate| gate.allows(event.node_id.as_str(), "attach") == Some(false)) {
491            return;
492        }
493        let sink = self.sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
494        if let Some(sink) = sink.as_ref() {
495            if control::queue_operator_event(&sink.outbound, &sink.budget, event).is_err() {
496                let _ = sink.disconnect.send(true);
497            }
498        }
499    }
500}
501
502fn relay_failure(
503    code: C2RelayFailureCode,
504    message: &'static str,
505    current_incarnation_id: Option<NodeIncarnationId>,
506) -> C2RelayFailure {
507    C2RelayFailure { code, message: message.to_owned(), current_incarnation_id }
508}
509
510#[derive(Debug, Error)]
511pub enum C2Error {
512    #[error("C2 API failed: {0}")]
513    Api(#[from] io::Error),
514    #[error("C2 task failed: {0}")]
515    Task(#[from] tokio::task::JoinError),
516}
517
518#[derive(Clone)]
519pub struct C2ShutdownHandle {
520    shutdown: watch::Sender<bool>,
521}
522
523impl C2ShutdownHandle {
524    pub fn shutdown(&self) { let _ = self.shutdown.send(true); }
525}
526
527pub struct C2Running {
528    api_addr: SocketAddr,
529    shutdown: C2ShutdownHandle,
530    task: Option<JoinHandle<Result<(), C2Error>>>,
531}
532
533impl C2Running {
534    pub async fn start(config: C2Config) -> Result<Self, C2Error> {
535        prepare_default_control_parent(&config.control_endpoint)?;
536        let listener = TcpListener::bind(config.api_listen).await?;
537        let api_addr = listener.local_addr()?;
538        let (shutdown_tx, shutdown_rx) = watch::channel(false);
539        let shutdown = C2ShutdownHandle { shutdown: shutdown_tx };
540        let task = tokio::spawn(run_bound(config, listener, shutdown_rx));
541        Ok(Self { api_addr, shutdown, task: Some(task) })
542    }
543
544    pub fn api_addr(&self) -> SocketAddr { self.api_addr }
545    pub fn shutdown_handle(&self) -> C2ShutdownHandle { self.shutdown.clone() }
546    pub async fn wait(mut self) -> Result<(), C2Error> {
547        self.task.take().expect("C2 task is present").await?
548    }
549}
550
551#[cfg(windows)]
552fn prepare_default_control_parent(_endpoint: &str) -> io::Result<()> { Ok(()) }
553
554#[cfg(unix)]
555fn prepare_default_control_parent(endpoint: &str) -> io::Result<()> {
556    let default = default_c2_control_endpoint()
557        .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error.to_string()))?;
558    if endpoint != default { return Ok(()); }
559    let parent = Path::new(endpoint).parent().ok_or_else(|| {
560        io::Error::new(io::ErrorKind::InvalidInput, "default C2 endpoint has no parent")
561    })?;
562    std::fs::create_dir_all(parent)?;
563    std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700))
564}
565
566impl Drop for C2Running {
567    fn drop(&mut self) {
568        self.shutdown.shutdown();
569        if let Some(task) = self.task.take() { task.abort(); }
570    }
571}
572
573/// A socket handed from the call-home listener to a node's relay worker, with the
574/// per-connection session token when the node authenticated by signature.
575type CallHomeSocket = (TcpStream, Option<String>);
576type CallHomeRoutes = Arc<Mutex<BTreeMap<NodeId, mpsc::Sender<CallHomeSocket>>>>;
577
578struct Spawner {
579    timings: C2Timings,
580    relay_senders: RelayRegistry,
581    call_home_routes: CallHomeRoutes,
582    ingress: mpsc::Sender<Attempt>,
583    status: watch::Receiver<Arc<StatusResponse>>,
584    hub: OperatorHub,
585}
586
587impl Spawner {
588    /// Create the relay worker for a runtime-enrolled node, once.
589    fn spawn_enrolled(&self, node_id: &NodeId, shutdown: &watch::Receiver<bool>, tasks: &mut JoinSet<io::Result<()>>) {
590        if self.relay_senders.snapshot().contains_key(node_id) {
591            return;
592        }
593        // The placeholder token is never used on the wire: signed call-home hands each
594        // connection its own session token.
595        let placeholder = "0".repeat(64);
596        let Ok(node) = C2NodeConfig::new(node_id.clone(), "accept", placeholder) else { return };
597        let (commands_tx, commands_rx) = mpsc::channel(8);
598        let (releases_tx, releases_rx) = mpsc::channel(1);
599        let (force_tx, force_rx) = watch::channel(0_u64);
600        let (call_home_tx, call_home_rx) = mpsc::channel(1);
601        self.call_home_routes.lock().expect("routes lock").insert(node_id.clone(), call_home_tx);
602        self.relay_senders.insert(node_id.clone(), RelayEndpoint { commands: commands_tx, releases: releases_tx, force_disconnect: force_tx });
603        tasks.spawn(node_relay_worker(
604            node,
605            self.timings,
606            commands_rx,
607            releases_rx,
608            force_rx,
609            Some(call_home_rx),
610            self.ingress.clone(),
611            self.status.clone(),
612            self.hub.clone(),
613            shutdown.clone(),
614        ));
615        tracing::info!(%node_id, "relay worker created for enrolled node");
616    }
617}
618
619async fn run_bound(config: C2Config, listener: TcpListener, mut shutdown: watch::Receiver<bool>) -> Result<(), C2Error> {
620    let now = unix_ms();
621    let nodes = config.nodes.iter().map(|node| {
622        (node.node_id.clone(), initial_observed_node(node))
623    }).collect();
624    let initial = Arc::new(StatusResponse { api_version: C2_API_VERSION, ready: config.nodes.is_empty(), observed_at_unix_ms: now, nodes });
625    let (status_tx, status_rx) = watch::channel(initial);
626    let (ingress_tx, ingress_rx) = mpsc::channel(config.nodes.len().saturating_mul(2).max(16));
627    let mut tasks = JoinSet::new();
628    let hub = OperatorHub::new();
629    let mut relay_senders = BTreeMap::new();
630    let mut relay_receivers = Vec::new();
631    // Where the call-home listener hands a freshly announced socket. One
632    // slot per waiting node and no more: a node has one live connection to
633    // this relay, so a second socket arriving while the first is still
634    // being served means the node reconnected and the relay has not
635    // noticed yet -- queueing more of those would only serve them stale.
636    let mut static_routes: BTreeMap<NodeId, mpsc::Sender<CallHomeSocket>> = BTreeMap::new();
637    for node in config.nodes.clone() {
638        let (commands_tx, commands_rx) = mpsc::channel(8);
639        let (releases_tx, releases_rx) = mpsc::channel(1);
640        let (force_tx, force_rx) = watch::channel(0_u64);
641        relay_senders.insert(node.node_id.clone(), RelayEndpoint {
642            commands: commands_tx,
643            releases: releases_tx,
644            force_disconnect: force_tx,
645        });
646        let call_home_rx = if node.route == C2NodeRoute::CallHome {
647            let (call_home_tx, call_home_rx) = mpsc::channel(1);
648            static_routes.insert(node.node_id.clone(), call_home_tx);
649            Some(call_home_rx)
650        } else {
651            None
652        };
653        relay_receivers.push((node, commands_rx, releases_rx, force_rx, call_home_rx));
654    }
655    let static_ids: BTreeSet<NodeId> = static_routes.keys().cloned().collect();
656    let call_home_routes: CallHomeRoutes = Arc::new(Mutex::new(static_routes));
657    let mut relay_senders = RelayRegistry::new(relay_senders);
658    // Enrollment: registry, listener, runtime node adds.
659    let mut enroll_events = None;
660    let mut enroll_service = None;
661    if let Some(enrollment) = &config.enrollment {
662        let (service, events) = crate::enroll_service::EnrollService::start(enrollment)
663            .map_err(|error| C2Error::Api(io::Error::other(error.to_string())))?;
664        let enroll_listener = TcpListener::bind(enrollment.listen).await?;
665        tasks.spawn(crate::enroll_service::serve_enroll(enroll_listener, Arc::clone(&service), shutdown.clone()));
666        relay_senders.gate = Some(Arc::clone(&service) as Arc<dyn crate::enroll_service::CapabilityGate>);
667        hub.set_gate(Arc::clone(&service) as Arc<dyn crate::enroll_service::CapabilityGate>);
668        enroll_events = Some(events);
669        enroll_service = Some(service);
670    }
671    if let Some(node_listen) = config.node_listen {
672        tasks.spawn(accept_call_home(node_listen, Arc::clone(&call_home_routes), static_ids, enroll_service.clone(), shutdown.clone()));
673    }
674    tasks.spawn(inventory_owner(config.timings.fresh_for, ingress_rx, status_tx, shutdown.clone()));
675    for (node, commands, releases, force_disconnect, call_home) in relay_receivers {
676        tasks.spawn(node_relay_worker(node, config.timings, commands, releases, force_disconnect, call_home, ingress_tx.clone(), status_rx.clone(), hub.clone(), shutdown.clone()));
677    }
678    let spawner = Spawner {
679        timings: config.timings,
680        relay_senders: relay_senders.clone(),
681        call_home_routes: Arc::clone(&call_home_routes),
682        ingress: ingress_tx.clone(),
683        status: status_rx.clone(),
684        hub: hub.clone(),
685    };
686    drop(ingress_tx);
687    tasks.spawn(http_server(listener, config.api_token.clone(), config.timings.http_io_deadline, status_rx.clone(), enroll_service.clone(), shutdown.clone()));
688    let slots = control::ControlSlots::new();
689    let tunnel_listen = config.control_tunnel_listen;
690    let tunnel_token = config.api_token.clone();
691    tasks.spawn(control::run(
692        config.control_endpoint,
693        config.api_token,
694        relay_senders.clone(),
695        status_rx.clone(),
696        hub.clone(),
697        shutdown.clone(),
698        slots.clone(),
699    ));
700    if let Some(listen) = tunnel_listen {
701        tasks.spawn(control::run_tcp(
702            listen,
703            tunnel_token,
704            relay_senders.clone(),
705            status_rx,
706            hub,
707            shutdown.clone(),
708            slots,
709        ));
710    }
711    loop {
712        tokio::select! {
713            changed = shutdown.changed() => {
714                if changed.is_err() || *shutdown.borrow() { break; }
715            }
716            event = async {
717                match enroll_events.as_mut() {
718                    Some(events) => events.recv().await,
719                    None => std::future::pending().await,
720                }
721            } => {
722                match event {
723                    Some(crate::enroll_service::EnrollEvent::Enrolled(id)) => {
724                        if let Ok(node_id) = NodeId::new(id) { spawner.spawn_enrolled(&node_id, &shutdown, &mut tasks); }
725                    }
726                    Some(crate::enroll_service::EnrollEvent::StatusChanged(id, _)) => {
727                        // A node that no longer holds the heartbeat capability is dropped.
728                        if let Ok(node_id) = NodeId::new(id) {
729                            if !relay_senders.allows(&node_id, "heartbeat") {
730                                if let Some(endpoint) = relay_senders.snapshot().get(&node_id) {
731                                    endpoint.force_disconnect.send_modify(|generation| *generation = generation.wrapping_add(1));
732                                }
733                            }
734                        }
735                    }
736                    Some(crate::enroll_service::EnrollEvent::PolicyChanged) => {
737                        if let Some(service) = &enroll_service {
738                            for id in service.nodes_without("heartbeat") {
739                                if let Ok(node_id) = NodeId::new(id) {
740                                    if let Some(endpoint) = relay_senders.snapshot().get(&node_id) {
741                                        endpoint.force_disconnect.send_modify(|generation| *generation = generation.wrapping_add(1));
742                                    }
743                                }
744                            }
745                        }
746                    }
747                    None => enroll_events = None,
748                }
749            }
750            result = tasks.join_next() => {
751                match result {
752                    Some(Ok(Ok(()))) if *shutdown.borrow() => break,
753                    Some(Ok(Ok(()))) => return Err(C2Error::Api(io::Error::new(io::ErrorKind::Other, "C2 task exited unexpectedly"))),
754                    Some(Ok(Err(error))) => return Err(C2Error::Api(error)),
755                    Some(Err(error)) => return Err(C2Error::Task(error)),
756                    None => break,
757                }
758            }
759        }
760    }
761    tasks.shutdown().await;
762    Ok(())
763}
764
765fn initial_enrolled_node() -> ObservedNode {
766    ObservedNode {
767        endpoint: "call-home".to_owned(), transport_label: "call-home".to_owned(),
768        transport: NodeTransportState::Offline, freshness: NodeFreshness::Unavailable,
769        cursor: None, inventory: None, last_attempt_unix_ms: None, last_success_unix_ms: None,
770        consecutive_failures: 0, last_error: None, gaps: Vec::new(), gaps_truncated: 0,
771    }
772}
773
774fn initial_observed_node(node: &C2NodeConfig) -> ObservedNode {
775    ObservedNode {
776        endpoint: node.endpoint.clone(), transport_label: node.transport_label().to_owned(),
777        transport: NodeTransportState::Offline, freshness: NodeFreshness::Unavailable,
778        cursor: None, inventory: None, last_attempt_unix_ms: None, last_success_unix_ms: None,
779        consecutive_failures: 0, last_error: None, gaps: Vec::new(), gaps_truncated: 0,
780    }
781}
782
783#[cfg(windows)]
784fn local_transport_label() -> &'static str { "windows-named-pipe" }
785
786#[cfg(unix)]
787fn local_transport_label() -> &'static str { "unix-domain-socket" }
788
789#[derive(Clone, Default)]
790struct ProviderContractManifest {
791    provider_contracts: Vec<ProviderContractSupport>,
792    provider_adapter_contracts: Vec<ProviderAdapterContractSupport>,
793}
794
795impl ProviderContractManifest {
796    fn from_compatibility(compatibility: Option<&NegotiatedNodeCompatibility>) -> Self {
797        let Some(compatibility) = compatibility.filter(|compatibility| {
798            compatibility.capabilities.iter().any(|capability| {
799                capability.as_str() == C2_PROVIDER_CONTRACT_MANIFEST_CAPABILITY
800            })
801        }) else {
802            return Self::default();
803        };
804        Self {
805            provider_contracts: compatibility.provider_contracts.clone(),
806            provider_adapter_contracts: compatibility.provider_adapter_contracts.clone(),
807        }
808    }
809}
810
811enum AttemptResult {
812    Connected {
813        cursor: NodeCursor,
814        snapshot: NodeSnapshot,
815        gaps: Vec<GapKind>,
816        provider_contract_manifest: ProviderContractManifest,
817    },
818    Success { cursor: NodeCursor, snapshot: NodeSnapshot, gaps: Vec<GapKind> },
819    Cursor {
820        cursor: NodeCursor,
821        gaps: Vec<GapKind>,
822        managed_worktree_events: Vec<NodeEvent>,
823    },
824    Failure { error: SanitizedError, hard: bool },
825}
826
827struct Attempt { node_id: NodeId, at_unix_ms: u64, result: AttemptResult }
828
829/// Accepts node connections and hands each one to the relay worker for
830/// whichever node it says it is.
831///
832/// This is the only place in the relay that learns a node's identity from
833/// the wire rather than from configuration, and it is careful about what
834/// that means: the announced name SELECTS a worker and nothing more. The
835/// worker then runs the same mutual challenge-response it runs on a
836/// dialled connection, against the token configured for that node, so a
837/// caller that announced a name it cannot prove gets no further than the
838/// next frame.
839///
840/// A socket for an unknown node, an unreadable preface, or a node whose
841/// slot is already full is dropped with a line saying which -- never
842/// silently, because "the node is calling but nothing happens" is exactly
843/// the failure that is impossible to diagnose from the node's end.
844async fn accept_call_home(
845    listen: SocketAddr,
846    routes: CallHomeRoutes,
847    static_ids: BTreeSet<NodeId>,
848    enroll: Option<Arc<crate::enroll_service::EnrollService>>,
849    mut shutdown: watch::Receiver<bool>,
850) -> io::Result<()> {
851    let listener = TcpListener::bind(listen).await?;
852    tracing::info!(%listen, waiting_nodes = routes.lock().expect("routes lock").len(), "call-home listener bound");
853    loop {
854        if *shutdown.borrow() {
855            return Ok(());
856        }
857        let (stream, peer) = tokio::select! {
858            accepted = listener.accept() => accepted?,
859            changed = shutdown.changed() => {
860                if changed.is_err() || *shutdown.borrow() { return Ok(()); }
861                continue;
862            }
863        };
864        let routes = Arc::clone(&routes);
865        let static_ids = static_ids.clone();
866        let enroll = enroll.clone();
867        // Off the accept loop: the preface has its own deadline, and one
868        // peer that connects and then says nothing must not stop every
869        // other node from being accepted meanwhile.
870        tokio::spawn(async move {
871            let mut stream = stream;
872            let _ = stream.set_nodelay(true);
873            let node_id = match read_call_home_announce(&mut stream).await {
874                Ok(node_id) => node_id,
875                Err(error) => {
876                    tracing::warn!(%peer, cause = %error, "call-home connection rejected");
877                    return;
878                }
879            };
880            // Token nodes (configured with --node) keep the original path. Enrolled nodes
881            // prove their identity by signing a challenge; the exchange yields a
882            // per-connection session token for the unchanged wire handshake.
883            let mut session_token = None;
884            if !static_ids.contains(&node_id) {
885                let Some(enroll) = enroll.as_ref() else {
886                    tracing::warn!(%peer, %node_id, "call-home connection named a node this relay does not wait for");
887                    return;
888                };
889                match enroll.signed_call_home(&mut stream, node_id.as_str()).await {
890                    Ok(token) => session_token = Some(token),
891                    Err(cause) => {
892                        tracing::warn!(%peer, %node_id, %cause, "signed call-home refused");
893                        return;
894                    }
895                }
896            }
897            let sender = routes.lock().expect("routes lock").get(&node_id).cloned();
898            let Some(sender) = sender else {
899                tracing::warn!(
900                    %peer,
901                    %node_id,
902                    "call-home connection named a node this relay does not wait for",
903                );
904                return;
905            };
906            match sender.try_send((stream, session_token)) {
907                Ok(()) => tracing::info!(%peer, %node_id, "node called home"),
908                Err(mpsc::error::TrySendError::Full(_)) => tracing::warn!(
909                    %peer,
910                    %node_id,
911                    "node called home while its previous connection is still being taken up",
912                ),
913                Err(mpsc::error::TrySendError::Closed(_)) => tracing::warn!(
914                    %peer,
915                    %node_id,
916                    "node called home but its relay worker is gone",
917                ),
918            }
919        });
920    }
921}
922
923/// Blocks until this node calls in, or shutdown -- with no deadline of its
924/// own on purpose.
925///
926/// A dialled connection either answers within `attempt_deadline` or has
927/// failed, and treating a slow answer as a failure is right there. Waiting
928/// for a node to call is the opposite: a node that has not called yet has
929/// not failed at anything, and it may be minutes from starting. Putting
930/// that wait under the attempt deadline would turn an idle relay into a
931/// stream of "connection deadline exceeded" warnings about nodes that are
932/// merely not running. The handshake that FOLLOWS is deadlined normally --
933/// a peer that connects and then stalls is a real failure.
934async fn await_call_home(
935    receiver: Option<&mut mpsc::Receiver<CallHomeSocket>>,
936    shutdown: &mut watch::Receiver<bool>,
937) -> Option<CallHomeSocket> {
938    let receiver = receiver?;
939    loop {
940        if *shutdown.borrow() {
941            return None;
942        }
943        tokio::select! {
944            stream = receiver.recv() => return stream,
945            changed = shutdown.changed() => {
946                if changed.is_err() || *shutdown.borrow() { return None; }
947            }
948        }
949    }
950}
951
952async fn node_relay_worker(
953    node: C2NodeConfig,
954    timings: C2Timings,
955    mut commands: mpsc::Receiver<RelayCommand>,
956    mut releases: mpsc::Receiver<oneshot::Sender<()>>,
957    mut force_disconnect: watch::Receiver<u64>,
958    mut call_home: Option<mpsc::Receiver<CallHomeSocket>>,
959    ingress: mpsc::Sender<Attempt>,
960    status: watch::Receiver<Arc<StatusResponse>>,
961    hub: OperatorHub,
962    mut shutdown: watch::Receiver<bool>,
963) -> io::Result<()> {
964    let mut failures = 0_usize;
965    loop {
966        if *shutdown.borrow() { return Ok(()); }
967        let previous = status.borrow().nodes.get(&node.node_id).and_then(|item| item.cursor);
968        // A waiting node's socket is collected BEFORE the deadline starts,
969        // because waiting for a node to call is not an attempt that can
970        // time out -- see `await_call_home`. Once it is here, the
971        // handshake over it is deadlined exactly like a dialled one.
972        let adopted = if node.route == C2NodeRoute::CallHome {
973            match await_call_home(call_home.as_mut(), &mut shutdown).await {
974                Some((stream, token)) => Some((stream, token)),
975                None => return Ok(()),
976            }
977        } else {
978            None
979        };
980        let (adopted, session_token) = match adopted {
981            Some((stream, token)) => (Some(stream), token),
982            None => (None, None),
983        };
984        let attempt_node = match session_token {
985            Some(token) => C2NodeConfig { token, ..node.clone() },
986            None => node.clone(),
987        };
988        let connected = timeout(timings.attempt_deadline, connect_operator(&attempt_node, adopted)).await;
989        let mut client = match connected {
990            Ok(Ok(client)) => client,
991            Ok(Err(error)) => {
992                failures = failures.saturating_add(1);
993                let (error, hard) = sanitize_node_error(&error);
994                tracing::warn!(
995                    node_id = %node.node_id,
996                    cause = %error.message,
997                    category = ?error.category,
998                    hard,
999                    "node connection attempt failed",
1000                );
1001                ingress_attempt(&ingress, &node.node_id, AttemptResult::Failure { error, hard }).await?;
1002                reject_disconnected_commands(&mut commands, previous);
1003                acknowledge_disconnected_releases(&mut releases);
1004                relay_backoff(&mut shutdown, timings, failures, hard).await?;
1005                continue;
1006            }
1007            Err(_) => {
1008                failures = failures.saturating_add(1);
1009                tracing::warn!(
1010                    node_id = %node.node_id,
1011                    cause = "node connection deadline exceeded",
1012                    "node connection attempt failed",
1013                );
1014                ingress_attempt(&ingress, &node.node_id, AttemptResult::Failure {
1015                    error: SanitizedError { category: C2ErrorCategory::Timeout, message: "node connection deadline exceeded".to_owned() },
1016                    hard: false,
1017                }).await?;
1018                reject_disconnected_commands(&mut commands, previous);
1019                acknowledge_disconnected_releases(&mut releases);
1020                relay_backoff(&mut shutdown, timings, failures, false).await?;
1021                continue;
1022            }
1023        };
1024        failures = 0;
1025        let hello = client.hello().clone();
1026        let provider_contract_manifest =
1027            ProviderContractManifest::from_compatibility(hello.compatibility.as_ref());
1028        let incarnation_id = hello.incarnation_id;
1029        let connection_id = hello.connection_id;
1030        tracing::info!(
1031            node_id = %node.node_id,
1032            connection_id,
1033            incarnation_id = ?incarnation_id,
1034            "node attached to relay",
1035        );
1036        let mut controller_owned = hello.controller.as_ref().is_some_and(|controller| controller.connection_id == connection_id);
1037        let mut cursor = NodeCursor { incarnation_id, sequence: hello.event_sequence };
1038        let mut snapshot = hello.snapshot;
1039        let mut gaps = Vec::new();
1040        let mut did_resync = false;
1041        if let Some(previous) = previous {
1042            if previous.incarnation_id != incarnation_id {
1043                gaps.push(GapKind::IncarnationChanged);
1044            } else if cursor.sequence < previous.sequence {
1045                gaps.push(GapKind::CursorRegression);
1046            } else if cursor.sequence > previous.sequence {
1047                match bounded_node_request(&mut client, NodeRequest::Resync { after_sequence: previous.sequence }).await {
1048                    Ok(NodeResponse::Resync {
1049                        event_sequence,
1050                        oldest_available_sequence,
1051                        snapshot: resync_snapshot,
1052                        events,
1053                    }) => {
1054                        let resync_gaps = validate_resync(
1055                            previous.sequence,
1056                            cursor.sequence,
1057                            event_sequence,
1058                            oldest_available_sequence,
1059                            &events,
1060                        );
1061                        if resync_gaps.is_empty() {
1062                            publish_recovered_events(&node.node_id, incarnation_id, &events, &hub);
1063                        } else {
1064                            hub.publish(RoutedNodeEvent {
1065                                node_id: node.node_id.clone(),
1066                                cursor: NodeCursor { incarnation_id, sequence: event_sequence },
1067                                event: C2NodeEvent::ResyncRequired {
1068                                    oldest_available_sequence,
1069                                },
1070                            });
1071                        }
1072                        gaps.extend(resync_gaps);
1073                        cursor.sequence = event_sequence;
1074                        snapshot = resync_snapshot;
1075                        did_resync = true;
1076                    }
1077                    Ok(_) => gaps.push(GapKind::NonContiguousEvents),
1078                    Err(error) => {
1079                        ingress_attempt(&ingress, &node.node_id, relay_failure_attempt(&error)).await?;
1080                        reject_disconnected_commands(&mut commands, Some(cursor));
1081                        continue;
1082                    }
1083                }
1084            }
1085        }
1086        ingress_attempt(&ingress, &node.node_id, AttemptResult::Connected {
1087            cursor,
1088            snapshot,
1089            gaps,
1090            provider_contract_manifest,
1091        }).await?;
1092        if let Err(error) = drain_pending_events(&mut client, &node.node_id, &mut cursor, &hub, &ingress, did_resync, None).await {
1093            ingress_attempt(&ingress, &node.node_id, relay_failure_attempt(&error)).await?;
1094            reject_disconnected_commands(&mut commands, Some(cursor));
1095            continue;
1096        }
1097
1098        let cadence = timings.poll_interval.max(Duration::from_millis(1)).min(Duration::from_millis(250));
1099        let mut snapshot_tick = tokio::time::interval(cadence);
1100        snapshot_tick.reset();
1101        let mut lease_tick = tokio::time::interval(Duration::from_secs(30));
1102        lease_tick.reset();
1103        let disconnect_error = loop {
1104            tokio::select! {
1105                command = commands.recv() => {
1106                    let Some(command) = command else { return Ok(()); };
1107                    tokio::select! {
1108                        result = handle_relay_command(
1109                            &mut client, command, &node.node_id, incarnation_id, connection_id,
1110                            &mut controller_owned, &hub, &mut cursor, &ingress,
1111                        ) => match result {
1112                            Ok(()) => {}
1113                            Err(error) => break error,
1114                        },
1115                        changed = force_disconnect.changed() => {
1116                            let _ = changed;
1117                            break NodeClientError::Io(io::Error::new(io::ErrorKind::ConnectionAborted, "node relay cleanup forced reconnect"));
1118                        }
1119                    }
1120                }
1121                release = releases.recv() => {
1122                    let Some(reply) = release else { return Ok(()); };
1123                    let result = release_controller(&mut client, &mut controller_owned).await;
1124                    let _ = reply.send(());
1125                    if let Err(error) = result { break error; }
1126                }
1127                changed = force_disconnect.changed() => {
1128                    let _ = changed;
1129                    break NodeClientError::Io(io::Error::new(io::ErrorKind::ConnectionAborted, "node relay cleanup forced reconnect"));
1130                }
1131                frame = client.recv() => {
1132                    match frame {
1133                        Ok(ServerFrame::Event(envelope)) => {
1134                            if let Err(error) = handle_live_node_event(
1135                                &mut client,
1136                                &node.node_id,
1137                                envelope,
1138                                &mut cursor,
1139                                &hub,
1140                                &ingress,
1141                            ).await {
1142                                break error;
1143                            }
1144                        }
1145                        Ok(ServerFrame::Reply(_)
1146                            | ServerFrame::Challenge(_)
1147                            | ServerFrame::Hello(_)) => {
1148                            break NodeClientError::Protocol(
1149                                "node sent an unexpected idle frame".to_owned(),
1150                            );
1151                        }
1152                        Err(error) => break error,
1153                    }
1154                }
1155                _ = snapshot_tick.tick() => {
1156                    match bounded_node_request(&mut client, NodeRequest::Snapshot).await {
1157                        Ok(NodeResponse::Snapshot { event_sequence, snapshot, .. }) => {
1158                            if let Err(error) = drain_pending_events(&mut client, &node.node_id, &mut cursor, &hub, &ingress, false, None).await { break error; }
1159                            if event_sequence >= cursor.sequence {
1160                                cursor.sequence = event_sequence;
1161                                ingress_attempt(&ingress, &node.node_id, AttemptResult::Success { cursor, snapshot, gaps: Vec::new() }).await?;
1162                            }
1163                        }
1164                        Ok(_) => break NodeClientError::Protocol("snapshot returned a different response".to_owned()),
1165                        Err(error) => break error,
1166                    }
1167                }
1168                _ = lease_tick.tick() => {
1169                    if controller_owned {
1170                        if hub.has_active_operator() {
1171                            match acquire_controller(&mut client, connection_id).await {
1172                                Ok(owned) => controller_owned = owned,
1173                                Err(error) => break error,
1174                            }
1175                        } else if let Err(error) = release_controller(&mut client, &mut controller_owned).await {
1176                            break error;
1177                        }
1178                        if let Err(error) = drain_pending_events(&mut client, &node.node_id, &mut cursor, &hub, &ingress, false, None).await { break error; }
1179                    }
1180                }
1181                changed = shutdown.changed() => {
1182                    if changed.is_err() || *shutdown.borrow() {
1183                        let _ = release_controller(&mut client, &mut controller_owned).await;
1184                        return Ok(());
1185                    }
1186                }
1187            }
1188        };
1189        let (error, hard) = sanitize_node_error(&disconnect_error);
1190        tracing::warn!(
1191            node_id = %node.node_id,
1192            connection_id,
1193            cause = %error.message,
1194            category = ?error.category,
1195            hard,
1196            "node dropped from relay",
1197        );
1198        ingress_attempt(&ingress, &node.node_id, AttemptResult::Failure { error, hard }).await?;
1199        reject_disconnected_commands(&mut commands, Some(cursor));
1200        failures = failures.saturating_add(1);
1201        relay_backoff(&mut shutdown, timings, failures, hard).await?;
1202    }
1203}
1204
1205/// Produces an authenticated operator client for `node`, whichever end
1206/// opened the socket.
1207///
1208/// All three arms end in the same handshake. The first two open a
1209/// connection and hand it straight to it; the third is handed one that a
1210/// node opened and the listener already matched to this node's name. That
1211/// symmetry is the point of the change: `LocalNodeClient::adopt` is what
1212/// `connect`/`connect_loopback` both do after their own `connect` call, so
1213/// a call-home node is authenticated by exactly the same code, against
1214/// exactly the same per-node token, as one this relay dialled.
1215async fn connect_operator(
1216    node: &C2NodeConfig,
1217    adopted: Option<TcpStream>,
1218) -> Result<LocalNodeClient, NodeClientError> {
1219    match node.route {
1220        C2NodeRoute::Local => {
1221            LocalNodeClient::connect(&node.endpoint, &node.node_id, ClientRole::Operator, &node.token).await
1222        }
1223        C2NodeRoute::SshForwardedLoopback(endpoint) => {
1224            LocalNodeClient::connect_loopback(endpoint, &node.node_id, ClientRole::Operator, &node.token).await
1225        }
1226        C2NodeRoute::CallHome => {
1227            // `node_relay_worker` collects the socket before calling this
1228            // and returns rather than calling without one, so `None` here
1229            // is a caller bug, not a runtime condition -- reported instead
1230            // of panicking because a relay worker is not worth aborting a
1231            // whole C2 over.
1232            let stream = adopted.ok_or_else(|| {
1233                NodeClientError::Protocol(
1234                    "call-home node reached the connect step with no adopted stream".to_owned(),
1235                )
1236            })?;
1237            LocalNodeClient::adopt(stream, &node.node_id, ClientRole::Operator, &node.token).await
1238        }
1239    }
1240}
1241
1242async fn bounded_node_request(
1243    client: &mut LocalNodeClient,
1244    request: NodeRequest,
1245) -> Result<NodeResponse, NodeClientError> {
1246    bounded_node_request_with_deadline(client, request, None).await
1247}
1248
1249async fn bounded_node_request_with_deadline(
1250    client: &mut LocalNodeClient,
1251    request: NodeRequest,
1252    relay_deadline: Option<Instant>,
1253) -> Result<NodeResponse, NodeClientError> {
1254    let deadline = request_budget(&request, relay_deadline, Instant::now());
1255    if deadline.is_zero() {
1256        return Err(NodeClientError::Frame(FrameError::PrefixTimedOut));
1257    }
1258    match timeout(deadline, client.request(request)).await {
1259        Ok(result) => result,
1260        Err(_) => Err(NodeClientError::Frame(FrameError::PrefixTimedOut)),
1261    }
1262}
1263
1264fn relay_request_deadline(request: &NodeRequest, now: Instant) -> Option<Instant> {
1265    matches!(request, NodeRequest::ResumeSessionRecord { .. })
1266        .then(|| now + node_request_deadline(request))
1267}
1268
1269fn request_budget(request: &NodeRequest, relay_deadline: Option<Instant>, now: Instant) -> Duration {
1270    relay_deadline
1271        .map(|deadline| deadline.saturating_duration_since(now))
1272        .unwrap_or_else(|| node_request_deadline(request))
1273}
1274
1275/// Relay bound for a workspace inspection. The node's own inspection
1276/// budget is 8s by default and 11s at most; this must clear that maximum
1277/// plus the round trip, or the relay kills a request the node was still
1278/// legitimately working on -- and killing it drops the node, not just the
1279/// request. See `node_request_deadline`'s own `InspectWorkspace` arm.
1280const WORKSPACE_INSPECTION_RELAY_DEADLINE: Duration = Duration::from_secs(15);
1281
1282fn node_request_deadline(request: &NodeRequest) -> Duration {
1283    match request {
1284        NodeRequest::Snapshot
1285        | NodeRequest::Resync { .. }
1286        | NodeRequest::ArmHarnessMcpReservation { .. }
1287        | NodeRequest::ActivateHarnessMcpReservation { .. }
1288        | NodeRequest::AbortHarnessMcpReservation { .. }
1289        | NodeRequest::PutHarnessMcpReplyChunk { .. }
1290        | NodeRequest::RejectHarnessMcpCall { .. }
1291        | NodeRequest::BrowseHostDirectories { .. }
1292        | NodeRequest::ReadWorkspaceFile { .. }
1293        | NodeRequest::WriteWorkspaceFile { .. }
1294        | NodeRequest::ReadGitHistory { .. }
1295        | NodeRequest::ReadGitDiff { .. }
1296        | NodeRequest::AcquireController { .. }
1297        | NodeRequest::ReleaseController
1298        | NodeRequest::RenameSessionRecord { .. }
1299        | NodeRequest::SetSessionTask { .. }
1300        | NodeRequest::ForgetSessionRecord { .. } => Duration::from_secs(5),
1301        // The node gives its own workspace inspection 8s by default and up
1302        // to 11s, and says so when it uses them
1303        // (`git_time_budget_exceeded`, elapsed over eight seconds on a
1304        // real repository here). Bounding it from out here at five made
1305        // that inner budget unreachable: any workspace big enough to spend
1306        // its own allowance timed out at the relay every single time.
1307        //
1308        // And a relay timeout is not a slow answer, it is a dead node --
1309        // the node is dropped from the relay, its controller lease is
1310        // released, and every read behind it starts answering
1311        // "unavailable" until it reattaches, which it then does, and the
1312        // cycle repeats. One number two seconds too small took the whole
1313        // stack down in a loop: no inventory, so a spawned session could
1314        // never be shown, while its process ran perfectly well.
1315        //
1316        // The outer bound must therefore clear the node's own MAXIMUM, not
1317        // its default, with room for the round trip on top.
1318        NodeRequest::InspectWorkspace { .. } => WORKSPACE_INSPECTION_RELAY_DEADLINE,
1319        NodeRequest::CreateWorkspaceFile { .. }
1320        | NodeRequest::CreateWorkspaceDirectory { .. } => {
1321            WORKSPACE_ENTRY_CREATE_RELAY_DEADLINE
1322        }
1323        NodeRequest::CatalogNativeSessions { .. }
1324        | NodeRequest::PageNativeSessions { .. }
1325        | NodeRequest::PreviewNativeSession { .. }
1326        | NodeRequest::IndexNativeSession { .. }
1327        | NodeRequest::PreviewSessionRecord { .. } => NATIVE_SESSION_REQUEST_DEADLINE,
1328        NodeRequest::CreateStandaloneWorkspace { .. }
1329        | NodeRequest::CreateWorktree { .. }
1330        | NodeRequest::RemoveWorktree { .. }
1331        | NodeRequest::CleanupManagedWorktree { .. } => Duration::from_secs(240),
1332        NodeRequest::Spawn { .. }
1333        | NodeRequest::Resume { .. }
1334        | NodeRequest::Stop { .. } => Duration::from_secs(15),
1335        NodeRequest::SpawnSpec { spec } =>
1336            Duration::from_millis(spec.deadline_ms.get()) + NODE_REQUEST_IO_HEADROOM,
1337        NodeRequest::SpawnSpecWithHarnessMcp { deadline_unix_ms, .. } =>
1338            Duration::from_millis(deadline_unix_ms.saturating_sub(unix_ms()))
1339                + NODE_REQUEST_IO_HEADROOM,
1340        NodeRequest::SpawnManagedWorktree { request } =>
1341            Duration::from_millis(request.spawn_spec.deadline_ms.get())
1342                + NODE_REQUEST_IO_HEADROOM,
1343        NodeRequest::ResumeSessionRecord { .. } =>
1344            MANAGED_RESUME_SETTLE_DEADLINE + NODE_REQUEST_IO_HEADROOM,
1345        _ => Duration::from_secs(10),
1346    }
1347}
1348
1349fn validate_resync(
1350    previous: u64,
1351    hello: u64,
1352    current: u64,
1353    oldest_available_sequence: u64,
1354    events: &[NodeEventEnvelope],
1355) -> Vec<GapKind> {
1356    let mut gaps = validate_events(
1357        previous,
1358        current,
1359        oldest_available_sequence,
1360        events,
1361    );
1362    if current < hello && !gaps.contains(&GapKind::CursorRegression) {
1363        gaps.push(GapKind::CursorRegression);
1364    }
1365    gaps
1366}
1367
1368async fn ingress_attempt(
1369    ingress: &mpsc::Sender<Attempt>,
1370    node_id: &NodeId,
1371    result: AttemptResult,
1372) -> io::Result<()> {
1373    ingress.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result })
1374        .await.map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "inventory owner closed"))
1375}
1376
1377async fn relay_backoff(
1378    shutdown: &mut watch::Receiver<bool>,
1379    timings: C2Timings,
1380    failures: usize,
1381    hard: bool,
1382) -> io::Result<()> {
1383    let delay = if hard || failures >= timings.transient_backoffs.len() {
1384        timings.parked_backoff
1385    } else {
1386        timings.transient_backoffs[failures.saturating_sub(1)]
1387    };
1388    tokio::select! {
1389        _ = sleep(delay) => Ok(()),
1390        _ = shutdown.changed() => {
1391            Ok(())
1392        }
1393    }
1394}
1395
1396fn relay_failure_attempt(error: &NodeClientError) -> AttemptResult {
1397    let (error, hard) = sanitize_node_error(error);
1398    AttemptResult::Failure { error, hard }
1399}
1400
1401async fn handle_relay_command(
1402    client: &mut LocalNodeClient,
1403    command: RelayCommand,
1404    node_id: &NodeId,
1405    incarnation_id: NodeIncarnationId,
1406    connection_id: u64,
1407    controller_owned: &mut bool,
1408    hub: &OperatorHub,
1409    cursor: &mut NodeCursor,
1410    ingress: &mpsc::Sender<Attempt>,
1411) -> Result<(), NodeClientError> {
1412    match command {
1413        RelayCommand::Request { operator_connection_id, expected_incarnation_id, request, reply } => {
1414            let relay_deadline = relay_request_deadline(&request, Instant::now());
1415            let expected_spawn = expected_spawn_request(&request);
1416            let expected_provider_session_index = matches!(
1417                &request,
1418                NodeRequest::IndexProviderSession { .. }
1419            ).then(|| request.clone());
1420            let expected_native_session = matches!(
1421                &request,
1422                NodeRequest::CatalogNativeSessions { .. }
1423                    | NodeRequest::PageNativeSessions { .. }
1424                    | NodeRequest::PreviewNativeSession { .. }
1425                    | NodeRequest::IndexNativeSession { .. }
1426            ).then(|| request.clone());
1427            let expected_workspace_content = matches!(
1428                &request,
1429                NodeRequest::ReadWorkspaceFile { .. }
1430                    | NodeRequest::WriteWorkspaceFile { .. }
1431                    | NodeRequest::CreateWorkspaceFile { .. }
1432                    | NodeRequest::CreateWorkspaceDirectory { .. }
1433                    | NodeRequest::ReadGitHistory { .. }
1434                    | NodeRequest::ReadGitDiff { .. }
1435            ).then(|| request.clone());
1436            let expected_session_task = matches!(
1437                &request,
1438                NodeRequest::SetSessionTask { .. }
1439            ).then(|| request.clone());
1440            let expected_harness_mcp = (request.required_capability()
1441                == Some(gate4agent_node_protocol::NODE_HARNESS_MCP_READ_PROXY_CAPABILITY))
1442                .then(|| request.clone());
1443            if !hub.is_active(operator_connection_id) {
1444                let _ = reply.send(Err(relay_failure(C2RelayFailureCode::ClientLagged, "C2 operator connection is no longer active", Some(incarnation_id))));
1445                return Ok(());
1446            }
1447            if expected_incarnation_id != incarnation_id {
1448                let _ = reply.send(Err(relay_failure(C2RelayFailureCode::StaleNodeIncarnation, "node incarnation changed", Some(incarnation_id))));
1449                return Ok(());
1450            }
1451            if matches!(&request, NodeRequest::IndexNativeSession { selection, .. }
1452                if selection.route.scope
1453                    != gate4agent_node_protocol::NativeSessionCatalogScope::Workspace)
1454            {
1455                let _ = reply.send(Err(relay_failure(
1456                    C2RelayFailureCode::RequestForbidden,
1457                    "external native sessions must be registered as workspaces before indexing",
1458                    Some(incarnation_id),
1459                )));
1460                return Ok(());
1461            }
1462            if !is_read_only_request(&request) && !*controller_owned {
1463                match acquire_controller_with_deadline(client, connection_id, relay_deadline).await {
1464                    Ok(owned) if owned => *controller_owned = true,
1465                    Ok(_) => {
1466                        let _ = reply.send(Err(relay_failure(C2RelayFailureCode::RelayBusy, "node controller lease is unavailable", Some(incarnation_id))));
1467                        return Ok(());
1468                    }
1469                    Err(error) if relay_node_failure(&error).is_some() => {
1470                        let failure = relay_node_failure(&error)
1471                            .expect("guarded node request failure");
1472                        let _ = reply.send(Ok(RoutedNodeResponse {
1473                            node_id: node_id.clone(),
1474                            incarnation_id,
1475                            response: Err(failure),
1476                        }));
1477                        return Ok(());
1478                    }
1479                    Err(error) => {
1480                        let _ = reply.send(Err(relay_failure(C2RelayFailureCode::NodeOffline, "node relay disconnected", Some(incarnation_id))));
1481                        return Err(error);
1482                    }
1483                }
1484            }
1485            let response = match bounded_node_request_with_deadline(client, request, relay_deadline).await {
1486                Ok(response) => Ok(response),
1487                Err(NodeClientError::Node(failure)) => Err(failure),
1488                Err(error @ NodeClientError::UnsupportedCapability(_)) => {
1489                    let failure = relay_node_failure(&error)
1490                        .expect("unsupported capability is a routed node failure");
1491                    let _ = reply.send(Ok(RoutedNodeResponse {
1492                        node_id: node_id.clone(),
1493                        incarnation_id,
1494                        response: Err(failure),
1495                    }));
1496                    return Ok(());
1497                }
1498                Err(error) => {
1499                    let _ = reply.send(Err(relay_failure(C2RelayFailureCode::NodeOffline, "node relay disconnected", Some(incarnation_id))));
1500                    return Err(error);
1501                }
1502            };
1503            if let Err(message) = validate_spawn_spec_response(
1504                expected_spawn.as_ref(),
1505                &response,
1506                incarnation_id,
1507            ) {
1508                let _ = reply.send(Err(relay_failure(
1509                    C2RelayFailureCode::NodeOffline,
1510                    "node relay returned an invalid spawn receipt",
1511                    Some(incarnation_id),
1512                )));
1513                return Err(NodeClientError::Protocol(message.to_owned()));
1514            }
1515            if let Err(message) = validate_provider_session_index_response(
1516                expected_provider_session_index.as_ref(),
1517                &response,
1518            ) {
1519                let _ = reply.send(Err(relay_failure(
1520                    C2RelayFailureCode::NodeOffline,
1521                    "node relay returned an invalid provider session index response",
1522                    Some(incarnation_id),
1523                )));
1524                return Err(NodeClientError::Protocol(message.to_owned()));
1525            }
1526            if let Err(message) = validate_native_session_response(
1527                expected_native_session.as_ref(),
1528                &response,
1529            ) {
1530                let _ = reply.send(Err(relay_failure(
1531                    C2RelayFailureCode::NodeOffline,
1532                    "node relay returned an invalid native session response",
1533                    Some(incarnation_id),
1534                )));
1535                return Err(NodeClientError::Protocol(message.to_owned()));
1536            }
1537            if let Err(message) = validate_workspace_content_response(
1538                expected_workspace_content.as_ref(),
1539                &response,
1540            ) {
1541                let _ = reply.send(Err(relay_failure(
1542                    C2RelayFailureCode::NodeOffline,
1543                    "node relay returned an invalid workspace content response",
1544                    Some(incarnation_id),
1545                )));
1546                return Err(NodeClientError::Protocol(message.to_owned()));
1547            }
1548            if let Err(message) = validate_session_task_response(
1549                expected_session_task.as_ref(),
1550                &response,
1551            ) {
1552                let _ = reply.send(Err(relay_failure(
1553                    C2RelayFailureCode::NodeOffline,
1554                    "node relay returned an invalid session task response",
1555                    Some(incarnation_id),
1556                )));
1557                return Err(NodeClientError::Protocol(message.to_owned()));
1558            }
1559            if let Err(message) = validate_harness_mcp_response(
1560                expected_harness_mcp.as_ref(),
1561                &response,
1562            ) {
1563                let _ = reply.send(Err(relay_failure(
1564                    C2RelayFailureCode::NodeOffline,
1565                    "node relay returned an invalid harness MCP response",
1566                    Some(incarnation_id),
1567                )));
1568                return Err(NodeClientError::Protocol(message.to_owned()));
1569            }
1570            drain_pending_events(client, node_id, cursor, hub, ingress, false, relay_deadline).await?;
1571            update_inventory_from_response(node_id, cursor, &response, ingress).await
1572                .map_err(NodeClientError::Io)?;
1573            let response = response
1574                .map(|response| C2NodeResponse::from_node_response_with_control_detail(&response))
1575                .map_err(|failure| C2NodeFailure::from(&failure));
1576            let _ = reply.send(Ok(RoutedNodeResponse { node_id: node_id.clone(), incarnation_id, response }));
1577        }
1578    }
1579    Ok(())
1580}
1581
1582fn validate_session_task_response(
1583    expected: Option<&NodeRequest>,
1584    response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1585) -> Result<(), &'static str> {
1586    match (expected, response) {
1587        (Some(_), Err(_)) | (None, Err(_)) => Ok(()),
1588        (
1589            Some(NodeRequest::SetSessionTask {
1590                record_id,
1591                expected_revision,
1592                target,
1593            }),
1594            Ok(NodeResponse::SessionRecordUpdated { record }),
1595        ) if session_task_record_matches(record, record_id, *expected_revision, target) => Ok(()),
1596        (Some(NodeRequest::SetSessionTask { .. }), Ok(_)) => {
1597            Err("session task response does not match the routed request")
1598        }
1599        (None, Ok(_)) => Ok(()),
1600        (Some(_), Ok(_)) => Ok(()),
1601    }
1602}
1603
1604fn validate_harness_mcp_response(
1605    expected: Option<&NodeRequest>,
1606    response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1607) -> Result<(), &'static str> {
1608    use NodeRequest as Request;
1609    use NodeResponse as Response;
1610    let valid = match (expected, response) {
1611        (Some(Request::ArmHarnessMcpReservation { reservation_id, activation_digest, expires_at_unix_ms, .. }),
1612            Ok(Response::Armed { reservation_id: echoed_id, activation_digest: echoed_digest, expires_at_unix_ms: echoed_expiry })) =>
1613            reservation_id == echoed_id && activation_digest == echoed_digest
1614                && expires_at_unix_ms == echoed_expiry,
1615        (Some(Request::SpawnSpecWithHarnessMcp { reservation_id, activation_digest, .. }),
1616            Ok(Response::Spawned { reservation_id: echoed_id, activation_digest: echoed_digest, receipt })) =>
1617            reservation_id == echoed_id && activation_digest == echoed_digest
1618                && receipt.harness_mcp_proxy.as_ref().is_some_and(|proxy| {
1619                    &proxy.reservation_id == reservation_id
1620                        && &proxy.activation_digest == activation_digest
1621                }),
1622        (Some(Request::ActivateHarnessMcpReservation { reservation_id, activation_digest, record_id, session }),
1623            Ok(Response::Activated { reservation_id: echoed_id, activation_digest: echoed_digest, record_id: echoed_record, session: echoed_session })) =>
1624            reservation_id == echoed_id && activation_digest == echoed_digest
1625                && record_id == echoed_record && session == echoed_session,
1626        (Some(Request::AbortHarnessMcpReservation { reservation_id, activation_digest }),
1627            Ok(Response::Aborted { reservation_id: echoed_id, activation_digest: echoed_digest })) =>
1628            reservation_id == echoed_id && activation_digest == echoed_digest,
1629        (Some(Request::PutHarnessMcpReplyChunk { reservation_id, activation_digest, record_id, session, call_id, offset, final_chunk, chunk_hex }),
1630            Ok(Response::ReplyChunkAccepted { reservation_id: echoed_id, activation_digest: echoed_digest, record_id: echoed_record, session: echoed_session, call_id: echoed_call, next_offset, completed })) =>
1631            reservation_id == echoed_id && activation_digest == echoed_digest
1632                && record_id == echoed_record && session == echoed_session && call_id == echoed_call
1633                && offset.checked_add(u32::try_from(chunk_hex.raw_len()).unwrap_or(u32::MAX))
1634                    == Some(*next_offset) && completed == final_chunk,
1635        (Some(Request::RejectHarnessMcpCall { reservation_id, activation_digest, record_id, session, call_id, .. }),
1636            Ok(Response::CallRejected { reservation_id: echoed_id, activation_digest: echoed_digest, record_id: echoed_record, session: echoed_session, call_id: echoed_call })) =>
1637            reservation_id == echoed_id && activation_digest == echoed_digest
1638                && record_id == echoed_record && session == echoed_session && call_id == echoed_call,
1639        (Some(_), Err(_)) | (None, Err(_)) => true,
1640        (Some(_), Ok(_)) => false,
1641        (None, Ok(response)) => !response.requires_harness_mcp_proxy_capability(),
1642    };
1643    if valid { Ok(()) } else { Err("harness MCP response does not match the routed request") }
1644}
1645
1646fn session_task_record_matches(
1647    record: &gate4agent_node_protocol::ManagedSessionRecord,
1648    record_id: &gate4agent_node_protocol::SessionRecordId,
1649    expected_revision: u64,
1650    target: &gate4agent_node_protocol::SessionTaskTargetV1,
1651) -> bool {
1652    if &record.record_id != record_id { return false; }
1653    let next_revision = expected_revision.checked_add(1);
1654    match target {
1655        gate4agent_node_protocol::SessionTaskTargetV1::New => record.task_binding.as_ref()
1656            .is_some_and(|binding| Some(binding.revision) == next_revision && binding.task_id.is_some()),
1657        gate4agent_node_protocol::SessionTaskTargetV1::Existing { task_id } => record.task_binding.as_ref()
1658            .is_some_and(|binding| (binding.revision == expected_revision || Some(binding.revision) == next_revision)
1659                && binding.task_id.as_ref() == Some(task_id)),
1660        gate4agent_node_protocol::SessionTaskTargetV1::Clear => match &record.task_binding {
1661            None => expected_revision == 0,
1662            Some(binding) => binding.task_id.is_none()
1663                && (binding.revision == expected_revision || Some(binding.revision) == next_revision),
1664        },
1665    }
1666}
1667
1668fn validate_workspace_content_response(
1669    expected: Option<&NodeRequest>,
1670    response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1671) -> Result<(), &'static str> {
1672    match (expected, response) {
1673        (Some(_), Err(_)) | (None, Err(_)) => Ok(()),
1674        (
1675            Some(NodeRequest::ReadWorkspaceFile { workspace_id, path }),
1676            Ok(NodeResponse::WorkspaceFileRead { file }),
1677        ) if &file.workspace_id == workspace_id && &file.path == path => Ok(()),
1678        (
1679            Some(NodeRequest::WriteWorkspaceFile { workspace_id, path, text, .. }),
1680            Ok(NodeResponse::WorkspaceFileWritten { file }),
1681        ) if &file.workspace_id == workspace_id
1682            && &file.path == path
1683            && matches!(
1684                &file.content,
1685                gate4agent_node_protocol::WorkspaceFileContent::Utf8 {
1686                    text: written,
1687                    byte_len,
1688                } if written == text
1689                    && u32::try_from(text.len()).ok() == Some(*byte_len)
1690            ) => Ok(()),
1691        (
1692            Some(NodeRequest::CreateWorkspaceFile { workspace_id, path }),
1693            Ok(NodeResponse::WorkspaceFileCreated { file }),
1694        ) if &file.workspace_id == workspace_id
1695            && &file.path == path
1696            && file.revision.is_some()
1697            && matches!(
1698                &file.content,
1699                gate4agent_node_protocol::WorkspaceFileContent::Utf8 {
1700                    text,
1701                    byte_len: 0,
1702                } if text.is_empty()
1703            ) => Ok(()),
1704        (
1705            Some(NodeRequest::CreateWorkspaceDirectory { workspace_id, path }),
1706            Ok(NodeResponse::WorkspaceDirectoryCreated {
1707                workspace_id: actual_workspace_id,
1708                entry,
1709            }),
1710        ) if actual_workspace_id == workspace_id
1711            && &entry.relative_path == path
1712            && entry.kind == gate4agent_node_protocol::WorkspaceEntryKind::Directory => Ok(()),
1713        (
1714            Some(NodeRequest::ReadGitHistory { workspace_id, .. }),
1715            Ok(NodeResponse::GitHistoryRead { workspace_id: actual, .. }),
1716        ) if actual == workspace_id => Ok(()),
1717        (
1718            Some(NodeRequest::ReadGitDiff { workspace_id, request }),
1719            Ok(NodeResponse::GitDiffRead { workspace_id: actual, diff }),
1720        ) if actual == workspace_id && diff.mode == request.mode && diff.path == request.path => Ok(()),
1721        (Some(_), Ok(_)) => Err("workspace content response does not match routed request"),
1722        (
1723            None,
1724            Ok(NodeResponse::WorkspaceFileRead { .. }
1725                | NodeResponse::WorkspaceFileWritten { .. }
1726                | NodeResponse::WorkspaceFileCreated { .. }
1727                | NodeResponse::WorkspaceDirectoryCreated { .. }
1728                | NodeResponse::GitHistoryRead { .. }
1729                | NodeResponse::GitDiffRead { .. }),
1730        ) => Err("unexpected workspace content response"),
1731        (None, Ok(_)) => Ok(()),
1732    }
1733}
1734
1735enum ExpectedSpawnRequest {
1736    Spec(SpawnSpec),
1737    ManagedV1(ManagedWorktreeSpawnRequest),
1738    ManagedV2(ManagedWorktreeSpawnRequestV2),
1739}
1740
1741fn expected_spawn_request(request: &NodeRequest) -> Option<ExpectedSpawnRequest> {
1742    match request {
1743        NodeRequest::SpawnSpec { spec } => Some(ExpectedSpawnRequest::Spec(spec.clone())),
1744        NodeRequest::SpawnManagedWorktree { request } => {
1745            Some(ExpectedSpawnRequest::ManagedV1(request.clone()))
1746        }
1747        NodeRequest::SpawnManagedWorktreeV2 { request } => {
1748            Some(ExpectedSpawnRequest::ManagedV2(request.clone()))
1749        }
1750        _ => None,
1751    }
1752}
1753
1754fn validate_provider_session_index_response(
1755    expected: Option<&NodeRequest>,
1756    response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1757) -> Result<(), &'static str> {
1758    match (expected, response) {
1759        (
1760            Some(NodeRequest::IndexProviderSession {
1761                workspace_id,
1762                provider,
1763                identity,
1764                ..
1765            }),
1766            Ok(NodeResponse::ProviderSessionIndexed { record }),
1767        ) if &record.workspace_id == workspace_id
1768            && &record.provider == provider
1769            && record.provider_session.as_ref() == Some(identity) => Ok(()),
1770        (Some(_), Ok(NodeResponse::ProviderSessionIndexed { .. })) => {
1771            Err("provider session index response does not match routed request")
1772        }
1773        (Some(_), Ok(_)) => Err("provider session index request returned a different response"),
1774        (None, Ok(NodeResponse::ProviderSessionIndexed { .. })) => {
1775            Err("unexpected provider session index response for a different node request")
1776        }
1777        (Some(_), Err(_)) | (None, _) => Ok(()),
1778    }
1779}
1780
1781fn validate_native_session_response(
1782    expected: Option<&NodeRequest>,
1783    response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1784) -> Result<(), &'static str> {
1785    match (expected, response) {
1786        (
1787            Some(NodeRequest::CatalogNativeSessions { route, .. }),
1788            Ok(NodeResponse::NativeSessionsCataloged {
1789                route: echoed_route,
1790                ..
1791            }),
1792        ) if echoed_route == route => Ok(()),
1793        (
1794            Some(NodeRequest::PageNativeSessions {
1795                route,
1796                window,
1797                catalog_revision,
1798                ..
1799            }),
1800            Ok(NodeResponse::NativeSessionsPaged {
1801                route: echoed_route,
1802                page,
1803            }),
1804        ) if echoed_route == route
1805            && page.window == *window
1806            && page.revision == *catalog_revision => Ok(()),
1807        (
1808            Some(NodeRequest::PreviewNativeSession { selection, .. }),
1809            Ok(NodeResponse::NativeSessionPreviewed {
1810                selection: echoed_selection,
1811                ..
1812            }),
1813        ) if echoed_selection == selection => Ok(()),
1814        (
1815            Some(NodeRequest::IndexNativeSession { selection, .. }),
1816            Ok(NodeResponse::NativeSessionIndexed {
1817                selection: echoed_selection,
1818                record,
1819            }),
1820        ) if echoed_selection == selection
1821            && selection.route.scope
1822                == gate4agent_node_protocol::NativeSessionCatalogScope::Workspace
1823            && selection.route.workspace_id.as_ref() == Some(&record.workspace_id)
1824            && selection.route.provider == record.provider => Ok(()),
1825        (Some(_), Err(_)) => Ok(()),
1826        (Some(_), Ok(_)) => Err("native session response does not match routed request"),
1827        (
1828            None,
1829            Ok(
1830                NodeResponse::NativeSessionsCataloged { .. }
1831                | NodeResponse::NativeSessionsPaged { .. }
1832                | NodeResponse::NativeSessionPreviewed { .. }
1833                | NodeResponse::NativeSessionIndexed { .. },
1834            ),
1835        ) => Err("unexpected native session response for a different node request"),
1836        (None, _) => Ok(()),
1837    }
1838}
1839
1840fn validate_spawn_spec_response(
1841    expected: Option<&ExpectedSpawnRequest>,
1842    response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
1843    relay_incarnation_id: NodeIncarnationId,
1844) -> Result<(), &'static str> {
1845    match (expected, response) {
1846        (Some(ExpectedSpawnRequest::Spec(spec)), Ok(NodeResponse::SpawnSpecAccepted { receipt })) => {
1847            validate_spawn_receipt(spec, receipt, relay_incarnation_id)
1848        }
1849        (
1850            Some(ExpectedSpawnRequest::ManagedV1(request)),
1851            Ok(NodeResponse::ManagedWorktreeSpawnAccepted { receipt }),
1852        ) => validate_managed_spawn_receipt(
1853            &request.spawn_spec,
1854            &request.worktree_profile_id,
1855            receipt,
1856            relay_incarnation_id,
1857        ),
1858        (
1859            Some(ExpectedSpawnRequest::ManagedV2(request)),
1860            Ok(NodeResponse::ManagedWorktreeSpawnAccepted { receipt }),
1861        ) if receipt.lease.profile_revision == request.expected_profile_revision => {
1862            validate_managed_spawn_receipt(
1863                &request.spawn_spec,
1864                &request.worktree_profile_id,
1865                receipt,
1866                relay_incarnation_id,
1867            )
1868        }
1869        (
1870            Some(ExpectedSpawnRequest::ManagedV2(_)),
1871            Ok(NodeResponse::ManagedWorktreeSpawnAccepted { .. }),
1872        ) => Err("managed spawn receipt profile revision does not match routed request"),
1873        (Some(_), Ok(_)) => return Err("spawn spec request returned a different response"),
1874        (Some(_), Err(_)) => return Ok(()),
1875        (None, Ok(NodeResponse::SpawnSpecAccepted { .. }
1876            | NodeResponse::ManagedWorktreeSpawnAccepted { .. })) => {
1877            return Err("unexpected spawn receipt for a different node request");
1878        }
1879        (None, _) => return Ok(()),
1880    }
1881}
1882
1883fn validate_managed_spawn_receipt(
1884    spawn_spec: &SpawnSpec,
1885    worktree_profile_id: &gate4agent_node_protocol::WorktreeProfileId,
1886    receipt: &gate4agent_node_protocol::ManagedWorktreeSpawnReceipt,
1887    relay_incarnation_id: NodeIncarnationId,
1888) -> Result<(), &'static str> {
1889    if receipt.lease.source_workspace_id != spawn_spec.target.workspace_id
1890        || &receipt.lease.profile_id != worktree_profile_id
1891        || receipt.lease.state != ManagedWorktreeLeaseState::InUse
1892        || receipt.lease.cleanup_failure.is_some()
1893        || receipt.lease.active_session_count != 1
1894        || receipt.spawn.target.node_id != spawn_spec.target.node_id
1895        || receipt.spawn.target.workspace_id != spawn_spec.target.workspace_id
1896        || receipt.spawn.target.worktree_id.as_ref() != Some(&receipt.lease.workspace_id)
1897        || receipt.spawn.session.workspace_id != receipt.lease.workspace_id
1898    {
1899        return Err("managed spawn receipt does not match routed request");
1900    }
1901    let mut resolved_spec = spawn_spec.clone();
1902    resolved_spec.target.worktree_id = Some(receipt.lease.workspace_id.clone());
1903    validate_spawn_receipt(&resolved_spec, &receipt.spawn, relay_incarnation_id)
1904}
1905
1906fn validate_spawn_receipt(
1907    spec: &SpawnSpec,
1908    receipt: &ResolvedSpawnReceipt,
1909    relay_incarnation_id: NodeIncarnationId,
1910) -> Result<(), &'static str> {
1911    if receipt.incarnation_id != relay_incarnation_id
1912        || !receipt.context_binding_is_valid()
1913        || receipt.target != spec.target
1914        || receipt.profile_id != spec.profile_id
1915        || receipt.idempotency_key != spec.idempotency_key
1916        || receipt.deadline_ms != spec.deadline_ms
1917        || receipt.required_capabilities != spec.required_capabilities
1918        || receipt.bundle.as_ref().is_some_and(|bundle| {
1919            receipt.bundle_id.as_ref() != Some(&bundle.id)
1920        })
1921        || &receipt.session.workspace_id
1922            != spec
1923                .target
1924                .worktree_id
1925                .as_ref()
1926                .unwrap_or(&spec.target.workspace_id)
1927    {
1928        return Err("spawn receipt does not match routed request");
1929    }
1930    if !required_override_matches(&spec.overrides.provider, &receipt.provider)
1931        || !required_override_matches(&spec.overrides.mode, &receipt.mode)
1932        || !required_override_matches(&spec.overrides.terminal_size, &receipt.terminal_size)
1933        || !optional_override_matches(&spec.overrides.bundle_id, &receipt.bundle_id)
1934        || !optional_override_matches(&spec.overrides.context_id, &receipt.context_id)
1935        || !environment_profile_override_matches(
1936            &spec.overrides.environment_profile_id,
1937            receipt.environment_profile.as_ref(),
1938        )
1939    {
1940        return Err("spawn receipt contradicts explicit overrides");
1941    }
1942    match &spec.overrides.prompt {
1943        SpawnOverride::Inherit => {}
1944        SpawnOverride::Set { value }
1945            if receipt.prompt.present
1946                && receipt.prompt.byte_len == u32::try_from(value.byte_len()).unwrap_or(0) => {}
1947        SpawnOverride::Clear if !receipt.prompt.present && receipt.prompt.byte_len == 0 => {}
1948        SpawnOverride::Set { .. } | SpawnOverride::Clear => {
1949            return Err("spawn receipt contradicts explicit prompt override");
1950        }
1951    }
1952    Ok(())
1953}
1954
1955fn environment_profile_override_matches(
1956    expected: &SpawnOverride<gate4agent_node_protocol::SpawnEnvironmentProfileId>,
1957    actual: Option<&gate4agent_node_protocol::ResolvedEnvironmentProfileReceipt>,
1958) -> bool {
1959    match expected {
1960        SpawnOverride::Inherit => true,
1961        SpawnOverride::Set { value } => {
1962            actual.is_some_and(|receipt| &receipt.profile_id == value)
1963        }
1964        SpawnOverride::Clear => actual.is_none(),
1965    }
1966}
1967
1968fn required_override_matches<T: Eq>(override_value: &SpawnOverride<T>, actual: &T) -> bool {
1969    match override_value {
1970        SpawnOverride::Inherit => true,
1971        SpawnOverride::Set { value } => value == actual,
1972        SpawnOverride::Clear => false,
1973    }
1974}
1975
1976fn optional_override_matches<T: Eq>(
1977    override_value: &SpawnOverride<T>,
1978    actual: &Option<T>,
1979) -> bool {
1980    match override_value {
1981        SpawnOverride::Inherit => true,
1982        SpawnOverride::Set { value } => actual.as_ref() == Some(value),
1983        SpawnOverride::Clear => actual.is_none(),
1984    }
1985}
1986
1987fn relay_node_failure(error: &NodeClientError) -> Option<C2NodeFailure> {
1988    match error {
1989        NodeClientError::Node(failure) => Some(C2NodeFailure::from(failure)),
1990        NodeClientError::UnsupportedCapability(_) => Some(C2NodeFailure {
1991            code: NodeFailureCode::UnsupportedCapability,
1992            message: "required capability unavailable".to_owned(),
1993        }),
1994        NodeClientError::Io(_)
1995        | NodeClientError::Frame(_)
1996        | NodeClientError::Protocol(_)
1997        | NodeClientError::BuildStampMismatch { .. }
1998        | NodeClientError::AuthenticationTimedOut
1999        | NodeClientError::Authentication(_)
2000        | NodeClientError::RequestIdExhausted => None,
2001    }
2002}
2003
2004fn is_read_only_request(request: &NodeRequest) -> bool {
2005    matches!(request,
2006        NodeRequest::Snapshot
2007        | NodeRequest::Resync { .. }
2008        | NodeRequest::BrowseHostDirectories { .. }
2009        | NodeRequest::InspectWorkspace { .. }
2010        | NodeRequest::ReadWorkspaceFile { .. }
2011        | NodeRequest::ReadGitHistory { .. }
2012        | NodeRequest::ReadGitDiff { .. }
2013        | NodeRequest::CatalogNativeSessions { .. }
2014        | NodeRequest::PageNativeSessions { .. }
2015        | NodeRequest::PreviewNativeSession { .. }
2016        | NodeRequest::PreviewSessionRecord { .. }
2017    )
2018}
2019
2020async fn acquire_controller(
2021    client: &mut LocalNodeClient,
2022    connection_id: u64,
2023) -> Result<bool, NodeClientError> {
2024    acquire_controller_with_deadline(client, connection_id, None).await
2025}
2026
2027async fn acquire_controller_with_deadline(
2028    client: &mut LocalNodeClient,
2029    connection_id: u64,
2030    relay_deadline: Option<Instant>,
2031) -> Result<bool, NodeClientError> {
2032    match bounded_node_request_with_deadline(
2033        client,
2034        NodeRequest::AcquireController { lease_ms: gate4agent_node_protocol::MAX_CONTROLLER_LEASE_MS },
2035        relay_deadline,
2036    ).await? {
2037        NodeResponse::Controller { controller } => Ok(controller.as_ref().is_some_and(|state| state.connection_id == connection_id)),
2038        _ => Err(NodeClientError::Protocol("controller acquisition returned a different response".to_owned())),
2039    }
2040}
2041
2042async fn release_controller(
2043    client: &mut LocalNodeClient,
2044    controller_owned: &mut bool,
2045) -> Result<(), NodeClientError> {
2046    if !*controller_owned { return Ok(()); }
2047    match bounded_node_request(client, NodeRequest::ReleaseController).await? {
2048        NodeResponse::Controller { .. } => { *controller_owned = false; Ok(()) }
2049        _ => Err(NodeClientError::Protocol("controller release returned a different response".to_owned())),
2050    }
2051}
2052
2053async fn update_inventory_from_response(
2054    node_id: &NodeId,
2055    cursor: &mut NodeCursor,
2056    response: &Result<NodeResponse, gate4agent_node_protocol::NodeFailure>,
2057    ingress: &mpsc::Sender<Attempt>,
2058) -> io::Result<()> {
2059    match response {
2060        Ok(NodeResponse::Snapshot { event_sequence, snapshot, .. })
2061        | Ok(NodeResponse::Resync { event_sequence, snapshot, .. }) => {
2062            if *event_sequence < cursor.sequence { return Ok(()); }
2063            cursor.sequence = *event_sequence;
2064            ingress_attempt(ingress, node_id, AttemptResult::Success { cursor: *cursor, snapshot: snapshot.clone(), gaps: Vec::new() }).await
2065        }
2066        _ => Ok(()),
2067    }
2068}
2069
2070fn publish_recovered_events(
2071    node_id: &NodeId,
2072    incarnation_id: NodeIncarnationId,
2073    events: &[NodeEventEnvelope],
2074    hub: &OperatorHub,
2075) {
2076    for envelope in events {
2077        if let Some(event) = routed_recovered_node_event(node_id, incarnation_id, envelope) {
2078            hub.publish(event);
2079        }
2080    }
2081}
2082
2083fn routed_recovered_node_event(
2084    node_id: &NodeId,
2085    incarnation_id: NodeIncarnationId,
2086    envelope: &NodeEventEnvelope,
2087) -> Option<RoutedNodeEvent> {
2088    (!matches!(&envelope.event, NodeEvent::HarnessMcpReadCall { .. })).then(|| {
2089        RoutedNodeEvent {
2090            node_id: node_id.clone(),
2091            cursor: NodeCursor { incarnation_id, sequence: envelope.sequence },
2092            event: C2NodeEvent::from_node_event_with_control_detail(&envelope.event),
2093        }
2094    })
2095}
2096
2097fn routed_transient_node_event(
2098    node_id: &NodeId,
2099    cursor: NodeCursor,
2100    envelope: &NodeEventEnvelope,
2101) -> Option<RoutedNodeEvent> {
2102    matches!(&envelope.event, NodeEvent::HarnessMcpReadCall { .. }).then(|| RoutedNodeEvent {
2103        node_id: node_id.clone(),
2104        cursor,
2105        event: C2NodeEvent::from(&envelope.event),
2106    })
2107}
2108
2109/// `NodeEvent::AgentStream` chunks are published unconditionally -- never
2110/// gated on `cursor` contiguity, and never the cause of a
2111/// `CursorRegression`/`NonContiguousEvents` gap -- but, unlike
2112/// `HarnessMcpReadCall` (always `sequence: 0`, see
2113/// `gate4agent-node/src/server.rs::publish_transient`), an
2114/// `AgentStreamChunkV1` envelope draws its `sequence` from the
2115/// SAME counter every durable `NodeEvent` shares
2116/// (`gate4agent-node/src/server.rs::publish_agent_stream_chunk`'s own doc,
2117/// there purely so the node's per-connection discard watermark can compare
2118/// it against everything else). So a chunk really does consume a slot in
2119/// the durable sequence space, and `routed_transient_node_event`'s
2120/// "leave `cursor` untouched" rule -- correct for a call that never had a
2121/// slot to begin with -- would otherwise make the very next durable event
2122/// look one short of contiguous, forcing a resync round trip on every
2123/// single provider turn that streams so much as one chunk.
2124///
2125/// This folds the chunk's sequence into `cursor` with `max` instead: on the
2126/// ordinary, non-bursty path (`Control(N)` then its own `AgentStream(N+1)`,
2127/// each one delivered as they are produced) that keeps `cursor` moving
2128/// exactly as it did before this function existed, so the very next
2129/// `Control(N+2)` still finds `cursor == N+1` and stays contiguous. On the
2130/// bursty path -- the one actually measured live: 19 chunks published
2131/// `source_sequence` 22-40 in a 17ms window, `envelope_sequence` interleaved
2132/// 1:1 with `Control`, only 2 of the 19 ever reached the harness -- the
2133/// node's connection loop can drain its durable channel far enough ahead of
2134/// this dedicated one that a lower-sequence chunk arrives at this relay
2135/// AFTER `cursor` has already been carried past it by a resync recovering
2136/// the higher-sequence `Control` envelopes around it. `max` refuses to let
2137/// that late chunk rewind `cursor` -- it is still published (chunks promise
2138/// no resync, so there is nothing to recover if it were dropped instead:
2139/// `gate4agent-node-protocol::AgentStreamChunkV1`'s own doc, "no
2140/// `ObservationV1` resync promise") -- it simply stops mattering to the
2141/// durable cursor's own contiguity bookkeeping once something newer has
2142/// already passed it by. Ordering and true loss within the agent-stream
2143/// channel itself remain the node's own broadcast `Lagged` warn to name,
2144/// not this cursor's to adjudicate.
2145fn route_agent_stream_event(
2146    node_id: &NodeId,
2147    cursor: &mut NodeCursor,
2148    envelope: &NodeEventEnvelope,
2149) -> Option<RoutedNodeEvent> {
2150    if !matches!(&envelope.event, NodeEvent::AgentStream { .. }) {
2151        return None;
2152    }
2153    let routed = RoutedNodeEvent {
2154        node_id: node_id.clone(),
2155        cursor: *cursor,
2156        event: C2NodeEvent::from(&envelope.event),
2157    };
2158    cursor.sequence = cursor.sequence.max(envelope.sequence);
2159    Some(routed)
2160}
2161
2162async fn drain_pending_events(
2163    client: &mut LocalNodeClient,
2164    node_id: &NodeId,
2165    cursor: &mut NodeCursor,
2166    hub: &OperatorHub,
2167    ingress: &mpsc::Sender<Attempt>,
2168    skip_replayed: bool,
2169    relay_deadline: Option<Instant>,
2170) -> Result<(), NodeClientError> {
2171    let mut skip_replayed = skip_replayed;
2172    for repair_pass in 0..=1 {
2173        let mut gaps = Vec::new();
2174        let mut managed_worktree_events = Vec::new();
2175        let mut changed = false;
2176        let mut repair = false;
2177        while let Some(envelope) = client.take_event() {
2178            if let Some(event) = routed_transient_node_event(node_id, *cursor, &envelope) {
2179                hub.publish(event);
2180                continue;
2181            }
2182            if let Some(event) = route_agent_stream_event(node_id, cursor, &envelope) {
2183                hub.publish(event);
2184                continue;
2185            }
2186            if skip_replayed && envelope.sequence <= cursor.sequence { continue; }
2187            let resync_required = matches!(&envelope.event, gate4agent_node_protocol::NodeEvent::ResyncRequired { .. });
2188            if resync_required || envelope.sequence != cursor.sequence.saturating_add(1) {
2189                gaps.push(if envelope.sequence <= cursor.sequence && !resync_required {
2190                    GapKind::CursorRegression
2191                } else if resync_required {
2192                    GapKind::HistoryEvicted
2193                } else {
2194                    GapKind::NonContiguousEvents
2195                });
2196                repair = true;
2197                continue;
2198            }
2199            let event_cursor = NodeCursor { incarnation_id: cursor.incarnation_id, sequence: envelope.sequence };
2200            if matches!(
2201                &envelope.event,
2202                NodeEvent::ManagedWorktreeUpserted { .. }
2203                    | NodeEvent::ManagedWorktreeRemoved { .. }
2204            ) {
2205                managed_worktree_events.push(envelope.event.clone());
2206            }
2207            hub.publish(RoutedNodeEvent {
2208                node_id: node_id.clone(),
2209                cursor: event_cursor,
2210                event: C2NodeEvent::from_node_event_with_control_detail(&envelope.event),
2211            });
2212            cursor.sequence = envelope.sequence;
2213            changed = true;
2214        }
2215        if !repair {
2216            if changed || !gaps.is_empty() {
2217                ingress_attempt(ingress, node_id, AttemptResult::Cursor {
2218                    cursor: *cursor,
2219                    gaps,
2220                    managed_worktree_events,
2221                }).await
2222                    .map_err(NodeClientError::Io)?;
2223            }
2224            return Ok(());
2225        }
2226        if repair_pass == 1 {
2227            ingress_attempt(ingress, node_id, AttemptResult::Cursor {
2228                cursor: *cursor,
2229                gaps,
2230                managed_worktree_events,
2231            }).await
2232                .map_err(NodeClientError::Io)?;
2233            return Err(NodeClientError::Protocol("node event stream remained noncontiguous after resync".to_owned()));
2234        }
2235        let after_sequence = cursor.sequence;
2236        let response = bounded_node_request_with_deadline(
2237            client,
2238            NodeRequest::Resync { after_sequence },
2239            relay_deadline,
2240        ).await?;
2241        let NodeResponse::Resync {
2242            event_sequence,
2243            oldest_available_sequence,
2244            snapshot,
2245            events,
2246        } = response else {
2247            return Err(NodeClientError::Protocol("event repair resync returned a different response".to_owned()));
2248        };
2249        let repair_gaps = validate_events(
2250            after_sequence,
2251            event_sequence,
2252            oldest_available_sequence,
2253            &events,
2254        );
2255        let contiguous = repair_gaps.is_empty();
2256        gaps = repair_gaps;
2257        if contiguous {
2258            for envelope in events.iter().filter(|event| event.sequence > after_sequence) {
2259                if let Some(event) = routed_recovered_node_event(
2260                    node_id,
2261                    cursor.incarnation_id,
2262                    envelope,
2263                ) {
2264                    hub.publish(event);
2265                }
2266            }
2267        } else {
2268            hub.publish(RoutedNodeEvent {
2269                node_id: node_id.clone(),
2270                cursor: NodeCursor { incarnation_id: cursor.incarnation_id, sequence: event_sequence },
2271                event: C2NodeEvent::ResyncRequired {
2272                    oldest_available_sequence,
2273                },
2274            });
2275        }
2276        cursor.sequence = event_sequence;
2277        ingress_attempt(ingress, node_id, AttemptResult::Success { cursor: *cursor, snapshot, gaps }).await
2278            .map_err(NodeClientError::Io)?;
2279        skip_replayed = true;
2280    }
2281    Ok(())
2282}
2283
2284async fn handle_live_node_event(
2285    client: &mut LocalNodeClient,
2286    node_id: &NodeId,
2287    envelope: NodeEventEnvelope,
2288    cursor: &mut NodeCursor,
2289    hub: &OperatorHub,
2290    ingress: &mpsc::Sender<Attempt>,
2291) -> Result<(), NodeClientError> {
2292    if let Some(event) = routed_transient_node_event(node_id, *cursor, &envelope) {
2293        hub.publish(event);
2294        return Ok(());
2295    }
2296    if let Some(event) = route_agent_stream_event(node_id, cursor, &envelope) {
2297        hub.publish(event);
2298        return Ok(());
2299    }
2300    if live_event_gap(cursor.sequence, &envelope).is_some() {
2301        let after_sequence = cursor.sequence;
2302        let response = bounded_node_request(
2303            client,
2304            NodeRequest::Resync { after_sequence },
2305        ).await?;
2306        let NodeResponse::Resync {
2307            event_sequence,
2308            oldest_available_sequence,
2309            snapshot,
2310            events,
2311        } = response else {
2312            return Err(NodeClientError::Protocol(
2313                "live event repair resync returned a different response".to_owned(),
2314            ));
2315        };
2316        let repair_gaps = validate_events(
2317            after_sequence,
2318            event_sequence,
2319            oldest_available_sequence,
2320            &events,
2321        );
2322        if repair_gaps.is_empty() {
2323            publish_recovered_events(node_id, cursor.incarnation_id, &events, hub);
2324        } else {
2325            hub.publish(RoutedNodeEvent {
2326                node_id: node_id.clone(),
2327                cursor: NodeCursor {
2328                    incarnation_id: cursor.incarnation_id,
2329                    sequence: event_sequence,
2330                },
2331                event: C2NodeEvent::ResyncRequired {
2332                    oldest_available_sequence,
2333                },
2334            });
2335        }
2336        cursor.sequence = event_sequence;
2337        ingress_attempt(
2338            ingress,
2339            node_id,
2340            AttemptResult::Success {
2341                cursor: *cursor,
2342                snapshot,
2343                gaps: repair_gaps,
2344            },
2345        ).await.map_err(NodeClientError::Io)?;
2346        return drain_pending_events(
2347            client,
2348            node_id,
2349            cursor,
2350            hub,
2351            ingress,
2352            true,
2353            None,
2354        ).await;
2355    }
2356
2357    cursor.sequence = envelope.sequence;
2358    let managed_worktree_events = matches!(
2359        &envelope.event,
2360        NodeEvent::ManagedWorktreeUpserted { .. }
2361            | NodeEvent::ManagedWorktreeRemoved { .. }
2362    )
2363    .then(|| vec![envelope.event.clone()])
2364    .unwrap_or_default();
2365    hub.publish(RoutedNodeEvent {
2366        node_id: node_id.clone(),
2367        cursor: *cursor,
2368        event: C2NodeEvent::from_node_event_with_control_detail(&envelope.event),
2369    });
2370    ingress_attempt(
2371        ingress,
2372        node_id,
2373        AttemptResult::Cursor {
2374            cursor: *cursor,
2375            gaps: Vec::new(),
2376            managed_worktree_events,
2377        },
2378    ).await.map_err(NodeClientError::Io)
2379}
2380
2381fn live_event_gap(previous: u64, envelope: &NodeEventEnvelope) -> Option<GapKind> {
2382    if matches!(
2383        &envelope.event,
2384        gate4agent_node_protocol::NodeEvent::ResyncRequired { .. }
2385    ) {
2386        return Some(GapKind::HistoryEvicted);
2387    }
2388    if envelope.sequence <= previous {
2389        return Some(GapKind::CursorRegression);
2390    }
2391    (envelope.sequence != previous.saturating_add(1))
2392        .then_some(GapKind::NonContiguousEvents)
2393}
2394
2395fn reject_disconnected_commands(commands: &mut mpsc::Receiver<RelayCommand>, cursor: Option<NodeCursor>) {
2396    while let Ok(command) = commands.try_recv() {
2397        match command {
2398            RelayCommand::Request { reply, .. } => {
2399                let _ = reply.send(Err(relay_failure(
2400                    C2RelayFailureCode::NodeOffline,
2401                    "node relay disconnected before request dispatch",
2402                    cursor.map(|value| value.incarnation_id),
2403                )));
2404            }
2405        }
2406    }
2407}
2408
2409fn acknowledge_disconnected_releases(releases: &mut mpsc::Receiver<oneshot::Sender<()>>) {
2410    while let Ok(reply) = releases.try_recv() { let _ = reply.send(()); }
2411}
2412
2413fn validate_events(
2414    previous: u64,
2415    current: u64,
2416    oldest_available_sequence: u64,
2417    events: &[NodeEventEnvelope],
2418) -> Vec<GapKind> {
2419    if current < previous { return vec![GapKind::CursorRegression]; }
2420    let max_floor = current.checked_add(1).unwrap_or(u64::MAX);
2421    let minimum_event_sequence = previous
2422        .checked_add(1)
2423        .unwrap_or(u64::MAX)
2424        .max(oldest_available_sequence);
2425    if oldest_available_sequence == 0
2426        || oldest_available_sequence > max_floor
2427        || (current == previous && !events.is_empty())
2428        || events.iter().any(|event| {
2429            event.sequence < minimum_event_sequence || event.sequence > current
2430        })
2431        || events.windows(2).any(|pair| pair[0].sequence >= pair[1].sequence)
2432    {
2433        return vec![GapKind::NonContiguousEvents];
2434    }
2435    if previous.saturating_add(1) < oldest_available_sequence {
2436        return vec![GapKind::HistoryEvicted];
2437    }
2438    Vec::new()
2439}
2440
2441fn sanitize_node_error(error: &NodeClientError) -> (SanitizedError, bool) {
2442    let (category, message, hard) = match error {
2443        NodeClientError::Protocol(message) if message.contains("identity mismatch") =>
2444            (C2ErrorCategory::Identity, "node identity mismatch", true),
2445        NodeClientError::Protocol(message) if message.contains("access-token proof") || message.contains("access denied") =>
2446            (C2ErrorCategory::Authentication, "node authentication failed", true),
2447        NodeClientError::Protocol(_) | NodeClientError::Frame(FrameError::Json(_) | FrameError::InvalidLength { .. }) =>
2448            (C2ErrorCategory::Protocol, "node protocol failed", true),
2449        NodeClientError::BuildStampMismatch { .. } =>
2450            (C2ErrorCategory::Protocol, "node build stamp mismatch", true),
2451        NodeClientError::UnsupportedCapability(_) =>
2452            (C2ErrorCategory::Protocol, "node capability unavailable", true),
2453        NodeClientError::Node(failure) if failure.code == NodeFailureCode::Unauthorized =>
2454            (C2ErrorCategory::Authentication, "node request authentication failed", true),
2455        NodeClientError::Node(_) =>
2456            (C2ErrorCategory::Protocol, "node rejected observer request", true),
2457        NodeClientError::Frame(FrameError::BodyTimedOut { .. } | FrameError::PrefixTimedOut)
2458            | NodeClientError::AuthenticationTimedOut =>
2459            (C2ErrorCategory::Timeout, "node observation deadline exceeded", false),
2460        NodeClientError::Authentication(_) | NodeClientError::RequestIdExhausted =>
2461            (C2ErrorCategory::Internal, "node client failed internally", true),
2462        NodeClientError::Io(_) | NodeClientError::Frame(FrameError::Io(_)) =>
2463            (C2ErrorCategory::Transport, "node transport unavailable", false),
2464    };
2465    (SanitizedError { category, message: message.to_owned() }, hard)
2466}
2467
2468async fn inventory_owner(
2469    fresh_for: Duration, mut ingress: mpsc::Receiver<Attempt>,
2470    status: watch::Sender<Arc<StatusResponse>>, mut shutdown: watch::Receiver<bool>,
2471) -> io::Result<()> {
2472    let mut current = (**status.borrow()).clone();
2473    let mut attempted = BTreeSet::new();
2474    loop {
2475        tokio::select! {
2476            attempt = ingress.recv() => {
2477                let Some(attempt) = attempt else { return Ok(()); };
2478                attempted.insert(attempt.node_id.clone());
2479                let node = current.nodes.entry(attempt.node_id.clone()).or_insert_with(initial_enrolled_node);
2480                node.last_attempt_unix_ms = Some(attempt.at_unix_ms);
2481                match attempt.result {
2482                    AttemptResult::Connected {
2483                        cursor,
2484                        snapshot,
2485                        gaps,
2486                        provider_contract_manifest,
2487                    } => {
2488                        let previous = node.cursor;
2489                        let mut inventory = SlimNodeInventory::from_snapshot(&snapshot);
2490                        inventory.provider_contracts =
2491                            provider_contract_manifest.provider_contracts;
2492                        inventory.provider_adapter_contracts =
2493                            provider_contract_manifest.provider_adapter_contracts;
2494                        node.transport = NodeTransportState::Online;
2495                        node.freshness = NodeFreshness::Fresh;
2496                        node.cursor = Some(cursor);
2497                        node.inventory = Some(inventory);
2498                        node.last_success_unix_ms = Some(attempt.at_unix_ms);
2499                        node.consecutive_failures = 0;
2500                        node.last_error = None;
2501                        for kind in gaps {
2502                            if node.gaps.len() == MAX_C2_GAPS_PER_NODE { node.gaps.remove(0); node.gaps_truncated += 1; }
2503                            node.gaps.push(NodeGap { kind, detected_at_unix_ms: attempt.at_unix_ms, previous, observed: cursor });
2504                        }
2505                    }
2506                    AttemptResult::Success { cursor, snapshot, gaps } => {
2507                        let previous = node.cursor;
2508                        let provider_contract_manifest = node.inventory.as_ref().map(|inventory| {
2509                            ProviderContractManifest {
2510                                provider_contracts: inventory.provider_contracts.clone(),
2511                                provider_adapter_contracts: inventory.provider_adapter_contracts.clone(),
2512                            }
2513                        }).unwrap_or_default();
2514                        let mut inventory = SlimNodeInventory::from_snapshot(&snapshot);
2515                        inventory.provider_contracts =
2516                            provider_contract_manifest.provider_contracts;
2517                        inventory.provider_adapter_contracts =
2518                            provider_contract_manifest.provider_adapter_contracts;
2519                        node.transport = NodeTransportState::Online;
2520                        node.freshness = NodeFreshness::Fresh;
2521                        node.cursor = Some(cursor);
2522                        node.inventory = Some(inventory);
2523                        node.last_success_unix_ms = Some(attempt.at_unix_ms);
2524                        node.consecutive_failures = 0;
2525                        node.last_error = None;
2526                        for kind in gaps {
2527                            if node.gaps.len() == MAX_C2_GAPS_PER_NODE { node.gaps.remove(0); node.gaps_truncated += 1; }
2528                            node.gaps.push(NodeGap { kind, detected_at_unix_ms: attempt.at_unix_ms, previous, observed: cursor });
2529                        }
2530                    }
2531                    AttemptResult::Cursor {
2532                        cursor,
2533                        gaps,
2534                        managed_worktree_events,
2535                    } => {
2536                        let previous = node.cursor;
2537                        let incarnation_changed = previous.is_some_and(|previous| {
2538                            previous.incarnation_id != cursor.incarnation_id
2539                        });
2540                        apply_managed_worktree_cursor(
2541                            node.inventory.as_mut(),
2542                            incarnation_changed,
2543                            &managed_worktree_events,
2544                        );
2545                        node.transport = NodeTransportState::Online;
2546                        node.freshness = NodeFreshness::Fresh;
2547                        node.cursor = Some(cursor);
2548                        node.last_success_unix_ms = Some(attempt.at_unix_ms);
2549                        node.consecutive_failures = 0;
2550                        node.last_error = None;
2551                        for kind in gaps {
2552                            if node.gaps.len() == MAX_C2_GAPS_PER_NODE { node.gaps.remove(0); node.gaps_truncated += 1; }
2553                            node.gaps.push(NodeGap { kind, detected_at_unix_ms: attempt.at_unix_ms, previous, observed: cursor });
2554                        }
2555                    }
2556                    AttemptResult::Failure { error, hard } => {
2557                        node.consecutive_failures = node.consecutive_failures.saturating_add(1);
2558                        node.transport = if hard || node.consecutive_failures >= 5 { NodeTransportState::Parked } else { NodeTransportState::Offline };
2559                        node.last_error = Some(error);
2560                    }
2561                }
2562                current.ready = current.nodes.keys().all(|id| attempted.contains(id));
2563                refresh_freshness(&mut current, fresh_for);
2564                current.observed_at_unix_ms = unix_ms();
2565                status.send_replace(Arc::new(current.clone()));
2566            }
2567            _ = sleep(Duration::from_millis(250)) => {
2568                refresh_freshness(&mut current, fresh_for);
2569                current.observed_at_unix_ms = unix_ms();
2570                status.send_replace(Arc::new(current.clone()));
2571            }
2572            changed = shutdown.changed() => if changed.is_err() || *shutdown.borrow() { return Ok(()); },
2573        }
2574    }
2575}
2576
2577fn apply_managed_worktree_cursor(
2578    inventory: Option<&mut SlimNodeInventory>,
2579    incarnation_changed: bool,
2580    events: &[NodeEvent],
2581) {
2582    let Some(inventory) = inventory else { return; };
2583    if incarnation_changed {
2584        inventory.provider_runtime_statuses.clear();
2585        inventory.managed_worktrees.clear();
2586        inventory.managed_worktree_count = 0;
2587        inventory.managed_worktrees_truncated = false;
2588        return;
2589    }
2590    for event in events {
2591        match event {
2592            NodeEvent::ManagedWorktreeUpserted { lease } => {
2593                inventory.managed_worktrees.retain(|existing| {
2594                    existing.lease_id != lease.lease_id
2595                        && existing.workspace_id != lease.workspace_id
2596                });
2597                inventory.managed_worktrees.push(lease.clone());
2598                inventory.managed_worktrees.sort_by(|left, right| {
2599                    left.lease_id.cmp(&right.lease_id)
2600                });
2601                inventory.managed_worktree_count = inventory.managed_worktrees.len();
2602                inventory.managed_worktrees.truncate(
2603                    crate::protocol::MAX_C2_MANAGED_WORKTREES_PER_NODE,
2604                );
2605                inventory.managed_worktrees_truncated =
2606                    inventory.managed_worktrees.len() < inventory.managed_worktree_count;
2607            }
2608            NodeEvent::ManagedWorktreeRemoved { lease_id } => {
2609                let before = inventory.managed_worktrees.len();
2610                inventory
2611                    .managed_worktrees
2612                    .retain(|lease| &lease.lease_id != lease_id);
2613                if inventory.managed_worktrees.len() < before
2614                    || inventory.managed_worktrees_truncated
2615                {
2616                    inventory.managed_worktree_count =
2617                        inventory.managed_worktree_count.saturating_sub(1);
2618                }
2619                inventory.managed_worktrees_truncated =
2620                    inventory.managed_worktrees.len() < inventory.managed_worktree_count;
2621            }
2622            _ => {}
2623        }
2624    }
2625}
2626
2627fn refresh_freshness(status: &mut StatusResponse, fresh_for: Duration) {
2628    let now = unix_ms();
2629    let fresh_ms = fresh_for.as_millis().min(u64::MAX as u128) as u64;
2630    for node in status.nodes.values_mut() {
2631        node.freshness = match node.last_success_unix_ms {
2632            None => NodeFreshness::Unavailable,
2633            Some(last) if now.saturating_sub(last) <= fresh_ms => NodeFreshness::Fresh,
2634            Some(_) => NodeFreshness::Stale,
2635        };
2636    }
2637}
2638
2639async fn http_server(
2640    listener: TcpListener, token: String, io_deadline: Duration,
2641    status: watch::Receiver<Arc<StatusResponse>>,
2642    enroll: Option<Arc<crate::enroll_service::EnrollService>>,
2643    mut shutdown: watch::Receiver<bool>,
2644) -> io::Result<()> {
2645    let permits = Arc::new(Semaphore::new(MAX_HTTP_CONNECTIONS));
2646    let mut connections = JoinSet::new();
2647    loop {
2648        tokio::select! {
2649            accepted = listener.accept() => {
2650                let (stream, _) = accepted?;
2651                let Ok(permit) = Arc::clone(&permits).try_acquire_owned() else { drop(stream); continue; };
2652                let token = token.clone();
2653                let status = status.clone();
2654                let enroll = enroll.clone();
2655                connections.spawn(async move { let _permit = permit; let _ = serve_http(stream, &token, io_deadline, &status, enroll.as_deref()).await; });
2656            }
2657            changed = shutdown.changed() => if changed.is_err() || *shutdown.borrow() { break; },
2658        }
2659        while let Some(result) = connections.try_join_next() { result.map_err(io::Error::other)?; }
2660    }
2661    connections.shutdown().await;
2662    Ok(())
2663}
2664
2665async fn serve_http(mut stream: TcpStream, token: &str, deadline: Duration, status: &watch::Receiver<Arc<StatusResponse>>, enroll: Option<&crate::enroll_service::EnrollService>) -> io::Result<()> {
2666    let request = match timeout(deadline, read_request(&mut stream)).await {
2667        Ok(Ok(request)) => request,
2668        Ok(Err(ReadError::TooLarge)) => return write_response(&mut stream, deadline, Response::plain(413, "Payload Too Large")).await,
2669        Ok(Err(ReadError::Io(error))) => return Err(error),
2670        _ => return Ok(()),
2671    };
2672    let response = route(request, token, status.borrow().as_ref(), enroll);
2673    write_response(&mut stream, deadline, response).await
2674}
2675
2676struct Request { method: String, path: String, authorization: Option<String>, body: Vec<u8> }
2677enum ReadError { Closed, Invalid, TooLarge, Io(io::Error) }
2678
2679async fn read_request(stream: &mut TcpStream) -> Result<Request, ReadError> {
2680    let mut bytes = Vec::with_capacity(1024);
2681    let mut chunk = [0_u8; 1024];
2682    let header_end = loop {
2683        let count = stream.read(&mut chunk).await.map_err(ReadError::Io)?;
2684        if count == 0 { return Err(ReadError::Closed); }
2685        if bytes.len().saturating_add(count) > HEADER_LIMIT_BYTES + OPERATOR_BODY_LIMIT_BYTES { return Err(ReadError::TooLarge); }
2686        bytes.extend_from_slice(&chunk[..count]);
2687        if let Some(position) = bytes.windows(4).position(|window| window == b"\r\n\r\n") { break position + 4; }
2688    };
2689    let text = std::str::from_utf8(&bytes[..header_end]).map_err(|_| ReadError::Invalid)?;
2690    let mut lines = text.split("\r\n");
2691    let mut first = lines.next().ok_or(ReadError::Invalid)?.split_whitespace();
2692    let method = first.next().ok_or(ReadError::Invalid)?;
2693    let path = first.next().ok_or(ReadError::Invalid)?;
2694    let version = first.next().ok_or(ReadError::Invalid)?;
2695    if first.next().is_some() || !version.starts_with("HTTP/1.") || !path.starts_with('/') { return Err(ReadError::Invalid); }
2696    let mut authorization = None;
2697    let mut content_length = 0_usize;
2698    for line in lines {
2699        if line.is_empty() { break; }
2700        let (name, value) = line.split_once(':').ok_or(ReadError::Invalid)?;
2701        if name.eq_ignore_ascii_case("authorization") {
2702            if authorization.is_some() { return Err(ReadError::Invalid); }
2703            authorization = Some(value.trim().to_owned());
2704        } else if name.eq_ignore_ascii_case("content-length") {
2705            content_length = value.trim().parse().map_err(|_| ReadError::Invalid)?;
2706        }
2707    }
2708    if content_length > OPERATOR_BODY_LIMIT_BYTES { return Err(ReadError::TooLarge); }
2709    let mut body = bytes[header_end..].to_vec();
2710    while body.len() < content_length {
2711        let count = stream.read(&mut chunk).await.map_err(ReadError::Io)?;
2712        if count == 0 { return Err(ReadError::Closed); }
2713        body.extend_from_slice(&chunk[..count]);
2714    }
2715    body.truncate(content_length);
2716    Ok(Request { method: method.to_owned(), path: path.to_owned(), authorization, body })
2717}
2718
2719fn route(request: Request, token: &str, status: &StatusResponse, enroll: Option<&crate::enroll_service::EnrollService>) -> Response {
2720    let path = request.path.split_once('?').map_or(request.path.as_str(), |pair| pair.0);
2721    // Operator enrollment endpoints: bearer-authenticated, loopback API only.
2722    if path == "/v1/enroll" || path.starts_with("/v1/enroll/") {
2723        let Some(service) = enroll else { return Response::plain(404, "Not Found"); };
2724        if !authorized(request.authorization.as_deref(), token) { return Response::plain(401, "Unauthorized").authenticate(); }
2725        let reply = service.handle_operator(&request.method, path, &request.body);
2726        return Response::raw_json(reply.status, reply.body);
2727    }
2728    if request.method != "GET" { return Response::plain(405, "Method Not Allowed").allow_get(); }
2729    match path {
2730        "/health" => Response::json(200, &HealthResponse { ok: true, service: "gate4agent-c2".to_owned(), api_version: C2_API_VERSION, pid: std::process::id(), version: env!("CARGO_PKG_VERSION").to_owned() }),
2731        "/ready" => {
2732            let online_nodes = status.nodes.values().filter(|node| node.transport == NodeTransportState::Online).count();
2733            let offline_nodes = status.nodes.values().filter(|node| node.transport == NodeTransportState::Offline).count();
2734            let parked_nodes = status.nodes.values().filter(|node| node.transport == NodeTransportState::Parked).count();
2735            let body = ReadyResponse { ready: status.ready, api_version: C2_API_VERSION, configured_nodes: status.nodes.len(), attempted_nodes: status.nodes.values().filter(|node| node.last_attempt_unix_ms.is_some()).count(), online_nodes, offline_nodes, parked_nodes };
2736            Response::json(if status.ready { 200 } else { 503 }, &body)
2737        }
2738        "/status" => {
2739            if !authorized(request.authorization.as_deref(), token) { Response::plain(401, "Unauthorized").authenticate() }
2740            else {
2741                // Nodes whose status does not hold `attach` show liveness only.
2742                let mut shown = status.clone();
2743                if let Some(service) = enroll {
2744                    for (id, node) in shown.nodes.iter_mut() {
2745                        if service.allows(id.as_str(), "attach") == Some(false) { node.inventory = None; }
2746                    }
2747                }
2748                Response::json(200, &shown)
2749            }
2750        }
2751        _ => Response::plain(404, "Not Found"),
2752    }
2753}
2754
2755fn authorized(header: Option<&str>, token: &str) -> bool {
2756    let Some((scheme, candidate)) = header.and_then(|value| value.split_once(' ')) else { return false; };
2757    scheme.eq_ignore_ascii_case("bearer") && constant_time_eq(candidate.as_bytes(), token.as_bytes())
2758}
2759
2760fn constant_time_eq(left: &[u8], right: &[u8]) -> bool {
2761    if left.len() != right.len() { return false; }
2762    left.iter().zip(right).fold(0_u8, |difference, (left, right)| difference | (left ^ right)) == 0
2763}
2764
2765struct Response { status: u16, reason: &'static str, content_type: &'static str, body: Vec<u8>, headers: Vec<(&'static str, &'static str)> }
2766impl Response {
2767    fn plain(status: u16, reason: &'static str) -> Self { Self { status, reason, content_type: "text/plain; charset=utf-8", body: reason.as_bytes().to_vec(), headers: Vec::new() } }
2768    fn json<T: serde::Serialize>(status: u16, value: &T) -> Self {
2769        let body = serde_json::to_vec(value).expect("C2 DTO must serialize");
2770        if body.len() > RESPONSE_BODY_LIMIT_BYTES { return Self::plain(503, "Service Unavailable"); }
2771        let reason = if status == 200 { "OK" } else { "Service Unavailable" };
2772        Self { status, reason, content_type: "application/json", body, headers: Vec::new() }
2773    }
2774    fn raw_json(status: u16, body: Vec<u8>) -> Self {
2775        let reason = match status { 200 => "OK", 400 => "Bad Request", 404 => "Not Found", 409 => "Conflict", 429 => "Too Many Requests", _ => "Service Unavailable" };
2776        Self { status, reason, content_type: "application/json", body, headers: Vec::new() }
2777    }
2778    fn allow_get(mut self) -> Self { self.headers.push(("Allow", "GET")); self }
2779    fn authenticate(mut self) -> Self { self.headers.push(("WWW-Authenticate", "Bearer")); self }
2780}
2781
2782async fn write_response(stream: &mut TcpStream, deadline: Duration, response: Response) -> io::Result<()> {
2783    let mut headers = format!("HTTP/1.1 {} {}\r\nContent-Type: {}\r\nContent-Length: {}\r\nConnection: close\r\n", response.status, response.reason, response.content_type, response.body.len());
2784    for (name, value) in response.headers { headers.push_str(name); headers.push_str(": "); headers.push_str(value); headers.push_str("\r\n"); }
2785    headers.push_str("\r\n");
2786    timeout(deadline, async { stream.write_all(headers.as_bytes()).await?; stream.write_all(&response.body).await?; stream.shutdown().await }).await
2787        .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "C2 HTTP write timed out"))?
2788}
2789
2790fn unix_ms() -> u64 {
2791    SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis().min(u64::MAX as u128) as u64
2792}
2793
2794#[cfg(test)]
2795mod endpoint_tests {
2796    use super::*;
2797    use crate::protocol::{C2RelayRoute, C2Topology};
2798
2799    fn node(endpoint: &str) -> Result<C2NodeConfig, C2ConfigError> {
2800        C2NodeConfig::new(NodeId::new("remote-node").unwrap(), endpoint, "safe-token")
2801    }
2802
2803    #[cfg(windows)]
2804    const LOCAL_ENDPOINT: &str = r"\\.\pipe\relay-route-fact";
2805    #[cfg(unix)]
2806    const LOCAL_ENDPOINT: &str = "/tmp/gate4agent-relay-route-fact.sock";
2807
2808    fn projected_route(node: &C2NodeConfig, transport: NodeTransportState) -> C2RelayRoute {
2809        let mut observed = initial_observed_node(node);
2810        observed.transport = transport;
2811        let status = StatusResponse {
2812            api_version: C2_API_VERSION,
2813            ready: false,
2814            observed_at_unix_ms: 1,
2815            nodes: BTreeMap::from([(node.node_id.clone(), observed)]),
2816        };
2817        C2Topology::from_status(&status).nodes[0].relay_route
2818    }
2819
2820    #[test]
2821    fn relay_route_fact_projects_exactly_and_survives_offline_and_parked() {
2822        let local = node(LOCAL_ENDPOINT).unwrap();
2823        let ssh = node("tcp://127.0.0.1:48100").unwrap();
2824
2825        for transport in [
2826            NodeTransportState::Online,
2827            NodeTransportState::Offline,
2828            NodeTransportState::Parked,
2829        ] {
2830            assert_eq!(projected_route(&local, transport), C2RelayRoute::LocalIpc);
2831            assert_eq!(
2832                projected_route(&ssh, transport),
2833                C2RelayRoute::SshForwardedLoopback,
2834            );
2835        }
2836    }
2837
2838    /// `accept` is the assignment that names no address, and the config
2839    /// has to treat it as a different KIND of route rather than as a
2840    /// malformed endpoint: it is how a node that cannot be dialled is
2841    /// declared.
2842    #[test]
2843    fn a_node_that_waits_to_be_called_is_declared_by_name_not_by_address() {
2844        let waiting = node("accept").unwrap();
2845        assert_eq!(waiting.route, C2NodeRoute::CallHome);
2846        assert_eq!(waiting.transport_label(), "call-home");
2847
2848        // Every waiting node carries the same `accept`, so the
2849        // duplicate-endpoint check must not see them as two nodes fighting
2850        // over one socket. Their uniqueness is `node_id`'s job, and that
2851        // check still applies.
2852        let second = C2NodeConfig::new(
2853            NodeId::new("second-waiting-node").unwrap(),
2854            "accept",
2855            "safe-token",
2856        )
2857        .unwrap();
2858        let config = C2Config::new(
2859            "127.0.0.1:0".parse().unwrap(),
2860            "safe-token",
2861            vec![waiting.clone(), second],
2862        )
2863        .expect("two waiting nodes are not a duplicate endpoint");
2864        assert_eq!(config.nodes.len(), 2);
2865    }
2866
2867    /// A node that waits for a call the relay never listens for is a
2868    /// deployment that can never work, so it is refused at startup instead
2869    /// of becoming a node that is silently offline forever.
2870    #[test]
2871    fn waiting_for_a_call_requires_somewhere_to_be_called() {
2872        let waiting = node("accept").unwrap();
2873        let config =
2874            C2Config::new("127.0.0.1:0".parse().unwrap(), "safe-token", vec![waiting]).unwrap();
2875        assert!(matches!(
2876            config.validate_call_home(),
2877            Err(C2ConfigError::CallHomeWithoutListener(_)),
2878        ));
2879        let config = config.with_node_listen("127.0.0.1:48200".parse().unwrap()).unwrap();
2880        assert!(config.validate_call_home().is_ok());
2881    }
2882
2883    /// Call-home may bind loopback or a concrete C2 WireGuard tunnel address.
2884    /// Unspecified (`0.0.0.0`) and port 0 stay refused so plaintext frames do
2885    /// not land on every interface.
2886    #[test]
2887    fn the_call_home_listener_refuses_unspecified_and_accepts_tunnel() {
2888        let waiting = node("accept").unwrap();
2889        let config =
2890            C2Config::new("127.0.0.1:9000".parse().unwrap(), "safe-token", vec![waiting]).unwrap();
2891        for refused in ["0.0.0.0:48200", "127.0.0.1:0", "[::]:48200"] {
2892            assert!(
2893                matches!(
2894                    config.clone().with_node_listen(refused.parse().unwrap()),
2895                    Err(C2ConfigError::NonLoopbackNodeListen(_)),
2896                ),
2897                "accepted {refused}",
2898            );
2899        }
2900        let tunnel: SocketAddr = "10.88.1.2:27442".parse().unwrap();
2901        let accepted = config.clone().with_node_listen(tunnel).unwrap();
2902        assert_eq!(accepted.node_listen, Some(tunnel));
2903        // Sharing the API's own address would mean HTTP and node frames
2904        // arriving on one socket; neither parser would survive the other.
2905        assert!(matches!(
2906            config.with_node_listen("127.0.0.1:9000".parse().unwrap()),
2907            Err(C2ConfigError::NodeListenConflict),
2908        ));
2909    }
2910
2911    #[test]
2912    fn ssh_forwarded_loopback_route_is_strict_canonical_and_control_stays_local() {
2913        let ipv4 = node("tcp://127.0.0.1:48100").unwrap();
2914        assert_eq!(ipv4.endpoint, "tcp://127.0.0.1:48100");
2915        assert_eq!(ipv4.route, C2NodeRoute::SshForwardedLoopback("127.0.0.1:48100".parse().unwrap()));
2916        assert_eq!(ipv4.transport_label(), "ssh-forwarded-loopback");
2917
2918        let ipv6 = node("tcp://[0:0:0:0:0:0:0:1]:48100").unwrap();
2919        assert_eq!(ipv6.endpoint, "tcp://[::1]:48100");
2920        assert_eq!(ipv6.transport_label(), "ssh-forwarded-loopback");
2921
2922        for invalid in [
2923            "tcp://localhost:48100",
2924            "tcp://127.0.0.2:48100",
2925            "tcp://0.0.0.0:48100",
2926            "tcp://127.0.0.1:0",
2927            "tcp://user@127.0.0.1:48100",
2928            "tcp://127.0.0.1:48100/path",
2929            "TCP://127.0.0.1:48100",
2930        ] {
2931            assert!(matches!(node(invalid), Err(C2ConfigError::InvalidEndpoint(_))), "accepted {invalid}");
2932        }
2933
2934        assert!(matches!(
2935            C2Config::new(
2936                "127.0.0.1:0".parse().unwrap(),
2937                "safe-token",
2938                vec![ipv4.clone(), C2NodeConfig::new(
2939                    NodeId::new("duplicate-route").unwrap(),
2940                    "tcp://127.0.0.1:48100",
2941                    "safe-token",
2942                ).unwrap()],
2943            ),
2944            Err(C2ConfigError::DuplicateEndpoint)
2945        ));
2946        assert!(matches!(
2947            C2Config::new("127.0.0.1:0".parse().unwrap(), "safe-token", vec![ipv4])
2948                .unwrap()
2949                .with_control_endpoint("tcp://127.0.0.1:48101"),
2950            Err(C2ConfigError::InvalidControlEndpoint)
2951        ));
2952    }
2953}
2954
2955#[cfg(all(test, windows))]
2956mod tests {
2957    use super::*;
2958    use gate4agent_node_protocol::{
2959        AgentId, AgentStreamChunkKindV1, AgentStreamChunkV1, CapabilityId, SessionAddress,
2960        SessionKey, SessionMode, SessionRecordId,
2961        HarnessMcpActivationDigest, HarnessMcpCallId, HarnessMcpContentTypeV1,
2962        HarnessMcpOpaquePayloadV1, HarnessMcpReservationId,
2963        SpawnDeadlineMs, SpawnFieldProvenance, SpawnIdempotencyKey, SpawnOverrides,
2964        SpawnProfileId, SpawnProfileRevision, SpawnPrompt, SpawnPromptMetadata,
2965        SpawnRequiredCapabilities, SpawnResolutionProvenance, SpawnTarget, WorkspaceId,
2966        ManagedWorktreeCleanupFailure, ManagedWorktreeLeaseId,
2967        ManagedWorktreeLeaseSnapshot, ManagedWorktreeRetention, ManagedWorktreeSpawnReceipt,
2968        WorktreeProfileId, WorktreeProfileRevision,
2969    };
2970    use gate4agent_types::{AgentInstanceId, SessionGeneration, TerminalSize};
2971    use std::collections::BTreeMap;
2972
2973    fn agent(value: &str) -> gate4agent_node_protocol::AgentId {
2974        gate4agent_node_protocol::AgentId::new(value).unwrap()
2975    }
2976
2977    fn managed_lease(
2978        lease_id: &str,
2979        workspace_id: &str,
2980        state: ManagedWorktreeLeaseState,
2981    ) -> ManagedWorktreeLeaseSnapshot {
2982        let in_use = state == ManagedWorktreeLeaseState::InUse;
2983        ManagedWorktreeLeaseSnapshot {
2984            lease_id: ManagedWorktreeLeaseId::new(lease_id).unwrap(),
2985            source_workspace_id: WorkspaceId::new("repo").unwrap(),
2986            workspace_id: WorkspaceId::new(workspace_id).unwrap(),
2987            profile_id: WorktreeProfileId::new("review").unwrap(),
2988            profile_revision: WorktreeProfileRevision::new("review.r1").unwrap(),
2989            retention: ManagedWorktreeRetention::RemoveWhenReleased,
2990            state,
2991            active_session_count: u16::from(in_use),
2992            managed_record_count: u16::from(in_use),
2993            cleanup_failure: None::<ManagedWorktreeCleanupFailure>,
2994            created_at_unix_ms: 1,
2995            updated_at_unix_ms: 2,
2996        }
2997    }
2998
2999    #[test]
3000    fn spawn_spec_receipt_correlation_rejects_mismatches_before_forwarding() {
3001        let incarnation_id = NodeIncarnationId::from_bytes([7; 16]);
3002        let terminal_size = TerminalSize { rows: 24, columns: 80 };
3003        let prompt = SpawnPrompt::new("hi").unwrap();
3004        let required_capabilities = SpawnRequiredCapabilities::new([
3005            CapabilityId::new("raw-pty-lifecycle").unwrap(),
3006        ]).unwrap();
3007        let spec = SpawnSpec {
3008            target: SpawnTarget {
3009                node_id: NodeId::new("node-a").unwrap(),
3010                workspace_id: WorkspaceId::new("repo").unwrap(),
3011                worktree_id: None,
3012            },
3013            profile_id: SpawnProfileId::new("default").unwrap(),
3014            expected_profile_revision: SpawnProfileRevision::new("r1").unwrap(),
3015            overrides: SpawnOverrides {
3016                provider: SpawnOverride::Set { value: AgentId::new("codex").unwrap() },
3017                mode: SpawnOverride::Set { value: SessionMode::Pty },
3018                terminal_size: SpawnOverride::Set { value: terminal_size },
3019                prompt: SpawnOverride::Set { value: prompt.clone() },
3020                bundle_id: SpawnOverride::Clear,
3021                context_id: SpawnOverride::Clear,
3022                environment_profile_id: SpawnOverride::Clear,
3023                approval_level: None,
3024                network_allowlist: None,
3025                browser_profile_id: None,
3026            },
3027            deadline_ms: SpawnDeadlineMs::new(5_000).unwrap(),
3028            idempotency_key: SpawnIdempotencyKey::new("spawn-1").unwrap(),
3029            required_capabilities: required_capabilities.clone(),
3030        };
3031        let receipt = ResolvedSpawnReceipt {
3032            incarnation_id,
3033            session: SessionAddress {
3034                workspace_id: spec.target.workspace_id.clone(),
3035                session: SessionKey {
3036                    instance_id: AgentInstanceId(7),
3037                    generation: SessionGeneration(1),
3038                },
3039            },
3040            target: spec.target.clone(),
3041            profile_id: spec.profile_id.clone(),
3042            profile_revision: SpawnProfileRevision::new("r1").unwrap(),
3043            provider: AgentId::new("codex").unwrap(),
3044            mode: SessionMode::Pty,
3045            terminal_size,
3046            prompt: SpawnPromptMetadata::from_prompt(Some(&prompt)),
3047            bundle_id: None,
3048            bundle: None,
3049            context_id: None,
3050            context: None,
3051            environment_profile: None,
3052            deadline_ms: spec.deadline_ms,
3053            idempotency_key: spec.idempotency_key.clone(),
3054            required_capabilities,
3055            provenance: SpawnResolutionProvenance {
3056                provider: SpawnFieldProvenance::Override,
3057                mode: SpawnFieldProvenance::Override,
3058                terminal_size: SpawnFieldProvenance::Override,
3059                prompt: SpawnFieldProvenance::Override,
3060                bundle_id: SpawnFieldProvenance::Cleared,
3061                context_id: SpawnFieldProvenance::Cleared,
3062                environment_profile_id: SpawnFieldProvenance::Cleared,
3063            },
3064            harness_mcp_proxy: None,
3065        };
3066        let response = |receipt| Ok(NodeResponse::SpawnSpecAccepted { receipt });
3067        let expected = ExpectedSpawnRequest::Spec(spec.clone());
3068        assert!(validate_spawn_spec_response(
3069            Some(&expected),
3070            &response(receipt.clone()),
3071            incarnation_id,
3072        ).is_ok());
3073
3074        let mut mismatches = Vec::new();
3075        let mut changed = receipt.clone();
3076        changed.incarnation_id = NodeIncarnationId::from_bytes([8; 16]);
3077        mismatches.push(changed);
3078        let mut changed = receipt.clone();
3079        changed.target.node_id = NodeId::new("node-b").unwrap();
3080        mismatches.push(changed);
3081        let mut changed = receipt.clone();
3082        changed.profile_id = SpawnProfileId::new("other").unwrap();
3083        mismatches.push(changed);
3084        let mut changed = receipt.clone();
3085        changed.idempotency_key = SpawnIdempotencyKey::new("spawn-2").unwrap();
3086        mismatches.push(changed);
3087        let mut changed = receipt.clone();
3088        changed.deadline_ms = SpawnDeadlineMs::new(4_999).unwrap();
3089        mismatches.push(changed);
3090        let mut changed = receipt.clone();
3091        changed.required_capabilities = SpawnRequiredCapabilities::default();
3092        mismatches.push(changed);
3093        let mut changed = receipt.clone();
3094        changed.session.workspace_id = WorkspaceId::new("other").unwrap();
3095        mismatches.push(changed);
3096        let mut changed = receipt.clone();
3097        changed.provider = AgentId::new("claude").unwrap();
3098        mismatches.push(changed);
3099        let mut changed = receipt.clone();
3100        changed.prompt = SpawnPromptMetadata { present: false, byte_len: 0 };
3101        mismatches.push(changed);
3102        let mut changed = receipt.clone();
3103        changed.environment_profile = Some(
3104            gate4agent_node_protocol::ResolvedEnvironmentProfileReceipt {
3105                profile_id:
3106                    gate4agent_node_protocol::SpawnEnvironmentProfileId::new(
3107                        "local-default",
3108                    )
3109                    .unwrap(),
3110                profile_revision:
3111                    gate4agent_node_protocol::SpawnEnvironmentProfileRevision::new(
3112                        "local-default.r1",
3113                    )
3114                    .unwrap(),
3115                    network_allowlist: None,
3116                    browser_profile_id: None,
3117            },
3118        );
3119        mismatches.push(changed);
3120        let mut changed = receipt.clone();
3121        changed.bundle = Some(gate4agent_node_protocol::ResolvedBundleReceipt {
3122            id: gate4agent_node_protocol::SpawnBundleId::new("unexpected-bundle")
3123                .unwrap(),
3124            revision: gate4agent_node_protocol::SpawnBundleRevision::new(
3125                "unexpected-bundle.r1",
3126            )
3127            .unwrap(),
3128            digest: gate4agent_node_protocol::SpawnBundleDigest::new(format!(
3129                "sha256:{}",
3130                "a".repeat(64),
3131            ))
3132            .unwrap(),
3133        });
3134        mismatches.push(changed);
3135        let mut changed = receipt.clone();
3136        changed.context = Some(gate4agent_node_protocol::ResolvedContextPackReceipt {
3137            id: gate4agent_node_protocol::SpawnContextId::new("unexpected-context")
3138                .unwrap(),
3139            digest: gate4agent_node_protocol::SpawnContextDigest::new(format!(
3140                "sha256:{}",
3141                "b".repeat(64),
3142            ))
3143            .unwrap(),
3144            lineage: gate4agent_node_protocol::ContextPackLineageReceipt {
3145                source_node_id: NodeId::new("node-a").unwrap(),
3146                source_session: receipt.session.clone(),
3147                source_provider: AgentId::new("codex").unwrap(),
3148            },
3149            source_message_count: 1,
3150            retained_message_count: 1,
3151            byte_len: 16,
3152            truncated: false,
3153        });
3154        mismatches.push(changed);
3155
3156        for mismatch in mismatches {
3157            assert!(validate_spawn_spec_response(
3158                Some(&expected),
3159                &response(mismatch),
3160                incarnation_id,
3161            ).is_err());
3162        }
3163
3164        let mut environment_spec = spec;
3165        environment_spec.overrides.environment_profile_id = SpawnOverride::Set {
3166            value: gate4agent_node_protocol::SpawnEnvironmentProfileId::new(
3167                "local-default",
3168            )
3169            .unwrap(),
3170        };
3171        let environment_expected = ExpectedSpawnRequest::Spec(environment_spec);
3172        let mut environment_receipt = receipt;
3173        environment_receipt.environment_profile = Some(
3174            gate4agent_node_protocol::ResolvedEnvironmentProfileReceipt {
3175                profile_id:
3176                    gate4agent_node_protocol::SpawnEnvironmentProfileId::new(
3177                        "local-default",
3178                    )
3179                    .unwrap(),
3180                profile_revision:
3181                    gate4agent_node_protocol::SpawnEnvironmentProfileRevision::new(
3182                        "local-default.r1",
3183                    )
3184                    .unwrap(),
3185                    network_allowlist: None,
3186                    browser_profile_id: None,
3187            },
3188        );
3189        assert!(validate_spawn_spec_response(
3190            Some(&environment_expected),
3191            &response(environment_receipt.clone()),
3192            incarnation_id,
3193        )
3194        .is_ok());
3195        environment_receipt.environment_profile.as_mut().unwrap().profile_id =
3196            gate4agent_node_protocol::SpawnEnvironmentProfileId::new("other")
3197                .unwrap();
3198        assert!(validate_spawn_spec_response(
3199            Some(&environment_expected),
3200            &response(environment_receipt),
3201            incarnation_id,
3202        )
3203        .is_err());
3204    }
3205
3206    #[test]
3207    fn provider_session_index_correlation_accepts_exact_identity_and_rejects_mismatches() {
3208        let identity = gate4agent_types::ProviderSessionIdentity {
3209            key: gate4agent_types::ProviderSessionKey::SessionId,
3210            id: "native-session-1".to_owned(),
3211            transcript_path: Some(r"C:\provider\sessions\native-session-1.jsonl".to_owned()),
3212        };
3213        let expected = NodeRequest::IndexProviderSession {
3214            workspace_id: WorkspaceId::new("primary").unwrap(),
3215            provider: AgentId::new("codex").unwrap(),
3216            identity: identity.clone(),
3217            display_name: "release shepherd".to_owned(),
3218        };
3219        let NodeRequest::IndexProviderSession { workspace_id, provider, .. } = &expected else {
3220            unreachable!();
3221        };
3222        let record = gate4agent_node_protocol::ManagedSessionRecord {
3223            record_id: SessionRecordId::new("session-001").unwrap(),
3224            display_name: "release shepherd".to_owned(),
3225            provider: provider.clone(),
3226            mode: SessionMode::Pty,
3227            state: gate4agent_node_protocol::ManagedSessionState::Dormant,
3228            workspace_id: workspace_id.clone(),
3229            canonical_root: gate4agent_node_protocol::OpaqueHostPath::utf8(
3230                r"C:\repo".to_owned(),
3231            ).unwrap(),
3232            provider_session: Some(identity),
3233            active_session: None,
3234            environment_profile: None,
3235            bundle: None,
3236            context_id: None,
3237            context: None,
3238            exported_context: None,
3239            task_binding: None,
3240            created_at_unix_ms: 1,
3241            updated_at_unix_ms: 2,
3242            last_error: None,
3243        };
3244        let response = |record| Ok(NodeResponse::ProviderSessionIndexed { record });
3245
3246        assert!(validate_provider_session_index_response(
3247            Some(&expected),
3248            &response(record.clone()),
3249        ).is_ok());
3250        assert!(validate_provider_session_index_response(
3251            None,
3252            &response(record.clone()),
3253        ).is_err());
3254
3255        let mut identity_mismatch = record.clone();
3256        identity_mismatch.provider_session.as_mut().unwrap().id =
3257            "native-session-2".to_owned();
3258        assert!(validate_provider_session_index_response(
3259            Some(&expected),
3260            &response(identity_mismatch),
3261        ).is_err());
3262
3263        let mut workspace_mismatch = record.clone();
3264        workspace_mismatch.workspace_id = WorkspaceId::new("other").unwrap();
3265        assert!(validate_provider_session_index_response(
3266            Some(&expected),
3267            &response(workspace_mismatch),
3268        ).is_err());
3269
3270        let mut provider_mismatch = record;
3271        provider_mismatch.provider = AgentId::new("claude").unwrap();
3272        assert!(validate_provider_session_index_response(
3273            Some(&expected),
3274            &response(provider_mismatch),
3275        ).is_err());
3276    }
3277
3278    #[test]
3279    fn native_session_route_correlation_rejects_mismatches_before_projection() {
3280        let route = gate4agent_node_protocol::NativeSessionCatalogRoute::workspace(
3281            WorkspaceId::new("primary").unwrap(),
3282            AgentId::new("codex").unwrap(),
3283        );
3284        let selection = gate4agent_node_protocol::NativeSessionSelection {
3285            route: route.clone(),
3286            catalog_revision: 7,
3287            recent_cutoff_unix_ms: 70,
3288            selection_id: "selection-7".to_owned(),
3289        };
3290        let catalog = NodeRequest::CatalogNativeSessions {
3291            route: route.clone(),
3292            limit: 10,
3293        };
3294        let catalog_response = Ok(NodeResponse::NativeSessionsCataloged {
3295            route: route.clone(),
3296            entries: Vec::new(),
3297            summary: None,
3298        });
3299        assert!(validate_native_session_response(Some(&catalog), &catalog_response).is_ok());
3300        let mismatched_route = gate4agent_node_protocol::NativeSessionCatalogRoute::workspace(
3301            WorkspaceId::new("other").unwrap(),
3302            AgentId::new("codex").unwrap(),
3303        );
3304        assert!(validate_native_session_response(
3305            Some(&catalog),
3306            &Ok(NodeResponse::NativeSessionsCataloged {
3307                route: mismatched_route,
3308                entries: Vec::new(),
3309                summary: None,
3310            }),
3311        )
3312        .is_err());
3313
3314        let page = NodeRequest::PageNativeSessions {
3315            route: route.clone(),
3316            window: gate4agent_node_protocol::NativeSessionCatalogWindow::Recent,
3317            catalog_revision: 7,
3318            recent_cutoff_unix_ms: 70,
3319            after_selection_id: None,
3320            limit: 10,
3321        };
3322        let page_response = |revision| {
3323            Ok(NodeResponse::NativeSessionsPaged {
3324                route: route.clone(),
3325                page: gate4agent_node_protocol::NativeSessionCatalogPage {
3326                    window: gate4agent_node_protocol::NativeSessionCatalogWindow::Recent,
3327                    revision,
3328                    entries: Vec::new(),
3329                    next_after_selection_id: None,
3330                    remaining_count: 0,
3331                    has_more: false,
3332                },
3333            })
3334        };
3335        assert!(validate_native_session_response(Some(&page), &page_response(7)).is_ok());
3336        assert!(validate_native_session_response(Some(&page), &page_response(8)).is_err());
3337
3338        let preview = NodeRequest::PreviewNativeSession {
3339            selection: selection.clone(),
3340            message_limit: 10,
3341        };
3342        let preview_response = |selection| {
3343            Ok(NodeResponse::NativeSessionPreviewed {
3344                selection,
3345                preview: gate4agent_node_protocol::SessionRecordPreview {
3346                    title: None,
3347                    modified_at_unix_ms: None,
3348                    model: None,
3349                    message_count: 0,
3350                    message_count_exact: true,
3351                    completed_turn_count: None,
3352                    total_tokens: None,
3353                    truncated: false,
3354                    messages: Vec::new(),
3355                },
3356            })
3357        };
3358        assert!(validate_native_session_response(
3359            Some(&preview),
3360            &preview_response(selection.clone()),
3361        )
3362        .is_ok());
3363        let mut mismatched_selection = selection.clone();
3364        mismatched_selection.catalog_revision = 8;
3365        assert!(validate_native_session_response(
3366            Some(&preview),
3367            &preview_response(mismatched_selection),
3368        )
3369        .is_err());
3370
3371        let index = NodeRequest::IndexNativeSession {
3372            selection: selection.clone(),
3373            display_name: "Indexed".to_owned(),
3374        };
3375        let record = gate4agent_node_protocol::ManagedSessionRecord {
3376            record_id: SessionRecordId::new("session-007").unwrap(),
3377            display_name: "Indexed".to_owned(),
3378            provider: route.provider.clone(),
3379            mode: SessionMode::Pty,
3380            state: gate4agent_node_protocol::ManagedSessionState::Dormant,
3381            workspace_id: route.workspace_id.clone().unwrap(),
3382            canonical_root: gate4agent_node_protocol::OpaqueHostPath::utf8(
3383                r"C:\repo".to_owned(),
3384            )
3385            .unwrap(),
3386            provider_session: None,
3387            active_session: None,
3388            environment_profile: None,
3389            bundle: None,
3390            context_id: None,
3391            context: None,
3392            exported_context: None,
3393            task_binding: None,
3394            created_at_unix_ms: 1,
3395            updated_at_unix_ms: 2,
3396            last_error: None,
3397        };
3398        assert!(validate_native_session_response(
3399            Some(&index),
3400            &Ok(NodeResponse::NativeSessionIndexed {
3401                selection: selection.clone(),
3402                record: record.clone(),
3403            }),
3404        )
3405        .is_ok());
3406        assert!(validate_native_session_response(
3407            Some(&index),
3408            &Ok(NodeResponse::ProviderSessionIndexed {
3409                record: record.clone(),
3410            }),
3411        )
3412        .is_err());
3413        let mut wrong_echo = selection.clone();
3414        wrong_echo.catalog_revision = 8;
3415        assert!(validate_native_session_response(
3416            Some(&index),
3417            &Ok(NodeResponse::NativeSessionIndexed {
3418                selection: wrong_echo,
3419                record: record.clone(),
3420            }),
3421        )
3422        .is_err());
3423        let mut wrong_provider = record.clone();
3424        wrong_provider.provider = AgentId::new("claude").unwrap();
3425        assert!(validate_native_session_response(
3426            Some(&index),
3427            &Ok(NodeResponse::NativeSessionIndexed {
3428                selection: selection.clone(),
3429                record: wrong_provider.clone(),
3430            }),
3431        )
3432        .is_err());
3433        let mut wrong_workspace = record.clone();
3434        wrong_workspace.workspace_id = WorkspaceId::new("other").unwrap();
3435        assert!(validate_native_session_response(
3436            Some(&index),
3437            &Ok(NodeResponse::NativeSessionIndexed {
3438                selection: selection.clone(),
3439                record: wrong_workspace,
3440            }),
3441        )
3442        .is_err());
3443
3444        let external = NodeRequest::IndexNativeSession {
3445            selection: gate4agent_node_protocol::NativeSessionSelection {
3446                route: gate4agent_node_protocol::NativeSessionCatalogRoute::unregistered(
3447                    AgentId::new("codex").unwrap(),
3448                ),
3449                ..selection
3450            },
3451            display_name: "External".to_owned(),
3452        };
3453        let NodeRequest::IndexNativeSession {
3454            selection: external_selection,
3455            ..
3456        } = &external else {
3457            unreachable!();
3458        };
3459        assert!(validate_native_session_response(
3460            Some(&external),
3461            &Ok(NodeResponse::NativeSessionIndexed {
3462                selection: external_selection.clone(),
3463                record: gate4agent_node_protocol::ManagedSessionRecord {
3464                    provider: AgentId::new("codex").unwrap(),
3465                    ..wrong_provider
3466                },
3467            }),
3468        )
3469        .is_err());
3470    }
3471
3472    #[test]
3473    fn managed_spawn_receipt_correlation_covers_legacy_and_v2_requests() {
3474        let incarnation_id = NodeIncarnationId::from_bytes([7; 16]);
3475        let spec = SpawnSpec {
3476            target: SpawnTarget {
3477                node_id: NodeId::new("node-a").unwrap(),
3478                workspace_id: WorkspaceId::new("repo").unwrap(),
3479                worktree_id: None,
3480            },
3481            profile_id: SpawnProfileId::new("default").unwrap(),
3482            expected_profile_revision:
3483                SpawnProfileRevision::new("default.r1").unwrap(),
3484            overrides: SpawnOverrides::default(),
3485            deadline_ms: SpawnDeadlineMs::new(5_000).unwrap(),
3486            idempotency_key: SpawnIdempotencyKey::new("managed-1").unwrap(),
3487            required_capabilities: SpawnRequiredCapabilities::default(),
3488        };
3489        let managed = ManagedWorktreeSpawnRequest {
3490            spawn_spec: spec.clone(),
3491            worktree_profile_id: WorktreeProfileId::new("review").unwrap(),
3492        };
3493        let workspace_id = WorkspaceId::new("managed-a").unwrap();
3494        let spawn = ResolvedSpawnReceipt {
3495            incarnation_id,
3496            session: SessionAddress {
3497                workspace_id: workspace_id.clone(),
3498                session: SessionKey {
3499                    instance_id: AgentInstanceId(8),
3500                    generation: SessionGeneration(1),
3501                },
3502            },
3503            target: SpawnTarget {
3504                node_id: spec.target.node_id.clone(),
3505                workspace_id: spec.target.workspace_id.clone(),
3506                worktree_id: Some(workspace_id.clone()),
3507            },
3508            profile_id: spec.profile_id.clone(),
3509            profile_revision: SpawnProfileRevision::new("default.r1").unwrap(),
3510            provider: AgentId::new("claude").unwrap(),
3511            mode: SessionMode::Pty,
3512            terminal_size: TerminalSize { rows: 24, columns: 80 },
3513            prompt: SpawnPromptMetadata { present: false, byte_len: 0 },
3514            bundle_id: None,
3515            bundle: None,
3516            context_id: None,
3517            context: None,
3518            environment_profile: None,
3519            deadline_ms: spec.deadline_ms,
3520            idempotency_key: spec.idempotency_key.clone(),
3521            required_capabilities: SpawnRequiredCapabilities::default(),
3522            provenance: SpawnResolutionProvenance {
3523                provider: SpawnFieldProvenance::Profile,
3524                mode: SpawnFieldProvenance::Profile,
3525                terminal_size: SpawnFieldProvenance::Profile,
3526                prompt: SpawnFieldProvenance::Profile,
3527                bundle_id: SpawnFieldProvenance::Profile,
3528                context_id: SpawnFieldProvenance::Profile,
3529                environment_profile_id: SpawnFieldProvenance::Profile,
3530            },
3531            harness_mcp_proxy: None,
3532        };
3533        let receipt = ManagedWorktreeSpawnReceipt {
3534            spawn,
3535            lease: managed_lease(
3536                "lease-a",
3537                "managed-a",
3538                ManagedWorktreeLeaseState::InUse,
3539            ),
3540        };
3541        let expected = ExpectedSpawnRequest::ManagedV1(managed.clone());
3542        let response = |receipt| Ok(NodeResponse::ManagedWorktreeSpawnAccepted { receipt });
3543        assert!(validate_spawn_spec_response(
3544            Some(&expected),
3545            &response(receipt.clone()),
3546            incarnation_id,
3547        )
3548        .is_ok());
3549
3550        let mut mismatches = Vec::new();
3551        let mut changed = receipt.clone();
3552        changed.lease.state = ManagedWorktreeLeaseState::Ready;
3553        mismatches.push(changed);
3554        let mut changed = receipt.clone();
3555        changed.lease.cleanup_failure = Some(ManagedWorktreeCleanupFailure::Busy);
3556        mismatches.push(changed);
3557        let mut changed = receipt.clone();
3558        changed.lease.active_session_count = 0;
3559        mismatches.push(changed);
3560        let mut changed = receipt.clone();
3561        changed.lease.profile_id = WorktreeProfileId::new("other").unwrap();
3562        mismatches.push(changed);
3563        let mut changed = receipt.clone();
3564        changed.lease.source_workspace_id = WorkspaceId::new("other").unwrap();
3565        mismatches.push(changed);
3566
3567        for mismatch in mismatches {
3568            assert!(validate_spawn_spec_response(
3569                Some(&expected),
3570                &response(mismatch),
3571                incarnation_id,
3572            )
3573            .is_err());
3574        }
3575
3576        let mut legacy_revision = receipt.clone();
3577        legacy_revision.lease.profile_revision =
3578            WorktreeProfileRevision::new("review.r2").unwrap();
3579        assert!(validate_spawn_spec_response(
3580            Some(&expected),
3581            &response(legacy_revision),
3582            incarnation_id,
3583        )
3584        .is_ok());
3585
3586        let v2_request = ManagedWorktreeSpawnRequestV2 {
3587            spawn_spec: managed.spawn_spec,
3588            worktree_profile_id: managed.worktree_profile_id,
3589            expected_profile_revision: WorktreeProfileRevision::new("review.r1").unwrap(),
3590        };
3591        let routed = NodeRequest::SpawnManagedWorktreeV2 {
3592            request: v2_request.clone(),
3593        };
3594        let captured_expected = expected_spawn_request(&routed).unwrap();
3595        let ExpectedSpawnRequest::ManagedV2(captured_request) = &captured_expected else {
3596            panic!("V2 managed spawn request was not captured separately");
3597        };
3598        assert_eq!(captured_request, &v2_request);
3599        assert!(validate_spawn_spec_response(
3600            Some(&captured_expected),
3601            &response(receipt.clone()),
3602            incarnation_id,
3603        )
3604        .is_ok());
3605
3606        let mut wrong_revision = receipt.clone();
3607        wrong_revision.lease.profile_revision =
3608            WorktreeProfileRevision::new("review.r2").unwrap();
3609        assert!(validate_spawn_spec_response(
3610            Some(&ExpectedSpawnRequest::ManagedV2(v2_request.clone())),
3611            &response(wrong_revision),
3612            incarnation_id,
3613        )
3614        .is_err());
3615        assert!(validate_spawn_spec_response(
3616            Some(&ExpectedSpawnRequest::ManagedV2(v2_request)),
3617            &Ok(NodeResponse::SpawnSpecAccepted {
3618                receipt: receipt.spawn,
3619            }),
3620            incarnation_id,
3621        )
3622        .is_err());
3623    }
3624
3625    #[test]
3626    fn windows_runtime_default_control_endpoint_is_exact_valid_and_distinct_from_a_node() {
3627        assert_eq!(DEFAULT_C2_CONTROL_ENDPOINT, r"\\.\pipe\gate4agent-c2");
3628        validate_control_endpoint(DEFAULT_C2_CONTROL_ENDPOINT).unwrap();
3629        let node = C2NodeConfig::new(
3630            NodeId::new("node-a").unwrap(),
3631            r"\\.\pipe\gate4agent-node",
3632            "safe-token",
3633        )
3634        .unwrap();
3635        let config = C2Config::new(
3636            "127.0.0.1:0".parse().unwrap(),
3637            "safe-token",
3638            vec![node],
3639        )
3640        .unwrap();
3641        assert_eq!(config.control_endpoint, DEFAULT_C2_CONTROL_ENDPOINT);
3642        assert!(!config.nodes[0]
3643            .endpoint
3644            .eq_ignore_ascii_case(&config.control_endpoint));
3645
3646        let conflicting_node = C2NodeConfig::new(
3647            NodeId::new("node-b").unwrap(),
3648            DEFAULT_C2_CONTROL_ENDPOINT,
3649            "safe-token",
3650        )
3651        .unwrap();
3652        assert!(matches!(
3653            C2Config::new(
3654                "127.0.0.1:0".parse().unwrap(),
3655                "safe-token",
3656                vec![conflicting_node],
3657            ),
3658            Err(C2ConfigError::ControlEndpointConflict)
3659        ));
3660    }
3661
3662    #[test]
3663    fn terminal_only_sequence_holes_do_not_mark_c2_partial() {
3664        use gate4agent_node_protocol::{NodeEvent, WorkspaceId};
3665        let event = |sequence| NodeEventEnvelope { sequence, event: NodeEvent::WorkspaceRemoved { workspace_id: WorkspaceId::new("work").unwrap() } };
3666        assert_eq!(validate_events(4, 7, 1, &[event(6)]), Vec::<GapKind>::new());
3667        assert_eq!(validate_events(4, 7, 1, &[]), Vec::<GapKind>::new());
3668        assert_eq!(validate_events(4, 7, 1, &[event(7), event(6)]), vec![GapKind::NonContiguousEvents]);
3669        assert_eq!(validate_events(7, 6, 1, &[]), vec![GapKind::CursorRegression]);
3670        assert_eq!(validate_resync(4, 6, 4, 1, &[]), vec![GapKind::CursorRegression]);
3671    }
3672
3673    #[test]
3674    fn real_durable_eviction_marks_c2_history_evicted() {
3675        use gate4agent_node_protocol::{NodeEvent, WorkspaceId};
3676        let event = |sequence| NodeEventEnvelope {
3677            sequence,
3678            event: NodeEvent::WorkspaceRemoved {
3679                workspace_id: WorkspaceId::new("work").unwrap(),
3680            },
3681        };
3682        assert_eq!(
3683            validate_events(4, 7, 6, &[event(6)]),
3684            vec![GapKind::HistoryEvicted],
3685        );
3686        assert_eq!(validate_events(5, 7, 6, &[event(6)]), Vec::<GapKind>::new());
3687    }
3688
3689    #[test]
3690    fn live_event_gap_preserves_cursor_and_resync_rules() {
3691        use gate4agent_node_protocol::{NodeEvent, WorkspaceId};
3692        let event = |sequence, event| NodeEventEnvelope { sequence, event };
3693        let removed = || NodeEvent::WorkspaceRemoved {
3694            workspace_id: WorkspaceId::new("work").unwrap(),
3695        };
3696
3697        assert_eq!(live_event_gap(4, &event(5, removed())), None);
3698        assert_eq!(
3699            live_event_gap(4, &event(4, removed())),
3700            Some(GapKind::CursorRegression),
3701        );
3702        assert_eq!(
3703            live_event_gap(4, &event(7, removed())),
3704            Some(GapKind::NonContiguousEvents),
3705        );
3706        assert_eq!(
3707            live_event_gap(4, &event(5, NodeEvent::ResyncRequired {
3708                oldest_available_sequence: 3,
3709            })),
3710            Some(GapKind::HistoryEvicted),
3711        );
3712    }
3713
3714    #[test]
3715    fn harness_mcp_transient_live_and_pending_projection_preserves_cursor_without_replay() {
3716        let node_id = NodeId::new("node-a").unwrap();
3717        let cursor = NodeCursor {
3718            incarnation_id: NodeIncarnationId::from_bytes([7; 16]),
3719            sequence: 41,
3720        };
3721        let envelope = NodeEventEnvelope {
3722            sequence: 0,
3723            event: NodeEvent::HarnessMcpReadCall {
3724                reservation_id: HarnessMcpReservationId::new(
3725                    format!("hmcpres_{}", "a".repeat(24)),
3726                ).unwrap(),
3727                activation_digest: HarnessMcpActivationDigest::new(
3728                    format!("sha256:{}", "b".repeat(64)),
3729                ).unwrap(),
3730                record_id: SessionRecordId::new("session-001").unwrap(),
3731                session: SessionAddress {
3732                    workspace_id: WorkspaceId::new("repo").unwrap(),
3733                    session: SessionKey {
3734                        instance_id: AgentInstanceId(8),
3735                        generation: SessionGeneration(1),
3736                    },
3737                },
3738                call_id: HarnessMcpCallId::new(
3739                    format!("hmcpcall_{}", "c".repeat(24)),
3740                ).unwrap(),
3741                request: HarnessMcpOpaquePayloadV1 {
3742                    content_type: HarnessMcpContentTypeV1::HarnessReadRequestJsonV1,
3743                    body: br#"{"kind":"context-get"}"#.to_vec(),
3744                },
3745                deadline_unix_ms: u64::MAX,
3746            },
3747        };
3748
3749        let live = routed_transient_node_event(&node_id, cursor, &envelope).unwrap();
3750        let pending = routed_transient_node_event(&node_id, cursor, &envelope).unwrap();
3751        assert_eq!(live, pending);
3752        assert_eq!(live.cursor, cursor);
3753        assert!(matches!(live.event, C2NodeEvent::HarnessMcpReadCall { .. }));
3754        assert!(routed_recovered_node_event(
3755            &node_id,
3756            cursor.incarnation_id,
3757            &envelope,
3758        ).is_none());
3759        assert_eq!(cursor.sequence, 41);
3760    }
3761
3762    fn agent_stream_test_address() -> SessionAddress {
3763        SessionAddress {
3764            workspace_id: WorkspaceId::new("repo").unwrap(),
3765            session: SessionKey {
3766                instance_id: AgentInstanceId(8),
3767                generation: SessionGeneration(1),
3768            },
3769        }
3770    }
3771
3772    fn agent_stream_test_envelope(sequence: u64, source_sequence: u64) -> NodeEventEnvelope {
3773        NodeEventEnvelope {
3774            sequence,
3775            event: NodeEvent::AgentStream {
3776                address: agent_stream_test_address(),
3777                chunk: AgentStreamChunkV1 {
3778                    source_sequence,
3779                    kind: AgentStreamChunkKindV1::Text {
3780                        text: format!("chunk-{source_sequence}"),
3781                        is_delta: true,
3782                    },
3783                },
3784            },
3785        }
3786    }
3787
3788    #[test]
3789    fn agent_stream_chunk_publishes_unconditionally_and_only_advances_cursor_forward() {
3790        let node_id = NodeId::new("node-a").unwrap();
3791        let mut cursor = NodeCursor {
3792            incarnation_id: NodeIncarnationId::from_bytes([7; 16]),
3793            sequence: 674,
3794        };
3795
3796        // A non-`AgentStream` envelope is not this function's to route.
3797        let control = NodeEventEnvelope {
3798            sequence: 675,
3799            event: NodeEvent::WorkspaceRemoved {
3800                workspace_id: WorkspaceId::new("work").unwrap(),
3801            },
3802        };
3803        assert!(route_agent_stream_event(&node_id, &mut cursor, &control).is_none());
3804        assert_eq!(cursor.sequence, 674);
3805
3806        // A chunk one past the cursor is published, and its OWN cursor
3807        // field on the wire reads the cursor as it stood before this
3808        // chunk (mirroring `HarnessMcpReadCall`'s convention exactly) --
3809        // then the live cursor moves forward to make room for it.
3810        let forward = agent_stream_test_envelope(675, 22);
3811        let routed = route_agent_stream_event(&node_id, &mut cursor, &forward).unwrap();
3812        assert_eq!(routed.cursor.sequence, 674);
3813        assert!(matches!(
3814            &routed.event,
3815            C2NodeEvent::AgentStream { chunk, .. } if chunk.source_sequence == 22
3816        ));
3817        assert_eq!(cursor.sequence, 675);
3818
3819        // A chunk that arrives (or, after a durable resync already carried
3820        // the cursor past it, is only now examined) BEHIND the cursor is
3821        // still published -- chunks promise no resync, so there is nothing
3822        // to recover if this one were dropped instead -- but it must never
3823        // rewind the cursor a genuinely later `Control` envelope already
3824        // advanced past.
3825        cursor.sequence = 711;
3826        let late = agent_stream_test_envelope(677, 23);
3827        let routed = route_agent_stream_event(&node_id, &mut cursor, &late).unwrap();
3828        assert_eq!(routed.cursor.sequence, 711);
3829        assert!(matches!(
3830            &routed.event,
3831            C2NodeEvent::AgentStream { chunk, .. } if chunk.source_sequence == 23
3832        ));
3833        assert_eq!(cursor.sequence, 711, "a late chunk must never rewind the cursor");
3834    }
3835
3836    #[test]
3837    fn agent_stream_burst_survives_a_durable_cursor_that_already_ran_past_it() {
3838        // Reproduces the measured live defect's mechanism at its worst: the
3839        // node's connection loop can drain its durable channel to
3840        // exhaustion across several ticks before this dedicated channel
3841        // gets any budget at all, so a whole burst's `Control` envelopes
3842        // can reach this relay, and carry `cursor` forward, before a
3843        // single one of the SAME burst's interleaved `AgentStream` chunks
3844        // is even looked at. `route_agent_stream_event` must publish every
3845        // one of them regardless.
3846        let node_id = NodeId::new("node-a").unwrap();
3847        let mut cursor = NodeCursor {
3848            incarnation_id: NodeIncarnationId::from_bytes([9; 16]),
3849            sequence: 0,
3850        };
3851        let chunk_count = 200_u64;
3852
3853        for turn in 1..=chunk_count {
3854            // Durable (`Control`) processing advancing the cursor is the
3855            // pre-existing, unchanged path (`live_event_gap` and its
3856            // resync repair, exercised by this module's other tests) --
3857            // simulated here only by its end effect on `cursor`, since
3858            // this test is about the `AgentStream` side's own contract.
3859            cursor.sequence = turn * 2;
3860        }
3861
3862        let mut recovered = Vec::new();
3863        for turn in 1..=chunk_count {
3864            let envelope = agent_stream_test_envelope(turn * 2 - 1, turn);
3865            let routed = route_agent_stream_event(&node_id, &mut cursor, &envelope)
3866                .expect("every chunk in the burst must be published, however far the durable cursor already ran past its sequence");
3867            recovered.push(routed);
3868        }
3869
3870        assert_eq!(recovered.len(), chunk_count as usize);
3871        for (turn, routed) in (1..=chunk_count).zip(recovered.iter()) {
3872            assert!(matches!(
3873                &routed.event,
3874                C2NodeEvent::AgentStream { chunk, .. } if chunk.source_sequence == turn
3875            ));
3876        }
3877        assert_eq!(
3878            cursor.sequence,
3879            chunk_count * 2,
3880            "a whole burst of stale chunks must never rewind the cursor the durable side already advanced",
3881        );
3882    }
3883
3884    #[test]
3885    fn config_rejects_header_injection_duplicate_nodes_and_non_loopback() {
3886        let id = NodeId::new("node-a").unwrap();
3887        assert!(matches!(C2NodeConfig::new(id.clone(), r"\\.\pipe\a", "bad\r\ntoken"), Err(C2ConfigError::InvalidToken)));
3888        let node = C2NodeConfig::new(id, r"\\.\pipe\a", "safe-token").unwrap();
3889        assert!(matches!(C2Config::new("0.0.0.0:0".parse().unwrap(), "safe", vec![node.clone()]), Err(C2ConfigError::NonLoopback(_))));
3890        assert!(matches!(C2Config::new("127.0.0.1:0".parse().unwrap(), "safe", vec![node.clone(), node]), Err(C2ConfigError::DuplicateNode)));
3891        let first = C2NodeConfig::new(NodeId::new("node-a").unwrap(), r"\\.\pipe\same", "safe").unwrap();
3892        let second = C2NodeConfig::new(NodeId::new("node-b").unwrap(), r"\\.\pipe\same", "safe").unwrap();
3893        assert!(matches!(C2Config::new("127.0.0.1:0".parse().unwrap(), "safe", vec![first, second]), Err(C2ConfigError::DuplicateEndpoint)));
3894        let oversized = format!(r"\\.\pipe\{}", "x".repeat(MAX_C2_ENDPOINT_BYTES));
3895        assert!(matches!(C2NodeConfig::new(NodeId::new("node-c").unwrap(), oversized, "safe"), Err(C2ConfigError::InvalidEndpoint(_))));
3896    }
3897
3898    #[test]
3899    fn durable_session_mutations_require_controller_and_use_bounded_deadlines() {
3900        let record_id = SessionRecordId::new("session-001").unwrap();
3901        let rename = NodeRequest::RenameSessionRecord {
3902            record_id: record_id.clone(),
3903            display_name: "release shepherd".to_owned(),
3904        };
3905        let resume = NodeRequest::ResumeSessionRecord {
3906            record_id: record_id.clone(),
3907            terminal_size: TerminalSize { rows: 40, columns: 120 },
3908            initial_prompt: None,
3909        };
3910        let forget = NodeRequest::ForgetSessionRecord { record_id };
3911
3912        assert!(!is_read_only_request(&rename));
3913        assert!(!is_read_only_request(&resume));
3914        assert!(!is_read_only_request(&forget));
3915        assert_eq!(node_request_deadline(&rename), Duration::from_secs(5));
3916        assert_eq!(node_request_deadline(&resume), Duration::from_secs(35));
3917        assert!(node_request_deadline(&resume) > MANAGED_RESUME_SETTLE_DEADLINE);
3918        assert_eq!(
3919            node_request_deadline(&resume) - MANAGED_RESUME_SETTLE_DEADLINE,
3920            NODE_REQUEST_IO_HEADROOM,
3921        );
3922        let started = Instant::now();
3923        let relay_deadline = relay_request_deadline(&resume, started).unwrap();
3924        assert_eq!(
3925            request_budget(&resume, Some(relay_deadline), started),
3926            Duration::from_secs(35),
3927        );
3928        assert_eq!(
3929            request_budget(
3930                &resume,
3931                Some(relay_deadline),
3932                started + Duration::from_secs(5),
3933            ),
3934            MANAGED_RESUME_SETTLE_DEADLINE,
3935        );
3936        assert_eq!(
3937            request_budget(
3938                &resume,
3939                Some(relay_deadline),
3940                started + Duration::from_secs(35),
3941            ),
3942            Duration::ZERO,
3943        );
3944        assert_eq!(node_request_deadline(&forget), Duration::from_secs(5));
3945    }
3946
3947    #[test]
3948    fn workspace_file_reads_are_read_only_with_five_second_deadline() {
3949        let request = NodeRequest::ReadWorkspaceFile {
3950            workspace_id: gate4agent_node_protocol::WorkspaceId::new("primary").unwrap(),
3951            path: gate4agent_node_protocol::RepositoryPath::utf8(
3952                "src/lib.rs".to_owned(),
3953            ).unwrap(),
3954        };
3955
3956        assert!(is_read_only_request(&request));
3957        assert_eq!(node_request_deadline(&request), Duration::from_secs(5));
3958        assert!(relay_request_deadline(&request, Instant::now()).is_none());
3959
3960    }
3961
3962    #[test]
3963    fn workspace_entry_create_relay_has_headroom_and_preserves_semantic_timeout() {
3964        let workspace_id = gate4agent_node_protocol::WorkspaceId::new("primary").unwrap();
3965        let file_path = gate4agent_node_protocol::RepositoryPath::utf8(
3966            "src/new.rs".to_owned(),
3967        ).unwrap();
3968        let create_file = NodeRequest::CreateWorkspaceFile {
3969            workspace_id: workspace_id.clone(),
3970            path: file_path.clone(),
3971        };
3972        assert!(!is_read_only_request(&create_file));
3973        assert_eq!(
3974            node_request_deadline(&create_file),
3975            WORKSPACE_ENTRY_CREATE_RELAY_DEADLINE,
3976        );
3977        assert!(
3978            node_request_deadline(&create_file)
3979                > WORKSPACE_ENTRY_CREATE_NODE_SEMANTIC_DEADLINE,
3980        );
3981        assert!(relay_request_deadline(&create_file, Instant::now()).is_none());
3982
3983        let propagated = relay_node_failure(&NodeClientError::Node(
3984            gate4agent_node_protocol::NodeFailure {
3985                code: NodeFailureCode::RepositoryEntryCreateTimedOut,
3986                message: "private node timeout detail".to_owned(),
3987            },
3988        )).unwrap();
3989        assert_eq!(
3990            propagated.code,
3991            NodeFailureCode::RepositoryEntryCreateTimedOut,
3992        );
3993        assert_eq!(propagated.message, "repository entry creation timed out");
3994
3995        let created_file = gate4agent_node_protocol::WorkspaceFileRead {
3996            workspace_id: workspace_id.clone(),
3997            path: file_path,
3998            content: gate4agent_node_protocol::WorkspaceFileContent::Utf8 {
3999                text: String::new(),
4000                byte_len: 0,
4001            },
4002            revision: Some(
4003                gate4agent_node_protocol::WorkspaceFileRevision::new(
4004                    "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4005                        .to_owned(),
4006                )
4007                .unwrap(),
4008            ),
4009        };
4010        assert!(validate_workspace_content_response(
4011            Some(&create_file),
4012            &Ok(NodeResponse::WorkspaceFileCreated {
4013                file: created_file.clone(),
4014            }),
4015        ).is_ok());
4016        let mut wrong_content = created_file;
4017        wrong_content.content = gate4agent_node_protocol::WorkspaceFileContent::Utf8 {
4018            text: "unexpected".to_owned(),
4019            byte_len: 10,
4020        };
4021        assert!(validate_workspace_content_response(
4022            Some(&create_file),
4023            &Ok(NodeResponse::WorkspaceFileCreated { file: wrong_content }),
4024        ).is_err());
4025
4026        let directory_path = gate4agent_node_protocol::RepositoryPath::utf8(
4027            "src/new".to_owned(),
4028        ).unwrap();
4029        let create_directory = NodeRequest::CreateWorkspaceDirectory {
4030            workspace_id: workspace_id.clone(),
4031            path: directory_path.clone(),
4032        };
4033        assert!(!is_read_only_request(&create_directory));
4034        assert_eq!(
4035            node_request_deadline(&create_directory),
4036            WORKSPACE_ENTRY_CREATE_RELAY_DEADLINE,
4037        );
4038        assert!(validate_workspace_content_response(
4039            Some(&create_directory),
4040            &Ok(NodeResponse::WorkspaceDirectoryCreated {
4041                workspace_id,
4042                entry: gate4agent_node_protocol::WorkspaceEntry {
4043                    relative_path: directory_path,
4044                    kind: gate4agent_node_protocol::WorkspaceEntryKind::Directory,
4045                },
4046            }),
4047        ).is_ok());
4048        assert!(validate_workspace_content_response(
4049            Some(&create_directory),
4050            &Ok(NodeResponse::Accepted),
4051        ).is_err());
4052    }
4053
4054    #[test]
4055    fn native_session_catalog_is_lease_free_read_only() {
4056        let route = gate4agent_node_protocol::NativeSessionCatalogRoute::workspace(
4057            gate4agent_node_protocol::WorkspaceId::new("primary").unwrap(),
4058            gate4agent_types::AgentId::new("codex").unwrap(),
4059        );
4060        let request = NodeRequest::CatalogNativeSessions {
4061            route: route.clone(),
4062            limit: 8,
4063        };
4064        assert!(is_read_only_request(&request));
4065        assert_eq!(
4066            node_request_deadline(&request),
4067            NATIVE_SESSION_REQUEST_DEADLINE,
4068        );
4069        assert!(relay_request_deadline(&request, Instant::now()).is_none());
4070
4071        let page = NodeRequest::PageNativeSessions {
4072            route: route.clone(),
4073            window: gate4agent_types::NativeSessionCatalogWindow::Older,
4074            catalog_revision: 7,
4075            recent_cutoff_unix_ms: 8,
4076            after_selection_id: Some("hist_selection_1".to_owned()),
4077            limit: 8,
4078        };
4079        assert!(is_read_only_request(&page));
4080        assert_eq!(node_request_deadline(&page), NATIVE_SESSION_REQUEST_DEADLINE);
4081        assert!(relay_request_deadline(&page, Instant::now()).is_none());
4082
4083        let preview = NodeRequest::PreviewNativeSession {
4084            selection: gate4agent_node_protocol::NativeSessionSelection {
4085                route,
4086                catalog_revision: 7,
4087                recent_cutoff_unix_ms: 8,
4088                selection_id: "hist_selection_1".to_owned(),
4089            },
4090            message_limit: 12,
4091        };
4092        assert!(is_read_only_request(&preview));
4093        assert_eq!(
4094            node_request_deadline(&preview),
4095            NATIVE_SESSION_REQUEST_DEADLINE,
4096        );
4097        assert!(relay_request_deadline(&preview, Instant::now()).is_none());
4098
4099        let record_preview = NodeRequest::PreviewSessionRecord {
4100            record_id: gate4agent_node_protocol::SessionRecordId::new("record-1").unwrap(),
4101            message_limit: 12,
4102        };
4103        assert!(is_read_only_request(&record_preview));
4104        assert_eq!(
4105            node_request_deadline(&record_preview),
4106            NATIVE_SESSION_REQUEST_DEADLINE,
4107        );
4108        assert!(relay_request_deadline(&record_preview, Instant::now()).is_none());
4109    }
4110
4111    #[test]
4112    fn standalone_workspace_creation_is_controller_mutation_with_worktree_deadline() {
4113        let request = NodeRequest::CreateStandaloneWorkspace {
4114            workspace_id: gate4agent_node_protocol::WorkspaceId::new("standalone").unwrap(),
4115            root: gate4agent_node_protocol::OpaqueHostPath::utf8(
4116                r"C:\standalone".to_owned(),
4117            ).unwrap(),
4118            initial_branch: Some("main".to_owned()),
4119        };
4120
4121        assert!(!is_read_only_request(&request));
4122        assert_eq!(node_request_deadline(&request), Duration::from_secs(240));
4123        assert!(relay_request_deadline(&request, Instant::now()).is_none());
4124    }
4125
4126    #[test]
4127    fn unsupported_node_capability_is_correlated_without_offline_classification() {
4128        let error = NodeClientError::UnsupportedCapability(
4129            "workspace-file-read-v1-private-detail".to_owned(),
4130        );
4131        let failure = relay_node_failure(&error)
4132            .expect("unsupported node capability must remain an in-band node failure");
4133
4134        assert_eq!(failure.code, NodeFailureCode::UnsupportedCapability);
4135        assert_eq!(failure.message, "required capability unavailable");
4136        assert!(!failure.message.contains("private-detail"));
4137        assert!(relay_node_failure(&NodeClientError::Io(io::Error::new(
4138            io::ErrorKind::BrokenPipe,
4139            "transport closed",
4140        ))).is_none());
4141    }
4142
4143    async fn raw_request(request: Vec<u8>, status: StatusResponse) -> Vec<u8> {
4144        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4145        let address = listener.local_addr().unwrap();
4146        let (_status_tx, status_rx) = watch::channel(Arc::new(status));
4147        let server = tokio::spawn(async move {
4148            let (stream, _) = listener.accept().await.unwrap();
4149            serve_http(stream, "api-token", Duration::from_secs(1), &status_rx).await.unwrap();
4150        });
4151        let mut stream = TcpStream::connect(address).await.unwrap();
4152        stream.write_all(&request).await.unwrap();
4153        let mut response = Vec::new();
4154        stream.read_to_end(&mut response).await.unwrap();
4155        server.await.unwrap();
4156        response
4157    }
4158
4159    fn empty_status(ready: bool) -> StatusResponse {
4160        StatusResponse { api_version: C2_API_VERSION, ready, observed_at_unix_ms: 0, nodes: BTreeMap::new() }
4161    }
4162
4163    #[tokio::test]
4164    async fn http_api_enforces_initializing_auth_method_path_and_header_bounds() {
4165        let initializing = raw_request(b"GET /ready HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), empty_status(false)).await;
4166        assert!(initializing.starts_with(b"HTTP/1.1 503 Service Unavailable\r\n"));
4167        assert!(String::from_utf8_lossy(&initializing).contains("\"ready\":false"));
4168
4169        let unauthorized = raw_request(b"GET /status HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), empty_status(true)).await;
4170        assert!(unauthorized.starts_with(b"HTTP/1.1 401 Unauthorized\r\n"));
4171        let method = raw_request(b"POST /health HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), empty_status(true)).await;
4172        assert!(method.starts_with(b"HTTP/1.1 405 Method Not Allowed\r\n"));
4173        let missing = raw_request(b"GET /missing HTTP/1.1\r\nHost: localhost\r\n\r\n".to_vec(), empty_status(true)).await;
4174        assert!(missing.starts_with(b"HTTP/1.1 404 Not Found\r\n"));
4175
4176        let mut oversized = b"GET /health HTTP/1.1\r\nX-Fill: ".to_vec();
4177        oversized.extend(std::iter::repeat(b'x').take(HEADER_LIMIT_BYTES));
4178        oversized.extend_from_slice(b"\r\n\r\n");
4179        let rejected = raw_request(oversized, empty_status(true)).await;
4180        assert!(rejected.starts_with(b"HTTP/1.1 413 Payload Too Large\r\n"));
4181    }
4182
4183    #[tokio::test]
4184    async fn inventory_state_transitions_offline_to_stale_parked_and_recovers() {
4185        let node_id = NodeId::new("node-a").unwrap();
4186        let mut nodes = BTreeMap::new();
4187        nodes.insert(node_id.clone(), ObservedNode {
4188            endpoint: r"\\.\pipe\a".to_owned(), transport_label: "windows-named-pipe".to_owned(),
4189            transport: NodeTransportState::Offline, freshness: NodeFreshness::Unavailable,
4190            cursor: None, inventory: None, last_attempt_unix_ms: None, last_success_unix_ms: None,
4191            consecutive_failures: 0, last_error: None, gaps: Vec::new(), gaps_truncated: 0,
4192        });
4193        let initial = Arc::new(StatusResponse { api_version: C2_API_VERSION, ready: false, observed_at_unix_ms: unix_ms(), nodes });
4194        let (status_tx, mut status_rx) = watch::channel(initial);
4195        let (ingress_tx, ingress_rx) = mpsc::channel(4);
4196        let (shutdown_tx, shutdown_rx) = watch::channel(false);
4197        let owner = tokio::spawn(inventory_owner(1, Duration::from_millis(10), ingress_rx, status_tx, shutdown_rx));
4198        let snapshot = NodeSnapshot {
4199            node_id: node_id.clone(),
4200            enabled_providers: Vec::new(),
4201            provider_runtime_statuses: crate::protocol::ProviderRuntimeStatuses::default(),
4202            workspaces: Vec::new(),
4203            session_records: Vec::new(),
4204            managed_worktrees: Vec::new(),
4205            launch_inventory: None,
4206            agent_progress: Vec::new(),
4207        };
4208        let cursor = NodeCursor { incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([1; 16]), sequence: 0 };
4209        let old_manifest = ProviderContractManifest {
4210            provider_contracts: vec![crate::protocol::ProviderContractSupport {
4211                provider: agent("codex"),
4212                revision: crate::protocol::ProviderContractRevision::new("old-contract").unwrap(),
4213            }],
4214            provider_adapter_contracts: vec![crate::protocol::ProviderAdapterContractSupport {
4215                provider: agent("codex"),
4216                family: crate::protocol::AdapterFamily::PtySemantic,
4217                adapter_id: crate::protocol::AdapterId::new("codex").unwrap(),
4218                revision: crate::protocol::AdapterContractRevision::new("old-adapter").unwrap(),
4219            }],
4220        };
4221        ingress_tx.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result: AttemptResult::Connected {
4222            cursor,
4223            snapshot: snapshot.clone(),
4224            gaps: Vec::new(),
4225            provider_contract_manifest: old_manifest,
4226        } }).await.unwrap();
4227        status_rx.changed().await.unwrap();
4228        assert_eq!(status_rx.borrow().nodes[&node_id].freshness, NodeFreshness::Fresh);
4229        assert_eq!(
4230            status_rx.borrow().nodes[&node_id].inventory.as_ref().unwrap()
4231                .provider_contracts[0].revision.as_str(),
4232            "old-contract",
4233        );
4234
4235        let failure = || AttemptResult::Failure { error: SanitizedError { category: C2ErrorCategory::Transport, message: "node transport unavailable".to_owned() }, hard: false };
4236        ingress_tx.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result: failure() }).await.unwrap();
4237        status_rx.changed().await.unwrap();
4238        assert_eq!(status_rx.borrow().nodes[&node_id].transport, NodeTransportState::Offline);
4239        timeout(Duration::from_secs(1), async {
4240            loop {
4241                status_rx.changed().await.unwrap();
4242                if status_rx.borrow().nodes[&node_id].freshness == NodeFreshness::Stale { break; }
4243            }
4244        }).await.unwrap();
4245        for _ in 0..4 {
4246            ingress_tx.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result: failure() }).await.unwrap();
4247            status_rx.changed().await.unwrap();
4248        }
4249        assert_eq!(status_rx.borrow().nodes[&node_id].transport, NodeTransportState::Parked);
4250        let replacement_manifest = ProviderContractManifest {
4251            provider_contracts: vec![crate::protocol::ProviderContractSupport {
4252                provider: agent("claude"),
4253                revision: crate::protocol::ProviderContractRevision::new("new-contract").unwrap(),
4254            }],
4255            provider_adapter_contracts: Vec::new(),
4256        };
4257        let replacement_cursor = NodeCursor {
4258            incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([2; 16]),
4259            sequence: 0,
4260        };
4261        ingress_tx.send(Attempt { node_id: node_id.clone(), at_unix_ms: unix_ms(), result: AttemptResult::Connected {
4262            cursor: replacement_cursor,
4263            snapshot,
4264            gaps: Vec::new(),
4265            provider_contract_manifest: replacement_manifest,
4266        } }).await.unwrap();
4267        status_rx.changed().await.unwrap();
4268        assert_eq!(status_rx.borrow().nodes[&node_id].transport, NodeTransportState::Online);
4269        assert_eq!(status_rx.borrow().nodes[&node_id].freshness, NodeFreshness::Fresh);
4270        let recovered_inventory = status_rx.borrow().nodes[&node_id].inventory.as_ref().unwrap().clone();
4271        assert_eq!(recovered_inventory.provider_contracts.len(), 1);
4272        assert_eq!(recovered_inventory.provider_contracts[0].provider, agent("claude"));
4273        assert_eq!(recovered_inventory.provider_contracts[0].revision.as_str(), "new-contract");
4274        assert!(recovered_inventory.provider_adapter_contracts.is_empty());
4275        ingress_tx.send(Attempt {
4276            node_id: node_id.clone(),
4277            at_unix_ms: unix_ms(),
4278            result: AttemptResult::Connected {
4279                cursor: NodeCursor {
4280                    incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([3; 16]),
4281                    sequence: 0,
4282                },
4283                snapshot: NodeSnapshot {
4284                    node_id: node_id.clone(),
4285                    enabled_providers: Vec::new(),
4286                    provider_runtime_statuses: crate::protocol::ProviderRuntimeStatuses::default(),
4287                    workspaces: Vec::new(),
4288                    session_records: Vec::new(),
4289                    managed_worktrees: Vec::new(),
4290                    launch_inventory: None,
4291                    agent_progress: Vec::new(),
4292                },
4293                gaps: Vec::new(),
4294                provider_contract_manifest: ProviderContractManifest::default(),
4295            },
4296        }).await.unwrap();
4297        status_rx.changed().await.unwrap();
4298        let unpublished = status_rx.borrow().nodes[&node_id].inventory.as_ref().unwrap().clone();
4299        assert!(unpublished.provider_contracts.is_empty());
4300        assert!(unpublished.provider_adapter_contracts.is_empty());
4301        shutdown_tx.send(true).unwrap();
4302        owner.await.unwrap().unwrap();
4303    }
4304
4305    fn runtime_statuses(
4306        provider: gate4agent_node_protocol::AgentId,
4307        version: &str,
4308    ) -> crate::protocol::ProviderRuntimeStatuses {
4309        crate::protocol::ProviderRuntimeStatuses::new([
4310            crate::protocol::ProviderRuntimeStatus::raw_passthrough(
4311                provider,
4312                Some(crate::protocol::ProviderRuntimeVersion::new(version).unwrap()),
4313            ),
4314        ])
4315        .unwrap()
4316    }
4317
4318    async fn runtime_inventory_owner() -> (
4319        NodeId,
4320        mpsc::Sender<Attempt>,
4321        watch::Receiver<Arc<StatusResponse>>,
4322        watch::Sender<bool>,
4323        tokio::task::JoinHandle<io::Result<()>>,
4324    ) {
4325        let node_id = NodeId::new("runtime-node").unwrap();
4326        let nodes = BTreeMap::from([(
4327            node_id.clone(),
4328            ObservedNode {
4329                endpoint: r"\\.\pipe\runtime-node".to_owned(),
4330                transport_label: "windows-named-pipe".to_owned(),
4331                transport: NodeTransportState::Offline,
4332                freshness: NodeFreshness::Unavailable,
4333                cursor: None,
4334                inventory: None,
4335                last_attempt_unix_ms: None,
4336                last_success_unix_ms: None,
4337                consecutive_failures: 0,
4338                last_error: None,
4339                gaps: Vec::new(),
4340                gaps_truncated: 0,
4341            },
4342        )]);
4343        let initial = Arc::new(StatusResponse {
4344            api_version: C2_API_VERSION,
4345            ready: false,
4346            observed_at_unix_ms: unix_ms(),
4347            nodes,
4348        });
4349        let (status_tx, status_rx) = watch::channel(initial);
4350        let (ingress_tx, ingress_rx) = mpsc::channel(4);
4351        let (shutdown_tx, shutdown_rx) = watch::channel(false);
4352        let owner = tokio::spawn(inventory_owner(
4353            1,
4354            Duration::from_secs(1),
4355            ingress_rx,
4356            status_tx,
4357            shutdown_rx,
4358        ));
4359        (node_id, ingress_tx, status_rx, shutdown_tx, owner)
4360    }
4361
4362    #[tokio::test]
4363    async fn incarnation_change_replaces_runtime_status() {
4364        let (node_id, ingress, mut status, shutdown, owner) = runtime_inventory_owner().await;
4365        for (incarnation, provider, version) in [
4366            (1, agent("claude"), "1.0.0"),
4367            (2, agent("codex"), "2.0.0"),
4368        ] {
4369            ingress
4370                .send(Attempt {
4371                    node_id: node_id.clone(),
4372                    at_unix_ms: unix_ms(),
4373                    result: AttemptResult::Connected {
4374                        cursor: NodeCursor {
4375                            incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([
4376                                incarnation;
4377                                16
4378                            ]),
4379                            sequence: 0,
4380                        },
4381                        snapshot: NodeSnapshot {
4382                            node_id: node_id.clone(),
4383                            enabled_providers: vec![provider.clone()],
4384                            provider_runtime_statuses: runtime_statuses(provider, version),
4385                            workspaces: Vec::new(),
4386                            session_records: Vec::new(),
4387                            managed_worktrees: Vec::new(),
4388                            launch_inventory: None,
4389                            agent_progress: Vec::new(),
4390                        },
4391                        gaps: Vec::new(),
4392                        provider_contract_manifest: ProviderContractManifest::default(),
4393                    },
4394                })
4395                .await
4396                .unwrap();
4397            status.changed().await.unwrap();
4398        }
4399        let current_status = status.borrow();
4400        let statuses = &current_status.nodes[&node_id]
4401            .inventory
4402            .as_ref()
4403            .unwrap()
4404            .provider_runtime_statuses;
4405        assert_eq!(statuses.as_slice().len(), 1);
4406        assert_eq!(
4407            statuses.as_slice()[0].provider(),
4408            &agent("codex"),
4409        );
4410        assert_eq!(statuses.as_slice()[0].version().unwrap().as_str(), "2.0.0");
4411        shutdown.send(true).unwrap();
4412        owner.await.unwrap().unwrap();
4413    }
4414
4415    #[tokio::test]
4416    async fn incarnation_change_without_snapshot_clears_dynamic_inventory() {
4417        let (node_id, ingress, mut status, shutdown, owner) = runtime_inventory_owner().await;
4418        ingress
4419            .send(Attempt {
4420                node_id: node_id.clone(),
4421                at_unix_ms: unix_ms(),
4422                result: AttemptResult::Connected {
4423                    cursor: NodeCursor {
4424                        incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([
4425                            3; 16
4426                        ]),
4427                        sequence: 0,
4428                    },
4429                    snapshot: NodeSnapshot {
4430                        node_id: node_id.clone(),
4431                        enabled_providers: vec![agent("claude")],
4432                        provider_runtime_statuses: runtime_statuses(
4433                            agent("claude"),
4434                            "3.0.0",
4435                        ),
4436                        workspaces: Vec::new(),
4437                        session_records: Vec::new(),
4438                        managed_worktrees: Vec::new(),
4439                        launch_inventory: None,
4440                        agent_progress: Vec::new(),
4441                    },
4442                    gaps: Vec::new(),
4443                    provider_contract_manifest: ProviderContractManifest::default(),
4444                },
4445            })
4446            .await
4447            .unwrap();
4448        status.changed().await.unwrap();
4449        ingress
4450            .send(Attempt {
4451                node_id: node_id.clone(),
4452                at_unix_ms: unix_ms(),
4453                result: AttemptResult::Cursor {
4454                    cursor: NodeCursor {
4455                        incarnation_id: gate4agent_node_protocol::NodeIncarnationId::from_bytes([
4456                            4; 16
4457                        ]),
4458                        sequence: 0,
4459                    },
4460                    gaps: vec![GapKind::IncarnationChanged],
4461                    managed_worktree_events: Vec::new(),
4462                },
4463            })
4464            .await
4465            .unwrap();
4466        status.changed().await.unwrap();
4467        assert!(status.borrow().nodes[&node_id]
4468            .inventory
4469            .as_ref()
4470            .unwrap()
4471            .provider_runtime_statuses
4472            .is_empty());
4473        shutdown.send(true).unwrap();
4474        owner.await.unwrap().unwrap();
4475    }
4476
4477    #[test]
4478    fn managed_worktree_inventory_events_are_exact_bounded_and_incarnation_fenced() {
4479        let mut inventory = SlimNodeInventory::from_snapshot(&NodeSnapshot {
4480            node_id: NodeId::new("node-a").unwrap(),
4481            enabled_providers: Vec::new(),
4482            provider_runtime_statuses: crate::protocol::ProviderRuntimeStatuses::default(),
4483            workspaces: Vec::new(),
4484            session_records: Vec::new(),
4485            managed_worktrees: vec![managed_lease(
4486                "lease-a",
4487                "managed-a",
4488                ManagedWorktreeLeaseState::Ready,
4489            )],
4490            launch_inventory: None,
4491            agent_progress: Vec::new(),
4492        });
4493        apply_managed_worktree_cursor(
4494            Some(&mut inventory),
4495            false,
4496            &[
4497                NodeEvent::ManagedWorktreeUpserted {
4498                    lease: managed_lease(
4499                        "lease-b",
4500                        "managed-b",
4501                        ManagedWorktreeLeaseState::Ready,
4502                    ),
4503                },
4504                NodeEvent::ManagedWorktreeUpserted {
4505                    lease: managed_lease(
4506                        "lease-a",
4507                        "managed-a",
4508                        ManagedWorktreeLeaseState::InUse,
4509                    ),
4510                },
4511            ],
4512        );
4513        assert_eq!(inventory.managed_worktree_count, 2);
4514        assert_eq!(inventory.managed_worktrees[0].lease_id.as_str(), "lease-a");
4515        assert_eq!(
4516            inventory.managed_worktrees[0].state,
4517            ManagedWorktreeLeaseState::InUse,
4518        );
4519
4520        apply_managed_worktree_cursor(
4521            Some(&mut inventory),
4522            false,
4523            &[NodeEvent::ManagedWorktreeRemoved {
4524                lease_id: ManagedWorktreeLeaseId::new("lease-a").unwrap(),
4525            }],
4526        );
4527        assert_eq!(inventory.managed_worktree_count, 1);
4528        assert_eq!(inventory.managed_worktrees[0].lease_id.as_str(), "lease-b");
4529
4530        apply_managed_worktree_cursor(
4531            Some(&mut inventory),
4532            true,
4533            &[NodeEvent::ManagedWorktreeUpserted {
4534                lease: managed_lease(
4535                    "lease-c",
4536                    "managed-c",
4537                    ManagedWorktreeLeaseState::Ready,
4538                ),
4539            }],
4540        );
4541        assert!(inventory.managed_worktrees.is_empty());
4542        assert_eq!(inventory.managed_worktree_count, 0);
4543        assert!(!inventory.managed_worktrees_truncated);
4544    }
4545}
4546
4547#[cfg(test)]
4548mod relay_deadline_tests {
4549    use super::*;
4550
4551    /// The defect this pins, stated as an invariant rather than a number:
4552    /// the relay's bound on a request must never sit below the budget the
4553    /// node itself is allowed to spend answering it. It sat at five
4554    /// seconds against the node's eight-to-eleven, so every inspection of
4555    /// a workspace large enough to use its own allowance timed out at the
4556    /// relay -- and a relay timeout drops the NODE, not the request, which
4557    /// took every read behind it down in a loop.
4558    #[test]
4559    fn workspace_inspection_relay_bound_clears_the_nodes_own_maximum() {
4560        // The node's own ceiling (`WORKSPACE_INSPECTION_TIME_BUDGET_MS_MAX`
4561        // in the node crate) restated here rather than imported: these are
4562        // separate crates by design, and the point of the test is that the
4563        // two numbers must be compared by a human when either moves.
4564        const NODE_INSPECTION_MAX: Duration = Duration::from_millis(11_000);
4565        assert!(
4566            WORKSPACE_INSPECTION_RELAY_DEADLINE > NODE_INSPECTION_MAX,
4567            "relay bound {:?} must exceed the node's own inspection maximum {:?}",
4568            WORKSPACE_INSPECTION_RELAY_DEADLINE,
4569            NODE_INSPECTION_MAX,
4570        );
4571    }
4572}
4573
4574#[cfg(test)]
4575mod control_tunnel_listen_tests {
4576    use super::*;
4577
4578    fn config() -> C2Config {
4579        let node = C2NodeConfig::new(
4580            NodeId::new("node-a").unwrap(),
4581            if cfg!(unix) { "/tmp/gate4agent-node-a.sock" } else { r"\\.\pipe\gate4agent-node-a" },
4582            "token-token",
4583        ).unwrap();
4584        C2Config::new("127.0.0.1:18321".parse().unwrap(), "api-token", vec![node]).unwrap()
4585    }
4586
4587    #[test]
4588    fn control_tunnel_rejects_unspecified_and_accepts_tunnel_address() {
4589        let bad: SocketAddr = "0.0.0.0:27441".parse().unwrap();
4590        assert!(matches!(
4591            config().with_control_tunnel_listen(bad),
4592            Err(C2ConfigError::InvalidControlTunnel(_))
4593        ));
4594        let ok: SocketAddr = "10.88.1.2:27441".parse().unwrap();
4595        let config = config().with_control_tunnel_listen(ok).unwrap();
4596        assert_eq!(config.control_tunnel_listen, Some(ok));
4597        assert!(!config.control_endpoint.is_empty());
4598    }
4599}