Skip to main content

agentos_sidecar_client/
transport.rs

1//! `SidecarTransport`: spawns a native sidecar binary and speaks the existing framed
2//! BARE protocol over its stdio.
3//!
4//! This mirrors the TypeScript `Sidecar`. Generated wire payloads are the native
5//! transport path.
6//!
7//! Request-id direction is load-bearing: host-initiated `Request`/`Response` frames use positive ids
8//! allocated by this transport, while sidecar-initiated `SidecarRequest`/`SidecarResponse` callbacks
9//! echo the id allocated by the sidecar.
10
11use std::process::Stdio;
12use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, AtomicUsize, Ordering};
13use std::sync::{Arc, Weak};
14
15#[cfg(unix)]
16use command_fds::{CommandFdExt, FdMapping};
17#[cfg(unix)]
18use std::os::unix::net::UnixStream as StdUnixStream;
19
20use scc::HashMap as SccHashMap;
21use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
22use tokio::process::{Child, Command};
23use tokio::sync::{broadcast, mpsc, oneshot};
24
25use crate::wire::{self, WireFrameCodec};
26use crate::TransportError;
27
28/// Broadcast capacity for the structured/lifecycle/process event fan-out.
29const EVENT_CHANNEL_CAPACITY: usize = 4096;
30
31/// Maximum outbound frames buffered while the writer task drains to sidecar stdin.
32const REQUEST_FRAME_QUEUE_CAPACITY: usize = 4096;
33
34/// Maximum callback/control response frames buffered ahead of regular host requests.
35const CONTROL_FRAME_QUEUE_CAPACITY: usize = 1024;
36
37/// Maximum in-flight host-initiated sidecar requests per transport.
38const PENDING_REQUEST_LIMIT: usize = 4096;
39
40/// Env var that overrides the sidecar binary path. Defaults to `agentos-native-sidecar` on `PATH`.
41/// Product clients can pass an explicit binary path to [`SidecarTransport::spawn`].
42const SIDECAR_BIN_ENV: &str = "AGENTOS_SIDECAR_BIN";
43
44/// Fixed inherited descriptor carrying the full-duplex response/control lane.
45#[cfg(unix)]
46const CONTROL_FD: std::os::fd::RawFd = 3;
47
48/// How long the host tolerates TOTAL inbound silence (no responses, events, sidecar requests, or
49/// heartbeats) before declaring the sidecar dead. The sidecar heartbeats every 10s from a dedicated
50/// thread even while busy, so this allows two missed beats plus margin; it bounds "sidecar is dead
51/// or wedged", never "this request is slow" — individual requests have no deadline of their own.
52/// Fixed protocol constant paired with the sidecar heartbeat cadence; mirrors the TS client.
53const SIDECAR_SILENCE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
54
55/// After a disconnect, how long the event lane may keep delivering frames the sidecar wrote before
56/// it exited. The killed sidecar's stdout normally reaches EOF at once; this only bounds the case
57/// where something else holds the pipe open, so event waiters never hang.
58const EVENT_DRAIN_GRACE: std::time::Duration = std::time::Duration::from_secs(2);
59
60type EventSenderSlot =
61    parking_lot::Mutex<Option<broadcast::Sender<(wire::OwnershipScope, wire::EventPayload)>>>;
62
63/// A registered callback that answers a sidecar-initiated request using generated wire types.
64pub type WireSidecarCallback = Arc<
65    dyn Fn(
66            wire::SidecarRequestPayload,
67            wire::OwnershipScope,
68        ) -> futures::future::BoxFuture<
69            'static,
70            Result<wire::SidecarResponsePayload, TransportError>,
71        > + Send
72        + Sync,
73>;
74
75fn next_transport_instance_id() -> u64 {
76    static NEXT_INSTANCE_ID: AtomicU64 = AtomicU64::new(1);
77    NEXT_INSTANCE_ID.fetch_add(1, Ordering::Relaxed)
78}
79
80/// Owns the spawned sidecar child, the framed BARE stdio I/O tasks, the pending-response map, the
81/// event fan-out, and the callback dispatch table.
82pub struct SidecarTransport {
83    /// Unique in this host process. Sidecar-issued ids repeat across sidecar processes.
84    instance_id: u64,
85    /// The spawned sidecar process (stdout/stdin taken by the I/O tasks; kept for kill on drop).
86    child: parking_lot::Mutex<Option<Child>>,
87    child_termination: tokio::sync::Mutex<()>,
88    /// Pending host-initiated requests, keyed by positive `RequestId`.
89    pending: SccHashMap<wire::RequestId, oneshot::Sender<wire::ResponsePayload>>,
90    pending_request_lock: parking_lot::Mutex<()>,
91    /// Host request-id counter (positive, starts at 1).
92    request_counter: AtomicI64,
93    /// Negotiated max frame size.
94    max_frame_bytes: AtomicUsize,
95    /// Most entries per write-files request, from the authenticate handshake. One entry per
96    /// request until the handshake sets it.
97    max_write_files_entries: AtomicUsize,
98    /// Structured-event fan-out for `Event` frames. Dropped once the event lane ends (or
99    /// `EVENT_DRAIN_GRACE` after a disconnect), so every subscriber sees `RecvError::Closed`
100    /// instead of waiting for an event that cannot come.
101    event_tx: Arc<EventSenderSlot>,
102    /// Registered host callbacks for `SidecarRequest` frames.
103    callbacks: SccHashMap<&'static str, WireSidecarCallback>,
104    /// Outbound host request frames drained by the writer task into the child's stdin.
105    request_writer_tx: mpsc::Sender<Vec<u8>>,
106    /// Outbound callback/control response frames. The writer drains this before regular requests.
107    control_writer_tx: mpsc::Sender<Vec<u8>>,
108    /// Silence the watchdog has counted since the reader last received any inbound frame.
109    counted_silence: parking_lot::Mutex<std::time::Duration>,
110    /// Set once the transport loses its sidecar: an inbound lane ended or broke protocol, or the
111    /// silence watchdog killed the child. No request can succeed after this.
112    disconnected: AtomicBool,
113}
114
115impl SidecarTransport {
116    /// Spawn the native sidecar binary and start the stdio I/O tasks.
117    ///
118    /// Does NOT run the handshake. Product clients drive Authenticate and any follow-up setup using
119    /// [`request_wire`](Self::request_wire) once the transport is live.
120    pub async fn spawn(binary_path: Option<String>) -> Result<Arc<Self>, TransportError> {
121        Self::spawn_with_args(binary_path, Vec::new()).await
122    }
123
124    /// Spawn with trusted operator-owned command-line configuration.
125    pub async fn spawn_with_args(
126        binary_path: Option<String>,
127        args: Vec<String>,
128    ) -> Result<Arc<Self>, TransportError> {
129        #[cfg(not(unix))]
130        {
131            let _ = (binary_path, args);
132            return Err(TransportError::Sidecar(
133                "the native sidecar response/control transport is unsupported on this platform"
134                    .to_string(),
135            ));
136        }
137
138        #[cfg(unix)]
139        {
140            Self::spawn_unix(binary_path, args).await
141        }
142    }
143
144    #[cfg(unix)]
145    async fn spawn_unix(
146        binary_path: Option<String>,
147        args: Vec<String>,
148    ) -> Result<Arc<Self>, TransportError> {
149        let bin = resolve_sidecar_binary_path(binary_path);
150        let (control_parent, control_child) = StdUnixStream::pair().map_err(|error| {
151            TransportError::Sidecar(format!(
152                "failed to create sidecar control socketpair: {error}"
153            ))
154        })?;
155        control_parent.set_nonblocking(true).map_err(|error| {
156            TransportError::Sidecar(format!(
157                "failed to configure sidecar control socket: {error}"
158            ))
159        })?;
160        let mut command = Command::new(&bin);
161        command
162            .args(&args)
163            .stdin(Stdio::piped())
164            .stdout(Stdio::piped())
165            .stderr(Stdio::inherit())
166            .kill_on_drop(true);
167        // `command-fds` performs the child-only dup after fork, avoiding the
168        // CLOEXEC race caused by making an arbitrary descriptor inheritable in
169        // this multithreaded process.
170        map_control_fd(&mut command, control_child)?;
171        let mut child = command.spawn().map_err(|error| {
172            TransportError::Sidecar(format!("failed to spawn sidecar '{bin}': {error}"))
173        })?;
174        drop(command);
175        let stdin = child
176            .stdin
177            .take()
178            .ok_or_else(|| TransportError::Sidecar("sidecar stdin was not piped".to_string()))?;
179        let stdout = child
180            .stdout
181            .take()
182            .ok_or_else(|| TransportError::Sidecar("sidecar stdout was not piped".to_string()))?;
183        let control = tokio::net::UnixStream::from_std(control_parent).map_err(|error| {
184            TransportError::Sidecar(format!("failed to adopt sidecar control socket: {error}"))
185        })?;
186        let (control_reader, control_writer) = control.into_split();
187
188        let (request_writer_tx, request_writer_rx) = mpsc::channel(REQUEST_FRAME_QUEUE_CAPACITY);
189        let (control_writer_tx, control_writer_rx) = mpsc::channel(CONTROL_FRAME_QUEUE_CAPACITY);
190        let (event_tx, _) = broadcast::channel(EVENT_CHANNEL_CAPACITY);
191
192        let transport = Arc::new(Self {
193            instance_id: next_transport_instance_id(),
194            child: parking_lot::Mutex::new(Some(child)),
195            child_termination: tokio::sync::Mutex::new(()),
196            pending: SccHashMap::new(),
197            pending_request_lock: parking_lot::Mutex::new(()),
198            request_counter: AtomicI64::new(1),
199            max_frame_bytes: AtomicUsize::new(wire::DEFAULT_MAX_FRAME_BYTES),
200            max_write_files_entries: AtomicUsize::new(1),
201            event_tx: Arc::new(parking_lot::Mutex::new(Some(event_tx))),
202            callbacks: SccHashMap::new(),
203            request_writer_tx,
204            control_writer_tx,
205            counted_silence: parking_lot::Mutex::new(std::time::Duration::ZERO),
206            disconnected: AtomicBool::new(false),
207        });
208
209        tokio::spawn(run_writer(
210            Arc::downgrade(&transport),
211            "ordinary request",
212            stdin,
213            request_writer_rx,
214        ));
215        tokio::spawn(run_writer(
216            Arc::downgrade(&transport),
217            "response/control",
218            control_writer,
219            control_writer_rx,
220        ));
221        tokio::spawn(run_reader(
222            Arc::downgrade(&transport),
223            stdout,
224            InboundLane::Event,
225        ));
226        tokio::spawn(run_reader(
227            Arc::downgrade(&transport),
228            control_reader,
229            InboundLane::Control,
230        ));
231        tokio::spawn(run_silence_watchdog(
232            Arc::downgrade(&transport),
233            SIDECAR_SILENCE_TIMEOUT,
234        ));
235
236        Ok(transport)
237    }
238
239    /// Allocate the next positive host request id.
240    pub fn next_request_id(&self) -> wire::RequestId {
241        self.request_counter.fetch_add(1, Ordering::SeqCst)
242    }
243
244    /// Issue a host request using generated wire protocol types and await a generated response.
245    pub async fn request_wire(
246        &self,
247        ownership: wire::OwnershipScope,
248        payload: wire::RequestPayload,
249    ) -> Result<wire::ResponsePayload, TransportError> {
250        self.request_wire_with_frame_limit(ownership, payload, None)
251            .await
252    }
253
254    /// Issue a host request using generated wire protocol types with a caller-specific frame limit.
255    pub async fn request_wire_bounded(
256        &self,
257        ownership: wire::OwnershipScope,
258        payload: wire::RequestPayload,
259        max_frame_bytes: usize,
260    ) -> Result<wire::ResponsePayload, TransportError> {
261        self.request_wire_with_frame_limit(ownership, payload, Some(max_frame_bytes))
262            .await
263    }
264
265    async fn request_wire_with_frame_limit(
266        &self,
267        ownership: wire::OwnershipScope,
268        payload: wire::RequestPayload,
269        max_frame_bytes: Option<usize>,
270    ) -> Result<wire::ResponsePayload, TransportError> {
271        let request_id = self.next_request_id();
272        let frame = wire::ProtocolFrame::RequestFrame(wire::RequestFrame {
273            schema: wire::protocol_schema(),
274            request_id,
275            ownership,
276            payload,
277        });
278        let bytes = self.encode_wire_frame(&frame, max_frame_bytes)?;
279
280        let (tx, rx) = oneshot::channel();
281        self.register_pending_request(request_id, tx)?;
282        let _pending_guard = PendingRequestGuard::new(self, request_id);
283
284        if self.request_writer_tx.send(bytes).await.is_err() {
285            self.pending.remove(&request_id);
286            return Err(TransportError::Disconnected(
287                "sidecar transport closed".to_string(),
288            ));
289        }
290
291        rx.await
292            .map_err(|_| TransportError::Disconnected("sidecar transport disconnected".to_string()))
293    }
294
295    /// Subscribe to structured/lifecycle/process events using generated wire protocol types.
296    pub fn subscribe_wire_events(
297        &self,
298    ) -> broadcast::Receiver<(wire::OwnershipScope, wire::EventPayload)> {
299        match self.event_tx.lock().as_ref() {
300            Some(event_tx) => event_tx.subscribe(),
301            // A lost transport delivers no more events; a receiver without a sender says so.
302            None => broadcast::channel(1).1,
303        }
304    }
305
306    /// Identify this transport in host-process-wide state.
307    pub fn instance_id(&self) -> u64 {
308        self.instance_id
309    }
310
311    /// Register a callback that answers a class of sidecar-initiated requests using generated wire
312    /// protocol types.
313    pub fn register_wire_callback(&self, key: &'static str, callback: WireSidecarCallback) {
314        let _ = self.callbacks.insert(key, callback);
315    }
316
317    /// Return the currently negotiated max frame size.
318    pub fn max_frame_bytes(&self) -> usize {
319        self.max_frame_bytes.load(Ordering::Relaxed)
320    }
321
322    /// Whether this transport lost its sidecar process. A lost transport never recovers; callers
323    /// must spawn a new sidecar.
324    pub fn is_disconnected(&self) -> bool {
325        self.disconnected.load(Ordering::SeqCst)
326    }
327
328    /// Update the negotiated max frame size after authentication.
329    pub fn set_max_frame_bytes(&self, max_frame_bytes: usize) {
330        self.max_frame_bytes
331            .store(max_frame_bytes, Ordering::SeqCst);
332    }
333
334    /// Most entries one `GuestFilesystemWriteFilesRequest` may carry, as negotiated.
335    pub fn max_write_files_entries(&self) -> usize {
336        self.max_write_files_entries.load(Ordering::Relaxed)
337    }
338
339    /// Update the negotiated write-files entry limit after authentication.
340    pub fn set_max_write_files_entries(&self, max_entries: usize) {
341        self.max_write_files_entries
342            .store(max_entries.max(1), Ordering::SeqCst);
343    }
344
345    /// Request graceful process termination through the physically independent
346    /// response/control lane. This does not reinterpret an ordinary request as
347    /// control traffic.
348    pub async fn shutdown(&self, reason: impl Into<String>) -> Result<(), TransportError> {
349        let frame = wire::ProtocolFrame::ControlFrame(wire::ControlFrame {
350            schema: wire::protocol_schema(),
351            payload: wire::ControlPayload::ShutdownControl(wire::ShutdownControl {
352                reason: reason.into(),
353            }),
354        });
355        let bytes = self.encode_wire_frame(&frame, None)?;
356        self.control_writer_tx.send(bytes).await.map_err(|_| {
357            TransportError::Sidecar("sidecar response/control transport closed".to_string())
358        })
359    }
360
361    /// Kill the child sidecar process if this transport still owns one.
362    pub fn kill_child(&self) {
363        if let Some(child) = self.child.lock().as_mut() {
364            match child.try_wait() {
365                Ok(None) => {
366                    if let Err(error) = child.start_kill() {
367                        tracing::error!(?error, "kill disconnected sidecar child");
368                    }
369                }
370                Ok(Some(_)) => {}
371                Err(error) => tracing::error!(?error, "poll disconnected sidecar child"),
372            }
373        }
374    }
375
376    /// Stop the child and reap it before its caller releases owned resources
377    /// such as a temporary package-cache directory.
378    pub async fn terminate_child(&self) -> Result<(), TransportError> {
379        let _termination = self.child_termination.lock().await;
380        let Some(child) = self.child.lock().take() else {
381            return Ok(());
382        };
383        let mut owned = ReapingChild {
384            slot: &self.child,
385            child: Some(child),
386        };
387        let child = owned.child.as_mut().expect("owned reaping child");
388        if child
389            .try_wait()
390            .map_err(|error| TransportError::Sidecar(format!("poll sidecar child: {error}")))?
391            .is_none()
392        {
393            if let Err(kill_error) = child.start_kill() {
394                if child
395                    .try_wait()
396                    .map_err(|error| {
397                        TransportError::Sidecar(format!("poll sidecar child after kill: {error}"))
398                    })?
399                    .is_none()
400                {
401                    return Err(TransportError::Sidecar(format!(
402                        "kill sidecar child: {kill_error}"
403                    )));
404                }
405            }
406        }
407        child
408            .wait()
409            .await
410            .map_err(|error| TransportError::Sidecar(format!("reap sidecar child: {error}")))?;
411        owned.child.take();
412        Ok(())
413    }
414
415    fn encode_wire_frame(
416        &self,
417        frame: &wire::ProtocolFrame,
418        max_frame_bytes: Option<usize>,
419    ) -> Result<Vec<u8>, TransportError> {
420        let transport_limit = self.max_frame_bytes.load(Ordering::Relaxed);
421        let max_frame_bytes = max_frame_bytes
422            .map(|limit| limit.min(transport_limit))
423            .unwrap_or(transport_limit);
424        let codec = WireFrameCodec::new(max_frame_bytes);
425        Ok(codec.encode(frame)?)
426    }
427
428    /// Route a decoded inbound frame. Host transports only legitimately receive `Response`, `Event`,
429    /// and `SidecarRequest` frames.
430    async fn handle_wire_frame(self: &Arc<Self>, frame: wire::ProtocolFrame) {
431        match frame {
432            wire::ProtocolFrame::ResponseFrame(response) => {
433                match self.pending.remove(&response.request_id) {
434                    Some((_, tx)) => {
435                        let _ = tx.send(response.payload);
436                    }
437                    None => {
438                        tracing::warn!(
439                            request_id = response.request_id,
440                            "response for unknown request id"
441                        )
442                    }
443                }
444            }
445            wire::ProtocolFrame::EventFrame(event) => {
446                // Transport-level liveness beats from the sidecar. Their arrival
447                // already reset the silence watchdog in the reader; they carry no
448                // meaning for event subscribers, so drop them here (mirrors the
449                // TS client's heartbeat swallow).
450                if matches!(
451                    &event.payload,
452                    wire::EventPayload::StructuredEvent(structured)
453                        if structured.name == "heartbeat"
454                ) {
455                    return;
456                }
457                if let Some(event_tx) = self.event_tx.lock().as_ref() {
458                    let _ = event_tx.send((event.ownership, event.payload));
459                }
460            }
461            wire::ProtocolFrame::SidecarRequestFrame(request) => {
462                self.dispatch_sidecar_request(request).await
463            }
464            wire::ProtocolFrame::SidecarResponseFrame(_)
465            | wire::ProtocolFrame::RequestFrame(_)
466            | wire::ProtocolFrame::ControlFrame(_) => {
467                tracing::warn!("unexpected inbound frame on host transport")
468            }
469        }
470    }
471
472    /// Dispatch a sidecar-initiated request to its registered callback. The callback runs in a
473    /// spawned task so long-running host callbacks (binding execution, permission prompts) cannot stall
474    /// the reader loop, which must keep draining responses for any requests the callback itself
475    /// issues through this transport.
476    async fn dispatch_sidecar_request(self: &Arc<Self>, frame: wire::SidecarRequestFrame) {
477        let key = sidecar_request_key(&frame.payload);
478        let callback = self.callbacks.read(&key, |_, value| value.clone());
479        match callback {
480            Some(callback) => {
481                let transport = Arc::downgrade(self);
482                tokio::spawn(async move {
483                    match callback(frame.payload, frame.ownership.clone()).await {
484                        Ok(payload) => {
485                            let response = wire::ProtocolFrame::SidecarResponseFrame(
486                                wire::SidecarResponseFrame {
487                                    schema: wire::protocol_schema(),
488                                    request_id: frame.request_id,
489                                    ownership: frame.ownership,
490                                    payload,
491                                },
492                            );
493                            // If the transport is gone, the child is being killed; drop the reply.
494                            let Some(transport) = transport.upgrade() else {
495                                return;
496                            };
497                            if let Ok(bytes) = transport.encode_wire_frame(&response, None) {
498                                let _ = transport.control_writer_tx.send(bytes).await;
499                            }
500                        }
501                        Err(error) => tracing::warn!(?error, key, "sidecar callback failed"),
502                    }
503                });
504            }
505            None => tracing::warn!(key, "no callback registered for sidecar request"),
506        }
507    }
508
509    /// Reject every in-flight request after the transport disconnects.
510    fn fail_all_pending(&self) {
511        // Serialized with `register_pending_request`, so no request can register after this
512        // clear without seeing `disconnected`.
513        let _guard = self.pending_request_lock.lock();
514        self.pending.clear();
515    }
516
517    fn disconnect(&self) {
518        self.disconnected.store(true, Ordering::SeqCst);
519        self.kill_child();
520        self.fail_all_pending();
521        // The event lane closes the event stream once it drains what the sidecar already wrote,
522        // such as a process exit. Bound that drain in case the lane never reaches EOF.
523        match tokio::runtime::Handle::try_current() {
524            Ok(runtime) => {
525                let event_tx = Arc::clone(&self.event_tx);
526                runtime.spawn(async move {
527                    tokio::time::sleep(EVENT_DRAIN_GRACE).await;
528                    event_tx.lock().take();
529                });
530            }
531            Err(_) => self.close_events(),
532        }
533    }
534
535    /// End the event stream: subscribers receive every event already sent, then `Closed`.
536    fn close_events(&self) {
537        self.event_tx.lock().take();
538    }
539
540    fn register_pending_request(
541        &self,
542        request_id: wire::RequestId,
543        tx: oneshot::Sender<wire::ResponsePayload>,
544    ) -> Result<(), TransportError> {
545        let _guard = self.pending_request_lock.lock();
546        if self.is_disconnected() {
547            return Err(TransportError::Disconnected(
548                "request sent after the sidecar process was lost".to_string(),
549            ));
550        }
551        if pending_request_count(self) >= PENDING_REQUEST_LIMIT {
552            return Err(TransportError::Sidecar(format!(
553                "sidecar pending request limit exceeded: at most {PENDING_REQUEST_LIMIT} requests can be in flight"
554            )));
555        }
556        let _ = self.pending.insert(request_id, tx);
557        Ok(())
558    }
559}
560
561#[cfg(unix)]
562fn map_control_fd(
563    command: &mut Command,
564    control_child: StdUnixStream,
565) -> Result<(), TransportError> {
566    command
567        .fd_mappings(vec![FdMapping {
568            parent_fd: control_child.into(),
569            child_fd: CONTROL_FD,
570        }])
571        .map_err(|error| {
572            TransportError::Sidecar(format!(
573                "failed to map sidecar response/control fd: {error}"
574            ))
575        })?;
576    Ok(())
577}
578
579struct PendingRequestGuard<'a> {
580    transport: &'a SidecarTransport,
581    request_id: wire::RequestId,
582}
583
584impl<'a> PendingRequestGuard<'a> {
585    fn new(transport: &'a SidecarTransport, request_id: wire::RequestId) -> Self {
586        Self {
587            transport,
588            request_id,
589        }
590    }
591}
592
593impl Drop for PendingRequestGuard<'_> {
594    fn drop(&mut self) {
595        let _ = self.transport.pending.remove(&self.request_id);
596    }
597}
598
599fn pending_request_count(transport: &SidecarTransport) -> usize {
600    let mut count = 0;
601    transport.pending.scan(|_, _| {
602        count += 1;
603    });
604    count
605}
606
607/// Map a sidecar-request payload to the callback registry key.
608fn sidecar_request_key(payload: &wire::SidecarRequestPayload) -> &'static str {
609    match payload {
610        wire::SidecarRequestPayload::HostCallbackRequest(_) => "host_callback",
611        wire::SidecarRequestPayload::JsBridgeCallRequest(_) => "js_bridge_call",
612        wire::SidecarRequestPayload::ExtEnvelope(_) => "ext",
613    }
614}
615
616/// Drain one bounded outbound lane into its physically independent stream.
617async fn run_writer<W>(
618    transport: Weak<SidecarTransport>,
619    lane: &'static str,
620    mut writer: W,
621    mut frames: mpsc::Receiver<Vec<u8>>,
622) where
623    W: AsyncWrite + Unpin,
624{
625    while let Some(bytes) = frames.recv().await {
626        let result = async {
627            writer.write_all(&bytes).await?;
628            writer.flush().await
629        }
630        .await;
631        if let Err(error) = result {
632            tracing::warn!(?error, lane, "sidecar writer failed");
633            if let Some(transport) = transport.upgrade() {
634                transport.disconnect();
635            }
636            return;
637        }
638    }
639}
640
641#[derive(Clone, Copy, Debug)]
642enum InboundLane {
643    Event,
644    Control,
645}
646
647impl InboundLane {
648    fn accepts(self, frame: &wire::ProtocolFrame) -> bool {
649        match self {
650            Self::Event => {
651                matches!(frame, wire::ProtocolFrame::EventFrame(event) if !is_heartbeat(event))
652            }
653            Self::Control => match frame {
654                wire::ProtocolFrame::ResponseFrame(_)
655                | wire::ProtocolFrame::SidecarRequestFrame(_) => true,
656                wire::ProtocolFrame::EventFrame(event) => is_heartbeat(event),
657                wire::ProtocolFrame::RequestFrame(_)
658                | wire::ProtocolFrame::SidecarResponseFrame(_)
659                | wire::ProtocolFrame::ControlFrame(_) => false,
660            },
661        }
662    }
663}
664
665fn is_heartbeat(event: &wire::EventFrame) -> bool {
666    matches!(
667        &event.payload,
668        wire::EventPayload::StructuredEvent(structured) if structured.name == "heartbeat"
669    )
670}
671
672/// Read length-prefixed BARE frames from one physical inbound lane and route them.
673async fn run_reader<R>(transport: Weak<SidecarTransport>, reader: R, lane: InboundLane)
674where
675    R: AsyncRead + Unpin,
676{
677    route_frames(transport.clone(), reader, lane).await;
678    // Only the event lane carries events, so once it ends no event can follow.
679    if matches!(lane, InboundLane::Event) {
680        if let Some(transport) = transport.upgrade() {
681            transport.close_events();
682        }
683    }
684}
685
686async fn route_frames<R>(transport: Weak<SidecarTransport>, mut reader: R, lane: InboundLane)
687where
688    R: AsyncRead + Unpin,
689{
690    loop {
691        let mut length_buf = [0u8; 4];
692        if let Err(error) = reader.read_exact(&mut length_buf).await {
693            if let Some(transport) = transport.upgrade() {
694                tracing::warn!(?error, ?lane, "sidecar reader ended");
695                transport.disconnect();
696            }
697            return;
698        }
699        let length = u32::from_be_bytes(length_buf) as usize;
700
701        let Some(transport) = transport.upgrade() else {
702            break;
703        };
704        let max_frame_bytes = transport.max_frame_bytes.load(Ordering::Relaxed);
705        if frame_length_exceeds_limit(length, max_frame_bytes) {
706            tracing::warn!(
707                size = length,
708                max = max_frame_bytes,
709                "sidecar frame exceeds negotiated limit"
710            );
711            transport.disconnect();
712            return;
713        }
714
715        let mut frame_bytes = vec![0u8; 4 + length];
716        frame_bytes[..4].copy_from_slice(&length_buf);
717        if let Err(error) = reader.read_exact(&mut frame_bytes[4..]).await {
718            tracing::warn!(?error, ?lane, "sidecar reader ended mid-frame");
719            transport.disconnect();
720            return;
721        }
722        // Any complete inbound frame proves the sidecar is alive; the silence
723        // watchdog counts from here.
724        *transport.counted_silence.lock() = std::time::Duration::ZERO;
725
726        let codec = WireFrameCodec::new(max_frame_bytes);
727        match codec.decode(&frame_bytes) {
728            Ok(frame) if lane.accepts(&frame) => transport.handle_wire_frame(frame).await,
729            Ok(frame) => {
730                tracing::warn!(?lane, frame = ?frame, "sidecar frame arrived on wrong transport lane");
731                transport.disconnect();
732                return;
733            }
734            Err(error) => {
735                tracing::warn!(?error, ?lane, "failed to decode sidecar frame");
736                transport.disconnect();
737                return;
738            }
739        }
740    }
741}
742
743fn frame_length_exceeds_limit(length: usize, max_frame_bytes: usize) -> bool {
744    length > max_frame_bytes
745}
746
747/// Kill the sidecar and fail all in-flight requests once the transport has seen no inbound frames
748/// (not even heartbeats) for `timeout`. A silent sidecar is dead or wedged, not busy: a busy
749/// sidecar still heartbeats every 10s from a dedicated thread. Exits when the transport drops.
750///
751/// Each tick counts at most one check interval. While the host runtime or process is stalled,
752/// frames wait unread in the pipe, and on wake this task can run before the readers do. That time
753/// says nothing about the sidecar. Every tick still counts, so a silent sidecar is detected after a
754/// fixed number of ticks however long the host stalls between them.
755async fn run_silence_watchdog(transport: Weak<SidecarTransport>, timeout: std::time::Duration) {
756    let check_interval = (timeout / 4).min(std::time::Duration::from_secs(1));
757    let mut last_tick_at = std::time::Instant::now();
758    loop {
759        tokio::time::sleep(check_interval).await;
760        let Some(transport) = transport.upgrade() else {
761            return;
762        };
763        let now = std::time::Instant::now();
764        let counted_tick = now.duration_since(last_tick_at).min(check_interval);
765        last_tick_at = now;
766        let silence = {
767            let mut counted_silence = transport.counted_silence.lock();
768            *counted_silence += counted_tick;
769            *counted_silence
770        };
771        if silence < timeout {
772            continue;
773        }
774        tracing::error!(
775            silence_ms = silence.as_millis() as u64,
776            "sidecar unresponsive: no protocol frames or heartbeats; killing sidecar",
777        );
778        transport.disconnect();
779        return;
780    }
781}
782
783fn resolve_sidecar_binary_path(binary_path: Option<String>) -> String {
784    binary_path
785        .or_else(|| std::env::var(SIDECAR_BIN_ENV).ok())
786        .unwrap_or_else(|| "agentos-native-sidecar".to_string())
787}
788
789// Child::wait is cancellation-safe, but moving Child out of its owner before
790// awaiting it is not. Restore ownership on cancellation/error so the next
791// terminate_child caller must confirm exit rather than seeing an empty slot.
792struct ReapingChild<'a> {
793    slot: &'a parking_lot::Mutex<Option<Child>>,
794    child: Option<Child>,
795}
796
797impl Drop for ReapingChild<'_> {
798    fn drop(&mut self) {
799        if let Some(child) = self.child.take() {
800            *self.slot.lock() = Some(child);
801        }
802    }
803}
804
805#[cfg(test)]
806mod tests {
807    use super::*;
808    use std::sync::Mutex;
809    use std::time::Duration;
810
811    static ENV_LOCK: Mutex<()> = Mutex::new(());
812
813    #[tokio::test]
814    async fn cancelled_reaping_restores_child_ownership_for_confirmed_retry() {
815        let transport = test_transport();
816        let child = Command::new("sleep")
817            .arg("30")
818            .kill_on_drop(true)
819            .spawn()
820            .unwrap();
821        let pid = child.id().unwrap();
822        transport.child.lock().replace(child);
823        let child = transport.child.lock().take();
824        let reaping = ReapingChild {
825            slot: &transport.child,
826            child,
827        };
828        assert!(transport.child.lock().is_none());
829        // This is the same ownership rollback run if terminate_child's wait
830        // future is dropped before it can confirm exit.
831        drop(reaping);
832        assert_eq!(transport.child.lock().as_ref().unwrap().id(), Some(pid));
833        tokio::time::timeout(Duration::from_secs(2), transport.terminate_child())
834            .await
835            .unwrap()
836            .unwrap();
837        assert!(transport.child.lock().is_none());
838        transport.terminate_child().await.unwrap();
839    }
840
841    #[tokio::test]
842    async fn disconnected_child_is_retained_until_reaped() {
843        let transport = test_transport();
844        let child = Command::new("sleep")
845            .arg("30")
846            .kill_on_drop(true)
847            .spawn()
848            .unwrap();
849        transport.child.lock().replace(child);
850        transport.kill_child();
851        assert!(transport.child.lock().is_some());
852        tokio::time::timeout(Duration::from_secs(2), transport.terminate_child())
853            .await
854            .unwrap()
855            .unwrap();
856        assert!(transport.child.lock().is_none());
857    }
858
859    fn test_transport() -> SidecarTransport {
860        let (request_writer_tx, _request_writer_rx) = mpsc::channel(REQUEST_FRAME_QUEUE_CAPACITY);
861        let (control_writer_tx, _control_writer_rx) = mpsc::channel(CONTROL_FRAME_QUEUE_CAPACITY);
862        let (event_tx, _) = broadcast::channel(EVENT_CHANNEL_CAPACITY);
863        SidecarTransport {
864            instance_id: next_transport_instance_id(),
865            child: parking_lot::Mutex::new(None),
866            child_termination: tokio::sync::Mutex::new(()),
867            pending: SccHashMap::new(),
868            pending_request_lock: parking_lot::Mutex::new(()),
869            request_counter: AtomicI64::new(1),
870            max_frame_bytes: AtomicUsize::new(wire::DEFAULT_MAX_FRAME_BYTES),
871            max_write_files_entries: AtomicUsize::new(1),
872            event_tx: Arc::new(parking_lot::Mutex::new(Some(event_tx))),
873            callbacks: SccHashMap::new(),
874            request_writer_tx,
875            control_writer_tx,
876            counted_silence: parking_lot::Mutex::new(std::time::Duration::ZERO),
877            disconnected: AtomicBool::new(false),
878        }
879    }
880
881    #[test]
882    fn binary_path_prefers_explicit_path_over_env() {
883        let _guard = ENV_LOCK.lock().expect("env lock");
884        let previous = std::env::var(SIDECAR_BIN_ENV).ok();
885        std::env::set_var(SIDECAR_BIN_ENV, "/tmp/from-env");
886
887        assert_eq!(
888            resolve_sidecar_binary_path(Some("/tmp/from-config".to_string())),
889            "/tmp/from-config"
890        );
891
892        restore_env(SIDECAR_BIN_ENV, previous);
893    }
894
895    #[test]
896    fn binary_path_uses_agentos_env_fallback() {
897        let _guard = ENV_LOCK.lock().expect("env lock");
898        let previous = std::env::var(SIDECAR_BIN_ENV).ok();
899        std::env::set_var(SIDECAR_BIN_ENV, "/tmp/agentos-native-sidecar");
900
901        assert_eq!(
902            resolve_sidecar_binary_path(None),
903            "/tmp/agentos-native-sidecar"
904        );
905
906        restore_env(SIDECAR_BIN_ENV, previous);
907    }
908
909    #[test]
910    fn binary_path_defaults_to_agentos_native_sidecar() {
911        let _guard = ENV_LOCK.lock().expect("env lock");
912        let previous = std::env::var(SIDECAR_BIN_ENV).ok();
913        std::env::remove_var(SIDECAR_BIN_ENV);
914
915        assert_eq!(resolve_sidecar_binary_path(None), "agentos-native-sidecar");
916
917        restore_env(SIDECAR_BIN_ENV, previous);
918    }
919
920    fn restore_env(key: &str, value: Option<String>) {
921        match value {
922            Some(value) => std::env::set_var(key, value),
923            None => std::env::remove_var(key),
924        }
925    }
926
927    #[test]
928    fn frame_length_limit_rejects_oversized_declared_length() {
929        assert!(!frame_length_exceeds_limit(1024, 1024));
930        assert!(frame_length_exceeds_limit(1025, 1024));
931    }
932
933    #[test]
934    fn transport_encodes_requests_with_generated_wire_codec() {
935        let transport = test_transport();
936        let frame = wire::ProtocolFrame::RequestFrame(wire::RequestFrame {
937            schema: wire::protocol_schema(),
938            request_id: 7,
939            ownership: wire::OwnershipScope::ConnectionOwnership(wire::ConnectionOwnership {
940                connection_id: "conn-1".to_string(),
941            }),
942            payload: wire::RequestPayload::AuthenticateRequest(wire::AuthenticateRequest {
943                client_name: "transport-test".to_string(),
944                auth_token: "token".to_string(),
945                protocol_version: wire::PROTOCOL_VERSION,
946                bridge_version: 1,
947            }),
948        });
949
950        let encoded = transport
951            .encode_wire_frame(&frame, None)
952            .expect("encode transport frame");
953        let decoded = WireFrameCodec::default()
954            .decode(&encoded)
955            .expect("decode generated wire frame");
956
957        assert!(matches!(
958            decoded,
959            wire::ProtocolFrame::RequestFrame(wire::RequestFrame {
960                payload: wire::RequestPayload::AuthenticateRequest(_),
961                ..
962            })
963        ));
964    }
965
966    #[tokio::test]
967    async fn transport_fans_out_generated_wire_events() {
968        let transport = Arc::new(test_transport());
969        let mut wire_events = transport.subscribe_wire_events();
970
971        transport
972            .handle_wire_frame(wire::ProtocolFrame::EventFrame(wire::EventFrame {
973                schema: wire::protocol_schema(),
974                ownership: wire::OwnershipScope::VmOwnership(wire::VmOwnership {
975                    connection_id: "conn-1".to_string(),
976                    session_id: "session-1".to_string(),
977                    vm_id: "vm-1".to_string(),
978                }),
979                payload: wire::EventPayload::ProcessOutputEvent(wire::ProcessOutputEvent {
980                    process_id: "proc-1".to_string(),
981                    channel: wire::StreamChannel::Stdout,
982                    chunk: b"hello".to_vec(),
983                    sequence: None,
984                    timestamp_ms: None,
985                }),
986            }))
987            .await;
988
989        let (ownership, payload) = wire_events.recv().await.expect("wire event");
990        assert!(matches!(
991            ownership,
992            wire::OwnershipScope::VmOwnership(wire::VmOwnership {
993                connection_id,
994                session_id,
995                vm_id,
996            }) if connection_id == "conn-1" && session_id == "session-1" && vm_id == "vm-1"
997        ));
998        assert!(matches!(
999            payload,
1000            wire::EventPayload::ProcessOutputEvent(wire::ProcessOutputEvent {
1001                process_id,
1002                channel: wire::StreamChannel::Stdout,
1003                chunk,
1004                sequence: None,
1005                timestamp_ms: None,
1006            }) if process_id == "proc-1" && chunk == b"hello".to_vec()
1007        ));
1008    }
1009
1010    #[tokio::test]
1011    async fn event_before_response_is_delivered_without_transport_history() {
1012        let transport = Arc::new(test_transport());
1013        let mut events = transport.subscribe_wire_events();
1014        let (response_tx, response_rx) = oneshot::channel();
1015        transport
1016            .register_pending_request(7, response_tx)
1017            .expect("register pending request");
1018        let ownership = wire::OwnershipScope::ConnectionOwnership(wire::ConnectionOwnership {
1019            connection_id: "conn-1".to_string(),
1020        });
1021
1022        transport
1023            .handle_wire_frame(wire::ProtocolFrame::EventFrame(wire::EventFrame {
1024                schema: wire::protocol_schema(),
1025                ownership: ownership.clone(),
1026                payload: wire::EventPayload::StructuredEvent(wire::StructuredEvent {
1027                    name: "process_started_before_reply".to_string(),
1028                    detail: std::collections::HashMap::new(),
1029                }),
1030            }))
1031            .await;
1032        transport
1033            .handle_wire_frame(wire::ProtocolFrame::ResponseFrame(wire::ResponseFrame {
1034                schema: wire::protocol_schema(),
1035                request_id: 7,
1036                ownership,
1037                payload: wire::ResponsePayload::ExtEnvelope(wire::ExtEnvelope {
1038                    namespace: "test".to_string(),
1039                    payload: Vec::new(),
1040                }),
1041            }))
1042            .await;
1043
1044        let (_, event) = events.recv().await.expect("live event");
1045        assert!(matches!(
1046            event,
1047            wire::EventPayload::StructuredEvent(wire::StructuredEvent { name, .. })
1048                if name == "process_started_before_reply"
1049        ));
1050        assert!(matches!(
1051            response_rx.await.expect("response"),
1052            wire::ResponsePayload::ExtEnvelope(_)
1053        ));
1054    }
1055
1056    #[tokio::test]
1057    async fn bounded_event_fanout_reports_lag_instead_of_retaining_history() {
1058        let (event_tx, mut events) = broadcast::channel(2);
1059        for index in 0..3 {
1060            event_tx
1061                .send((
1062                    wire::OwnershipScope::ConnectionOwnership(wire::ConnectionOwnership {
1063                        connection_id: "conn-1".to_string(),
1064                    }),
1065                    wire::EventPayload::StructuredEvent(wire::StructuredEvent {
1066                        name: format!("event-{index}"),
1067                        detail: std::collections::HashMap::new(),
1068                    }),
1069                ))
1070                .expect("active receiver");
1071        }
1072
1073        assert!(matches!(
1074            events.recv().await,
1075            Err(broadcast::error::RecvError::Lagged(1))
1076        ));
1077    }
1078
1079    #[tokio::test]
1080    async fn silence_watchdog_fails_pending_requests_after_sustained_silence() {
1081        let transport = Arc::new(test_transport());
1082        let (tx, rx) = oneshot::channel();
1083        transport
1084            .register_pending_request(1, tx)
1085            .expect("register pending request");
1086
1087        tokio::spawn(run_silence_watchdog(
1088            Arc::downgrade(&transport),
1089            std::time::Duration::from_millis(40),
1090        ));
1091
1092        // No inbound activity at all: the watchdog must reject the pending
1093        // request (dropped sender -> disconnected error at the caller).
1094        rx.await
1095            .expect_err("watchdog should drop the pending sender");
1096        assert_eq!(pending_request_count(&transport), 0);
1097    }
1098
1099    #[tokio::test]
1100    async fn silence_watchdog_stays_quiet_while_frames_arrive() {
1101        let transport = Arc::new(test_transport());
1102        let (tx, mut rx) = oneshot::channel();
1103        transport
1104            .register_pending_request(1, tx)
1105            .expect("register pending request");
1106
1107        tokio::spawn(run_silence_watchdog(
1108            Arc::downgrade(&transport),
1109            std::time::Duration::from_millis(120),
1110        ));
1111
1112        // Simulate steady inbound activity (what the reader does per frame)
1113        // for well past the silence window; the watchdog must not fire.
1114        for _ in 0..6 {
1115            tokio::time::sleep(std::time::Duration::from_millis(40)).await;
1116            *transport.counted_silence.lock() = std::time::Duration::ZERO;
1117            assert!(
1118                rx.try_recv().is_err(),
1119                "pending request must remain registered while frames arrive"
1120            );
1121        }
1122        assert_eq!(pending_request_count(&transport), 1);
1123    }
1124
1125    #[tokio::test]
1126    async fn silence_watchdog_ignores_a_stalled_host_but_still_detects_silence() {
1127        let transport = Arc::new(test_transport());
1128        let (tx, mut rx) = oneshot::channel();
1129        transport
1130            .register_pending_request(1, tx)
1131            .expect("register pending request");
1132
1133        let timeout = std::time::Duration::from_millis(200);
1134        tokio::spawn(run_silence_watchdog(Arc::downgrade(&transport), timeout));
1135        tokio::task::yield_now().await;
1136
1137        // Block this single-threaded runtime past the window. The stall says
1138        // nothing about the sidecar, so the watchdog must not fire on wake.
1139        std::thread::sleep(timeout * 3);
1140        tokio::time::sleep(std::time::Duration::from_millis(60)).await;
1141        assert!(
1142            matches!(rx.try_recv(), Err(oneshot::error::TryRecvError::Empty)),
1143            "a host stall must not count as sidecar silence"
1144        );
1145
1146        // The sidecar stays silent while the host keeps stalling in bursts longer
1147        // than the check interval: the watchdog still fires.
1148        for _ in 0..100 {
1149            std::thread::sleep(std::time::Duration::from_millis(80));
1150            tokio::time::sleep(std::time::Duration::from_millis(1)).await;
1151            if matches!(rx.try_recv(), Err(oneshot::error::TryRecvError::Closed)) {
1152                assert_eq!(pending_request_count(&transport), 0);
1153                return;
1154            }
1155        }
1156        panic!("a silent sidecar must be detected while the host stalls in bursts");
1157    }
1158
1159    #[tokio::test]
1160    async fn heartbeat_events_are_swallowed_before_the_event_fanout() {
1161        let transport = Arc::new(test_transport());
1162        let mut wire_events = transport.subscribe_wire_events();
1163
1164        transport
1165            .handle_wire_frame(wire::ProtocolFrame::EventFrame(wire::EventFrame {
1166                schema: wire::protocol_schema(),
1167                ownership: wire::OwnershipScope::ConnectionOwnership(wire::ConnectionOwnership {
1168                    connection_id: "sidecar-transport".to_string(),
1169                }),
1170                payload: wire::EventPayload::StructuredEvent(wire::StructuredEvent {
1171                    name: "heartbeat".to_string(),
1172                    detail: std::collections::HashMap::new(),
1173                }),
1174            }))
1175            .await;
1176        // A non-heartbeat structured event still fans out, proving the filter
1177        // is name-scoped rather than dropping all structured events.
1178        transport
1179            .handle_wire_frame(wire::ProtocolFrame::EventFrame(wire::EventFrame {
1180                schema: wire::protocol_schema(),
1181                ownership: wire::OwnershipScope::ConnectionOwnership(wire::ConnectionOwnership {
1182                    connection_id: "conn-1".to_string(),
1183                }),
1184                payload: wire::EventPayload::StructuredEvent(wire::StructuredEvent {
1185                    name: "limit_warning".to_string(),
1186                    detail: std::collections::HashMap::new(),
1187                }),
1188            }))
1189            .await;
1190
1191        let (_, payload) = wire_events.recv().await.expect("structured event");
1192        assert!(matches!(
1193            payload,
1194            wire::EventPayload::StructuredEvent(wire::StructuredEvent { name, .. })
1195                if name == "limit_warning"
1196        ));
1197        assert!(
1198            wire_events.try_recv().is_err(),
1199            "heartbeat must not fan out"
1200        );
1201    }
1202
1203    #[test]
1204    fn pending_request_guard_removes_registered_slot_on_drop() {
1205        let transport = test_transport();
1206        let (tx, _rx) = oneshot::channel();
1207        transport
1208            .register_pending_request(1, tx)
1209            .expect("register pending request");
1210
1211        {
1212            let _guard = PendingRequestGuard::new(&transport, 1);
1213            assert_eq!(pending_request_count(&transport), 1);
1214        }
1215
1216        assert_eq!(pending_request_count(&transport), 0);
1217    }
1218
1219    #[test]
1220    fn pending_request_limit_rejects_full_transport() {
1221        let transport = test_transport();
1222        for request_id in 1..=PENDING_REQUEST_LIMIT as wire::RequestId {
1223            let (tx, _rx) = oneshot::channel();
1224            transport
1225                .register_pending_request(request_id, tx)
1226                .expect("register pending request");
1227        }
1228        let (tx, _rx) = oneshot::channel();
1229        let error = transport
1230            .register_pending_request((PENDING_REQUEST_LIMIT + 1) as wire::RequestId, tx)
1231            .expect_err("full pending map should reject");
1232
1233        assert!(
1234            error
1235                .to_string()
1236                .contains("sidecar pending request limit exceeded"),
1237            "unexpected error: {error}"
1238        );
1239    }
1240
1241    #[tokio::test]
1242    async fn shutdown_uses_typed_control_frame() {
1243        let (request_writer_tx, _request_writer_rx) = mpsc::channel(4);
1244        let (control_writer_tx, mut control_writer_rx) = mpsc::channel(4);
1245        let (event_tx, _) = broadcast::channel(4);
1246        let transport = SidecarTransport {
1247            instance_id: next_transport_instance_id(),
1248            child: parking_lot::Mutex::new(None),
1249            child_termination: tokio::sync::Mutex::new(()),
1250            pending: SccHashMap::new(),
1251            pending_request_lock: parking_lot::Mutex::new(()),
1252            request_counter: AtomicI64::new(1),
1253            max_frame_bytes: AtomicUsize::new(wire::DEFAULT_MAX_FRAME_BYTES),
1254            max_write_files_entries: AtomicUsize::new(1),
1255            event_tx: Arc::new(parking_lot::Mutex::new(Some(event_tx))),
1256            callbacks: SccHashMap::new(),
1257            request_writer_tx,
1258            control_writer_tx,
1259            counted_silence: parking_lot::Mutex::new(std::time::Duration::ZERO),
1260            disconnected: AtomicBool::new(false),
1261        };
1262
1263        transport
1264            .shutdown("test complete")
1265            .await
1266            .expect("enqueue typed shutdown");
1267        let bytes = control_writer_rx.recv().await.expect("control frame bytes");
1268        let frame = WireFrameCodec::default()
1269            .decode(&bytes)
1270            .expect("decode shutdown frame");
1271        assert!(matches!(
1272            frame,
1273            wire::ProtocolFrame::ControlFrame(wire::ControlFrame {
1274                payload: wire::ControlPayload::ShutdownControl(wire::ShutdownControl { reason }),
1275                ..
1276            }) if reason == "test complete"
1277        ));
1278    }
1279
1280    #[cfg(unix)]
1281    #[tokio::test]
1282    async fn unix_spawn_maps_duplex_control_socket_to_fd_three() {
1283        let (mut parent, child) = StdUnixStream::pair().expect("control socketpair");
1284        parent
1285            .set_read_timeout(Some(std::time::Duration::from_secs(2)))
1286            .expect("control read timeout");
1287        let mut command = Command::new("sh");
1288        command.arg("-c").arg("printf mapped-control >&3");
1289        map_control_fd(&mut command, child).expect("map child control fd");
1290        let mut child = command.spawn().expect("spawn fd mapping probe");
1291        drop(command);
1292
1293        let mut received = [0_u8; 14];
1294        std::io::Read::read_exact(&mut parent, &mut received).expect("read mapped fd output");
1295        assert_eq!(&received, b"mapped-control");
1296        assert!(child
1297            .wait()
1298            .await
1299            .expect("wait for mapping probe")
1300            .success());
1301    }
1302
1303    #[tokio::test]
1304    async fn control_writer_progresses_while_ordinary_stream_is_blocked() {
1305        let transport = Arc::new(test_transport());
1306        let (ordinary_client, _ordinary_server) = tokio::io::duplex(1);
1307        let (control_client, mut control_server) = tokio::io::duplex(64);
1308        let (control_tx, control_rx) = mpsc::channel(CONTROL_FRAME_QUEUE_CAPACITY);
1309        let (request_tx, request_rx) = mpsc::channel(REQUEST_FRAME_QUEUE_CAPACITY);
1310        request_tx
1311            .send(vec![b'r'; 64])
1312            .await
1313            .expect("send request frame");
1314        control_tx
1315            .send(vec![b'c'])
1316            .await
1317            .expect("send control frame");
1318        let ordinary_writer = tokio::spawn(run_writer(
1319            Arc::downgrade(&transport),
1320            "ordinary",
1321            ordinary_client,
1322            request_rx,
1323        ));
1324        let control_writer = tokio::spawn(run_writer(
1325            Arc::downgrade(&transport),
1326            "control",
1327            control_client,
1328            control_rx,
1329        ));
1330        let mut first = [0u8; 1];
1331        tokio::time::timeout(
1332            std::time::Duration::from_secs(1),
1333            control_server.read_exact(&mut first),
1334        )
1335        .await
1336        .expect("control write must not wait for ordinary stream")
1337        .expect("read first byte");
1338        assert_eq!(first, [b'c']);
1339        ordinary_writer.abort();
1340        control_writer.abort();
1341    }
1342
1343    #[tokio::test]
1344    async fn eof_on_either_inbound_lane_fails_pending_requests() {
1345        for lane in [InboundLane::Event, InboundLane::Control] {
1346            let transport = Arc::new(test_transport());
1347            let (tx, rx) = oneshot::channel();
1348            transport
1349                .register_pending_request(1, tx)
1350                .expect("register pending request");
1351            let (reader, peer) = tokio::io::duplex(64);
1352            drop(peer);
1353
1354            tokio::time::timeout(
1355                std::time::Duration::from_secs(1),
1356                run_reader(Arc::downgrade(&transport), reader, lane),
1357            )
1358            .await
1359            .expect("reader must terminate on EOF");
1360
1361            rx.await.expect_err("EOF must fail pending requests");
1362            assert_eq!(pending_request_count(&transport), 0);
1363        }
1364    }
1365
1366    /// The control lane can close before the event lane drains, for example when the sidecar
1367    /// exits right after writing a process exit. Events already written to the event lane must
1368    /// still reach subscribers before they see the transport close.
1369    #[tokio::test]
1370    async fn events_written_before_a_disconnect_still_reach_subscribers() {
1371        let transport = Arc::new(test_transport());
1372        let mut events = transport.subscribe_wire_events();
1373
1374        let (control_reader, control_peer) = tokio::io::duplex(64);
1375        drop(control_peer);
1376        run_reader(
1377            Arc::downgrade(&transport),
1378            control_reader,
1379            InboundLane::Control,
1380        )
1381        .await;
1382        assert!(transport.is_disconnected());
1383
1384        let (event_reader, mut event_peer) = tokio::io::duplex(4096);
1385        let frame = WireFrameCodec::default()
1386            .encode(&wire::ProtocolFrame::EventFrame(wire::EventFrame {
1387                schema: wire::protocol_schema(),
1388                ownership: wire::OwnershipScope::ConnectionOwnership(wire::ConnectionOwnership {
1389                    connection_id: "conn-1".to_string(),
1390                }),
1391                payload: wire::EventPayload::StructuredEvent(wire::StructuredEvent {
1392                    name: "last_words".to_string(),
1393                    detail: std::collections::HashMap::new(),
1394                }),
1395            }))
1396            .expect("encode event");
1397        event_peer.write_all(&frame).await.expect("write event");
1398        drop(event_peer);
1399        tokio::time::timeout(
1400            Duration::from_secs(1),
1401            run_reader(Arc::downgrade(&transport), event_reader, InboundLane::Event),
1402        )
1403        .await
1404        .expect("event reader must end at EOF");
1405
1406        let (_, payload) = events
1407            .recv()
1408            .await
1409            .expect("the buffered event is delivered");
1410        assert!(matches!(
1411            payload,
1412            wire::EventPayload::StructuredEvent(wire::StructuredEvent { name, .. }) if name == "last_words"
1413        ));
1414        assert!(matches!(
1415            events.recv().await,
1416            Err(broadcast::error::RecvError::Closed)
1417        ));
1418    }
1419
1420    #[tokio::test]
1421    async fn writer_preserves_order_within_one_lane() {
1422        let transport = Arc::new(test_transport());
1423        let (client, mut server) = tokio::io::duplex(64);
1424        let (tx, rx) = mpsc::channel(CONTROL_FRAME_QUEUE_CAPACITY);
1425        tx.send(vec![b'c']).await.expect("control one");
1426        tx.send(vec![b'C']).await.expect("control two");
1427        drop(tx);
1428
1429        let writer = tokio::spawn(run_writer(
1430            Arc::downgrade(&transport),
1431            "control",
1432            client,
1433            rx,
1434        ));
1435        let mut output = [0u8; 2];
1436        server.read_exact(&mut output).await.expect("read output");
1437        writer.await.expect("writer task");
1438
1439        assert_eq!(output, [b'c', b'C']);
1440    }
1441}