1use std::process::Stdio;
12use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, AtomicUsize, Ordering};
13use std::sync::{Arc, Weak};
14
15#[cfg(unix)]
16use command_fds::{CommandFdExt, FdMapping};
17#[cfg(unix)]
18use std::os::unix::net::UnixStream as StdUnixStream;
19
20use scc::HashMap as SccHashMap;
21use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
22use tokio::process::{Child, Command};
23use tokio::sync::{broadcast, mpsc, oneshot};
24
25use crate::wire::{self, WireFrameCodec};
26use crate::TransportError;
27
28const EVENT_CHANNEL_CAPACITY: usize = 4096;
30
31const REQUEST_FRAME_QUEUE_CAPACITY: usize = 4096;
33
34const CONTROL_FRAME_QUEUE_CAPACITY: usize = 1024;
36
37const PENDING_REQUEST_LIMIT: usize = 4096;
39
40const SIDECAR_BIN_ENV: &str = "AGENTOS_SIDECAR_BIN";
43
44#[cfg(unix)]
46const CONTROL_FD: std::os::fd::RawFd = 3;
47
48const SIDECAR_SILENCE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
54
55const EVENT_DRAIN_GRACE: std::time::Duration = std::time::Duration::from_secs(2);
59
60type EventSenderSlot =
61 parking_lot::Mutex<Option<broadcast::Sender<(wire::OwnershipScope, wire::EventPayload)>>>;
62
63pub type WireSidecarCallback = Arc<
65 dyn Fn(
66 wire::SidecarRequestPayload,
67 wire::OwnershipScope,
68 ) -> futures::future::BoxFuture<
69 'static,
70 Result<wire::SidecarResponsePayload, TransportError>,
71 > + Send
72 + Sync,
73>;
74
75fn next_transport_instance_id() -> u64 {
76 static NEXT_INSTANCE_ID: AtomicU64 = AtomicU64::new(1);
77 NEXT_INSTANCE_ID.fetch_add(1, Ordering::Relaxed)
78}
79
80pub struct SidecarTransport {
83 instance_id: u64,
85 child: parking_lot::Mutex<Option<Child>>,
87 child_termination: tokio::sync::Mutex<()>,
88 pending: SccHashMap<wire::RequestId, oneshot::Sender<wire::ResponsePayload>>,
90 pending_request_lock: parking_lot::Mutex<()>,
91 request_counter: AtomicI64,
93 max_frame_bytes: AtomicUsize,
95 max_write_files_entries: AtomicUsize,
98 event_tx: Arc<EventSenderSlot>,
102 callbacks: SccHashMap<&'static str, WireSidecarCallback>,
104 request_writer_tx: mpsc::Sender<Vec<u8>>,
106 control_writer_tx: mpsc::Sender<Vec<u8>>,
108 counted_silence: parking_lot::Mutex<std::time::Duration>,
110 disconnected: AtomicBool,
113}
114
115impl SidecarTransport {
116 pub async fn spawn(binary_path: Option<String>) -> Result<Arc<Self>, TransportError> {
121 Self::spawn_with_args(binary_path, Vec::new()).await
122 }
123
124 pub async fn spawn_with_args(
126 binary_path: Option<String>,
127 args: Vec<String>,
128 ) -> Result<Arc<Self>, TransportError> {
129 #[cfg(not(unix))]
130 {
131 let _ = (binary_path, args);
132 return Err(TransportError::Sidecar(
133 "the native sidecar response/control transport is unsupported on this platform"
134 .to_string(),
135 ));
136 }
137
138 #[cfg(unix)]
139 {
140 Self::spawn_unix(binary_path, args).await
141 }
142 }
143
144 #[cfg(unix)]
145 async fn spawn_unix(
146 binary_path: Option<String>,
147 args: Vec<String>,
148 ) -> Result<Arc<Self>, TransportError> {
149 let bin = resolve_sidecar_binary_path(binary_path);
150 let (control_parent, control_child) = StdUnixStream::pair().map_err(|error| {
151 TransportError::Sidecar(format!(
152 "failed to create sidecar control socketpair: {error}"
153 ))
154 })?;
155 control_parent.set_nonblocking(true).map_err(|error| {
156 TransportError::Sidecar(format!(
157 "failed to configure sidecar control socket: {error}"
158 ))
159 })?;
160 let mut command = Command::new(&bin);
161 command
162 .args(&args)
163 .stdin(Stdio::piped())
164 .stdout(Stdio::piped())
165 .stderr(Stdio::inherit())
166 .kill_on_drop(true);
167 map_control_fd(&mut command, control_child)?;
171 let mut child = command.spawn().map_err(|error| {
172 TransportError::Sidecar(format!("failed to spawn sidecar '{bin}': {error}"))
173 })?;
174 drop(command);
175 let stdin = child
176 .stdin
177 .take()
178 .ok_or_else(|| TransportError::Sidecar("sidecar stdin was not piped".to_string()))?;
179 let stdout = child
180 .stdout
181 .take()
182 .ok_or_else(|| TransportError::Sidecar("sidecar stdout was not piped".to_string()))?;
183 let control = tokio::net::UnixStream::from_std(control_parent).map_err(|error| {
184 TransportError::Sidecar(format!("failed to adopt sidecar control socket: {error}"))
185 })?;
186 let (control_reader, control_writer) = control.into_split();
187
188 let (request_writer_tx, request_writer_rx) = mpsc::channel(REQUEST_FRAME_QUEUE_CAPACITY);
189 let (control_writer_tx, control_writer_rx) = mpsc::channel(CONTROL_FRAME_QUEUE_CAPACITY);
190 let (event_tx, _) = broadcast::channel(EVENT_CHANNEL_CAPACITY);
191
192 let transport = Arc::new(Self {
193 instance_id: next_transport_instance_id(),
194 child: parking_lot::Mutex::new(Some(child)),
195 child_termination: tokio::sync::Mutex::new(()),
196 pending: SccHashMap::new(),
197 pending_request_lock: parking_lot::Mutex::new(()),
198 request_counter: AtomicI64::new(1),
199 max_frame_bytes: AtomicUsize::new(wire::DEFAULT_MAX_FRAME_BYTES),
200 max_write_files_entries: AtomicUsize::new(1),
201 event_tx: Arc::new(parking_lot::Mutex::new(Some(event_tx))),
202 callbacks: SccHashMap::new(),
203 request_writer_tx,
204 control_writer_tx,
205 counted_silence: parking_lot::Mutex::new(std::time::Duration::ZERO),
206 disconnected: AtomicBool::new(false),
207 });
208
209 tokio::spawn(run_writer(
210 Arc::downgrade(&transport),
211 "ordinary request",
212 stdin,
213 request_writer_rx,
214 ));
215 tokio::spawn(run_writer(
216 Arc::downgrade(&transport),
217 "response/control",
218 control_writer,
219 control_writer_rx,
220 ));
221 tokio::spawn(run_reader(
222 Arc::downgrade(&transport),
223 stdout,
224 InboundLane::Event,
225 ));
226 tokio::spawn(run_reader(
227 Arc::downgrade(&transport),
228 control_reader,
229 InboundLane::Control,
230 ));
231 tokio::spawn(run_silence_watchdog(
232 Arc::downgrade(&transport),
233 SIDECAR_SILENCE_TIMEOUT,
234 ));
235
236 Ok(transport)
237 }
238
239 pub fn next_request_id(&self) -> wire::RequestId {
241 self.request_counter.fetch_add(1, Ordering::SeqCst)
242 }
243
244 pub async fn request_wire(
246 &self,
247 ownership: wire::OwnershipScope,
248 payload: wire::RequestPayload,
249 ) -> Result<wire::ResponsePayload, TransportError> {
250 self.request_wire_with_frame_limit(ownership, payload, None)
251 .await
252 }
253
254 pub async fn request_wire_bounded(
256 &self,
257 ownership: wire::OwnershipScope,
258 payload: wire::RequestPayload,
259 max_frame_bytes: usize,
260 ) -> Result<wire::ResponsePayload, TransportError> {
261 self.request_wire_with_frame_limit(ownership, payload, Some(max_frame_bytes))
262 .await
263 }
264
265 async fn request_wire_with_frame_limit(
266 &self,
267 ownership: wire::OwnershipScope,
268 payload: wire::RequestPayload,
269 max_frame_bytes: Option<usize>,
270 ) -> Result<wire::ResponsePayload, TransportError> {
271 let request_id = self.next_request_id();
272 let frame = wire::ProtocolFrame::RequestFrame(wire::RequestFrame {
273 schema: wire::protocol_schema(),
274 request_id,
275 ownership,
276 payload,
277 });
278 let bytes = self.encode_wire_frame(&frame, max_frame_bytes)?;
279
280 let (tx, rx) = oneshot::channel();
281 self.register_pending_request(request_id, tx)?;
282 let _pending_guard = PendingRequestGuard::new(self, request_id);
283
284 if self.request_writer_tx.send(bytes).await.is_err() {
285 self.pending.remove(&request_id);
286 return Err(TransportError::Disconnected(
287 "sidecar transport closed".to_string(),
288 ));
289 }
290
291 rx.await
292 .map_err(|_| TransportError::Disconnected("sidecar transport disconnected".to_string()))
293 }
294
295 pub fn subscribe_wire_events(
297 &self,
298 ) -> broadcast::Receiver<(wire::OwnershipScope, wire::EventPayload)> {
299 match self.event_tx.lock().as_ref() {
300 Some(event_tx) => event_tx.subscribe(),
301 None => broadcast::channel(1).1,
303 }
304 }
305
306 pub fn instance_id(&self) -> u64 {
308 self.instance_id
309 }
310
311 pub fn register_wire_callback(&self, key: &'static str, callback: WireSidecarCallback) {
314 let _ = self.callbacks.insert(key, callback);
315 }
316
317 pub fn max_frame_bytes(&self) -> usize {
319 self.max_frame_bytes.load(Ordering::Relaxed)
320 }
321
322 pub fn is_disconnected(&self) -> bool {
325 self.disconnected.load(Ordering::SeqCst)
326 }
327
328 pub fn set_max_frame_bytes(&self, max_frame_bytes: usize) {
330 self.max_frame_bytes
331 .store(max_frame_bytes, Ordering::SeqCst);
332 }
333
334 pub fn max_write_files_entries(&self) -> usize {
336 self.max_write_files_entries.load(Ordering::Relaxed)
337 }
338
339 pub fn set_max_write_files_entries(&self, max_entries: usize) {
341 self.max_write_files_entries
342 .store(max_entries.max(1), Ordering::SeqCst);
343 }
344
345 pub async fn shutdown(&self, reason: impl Into<String>) -> Result<(), TransportError> {
349 let frame = wire::ProtocolFrame::ControlFrame(wire::ControlFrame {
350 schema: wire::protocol_schema(),
351 payload: wire::ControlPayload::ShutdownControl(wire::ShutdownControl {
352 reason: reason.into(),
353 }),
354 });
355 let bytes = self.encode_wire_frame(&frame, None)?;
356 self.control_writer_tx.send(bytes).await.map_err(|_| {
357 TransportError::Sidecar("sidecar response/control transport closed".to_string())
358 })
359 }
360
361 pub fn kill_child(&self) {
363 if let Some(child) = self.child.lock().as_mut() {
364 match child.try_wait() {
365 Ok(None) => {
366 if let Err(error) = child.start_kill() {
367 tracing::error!(?error, "kill disconnected sidecar child");
368 }
369 }
370 Ok(Some(_)) => {}
371 Err(error) => tracing::error!(?error, "poll disconnected sidecar child"),
372 }
373 }
374 }
375
376 pub async fn terminate_child(&self) -> Result<(), TransportError> {
379 let _termination = self.child_termination.lock().await;
380 let Some(child) = self.child.lock().take() else {
381 return Ok(());
382 };
383 let mut owned = ReapingChild {
384 slot: &self.child,
385 child: Some(child),
386 };
387 let child = owned.child.as_mut().expect("owned reaping child");
388 if child
389 .try_wait()
390 .map_err(|error| TransportError::Sidecar(format!("poll sidecar child: {error}")))?
391 .is_none()
392 {
393 if let Err(kill_error) = child.start_kill() {
394 if child
395 .try_wait()
396 .map_err(|error| {
397 TransportError::Sidecar(format!("poll sidecar child after kill: {error}"))
398 })?
399 .is_none()
400 {
401 return Err(TransportError::Sidecar(format!(
402 "kill sidecar child: {kill_error}"
403 )));
404 }
405 }
406 }
407 child
408 .wait()
409 .await
410 .map_err(|error| TransportError::Sidecar(format!("reap sidecar child: {error}")))?;
411 owned.child.take();
412 Ok(())
413 }
414
415 fn encode_wire_frame(
416 &self,
417 frame: &wire::ProtocolFrame,
418 max_frame_bytes: Option<usize>,
419 ) -> Result<Vec<u8>, TransportError> {
420 let transport_limit = self.max_frame_bytes.load(Ordering::Relaxed);
421 let max_frame_bytes = max_frame_bytes
422 .map(|limit| limit.min(transport_limit))
423 .unwrap_or(transport_limit);
424 let codec = WireFrameCodec::new(max_frame_bytes);
425 Ok(codec.encode(frame)?)
426 }
427
428 async fn handle_wire_frame(self: &Arc<Self>, frame: wire::ProtocolFrame) {
431 match frame {
432 wire::ProtocolFrame::ResponseFrame(response) => {
433 match self.pending.remove(&response.request_id) {
434 Some((_, tx)) => {
435 let _ = tx.send(response.payload);
436 }
437 None => {
438 tracing::warn!(
439 request_id = response.request_id,
440 "response for unknown request id"
441 )
442 }
443 }
444 }
445 wire::ProtocolFrame::EventFrame(event) => {
446 if matches!(
451 &event.payload,
452 wire::EventPayload::StructuredEvent(structured)
453 if structured.name == "heartbeat"
454 ) {
455 return;
456 }
457 if let Some(event_tx) = self.event_tx.lock().as_ref() {
458 let _ = event_tx.send((event.ownership, event.payload));
459 }
460 }
461 wire::ProtocolFrame::SidecarRequestFrame(request) => {
462 self.dispatch_sidecar_request(request).await
463 }
464 wire::ProtocolFrame::SidecarResponseFrame(_)
465 | wire::ProtocolFrame::RequestFrame(_)
466 | wire::ProtocolFrame::ControlFrame(_) => {
467 tracing::warn!("unexpected inbound frame on host transport")
468 }
469 }
470 }
471
472 async fn dispatch_sidecar_request(self: &Arc<Self>, frame: wire::SidecarRequestFrame) {
477 let key = sidecar_request_key(&frame.payload);
478 let callback = self.callbacks.read(&key, |_, value| value.clone());
479 match callback {
480 Some(callback) => {
481 let transport = Arc::downgrade(self);
482 tokio::spawn(async move {
483 match callback(frame.payload, frame.ownership.clone()).await {
484 Ok(payload) => {
485 let response = wire::ProtocolFrame::SidecarResponseFrame(
486 wire::SidecarResponseFrame {
487 schema: wire::protocol_schema(),
488 request_id: frame.request_id,
489 ownership: frame.ownership,
490 payload,
491 },
492 );
493 let Some(transport) = transport.upgrade() else {
495 return;
496 };
497 if let Ok(bytes) = transport.encode_wire_frame(&response, None) {
498 let _ = transport.control_writer_tx.send(bytes).await;
499 }
500 }
501 Err(error) => tracing::warn!(?error, key, "sidecar callback failed"),
502 }
503 });
504 }
505 None => tracing::warn!(key, "no callback registered for sidecar request"),
506 }
507 }
508
509 fn fail_all_pending(&self) {
511 let _guard = self.pending_request_lock.lock();
514 self.pending.clear();
515 }
516
517 fn disconnect(&self) {
518 self.disconnected.store(true, Ordering::SeqCst);
519 self.kill_child();
520 self.fail_all_pending();
521 match tokio::runtime::Handle::try_current() {
524 Ok(runtime) => {
525 let event_tx = Arc::clone(&self.event_tx);
526 runtime.spawn(async move {
527 tokio::time::sleep(EVENT_DRAIN_GRACE).await;
528 event_tx.lock().take();
529 });
530 }
531 Err(_) => self.close_events(),
532 }
533 }
534
535 fn close_events(&self) {
537 self.event_tx.lock().take();
538 }
539
540 fn register_pending_request(
541 &self,
542 request_id: wire::RequestId,
543 tx: oneshot::Sender<wire::ResponsePayload>,
544 ) -> Result<(), TransportError> {
545 let _guard = self.pending_request_lock.lock();
546 if self.is_disconnected() {
547 return Err(TransportError::Disconnected(
548 "request sent after the sidecar process was lost".to_string(),
549 ));
550 }
551 if pending_request_count(self) >= PENDING_REQUEST_LIMIT {
552 return Err(TransportError::Sidecar(format!(
553 "sidecar pending request limit exceeded: at most {PENDING_REQUEST_LIMIT} requests can be in flight"
554 )));
555 }
556 let _ = self.pending.insert(request_id, tx);
557 Ok(())
558 }
559}
560
561#[cfg(unix)]
562fn map_control_fd(
563 command: &mut Command,
564 control_child: StdUnixStream,
565) -> Result<(), TransportError> {
566 command
567 .fd_mappings(vec![FdMapping {
568 parent_fd: control_child.into(),
569 child_fd: CONTROL_FD,
570 }])
571 .map_err(|error| {
572 TransportError::Sidecar(format!(
573 "failed to map sidecar response/control fd: {error}"
574 ))
575 })?;
576 Ok(())
577}
578
579struct PendingRequestGuard<'a> {
580 transport: &'a SidecarTransport,
581 request_id: wire::RequestId,
582}
583
584impl<'a> PendingRequestGuard<'a> {
585 fn new(transport: &'a SidecarTransport, request_id: wire::RequestId) -> Self {
586 Self {
587 transport,
588 request_id,
589 }
590 }
591}
592
593impl Drop for PendingRequestGuard<'_> {
594 fn drop(&mut self) {
595 let _ = self.transport.pending.remove(&self.request_id);
596 }
597}
598
599fn pending_request_count(transport: &SidecarTransport) -> usize {
600 let mut count = 0;
601 transport.pending.scan(|_, _| {
602 count += 1;
603 });
604 count
605}
606
607fn sidecar_request_key(payload: &wire::SidecarRequestPayload) -> &'static str {
609 match payload {
610 wire::SidecarRequestPayload::HostCallbackRequest(_) => "host_callback",
611 wire::SidecarRequestPayload::JsBridgeCallRequest(_) => "js_bridge_call",
612 wire::SidecarRequestPayload::ExtEnvelope(_) => "ext",
613 }
614}
615
616async fn run_writer<W>(
618 transport: Weak<SidecarTransport>,
619 lane: &'static str,
620 mut writer: W,
621 mut frames: mpsc::Receiver<Vec<u8>>,
622) where
623 W: AsyncWrite + Unpin,
624{
625 while let Some(bytes) = frames.recv().await {
626 let result = async {
627 writer.write_all(&bytes).await?;
628 writer.flush().await
629 }
630 .await;
631 if let Err(error) = result {
632 tracing::warn!(?error, lane, "sidecar writer failed");
633 if let Some(transport) = transport.upgrade() {
634 transport.disconnect();
635 }
636 return;
637 }
638 }
639}
640
641#[derive(Clone, Copy, Debug)]
642enum InboundLane {
643 Event,
644 Control,
645}
646
647impl InboundLane {
648 fn accepts(self, frame: &wire::ProtocolFrame) -> bool {
649 match self {
650 Self::Event => {
651 matches!(frame, wire::ProtocolFrame::EventFrame(event) if !is_heartbeat(event))
652 }
653 Self::Control => match frame {
654 wire::ProtocolFrame::ResponseFrame(_)
655 | wire::ProtocolFrame::SidecarRequestFrame(_) => true,
656 wire::ProtocolFrame::EventFrame(event) => is_heartbeat(event),
657 wire::ProtocolFrame::RequestFrame(_)
658 | wire::ProtocolFrame::SidecarResponseFrame(_)
659 | wire::ProtocolFrame::ControlFrame(_) => false,
660 },
661 }
662 }
663}
664
665fn is_heartbeat(event: &wire::EventFrame) -> bool {
666 matches!(
667 &event.payload,
668 wire::EventPayload::StructuredEvent(structured) if structured.name == "heartbeat"
669 )
670}
671
672async fn run_reader<R>(transport: Weak<SidecarTransport>, reader: R, lane: InboundLane)
674where
675 R: AsyncRead + Unpin,
676{
677 route_frames(transport.clone(), reader, lane).await;
678 if matches!(lane, InboundLane::Event) {
680 if let Some(transport) = transport.upgrade() {
681 transport.close_events();
682 }
683 }
684}
685
686async fn route_frames<R>(transport: Weak<SidecarTransport>, mut reader: R, lane: InboundLane)
687where
688 R: AsyncRead + Unpin,
689{
690 loop {
691 let mut length_buf = [0u8; 4];
692 if let Err(error) = reader.read_exact(&mut length_buf).await {
693 if let Some(transport) = transport.upgrade() {
694 tracing::warn!(?error, ?lane, "sidecar reader ended");
695 transport.disconnect();
696 }
697 return;
698 }
699 let length = u32::from_be_bytes(length_buf) as usize;
700
701 let Some(transport) = transport.upgrade() else {
702 break;
703 };
704 let max_frame_bytes = transport.max_frame_bytes.load(Ordering::Relaxed);
705 if frame_length_exceeds_limit(length, max_frame_bytes) {
706 tracing::warn!(
707 size = length,
708 max = max_frame_bytes,
709 "sidecar frame exceeds negotiated limit"
710 );
711 transport.disconnect();
712 return;
713 }
714
715 let mut frame_bytes = vec![0u8; 4 + length];
716 frame_bytes[..4].copy_from_slice(&length_buf);
717 if let Err(error) = reader.read_exact(&mut frame_bytes[4..]).await {
718 tracing::warn!(?error, ?lane, "sidecar reader ended mid-frame");
719 transport.disconnect();
720 return;
721 }
722 *transport.counted_silence.lock() = std::time::Duration::ZERO;
725
726 let codec = WireFrameCodec::new(max_frame_bytes);
727 match codec.decode(&frame_bytes) {
728 Ok(frame) if lane.accepts(&frame) => transport.handle_wire_frame(frame).await,
729 Ok(frame) => {
730 tracing::warn!(?lane, frame = ?frame, "sidecar frame arrived on wrong transport lane");
731 transport.disconnect();
732 return;
733 }
734 Err(error) => {
735 tracing::warn!(?error, ?lane, "failed to decode sidecar frame");
736 transport.disconnect();
737 return;
738 }
739 }
740 }
741}
742
743fn frame_length_exceeds_limit(length: usize, max_frame_bytes: usize) -> bool {
744 length > max_frame_bytes
745}
746
747async fn run_silence_watchdog(transport: Weak<SidecarTransport>, timeout: std::time::Duration) {
756 let check_interval = (timeout / 4).min(std::time::Duration::from_secs(1));
757 let mut last_tick_at = std::time::Instant::now();
758 loop {
759 tokio::time::sleep(check_interval).await;
760 let Some(transport) = transport.upgrade() else {
761 return;
762 };
763 let now = std::time::Instant::now();
764 let counted_tick = now.duration_since(last_tick_at).min(check_interval);
765 last_tick_at = now;
766 let silence = {
767 let mut counted_silence = transport.counted_silence.lock();
768 *counted_silence += counted_tick;
769 *counted_silence
770 };
771 if silence < timeout {
772 continue;
773 }
774 tracing::error!(
775 silence_ms = silence.as_millis() as u64,
776 "sidecar unresponsive: no protocol frames or heartbeats; killing sidecar",
777 );
778 transport.disconnect();
779 return;
780 }
781}
782
783fn resolve_sidecar_binary_path(binary_path: Option<String>) -> String {
784 binary_path
785 .or_else(|| std::env::var(SIDECAR_BIN_ENV).ok())
786 .unwrap_or_else(|| "agentos-native-sidecar".to_string())
787}
788
789struct ReapingChild<'a> {
793 slot: &'a parking_lot::Mutex<Option<Child>>,
794 child: Option<Child>,
795}
796
797impl Drop for ReapingChild<'_> {
798 fn drop(&mut self) {
799 if let Some(child) = self.child.take() {
800 *self.slot.lock() = Some(child);
801 }
802 }
803}
804
805#[cfg(test)]
806mod tests {
807 use super::*;
808 use std::sync::Mutex;
809 use std::time::Duration;
810
811 static ENV_LOCK: Mutex<()> = Mutex::new(());
812
813 #[tokio::test]
814 async fn cancelled_reaping_restores_child_ownership_for_confirmed_retry() {
815 let transport = test_transport();
816 let child = Command::new("sleep")
817 .arg("30")
818 .kill_on_drop(true)
819 .spawn()
820 .unwrap();
821 let pid = child.id().unwrap();
822 transport.child.lock().replace(child);
823 let child = transport.child.lock().take();
824 let reaping = ReapingChild {
825 slot: &transport.child,
826 child,
827 };
828 assert!(transport.child.lock().is_none());
829 drop(reaping);
832 assert_eq!(transport.child.lock().as_ref().unwrap().id(), Some(pid));
833 tokio::time::timeout(Duration::from_secs(2), transport.terminate_child())
834 .await
835 .unwrap()
836 .unwrap();
837 assert!(transport.child.lock().is_none());
838 transport.terminate_child().await.unwrap();
839 }
840
841 #[tokio::test]
842 async fn disconnected_child_is_retained_until_reaped() {
843 let transport = test_transport();
844 let child = Command::new("sleep")
845 .arg("30")
846 .kill_on_drop(true)
847 .spawn()
848 .unwrap();
849 transport.child.lock().replace(child);
850 transport.kill_child();
851 assert!(transport.child.lock().is_some());
852 tokio::time::timeout(Duration::from_secs(2), transport.terminate_child())
853 .await
854 .unwrap()
855 .unwrap();
856 assert!(transport.child.lock().is_none());
857 }
858
859 fn test_transport() -> SidecarTransport {
860 let (request_writer_tx, _request_writer_rx) = mpsc::channel(REQUEST_FRAME_QUEUE_CAPACITY);
861 let (control_writer_tx, _control_writer_rx) = mpsc::channel(CONTROL_FRAME_QUEUE_CAPACITY);
862 let (event_tx, _) = broadcast::channel(EVENT_CHANNEL_CAPACITY);
863 SidecarTransport {
864 instance_id: next_transport_instance_id(),
865 child: parking_lot::Mutex::new(None),
866 child_termination: tokio::sync::Mutex::new(()),
867 pending: SccHashMap::new(),
868 pending_request_lock: parking_lot::Mutex::new(()),
869 request_counter: AtomicI64::new(1),
870 max_frame_bytes: AtomicUsize::new(wire::DEFAULT_MAX_FRAME_BYTES),
871 max_write_files_entries: AtomicUsize::new(1),
872 event_tx: Arc::new(parking_lot::Mutex::new(Some(event_tx))),
873 callbacks: SccHashMap::new(),
874 request_writer_tx,
875 control_writer_tx,
876 counted_silence: parking_lot::Mutex::new(std::time::Duration::ZERO),
877 disconnected: AtomicBool::new(false),
878 }
879 }
880
881 #[test]
882 fn binary_path_prefers_explicit_path_over_env() {
883 let _guard = ENV_LOCK.lock().expect("env lock");
884 let previous = std::env::var(SIDECAR_BIN_ENV).ok();
885 std::env::set_var(SIDECAR_BIN_ENV, "/tmp/from-env");
886
887 assert_eq!(
888 resolve_sidecar_binary_path(Some("/tmp/from-config".to_string())),
889 "/tmp/from-config"
890 );
891
892 restore_env(SIDECAR_BIN_ENV, previous);
893 }
894
895 #[test]
896 fn binary_path_uses_agentos_env_fallback() {
897 let _guard = ENV_LOCK.lock().expect("env lock");
898 let previous = std::env::var(SIDECAR_BIN_ENV).ok();
899 std::env::set_var(SIDECAR_BIN_ENV, "/tmp/agentos-native-sidecar");
900
901 assert_eq!(
902 resolve_sidecar_binary_path(None),
903 "/tmp/agentos-native-sidecar"
904 );
905
906 restore_env(SIDECAR_BIN_ENV, previous);
907 }
908
909 #[test]
910 fn binary_path_defaults_to_agentos_native_sidecar() {
911 let _guard = ENV_LOCK.lock().expect("env lock");
912 let previous = std::env::var(SIDECAR_BIN_ENV).ok();
913 std::env::remove_var(SIDECAR_BIN_ENV);
914
915 assert_eq!(resolve_sidecar_binary_path(None), "agentos-native-sidecar");
916
917 restore_env(SIDECAR_BIN_ENV, previous);
918 }
919
920 fn restore_env(key: &str, value: Option<String>) {
921 match value {
922 Some(value) => std::env::set_var(key, value),
923 None => std::env::remove_var(key),
924 }
925 }
926
927 #[test]
928 fn frame_length_limit_rejects_oversized_declared_length() {
929 assert!(!frame_length_exceeds_limit(1024, 1024));
930 assert!(frame_length_exceeds_limit(1025, 1024));
931 }
932
933 #[test]
934 fn transport_encodes_requests_with_generated_wire_codec() {
935 let transport = test_transport();
936 let frame = wire::ProtocolFrame::RequestFrame(wire::RequestFrame {
937 schema: wire::protocol_schema(),
938 request_id: 7,
939 ownership: wire::OwnershipScope::ConnectionOwnership(wire::ConnectionOwnership {
940 connection_id: "conn-1".to_string(),
941 }),
942 payload: wire::RequestPayload::AuthenticateRequest(wire::AuthenticateRequest {
943 client_name: "transport-test".to_string(),
944 auth_token: "token".to_string(),
945 protocol_version: wire::PROTOCOL_VERSION,
946 bridge_version: 1,
947 }),
948 });
949
950 let encoded = transport
951 .encode_wire_frame(&frame, None)
952 .expect("encode transport frame");
953 let decoded = WireFrameCodec::default()
954 .decode(&encoded)
955 .expect("decode generated wire frame");
956
957 assert!(matches!(
958 decoded,
959 wire::ProtocolFrame::RequestFrame(wire::RequestFrame {
960 payload: wire::RequestPayload::AuthenticateRequest(_),
961 ..
962 })
963 ));
964 }
965
966 #[tokio::test]
967 async fn transport_fans_out_generated_wire_events() {
968 let transport = Arc::new(test_transport());
969 let mut wire_events = transport.subscribe_wire_events();
970
971 transport
972 .handle_wire_frame(wire::ProtocolFrame::EventFrame(wire::EventFrame {
973 schema: wire::protocol_schema(),
974 ownership: wire::OwnershipScope::VmOwnership(wire::VmOwnership {
975 connection_id: "conn-1".to_string(),
976 session_id: "session-1".to_string(),
977 vm_id: "vm-1".to_string(),
978 }),
979 payload: wire::EventPayload::ProcessOutputEvent(wire::ProcessOutputEvent {
980 process_id: "proc-1".to_string(),
981 channel: wire::StreamChannel::Stdout,
982 chunk: b"hello".to_vec(),
983 sequence: None,
984 timestamp_ms: None,
985 }),
986 }))
987 .await;
988
989 let (ownership, payload) = wire_events.recv().await.expect("wire event");
990 assert!(matches!(
991 ownership,
992 wire::OwnershipScope::VmOwnership(wire::VmOwnership {
993 connection_id,
994 session_id,
995 vm_id,
996 }) if connection_id == "conn-1" && session_id == "session-1" && vm_id == "vm-1"
997 ));
998 assert!(matches!(
999 payload,
1000 wire::EventPayload::ProcessOutputEvent(wire::ProcessOutputEvent {
1001 process_id,
1002 channel: wire::StreamChannel::Stdout,
1003 chunk,
1004 sequence: None,
1005 timestamp_ms: None,
1006 }) if process_id == "proc-1" && chunk == b"hello".to_vec()
1007 ));
1008 }
1009
1010 #[tokio::test]
1011 async fn event_before_response_is_delivered_without_transport_history() {
1012 let transport = Arc::new(test_transport());
1013 let mut events = transport.subscribe_wire_events();
1014 let (response_tx, response_rx) = oneshot::channel();
1015 transport
1016 .register_pending_request(7, response_tx)
1017 .expect("register pending request");
1018 let ownership = wire::OwnershipScope::ConnectionOwnership(wire::ConnectionOwnership {
1019 connection_id: "conn-1".to_string(),
1020 });
1021
1022 transport
1023 .handle_wire_frame(wire::ProtocolFrame::EventFrame(wire::EventFrame {
1024 schema: wire::protocol_schema(),
1025 ownership: ownership.clone(),
1026 payload: wire::EventPayload::StructuredEvent(wire::StructuredEvent {
1027 name: "process_started_before_reply".to_string(),
1028 detail: std::collections::HashMap::new(),
1029 }),
1030 }))
1031 .await;
1032 transport
1033 .handle_wire_frame(wire::ProtocolFrame::ResponseFrame(wire::ResponseFrame {
1034 schema: wire::protocol_schema(),
1035 request_id: 7,
1036 ownership,
1037 payload: wire::ResponsePayload::ExtEnvelope(wire::ExtEnvelope {
1038 namespace: "test".to_string(),
1039 payload: Vec::new(),
1040 }),
1041 }))
1042 .await;
1043
1044 let (_, event) = events.recv().await.expect("live event");
1045 assert!(matches!(
1046 event,
1047 wire::EventPayload::StructuredEvent(wire::StructuredEvent { name, .. })
1048 if name == "process_started_before_reply"
1049 ));
1050 assert!(matches!(
1051 response_rx.await.expect("response"),
1052 wire::ResponsePayload::ExtEnvelope(_)
1053 ));
1054 }
1055
1056 #[tokio::test]
1057 async fn bounded_event_fanout_reports_lag_instead_of_retaining_history() {
1058 let (event_tx, mut events) = broadcast::channel(2);
1059 for index in 0..3 {
1060 event_tx
1061 .send((
1062 wire::OwnershipScope::ConnectionOwnership(wire::ConnectionOwnership {
1063 connection_id: "conn-1".to_string(),
1064 }),
1065 wire::EventPayload::StructuredEvent(wire::StructuredEvent {
1066 name: format!("event-{index}"),
1067 detail: std::collections::HashMap::new(),
1068 }),
1069 ))
1070 .expect("active receiver");
1071 }
1072
1073 assert!(matches!(
1074 events.recv().await,
1075 Err(broadcast::error::RecvError::Lagged(1))
1076 ));
1077 }
1078
1079 #[tokio::test]
1080 async fn silence_watchdog_fails_pending_requests_after_sustained_silence() {
1081 let transport = Arc::new(test_transport());
1082 let (tx, rx) = oneshot::channel();
1083 transport
1084 .register_pending_request(1, tx)
1085 .expect("register pending request");
1086
1087 tokio::spawn(run_silence_watchdog(
1088 Arc::downgrade(&transport),
1089 std::time::Duration::from_millis(40),
1090 ));
1091
1092 rx.await
1095 .expect_err("watchdog should drop the pending sender");
1096 assert_eq!(pending_request_count(&transport), 0);
1097 }
1098
1099 #[tokio::test]
1100 async fn silence_watchdog_stays_quiet_while_frames_arrive() {
1101 let transport = Arc::new(test_transport());
1102 let (tx, mut rx) = oneshot::channel();
1103 transport
1104 .register_pending_request(1, tx)
1105 .expect("register pending request");
1106
1107 tokio::spawn(run_silence_watchdog(
1108 Arc::downgrade(&transport),
1109 std::time::Duration::from_millis(120),
1110 ));
1111
1112 for _ in 0..6 {
1115 tokio::time::sleep(std::time::Duration::from_millis(40)).await;
1116 *transport.counted_silence.lock() = std::time::Duration::ZERO;
1117 assert!(
1118 rx.try_recv().is_err(),
1119 "pending request must remain registered while frames arrive"
1120 );
1121 }
1122 assert_eq!(pending_request_count(&transport), 1);
1123 }
1124
1125 #[tokio::test]
1126 async fn silence_watchdog_ignores_a_stalled_host_but_still_detects_silence() {
1127 let transport = Arc::new(test_transport());
1128 let (tx, mut rx) = oneshot::channel();
1129 transport
1130 .register_pending_request(1, tx)
1131 .expect("register pending request");
1132
1133 let timeout = std::time::Duration::from_millis(200);
1134 tokio::spawn(run_silence_watchdog(Arc::downgrade(&transport), timeout));
1135 tokio::task::yield_now().await;
1136
1137 std::thread::sleep(timeout * 3);
1140 tokio::time::sleep(std::time::Duration::from_millis(60)).await;
1141 assert!(
1142 matches!(rx.try_recv(), Err(oneshot::error::TryRecvError::Empty)),
1143 "a host stall must not count as sidecar silence"
1144 );
1145
1146 for _ in 0..100 {
1149 std::thread::sleep(std::time::Duration::from_millis(80));
1150 tokio::time::sleep(std::time::Duration::from_millis(1)).await;
1151 if matches!(rx.try_recv(), Err(oneshot::error::TryRecvError::Closed)) {
1152 assert_eq!(pending_request_count(&transport), 0);
1153 return;
1154 }
1155 }
1156 panic!("a silent sidecar must be detected while the host stalls in bursts");
1157 }
1158
1159 #[tokio::test]
1160 async fn heartbeat_events_are_swallowed_before_the_event_fanout() {
1161 let transport = Arc::new(test_transport());
1162 let mut wire_events = transport.subscribe_wire_events();
1163
1164 transport
1165 .handle_wire_frame(wire::ProtocolFrame::EventFrame(wire::EventFrame {
1166 schema: wire::protocol_schema(),
1167 ownership: wire::OwnershipScope::ConnectionOwnership(wire::ConnectionOwnership {
1168 connection_id: "sidecar-transport".to_string(),
1169 }),
1170 payload: wire::EventPayload::StructuredEvent(wire::StructuredEvent {
1171 name: "heartbeat".to_string(),
1172 detail: std::collections::HashMap::new(),
1173 }),
1174 }))
1175 .await;
1176 transport
1179 .handle_wire_frame(wire::ProtocolFrame::EventFrame(wire::EventFrame {
1180 schema: wire::protocol_schema(),
1181 ownership: wire::OwnershipScope::ConnectionOwnership(wire::ConnectionOwnership {
1182 connection_id: "conn-1".to_string(),
1183 }),
1184 payload: wire::EventPayload::StructuredEvent(wire::StructuredEvent {
1185 name: "limit_warning".to_string(),
1186 detail: std::collections::HashMap::new(),
1187 }),
1188 }))
1189 .await;
1190
1191 let (_, payload) = wire_events.recv().await.expect("structured event");
1192 assert!(matches!(
1193 payload,
1194 wire::EventPayload::StructuredEvent(wire::StructuredEvent { name, .. })
1195 if name == "limit_warning"
1196 ));
1197 assert!(
1198 wire_events.try_recv().is_err(),
1199 "heartbeat must not fan out"
1200 );
1201 }
1202
1203 #[test]
1204 fn pending_request_guard_removes_registered_slot_on_drop() {
1205 let transport = test_transport();
1206 let (tx, _rx) = oneshot::channel();
1207 transport
1208 .register_pending_request(1, tx)
1209 .expect("register pending request");
1210
1211 {
1212 let _guard = PendingRequestGuard::new(&transport, 1);
1213 assert_eq!(pending_request_count(&transport), 1);
1214 }
1215
1216 assert_eq!(pending_request_count(&transport), 0);
1217 }
1218
1219 #[test]
1220 fn pending_request_limit_rejects_full_transport() {
1221 let transport = test_transport();
1222 for request_id in 1..=PENDING_REQUEST_LIMIT as wire::RequestId {
1223 let (tx, _rx) = oneshot::channel();
1224 transport
1225 .register_pending_request(request_id, tx)
1226 .expect("register pending request");
1227 }
1228 let (tx, _rx) = oneshot::channel();
1229 let error = transport
1230 .register_pending_request((PENDING_REQUEST_LIMIT + 1) as wire::RequestId, tx)
1231 .expect_err("full pending map should reject");
1232
1233 assert!(
1234 error
1235 .to_string()
1236 .contains("sidecar pending request limit exceeded"),
1237 "unexpected error: {error}"
1238 );
1239 }
1240
1241 #[tokio::test]
1242 async fn shutdown_uses_typed_control_frame() {
1243 let (request_writer_tx, _request_writer_rx) = mpsc::channel(4);
1244 let (control_writer_tx, mut control_writer_rx) = mpsc::channel(4);
1245 let (event_tx, _) = broadcast::channel(4);
1246 let transport = SidecarTransport {
1247 instance_id: next_transport_instance_id(),
1248 child: parking_lot::Mutex::new(None),
1249 child_termination: tokio::sync::Mutex::new(()),
1250 pending: SccHashMap::new(),
1251 pending_request_lock: parking_lot::Mutex::new(()),
1252 request_counter: AtomicI64::new(1),
1253 max_frame_bytes: AtomicUsize::new(wire::DEFAULT_MAX_FRAME_BYTES),
1254 max_write_files_entries: AtomicUsize::new(1),
1255 event_tx: Arc::new(parking_lot::Mutex::new(Some(event_tx))),
1256 callbacks: SccHashMap::new(),
1257 request_writer_tx,
1258 control_writer_tx,
1259 counted_silence: parking_lot::Mutex::new(std::time::Duration::ZERO),
1260 disconnected: AtomicBool::new(false),
1261 };
1262
1263 transport
1264 .shutdown("test complete")
1265 .await
1266 .expect("enqueue typed shutdown");
1267 let bytes = control_writer_rx.recv().await.expect("control frame bytes");
1268 let frame = WireFrameCodec::default()
1269 .decode(&bytes)
1270 .expect("decode shutdown frame");
1271 assert!(matches!(
1272 frame,
1273 wire::ProtocolFrame::ControlFrame(wire::ControlFrame {
1274 payload: wire::ControlPayload::ShutdownControl(wire::ShutdownControl { reason }),
1275 ..
1276 }) if reason == "test complete"
1277 ));
1278 }
1279
1280 #[cfg(unix)]
1281 #[tokio::test]
1282 async fn unix_spawn_maps_duplex_control_socket_to_fd_three() {
1283 let (mut parent, child) = StdUnixStream::pair().expect("control socketpair");
1284 parent
1285 .set_read_timeout(Some(std::time::Duration::from_secs(2)))
1286 .expect("control read timeout");
1287 let mut command = Command::new("sh");
1288 command.arg("-c").arg("printf mapped-control >&3");
1289 map_control_fd(&mut command, child).expect("map child control fd");
1290 let mut child = command.spawn().expect("spawn fd mapping probe");
1291 drop(command);
1292
1293 let mut received = [0_u8; 14];
1294 std::io::Read::read_exact(&mut parent, &mut received).expect("read mapped fd output");
1295 assert_eq!(&received, b"mapped-control");
1296 assert!(child
1297 .wait()
1298 .await
1299 .expect("wait for mapping probe")
1300 .success());
1301 }
1302
1303 #[tokio::test]
1304 async fn control_writer_progresses_while_ordinary_stream_is_blocked() {
1305 let transport = Arc::new(test_transport());
1306 let (ordinary_client, _ordinary_server) = tokio::io::duplex(1);
1307 let (control_client, mut control_server) = tokio::io::duplex(64);
1308 let (control_tx, control_rx) = mpsc::channel(CONTROL_FRAME_QUEUE_CAPACITY);
1309 let (request_tx, request_rx) = mpsc::channel(REQUEST_FRAME_QUEUE_CAPACITY);
1310 request_tx
1311 .send(vec![b'r'; 64])
1312 .await
1313 .expect("send request frame");
1314 control_tx
1315 .send(vec![b'c'])
1316 .await
1317 .expect("send control frame");
1318 let ordinary_writer = tokio::spawn(run_writer(
1319 Arc::downgrade(&transport),
1320 "ordinary",
1321 ordinary_client,
1322 request_rx,
1323 ));
1324 let control_writer = tokio::spawn(run_writer(
1325 Arc::downgrade(&transport),
1326 "control",
1327 control_client,
1328 control_rx,
1329 ));
1330 let mut first = [0u8; 1];
1331 tokio::time::timeout(
1332 std::time::Duration::from_secs(1),
1333 control_server.read_exact(&mut first),
1334 )
1335 .await
1336 .expect("control write must not wait for ordinary stream")
1337 .expect("read first byte");
1338 assert_eq!(first, [b'c']);
1339 ordinary_writer.abort();
1340 control_writer.abort();
1341 }
1342
1343 #[tokio::test]
1344 async fn eof_on_either_inbound_lane_fails_pending_requests() {
1345 for lane in [InboundLane::Event, InboundLane::Control] {
1346 let transport = Arc::new(test_transport());
1347 let (tx, rx) = oneshot::channel();
1348 transport
1349 .register_pending_request(1, tx)
1350 .expect("register pending request");
1351 let (reader, peer) = tokio::io::duplex(64);
1352 drop(peer);
1353
1354 tokio::time::timeout(
1355 std::time::Duration::from_secs(1),
1356 run_reader(Arc::downgrade(&transport), reader, lane),
1357 )
1358 .await
1359 .expect("reader must terminate on EOF");
1360
1361 rx.await.expect_err("EOF must fail pending requests");
1362 assert_eq!(pending_request_count(&transport), 0);
1363 }
1364 }
1365
1366 #[tokio::test]
1370 async fn events_written_before_a_disconnect_still_reach_subscribers() {
1371 let transport = Arc::new(test_transport());
1372 let mut events = transport.subscribe_wire_events();
1373
1374 let (control_reader, control_peer) = tokio::io::duplex(64);
1375 drop(control_peer);
1376 run_reader(
1377 Arc::downgrade(&transport),
1378 control_reader,
1379 InboundLane::Control,
1380 )
1381 .await;
1382 assert!(transport.is_disconnected());
1383
1384 let (event_reader, mut event_peer) = tokio::io::duplex(4096);
1385 let frame = WireFrameCodec::default()
1386 .encode(&wire::ProtocolFrame::EventFrame(wire::EventFrame {
1387 schema: wire::protocol_schema(),
1388 ownership: wire::OwnershipScope::ConnectionOwnership(wire::ConnectionOwnership {
1389 connection_id: "conn-1".to_string(),
1390 }),
1391 payload: wire::EventPayload::StructuredEvent(wire::StructuredEvent {
1392 name: "last_words".to_string(),
1393 detail: std::collections::HashMap::new(),
1394 }),
1395 }))
1396 .expect("encode event");
1397 event_peer.write_all(&frame).await.expect("write event");
1398 drop(event_peer);
1399 tokio::time::timeout(
1400 Duration::from_secs(1),
1401 run_reader(Arc::downgrade(&transport), event_reader, InboundLane::Event),
1402 )
1403 .await
1404 .expect("event reader must end at EOF");
1405
1406 let (_, payload) = events
1407 .recv()
1408 .await
1409 .expect("the buffered event is delivered");
1410 assert!(matches!(
1411 payload,
1412 wire::EventPayload::StructuredEvent(wire::StructuredEvent { name, .. }) if name == "last_words"
1413 ));
1414 assert!(matches!(
1415 events.recv().await,
1416 Err(broadcast::error::RecvError::Closed)
1417 ));
1418 }
1419
1420 #[tokio::test]
1421 async fn writer_preserves_order_within_one_lane() {
1422 let transport = Arc::new(test_transport());
1423 let (client, mut server) = tokio::io::duplex(64);
1424 let (tx, rx) = mpsc::channel(CONTROL_FRAME_QUEUE_CAPACITY);
1425 tx.send(vec![b'c']).await.expect("control one");
1426 tx.send(vec![b'C']).await.expect("control two");
1427 drop(tx);
1428
1429 let writer = tokio::spawn(run_writer(
1430 Arc::downgrade(&transport),
1431 "control",
1432 client,
1433 rx,
1434 ));
1435 let mut output = [0u8; 2];
1436 server.read_exact(&mut output).await.expect("read output");
1437 writer.await.expect("writer task");
1438
1439 assert_eq!(output, [b'c', b'C']);
1440 }
1441}