Skip to main content

agentos_client/
shell.rs

1//! Network (fetch) and Shell / terminal methods + supporting types.
2//!
3//! Ported from `packages/core/src/agent-os.ts` (`fetch` + shell methods) and `runtime-compat.ts`
4//! (`ShellHandle`, `OpenShellOptions`, `ConnectTerminalOptions`).
5//!
6//! Id-vs-PID is load-bearing: `open_shell` returns a synthetic `shell-N` id; `connect_terminal`
7//! returns a PID and is NOT tracked in the shells map.
8//!
9//! The native wire protocol has no PTY/winsize request, so a shell is modeled as a guest process
10//! spawned via [`ExecuteRequest`]: its `process_id` is what `write_shell`/`close_shell` address on
11//! the wire, while the public boundary keeps the synthetic `shell-N` id.
12//!
13//! Stream routing mirrors the TS PTY path: the public `data` stream (`on_shell_data`) carries stdout
14//! and stderr in the order received from the sidecar. stderr is also delivered on an optional
15//! channel-specific diagnostic tap (`on_shell_stderr` + [`OpenShellOptions::on_stderr`]); terminal
16//! renderers consume only `data` so prompts and control sequences are neither reordered nor doubled.
17
18use std::collections::{BTreeMap, VecDeque};
19use std::sync::atomic::{AtomicUsize, Ordering};
20
21use anyhow::Result;
22use tokio::sync::{broadcast, watch};
23use uuid::Uuid;
24
25use agentos_sidecar_client::wire::{self, EventPayload, StreamChannel};
26
27use crate::agent_os::{AgentOs, ClosedShellEntry, ShellEntry, TerminalEntry};
28use crate::error::ClientError;
29use crate::process::{
30    install_output_callback, OutputCallback, ProcessStatus, ProcessStream, StdinInput,
31};
32
33/// Channel capacity for a shell's ordered terminal-data and diagnostic-stderr broadcasts.
34const SHELL_DATA_CHANNEL_CAPACITY: usize = 1024;
35
36/// Maximum active or spawning shells and terminals per VM.
37const TERMINAL_LIMIT: usize = 1024;
38
39/// Default shell command used when [`OpenShellOptions::command`] is omitted (matches the kernel's
40/// PTY-backed `sh`).
41const DEFAULT_SHELL_COMMAND: &str = "sh";
42
43type ShellOutcome = Option<std::result::Result<i32, ClientError>>;
44type ShellSpawnReadiness = watch::Receiver<Option<std::result::Result<(), ClientError>>>;
45
46fn publish_shell_outcome(
47    sender: &watch::Sender<ShellOutcome>,
48    result: std::result::Result<i32, ClientError>,
49) {
50    sender.send_if_modified(|outcome| {
51        // Neither a late observation failure nor repeated replay may replace a
52        // confirmed exit or emit another completion notification.
53        if matches!(outcome, Some(Ok(_))) {
54            return false;
55        }
56        *outcome = Some(result);
57        true
58    });
59}
60
61fn retain_shell_result(retained: &mut VecDeque<ClosedShellEntry>, entry: ClosedShellEntry) {
62    if let Some(existing) = retained.iter_mut().find(|existing| {
63        existing.shell_id == entry.shell_id && existing.process_id == entry.process_id
64    }) {
65        if existing.result.is_err() {
66            *existing = entry;
67        }
68        return;
69    }
70    retained.push_back(entry);
71    while retained.len() > crate::CLOSED_SHELL_EXIT_CODE_RETENTION_LIMIT {
72        retained.pop_front();
73    }
74}
75
76async fn next_shell_event(
77    events: &mut broadcast::Receiver<(wire::OwnershipScope, EventPayload)>,
78    ownership: &wire::OwnershipScope,
79    outcome: &watch::Sender<ShellOutcome>,
80) -> Option<std::result::Result<EventPayload, broadcast::error::RecvError>> {
81    let mut replay_outcome = outcome.subscribe();
82    loop {
83        if matches!(*replay_outcome.borrow(), Some(Ok(_))) {
84            return None;
85        }
86        tokio::select! {
87            event = events.recv() => match event {
88                Ok((scope, payload)) if scope == *ownership => return Some(Ok(payload)),
89                Ok(_) => continue,
90                Err(error) => return Some(Err(error)),
91            },
92            changed = replay_outcome.changed() => {
93                if let Err(error) = changed {
94                    tracing::error!(%error, "terminal outcome channel closed during observation");
95                    return None;
96                }
97            }
98        }
99    }
100}
101
102async fn observe_shell_exit(mut receiver: watch::Receiver<ShellOutcome>) -> ShellOutcome {
103    loop {
104        if let Some(result) = receiver.borrow_and_update().clone() {
105            return Some(result);
106        }
107        if receiver.changed().await.is_err() {
108            return None;
109        }
110    }
111}
112
113// ---------------------------------------------------------------------------
114// Supporting types
115// ---------------------------------------------------------------------------
116
117/// Callback-free options for portable `open_shell`.
118#[derive(Default)]
119pub struct OpenShellOptions {
120    pub command: Option<String>,
121    pub args: Vec<String>,
122    pub env: BTreeMap<String, String>,
123    pub cwd: Option<String>,
124    pub cols: Option<u16>,
125    pub rows: Option<u16>,
126}
127
128/// Options for `connect_terminal` (extends [`OpenShellOptions`]).
129///
130/// `on_data` mirrors the TS `ConnectTerminalOptions.onData` raw-byte callback. When omitted, TS pipes
131/// shell output to host stdout; the Rust port routes it through the shell's data subscription and
132/// requires the caller to provide the sink because there is no host-process stdio to bind to.
133#[derive(Default)]
134pub struct ConnectTerminalOptions {
135    pub base: OpenShellOptions,
136    pub on_data: Option<OutputCallback>,
137    pub on_stderr: Option<OutputCallback>,
138}
139
140/// The synthetic shell id returned by `open_shell` (`shell-N`, NOT a pid).
141#[derive(Debug, Clone, PartialEq, Eq)]
142pub struct ShellHandle {
143    pub shell_id: String,
144}
145
146#[derive(Debug, Clone, PartialEq, Eq)]
147pub struct ShellData {
148    pub shell_id: String,
149    pub data: Vec<u8>,
150}
151
152#[derive(Debug, Clone, PartialEq, Eq)]
153pub struct ShellExit {
154    pub shell_id: String,
155    pub exit_code: i32,
156}
157
158#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
159#[serde(rename_all = "camelCase")]
160pub struct TerminalInfo {
161    pub shell_id: String,
162    pub pid: u32,
163    pub running: bool,
164    pub exit_code: Option<i32>,
165}
166
167#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
168#[serde(rename_all = "camelCase")]
169pub struct TerminalOutputEvent {
170    pub sequence: u64,
171    pub stream: ProcessStream,
172    pub data: Vec<u8>,
173    pub timestamp_ms: i64,
174}
175
176#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
177#[serde(rename_all = "camelCase")]
178pub struct TerminalSnapshot {
179    pub shell_id: String,
180    pub pid: u32,
181    pub events: Vec<TerminalOutputEvent>,
182    pub next_cursor: Option<u64>,
183    pub has_more: bool,
184    pub truncated: bool,
185    pub exit_code: Option<i32>,
186}
187
188// ---------------------------------------------------------------------------
189// Helpers
190// ---------------------------------------------------------------------------
191
192/// Map a [`RejectedResponse`] into a [`ClientError::Kernel`] so the errno `code` survives.
193fn rejected_to_error(rejected: wire::RejectedResponse) -> ClientError {
194    ClientError::from_rejection(rejected)
195}
196
197fn shell_started(response: wire::ResponsePayload) -> std::result::Result<u32, ClientError> {
198    match response {
199        wire::ResponsePayload::ProcessStartedResponse(wire::ProcessStartedResponse {
200            pid: Some(pid),
201            ..
202        }) => Ok(pid),
203        wire::ResponsePayload::RejectedResponse(rejected) => Err(rejected_to_error(rejected)),
204        other => Err(ClientError::Sidecar(format!(
205            "open_shell: expected a started process with a PID, received {other:?}"
206        ))),
207    }
208}
209
210struct ShellReservation(AgentOs);
211
212impl Drop for ShellReservation {
213    fn drop(&mut self) {
214        release_counter(&self.0.inner().terminal_count);
215    }
216}
217
218/// Encode a [`StdinInput`] into the wire `chunk` bytes. The wire `chunk` field is bare `data`
219/// (`Vec<u8>`), so raw Binary stdin is carried verbatim (no lossy UTF-8 conversion), matching the
220/// byte-exact TS `proc.writeStdin` contract.
221fn stdin_chunk(data: StdinInput) -> Vec<u8> {
222    match data {
223        StdinInput::Text(text) => text.into_bytes(),
224        StdinInput::Bytes(bytes) => bytes,
225    }
226}
227
228fn try_reserve_counter(counter: &AtomicUsize, limit: usize) -> bool {
229    counter
230        .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |count| {
231            (count < limit).then_some(count + 1)
232        })
233        .is_ok()
234}
235
236fn release_counter(counter: &AtomicUsize) {
237    let _ = counter.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |count| {
238        Some(count.saturating_sub(1))
239    });
240}
241
242struct TerminalReservation<'a> {
243    agent: &'a AgentOs,
244    active: bool,
245}
246
247impl<'a> TerminalReservation<'a> {
248    fn new(agent: &'a AgentOs) -> std::result::Result<Self, ClientError> {
249        if !try_reserve_counter(&agent.inner().terminal_count, TERMINAL_LIMIT) {
250            return Err(ClientError::ResourceLimit {
251                code: "ERR_AGENTOS_RESOURCE_LIMIT".into(),
252                message: format!("terminal limit {TERMINAL_LIMIT} reached; wait for existing terminals to exit or raise TERMINAL_LIMIT"),
253                details: Box::new(crate::ResourceLimitDetails {
254                    limit_name: Some("active_terminals".into()),
255                    configured_limit: Some(TERMINAL_LIMIT as u64),
256                    requested: Some(TERMINAL_LIMIT as u64 + 1),
257                    configuration_path: Some("TERMINAL_LIMIT".into()),
258                    unit: Some("terminals".into()),
259                    scope: Some("vm".into()),
260                    retryable: Some(true),
261                    ..Default::default()
262                }),
263            });
264        }
265        let active = agent.inner().terminal_count.load(Ordering::SeqCst);
266        if active >= TERMINAL_LIMIT * 4 / 5 {
267            tracing::warn!(
268                active,
269                limit = TERMINAL_LIMIT,
270                "terminal admission approaches TERMINAL_LIMIT; close unused terminals"
271            );
272        }
273        Ok(Self {
274            agent,
275            active: true,
276        })
277    }
278
279    fn disarm(&mut self) {
280        self.active = false;
281    }
282}
283
284impl Drop for TerminalReservation<'_> {
285    fn drop(&mut self) {
286        if self.active {
287            release_counter(&self.agent.inner().terminal_count);
288        }
289    }
290}
291
292impl AgentOs {
293    /// The VM-scoped ownership scope used for every shell/fetch wire request.
294    fn vm_ownership(&self) -> wire::OwnershipScope {
295        wire::OwnershipScope::VmOwnership(wire::VmOwnership {
296            connection_id: self.connection_id().to_string(),
297            session_id: self.wire_session_id().to_string(),
298            vm_id: self.vm_id().to_string(),
299        })
300    }
301
302    pub(crate) fn finish_terminal(&self, process_id: &str) {
303        if self.inner().terminals.remove(process_id).is_some() {
304            release_counter(&self.inner().terminal_count);
305        }
306    }
307
308    async fn start_terminal(
309        &self,
310        execute: wire::ExecuteRequest,
311        ownership: wire::OwnershipScope,
312        pid_tx: tokio::sync::oneshot::Sender<std::result::Result<u32, ClientError>>,
313        process_id: &str,
314    ) -> Option<u32> {
315        {
316            let _terminal_lifecycle_guard = self.inner().terminal_lifecycle_lock.lock().await;
317            if self.inner().disposed.load(Ordering::SeqCst) {
318                let error = ClientError::Sidecar(
319                    "cannot connect terminal after VM shutdown has started".to_string(),
320                );
321                let _ = pid_tx.send(Err(error));
322                self.finish_terminal(process_id);
323                return None;
324            }
325        }
326
327        let result = match self
328            .transport()
329            .request_wire(ownership, wire::RequestPayload::ExecuteRequest(execute))
330            .await
331        {
332            Ok(wire::ResponsePayload::ProcessStartedResponse(wire::ProcessStartedResponse {
333                pid,
334                ..
335            })) => pid.ok_or_else(|| {
336                ClientError::Sidecar("connect_terminal: sidecar did not return a pid".to_string())
337            }),
338            Ok(wire::ResponsePayload::RejectedResponse(rejected)) => {
339                Err(rejected_to_error(rejected))
340            }
341            Ok(other) => Err(ClientError::Sidecar(format!(
342                "unexpected response to connect_terminal: {other:?}"
343            ))),
344            Err(error) => Err(error.into()),
345        };
346
347        match result {
348            Ok(pid) => {
349                let _ = pid_tx.send(Ok(pid));
350                Some(pid)
351            }
352            Err(error) => {
353                let _ = pid_tx.send(Err(error));
354                self.finish_terminal(process_id);
355                None
356            }
357        }
358    }
359}
360
361// ---------------------------------------------------------------------------
362// Shell / terminal
363// ---------------------------------------------------------------------------
364//
365// Note: `fetch` (the Network half of this reference section) is scaffolded in `net.rs`, which owns
366// the `impl AgentOs { fn fetch }` block. It is intentionally NOT defined here to avoid a duplicate
367// definition; the helpers below (`rejected_to_error`, `vm_ownership`) are shared by both halves.
368
369impl AgentOs {
370    /// Open a PTY-backed shell. SYNC. Returns a synthetic `shell-N` id (NOT a pid).
371    ///
372    /// The shell id and its registry entry are allocated synchronously (matching the TS sync
373    /// contract); the actual guest-process spawn, output fan-out, and exit-task registration happen
374    /// on a background task because the wire spawn is async. The exit task is tracked in the
375    /// pending-shell-exit set so `dispose` can drain it (two-phase teardown).
376    ///
377    /// Stdout and stderr are fanned into the shell's ordered `data` broadcast (`on_shell_data`).
378    /// Stderr is also fanned into a dedicated diagnostic broadcast (`on_shell_stderr` and the
379    /// [`OpenShellOptions::on_stderr`] callback); terminal renderers should consume only `data`.
380    pub fn open_shell(&self, mut options: OpenShellOptions) -> Result<ShellHandle> {
381        let inner = self.inner();
382        if inner.disposed.load(Ordering::SeqCst) {
383            return Err(ClientError::Sidecar(
384                "cannot open terminal after VM shutdown has started".into(),
385            )
386            .into());
387        }
388        let mut reservation = TerminalReservation::new(self)?;
389        let counter = inner.shell_counter.fetch_add(1, Ordering::SeqCst) + 1;
390        let shell_id = format!("shell-{counter}");
391        // The wire-side process id used by write_shell/close_shell and event routing.
392        let process_id = format!("shell-{}", Uuid::new_v4());
393
394        let (data_tx, _) = tokio::sync::broadcast::channel(SHELL_DATA_CHANNEL_CAPACITY);
395        let (stderr_tx, _) = tokio::sync::broadcast::channel(SHELL_DATA_CHANNEL_CAPACITY);
396        let (event_tx, _) = tokio::sync::broadcast::channel(SHELL_DATA_CHANNEL_CAPACITY);
397        // Spawn-readiness gate: write/close await this before issuing their wire request.
398        let (spawned_tx, _) = tokio::sync::watch::channel(None);
399        // Exit-code channel backing `wait_shell`.
400        let (exit_tx, _) = tokio::sync::watch::channel(None);
401        // Register the entry up front so write/resize/close can address it immediately, exactly like
402        // the TS map insert before the handle's async work settles.
403        let entry = ShellEntry {
404            pid: 0,
405            data_tx: data_tx.clone(),
406            stderr_tx: stderr_tx.clone(),
407            event_tx: event_tx.clone(),
408            process_id: process_id.clone(),
409            spawned_tx: spawned_tx.clone(),
410            exit_tx: exit_tx.clone(),
411        };
412        // `insert` fails only if the key already exists; the monotonic counter guarantees it cannot.
413        let _ = inner.shells.insert(shell_id.clone(), entry);
414
415        let command = options
416            .command
417            .clone()
418            .unwrap_or_else(|| DEFAULT_SHELL_COMMAND.to_string());
419        options
420            .env
421            .insert(String::from("AGENTOS_EXEC_TTY"), String::from("1"));
422        // Seed the PTY winsize env exactly like the TS openShell (COLUMNS/LINES).
423        if let Some(cols) = options.cols {
424            options
425                .env
426                .insert(String::from("COLUMNS"), cols.to_string());
427        }
428        if let Some(rows) = options.rows {
429            options.env.insert(String::from("LINES"), rows.to_string());
430        }
431        let execute = wire::ExecuteRequest {
432            process_id: process_id.clone(),
433            command: Some(command),
434            runtime: None,
435            entrypoint: None,
436            args: options.args.clone(),
437            env: options.env.clone().into_iter().collect(),
438            cwd: options.cwd.clone(),
439            wasm_permission_tier: None,
440            retain_output: true,
441        };
442
443        // Background: subscribe to events first (so no output is missed), issue the spawn, fan
444        // stdout into the data broadcast and stderr into the stderr broadcast, and complete when the
445        // process exits.
446        let agent = self.clone();
447        let ownership = self.vm_ownership();
448        let route_process_id = process_id.clone();
449        let exit_shell_id = shell_id.clone();
450        let exit_key = counter;
451        let (start_tx, start_rx) = tokio::sync::oneshot::channel();
452        let task_reservation = ShellReservation(self.clone());
453        reservation.disarm();
454        let handle = tokio::spawn(async move {
455            let _reservation = task_reservation;
456            if start_rx.await.is_err() {
457                return;
458            }
459            let mut events = agent.transport().subscribe_wire_events();
460
461            let response = match agent
462                .transport()
463                .request_wire(
464                    ownership.clone(),
465                    wire::RequestPayload::ExecuteRequest(execute),
466                )
467                .await
468            {
469                Ok(response) => shell_started(response),
470                Err(error) => Err(error.into()),
471            };
472            let kernel_pid = match response {
473                Ok(pid) => pid,
474                Err(error) => {
475                    tracing::warn!(?error, shell_id = %exit_shell_id, "open_shell spawn failed");
476                    agent.retain_shell_outcome(
477                        &exit_shell_id,
478                        &route_process_id,
479                        0,
480                        Err(error.clone()),
481                    );
482                    spawned_tx.send_replace(Some(Err(error.clone())));
483                    publish_shell_outcome(&exit_tx, Err(error));
484                    agent.inner().shells.remove(&exit_shell_id);
485                    agent.inner().pending_shell_exits.remove(&exit_key);
486                    return;
487                }
488            };
489
490            // Record the real kernel pid on the entry (TS `ShellHandle.pid`) and release the write
491            // gate so any queued `write_shell`/`close_shell` proceed against the live spawn.
492            agent
493                .inner()
494                .shells
495                .update(&exit_shell_id, |_, existing| existing.pid = kernel_pid);
496            // send_replace, not send: `watch::Sender::send` REFUSES to store the
497            // value while no receiver exists (and the initial receiver is dropped
498            // at channel creation), which left the spawn gate permanently false
499            // for any write/resize issued after this point — they hung forever in
500            // wait_for_spawn. send_replace stores unconditionally.
501            spawned_tx.send_replace(Some(Ok(())));
502
503            while let Some(event) = next_shell_event(&mut events, &ownership, &exit_tx).await {
504                let payload = match event {
505                    Ok(value) => value,
506                    Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => {
507                        tracing::warn!(skipped, shell_id = %exit_shell_id, "terminal live events lost; recover output with terminal.output.read");
508                        continue;
509                    }
510                    Err(tokio::sync::broadcast::error::RecvError::Closed) => {
511                        let error = crate::process::sidecar_exited_before_exit(&route_process_id);
512                        tracing::warn!(?error, "terminal exit unconfirmed");
513                        agent.retain_shell_outcome(
514                            &exit_shell_id,
515                            &route_process_id,
516                            kernel_pid,
517                            Err(error.clone()),
518                        );
519                        publish_shell_outcome(&exit_tx, Err(error));
520                        break;
521                    }
522                };
523                match payload {
524                    EventPayload::ProcessOutputEvent(output) => {
525                        if output.process_id != route_process_id {
526                            continue;
527                        }
528                        let stream = match output.channel {
529                            StreamChannel::Stdout => ProcessStream::Stdout,
530                            StreamChannel::Stderr => ProcessStream::Stderr,
531                        };
532                        let (Some(sequence), Some(timestamp_ms)) =
533                            (output.sequence, output.timestamp_ms)
534                        else {
535                            tracing::error!(
536                                shell_id = %exit_shell_id,
537                                "retained terminal output event is missing its replay identity"
538                            );
539                            continue;
540                        };
541                        let _ = event_tx.send(TerminalOutputEvent {
542                            sequence,
543                            stream,
544                            data: output.chunk.clone(),
545                            timestamp_ms: timestamp_ms.min(i64::MAX as u64) as i64,
546                        });
547                        // Publish every PTY chunk from this single wire-event consumer so terminal
548                        // control sequences retain their original stdout/stderr order.
549                        let _ = data_tx.send(output.chunk.clone());
550                        if output.channel == StreamChannel::Stderr {
551                            // Channel identity remains available as an optional diagnostic tap.
552                            let _ = stderr_tx.send(output.chunk);
553                        }
554                    }
555                    EventPayload::ProcessExitedEvent(exited) => {
556                        if exited.process_id == route_process_id {
557                            // Record the exit code for `wait_shell`: live waiters observe the watch
558                            // update; late waiters (after the entry is dropped below) find it in the
559                            // bounded retention map, mirroring the TS closed-shell retention.
560                            agent.retain_shell_outcome(
561                                &exit_shell_id,
562                                &route_process_id,
563                                kernel_pid,
564                                Ok(exited.exit_code),
565                            );
566                            publish_shell_outcome(&exit_tx, Ok(exited.exit_code));
567                            break;
568                        }
569                    }
570                    EventPayload::VmLifecycleEvent(_)
571                    | EventPayload::ExecutionOutputEvent(_)
572                    | EventPayload::ExecutionCompletedEvent(_)
573                    | EventPayload::StructuredEvent(_)
574                    | EventPayload::ExtEnvelope(_) => {}
575                }
576            }
577
578            // The `.finally` equivalent: remove from both the tracking set and the shells map (only
579            // if it is still our entry, matching the TS identity check).
580            agent.inner().pending_shell_exits.remove(&exit_key);
581            agent.inner().shells.remove_if(&exit_shell_id, |existing| {
582                existing.process_id == route_process_id
583            });
584            // remove_if takes `&mut V`; the comparison only reads, which is fine.
585        });
586
587        let _ = inner.pending_shell_exits.insert(counter, handle);
588        if start_tx.send(()).is_err() {
589            tracing::warn!(%shell_id, "terminal task closed before launch");
590        }
591
592        Ok(ShellHandle { shell_id })
593    }
594
595    fn retain_shell_outcome(
596        &self,
597        shell_id: &str,
598        process_id: &str,
599        pid: u32,
600        result: std::result::Result<i32, ClientError>,
601    ) {
602        let mut retained = self.inner().closed_shells.lock();
603        retain_shell_result(
604            &mut retained,
605            ClosedShellEntry {
606                shell_id: shell_id.to_owned(),
607                process_id: process_id.to_owned(),
608                pid,
609                result,
610            },
611        );
612    }
613
614    fn retained_shell_entry(&self, shell_id: &str) -> Option<ClosedShellEntry> {
615        self.inner()
616            .closed_shells
617            .lock()
618            .iter()
619            .rev()
620            .find(|entry| entry.shell_id == shell_id)
621            .cloned()
622    }
623
624    fn retained_shell_outcome(&self, shell_id: &str) -> std::result::Result<i32, ClientError> {
625        self.retained_shell_entry(shell_id)
626            .map(|entry| entry.result)
627            .unwrap_or_else(|| Err(ClientError::ShellNotFound(shell_id.to_owned())))
628    }
629
630    /// Connect a terminal bound to host stdio. Returns a PID. NOT tracked in the shells map; cannot
631    /// be addressed by other shell methods. Killed during dispose via the terminal registry.
632    ///
633    /// Mirrors the TS `connectTerminal`, which routes its `onData`/`onStderr` callbacks through
634    /// `openShell`. The Rust port opens a shell, wires the caller's `on_data` to ordered terminal data
635    /// and `on_stderr` to the optional diagnostic tap, then returns the shell's pid. Host
636    /// stdin binding, terminal raw-mode, and SIGWINCH/resize forwarding are host-process concerns
637    /// that have no native wire op and are intentionally not bound here.
638    pub async fn connect_terminal(&self, options: ConnectTerminalOptions) -> Result<u32> {
639        let ConnectTerminalOptions {
640            base,
641            on_data,
642            on_stderr,
643        } = options;
644
645        let process_id = format!("terminal-{}", Uuid::new_v4());
646        let command = base
647            .command
648            .clone()
649            .unwrap_or_else(|| DEFAULT_SHELL_COMMAND.to_string());
650        let (data_tx, _) = tokio::sync::broadcast::channel::<Vec<u8>>(SHELL_DATA_CHANNEL_CAPACITY);
651        let (stderr_tx, _) =
652            tokio::sync::broadcast::channel::<Vec<u8>>(SHELL_DATA_CHANNEL_CAPACITY);
653
654        // onData defaults to host stdout in TS; the Rust port has no host process stdout to bind to,
655        // so it only fans out when a sink is supplied. onStderr is diagnostic and independent.
656        if let Some(cb) = on_data {
657            install_output_callback(data_tx.clone(), cb);
658        }
659        if let Some(cb) = on_stderr {
660            install_output_callback(stderr_tx.clone(), cb);
661        }
662
663        let execute = wire::ExecuteRequest {
664            process_id: process_id.clone(),
665            command: Some(command),
666            runtime: None,
667            entrypoint: None,
668            args: base.args.clone(),
669            env: base.env.clone().into_iter().collect(),
670            cwd: base.cwd.clone(),
671            wasm_permission_tier: None,
672            retain_output: true,
673        };
674
675        // Subscribe before issuing the spawn so no output is missed.
676        let events = self.transport().subscribe_wire_events();
677        let ownership = self.vm_ownership();
678        let (pid_tx, pid_rx) = tokio::sync::oneshot::channel();
679        let (start_tx, start_rx) = tokio::sync::oneshot::channel::<()>();
680        let agent = self.clone();
681        let route_process_id = process_id.clone();
682        let exit_task = tokio::spawn(async move {
683            if start_rx.await.is_err() {
684                return;
685            }
686            let terminal_pid = match agent
687                .start_terminal(execute, ownership, pid_tx, &route_process_id)
688                .await
689            {
690                Some(pid) => pid,
691                None => return,
692            };
693            let mut events = events;
694            loop {
695                let (_scope, payload) = match events.recv().await {
696                    Ok(value) => value,
697                    Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
698                        if terminal_process_finished(&agent, terminal_pid).await {
699                            break;
700                        }
701                        continue;
702                    }
703                    Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
704                };
705                match payload {
706                    EventPayload::ProcessOutputEvent(output) => {
707                        if output.process_id != route_process_id {
708                            continue;
709                        }
710                        let _ = data_tx.send(output.chunk.clone());
711                        if output.channel == StreamChannel::Stderr {
712                            let _ = stderr_tx.send(output.chunk);
713                        }
714                    }
715                    EventPayload::ProcessExitedEvent(exited) => {
716                        if exited.process_id == route_process_id {
717                            break;
718                        }
719                    }
720                    EventPayload::VmLifecycleEvent(_)
721                    | EventPayload::ExecutionOutputEvent(_)
722                    | EventPayload::ExecutionCompletedEvent(_)
723                    | EventPayload::StructuredEvent(_)
724                    | EventPayload::ExtEnvelope(_) => {}
725                }
726            }
727            agent.finish_terminal(&route_process_id);
728        });
729
730        {
731            let _terminal_lifecycle_guard = self.inner().terminal_lifecycle_lock.lock().await;
732            if self.inner().disposed.load(Ordering::SeqCst) {
733                exit_task.abort();
734                return Err(ClientError::Sidecar(
735                    "cannot connect terminal after VM shutdown has started".to_string(),
736                )
737                .into());
738            }
739            let mut terminal_reservation = TerminalReservation::new(self)?;
740            match self
741                .inner()
742                .terminals
743                .insert(process_id.clone(), TerminalEntry { exit_task })
744            {
745                Ok(()) => {}
746                Err((_, entry)) => {
747                    entry.exit_task.abort();
748                    return Err(ClientError::Sidecar(format!(
749                        "terminal process id collision while tracking terminal: {process_id}"
750                    ))
751                    .into());
752                }
753            }
754            terminal_reservation.disarm();
755            if start_tx.send(()).is_err() {
756                self.finish_terminal(&process_id);
757                return Err(ClientError::Sidecar(
758                    "terminal startup task ended before registration completed".to_string(),
759                )
760                .into());
761            }
762        }
763
764        pid_rx
765            .await
766            .map_err(|_| {
767                ClientError::Sidecar(
768                    "terminal startup task ended before returning a pid".to_string(),
769                )
770            })?
771            .map_err(Into::into)
772    }
773
774    /// Write to a shell. SYNC fire-and-forget. Errors with [`ClientError::ShellNotFound`].
775    pub fn write_shell(
776        &self,
777        shell_id: &str,
778        data: StdinInput,
779    ) -> std::result::Result<(), ClientError> {
780        let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
781        let chunk = stdin_chunk(data);
782
783        // Fire-and-forget: the TS handle.write returns void; surface only the synchronous
784        // ShellNotFound, and dispatch the wire write in the background after the spawn lands. TS
785        // openShell is fully synchronous so the spawn is always live by the time write runs; awaiting
786        // the readiness gate reproduces that ordering and avoids dropping early input.
787        let agent = self.clone();
788        let ownership = self.vm_ownership();
789        tokio::spawn(async move {
790            if let Err(error) = wait_for_spawn(spawned_rx).await {
791                tracing::warn!(?error, "write_shell launch failed");
792                return;
793            }
794            let payload = wire::RequestPayload::WriteStdinRequest(wire::WriteStdinRequest {
795                process_id,
796                chunk,
797            });
798            if let Err(error) = agent.transport().request_wire(ownership, payload).await {
799                tracing::warn!(?error, "write_shell failed");
800            }
801        });
802
803        Ok(())
804    }
805
806    /// List actor-addressable shells currently retained by Core.
807    pub fn list_shells(&self) -> Vec<TerminalInfo> {
808        let mut shells = Vec::new();
809        self.inner().shells.scan(|shell_id, entry| {
810            let outcome = entry.exit_tx.borrow();
811            let exit_code = outcome
812                .as_ref()
813                .and_then(|result| result.as_ref().ok())
814                .copied();
815            shells.push(TerminalInfo {
816                shell_id: shell_id.clone(),
817                pid: entry.pid,
818                running: outcome.is_none(),
819                exit_code,
820            });
821        });
822        shells.sort_by(|left, right| left.shell_id.cmp(&right.shell_id));
823        shells
824    }
825
826    /// Read a bounded raw-byte terminal replay. Screen rendering is a client concern.
827    pub async fn snapshot_shell(
828        &self,
829        shell_id: &str,
830        after: Option<u64>,
831        max_bytes: Option<usize>,
832    ) -> std::result::Result<TerminalSnapshot, ClientError> {
833        self.snapshot_shell_page(shell_id, after, None, max_bytes)
834            .await
835    }
836
837    /// Read a terminal replay page with the same event and byte bounds as
838    /// process output replay.
839    pub async fn snapshot_shell_page(
840        &self,
841        shell_id: &str,
842        after: Option<u64>,
843        max_events: Option<usize>,
844        max_bytes: Option<usize>,
845    ) -> std::result::Result<TerminalSnapshot, ClientError> {
846        let (max_events, max_bytes) =
847            crate::output_replay::wire_page_limits(max_events, max_bytes, "terminal.output.read")?;
848        let live = self.inner().shells.read(shell_id, |_, entry| {
849            (
850                entry.pid,
851                entry.process_id.clone(),
852                entry.exit_tx.borrow().clone(),
853            )
854        });
855        let (pid, process_id, retained_exit) = match live {
856            Some((pid, process_id, exit)) => (pid, process_id, exit),
857            None => {
858                let retained = self
859                    .retained_shell_entry(shell_id)
860                    .ok_or_else(|| ClientError::ShellNotFound(shell_id.to_owned()))?;
861                (retained.pid, retained.process_id, Some(retained.result))
862            }
863        };
864        let response = self
865            .transport()
866            .request_wire(
867                self.vm_ownership(),
868                wire::RequestPayload::ReadProcessOutputRequest(wire::ReadProcessOutputRequest {
869                    process_id: process_id.clone(),
870                    after,
871                    max_events: u32::try_from(max_events).map_err(|_| {
872                        ClientError::Sidecar(String::from(
873                            "terminal.output.read maxEvents exceeds the wire u32 range",
874                        ))
875                    })?,
876                    max_bytes: u32::try_from(max_bytes).map_err(|_| {
877                        ClientError::Sidecar(String::from(
878                            "terminal.output.read maxBytes exceeds the wire u32 range",
879                        ))
880                    })?,
881                }),
882            )
883            .await?;
884        let page = match response {
885            wire::ResponsePayload::ProcessOutputPageResponse(page) => page,
886            wire::ResponsePayload::RejectedResponse(rejected) => {
887                return Err(rejected_to_error(rejected));
888            }
889            other => {
890                return Err(ClientError::Sidecar(format!(
891                    "terminal.output.read: unexpected response {other:?}"
892                )));
893            }
894        };
895        let exit_code = match page.exit_code {
896            Some(exit_code) => {
897                // Persist before waking live waiters so removal of the live
898                // entry cannot create a gap for a late wait_shell caller.
899                self.retain_shell_outcome(shell_id, &process_id, pid, Ok(exit_code));
900                self.inner().shells.read(shell_id, |_, entry| {
901                    if entry.process_id == process_id {
902                        publish_shell_outcome(&entry.exit_tx, Ok(exit_code));
903                    }
904                });
905                Some(exit_code)
906            }
907            None => retained_exit.transpose()?,
908        };
909        Ok(TerminalSnapshot {
910            shell_id: shell_id.to_owned(),
911            pid,
912            events: page
913                .events
914                .into_iter()
915                .map(|event| TerminalOutputEvent {
916                    sequence: event.sequence,
917                    stream: match event.channel {
918                        StreamChannel::Stdout => ProcessStream::Stdout,
919                        StreamChannel::Stderr => ProcessStream::Stderr,
920                    },
921                    data: event.chunk,
922                    timestamp_ms: event.timestamp_ms.min(i64::MAX as u64) as i64,
923                })
924                .collect(),
925            next_cursor: page.next_cursor,
926            has_more: page.has_more,
927            truncated: page.truncated,
928            exit_code,
929        })
930    }
931
932    /// Write to a shell and AWAIT the wire write. Same routing as [`Self::write_shell`], but the
933    /// caller observes wire failures instead of a fire-and-forget warn — used by the actor plugin's
934    /// `writeShell` action so a failed write rejects the action.
935    pub async fn write_shell_awaited(
936        &self,
937        shell_id: &str,
938        data: StdinInput,
939    ) -> std::result::Result<(), ClientError> {
940        let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
941        let chunk = stdin_chunk(data);
942        tracing::debug!(shell_id, "write_shell_awaited: waiting for spawn gate");
943        wait_for_spawn(spawned_rx).await?;
944        tracing::debug!(shell_id, "write_shell_awaited: issuing wire write");
945        let payload =
946            wire::RequestPayload::WriteStdinRequest(wire::WriteStdinRequest { process_id, chunk });
947        let response = self
948            .transport()
949            .request_wire(self.vm_ownership(), payload)
950            .await?;
951        tracing::debug!(shell_id, "write_shell_awaited: wire write acked");
952        match response {
953            wire::ResponsePayload::RejectedResponse(rejected) => Err(rejected_to_error(rejected)),
954            _ => Ok(()),
955        }
956    }
957
958    /// Subscribe to a shell's ordered terminal data. SYNC register; multi-handler; dropping the
959    /// returned stream is the unsubscribe. Carries stdout and stderr exactly once in wire order.
960    /// Use [`Self::on_shell_stderr`] only as a channel-specific diagnostic tap, not as a second
961    /// terminal-rendering stream. Errors with [`ClientError::ShellNotFound`].
962    pub fn on_shell_data(
963        &self,
964        shell_id: &str,
965        mut handler: impl FnMut(ShellData) + Send + 'static,
966    ) -> std::result::Result<crate::stream::Subscription, ClientError> {
967        let mut rx = self
968            .inner()
969            .shells
970            .read(shell_id, |_, entry| entry.data_tx.subscribe())
971            .ok_or_else(|| ClientError::ShellNotFound(shell_id.to_string()))?;
972        let shell_id = shell_id.to_string();
973        let task = tokio::spawn(async move {
974            loop {
975                match rx.recv().await {
976                    Ok(data) => handler(ShellData {
977                        shell_id: shell_id.clone(),
978                        data,
979                    }),
980                    Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
981                    Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
982                }
983            }
984        });
985        Ok(crate::stream::Subscription::new(move || task.abort()))
986    }
987
988    /// Subscribe to sequenced terminal output retained by Core.
989    pub fn on_shell_output(
990        &self,
991        shell_id: &str,
992        mut handler: impl FnMut(TerminalOutputEvent) + Send + 'static,
993    ) -> std::result::Result<crate::stream::Subscription, ClientError> {
994        let rx = self
995            .inner()
996            .shells
997            .read(shell_id, |_, entry| entry.event_tx.subscribe());
998        let Some(mut rx) = rx else {
999            // A fast exit can race actor event registration. The final output
1000            // is available through replay; no future hints will be emitted.
1001            self.retained_shell_outcome(shell_id)?;
1002            return Ok(crate::stream::Subscription::noop());
1003        };
1004        let task = tokio::spawn(async move {
1005            loop {
1006                match rx.recv().await {
1007                    Ok(event) => handler(event),
1008                    Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => {
1009                        tracing::warn!(
1010                            skipped,
1011                            "terminal output subscriber lagged; recover with snapshot_shell"
1012                        );
1013                    }
1014                    Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
1015                }
1016            }
1017        });
1018        Ok(crate::stream::Subscription::new(move || task.abort()))
1019    }
1020
1021    /// Subscribe to a shell's stderr. SYNC register; multi-handler; dropping the returned stream is
1022    /// the unsubscribe. This is the optional diagnostic channel backing the TS `onStderr` option;
1023    /// stderr is also present once in ordered `on_shell_data`. Errors with
1024    /// [`ClientError::ShellNotFound`].
1025    pub fn on_shell_stderr(
1026        &self,
1027        shell_id: &str,
1028        mut handler: impl FnMut(ShellData) + Send + 'static,
1029    ) -> std::result::Result<crate::stream::Subscription, ClientError> {
1030        let mut rx = self
1031            .inner()
1032            .shells
1033            .read(shell_id, |_, entry| entry.stderr_tx.subscribe())
1034            .ok_or_else(|| ClientError::ShellNotFound(shell_id.to_string()))?;
1035        let shell_id = shell_id.to_string();
1036        let task = tokio::spawn(async move {
1037            loop {
1038                match rx.recv().await {
1039                    Ok(data) => handler(ShellData {
1040                        shell_id: shell_id.clone(),
1041                        data,
1042                    }),
1043                    Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
1044                    Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
1045                }
1046            }
1047        });
1048        Ok(crate::stream::Subscription::new(move || task.abort()))
1049    }
1050
1051    pub fn on_shell_exit(
1052        &self,
1053        shell_id: &str,
1054        handler: impl FnOnce(ShellExit) + Send + 'static,
1055    ) -> std::result::Result<crate::stream::Subscription, ClientError> {
1056        let rx = self
1057            .inner()
1058            .shells
1059            .read(shell_id, |_, entry| entry.exit_tx.subscribe());
1060        let Some(mut rx) = rx else {
1061            handler(ShellExit {
1062                shell_id: shell_id.to_owned(),
1063                exit_code: self.retained_shell_outcome(shell_id)?,
1064            });
1065            return Ok(crate::stream::Subscription::noop());
1066        };
1067        if let Some(result) = rx.borrow().clone() {
1068            let exit_code = result?;
1069            handler(ShellExit {
1070                shell_id: shell_id.to_string(),
1071                exit_code,
1072            });
1073            return Ok(crate::stream::Subscription::noop());
1074        }
1075        let shell_id = shell_id.to_string();
1076        let task = tokio::spawn(async move {
1077            while rx.changed().await.is_ok() {
1078                if let Some(result) = rx.borrow().clone() {
1079                    match result {
1080                        Ok(exit_code) => handler(ShellExit {
1081                            shell_id,
1082                            exit_code,
1083                        }),
1084                        Err(error) => {
1085                            tracing::warn!(?error, %shell_id, "terminal subscription failed before confirmed exit")
1086                        }
1087                    }
1088                    return;
1089                }
1090            }
1091        });
1092        Ok(crate::stream::Subscription::new(move || task.abort()))
1093    }
1094
1095    /// Resize a shell's PTY winsize. SYNC fire-and-forget, mirroring the TS `ShellHandle.resize`
1096    /// (which dispatches `resizePty` in the background after the spawn lands). Errors with
1097    /// [`ClientError::ShellNotFound`].
1098    pub fn resize_shell(
1099        &self,
1100        shell_id: &str,
1101        cols: u16,
1102        rows: u16,
1103    ) -> std::result::Result<(), ClientError> {
1104        // Existence check matches the TS `if (!entry) throw Shell not found`.
1105        let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
1106
1107        let agent = self.clone();
1108        let ownership = self.vm_ownership();
1109        tokio::spawn(async move {
1110            if let Err(error) = wait_for_spawn(spawned_rx).await {
1111                tracing::warn!(?error, "resize_shell launch failed");
1112                return;
1113            }
1114            let payload = wire::RequestPayload::ResizePtyRequest(wire::ResizePtyRequest {
1115                process_id,
1116                cols,
1117                rows,
1118            });
1119            if let Err(error) = agent.transport().request_wire(ownership, payload).await {
1120                tracing::warn!(?error, "resize_shell failed");
1121            }
1122        });
1123
1124        Ok(())
1125    }
1126
1127    /// Resize a shell and wait for the sidecar acknowledgement.
1128    pub async fn resize_shell_awaited(
1129        &self,
1130        shell_id: &str,
1131        cols: u16,
1132        rows: u16,
1133    ) -> std::result::Result<(), ClientError> {
1134        let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
1135        wait_for_spawn(spawned_rx).await?;
1136        let response = self
1137            .transport()
1138            .request_wire(
1139                self.vm_ownership(),
1140                wire::RequestPayload::ResizePtyRequest(wire::ResizePtyRequest {
1141                    process_id,
1142                    cols,
1143                    rows,
1144                }),
1145            )
1146            .await?;
1147        match response {
1148            wire::ResponsePayload::PtyResizedResponse(_) => Ok(()),
1149            wire::ResponsePayload::RejectedResponse(rejected) => Err(rejected_to_error(rejected)),
1150            other => Err(ClientError::Sidecar(format!(
1151                "resize shell: unexpected response {other:?}"
1152            ))),
1153        }
1154    }
1155
1156    /// Wait for a shell to exit and return its process exit code (TS `waitShell`). Resolves
1157    /// immediately for a shell that already exited within the bounded retention window. Errors with
1158    /// [`ClientError::ShellNotFound`] for an unknown id.
1159    pub async fn wait_shell(&self, shell_id: &str) -> std::result::Result<i32, ClientError> {
1160        let exit_rx = self
1161            .inner()
1162            .shells
1163            .read(shell_id, |_, entry| entry.exit_tx.subscribe());
1164        let Some(exit_rx) = exit_rx else {
1165            // Entry already dropped: fall back to the recorded exit code (TS retention behavior).
1166            return self.retained_shell_outcome(shell_id);
1167        };
1168        // Sender teardown without an outcome falls back to bounded retention.
1169        observe_shell_exit(exit_rx)
1170            .await
1171            .unwrap_or_else(|| self.retained_shell_outcome(shell_id))
1172    }
1173
1174    /// Close a shell. SYNC. `kill()` + immediate map delete; the exit task is still drained by
1175    /// `dispose`. Errors with [`ClientError::ShellNotFound`].
1176    pub fn close_shell(&self, shell_id: &str) -> std::result::Result<(), ClientError> {
1177        let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
1178
1179        // Immediate map delete, exactly like the TS `_shells.delete(shellId)`; the pending-exit task
1180        // remains tracked so `dispose` still drains it (two-phase teardown).
1181        self.inner().shells.remove(shell_id);
1182
1183        // Fire-and-forget kill (SIGTERM) after the spawn lands so the kill addresses a live process.
1184        let agent = self.clone();
1185        let ownership = self.vm_ownership();
1186        tokio::spawn(async move {
1187            if let Err(error) = wait_for_spawn(spawned_rx).await {
1188                tracing::warn!(?error, "close_shell launch failed");
1189                return;
1190            }
1191            let payload = wire::RequestPayload::KillProcessRequest(wire::KillProcessRequest {
1192                process_id,
1193                signal: String::from("SIGTERM"),
1194            });
1195            if let Err(error) = agent.transport().request_wire(ownership, payload).await {
1196                tracing::warn!(?error, "close_shell kill failed");
1197            }
1198        });
1199
1200        Ok(())
1201    }
1202
1203    /// Close a shell and wait for signal delivery to be acknowledged.
1204    pub async fn close_shell_awaited(
1205        &self,
1206        shell_id: &str,
1207    ) -> std::result::Result<(), ClientError> {
1208        let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
1209        wait_for_spawn(spawned_rx).await?;
1210        let response = self
1211            .transport()
1212            .request_wire(
1213                self.vm_ownership(),
1214                wire::RequestPayload::KillProcessRequest(wire::KillProcessRequest {
1215                    process_id,
1216                    signal: String::from("SIGTERM"),
1217                }),
1218            )
1219            .await?;
1220        match response {
1221            wire::ResponsePayload::ProcessKilledResponse(_) => {
1222                self.inner().shells.remove(shell_id);
1223                Ok(())
1224            }
1225            wire::ResponsePayload::RejectedResponse(rejected) => Err(rejected_to_error(rejected)),
1226            other => Err(ClientError::Sidecar(format!(
1227                "close shell: unexpected response {other:?}"
1228            ))),
1229        }
1230    }
1231
1232    /// Look up the wire-side `process_id` and the spawn-readiness receiver for a shell id, or
1233    /// [`ClientError::ShellNotFound`].
1234    fn shell_wire_handle(
1235        &self,
1236        shell_id: &str,
1237    ) -> std::result::Result<(String, ShellSpawnReadiness), ClientError> {
1238        self.inner()
1239            .shells
1240            .read(shell_id, |_, entry| {
1241                (entry.process_id.clone(), entry.spawned_tx.subscribe())
1242            })
1243            .ok_or_else(|| {
1244                self.retained_shell_outcome(shell_id)
1245                    .err()
1246                    .unwrap_or_else(|| ClientError::ShellNotFound(shell_id.to_owned()))
1247            })
1248    }
1249}
1250
1251/// Wait until the shell's background `Execute` request has been acked (the readiness gate flips to
1252/// ready). A failed launch must not issue follow-up I/O against a nonexistent process.
1253async fn wait_for_spawn(
1254    mut spawned_rx: ShellSpawnReadiness,
1255) -> std::result::Result<(), ClientError> {
1256    loop {
1257        if let Some(result) = spawned_rx.borrow_and_update().clone() {
1258            return result;
1259        }
1260        spawned_rx.changed().await.map_err(|_| {
1261            ClientError::Sidecar("terminal launch ended before readiness was confirmed".into())
1262        })?;
1263    }
1264}
1265
1266async fn terminal_process_finished(agent: &AgentOs, pid: u32) -> bool {
1267    match agent.all_processes().await {
1268        Ok(processes) => match processes.into_iter().find(|process| process.pid == pid) {
1269            Some(process) => process.status != ProcessStatus::Running,
1270            None => true,
1271        },
1272        Err(error) => {
1273            tracing::warn!(?error, pid, "terminal process snapshot failed");
1274            false
1275        }
1276    }
1277}
1278
1279#[cfg(test)]
1280mod tests {
1281    #[tokio::test]
1282    async fn terminal_replayed_exit_recovers_waiters_and_retention_without_live_exit() {
1283        for failed in [false, true] {
1284            let failure = ClientError::TerminationFailed {
1285                process_id: "terminal-p".into(),
1286                reason: "terminal event stream lost its exit".into(),
1287            };
1288            let (outcome, _) = watch::channel(failed.then(|| Err(failure.clone())));
1289            let (_event_sender, mut events) = broadcast::channel(1);
1290            let ownership = wire::OwnershipScope::vm("connection", "session", "vm");
1291            let observer = next_shell_event(&mut events, &ownership, &outcome);
1292            tokio::pin!(observer);
1293            tokio::select! {
1294                _ = &mut observer => panic!("there is no confirmed terminal exit"),
1295                _ = tokio::task::yield_now() => {}
1296            }
1297            let mut retained = VecDeque::new();
1298            let entry = |result| ClosedShellEntry {
1299                shell_id: "shell-1".into(),
1300                process_id: "terminal-p".into(),
1301                pid: 42,
1302                result,
1303            };
1304            if failed {
1305                retain_shell_result(&mut retained, entry(Err(failure.clone())));
1306            }
1307            let waiter = observe_shell_exit(outcome.subscribe());
1308            tokio::pin!(waiter);
1309            if !failed {
1310                tokio::select! {
1311                    _ = &mut waiter => panic!("wait_shell must wait for a confirmed exit"),
1312                    _ = tokio::task::yield_now() => {}
1313                }
1314            }
1315            retain_shell_result(&mut retained, entry(Ok(17)));
1316            publish_shell_outcome(&outcome, Ok(17));
1317            assert_eq!(
1318                tokio::time::timeout(std::time::Duration::from_secs(1), &mut waiter)
1319                    .await
1320                    .unwrap()
1321                    .unwrap()
1322                    .unwrap(),
1323                17
1324            );
1325            assert!(
1326                tokio::time::timeout(std::time::Duration::from_secs(1), observer)
1327                    .await
1328                    .unwrap()
1329                    .is_none()
1330            );
1331            let late_waiter = outcome.subscribe();
1332            publish_shell_outcome(&outcome, Ok(17));
1333            publish_shell_outcome(&outcome, Err(failure.clone()));
1334            retain_shell_result(&mut retained, entry(Ok(17)));
1335            retain_shell_result(&mut retained, entry(Err(failure)));
1336            assert!(!late_waiter.has_changed().unwrap());
1337            assert_eq!(observe_shell_exit(late_waiter).await.unwrap().unwrap(), 17);
1338            assert_eq!(
1339                retained.len(),
1340                1,
1341                "repeat replay must not consume retention"
1342            );
1343            assert_eq!(*retained[0].result.as_ref().unwrap(), 17);
1344        }
1345    }
1346
1347    #[tokio::test]
1348    async fn terminal_launch_rejection_is_typed_and_does_not_open_the_io_gate() {
1349        let error = shell_started(wire::ResponsePayload::RejectedResponse(
1350            wire::RejectedResponse {
1351                code: "EACCES".into(),
1352                message: "execution denied".into(),
1353                limit_name: None,
1354                configured_limit: None,
1355                current_usage: None,
1356                requested: None,
1357                unit: None,
1358                scope: None,
1359                vm_id: None,
1360                session_generation: None,
1361                capability_id: None,
1362                operation: None,
1363                configuration_path: None,
1364                retryable: None,
1365                errno: None,
1366            },
1367        ))
1368        .unwrap_err();
1369        assert!(matches!(&error, ClientError::Kernel { code, .. } if code == "EACCES"));
1370        let (sender, receiver) = tokio::sync::watch::channel(None);
1371        sender.send_replace(Some(Err(error)));
1372        assert!(
1373            matches!(wait_for_spawn(receiver).await, Err(ClientError::Kernel { code, .. }) if code == "EACCES")
1374        );
1375        let (sender, receiver) = tokio::sync::watch::channel(None);
1376        drop(sender);
1377        assert!(wait_for_spawn(receiver).await.is_err());
1378    }
1379
1380    use super::*;
1381
1382    #[test]
1383    fn reserve_counter_enforces_limit_and_release_reopens_slot() {
1384        let counter = AtomicUsize::new(0);
1385
1386        assert!(try_reserve_counter(&counter, 2));
1387        assert!(try_reserve_counter(&counter, 2));
1388        assert!(!try_reserve_counter(&counter, 2));
1389        release_counter(&counter);
1390        assert!(try_reserve_counter(&counter, 2));
1391    }
1392}