1use std::collections::{BTreeMap, VecDeque};
19use std::sync::atomic::{AtomicUsize, Ordering};
20
21use anyhow::Result;
22use tokio::sync::{broadcast, watch};
23use uuid::Uuid;
24
25use agentos_sidecar_client::wire::{self, EventPayload, StreamChannel};
26
27use crate::agent_os::{AgentOs, ClosedShellEntry, ShellEntry, TerminalEntry};
28use crate::error::ClientError;
29use crate::process::{
30 install_output_callback, OutputCallback, ProcessStatus, ProcessStream, StdinInput,
31};
32
33const SHELL_DATA_CHANNEL_CAPACITY: usize = 1024;
35
36const TERMINAL_LIMIT: usize = 1024;
38
39const DEFAULT_SHELL_COMMAND: &str = "sh";
42
43type ShellOutcome = Option<std::result::Result<i32, ClientError>>;
44type ShellSpawnReadiness = watch::Receiver<Option<std::result::Result<(), ClientError>>>;
45
46fn publish_shell_outcome(
47 sender: &watch::Sender<ShellOutcome>,
48 result: std::result::Result<i32, ClientError>,
49) {
50 sender.send_if_modified(|outcome| {
51 if matches!(outcome, Some(Ok(_))) {
54 return false;
55 }
56 *outcome = Some(result);
57 true
58 });
59}
60
61fn retain_shell_result(retained: &mut VecDeque<ClosedShellEntry>, entry: ClosedShellEntry) {
62 if let Some(existing) = retained.iter_mut().find(|existing| {
63 existing.shell_id == entry.shell_id && existing.process_id == entry.process_id
64 }) {
65 if existing.result.is_err() {
66 *existing = entry;
67 }
68 return;
69 }
70 retained.push_back(entry);
71 while retained.len() > crate::CLOSED_SHELL_EXIT_CODE_RETENTION_LIMIT {
72 retained.pop_front();
73 }
74}
75
76async fn next_shell_event(
77 events: &mut broadcast::Receiver<(wire::OwnershipScope, EventPayload)>,
78 ownership: &wire::OwnershipScope,
79 outcome: &watch::Sender<ShellOutcome>,
80) -> Option<std::result::Result<EventPayload, broadcast::error::RecvError>> {
81 let mut replay_outcome = outcome.subscribe();
82 loop {
83 if matches!(*replay_outcome.borrow(), Some(Ok(_))) {
84 return None;
85 }
86 tokio::select! {
87 event = events.recv() => match event {
88 Ok((scope, payload)) if scope == *ownership => return Some(Ok(payload)),
89 Ok(_) => continue,
90 Err(error) => return Some(Err(error)),
91 },
92 changed = replay_outcome.changed() => {
93 if let Err(error) = changed {
94 tracing::error!(%error, "terminal outcome channel closed during observation");
95 return None;
96 }
97 }
98 }
99 }
100}
101
102async fn observe_shell_exit(mut receiver: watch::Receiver<ShellOutcome>) -> ShellOutcome {
103 loop {
104 if let Some(result) = receiver.borrow_and_update().clone() {
105 return Some(result);
106 }
107 if receiver.changed().await.is_err() {
108 return None;
109 }
110 }
111}
112
113#[derive(Default)]
119pub struct OpenShellOptions {
120 pub command: Option<String>,
121 pub args: Vec<String>,
122 pub env: BTreeMap<String, String>,
123 pub cwd: Option<String>,
124 pub cols: Option<u16>,
125 pub rows: Option<u16>,
126}
127
128#[derive(Default)]
134pub struct ConnectTerminalOptions {
135 pub base: OpenShellOptions,
136 pub on_data: Option<OutputCallback>,
137 pub on_stderr: Option<OutputCallback>,
138}
139
140#[derive(Debug, Clone, PartialEq, Eq)]
142pub struct ShellHandle {
143 pub shell_id: String,
144}
145
146#[derive(Debug, Clone, PartialEq, Eq)]
147pub struct ShellData {
148 pub shell_id: String,
149 pub data: Vec<u8>,
150}
151
152#[derive(Debug, Clone, PartialEq, Eq)]
153pub struct ShellExit {
154 pub shell_id: String,
155 pub exit_code: i32,
156}
157
158#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
159#[serde(rename_all = "camelCase")]
160pub struct TerminalInfo {
161 pub shell_id: String,
162 pub pid: u32,
163 pub running: bool,
164 pub exit_code: Option<i32>,
165}
166
167#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
168#[serde(rename_all = "camelCase")]
169pub struct TerminalOutputEvent {
170 pub sequence: u64,
171 pub stream: ProcessStream,
172 pub data: Vec<u8>,
173 pub timestamp_ms: i64,
174}
175
176#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
177#[serde(rename_all = "camelCase")]
178pub struct TerminalSnapshot {
179 pub shell_id: String,
180 pub pid: u32,
181 pub events: Vec<TerminalOutputEvent>,
182 pub next_cursor: Option<u64>,
183 pub has_more: bool,
184 pub truncated: bool,
185 pub exit_code: Option<i32>,
186}
187
188fn rejected_to_error(rejected: wire::RejectedResponse) -> ClientError {
194 ClientError::from_rejection(rejected)
195}
196
197fn shell_started(response: wire::ResponsePayload) -> std::result::Result<u32, ClientError> {
198 match response {
199 wire::ResponsePayload::ProcessStartedResponse(wire::ProcessStartedResponse {
200 pid: Some(pid),
201 ..
202 }) => Ok(pid),
203 wire::ResponsePayload::RejectedResponse(rejected) => Err(rejected_to_error(rejected)),
204 other => Err(ClientError::Sidecar(format!(
205 "open_shell: expected a started process with a PID, received {other:?}"
206 ))),
207 }
208}
209
210struct ShellReservation(AgentOs);
211
212impl Drop for ShellReservation {
213 fn drop(&mut self) {
214 release_counter(&self.0.inner().terminal_count);
215 }
216}
217
218fn stdin_chunk(data: StdinInput) -> Vec<u8> {
222 match data {
223 StdinInput::Text(text) => text.into_bytes(),
224 StdinInput::Bytes(bytes) => bytes,
225 }
226}
227
228fn try_reserve_counter(counter: &AtomicUsize, limit: usize) -> bool {
229 counter
230 .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |count| {
231 (count < limit).then_some(count + 1)
232 })
233 .is_ok()
234}
235
236fn release_counter(counter: &AtomicUsize) {
237 let _ = counter.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |count| {
238 Some(count.saturating_sub(1))
239 });
240}
241
242struct TerminalReservation<'a> {
243 agent: &'a AgentOs,
244 active: bool,
245}
246
247impl<'a> TerminalReservation<'a> {
248 fn new(agent: &'a AgentOs) -> std::result::Result<Self, ClientError> {
249 if !try_reserve_counter(&agent.inner().terminal_count, TERMINAL_LIMIT) {
250 return Err(ClientError::ResourceLimit {
251 code: "ERR_AGENTOS_RESOURCE_LIMIT".into(),
252 message: format!("terminal limit {TERMINAL_LIMIT} reached; wait for existing terminals to exit or raise TERMINAL_LIMIT"),
253 details: Box::new(crate::ResourceLimitDetails {
254 limit_name: Some("active_terminals".into()),
255 configured_limit: Some(TERMINAL_LIMIT as u64),
256 requested: Some(TERMINAL_LIMIT as u64 + 1),
257 configuration_path: Some("TERMINAL_LIMIT".into()),
258 unit: Some("terminals".into()),
259 scope: Some("vm".into()),
260 retryable: Some(true),
261 ..Default::default()
262 }),
263 });
264 }
265 let active = agent.inner().terminal_count.load(Ordering::SeqCst);
266 if active >= TERMINAL_LIMIT * 4 / 5 {
267 tracing::warn!(
268 active,
269 limit = TERMINAL_LIMIT,
270 "terminal admission approaches TERMINAL_LIMIT; close unused terminals"
271 );
272 }
273 Ok(Self {
274 agent,
275 active: true,
276 })
277 }
278
279 fn disarm(&mut self) {
280 self.active = false;
281 }
282}
283
284impl Drop for TerminalReservation<'_> {
285 fn drop(&mut self) {
286 if self.active {
287 release_counter(&self.agent.inner().terminal_count);
288 }
289 }
290}
291
292impl AgentOs {
293 fn vm_ownership(&self) -> wire::OwnershipScope {
295 wire::OwnershipScope::VmOwnership(wire::VmOwnership {
296 connection_id: self.connection_id().to_string(),
297 session_id: self.wire_session_id().to_string(),
298 vm_id: self.vm_id().to_string(),
299 })
300 }
301
302 pub(crate) fn finish_terminal(&self, process_id: &str) {
303 if self.inner().terminals.remove(process_id).is_some() {
304 release_counter(&self.inner().terminal_count);
305 }
306 }
307
308 async fn start_terminal(
309 &self,
310 execute: wire::ExecuteRequest,
311 ownership: wire::OwnershipScope,
312 pid_tx: tokio::sync::oneshot::Sender<std::result::Result<u32, ClientError>>,
313 process_id: &str,
314 ) -> Option<u32> {
315 {
316 let _terminal_lifecycle_guard = self.inner().terminal_lifecycle_lock.lock().await;
317 if self.inner().disposed.load(Ordering::SeqCst) {
318 let error = ClientError::Sidecar(
319 "cannot connect terminal after VM shutdown has started".to_string(),
320 );
321 let _ = pid_tx.send(Err(error));
322 self.finish_terminal(process_id);
323 return None;
324 }
325 }
326
327 let result = match self
328 .transport()
329 .request_wire(ownership, wire::RequestPayload::ExecuteRequest(execute))
330 .await
331 {
332 Ok(wire::ResponsePayload::ProcessStartedResponse(wire::ProcessStartedResponse {
333 pid,
334 ..
335 })) => pid.ok_or_else(|| {
336 ClientError::Sidecar("connect_terminal: sidecar did not return a pid".to_string())
337 }),
338 Ok(wire::ResponsePayload::RejectedResponse(rejected)) => {
339 Err(rejected_to_error(rejected))
340 }
341 Ok(other) => Err(ClientError::Sidecar(format!(
342 "unexpected response to connect_terminal: {other:?}"
343 ))),
344 Err(error) => Err(error.into()),
345 };
346
347 match result {
348 Ok(pid) => {
349 let _ = pid_tx.send(Ok(pid));
350 Some(pid)
351 }
352 Err(error) => {
353 let _ = pid_tx.send(Err(error));
354 self.finish_terminal(process_id);
355 None
356 }
357 }
358 }
359}
360
361impl AgentOs {
370 pub fn open_shell(&self, mut options: OpenShellOptions) -> Result<ShellHandle> {
381 let inner = self.inner();
382 if inner.disposed.load(Ordering::SeqCst) {
383 return Err(ClientError::Sidecar(
384 "cannot open terminal after VM shutdown has started".into(),
385 )
386 .into());
387 }
388 let mut reservation = TerminalReservation::new(self)?;
389 let counter = inner.shell_counter.fetch_add(1, Ordering::SeqCst) + 1;
390 let shell_id = format!("shell-{counter}");
391 let process_id = format!("shell-{}", Uuid::new_v4());
393
394 let (data_tx, _) = tokio::sync::broadcast::channel(SHELL_DATA_CHANNEL_CAPACITY);
395 let (stderr_tx, _) = tokio::sync::broadcast::channel(SHELL_DATA_CHANNEL_CAPACITY);
396 let (event_tx, _) = tokio::sync::broadcast::channel(SHELL_DATA_CHANNEL_CAPACITY);
397 let (spawned_tx, _) = tokio::sync::watch::channel(None);
399 let (exit_tx, _) = tokio::sync::watch::channel(None);
401 let entry = ShellEntry {
404 pid: 0,
405 data_tx: data_tx.clone(),
406 stderr_tx: stderr_tx.clone(),
407 event_tx: event_tx.clone(),
408 process_id: process_id.clone(),
409 spawned_tx: spawned_tx.clone(),
410 exit_tx: exit_tx.clone(),
411 };
412 let _ = inner.shells.insert(shell_id.clone(), entry);
414
415 let command = options
416 .command
417 .clone()
418 .unwrap_or_else(|| DEFAULT_SHELL_COMMAND.to_string());
419 options
420 .env
421 .insert(String::from("AGENTOS_EXEC_TTY"), String::from("1"));
422 if let Some(cols) = options.cols {
424 options
425 .env
426 .insert(String::from("COLUMNS"), cols.to_string());
427 }
428 if let Some(rows) = options.rows {
429 options.env.insert(String::from("LINES"), rows.to_string());
430 }
431 let execute = wire::ExecuteRequest {
432 process_id: process_id.clone(),
433 command: Some(command),
434 runtime: None,
435 entrypoint: None,
436 args: options.args.clone(),
437 env: options.env.clone().into_iter().collect(),
438 cwd: options.cwd.clone(),
439 wasm_permission_tier: None,
440 retain_output: true,
441 };
442
443 let agent = self.clone();
447 let ownership = self.vm_ownership();
448 let route_process_id = process_id.clone();
449 let exit_shell_id = shell_id.clone();
450 let exit_key = counter;
451 let (start_tx, start_rx) = tokio::sync::oneshot::channel();
452 let task_reservation = ShellReservation(self.clone());
453 reservation.disarm();
454 let handle = tokio::spawn(async move {
455 let _reservation = task_reservation;
456 if start_rx.await.is_err() {
457 return;
458 }
459 let mut events = agent.transport().subscribe_wire_events();
460
461 let response = match agent
462 .transport()
463 .request_wire(
464 ownership.clone(),
465 wire::RequestPayload::ExecuteRequest(execute),
466 )
467 .await
468 {
469 Ok(response) => shell_started(response),
470 Err(error) => Err(error.into()),
471 };
472 let kernel_pid = match response {
473 Ok(pid) => pid,
474 Err(error) => {
475 tracing::warn!(?error, shell_id = %exit_shell_id, "open_shell spawn failed");
476 agent.retain_shell_outcome(
477 &exit_shell_id,
478 &route_process_id,
479 0,
480 Err(error.clone()),
481 );
482 spawned_tx.send_replace(Some(Err(error.clone())));
483 publish_shell_outcome(&exit_tx, Err(error));
484 agent.inner().shells.remove(&exit_shell_id);
485 agent.inner().pending_shell_exits.remove(&exit_key);
486 return;
487 }
488 };
489
490 agent
493 .inner()
494 .shells
495 .update(&exit_shell_id, |_, existing| existing.pid = kernel_pid);
496 spawned_tx.send_replace(Some(Ok(())));
502
503 while let Some(event) = next_shell_event(&mut events, &ownership, &exit_tx).await {
504 let payload = match event {
505 Ok(value) => value,
506 Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => {
507 tracing::warn!(skipped, shell_id = %exit_shell_id, "terminal live events lost; recover output with terminal.output.read");
508 continue;
509 }
510 Err(tokio::sync::broadcast::error::RecvError::Closed) => {
511 let error = crate::process::sidecar_exited_before_exit(&route_process_id);
512 tracing::warn!(?error, "terminal exit unconfirmed");
513 agent.retain_shell_outcome(
514 &exit_shell_id,
515 &route_process_id,
516 kernel_pid,
517 Err(error.clone()),
518 );
519 publish_shell_outcome(&exit_tx, Err(error));
520 break;
521 }
522 };
523 match payload {
524 EventPayload::ProcessOutputEvent(output) => {
525 if output.process_id != route_process_id {
526 continue;
527 }
528 let stream = match output.channel {
529 StreamChannel::Stdout => ProcessStream::Stdout,
530 StreamChannel::Stderr => ProcessStream::Stderr,
531 };
532 let (Some(sequence), Some(timestamp_ms)) =
533 (output.sequence, output.timestamp_ms)
534 else {
535 tracing::error!(
536 shell_id = %exit_shell_id,
537 "retained terminal output event is missing its replay identity"
538 );
539 continue;
540 };
541 let _ = event_tx.send(TerminalOutputEvent {
542 sequence,
543 stream,
544 data: output.chunk.clone(),
545 timestamp_ms: timestamp_ms.min(i64::MAX as u64) as i64,
546 });
547 let _ = data_tx.send(output.chunk.clone());
550 if output.channel == StreamChannel::Stderr {
551 let _ = stderr_tx.send(output.chunk);
553 }
554 }
555 EventPayload::ProcessExitedEvent(exited) => {
556 if exited.process_id == route_process_id {
557 agent.retain_shell_outcome(
561 &exit_shell_id,
562 &route_process_id,
563 kernel_pid,
564 Ok(exited.exit_code),
565 );
566 publish_shell_outcome(&exit_tx, Ok(exited.exit_code));
567 break;
568 }
569 }
570 EventPayload::VmLifecycleEvent(_)
571 | EventPayload::ExecutionOutputEvent(_)
572 | EventPayload::ExecutionCompletedEvent(_)
573 | EventPayload::StructuredEvent(_)
574 | EventPayload::ExtEnvelope(_) => {}
575 }
576 }
577
578 agent.inner().pending_shell_exits.remove(&exit_key);
581 agent.inner().shells.remove_if(&exit_shell_id, |existing| {
582 existing.process_id == route_process_id
583 });
584 });
586
587 let _ = inner.pending_shell_exits.insert(counter, handle);
588 if start_tx.send(()).is_err() {
589 tracing::warn!(%shell_id, "terminal task closed before launch");
590 }
591
592 Ok(ShellHandle { shell_id })
593 }
594
595 fn retain_shell_outcome(
596 &self,
597 shell_id: &str,
598 process_id: &str,
599 pid: u32,
600 result: std::result::Result<i32, ClientError>,
601 ) {
602 let mut retained = self.inner().closed_shells.lock();
603 retain_shell_result(
604 &mut retained,
605 ClosedShellEntry {
606 shell_id: shell_id.to_owned(),
607 process_id: process_id.to_owned(),
608 pid,
609 result,
610 },
611 );
612 }
613
614 fn retained_shell_entry(&self, shell_id: &str) -> Option<ClosedShellEntry> {
615 self.inner()
616 .closed_shells
617 .lock()
618 .iter()
619 .rev()
620 .find(|entry| entry.shell_id == shell_id)
621 .cloned()
622 }
623
624 fn retained_shell_outcome(&self, shell_id: &str) -> std::result::Result<i32, ClientError> {
625 self.retained_shell_entry(shell_id)
626 .map(|entry| entry.result)
627 .unwrap_or_else(|| Err(ClientError::ShellNotFound(shell_id.to_owned())))
628 }
629
630 pub async fn connect_terminal(&self, options: ConnectTerminalOptions) -> Result<u32> {
639 let ConnectTerminalOptions {
640 base,
641 on_data,
642 on_stderr,
643 } = options;
644
645 let process_id = format!("terminal-{}", Uuid::new_v4());
646 let command = base
647 .command
648 .clone()
649 .unwrap_or_else(|| DEFAULT_SHELL_COMMAND.to_string());
650 let (data_tx, _) = tokio::sync::broadcast::channel::<Vec<u8>>(SHELL_DATA_CHANNEL_CAPACITY);
651 let (stderr_tx, _) =
652 tokio::sync::broadcast::channel::<Vec<u8>>(SHELL_DATA_CHANNEL_CAPACITY);
653
654 if let Some(cb) = on_data {
657 install_output_callback(data_tx.clone(), cb);
658 }
659 if let Some(cb) = on_stderr {
660 install_output_callback(stderr_tx.clone(), cb);
661 }
662
663 let execute = wire::ExecuteRequest {
664 process_id: process_id.clone(),
665 command: Some(command),
666 runtime: None,
667 entrypoint: None,
668 args: base.args.clone(),
669 env: base.env.clone().into_iter().collect(),
670 cwd: base.cwd.clone(),
671 wasm_permission_tier: None,
672 retain_output: true,
673 };
674
675 let events = self.transport().subscribe_wire_events();
677 let ownership = self.vm_ownership();
678 let (pid_tx, pid_rx) = tokio::sync::oneshot::channel();
679 let (start_tx, start_rx) = tokio::sync::oneshot::channel::<()>();
680 let agent = self.clone();
681 let route_process_id = process_id.clone();
682 let exit_task = tokio::spawn(async move {
683 if start_rx.await.is_err() {
684 return;
685 }
686 let terminal_pid = match agent
687 .start_terminal(execute, ownership, pid_tx, &route_process_id)
688 .await
689 {
690 Some(pid) => pid,
691 None => return,
692 };
693 let mut events = events;
694 loop {
695 let (_scope, payload) = match events.recv().await {
696 Ok(value) => value,
697 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
698 if terminal_process_finished(&agent, terminal_pid).await {
699 break;
700 }
701 continue;
702 }
703 Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
704 };
705 match payload {
706 EventPayload::ProcessOutputEvent(output) => {
707 if output.process_id != route_process_id {
708 continue;
709 }
710 let _ = data_tx.send(output.chunk.clone());
711 if output.channel == StreamChannel::Stderr {
712 let _ = stderr_tx.send(output.chunk);
713 }
714 }
715 EventPayload::ProcessExitedEvent(exited) => {
716 if exited.process_id == route_process_id {
717 break;
718 }
719 }
720 EventPayload::VmLifecycleEvent(_)
721 | EventPayload::ExecutionOutputEvent(_)
722 | EventPayload::ExecutionCompletedEvent(_)
723 | EventPayload::StructuredEvent(_)
724 | EventPayload::ExtEnvelope(_) => {}
725 }
726 }
727 agent.finish_terminal(&route_process_id);
728 });
729
730 {
731 let _terminal_lifecycle_guard = self.inner().terminal_lifecycle_lock.lock().await;
732 if self.inner().disposed.load(Ordering::SeqCst) {
733 exit_task.abort();
734 return Err(ClientError::Sidecar(
735 "cannot connect terminal after VM shutdown has started".to_string(),
736 )
737 .into());
738 }
739 let mut terminal_reservation = TerminalReservation::new(self)?;
740 match self
741 .inner()
742 .terminals
743 .insert(process_id.clone(), TerminalEntry { exit_task })
744 {
745 Ok(()) => {}
746 Err((_, entry)) => {
747 entry.exit_task.abort();
748 return Err(ClientError::Sidecar(format!(
749 "terminal process id collision while tracking terminal: {process_id}"
750 ))
751 .into());
752 }
753 }
754 terminal_reservation.disarm();
755 if start_tx.send(()).is_err() {
756 self.finish_terminal(&process_id);
757 return Err(ClientError::Sidecar(
758 "terminal startup task ended before registration completed".to_string(),
759 )
760 .into());
761 }
762 }
763
764 pid_rx
765 .await
766 .map_err(|_| {
767 ClientError::Sidecar(
768 "terminal startup task ended before returning a pid".to_string(),
769 )
770 })?
771 .map_err(Into::into)
772 }
773
774 pub fn write_shell(
776 &self,
777 shell_id: &str,
778 data: StdinInput,
779 ) -> std::result::Result<(), ClientError> {
780 let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
781 let chunk = stdin_chunk(data);
782
783 let agent = self.clone();
788 let ownership = self.vm_ownership();
789 tokio::spawn(async move {
790 if let Err(error) = wait_for_spawn(spawned_rx).await {
791 tracing::warn!(?error, "write_shell launch failed");
792 return;
793 }
794 let payload = wire::RequestPayload::WriteStdinRequest(wire::WriteStdinRequest {
795 process_id,
796 chunk,
797 });
798 if let Err(error) = agent.transport().request_wire(ownership, payload).await {
799 tracing::warn!(?error, "write_shell failed");
800 }
801 });
802
803 Ok(())
804 }
805
806 pub fn list_shells(&self) -> Vec<TerminalInfo> {
808 let mut shells = Vec::new();
809 self.inner().shells.scan(|shell_id, entry| {
810 let outcome = entry.exit_tx.borrow();
811 let exit_code = outcome
812 .as_ref()
813 .and_then(|result| result.as_ref().ok())
814 .copied();
815 shells.push(TerminalInfo {
816 shell_id: shell_id.clone(),
817 pid: entry.pid,
818 running: outcome.is_none(),
819 exit_code,
820 });
821 });
822 shells.sort_by(|left, right| left.shell_id.cmp(&right.shell_id));
823 shells
824 }
825
826 pub async fn snapshot_shell(
828 &self,
829 shell_id: &str,
830 after: Option<u64>,
831 max_bytes: Option<usize>,
832 ) -> std::result::Result<TerminalSnapshot, ClientError> {
833 self.snapshot_shell_page(shell_id, after, None, max_bytes)
834 .await
835 }
836
837 pub async fn snapshot_shell_page(
840 &self,
841 shell_id: &str,
842 after: Option<u64>,
843 max_events: Option<usize>,
844 max_bytes: Option<usize>,
845 ) -> std::result::Result<TerminalSnapshot, ClientError> {
846 let (max_events, max_bytes) =
847 crate::output_replay::wire_page_limits(max_events, max_bytes, "terminal.output.read")?;
848 let live = self.inner().shells.read(shell_id, |_, entry| {
849 (
850 entry.pid,
851 entry.process_id.clone(),
852 entry.exit_tx.borrow().clone(),
853 )
854 });
855 let (pid, process_id, retained_exit) = match live {
856 Some((pid, process_id, exit)) => (pid, process_id, exit),
857 None => {
858 let retained = self
859 .retained_shell_entry(shell_id)
860 .ok_or_else(|| ClientError::ShellNotFound(shell_id.to_owned()))?;
861 (retained.pid, retained.process_id, Some(retained.result))
862 }
863 };
864 let response = self
865 .transport()
866 .request_wire(
867 self.vm_ownership(),
868 wire::RequestPayload::ReadProcessOutputRequest(wire::ReadProcessOutputRequest {
869 process_id: process_id.clone(),
870 after,
871 max_events: u32::try_from(max_events).map_err(|_| {
872 ClientError::Sidecar(String::from(
873 "terminal.output.read maxEvents exceeds the wire u32 range",
874 ))
875 })?,
876 max_bytes: u32::try_from(max_bytes).map_err(|_| {
877 ClientError::Sidecar(String::from(
878 "terminal.output.read maxBytes exceeds the wire u32 range",
879 ))
880 })?,
881 }),
882 )
883 .await?;
884 let page = match response {
885 wire::ResponsePayload::ProcessOutputPageResponse(page) => page,
886 wire::ResponsePayload::RejectedResponse(rejected) => {
887 return Err(rejected_to_error(rejected));
888 }
889 other => {
890 return Err(ClientError::Sidecar(format!(
891 "terminal.output.read: unexpected response {other:?}"
892 )));
893 }
894 };
895 let exit_code = match page.exit_code {
896 Some(exit_code) => {
897 self.retain_shell_outcome(shell_id, &process_id, pid, Ok(exit_code));
900 self.inner().shells.read(shell_id, |_, entry| {
901 if entry.process_id == process_id {
902 publish_shell_outcome(&entry.exit_tx, Ok(exit_code));
903 }
904 });
905 Some(exit_code)
906 }
907 None => retained_exit.transpose()?,
908 };
909 Ok(TerminalSnapshot {
910 shell_id: shell_id.to_owned(),
911 pid,
912 events: page
913 .events
914 .into_iter()
915 .map(|event| TerminalOutputEvent {
916 sequence: event.sequence,
917 stream: match event.channel {
918 StreamChannel::Stdout => ProcessStream::Stdout,
919 StreamChannel::Stderr => ProcessStream::Stderr,
920 },
921 data: event.chunk,
922 timestamp_ms: event.timestamp_ms.min(i64::MAX as u64) as i64,
923 })
924 .collect(),
925 next_cursor: page.next_cursor,
926 has_more: page.has_more,
927 truncated: page.truncated,
928 exit_code,
929 })
930 }
931
932 pub async fn write_shell_awaited(
936 &self,
937 shell_id: &str,
938 data: StdinInput,
939 ) -> std::result::Result<(), ClientError> {
940 let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
941 let chunk = stdin_chunk(data);
942 tracing::debug!(shell_id, "write_shell_awaited: waiting for spawn gate");
943 wait_for_spawn(spawned_rx).await?;
944 tracing::debug!(shell_id, "write_shell_awaited: issuing wire write");
945 let payload =
946 wire::RequestPayload::WriteStdinRequest(wire::WriteStdinRequest { process_id, chunk });
947 let response = self
948 .transport()
949 .request_wire(self.vm_ownership(), payload)
950 .await?;
951 tracing::debug!(shell_id, "write_shell_awaited: wire write acked");
952 match response {
953 wire::ResponsePayload::RejectedResponse(rejected) => Err(rejected_to_error(rejected)),
954 _ => Ok(()),
955 }
956 }
957
958 pub fn on_shell_data(
963 &self,
964 shell_id: &str,
965 mut handler: impl FnMut(ShellData) + Send + 'static,
966 ) -> std::result::Result<crate::stream::Subscription, ClientError> {
967 let mut rx = self
968 .inner()
969 .shells
970 .read(shell_id, |_, entry| entry.data_tx.subscribe())
971 .ok_or_else(|| ClientError::ShellNotFound(shell_id.to_string()))?;
972 let shell_id = shell_id.to_string();
973 let task = tokio::spawn(async move {
974 loop {
975 match rx.recv().await {
976 Ok(data) => handler(ShellData {
977 shell_id: shell_id.clone(),
978 data,
979 }),
980 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
981 Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
982 }
983 }
984 });
985 Ok(crate::stream::Subscription::new(move || task.abort()))
986 }
987
988 pub fn on_shell_output(
990 &self,
991 shell_id: &str,
992 mut handler: impl FnMut(TerminalOutputEvent) + Send + 'static,
993 ) -> std::result::Result<crate::stream::Subscription, ClientError> {
994 let rx = self
995 .inner()
996 .shells
997 .read(shell_id, |_, entry| entry.event_tx.subscribe());
998 let Some(mut rx) = rx else {
999 self.retained_shell_outcome(shell_id)?;
1002 return Ok(crate::stream::Subscription::noop());
1003 };
1004 let task = tokio::spawn(async move {
1005 loop {
1006 match rx.recv().await {
1007 Ok(event) => handler(event),
1008 Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => {
1009 tracing::warn!(
1010 skipped,
1011 "terminal output subscriber lagged; recover with snapshot_shell"
1012 );
1013 }
1014 Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
1015 }
1016 }
1017 });
1018 Ok(crate::stream::Subscription::new(move || task.abort()))
1019 }
1020
1021 pub fn on_shell_stderr(
1026 &self,
1027 shell_id: &str,
1028 mut handler: impl FnMut(ShellData) + Send + 'static,
1029 ) -> std::result::Result<crate::stream::Subscription, ClientError> {
1030 let mut rx = self
1031 .inner()
1032 .shells
1033 .read(shell_id, |_, entry| entry.stderr_tx.subscribe())
1034 .ok_or_else(|| ClientError::ShellNotFound(shell_id.to_string()))?;
1035 let shell_id = shell_id.to_string();
1036 let task = tokio::spawn(async move {
1037 loop {
1038 match rx.recv().await {
1039 Ok(data) => handler(ShellData {
1040 shell_id: shell_id.clone(),
1041 data,
1042 }),
1043 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
1044 Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
1045 }
1046 }
1047 });
1048 Ok(crate::stream::Subscription::new(move || task.abort()))
1049 }
1050
1051 pub fn on_shell_exit(
1052 &self,
1053 shell_id: &str,
1054 handler: impl FnOnce(ShellExit) + Send + 'static,
1055 ) -> std::result::Result<crate::stream::Subscription, ClientError> {
1056 let rx = self
1057 .inner()
1058 .shells
1059 .read(shell_id, |_, entry| entry.exit_tx.subscribe());
1060 let Some(mut rx) = rx else {
1061 handler(ShellExit {
1062 shell_id: shell_id.to_owned(),
1063 exit_code: self.retained_shell_outcome(shell_id)?,
1064 });
1065 return Ok(crate::stream::Subscription::noop());
1066 };
1067 if let Some(result) = rx.borrow().clone() {
1068 let exit_code = result?;
1069 handler(ShellExit {
1070 shell_id: shell_id.to_string(),
1071 exit_code,
1072 });
1073 return Ok(crate::stream::Subscription::noop());
1074 }
1075 let shell_id = shell_id.to_string();
1076 let task = tokio::spawn(async move {
1077 while rx.changed().await.is_ok() {
1078 if let Some(result) = rx.borrow().clone() {
1079 match result {
1080 Ok(exit_code) => handler(ShellExit {
1081 shell_id,
1082 exit_code,
1083 }),
1084 Err(error) => {
1085 tracing::warn!(?error, %shell_id, "terminal subscription failed before confirmed exit")
1086 }
1087 }
1088 return;
1089 }
1090 }
1091 });
1092 Ok(crate::stream::Subscription::new(move || task.abort()))
1093 }
1094
1095 pub fn resize_shell(
1099 &self,
1100 shell_id: &str,
1101 cols: u16,
1102 rows: u16,
1103 ) -> std::result::Result<(), ClientError> {
1104 let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
1106
1107 let agent = self.clone();
1108 let ownership = self.vm_ownership();
1109 tokio::spawn(async move {
1110 if let Err(error) = wait_for_spawn(spawned_rx).await {
1111 tracing::warn!(?error, "resize_shell launch failed");
1112 return;
1113 }
1114 let payload = wire::RequestPayload::ResizePtyRequest(wire::ResizePtyRequest {
1115 process_id,
1116 cols,
1117 rows,
1118 });
1119 if let Err(error) = agent.transport().request_wire(ownership, payload).await {
1120 tracing::warn!(?error, "resize_shell failed");
1121 }
1122 });
1123
1124 Ok(())
1125 }
1126
1127 pub async fn resize_shell_awaited(
1129 &self,
1130 shell_id: &str,
1131 cols: u16,
1132 rows: u16,
1133 ) -> std::result::Result<(), ClientError> {
1134 let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
1135 wait_for_spawn(spawned_rx).await?;
1136 let response = self
1137 .transport()
1138 .request_wire(
1139 self.vm_ownership(),
1140 wire::RequestPayload::ResizePtyRequest(wire::ResizePtyRequest {
1141 process_id,
1142 cols,
1143 rows,
1144 }),
1145 )
1146 .await?;
1147 match response {
1148 wire::ResponsePayload::PtyResizedResponse(_) => Ok(()),
1149 wire::ResponsePayload::RejectedResponse(rejected) => Err(rejected_to_error(rejected)),
1150 other => Err(ClientError::Sidecar(format!(
1151 "resize shell: unexpected response {other:?}"
1152 ))),
1153 }
1154 }
1155
1156 pub async fn wait_shell(&self, shell_id: &str) -> std::result::Result<i32, ClientError> {
1160 let exit_rx = self
1161 .inner()
1162 .shells
1163 .read(shell_id, |_, entry| entry.exit_tx.subscribe());
1164 let Some(exit_rx) = exit_rx else {
1165 return self.retained_shell_outcome(shell_id);
1167 };
1168 observe_shell_exit(exit_rx)
1170 .await
1171 .unwrap_or_else(|| self.retained_shell_outcome(shell_id))
1172 }
1173
1174 pub fn close_shell(&self, shell_id: &str) -> std::result::Result<(), ClientError> {
1177 let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
1178
1179 self.inner().shells.remove(shell_id);
1182
1183 let agent = self.clone();
1185 let ownership = self.vm_ownership();
1186 tokio::spawn(async move {
1187 if let Err(error) = wait_for_spawn(spawned_rx).await {
1188 tracing::warn!(?error, "close_shell launch failed");
1189 return;
1190 }
1191 let payload = wire::RequestPayload::KillProcessRequest(wire::KillProcessRequest {
1192 process_id,
1193 signal: String::from("SIGTERM"),
1194 });
1195 if let Err(error) = agent.transport().request_wire(ownership, payload).await {
1196 tracing::warn!(?error, "close_shell kill failed");
1197 }
1198 });
1199
1200 Ok(())
1201 }
1202
1203 pub async fn close_shell_awaited(
1205 &self,
1206 shell_id: &str,
1207 ) -> std::result::Result<(), ClientError> {
1208 let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
1209 wait_for_spawn(spawned_rx).await?;
1210 let response = self
1211 .transport()
1212 .request_wire(
1213 self.vm_ownership(),
1214 wire::RequestPayload::KillProcessRequest(wire::KillProcessRequest {
1215 process_id,
1216 signal: String::from("SIGTERM"),
1217 }),
1218 )
1219 .await?;
1220 match response {
1221 wire::ResponsePayload::ProcessKilledResponse(_) => {
1222 self.inner().shells.remove(shell_id);
1223 Ok(())
1224 }
1225 wire::ResponsePayload::RejectedResponse(rejected) => Err(rejected_to_error(rejected)),
1226 other => Err(ClientError::Sidecar(format!(
1227 "close shell: unexpected response {other:?}"
1228 ))),
1229 }
1230 }
1231
1232 fn shell_wire_handle(
1235 &self,
1236 shell_id: &str,
1237 ) -> std::result::Result<(String, ShellSpawnReadiness), ClientError> {
1238 self.inner()
1239 .shells
1240 .read(shell_id, |_, entry| {
1241 (entry.process_id.clone(), entry.spawned_tx.subscribe())
1242 })
1243 .ok_or_else(|| {
1244 self.retained_shell_outcome(shell_id)
1245 .err()
1246 .unwrap_or_else(|| ClientError::ShellNotFound(shell_id.to_owned()))
1247 })
1248 }
1249}
1250
1251async fn wait_for_spawn(
1254 mut spawned_rx: ShellSpawnReadiness,
1255) -> std::result::Result<(), ClientError> {
1256 loop {
1257 if let Some(result) = spawned_rx.borrow_and_update().clone() {
1258 return result;
1259 }
1260 spawned_rx.changed().await.map_err(|_| {
1261 ClientError::Sidecar("terminal launch ended before readiness was confirmed".into())
1262 })?;
1263 }
1264}
1265
1266async fn terminal_process_finished(agent: &AgentOs, pid: u32) -> bool {
1267 match agent.all_processes().await {
1268 Ok(processes) => match processes.into_iter().find(|process| process.pid == pid) {
1269 Some(process) => process.status != ProcessStatus::Running,
1270 None => true,
1271 },
1272 Err(error) => {
1273 tracing::warn!(?error, pid, "terminal process snapshot failed");
1274 false
1275 }
1276 }
1277}
1278
1279#[cfg(test)]
1280mod tests {
1281 #[tokio::test]
1282 async fn terminal_replayed_exit_recovers_waiters_and_retention_without_live_exit() {
1283 for failed in [false, true] {
1284 let failure = ClientError::TerminationFailed {
1285 process_id: "terminal-p".into(),
1286 reason: "terminal event stream lost its exit".into(),
1287 };
1288 let (outcome, _) = watch::channel(failed.then(|| Err(failure.clone())));
1289 let (_event_sender, mut events) = broadcast::channel(1);
1290 let ownership = wire::OwnershipScope::vm("connection", "session", "vm");
1291 let observer = next_shell_event(&mut events, &ownership, &outcome);
1292 tokio::pin!(observer);
1293 tokio::select! {
1294 _ = &mut observer => panic!("there is no confirmed terminal exit"),
1295 _ = tokio::task::yield_now() => {}
1296 }
1297 let mut retained = VecDeque::new();
1298 let entry = |result| ClosedShellEntry {
1299 shell_id: "shell-1".into(),
1300 process_id: "terminal-p".into(),
1301 pid: 42,
1302 result,
1303 };
1304 if failed {
1305 retain_shell_result(&mut retained, entry(Err(failure.clone())));
1306 }
1307 let waiter = observe_shell_exit(outcome.subscribe());
1308 tokio::pin!(waiter);
1309 if !failed {
1310 tokio::select! {
1311 _ = &mut waiter => panic!("wait_shell must wait for a confirmed exit"),
1312 _ = tokio::task::yield_now() => {}
1313 }
1314 }
1315 retain_shell_result(&mut retained, entry(Ok(17)));
1316 publish_shell_outcome(&outcome, Ok(17));
1317 assert_eq!(
1318 tokio::time::timeout(std::time::Duration::from_secs(1), &mut waiter)
1319 .await
1320 .unwrap()
1321 .unwrap()
1322 .unwrap(),
1323 17
1324 );
1325 assert!(
1326 tokio::time::timeout(std::time::Duration::from_secs(1), observer)
1327 .await
1328 .unwrap()
1329 .is_none()
1330 );
1331 let late_waiter = outcome.subscribe();
1332 publish_shell_outcome(&outcome, Ok(17));
1333 publish_shell_outcome(&outcome, Err(failure.clone()));
1334 retain_shell_result(&mut retained, entry(Ok(17)));
1335 retain_shell_result(&mut retained, entry(Err(failure)));
1336 assert!(!late_waiter.has_changed().unwrap());
1337 assert_eq!(observe_shell_exit(late_waiter).await.unwrap().unwrap(), 17);
1338 assert_eq!(
1339 retained.len(),
1340 1,
1341 "repeat replay must not consume retention"
1342 );
1343 assert_eq!(*retained[0].result.as_ref().unwrap(), 17);
1344 }
1345 }
1346
1347 #[tokio::test]
1348 async fn terminal_launch_rejection_is_typed_and_does_not_open_the_io_gate() {
1349 let error = shell_started(wire::ResponsePayload::RejectedResponse(
1350 wire::RejectedResponse {
1351 code: "EACCES".into(),
1352 message: "execution denied".into(),
1353 limit_name: None,
1354 configured_limit: None,
1355 current_usage: None,
1356 requested: None,
1357 unit: None,
1358 scope: None,
1359 vm_id: None,
1360 session_generation: None,
1361 capability_id: None,
1362 operation: None,
1363 configuration_path: None,
1364 retryable: None,
1365 errno: None,
1366 },
1367 ))
1368 .unwrap_err();
1369 assert!(matches!(&error, ClientError::Kernel { code, .. } if code == "EACCES"));
1370 let (sender, receiver) = tokio::sync::watch::channel(None);
1371 sender.send_replace(Some(Err(error)));
1372 assert!(
1373 matches!(wait_for_spawn(receiver).await, Err(ClientError::Kernel { code, .. }) if code == "EACCES")
1374 );
1375 let (sender, receiver) = tokio::sync::watch::channel(None);
1376 drop(sender);
1377 assert!(wait_for_spawn(receiver).await.is_err());
1378 }
1379
1380 use super::*;
1381
1382 #[test]
1383 fn reserve_counter_enforces_limit_and_release_reopens_slot() {
1384 let counter = AtomicUsize::new(0);
1385
1386 assert!(try_reserve_counter(&counter, 2));
1387 assert!(try_reserve_counter(&counter, 2));
1388 assert!(!try_reserve_counter(&counter, 2));
1389 release_counter(&counter);
1390 assert!(try_reserve_counter(&counter, 2));
1391 }
1392}