1use std::collections::{BTreeMap, VecDeque};
19use std::sync::atomic::{AtomicUsize, Ordering};
20
21use anyhow::Result;
22use tokio::sync::{broadcast, watch};
23use uuid::Uuid;
24
25use agentos_sidecar_client::wire::{self, EventPayload, StreamChannel};
26
27use crate::agent_os::{AgentOs, ClosedShellEntry, ShellEntry, TerminalEntry};
28use crate::error::ClientError;
29use crate::process::{
30 install_output_callback, OutputCallback, ProcessStatus, ProcessStream, StdinInput,
31};
32
33const SHELL_DATA_CHANNEL_CAPACITY: usize = 1024;
35
36const TERMINAL_LIMIT: usize = 1024;
38
39const DEFAULT_SHELL_COMMAND: &str = "sh";
42
43type ShellOutcome = Option<std::result::Result<i32, ClientError>>;
44type ShellSpawnReadiness = watch::Receiver<Option<std::result::Result<(), ClientError>>>;
45
46fn publish_shell_outcome(
47 sender: &watch::Sender<ShellOutcome>,
48 result: std::result::Result<i32, ClientError>,
49) {
50 sender.send_if_modified(|outcome| {
51 if matches!(outcome, Some(Ok(_))) {
54 return false;
55 }
56 *outcome = Some(result);
57 true
58 });
59}
60
61fn retain_shell_result(retained: &mut VecDeque<ClosedShellEntry>, entry: ClosedShellEntry) {
62 if let Some(existing) = retained.iter_mut().find(|existing| {
63 existing.shell_id == entry.shell_id && existing.process_id == entry.process_id
64 }) {
65 if existing.result.is_err() {
66 *existing = entry;
67 }
68 return;
69 }
70 retained.push_back(entry);
71 while retained.len() > crate::CLOSED_SHELL_EXIT_CODE_RETENTION_LIMIT {
72 retained.pop_front();
73 }
74}
75
76async fn next_shell_event(
77 events: &mut broadcast::Receiver<(wire::OwnershipScope, EventPayload)>,
78 ownership: &wire::OwnershipScope,
79 outcome: &watch::Sender<ShellOutcome>,
80) -> Option<std::result::Result<EventPayload, broadcast::error::RecvError>> {
81 let mut replay_outcome = outcome.subscribe();
82 loop {
83 if matches!(*replay_outcome.borrow(), Some(Ok(_))) {
84 return None;
85 }
86 tokio::select! {
87 event = events.recv() => match event {
88 Ok((scope, payload)) if scope == *ownership => return Some(Ok(payload)),
89 Ok(_) => continue,
90 Err(error) => return Some(Err(error)),
91 },
92 changed = replay_outcome.changed() => {
93 if let Err(error) = changed {
94 tracing::error!(%error, "terminal outcome channel closed during observation");
95 return None;
96 }
97 }
98 }
99 }
100}
101
102async fn observe_shell_exit(mut receiver: watch::Receiver<ShellOutcome>) -> ShellOutcome {
103 loop {
104 if let Some(result) = receiver.borrow_and_update().clone() {
105 return Some(result);
106 }
107 if receiver.changed().await.is_err() {
108 return None;
109 }
110 }
111}
112
113#[derive(Default)]
119pub struct OpenShellOptions {
120 pub command: Option<String>,
121 pub args: Vec<String>,
122 pub env: BTreeMap<String, String>,
123 pub cwd: Option<String>,
124 pub cols: Option<u16>,
125 pub rows: Option<u16>,
126}
127
128#[derive(Default)]
134pub struct ConnectTerminalOptions {
135 pub base: OpenShellOptions,
136 pub on_data: Option<OutputCallback>,
137 pub on_stderr: Option<OutputCallback>,
138}
139
140#[derive(Debug, Clone, PartialEq, Eq)]
142pub struct ShellHandle {
143 pub shell_id: String,
144}
145
146#[derive(Debug, Clone, PartialEq, Eq)]
147pub struct ShellData {
148 pub shell_id: String,
149 pub data: Vec<u8>,
150}
151
152#[derive(Debug, Clone, PartialEq, Eq)]
153pub struct ShellExit {
154 pub shell_id: String,
155 pub exit_code: i32,
156}
157
158#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
159#[serde(rename_all = "camelCase")]
160pub struct TerminalInfo {
161 pub shell_id: String,
162 pub pid: u32,
163 pub running: bool,
164 pub exit_code: Option<i32>,
165}
166
167#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
168#[serde(rename_all = "camelCase")]
169pub struct TerminalOutputEvent {
170 pub sequence: u64,
171 pub stream: ProcessStream,
172 pub data: Vec<u8>,
173 pub timestamp_ms: i64,
174}
175
176#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
177#[serde(rename_all = "camelCase")]
178pub struct TerminalSnapshot {
179 pub shell_id: String,
180 pub pid: u32,
181 pub events: Vec<TerminalOutputEvent>,
182 pub next_cursor: Option<u64>,
183 pub has_more: bool,
184 pub truncated: bool,
185 pub exit_code: Option<i32>,
186}
187
188fn rejected_to_error(rejected: wire::RejectedResponse) -> ClientError {
194 ClientError::from_rejection(rejected)
195}
196
197fn shell_started(response: wire::ResponsePayload) -> std::result::Result<u32, ClientError> {
198 match response {
199 wire::ResponsePayload::ProcessStartedResponse(wire::ProcessStartedResponse {
200 pid: Some(pid),
201 ..
202 }) => Ok(pid),
203 wire::ResponsePayload::RejectedResponse(rejected) => Err(rejected_to_error(rejected)),
204 other => Err(ClientError::Sidecar(format!(
205 "open_shell: expected a started process with a PID, received {other:?}"
206 ))),
207 }
208}
209
210struct ShellReservation(AgentOs);
211
212impl Drop for ShellReservation {
213 fn drop(&mut self) {
214 release_counter(&self.0.inner().terminal_count);
215 }
216}
217
218fn stdin_chunk(data: StdinInput) -> Vec<u8> {
222 match data {
223 StdinInput::Text(text) => text.into_bytes(),
224 StdinInput::Bytes(bytes) => bytes,
225 }
226}
227
228fn try_reserve_counter(counter: &AtomicUsize, limit: usize) -> bool {
229 counter
230 .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |count| {
231 (count < limit).then_some(count + 1)
232 })
233 .is_ok()
234}
235
236fn release_counter(counter: &AtomicUsize) {
237 let _ = counter.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |count| {
238 Some(count.saturating_sub(1))
239 });
240}
241
242struct TerminalReservation<'a> {
243 agent: &'a AgentOs,
244 active: bool,
245}
246
247impl<'a> TerminalReservation<'a> {
248 fn new(agent: &'a AgentOs) -> std::result::Result<Self, ClientError> {
249 if !try_reserve_counter(&agent.inner().terminal_count, TERMINAL_LIMIT) {
250 return Err(ClientError::ResourceLimit {
251 code: "ERR_AGENTOS_RESOURCE_LIMIT".into(),
252 message: format!("terminal limit {TERMINAL_LIMIT} reached; wait for existing terminals to exit or raise TERMINAL_LIMIT"),
253 details: Box::new(crate::ResourceLimitDetails {
254 limit_name: Some("active_terminals".into()),
255 configured_limit: Some(TERMINAL_LIMIT as u64),
256 requested: Some(TERMINAL_LIMIT as u64 + 1),
257 configuration_path: Some("TERMINAL_LIMIT".into()),
258 unit: Some("terminals".into()),
259 scope: Some("vm".into()),
260 retryable: Some(true),
261 ..Default::default()
262 }),
263 });
264 }
265 let active = agent.inner().terminal_count.load(Ordering::SeqCst);
266 if active >= TERMINAL_LIMIT * 4 / 5 {
267 tracing::warn!(
268 active,
269 limit = TERMINAL_LIMIT,
270 "terminal admission approaches TERMINAL_LIMIT; close unused terminals"
271 );
272 }
273 Ok(Self {
274 agent,
275 active: true,
276 })
277 }
278
279 fn disarm(&mut self) {
280 self.active = false;
281 }
282}
283
284impl Drop for TerminalReservation<'_> {
285 fn drop(&mut self) {
286 if self.active {
287 release_counter(&self.agent.inner().terminal_count);
288 }
289 }
290}
291
292impl AgentOs {
293 fn vm_ownership(&self) -> wire::OwnershipScope {
295 wire::OwnershipScope::VmOwnership(wire::VmOwnership {
296 connection_id: self.connection_id().to_string(),
297 session_id: self.wire_session_id().to_string(),
298 vm_id: self.vm_id().to_string(),
299 })
300 }
301
302 pub(crate) fn finish_terminal(&self, process_id: &str) {
303 if self.inner().terminals.remove(process_id).is_some() {
304 release_counter(&self.inner().terminal_count);
305 }
306 }
307
308 async fn start_terminal(
309 &self,
310 execute: wire::ExecuteRequest,
311 ownership: wire::OwnershipScope,
312 pid_tx: tokio::sync::oneshot::Sender<std::result::Result<u32, ClientError>>,
313 process_id: &str,
314 ) -> Option<u32> {
315 {
316 let _terminal_lifecycle_guard = self.inner().terminal_lifecycle_lock.lock().await;
317 if self.inner().disposed.load(Ordering::SeqCst) {
318 let error = ClientError::Sidecar(
319 "cannot connect terminal after VM shutdown has started".to_string(),
320 );
321 let _ = pid_tx.send(Err(error));
322 self.finish_terminal(process_id);
323 return None;
324 }
325 }
326
327 let result = match self
328 .transport()
329 .request_wire(ownership, wire::RequestPayload::ExecuteRequest(execute))
330 .await
331 {
332 Ok(wire::ResponsePayload::ProcessStartedResponse(wire::ProcessStartedResponse {
333 pid,
334 ..
335 })) => pid.ok_or_else(|| {
336 ClientError::Sidecar("connect_terminal: sidecar did not return a pid".to_string())
337 }),
338 Ok(wire::ResponsePayload::RejectedResponse(rejected)) => {
339 Err(rejected_to_error(rejected))
340 }
341 Ok(other) => Err(ClientError::Sidecar(format!(
342 "unexpected response to connect_terminal: {other:?}"
343 ))),
344 Err(error) => Err(error.into()),
345 };
346
347 match result {
348 Ok(pid) => {
349 let _ = pid_tx.send(Ok(pid));
350 Some(pid)
351 }
352 Err(error) => {
353 let _ = pid_tx.send(Err(error));
354 self.finish_terminal(process_id);
355 None
356 }
357 }
358 }
359}
360
361impl AgentOs {
370 pub fn open_shell(&self, mut options: OpenShellOptions) -> Result<ShellHandle> {
381 let inner = self.inner();
382 if inner.disposed.load(Ordering::SeqCst) {
383 return Err(ClientError::Sidecar(
384 "cannot open terminal after VM shutdown has started".into(),
385 )
386 .into());
387 }
388 let mut reservation = TerminalReservation::new(self)?;
389 let counter = inner.shell_counter.fetch_add(1, Ordering::SeqCst) + 1;
390 let shell_id = format!("shell-{counter}");
391 let process_id = format!("shell-{}", Uuid::new_v4());
393
394 let (data_tx, _) = tokio::sync::broadcast::channel(SHELL_DATA_CHANNEL_CAPACITY);
395 let (stderr_tx, _) = tokio::sync::broadcast::channel(SHELL_DATA_CHANNEL_CAPACITY);
396 let (event_tx, _) = tokio::sync::broadcast::channel(SHELL_DATA_CHANNEL_CAPACITY);
397 let (spawned_tx, _) = tokio::sync::watch::channel(None);
399 let (exit_tx, _) = tokio::sync::watch::channel(None);
401 let entry = ShellEntry {
404 pid: 0,
405 data_tx: data_tx.clone(),
406 stderr_tx: stderr_tx.clone(),
407 event_tx: event_tx.clone(),
408 process_id: process_id.clone(),
409 spawned_tx: spawned_tx.clone(),
410 exit_tx: exit_tx.clone(),
411 };
412 let _ = inner.shells.insert(shell_id.clone(), entry);
414
415 let command = options
416 .command
417 .clone()
418 .unwrap_or_else(|| DEFAULT_SHELL_COMMAND.to_string());
419 options
420 .env
421 .insert(String::from("AGENTOS_EXEC_TTY"), String::from("1"));
422 if let Some(cols) = options.cols {
424 options
425 .env
426 .insert(String::from("COLUMNS"), cols.to_string());
427 }
428 if let Some(rows) = options.rows {
429 options.env.insert(String::from("LINES"), rows.to_string());
430 }
431 let execute = wire::ExecuteRequest {
432 process_id: process_id.clone(),
433 command: Some(command),
434 runtime: None,
435 entrypoint: None,
436 args: options.args.clone(),
437 env: options.env.clone().into_iter().collect(),
438 cwd: options.cwd.clone(),
439 wasm_permission_tier: None,
440 retain_output: true,
441 };
442
443 let agent = self.clone();
447 let ownership = self.vm_ownership();
448 let route_process_id = process_id.clone();
449 let exit_shell_id = shell_id.clone();
450 let exit_key = counter;
451 let (start_tx, start_rx) = tokio::sync::oneshot::channel();
452 let task_reservation = ShellReservation(self.clone());
453 reservation.disarm();
454 let handle = tokio::spawn(async move {
455 let _reservation = task_reservation;
456 if start_rx.await.is_err() {
457 return;
458 }
459 let mut events = agent.transport().subscribe_wire_events();
460
461 let response = match agent
462 .transport()
463 .request_wire(
464 ownership.clone(),
465 wire::RequestPayload::ExecuteRequest(execute),
466 )
467 .await
468 {
469 Ok(response) => shell_started(response),
470 Err(error) => Err(error.into()),
471 };
472 let kernel_pid = match response {
473 Ok(pid) => pid,
474 Err(error) => {
475 tracing::warn!(?error, shell_id = %exit_shell_id, "open_shell spawn failed");
476 agent.retain_shell_outcome(
477 &exit_shell_id,
478 &route_process_id,
479 0,
480 Err(error.clone()),
481 );
482 spawned_tx.send_replace(Some(Err(error.clone())));
483 publish_shell_outcome(&exit_tx, Err(error));
484 agent.inner().shells.remove(&exit_shell_id);
485 agent.inner().pending_shell_exits.remove(&exit_key);
486 return;
487 }
488 };
489
490 agent
493 .inner()
494 .shells
495 .update(&exit_shell_id, |_, existing| existing.pid = kernel_pid);
496 spawned_tx.send_replace(Some(Ok(())));
502
503 while let Some(event) = next_shell_event(&mut events, &ownership, &exit_tx).await {
504 let payload = match event {
505 Ok(value) => value,
506 Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => {
507 tracing::warn!(skipped, shell_id = %exit_shell_id, "terminal live events lost; recover output with terminal.output.read");
508 continue;
509 }
510 Err(tokio::sync::broadcast::error::RecvError::Closed) => {
511 let error = ClientError::TerminationFailed {
512 process_id: route_process_id.clone(),
513 reason: "terminal event stream closed before a confirmed exit".into(),
514 };
515 tracing::warn!(?error, "terminal exit unconfirmed");
516 agent.retain_shell_outcome(
517 &exit_shell_id,
518 &route_process_id,
519 kernel_pid,
520 Err(error.clone()),
521 );
522 publish_shell_outcome(&exit_tx, Err(error));
523 break;
524 }
525 };
526 match payload {
527 EventPayload::ProcessOutputEvent(output) => {
528 if output.process_id != route_process_id {
529 continue;
530 }
531 let stream = match output.channel {
532 StreamChannel::Stdout => ProcessStream::Stdout,
533 StreamChannel::Stderr => ProcessStream::Stderr,
534 };
535 let (Some(sequence), Some(timestamp_ms)) =
536 (output.sequence, output.timestamp_ms)
537 else {
538 tracing::error!(
539 shell_id = %exit_shell_id,
540 "retained terminal output event is missing its replay identity"
541 );
542 continue;
543 };
544 let _ = event_tx.send(TerminalOutputEvent {
545 sequence,
546 stream,
547 data: output.chunk.clone(),
548 timestamp_ms: timestamp_ms.min(i64::MAX as u64) as i64,
549 });
550 let _ = data_tx.send(output.chunk.clone());
553 if output.channel == StreamChannel::Stderr {
554 let _ = stderr_tx.send(output.chunk);
556 }
557 }
558 EventPayload::ProcessExitedEvent(exited) => {
559 if exited.process_id == route_process_id {
560 agent.retain_shell_outcome(
564 &exit_shell_id,
565 &route_process_id,
566 kernel_pid,
567 Ok(exited.exit_code),
568 );
569 publish_shell_outcome(&exit_tx, Ok(exited.exit_code));
570 break;
571 }
572 }
573 EventPayload::VmLifecycleEvent(_)
574 | EventPayload::ExecutionOutputEvent(_)
575 | EventPayload::ExecutionCompletedEvent(_)
576 | EventPayload::StructuredEvent(_)
577 | EventPayload::ExtEnvelope(_) => {}
578 }
579 }
580
581 agent.inner().pending_shell_exits.remove(&exit_key);
584 agent.inner().shells.remove_if(&exit_shell_id, |existing| {
585 existing.process_id == route_process_id
586 });
587 });
589
590 let _ = inner.pending_shell_exits.insert(counter, handle);
591 if start_tx.send(()).is_err() {
592 tracing::warn!(%shell_id, "terminal task closed before launch");
593 }
594
595 Ok(ShellHandle { shell_id })
596 }
597
598 fn retain_shell_outcome(
599 &self,
600 shell_id: &str,
601 process_id: &str,
602 pid: u32,
603 result: std::result::Result<i32, ClientError>,
604 ) {
605 let mut retained = self.inner().closed_shells.lock();
606 retain_shell_result(
607 &mut retained,
608 ClosedShellEntry {
609 shell_id: shell_id.to_owned(),
610 process_id: process_id.to_owned(),
611 pid,
612 result,
613 },
614 );
615 }
616
617 fn retained_shell_entry(&self, shell_id: &str) -> Option<ClosedShellEntry> {
618 self.inner()
619 .closed_shells
620 .lock()
621 .iter()
622 .rev()
623 .find(|entry| entry.shell_id == shell_id)
624 .cloned()
625 }
626
627 fn retained_shell_outcome(&self, shell_id: &str) -> std::result::Result<i32, ClientError> {
628 self.retained_shell_entry(shell_id)
629 .map(|entry| entry.result)
630 .unwrap_or_else(|| Err(ClientError::ShellNotFound(shell_id.to_owned())))
631 }
632
633 pub async fn connect_terminal(&self, options: ConnectTerminalOptions) -> Result<u32> {
642 let ConnectTerminalOptions {
643 base,
644 on_data,
645 on_stderr,
646 } = options;
647
648 let process_id = format!("terminal-{}", Uuid::new_v4());
649 let command = base
650 .command
651 .clone()
652 .unwrap_or_else(|| DEFAULT_SHELL_COMMAND.to_string());
653 let (data_tx, _) = tokio::sync::broadcast::channel::<Vec<u8>>(SHELL_DATA_CHANNEL_CAPACITY);
654 let (stderr_tx, _) =
655 tokio::sync::broadcast::channel::<Vec<u8>>(SHELL_DATA_CHANNEL_CAPACITY);
656
657 if let Some(cb) = on_data {
660 install_output_callback(data_tx.clone(), cb);
661 }
662 if let Some(cb) = on_stderr {
663 install_output_callback(stderr_tx.clone(), cb);
664 }
665
666 let execute = wire::ExecuteRequest {
667 process_id: process_id.clone(),
668 command: Some(command),
669 runtime: None,
670 entrypoint: None,
671 args: base.args.clone(),
672 env: base.env.clone().into_iter().collect(),
673 cwd: base.cwd.clone(),
674 wasm_permission_tier: None,
675 retain_output: true,
676 };
677
678 let events = self.transport().subscribe_wire_events();
680 let ownership = self.vm_ownership();
681 let (pid_tx, pid_rx) = tokio::sync::oneshot::channel();
682 let (start_tx, start_rx) = tokio::sync::oneshot::channel::<()>();
683 let agent = self.clone();
684 let route_process_id = process_id.clone();
685 let exit_task = tokio::spawn(async move {
686 if start_rx.await.is_err() {
687 return;
688 }
689 let terminal_pid = match agent
690 .start_terminal(execute, ownership, pid_tx, &route_process_id)
691 .await
692 {
693 Some(pid) => pid,
694 None => return,
695 };
696 let mut events = events;
697 loop {
698 let (_scope, payload) = match events.recv().await {
699 Ok(value) => value,
700 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
701 if terminal_process_finished(&agent, terminal_pid).await {
702 break;
703 }
704 continue;
705 }
706 Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
707 };
708 match payload {
709 EventPayload::ProcessOutputEvent(output) => {
710 if output.process_id != route_process_id {
711 continue;
712 }
713 let _ = data_tx.send(output.chunk.clone());
714 if output.channel == StreamChannel::Stderr {
715 let _ = stderr_tx.send(output.chunk);
716 }
717 }
718 EventPayload::ProcessExitedEvent(exited) => {
719 if exited.process_id == route_process_id {
720 break;
721 }
722 }
723 EventPayload::VmLifecycleEvent(_)
724 | EventPayload::ExecutionOutputEvent(_)
725 | EventPayload::ExecutionCompletedEvent(_)
726 | EventPayload::StructuredEvent(_)
727 | EventPayload::ExtEnvelope(_) => {}
728 }
729 }
730 agent.finish_terminal(&route_process_id);
731 });
732
733 {
734 let _terminal_lifecycle_guard = self.inner().terminal_lifecycle_lock.lock().await;
735 if self.inner().disposed.load(Ordering::SeqCst) {
736 exit_task.abort();
737 return Err(ClientError::Sidecar(
738 "cannot connect terminal after VM shutdown has started".to_string(),
739 )
740 .into());
741 }
742 let mut terminal_reservation = TerminalReservation::new(self)?;
743 match self
744 .inner()
745 .terminals
746 .insert(process_id.clone(), TerminalEntry { exit_task })
747 {
748 Ok(()) => {}
749 Err((_, entry)) => {
750 entry.exit_task.abort();
751 return Err(ClientError::Sidecar(format!(
752 "terminal process id collision while tracking terminal: {process_id}"
753 ))
754 .into());
755 }
756 }
757 terminal_reservation.disarm();
758 if start_tx.send(()).is_err() {
759 self.finish_terminal(&process_id);
760 return Err(ClientError::Sidecar(
761 "terminal startup task ended before registration completed".to_string(),
762 )
763 .into());
764 }
765 }
766
767 pid_rx
768 .await
769 .map_err(|_| {
770 ClientError::Sidecar(
771 "terminal startup task ended before returning a pid".to_string(),
772 )
773 })?
774 .map_err(Into::into)
775 }
776
777 pub fn write_shell(
779 &self,
780 shell_id: &str,
781 data: StdinInput,
782 ) -> std::result::Result<(), ClientError> {
783 let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
784 let chunk = stdin_chunk(data);
785
786 let agent = self.clone();
791 let ownership = self.vm_ownership();
792 tokio::spawn(async move {
793 if let Err(error) = wait_for_spawn(spawned_rx).await {
794 tracing::warn!(?error, "write_shell launch failed");
795 return;
796 }
797 let payload = wire::RequestPayload::WriteStdinRequest(wire::WriteStdinRequest {
798 process_id,
799 chunk,
800 });
801 if let Err(error) = agent.transport().request_wire(ownership, payload).await {
802 tracing::warn!(?error, "write_shell failed");
803 }
804 });
805
806 Ok(())
807 }
808
809 pub fn list_shells(&self) -> Vec<TerminalInfo> {
811 let mut shells = Vec::new();
812 self.inner().shells.scan(|shell_id, entry| {
813 let outcome = entry.exit_tx.borrow();
814 let exit_code = outcome
815 .as_ref()
816 .and_then(|result| result.as_ref().ok())
817 .copied();
818 shells.push(TerminalInfo {
819 shell_id: shell_id.clone(),
820 pid: entry.pid,
821 running: outcome.is_none(),
822 exit_code,
823 });
824 });
825 shells.sort_by(|left, right| left.shell_id.cmp(&right.shell_id));
826 shells
827 }
828
829 pub async fn snapshot_shell(
831 &self,
832 shell_id: &str,
833 after: Option<u64>,
834 max_bytes: Option<usize>,
835 ) -> std::result::Result<TerminalSnapshot, ClientError> {
836 self.snapshot_shell_page(shell_id, after, None, max_bytes)
837 .await
838 }
839
840 pub async fn snapshot_shell_page(
843 &self,
844 shell_id: &str,
845 after: Option<u64>,
846 max_events: Option<usize>,
847 max_bytes: Option<usize>,
848 ) -> std::result::Result<TerminalSnapshot, ClientError> {
849 let (max_events, max_bytes) =
850 crate::output_replay::wire_page_limits(max_events, max_bytes, "terminal.output.read")?;
851 let live = self.inner().shells.read(shell_id, |_, entry| {
852 (
853 entry.pid,
854 entry.process_id.clone(),
855 entry.exit_tx.borrow().clone(),
856 )
857 });
858 let (pid, process_id, retained_exit) = match live {
859 Some((pid, process_id, exit)) => (pid, process_id, exit),
860 None => {
861 let retained = self
862 .retained_shell_entry(shell_id)
863 .ok_or_else(|| ClientError::ShellNotFound(shell_id.to_owned()))?;
864 (retained.pid, retained.process_id, Some(retained.result))
865 }
866 };
867 let response = self
868 .transport()
869 .request_wire(
870 self.vm_ownership(),
871 wire::RequestPayload::ReadProcessOutputRequest(wire::ReadProcessOutputRequest {
872 process_id: process_id.clone(),
873 after,
874 max_events: u32::try_from(max_events).map_err(|_| {
875 ClientError::Sidecar(String::from(
876 "terminal.output.read maxEvents exceeds the wire u32 range",
877 ))
878 })?,
879 max_bytes: u32::try_from(max_bytes).map_err(|_| {
880 ClientError::Sidecar(String::from(
881 "terminal.output.read maxBytes exceeds the wire u32 range",
882 ))
883 })?,
884 }),
885 )
886 .await?;
887 let page = match response {
888 wire::ResponsePayload::ProcessOutputPageResponse(page) => page,
889 wire::ResponsePayload::RejectedResponse(rejected) => {
890 return Err(rejected_to_error(rejected));
891 }
892 other => {
893 return Err(ClientError::Sidecar(format!(
894 "terminal.output.read: unexpected response {other:?}"
895 )));
896 }
897 };
898 let exit_code = match page.exit_code {
899 Some(exit_code) => {
900 self.retain_shell_outcome(shell_id, &process_id, pid, Ok(exit_code));
903 self.inner().shells.read(shell_id, |_, entry| {
904 if entry.process_id == process_id {
905 publish_shell_outcome(&entry.exit_tx, Ok(exit_code));
906 }
907 });
908 Some(exit_code)
909 }
910 None => retained_exit.transpose()?,
911 };
912 Ok(TerminalSnapshot {
913 shell_id: shell_id.to_owned(),
914 pid,
915 events: page
916 .events
917 .into_iter()
918 .map(|event| TerminalOutputEvent {
919 sequence: event.sequence,
920 stream: match event.channel {
921 StreamChannel::Stdout => ProcessStream::Stdout,
922 StreamChannel::Stderr => ProcessStream::Stderr,
923 },
924 data: event.chunk,
925 timestamp_ms: event.timestamp_ms.min(i64::MAX as u64) as i64,
926 })
927 .collect(),
928 next_cursor: page.next_cursor,
929 has_more: page.has_more,
930 truncated: page.truncated,
931 exit_code,
932 })
933 }
934
935 pub async fn write_shell_awaited(
939 &self,
940 shell_id: &str,
941 data: StdinInput,
942 ) -> std::result::Result<(), ClientError> {
943 let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
944 let chunk = stdin_chunk(data);
945 tracing::debug!(shell_id, "write_shell_awaited: waiting for spawn gate");
946 wait_for_spawn(spawned_rx).await?;
947 tracing::debug!(shell_id, "write_shell_awaited: issuing wire write");
948 let payload =
949 wire::RequestPayload::WriteStdinRequest(wire::WriteStdinRequest { process_id, chunk });
950 let response = self
951 .transport()
952 .request_wire(self.vm_ownership(), payload)
953 .await?;
954 tracing::debug!(shell_id, "write_shell_awaited: wire write acked");
955 match response {
956 wire::ResponsePayload::RejectedResponse(rejected) => Err(rejected_to_error(rejected)),
957 _ => Ok(()),
958 }
959 }
960
961 pub fn on_shell_data(
966 &self,
967 shell_id: &str,
968 mut handler: impl FnMut(ShellData) + Send + 'static,
969 ) -> std::result::Result<crate::stream::Subscription, ClientError> {
970 let mut rx = self
971 .inner()
972 .shells
973 .read(shell_id, |_, entry| entry.data_tx.subscribe())
974 .ok_or_else(|| ClientError::ShellNotFound(shell_id.to_string()))?;
975 let shell_id = shell_id.to_string();
976 let task = tokio::spawn(async move {
977 loop {
978 match rx.recv().await {
979 Ok(data) => handler(ShellData {
980 shell_id: shell_id.clone(),
981 data,
982 }),
983 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
984 Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
985 }
986 }
987 });
988 Ok(crate::stream::Subscription::new(move || task.abort()))
989 }
990
991 pub fn on_shell_output(
993 &self,
994 shell_id: &str,
995 mut handler: impl FnMut(TerminalOutputEvent) + Send + 'static,
996 ) -> std::result::Result<crate::stream::Subscription, ClientError> {
997 let rx = self
998 .inner()
999 .shells
1000 .read(shell_id, |_, entry| entry.event_tx.subscribe());
1001 let Some(mut rx) = rx else {
1002 self.retained_shell_outcome(shell_id)?;
1005 return Ok(crate::stream::Subscription::noop());
1006 };
1007 let task = tokio::spawn(async move {
1008 loop {
1009 match rx.recv().await {
1010 Ok(event) => handler(event),
1011 Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => {
1012 tracing::warn!(
1013 skipped,
1014 "terminal output subscriber lagged; recover with snapshot_shell"
1015 );
1016 }
1017 Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
1018 }
1019 }
1020 });
1021 Ok(crate::stream::Subscription::new(move || task.abort()))
1022 }
1023
1024 pub fn on_shell_stderr(
1029 &self,
1030 shell_id: &str,
1031 mut handler: impl FnMut(ShellData) + Send + 'static,
1032 ) -> std::result::Result<crate::stream::Subscription, ClientError> {
1033 let mut rx = self
1034 .inner()
1035 .shells
1036 .read(shell_id, |_, entry| entry.stderr_tx.subscribe())
1037 .ok_or_else(|| ClientError::ShellNotFound(shell_id.to_string()))?;
1038 let shell_id = shell_id.to_string();
1039 let task = tokio::spawn(async move {
1040 loop {
1041 match rx.recv().await {
1042 Ok(data) => handler(ShellData {
1043 shell_id: shell_id.clone(),
1044 data,
1045 }),
1046 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
1047 Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
1048 }
1049 }
1050 });
1051 Ok(crate::stream::Subscription::new(move || task.abort()))
1052 }
1053
1054 pub fn on_shell_exit(
1055 &self,
1056 shell_id: &str,
1057 handler: impl FnOnce(ShellExit) + Send + 'static,
1058 ) -> std::result::Result<crate::stream::Subscription, ClientError> {
1059 let rx = self
1060 .inner()
1061 .shells
1062 .read(shell_id, |_, entry| entry.exit_tx.subscribe());
1063 let Some(mut rx) = rx else {
1064 handler(ShellExit {
1065 shell_id: shell_id.to_owned(),
1066 exit_code: self.retained_shell_outcome(shell_id)?,
1067 });
1068 return Ok(crate::stream::Subscription::noop());
1069 };
1070 if let Some(result) = rx.borrow().clone() {
1071 let exit_code = result?;
1072 handler(ShellExit {
1073 shell_id: shell_id.to_string(),
1074 exit_code,
1075 });
1076 return Ok(crate::stream::Subscription::noop());
1077 }
1078 let shell_id = shell_id.to_string();
1079 let task = tokio::spawn(async move {
1080 while rx.changed().await.is_ok() {
1081 if let Some(result) = rx.borrow().clone() {
1082 match result {
1083 Ok(exit_code) => handler(ShellExit {
1084 shell_id,
1085 exit_code,
1086 }),
1087 Err(error) => {
1088 tracing::warn!(?error, %shell_id, "terminal subscription failed before confirmed exit")
1089 }
1090 }
1091 return;
1092 }
1093 }
1094 });
1095 Ok(crate::stream::Subscription::new(move || task.abort()))
1096 }
1097
1098 pub fn resize_shell(
1102 &self,
1103 shell_id: &str,
1104 cols: u16,
1105 rows: u16,
1106 ) -> std::result::Result<(), ClientError> {
1107 let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
1109
1110 let agent = self.clone();
1111 let ownership = self.vm_ownership();
1112 tokio::spawn(async move {
1113 if let Err(error) = wait_for_spawn(spawned_rx).await {
1114 tracing::warn!(?error, "resize_shell launch failed");
1115 return;
1116 }
1117 let payload = wire::RequestPayload::ResizePtyRequest(wire::ResizePtyRequest {
1118 process_id,
1119 cols,
1120 rows,
1121 });
1122 if let Err(error) = agent.transport().request_wire(ownership, payload).await {
1123 tracing::warn!(?error, "resize_shell failed");
1124 }
1125 });
1126
1127 Ok(())
1128 }
1129
1130 pub async fn resize_shell_awaited(
1132 &self,
1133 shell_id: &str,
1134 cols: u16,
1135 rows: u16,
1136 ) -> std::result::Result<(), ClientError> {
1137 let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
1138 wait_for_spawn(spawned_rx).await?;
1139 let response = self
1140 .transport()
1141 .request_wire(
1142 self.vm_ownership(),
1143 wire::RequestPayload::ResizePtyRequest(wire::ResizePtyRequest {
1144 process_id,
1145 cols,
1146 rows,
1147 }),
1148 )
1149 .await?;
1150 match response {
1151 wire::ResponsePayload::PtyResizedResponse(_) => Ok(()),
1152 wire::ResponsePayload::RejectedResponse(rejected) => Err(rejected_to_error(rejected)),
1153 other => Err(ClientError::Sidecar(format!(
1154 "resize shell: unexpected response {other:?}"
1155 ))),
1156 }
1157 }
1158
1159 pub async fn wait_shell(&self, shell_id: &str) -> std::result::Result<i32, ClientError> {
1163 let exit_rx = self
1164 .inner()
1165 .shells
1166 .read(shell_id, |_, entry| entry.exit_tx.subscribe());
1167 let Some(exit_rx) = exit_rx else {
1168 return self.retained_shell_outcome(shell_id);
1170 };
1171 observe_shell_exit(exit_rx)
1173 .await
1174 .unwrap_or_else(|| self.retained_shell_outcome(shell_id))
1175 }
1176
1177 pub fn close_shell(&self, shell_id: &str) -> std::result::Result<(), ClientError> {
1180 let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
1181
1182 self.inner().shells.remove(shell_id);
1185
1186 let agent = self.clone();
1188 let ownership = self.vm_ownership();
1189 tokio::spawn(async move {
1190 if let Err(error) = wait_for_spawn(spawned_rx).await {
1191 tracing::warn!(?error, "close_shell launch failed");
1192 return;
1193 }
1194 let payload = wire::RequestPayload::KillProcessRequest(wire::KillProcessRequest {
1195 process_id,
1196 signal: String::from("SIGTERM"),
1197 });
1198 if let Err(error) = agent.transport().request_wire(ownership, payload).await {
1199 tracing::warn!(?error, "close_shell kill failed");
1200 }
1201 });
1202
1203 Ok(())
1204 }
1205
1206 pub async fn close_shell_awaited(
1208 &self,
1209 shell_id: &str,
1210 ) -> std::result::Result<(), ClientError> {
1211 let (process_id, spawned_rx) = self.shell_wire_handle(shell_id)?;
1212 wait_for_spawn(spawned_rx).await?;
1213 let response = self
1214 .transport()
1215 .request_wire(
1216 self.vm_ownership(),
1217 wire::RequestPayload::KillProcessRequest(wire::KillProcessRequest {
1218 process_id,
1219 signal: String::from("SIGTERM"),
1220 }),
1221 )
1222 .await?;
1223 match response {
1224 wire::ResponsePayload::ProcessKilledResponse(_) => {
1225 self.inner().shells.remove(shell_id);
1226 Ok(())
1227 }
1228 wire::ResponsePayload::RejectedResponse(rejected) => Err(rejected_to_error(rejected)),
1229 other => Err(ClientError::Sidecar(format!(
1230 "close shell: unexpected response {other:?}"
1231 ))),
1232 }
1233 }
1234
1235 fn shell_wire_handle(
1238 &self,
1239 shell_id: &str,
1240 ) -> std::result::Result<(String, ShellSpawnReadiness), ClientError> {
1241 self.inner()
1242 .shells
1243 .read(shell_id, |_, entry| {
1244 (entry.process_id.clone(), entry.spawned_tx.subscribe())
1245 })
1246 .ok_or_else(|| {
1247 self.retained_shell_outcome(shell_id)
1248 .err()
1249 .unwrap_or_else(|| ClientError::ShellNotFound(shell_id.to_owned()))
1250 })
1251 }
1252}
1253
1254async fn wait_for_spawn(
1257 mut spawned_rx: ShellSpawnReadiness,
1258) -> std::result::Result<(), ClientError> {
1259 loop {
1260 if let Some(result) = spawned_rx.borrow_and_update().clone() {
1261 return result;
1262 }
1263 spawned_rx.changed().await.map_err(|_| {
1264 ClientError::Sidecar("terminal launch ended before readiness was confirmed".into())
1265 })?;
1266 }
1267}
1268
1269async fn terminal_process_finished(agent: &AgentOs, pid: u32) -> bool {
1270 match agent.all_processes().await {
1271 Ok(processes) => match processes.into_iter().find(|process| process.pid == pid) {
1272 Some(process) => process.status != ProcessStatus::Running,
1273 None => true,
1274 },
1275 Err(error) => {
1276 tracing::warn!(?error, pid, "terminal process snapshot failed");
1277 false
1278 }
1279 }
1280}
1281
1282#[cfg(test)]
1283mod tests {
1284 #[tokio::test]
1285 async fn terminal_replayed_exit_recovers_waiters_and_retention_without_live_exit() {
1286 for failed in [false, true] {
1287 let failure = ClientError::TerminationFailed {
1288 process_id: "terminal-p".into(),
1289 reason: "terminal event stream lost its exit".into(),
1290 };
1291 let (outcome, _) = watch::channel(failed.then(|| Err(failure.clone())));
1292 let (_event_sender, mut events) = broadcast::channel(1);
1293 let ownership = wire::OwnershipScope::vm("connection", "session", "vm");
1294 let observer = next_shell_event(&mut events, &ownership, &outcome);
1295 tokio::pin!(observer);
1296 tokio::select! {
1297 _ = &mut observer => panic!("there is no confirmed terminal exit"),
1298 _ = tokio::task::yield_now() => {}
1299 }
1300 let mut retained = VecDeque::new();
1301 let entry = |result| ClosedShellEntry {
1302 shell_id: "shell-1".into(),
1303 process_id: "terminal-p".into(),
1304 pid: 42,
1305 result,
1306 };
1307 if failed {
1308 retain_shell_result(&mut retained, entry(Err(failure.clone())));
1309 }
1310 let waiter = observe_shell_exit(outcome.subscribe());
1311 tokio::pin!(waiter);
1312 if !failed {
1313 tokio::select! {
1314 _ = &mut waiter => panic!("wait_shell must wait for a confirmed exit"),
1315 _ = tokio::task::yield_now() => {}
1316 }
1317 }
1318 retain_shell_result(&mut retained, entry(Ok(17)));
1319 publish_shell_outcome(&outcome, Ok(17));
1320 assert_eq!(
1321 tokio::time::timeout(std::time::Duration::from_secs(1), &mut waiter)
1322 .await
1323 .unwrap()
1324 .unwrap()
1325 .unwrap(),
1326 17
1327 );
1328 assert!(
1329 tokio::time::timeout(std::time::Duration::from_secs(1), observer)
1330 .await
1331 .unwrap()
1332 .is_none()
1333 );
1334 let late_waiter = outcome.subscribe();
1335 publish_shell_outcome(&outcome, Ok(17));
1336 publish_shell_outcome(&outcome, Err(failure.clone()));
1337 retain_shell_result(&mut retained, entry(Ok(17)));
1338 retain_shell_result(&mut retained, entry(Err(failure)));
1339 assert!(!late_waiter.has_changed().unwrap());
1340 assert_eq!(observe_shell_exit(late_waiter).await.unwrap().unwrap(), 17);
1341 assert_eq!(
1342 retained.len(),
1343 1,
1344 "repeat replay must not consume retention"
1345 );
1346 assert_eq!(*retained[0].result.as_ref().unwrap(), 17);
1347 }
1348 }
1349
1350 #[tokio::test]
1351 async fn terminal_launch_rejection_is_typed_and_does_not_open_the_io_gate() {
1352 let error = shell_started(wire::ResponsePayload::RejectedResponse(
1353 wire::RejectedResponse {
1354 code: "EACCES".into(),
1355 message: "execution denied".into(),
1356 limit_name: None,
1357 configured_limit: None,
1358 current_usage: None,
1359 requested: None,
1360 unit: None,
1361 scope: None,
1362 vm_id: None,
1363 session_generation: None,
1364 capability_id: None,
1365 operation: None,
1366 configuration_path: None,
1367 retryable: None,
1368 errno: None,
1369 },
1370 ))
1371 .unwrap_err();
1372 assert!(matches!(&error, ClientError::Kernel { code, .. } if code == "EACCES"));
1373 let (sender, receiver) = tokio::sync::watch::channel(None);
1374 sender.send_replace(Some(Err(error)));
1375 assert!(
1376 matches!(wait_for_spawn(receiver).await, Err(ClientError::Kernel { code, .. }) if code == "EACCES")
1377 );
1378 let (sender, receiver) = tokio::sync::watch::channel(None);
1379 drop(sender);
1380 assert!(wait_for_spawn(receiver).await.is_err());
1381 }
1382
1383 use super::*;
1384
1385 #[test]
1386 fn reserve_counter_enforces_limit_and_release_reopens_slot() {
1387 let counter = AtomicUsize::new(0);
1388
1389 assert!(try_reserve_counter(&counter, 2));
1390 assert!(try_reserve_counter(&counter, 2));
1391 assert!(!try_reserve_counter(&counter, 2));
1392 release_counter(&counter);
1393 assert!(try_reserve_counter(&counter, 2));
1394 }
1395}