use std::collections::BTreeMap;
use std::io::{self, Read, Write};
use std::net::Shutdown;
use std::os::unix::net::UnixStream;
use std::path::Path;
use std::path::PathBuf;
use std::process::ExitStatus;
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
use std::process::{ChildStdin, Command, Stdio};
use std::sync::atomic::AtomicBool;
use std::sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender, TryRecvError};
use std::sync::{Arc, Mutex};
use std::thread::{self, JoinHandle};
use std::time::{Duration, Instant};
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
use serde::Deserialize;
use serde::Serialize;
const FRAME_REQUEST: u8 = 0xff;
const FRAME_CONTROL: u8 = 0xfe;
const FRAME_STDIN: u8 = 0;
const FRAME_STDOUT: u8 = 1;
const FRAME_STDERR: u8 = 2;
const FRAME_RESIZE: u8 = 3;
const FRAME_SIGNAL: u8 = 4;
const FRAME_EXIT: u8 = 5;
const FRAME_ERROR: u8 = 6;
#[derive(Clone)]
pub struct ExecBuilder {
exec_path: PathBuf,
argv: Vec<String>,
env: BTreeMap<String, String>,
cwd: Option<String>,
tty: bool,
cols: Option<u16>,
rows: Option<u16>,
timeout: Option<Duration>,
stage_files: Vec<StageFile>,
chain: Vec<Vec<String>>,
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
warm_pool: Option<std::sync::Arc<crate::sentry::Pool>>,
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
runsc: Option<RunscExecTarget>,
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
#[derive(Clone, Debug, Serialize, Deserialize)]
#[doc(hidden)]
pub struct RunscExecTarget {
pub runsc_bin: PathBuf,
pub runsc_root: PathBuf,
#[serde(default)]
pub network: Option<String>,
pub container_id: String,
pub image_env: Vec<(String, String)>,
#[serde(default)]
pub agent_path: Option<String>,
pub file_agent_rootfs: Option<PathBuf>,
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
pub(crate) const RUNSC_EXEC_TOKEN_PREFIX: &str = "supermachine-runsc-exec-v1:";
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
pub(crate) const RUNSC_NETWORK_NONE: &str = "none";
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
pub(crate) const RUNSC_NETWORK_SANDBOX: &str = "sandbox";
#[derive(Clone)]
struct StageFile {
path: String,
data: Vec<u8>,
mode: Option<u32>,
}
impl ExecBuilder {
pub fn new(exec_path: PathBuf) -> Self {
Self {
exec_path,
argv: Vec::new(),
env: BTreeMap::new(),
cwd: None,
tty: false,
cols: None,
rows: None,
timeout: None,
stage_files: Vec::new(),
chain: Vec::new(),
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
warm_pool: None,
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
runsc: None,
}
}
#[doc(hidden)]
pub fn from_binding_token(token: impl AsRef<str>) -> io::Result<Self> {
let token = token.as_ref();
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
if let Some(raw) = token.strip_prefix(RUNSC_EXEC_TOKEN_PREFIX) {
let bytes = crate::api::b64_decode(raw)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
let target: RunscExecTarget = serde_json::from_slice(&bytes)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
return Ok(Self::new(PathBuf::from("runsc-exec-token")).with_runsc_exec(target));
}
Ok(Self::new(PathBuf::from(token)))
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
pub(crate) fn with_warm_pool(mut self, pool: std::sync::Arc<crate::sentry::Pool>) -> Self {
self.warm_pool = Some(pool);
self
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
#[doc(hidden)]
pub fn with_runsc_exec(mut self, target: RunscExecTarget) -> Self {
self.runsc = Some(target);
self
}
pub fn argv<I, S>(mut self, argv: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.argv = argv.into_iter().map(Into::into).collect();
self
}
pub fn env(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
self.env.insert(key.into(), value.into());
self
}
pub fn cwd(mut self, path: impl Into<String>) -> Self {
self.cwd = Some(path.into());
self
}
pub fn tty(mut self, on: bool) -> Self {
self.tty = on;
self
}
pub fn winsize(mut self, cols: u16, rows: u16) -> Self {
self.cols = Some(cols);
self.rows = Some(rows);
self
}
pub fn timeout(mut self, d: Duration) -> Self {
self.timeout = Some(d);
self
}
pub fn stage_file(mut self, path: impl Into<String>, bytes: impl Into<Vec<u8>>) -> Self {
self.stage_files.push(StageFile {
path: path.into(),
data: bytes.into(),
mode: None,
});
self
}
pub fn stage_file_mode(
mut self,
path: impl Into<String>,
bytes: impl Into<Vec<u8>>,
mode: u32,
) -> Self {
self.stage_files.push(StageFile {
path: path.into(),
data: bytes.into(),
mode: Some(mode),
});
self
}
pub fn chain<I, S>(mut self, argv: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.chain.push(argv.into_iter().map(Into::into).collect());
self
}
pub fn spawn(self) -> io::Result<ExecChild> {
if self.argv.is_empty() {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"exec: argv is empty",
));
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
if self.runsc.is_some() {
return runsc_spawn(self);
}
spawn(self)
}
pub fn output(self) -> io::Result<ExecOutcome> {
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
if let Some(pool) = &self.warm_pool {
let started = std::time::Instant::now();
let stage_payload: Vec<StageFilePayload> = self
.stage_files
.iter()
.map(|s| StageFilePayload {
path: &s.path,
data_b64: crate::api::b64_encode(&s.data),
mode: s.mode,
})
.collect();
let payload = RequestPayload {
argv: &self.argv,
env: &self.env,
cwd: self.cwd.as_deref(),
tty: false,
cols: None,
rows: None,
stage_files: stage_payload,
chain: &self.chain,
};
let body = serde_json::to_vec(&payload)
.map_err(|e| io::Error::other(format!("sentry exec request encode: {e}")))?;
let (code, out) = pool
.exec_capture_full(&body)
.map_err(|e| io::Error::other(format!("sentry warm-daemon exec: {e}")))?;
return Ok(ExecOutcome {
status: synthesize_exit(code as u32),
stdout: out,
stderr: Vec::new(),
duration: started.elapsed(),
peak_rss_kib: None,
timed_out: false,
});
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
if self.runsc.is_some() {
return runsc_output(self);
}
let (never_tx, never_rx) = channel::<()>();
let result = self.output_with_cancel(never_rx);
drop(never_tx);
result
}
pub fn output_resilient(self, retries: u8) -> io::Result<ExecOutcome> {
if retries == 0 {
return self.output();
}
let mut attempt: u8 = 0;
loop {
let result = self.clone().output();
let setup_failure = result
.as_ref()
.err()
.and_then(|e| e.get_ref())
.and_then(|e| e.downcast_ref::<ExecEof>())
.map(ExecEof::is_setup_failure)
.unwrap_or(false);
if setup_failure && attempt < retries {
attempt += 1;
eprintln!(
"[exec] connection-setup EOF (agent not ready / worker gone) — retry \
{attempt}/{retries}"
);
thread::sleep(Duration::from_millis(50 * attempt as u64));
continue;
}
return result;
}
}
pub fn output_with_cancel(self, cancel: Receiver<()>) -> io::Result<ExecOutcome> {
let timeout = self.timeout;
let t0 = Instant::now();
let mut child = self.spawn()?;
drop(child.stdin());
let mut stdout = child.stdout();
let mut stderr = child.stderr();
let stdout_handle = stdout.take().map(|mut s| {
thread::spawn(move || {
let mut buf = Vec::new();
let _ = s.read_to_end(&mut buf);
buf
})
});
let stderr_handle = stderr.take().map(|mut s| {
thread::spawn(move || {
let mut buf = Vec::new();
let _ = s.read_to_end(&mut buf);
buf
})
});
let (stop_tx, stop_rx) = channel::<()>();
let timed_out = Arc::new(std::sync::atomic::AtomicBool::new(false));
let signaler = child.signaler();
let watchdog = {
let timed_out = Arc::clone(&timed_out);
Some(thread::spawn(move || {
const TICK: Duration = Duration::from_millis(10);
const ABORT_GRACE: Duration = Duration::from_secs(2);
let deadline = timeout.map(|d| Instant::now() + d);
let escalate = |stop_rx: &Receiver<()>| {
let _ = signaler.kill();
match stop_rx.recv_timeout(ABORT_GRACE) {
Ok(()) | Err(RecvTimeoutError::Disconnected) => {
}
Err(RecvTimeoutError::Timeout) => {
let _ = signaler.abort();
}
}
};
loop {
match stop_rx.try_recv() {
Ok(()) | Err(TryRecvError::Disconnected) => return,
Err(TryRecvError::Empty) => {}
}
match cancel.try_recv() {
Ok(()) => {
escalate(&stop_rx);
return;
}
Err(TryRecvError::Disconnected) => {
}
Err(TryRecvError::Empty) => {}
}
if let Some(dl) = deadline {
if Instant::now() >= dl {
timed_out.store(true, std::sync::atomic::Ordering::SeqCst);
if crate::trace::enabled("timeout") {
eprintln!("[exec.timeout] fired at +{:?}", timeout.unwrap());
}
escalate(&stop_rx);
return;
}
}
let sleep_for = deadline
.map(|dl| dl.saturating_duration_since(Instant::now()).min(TICK))
.unwrap_or(TICK);
if sleep_for.is_zero() {
continue;
}
match stop_rx.recv_timeout(sleep_for) {
Ok(()) => return,
Err(RecvTimeoutError::Timeout) => continue,
Err(RecvTimeoutError::Disconnected) => return,
}
}
}))
};
let exit = child.wait_with_rss();
let _ = stop_tx.send(());
if let Some(h) = watchdog {
let _ = h.join();
}
let stdout_bytes = stdout_handle
.map(|h| h.join().unwrap_or_default())
.unwrap_or_default();
let stderr_bytes = stderr_handle
.map(|h| h.join().unwrap_or_default())
.unwrap_or_default();
let (status, peak_rss_kib) = exit?;
Ok(ExecOutcome {
status,
stdout: stdout_bytes,
stderr: stderr_bytes,
duration: t0.elapsed(),
peak_rss_kib,
timed_out: timed_out.load(std::sync::atomic::Ordering::SeqCst),
})
}
}
#[derive(Debug)]
pub struct ExecOutcome {
pub status: ExitStatus,
pub stdout: Vec<u8>,
pub stderr: Vec<u8>,
pub duration: Duration,
pub peak_rss_kib: Option<u64>,
pub timed_out: bool,
}
impl ExecOutcome {
pub fn success(&self) -> bool {
self.status.success() && !self.timed_out
}
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn runsc_output(builder: ExecBuilder) -> io::Result<ExecOutcome> {
let target = builder.runsc.clone().ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidInput,
"runsc_output called without a runsc target",
)
})?;
if builder.argv.is_empty() {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"exec: argv is empty",
));
}
if builder.tty {
if target.agent_path.is_some() {
return runsc_agent_output(builder, target);
}
return Err(io::Error::new(
io::ErrorKind::Unsupported,
"runsc backend does not support TTY exec without a bundled agent",
));
}
if builder.tty || builder.cols.is_some() || builder.rows.is_some() {
return Err(io::Error::new(
io::ErrorKind::Unsupported,
"runsc backend does not support winsize without TTY",
));
}
runsc_stage_files(&target, &builder.stage_files)?;
let started = Instant::now();
let mut stdout = Vec::new();
let mut stderr = Vec::new();
let mut status = synthesize_exit(0);
let mut timed_out = false;
let mut argvs = Vec::with_capacity(1 + builder.chain.len());
argvs.push(builder.argv.clone());
argvs.extend(builder.chain.iter().cloned());
for argv in argvs {
let out = if target.file_agent_rootfs.is_some() {
runsc_file_agent_output_one(&target, &builder, &argv, started)?
} else {
runsc_output_one(&target, &builder, &argv, started)?
};
timed_out |= out.timed_out;
stdout.extend(out.stdout);
stderr.extend(out.stderr);
status = out.status;
if timed_out || !status.success() {
break;
}
}
Ok(ExecOutcome {
status,
stdout,
stderr,
duration: started.elapsed(),
peak_rss_kib: None,
timed_out,
})
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn runsc_agent_output(builder: ExecBuilder, target: RunscExecTarget) -> io::Result<ExecOutcome> {
let started = Instant::now();
runsc_stage_files(&target, &builder.stage_files)?;
let agent_path = target.agent_path.as_deref().ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidInput,
"runsc agent exec target missing agent path",
)
})?;
let mut cmd = Command::new(&target.runsc_bin);
add_runsc_global_flags_with_network(&mut cmd, &target.runsc_root, runsc_network(&target));
cmd.arg("exec")
.arg(&target.container_id)
.arg(agent_path)
.arg("--exec-stdio")
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
let mut child = cmd.spawn().map_err(|e| {
io::Error::new(
e.kind(),
format!(
"runsc exec {} {} --exec-stdio: {e}",
target.container_id, agent_path
),
)
})?;
let stage_payload: Vec<StageFilePayload> = builder
.stage_files
.iter()
.map(|s| StageFilePayload {
path: &s.path,
data_b64: crate::api::b64_encode(&s.data),
mode: s.mode,
})
.collect();
let payload = RequestPayload {
argv: &builder.argv,
env: &builder.env,
cwd: builder.cwd.as_deref(),
tty: builder.tty,
cols: builder.cols,
rows: builder.rows,
stage_files: stage_payload,
chain: &builder.chain,
};
let json = serde_json::to_vec(&payload).map_err(|e| {
io::Error::new(
io::ErrorKind::InvalidInput,
format!("runsc agent exec: encode REQUEST: {e}"),
)
})?;
{
let stdin = child.stdin.as_mut().ok_or_else(|| {
io::Error::new(io::ErrorKind::Other, "runsc agent exec stdin pipe missing")
})?;
write_raw_frame(stdin, FRAME_REQUEST, &json)?;
write_raw_frame(stdin, FRAME_STDIN, &[])?;
}
let mut child_stdout = child.stdout.take().ok_or_else(|| {
io::Error::new(io::ErrorKind::Other, "runsc agent exec stdout pipe missing")
})?;
let mut child_stderr = child.stderr.take().ok_or_else(|| {
io::Error::new(io::ErrorKind::Other, "runsc agent exec stderr pipe missing")
})?;
let stderr_handle = thread::Builder::new()
.name("supermachine-runsc-agent-output-stderr".into())
.spawn(move || {
let mut buf = Vec::new();
let _ = child_stderr.read_to_end(&mut buf);
buf
})
.map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("runsc agent output stderr thread: {e}"),
)
})?;
let mut stdout = Vec::new();
let mut stderr = Vec::new();
let mut status = synthesize_exit(0);
let done = Arc::new(AtomicBool::new(false));
let timed_out = Arc::new(AtomicBool::new(false));
let watchdog = builder.timeout.map(|timeout| {
let done = Arc::clone(&done);
let timed_out = Arc::clone(&timed_out);
let child_pid = child.id() as i32;
thread::spawn(move || {
let elapsed = started.elapsed();
if elapsed < timeout {
thread::sleep(timeout - elapsed);
}
if !done.load(std::sync::atomic::Ordering::SeqCst) {
timed_out.store(true, std::sync::atomic::Ordering::SeqCst);
let _ = unsafe { libc::kill(child_pid, libc::SIGKILL) };
}
})
});
loop {
let (kind, payload) = match read_frame(&mut child_stdout) {
Ok(frame) => frame,
Err(e) => {
if timed_out.load(std::sync::atomic::Ordering::SeqCst) {
break;
}
let _ = child.kill();
let _ = child.wait();
done.store(true, std::sync::atomic::Ordering::SeqCst);
if let Some(h) = watchdog {
let _ = h.join();
}
let stderr = stderr_handle.join().unwrap_or_default();
if !stderr.is_empty() {
return Err(io::Error::new(
e.kind(),
format!(
"runsc agent exec frame read failed: {e}; runsc stderr={}",
String::from_utf8_lossy(&stderr)
),
));
}
return Err(e);
}
};
match kind {
FRAME_STDOUT => stdout.extend_from_slice(&payload),
FRAME_STDERR => stderr.extend_from_slice(&payload),
FRAME_EXIT => {
match decode_exit_payload(&payload) {
DemuxExit::Status { code, .. } => status = synthesize_exit(code),
DemuxExit::Error(msg) => {
let _ = child.kill();
let _ = child.wait();
done.store(true, std::sync::atomic::Ordering::SeqCst);
if let Some(h) = watchdog {
let _ = h.join();
}
let mut runsc_stderr = stderr_handle.join().unwrap_or_default();
stderr.append(&mut runsc_stderr);
return Err(io::Error::new(io::ErrorKind::InvalidData, msg));
}
_ => {}
}
break;
}
FRAME_ERROR => {
let msg = String::from_utf8_lossy(&payload).into_owned();
let _ = child.kill();
let _ = child.wait();
done.store(true, std::sync::atomic::Ordering::SeqCst);
if let Some(h) = watchdog {
let _ = h.join();
}
let mut runsc_stderr = stderr_handle.join().unwrap_or_default();
stderr.append(&mut runsc_stderr);
return Err(io::Error::new(io::ErrorKind::Other, msg));
}
_ => {}
}
}
drop(child.stdin.take());
let host_status = child.wait()?;
done.store(true, std::sync::atomic::Ordering::SeqCst);
if let Some(h) = watchdog {
let _ = h.join();
}
let mut runsc_stderr = stderr_handle.join().unwrap_or_default();
stderr.append(&mut runsc_stderr);
if timed_out.load(std::sync::atomic::Ordering::SeqCst) {
status = synthesize_exit(runsc_status_code(host_status));
}
Ok(ExecOutcome {
status,
stdout,
stderr,
duration: started.elapsed(),
peak_rss_kib: None,
timed_out: timed_out.load(std::sync::atomic::Ordering::SeqCst),
})
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn runsc_file_agent_output_one(
target: &RunscExecTarget,
builder: &ExecBuilder,
argv: &[String],
started: Instant,
) -> io::Result<ExecOutcome> {
let rootfs = target.file_agent_rootfs.as_ref().ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidInput,
"runsc file-agent exec target missing rootfs",
)
})?;
let req_dir = rootfs
.join(".supermachine")
.join("runsc-exec")
.join("requests")
.join(format!(
"req-{}-{}",
std::process::id(),
unique_suffix_nanos()
));
std::fs::create_dir_all(&req_dir)?;
let mut script = String::from("set -e\n");
if let Some(cwd) = builder.cwd.as_deref() {
script.push_str("cd ");
script.push_str(&sh_quote(cwd));
script.push('\n');
}
let mut env = BTreeMap::new();
for (k, v) in &target.image_env {
env.insert(k.clone(), v.clone());
}
for (k, v) in &builder.env {
env.insert(k.clone(), v.clone());
}
for (k, v) in env {
script.push_str("export ");
script.push_str(&k);
script.push('=');
script.push_str(&sh_quote(&v));
script.push('\n');
}
script.push_str("exec");
for arg in argv {
script.push(' ');
script.push_str(&sh_quote(arg));
}
script.push('\n');
std::fs::write(req_dir.join("script"), script)?;
std::fs::write(req_dir.join("ready"), b"1\n")?;
let mut timed_out = false;
loop {
if req_dir.join("done").is_file() {
break;
}
if let Some(timeout) = builder.timeout {
if started.elapsed() >= timeout {
timed_out = true;
break;
}
}
thread::sleep(Duration::from_millis(20));
}
let stdout = std::fs::read(req_dir.join("stdout")).unwrap_or_default();
let stderr = std::fs::read(req_dir.join("stderr")).unwrap_or_default();
let code = std::fs::read_to_string(req_dir.join("status"))
.ok()
.and_then(|s| s.trim().parse::<u32>().ok())
.unwrap_or(if timed_out { 124 } else { 1 });
let _ = std::fs::remove_dir_all(&req_dir);
Ok(ExecOutcome {
status: synthesize_exit(code),
stdout,
stderr,
duration: started.elapsed(),
peak_rss_kib: None,
timed_out,
})
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn runsc_output_one(
target: &RunscExecTarget,
builder: &ExecBuilder,
argv: &[String],
started: Instant,
) -> io::Result<ExecOutcome> {
let mut cmd = Command::new(&target.runsc_bin);
add_runsc_global_flags_with_network(&mut cmd, &target.runsc_root, runsc_network(target));
cmd.arg("exec");
if let Some(cwd) = builder.cwd.as_deref() {
cmd.arg("-cwd").arg(cwd);
}
let mut env = BTreeMap::new();
for (k, v) in &target.image_env {
env.insert(k.clone(), v.clone());
}
for (k, v) in &builder.env {
env.insert(k.clone(), v.clone());
}
for (k, v) in env {
cmd.arg("-env").arg(format!("{k}={v}"));
}
cmd.arg(&target.container_id);
cmd.args(argv);
cmd.stdin(Stdio::null());
cmd.stdout(Stdio::piped());
cmd.stderr(Stdio::piped());
let mut child = cmd.spawn().map_err(|e| {
io::Error::new(e.kind(), format!("runsc exec {}: {e}", target.container_id))
})?;
let mut timed_out = false;
loop {
if child.try_wait()?.is_some() {
break;
}
if let Some(timeout) = builder.timeout {
if started.elapsed() >= timeout {
timed_out = true;
let _ = child.kill();
break;
}
}
thread::sleep(Duration::from_millis(10));
}
let out = child.wait_with_output()?;
Ok(ExecOutcome {
status: out.status,
stdout: out.stdout,
stderr: out.stderr,
duration: started.elapsed(),
peak_rss_kib: None,
timed_out,
})
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn runsc_spawn(builder: ExecBuilder) -> io::Result<ExecChild> {
let target = builder.runsc.clone().ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidInput,
"runsc_spawn called without a runsc target",
)
})?;
if builder.argv.is_empty() {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"exec: argv is empty",
));
}
if target.agent_path.is_some() {
return runsc_agent_spawn(builder, target);
}
if builder.tty || builder.cols.is_some() || builder.rows.is_some() {
return Err(io::Error::new(
io::ErrorKind::Unsupported,
"runsc backend does not support TTY exec yet",
));
}
runsc_stage_files(&target, &builder.stage_files)?;
let argv = runsc_spawn_argv(&builder);
let mut cmd = runsc_exec_command(&target, &builder, &argv);
cmd.stdin(Stdio::piped());
cmd.stdout(Stdio::piped());
cmd.stderr(Stdio::piped());
let mut child = cmd.spawn().map_err(|e| {
io::Error::new(e.kind(), format!("runsc exec {}: {e}", target.container_id))
})?;
let child_pid = child.id() as i32;
let child_stdin = Arc::new(Mutex::new(child.stdin.take()));
let child_stdout = child
.stdout
.take()
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "runsc exec stdout pipe missing"))?;
let child_stderr = child
.stderr
.take()
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "runsc exec stderr pipe missing"))?;
let (host_sock, bridge_sock) = UnixStream::pair()?;
let sock_r = host_sock.try_clone()?;
let interrupt_fd = Arc::new(host_sock.try_clone()?);
let sock_w = Arc::new(Mutex::new(host_sock));
let (stdout_tx, stdout_r) = channel::<Vec<u8>>();
let (stderr_tx, stderr_r) = channel::<Vec<u8>>();
let (exit_tx, exit_rx) = channel::<DemuxExit>();
let demux = thread::Builder::new()
.name("supermachine-runsc-exec-demux".into())
.spawn(move || {
demux_loop(sock_r, stdout_tx, stderr_tx, exit_tx);
})
.map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("runsc exec: demux thread: {e}"),
)
})?;
let bridge_w = Arc::new(Mutex::new(bridge_sock.try_clone()?));
let control_stdin = Arc::clone(&child_stdin);
let process_done = Arc::new(AtomicBool::new(false));
let control_done = Arc::clone(&process_done);
let mut control_r = bridge_sock.try_clone()?;
thread::Builder::new()
.name("supermachine-runsc-exec-control".into())
.spawn(move || {
runsc_spawn_control_loop(&mut control_r, child_pid, control_stdin, control_done);
})
.map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("runsc exec: control thread: {e}"),
)
})?;
let stdout_pump = runsc_spawn_pump(
"supermachine-runsc-exec-stdout",
child_stdout,
FRAME_STDOUT,
Arc::clone(&bridge_w),
)?;
let stderr_pump = runsc_spawn_pump(
"supermachine-runsc-exec-stderr",
child_stderr,
FRAME_STDERR,
Arc::clone(&bridge_w),
)?;
thread::Builder::new()
.name("supermachine-runsc-exec-wait".into())
.spawn(move || {
drop(child_stdin);
let status = child.wait();
let payload = match status {
Ok(status) => runsc_status_code(status).to_be_bytes(),
Err(_) => 1u32.to_be_bytes(),
};
process_done.store(true, std::sync::atomic::Ordering::SeqCst);
let _ = send_frame(&bridge_w, FRAME_EXIT, &payload);
if let Ok(sock) = bridge_w.lock() {
let _ = sock.shutdown(Shutdown::Both);
}
let _ = stdout_pump.join();
let _ = stderr_pump.join();
})
.map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("runsc exec: wait thread: {e}"),
)
})?;
let stdin = ExecStdin::new(sock_w.clone());
Ok(ExecChild {
sock_w,
we_signaled: Arc::new(AtomicBool::new(false)),
aborted: Arc::new(AtomicBool::new(false)),
interrupt_fd,
stdout_r: Some(stdout_r),
stderr_r: Some(stderr_r),
stdin: Some(stdin),
demux: Some(demux),
exit_rx,
})
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn runsc_agent_spawn(builder: ExecBuilder, target: RunscExecTarget) -> io::Result<ExecChild> {
let debug = std::env::var_os("SUPERMACHINE_RUNSC_EXECDBG").is_some();
let agent_path = target.agent_path.as_deref().ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidInput,
"runsc agent exec target missing agent path",
)
})?;
let mut cmd = Command::new(&target.runsc_bin);
add_runsc_global_flags_with_network(&mut cmd, &target.runsc_root, runsc_network(&target));
cmd.arg("exec")
.arg(&target.container_id)
.arg(agent_path)
.arg("--exec-stdio");
cmd.stdin(Stdio::piped());
cmd.stdout(Stdio::piped());
cmd.stderr(Stdio::piped());
if debug {
eprintln!(
"[runsc.execdbg] spawn agent container={} path={} argv0={}",
target.container_id,
agent_path,
builder.argv.first().map(String::as_str).unwrap_or("")
);
}
let mut child = cmd.spawn().map_err(|e| {
io::Error::new(
e.kind(),
format!(
"runsc exec {} {} --exec-stdio: {e}",
target.container_id, agent_path
),
)
})?;
let child_pid = child.id() as i32;
let mut child_stdin = child.stdin.take().ok_or_else(|| {
io::Error::new(io::ErrorKind::Other, "runsc agent exec stdin pipe missing")
})?;
let mut child_stdout = child.stdout.take().ok_or_else(|| {
io::Error::new(io::ErrorKind::Other, "runsc agent exec stdout pipe missing")
})?;
let child_stderr = child.stderr.take().ok_or_else(|| {
io::Error::new(io::ErrorKind::Other, "runsc agent exec stderr pipe missing")
})?;
let (host_sock, bridge_sock) = UnixStream::pair()?;
let sock_r = host_sock.try_clone()?;
let interrupt_fd = Arc::new(host_sock.try_clone()?);
let sock_w = Arc::new(Mutex::new(host_sock));
let (stdout_tx, stdout_r) = channel::<Vec<u8>>();
let (stderr_tx, stderr_r) = channel::<Vec<u8>>();
let (exit_tx, exit_rx) = channel::<DemuxExit>();
let demux = thread::Builder::new()
.name("supermachine-runsc-agent-demux".into())
.spawn(move || {
demux_loop(sock_r, stdout_tx, stderr_tx, exit_tx);
})
.map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("runsc agent exec: demux thread: {e}"),
)
})?;
let bridge_w = Arc::new(Mutex::new(bridge_sock.try_clone()?));
let mut bridge_to_agent = bridge_sock.try_clone()?;
let process_done = Arc::new(AtomicBool::new(false));
let pump_done = Arc::clone(&process_done);
let debug_c2g = debug;
thread::Builder::new()
.name("supermachine-runsc-agent-c2g".into())
.spawn(move || {
let mut copied: u64 = 0;
let mut buf = [0u8; 8192];
let copied_result: io::Result<u64> = loop {
match bridge_to_agent.read(&mut buf) {
Ok(0) => break Ok(copied),
Ok(n) => {
copied += n as u64;
if debug_c2g {
eprintln!("[runsc.execdbg] c2g read n={n} total={copied}");
}
if let Err(e) = child_stdin.write_all(&buf[..n]) {
break Err(e);
}
if let Err(e) = child_stdin.flush() {
break Err(e);
}
}
Err(e) => break Err(e),
}
};
if debug_c2g {
eprintln!("[runsc.execdbg] c2g done copied={copied_result:?}");
}
let _ = child_stdin.flush();
drop(child_stdin);
if !pump_done.load(std::sync::atomic::Ordering::SeqCst) {
if debug_c2g {
eprintln!("[runsc.execdbg] c2g killing host runsc exec pid={child_pid}");
}
let _ = unsafe { libc::kill(child_pid, libc::SIGKILL) };
}
})
.map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("runsc agent exec: stdin bridge thread: {e}"),
)
})?;
let stdout_bridge_w = Arc::clone(&bridge_w);
let debug_g2c = debug;
let stdout_pump = thread::Builder::new()
.name("supermachine-runsc-agent-g2c".into())
.spawn(move || {
let mut buf = [0u8; 8192];
let mut total: u64 = 0;
loop {
match child_stdout.read(&mut buf) {
Ok(0) => {
if debug_g2c {
eprintln!("[runsc.execdbg] g2c eof total={total}");
}
return;
}
Ok(n) => {
total += n as u64;
if debug_g2c {
eprintln!("[runsc.execdbg] g2c read n={n} total={total}");
}
let mut guard = match stdout_bridge_w.lock() {
Ok(g) => g,
Err(_) => return,
};
if guard.write_all(&buf[..n]).is_err() {
return;
}
}
Err(e) => {
if debug_g2c {
eprintln!("[runsc.execdbg] g2c read error {e}");
}
return;
}
}
}
})
.map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("runsc agent exec: stdout bridge thread: {e}"),
)
})?;
let stderr_pump = runsc_spawn_pump(
"supermachine-runsc-agent-stderr",
child_stderr,
FRAME_STDERR,
Arc::clone(&bridge_w),
)?;
thread::Builder::new()
.name("supermachine-runsc-agent-wait".into())
.spawn(move || {
let status = child.wait();
if debug {
eprintln!("[runsc.execdbg] wait status={status:?}");
}
process_done.store(true, std::sync::atomic::Ordering::SeqCst);
if let Ok(sock) = bridge_w.lock() {
let _ = sock.shutdown(Shutdown::Both);
}
let _ = stdout_pump.join();
let _ = stderr_pump.join();
})
.map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("runsc agent exec: wait thread: {e}"),
)
})?;
let stage_payload: Vec<StageFilePayload> = builder
.stage_files
.iter()
.map(|s| StageFilePayload {
path: &s.path,
data_b64: crate::api::b64_encode(&s.data),
mode: s.mode,
})
.collect();
let payload = RequestPayload {
argv: &builder.argv,
env: &builder.env,
cwd: builder.cwd.as_deref(),
tty: builder.tty,
cols: builder.cols,
rows: builder.rows,
stage_files: stage_payload,
chain: &builder.chain,
};
let json = serde_json::to_vec(&payload).map_err(|e| {
io::Error::new(
io::ErrorKind::InvalidInput,
format!("runsc agent exec: encode REQUEST: {e}"),
)
})?;
send_frame(&sock_w, FRAME_REQUEST, &json)?;
let stdin = ExecStdin::new(sock_w.clone());
Ok(ExecChild {
sock_w,
we_signaled: Arc::new(AtomicBool::new(false)),
aborted: Arc::new(AtomicBool::new(false)),
interrupt_fd,
stdout_r: Some(stdout_r),
stderr_r: Some(stderr_r),
stdin: Some(stdin),
demux: Some(demux),
exit_rx,
})
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn runsc_exec_command(target: &RunscExecTarget, builder: &ExecBuilder, argv: &[String]) -> Command {
let mut cmd = Command::new(&target.runsc_bin);
add_runsc_global_flags_with_network(&mut cmd, &target.runsc_root, runsc_network(target));
cmd.arg("exec");
if let Some(cwd) = builder.cwd.as_deref() {
cmd.arg("-cwd").arg(cwd);
}
let mut env = BTreeMap::new();
for (k, v) in &target.image_env {
env.insert(k.clone(), v.clone());
}
for (k, v) in &builder.env {
env.insert(k.clone(), v.clone());
}
for (k, v) in env {
cmd.arg("-env").arg(format!("{k}={v}"));
}
cmd.arg(&target.container_id);
cmd.args(argv);
cmd
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn runsc_spawn_argv(builder: &ExecBuilder) -> Vec<String> {
if builder.chain.is_empty() {
return builder.argv.clone();
}
let mut commands = Vec::with_capacity(1 + builder.chain.len());
commands.push(shell_join_argv(&builder.argv));
for argv in &builder.chain {
commands.push(shell_join_argv(argv));
}
vec!["/bin/sh".into(), "-c".into(), commands.join(" && ")]
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn shell_join_argv(argv: &[String]) -> String {
argv.iter()
.map(|arg| sh_quote(arg))
.collect::<Vec<_>>()
.join(" ")
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn runsc_stage_files(target: &RunscExecTarget, files: &[StageFile]) -> io::Result<()> {
for file in files {
let q_path = sh_quote(&file.path);
let mode = file.mode.unwrap_or(0o644);
let script = format!(
"set -e\n\
p={q_path}\n\
d=$(dirname -- \"$p\")\n\
mkdir -p -- \"$d\"\n\
t=\"$d/.supermachine-stage-$$-$(basename -- \"$p\")\"\n\
cat > \"$t\"\n\
chmod {mode:o} \"$t\"\n\
mv -f -- \"$t\" \"$p\"\n"
);
let mut cmd = Command::new(&target.runsc_bin);
add_runsc_global_flags_with_network(&mut cmd, &target.runsc_root, runsc_network(target));
cmd.arg("exec")
.arg(&target.container_id)
.arg("/bin/sh")
.arg("-c")
.arg(script);
cmd.stdin(Stdio::piped());
cmd.stdout(Stdio::piped());
cmd.stderr(Stdio::piped());
let mut child = cmd.spawn().map_err(|e| {
io::Error::new(
e.kind(),
format!(
"runsc stage_file {} in {}: {e}",
file.path, target.container_id
),
)
})?;
if let Some(mut stdin) = child.stdin.take() {
stdin.write_all(&file.data)?;
}
let out = child.wait_with_output()?;
if !out.status.success() {
return Err(io::Error::other(format!(
"runsc stage_file {} failed: status={} stdout={} stderr={}",
file.path,
out.status,
String::from_utf8_lossy(&out.stdout),
String::from_utf8_lossy(&out.stderr)
)));
}
}
Ok(())
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn runsc_spawn_control_loop(
sock: &mut UnixStream,
child_pid: i32,
child_stdin: Arc<Mutex<Option<ChildStdin>>>,
process_done: Arc<AtomicBool>,
) {
loop {
let (kind, payload) = match read_frame(sock) {
Ok(frame) => frame,
Err(_) => {
if !process_done.load(std::sync::atomic::Ordering::SeqCst) {
let _ = unsafe { libc::kill(child_pid, libc::SIGKILL) };
}
return;
}
};
match kind {
FRAME_STDIN => {
let mut guard = match child_stdin.lock() {
Ok(g) => g,
Err(_) => return,
};
if payload.is_empty() {
guard.take();
} else if let Some(stdin) = guard.as_mut() {
let _ = stdin.write_all(&payload);
let _ = stdin.flush();
}
}
FRAME_SIGNAL => {
if let Some(signum) = payload.first().copied() {
let _ = unsafe { libc::kill(child_pid, signum as i32) };
}
}
FRAME_RESIZE => {}
_ => {}
}
}
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn runsc_spawn_pump<R>(
name: &'static str,
mut reader: R,
kind: u8,
bridge_w: Arc<Mutex<UnixStream>>,
) -> io::Result<JoinHandle<()>>
where
R: Read + Send + 'static,
{
thread::Builder::new()
.name(name.into())
.spawn(move || {
let mut buf = [0u8; 8192];
loop {
match reader.read(&mut buf) {
Ok(0) => return,
Ok(n) => {
if send_frame(&bridge_w, kind, &buf[..n]).is_err() {
return;
}
}
Err(_) => return,
}
}
})
.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("runsc exec: {name}: {e}")))
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn write_raw_frame<W: Write>(writer: &mut W, kind: u8, payload: &[u8]) -> io::Result<()> {
let mut header = [0u8; 5];
header[0] = kind;
header[1..5].copy_from_slice(&(payload.len() as u32).to_be_bytes());
writer.write_all(&header)?;
if !payload.is_empty() {
writer.write_all(payload)?;
}
writer.flush()
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn runsc_status_code(status: ExitStatus) -> u32 {
#[cfg(unix)]
{
use std::os::unix::process::ExitStatusExt;
if let Some(code) = status.code() {
return code as u32;
}
if let Some(signal) = status.signal() {
return 128 + signal as u32;
}
}
1
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn sh_quote(s: &str) -> String {
let mut out = String::from("'");
for ch in s.chars() {
if ch == '\'' {
out.push_str("'\\''");
} else {
out.push(ch);
}
}
out.push('\'');
out
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn unique_suffix_nanos() -> u128 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0)
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
pub(crate) fn add_runsc_global_flags(cmd: &mut Command, root: &Path) {
add_runsc_global_flags_with_network(cmd, root, RUNSC_NETWORK_NONE);
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
pub(crate) fn add_runsc_global_flags_with_network(cmd: &mut Command, root: &Path, network: &str) {
cmd.arg("--root")
.arg(root)
.arg("--platform=systrap")
.arg(format!("--network={network}"))
.arg("--overlay2=none")
.arg("--file-access=shared");
if network == RUNSC_NETWORK_SANDBOX {
cmd.arg("--reproduce-nat").arg("--reproduce-nftables");
}
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn runsc_network(target: &RunscExecTarget) -> &str {
target.network.as_deref().unwrap_or(RUNSC_NETWORK_NONE)
}
#[derive(Serialize)]
struct RequestPayload<'a> {
argv: &'a [String],
env: &'a BTreeMap<String, String>,
#[serde(skip_serializing_if = "Option::is_none")]
cwd: Option<&'a str>,
tty: bool,
#[serde(skip_serializing_if = "Option::is_none")]
cols: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")]
rows: Option<u16>,
#[serde(skip_serializing_if = "Vec::is_empty")]
stage_files: Vec<StageFilePayload<'a>>,
#[serde(skip_serializing_if = "<[_]>::is_empty")]
chain: &'a [Vec<String>],
}
#[derive(Serialize)]
struct StageFilePayload<'a> {
path: &'a str,
data_b64: String,
#[serde(skip_serializing_if = "Option::is_none")]
mode: Option<u32>,
}
pub struct ExecChild {
sock_w: Arc<Mutex<UnixStream>>,
we_signaled: Arc<AtomicBool>,
aborted: Arc<AtomicBool>,
interrupt_fd: Arc<UnixStream>,
stdout_r: Option<Receiver<Vec<u8>>>,
stderr_r: Option<Receiver<Vec<u8>>>,
stdin: Option<ExecStdin>,
demux: Option<JoinHandle<()>>,
exit_rx: Receiver<DemuxExit>,
}
#[derive(Clone)]
pub struct ExecSignaler {
sock_w: Arc<Mutex<UnixStream>>,
interrupt_fd: Arc<UnixStream>,
we_signaled: Arc<AtomicBool>,
aborted: Arc<AtomicBool>,
}
impl std::fmt::Debug for ExecSignaler {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ExecSignaler").finish_non_exhaustive()
}
}
impl ExecSignaler {
pub fn signal(&self, signum: i32) -> io::Result<()> {
self.we_signaled
.store(true, std::sync::atomic::Ordering::SeqCst);
let payload = [signum as u8];
send_frame(&self.sock_w, FRAME_SIGNAL, &payload)
}
pub fn kill(&self) -> io::Result<()> {
self.signal(libc::SIGKILL)
}
pub fn abort(&self) -> io::Result<()> {
self.aborted
.store(true, std::sync::atomic::Ordering::SeqCst);
match self.interrupt_fd.shutdown(Shutdown::Both) {
Ok(()) => Ok(()),
Err(e) if e.kind() == io::ErrorKind::NotConnected => Ok(()),
Err(e) => Err(e),
}
}
}
enum DemuxExit {
Status {
code: u32,
peak_rss_kib: Option<u64>,
},
EofBeforeExit { frames: u64, bytes: u64 },
Error(String),
Io(io::Error),
}
impl ExecChild {
pub fn stdin(&mut self) -> Option<ExecStdin> {
self.stdin.take()
}
pub fn stdout(&mut self) -> Option<ExecStdout> {
self.stdout_r.take().map(ExecStdout::from_rx)
}
pub fn stderr(&mut self) -> Option<ExecStderr> {
self.stderr_r.take().map(ExecStderr::from_rx)
}
pub fn signal(&self, signum: i32) -> io::Result<()> {
self.we_signaled
.store(true, std::sync::atomic::Ordering::SeqCst);
let payload = [signum as u8];
send_frame(&self.sock_w, FRAME_SIGNAL, &payload)
}
pub fn signaler(&self) -> ExecSignaler {
ExecSignaler {
sock_w: Arc::clone(&self.sock_w),
interrupt_fd: Arc::clone(&self.interrupt_fd),
we_signaled: Arc::clone(&self.we_signaled),
aborted: Arc::clone(&self.aborted),
}
}
pub fn abort(&self) -> io::Result<()> {
self.aborted
.store(true, std::sync::atomic::Ordering::SeqCst);
match self.interrupt_fd.shutdown(Shutdown::Both) {
Ok(()) => Ok(()),
Err(e) if e.kind() == io::ErrorKind::NotConnected => Ok(()),
Err(e) => Err(e),
}
}
pub fn try_wait(&self) -> io::Result<Option<(ExitStatus, Option<u64>)>> {
match self.exit_rx.try_recv() {
Ok(DemuxExit::Status { code, peak_rss_kib }) => {
Ok(Some((synthesize_exit(code), peak_rss_kib)))
}
Ok(DemuxExit::EofBeforeExit { frames, bytes }) => {
if self.we_signaled.load(std::sync::atomic::Ordering::SeqCst) {
Ok(Some((synthesize_killed_by_signal(libc::SIGKILL), None)))
} else {
Err(eof_before_exit_error(
self.aborted.load(std::sync::atomic::Ordering::SeqCst),
frames,
bytes,
))
}
}
Ok(DemuxExit::Error(msg)) => Err(io::Error::new(io::ErrorKind::Other, msg)),
Ok(DemuxExit::Io(e)) => Err(e),
Err(TryRecvError::Empty) => Ok(None),
Err(TryRecvError::Disconnected) => Err(io::Error::new(
io::ErrorKind::Other,
"exec: demux thread died",
)),
}
}
pub fn resize(&self, cols: u16, rows: u16) -> io::Result<()> {
let mut payload = [0u8; 4];
payload[0..2].copy_from_slice(&cols.to_be_bytes());
payload[2..4].copy_from_slice(&rows.to_be_bytes());
send_frame(&self.sock_w, FRAME_RESIZE, &payload)
}
pub fn wait(self) -> io::Result<ExitStatus> {
Ok(self.wait_with_rss()?.0)
}
pub fn wait_with_rss(mut self) -> io::Result<(ExitStatus, Option<u64>)> {
self.stdin.take();
let exit = self
.exit_rx
.recv()
.map_err(|_| io::Error::new(io::ErrorKind::Other, "exec: demux thread died"))?;
if let Some(h) = self.demux.take() {
let _ = h.join();
}
match exit {
DemuxExit::Status { code, peak_rss_kib } => Ok((synthesize_exit(code), peak_rss_kib)),
DemuxExit::EofBeforeExit { frames, bytes } => {
if self.we_signaled.load(std::sync::atomic::Ordering::SeqCst) {
Ok((synthesize_killed_by_signal(libc::SIGKILL), None))
} else {
Err(eof_before_exit_error(
self.aborted.load(std::sync::atomic::Ordering::SeqCst),
frames,
bytes,
))
}
}
DemuxExit::Error(msg) => Err(io::Error::new(io::ErrorKind::Other, msg)),
DemuxExit::Io(e) => Err(e),
}
}
}
impl Drop for ExecChild {
fn drop(&mut self) {
self.stdin.take();
self.aborted
.store(true, std::sync::atomic::Ordering::SeqCst);
let _ = self.interrupt_fd.shutdown(Shutdown::Both);
}
}
pub struct ExecStdin {
sock_w: Arc<Mutex<UnixStream>>,
closed: bool,
}
impl ExecStdin {
fn new(sock_w: Arc<Mutex<UnixStream>>) -> Self {
Self {
sock_w,
closed: false,
}
}
pub fn close(mut self) -> io::Result<()> {
if self.closed {
return Ok(());
}
self.closed = true;
send_frame(&self.sock_w, FRAME_STDIN, &[])
}
}
impl Write for ExecStdin {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
if buf.is_empty() {
return Ok(0);
}
send_frame(&self.sock_w, FRAME_STDIN, buf)?;
Ok(buf.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
impl Drop for ExecStdin {
fn drop(&mut self) {
if !self.closed {
let _ = send_frame(&self.sock_w, FRAME_STDIN, &[]);
}
}
}
pub struct ExecStdout {
rx: Receiver<Vec<u8>>,
leftover: Vec<u8>,
}
impl ExecStdout {
fn from_rx(rx: Receiver<Vec<u8>>) -> Self {
Self {
rx,
leftover: Vec::new(),
}
}
pub fn try_read(&mut self, buf: &mut [u8]) -> io::Result<Option<usize>> {
chunk_channel_try_read(&self.rx, &mut self.leftover, buf)
}
}
impl Read for ExecStdout {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
chunk_channel_read(&self.rx, &mut self.leftover, buf)
}
}
pub struct ExecStderr {
rx: Receiver<Vec<u8>>,
leftover: Vec<u8>,
}
impl ExecStderr {
fn from_rx(rx: Receiver<Vec<u8>>) -> Self {
Self {
rx,
leftover: Vec::new(),
}
}
pub fn try_read(&mut self, buf: &mut [u8]) -> io::Result<Option<usize>> {
chunk_channel_try_read(&self.rx, &mut self.leftover, buf)
}
}
impl Read for ExecStderr {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
chunk_channel_read(&self.rx, &mut self.leftover, buf)
}
}
fn chunk_channel_read(
rx: &Receiver<Vec<u8>>,
leftover: &mut Vec<u8>,
buf: &mut [u8],
) -> io::Result<usize> {
if buf.is_empty() {
return Ok(0);
}
if leftover.is_empty() {
match rx.recv() {
Ok(chunk) => {
if chunk.is_empty() {
return chunk_channel_read(rx, leftover, buf);
}
*leftover = chunk;
}
Err(_) => return Ok(0), }
}
let n = buf.len().min(leftover.len());
buf[..n].copy_from_slice(&leftover[..n]);
leftover.drain(..n);
Ok(n)
}
fn chunk_channel_try_read(
rx: &Receiver<Vec<u8>>,
leftover: &mut Vec<u8>,
buf: &mut [u8],
) -> io::Result<Option<usize>> {
if buf.is_empty() {
return Ok(Some(0));
}
while leftover.is_empty() {
match rx.try_recv() {
Ok(chunk) => {
if chunk.is_empty() {
continue;
}
*leftover = chunk;
}
Err(TryRecvError::Empty) => return Ok(None),
Err(TryRecvError::Disconnected) => return Ok(Some(0)),
}
}
let n = buf.len().min(leftover.len());
buf[..n].copy_from_slice(&leftover[..n]);
leftover.drain(..n);
Ok(Some(n))
}
pub fn send_control(exec_path: &Path, action_json: &serde_json::Value) -> io::Result<()> {
let _ = send_control_with_ack(exec_path, action_json, None)?;
Ok(())
}
pub fn send_control_with_ack(
exec_path: &Path,
action_json: &serde_json::Value,
read_timeout: Option<std::time::Duration>,
) -> io::Result<serde_json::Value> {
use std::io::Read;
let mut sock = UnixStream::connect(exec_path)?;
sock.set_read_timeout(read_timeout.or(Some(std::time::Duration::from_secs(5))))?;
sock.set_write_timeout(Some(std::time::Duration::from_secs(5)))?;
let body = serde_json::to_vec(action_json)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, format!("encode CONTROL: {e}")))?;
let mut header = [0u8; 5];
header[0] = FRAME_CONTROL;
header[1..5].copy_from_slice(&(body.len() as u32).to_be_bytes());
sock.write_all(&header)?;
if !body.is_empty() {
sock.write_all(&body)?;
}
let mut ack_hdr = [0u8; 5];
sock.read_exact(&mut ack_hdr)?;
let kind = ack_hdr[0];
let len = u32::from_be_bytes([ack_hdr[1], ack_hdr[2], ack_hdr[3], ack_hdr[4]]) as usize;
if len > 16 * 1024 * 1024 {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("CONTROL ack frame too large: {len}"),
));
}
let mut ack_body = vec![0u8; len];
if !ack_body.is_empty() {
sock.read_exact(&mut ack_body)?;
}
if kind != FRAME_CONTROL {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("expected CONTROL ack frame, got {kind:#x}"),
));
}
let ack: serde_json::Value = serde_json::from_slice(&ack_body).map_err(|e| {
io::Error::new(io::ErrorKind::InvalidData, format!("CONTROL ack JSON: {e}"))
})?;
if ack.get("ok").and_then(|v| v.as_bool()) == Some(true) {
return Ok(ack);
}
let msg = ack
.get("error")
.and_then(|v| v.as_str())
.unwrap_or("CONTROL: agent reported failure")
.to_owned();
Err(io::Error::new(io::ErrorKind::Other, msg))
}
fn spawn(builder: ExecBuilder) -> io::Result<ExecChild> {
let sock = UnixStream::connect(&builder.exec_path)?;
let sock_r = sock.try_clone()?;
let interrupt_fd = Arc::new(sock.try_clone()?);
let sock_w = Arc::new(Mutex::new(sock));
let stage_payload: Vec<StageFilePayload> = builder
.stage_files
.iter()
.map(|s| StageFilePayload {
path: &s.path,
data_b64: crate::api::b64_encode(&s.data),
mode: s.mode,
})
.collect();
let payload = RequestPayload {
argv: &builder.argv,
env: &builder.env,
cwd: builder.cwd.as_deref(),
tty: builder.tty,
cols: builder.cols,
rows: builder.rows,
stage_files: stage_payload,
chain: &builder.chain,
};
let json = serde_json::to_vec(&payload).map_err(|e| {
io::Error::new(
io::ErrorKind::InvalidInput,
format!("exec: encode REQUEST: {e}"),
)
})?;
send_frame(&sock_w, FRAME_REQUEST, &json)?;
let (stdout_tx, stdout_r) = channel::<Vec<u8>>();
let (stderr_tx, stderr_r) = channel::<Vec<u8>>();
let (exit_tx, exit_rx) = channel::<DemuxExit>();
let demux = thread::Builder::new()
.name("supermachine-exec-demux".into())
.spawn(move || {
demux_loop(sock_r, stdout_tx, stderr_tx, exit_tx);
})
.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("exec: demux thread: {e}")))?;
let stdin = ExecStdin::new(sock_w.clone());
Ok(ExecChild {
sock_w,
we_signaled: Arc::new(AtomicBool::new(false)),
aborted: Arc::new(AtomicBool::new(false)),
interrupt_fd,
stdout_r: Some(stdout_r),
stderr_r: Some(stderr_r),
stdin: Some(stdin),
demux: Some(demux),
exit_rx,
})
}
fn demux_loop(
mut sock: UnixStream,
stdout_tx: Sender<Vec<u8>>,
stderr_tx: Sender<Vec<u8>>,
exit_tx: Sender<DemuxExit>,
) {
let mut frames: u64 = 0;
let mut bytes: u64 = 0;
loop {
let (kind, payload) = match read_frame(&mut sock) {
Ok(f) => f,
Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => {
let _ = exit_tx.send(DemuxExit::EofBeforeExit { frames, bytes });
return;
}
Err(e) => {
let _ = exit_tx.send(DemuxExit::Io(e));
return;
}
};
match kind {
FRAME_STDOUT => {
frames += 1;
bytes += payload.len() as u64;
let _ = stdout_tx.send(payload);
}
FRAME_STDERR => {
frames += 1;
bytes += payload.len() as u64;
let _ = stderr_tx.send(payload);
}
FRAME_EXIT => {
let _ = exit_tx.send(decode_exit_payload(&payload));
return;
}
FRAME_ERROR => {
let msg = String::from_utf8_lossy(&payload).into_owned();
let _ = exit_tx.send(DemuxExit::Error(msg));
return;
}
_ => {
}
}
}
}
fn send_frame(sock: &Arc<Mutex<UnixStream>>, kind: u8, payload: &[u8]) -> io::Result<()> {
let mut header = [0u8; 5];
header[0] = kind;
header[1..5].copy_from_slice(&(payload.len() as u32).to_be_bytes());
let mut g = sock
.lock()
.map_err(|_| io::Error::new(io::ErrorKind::Other, "exec: socket mutex poisoned"))?;
g.write_all(&header)?;
if !payload.is_empty() {
g.write_all(payload)?;
}
Ok(())
}
fn read_frame<R: Read>(sock: &mut R) -> io::Result<(u8, Vec<u8>)> {
let mut header = [0u8; 5];
sock.read_exact(&mut header)?;
let kind = header[0];
let len = u32::from_be_bytes([header[1], header[2], header[3], header[4]]) as usize;
if len > 16 * 1024 * 1024 {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("exec: frame len {len} > 16 MiB"),
));
}
let mut payload = vec![0u8; len];
if !payload.is_empty() {
sock.read_exact(&mut payload)?;
}
Ok((kind, payload))
}
#[derive(Debug, Clone, Copy)]
pub struct ExecEof {
pub aborted: bool,
pub frames: u64,
pub bytes: u64,
}
impl ExecEof {
pub fn is_setup_failure(&self) -> bool {
!self.aborted && self.frames == 0 && self.bytes == 0
}
}
impl std::fmt::Display for ExecEof {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if self.aborted {
f.write_str(
"exec: connection closed before EXIT because the host aborted it locally \
(abort()/Drop) — not an agent-side failure",
)
} else if self.frames == 0 && self.bytes == 0 {
f.write_str(
"exec: agent closed the connection before producing any output or EXIT — the \
in-guest agent likely failed to start the process, or the VM went away \
(guest panic / reboot / OOM-kill)",
)
} else {
write!(
f,
"exec: agent closed the connection after {} output frame(s) / {} byte(s) \
without sending EXIT — the channel dropped mid-stream (the host may be \
oversubscribed; set SUPERMACHINE_WORKER_LOG_FILE to capture worker logs \
for a post-mortem)",
self.frames, self.bytes
)
}
}
}
impl std::error::Error for ExecEof {}
fn eof_before_exit_error(aborted: bool, frames: u64, bytes: u64) -> io::Error {
io::Error::new(
io::ErrorKind::UnexpectedEof,
ExecEof {
aborted,
frames,
bytes,
},
)
}
fn decode_exit_payload(payload: &[u8]) -> DemuxExit {
if payload.len() < 4 {
return DemuxExit::Error(format!(
"truncated EXIT frame: {} byte(s), need >= 4 for the exit code",
payload.len()
));
}
let code = u32::from_be_bytes([payload[0], payload[1], payload[2], payload[3]]);
let peak_rss_kib = if payload.len() >= 12 {
Some(u64::from_be_bytes([
payload[4],
payload[5],
payload[6],
payload[7],
payload[8],
payload[9],
payload[10],
payload[11],
]))
} else {
None
};
DemuxExit::Status { code, peak_rss_kib }
}
#[cfg(unix)]
fn synthesize_exit(code: u32) -> ExitStatus {
use std::os::unix::process::ExitStatusExt;
ExitStatus::from_raw(((code as i32) & 0xff) << 8)
}
#[cfg(unix)]
fn synthesize_killed_by_signal(sig: i32) -> ExitStatus {
use std::os::unix::process::ExitStatusExt;
ExitStatus::from_raw(sig & 0x7f)
}
#[cfg(test)]
mod tests {
use super::*;
use std::os::unix::net::UnixListener;
use std::os::unix::process::ExitStatusExt;
use std::path::PathBuf;
#[test]
fn decode_exit_payload_reads_code_and_optional_rss() {
match decode_exit_payload(&[0, 0, 0, 42]) {
DemuxExit::Status { code, peak_rss_kib } => {
assert_eq!(code, 42);
assert_eq!(peak_rss_kib, None);
}
_ => panic!("expected Status"),
}
let mut p = vec![0, 0, 0, 7];
p.extend_from_slice(&1234u64.to_be_bytes());
match decode_exit_payload(&p) {
DemuxExit::Status { code, peak_rss_kib } => {
assert_eq!(code, 7);
assert_eq!(peak_rss_kib, Some(1234));
}
_ => panic!("expected Status"),
}
match decode_exit_payload(&[0, 0, 1, 0, 0xaa, 0xbb]) {
DemuxExit::Status { code, peak_rss_kib } => {
assert_eq!(code, 256);
assert_eq!(peak_rss_kib, None);
}
_ => panic!("expected Status"),
}
}
#[test]
fn decode_exit_payload_truncated_is_error_not_exit_zero() {
for short in [&[][..], &[0][..], &[0, 0][..], &[0, 0, 0][..]] {
match decode_exit_payload(short) {
DemuxExit::Error(msg) => assert!(msg.contains("truncated EXIT"), "{msg}"),
_ => panic!("{}-byte EXIT frame must be an Error", short.len()),
}
}
}
#[test]
fn synthesize_exit_round_trips_exit_code() {
for c in [0u32, 1, 42, 255] {
assert_eq!(synthesize_exit(c).code(), Some(c as i32), "code {c}");
}
assert!(synthesize_exit(0).signal().is_none());
}
#[test]
fn synthesize_killed_by_signal_reports_signal_not_code() {
assert_eq!(synthesize_killed_by_signal(9).signal(), Some(9));
assert_eq!(synthesize_killed_by_signal(9).code(), None);
assert_eq!(synthesize_killed_by_signal(15).signal(), Some(15));
}
#[test]
fn eof_before_exit_error_is_self_diagnosing() {
let aborted = eof_before_exit_error(true, 0, 0);
assert_eq!(aborted.kind(), io::ErrorKind::UnexpectedEof);
assert!(aborted.to_string().contains("aborted it locally"));
let never_started = eof_before_exit_error(false, 0, 0);
assert_eq!(never_started.kind(), io::ErrorKind::UnexpectedEof);
assert!(never_started
.to_string()
.contains("before producing any output"));
let mid_stream = eof_before_exit_error(false, 7, 4096);
assert_eq!(mid_stream.kind(), io::ErrorKind::UnexpectedEof);
let msg = mid_stream.to_string();
assert!(
msg.contains("7 output frame"),
"should report frame count: {msg}"
);
assert!(msg.contains("4096 byte"), "should report byte count: {msg}");
assert!(
msg.contains("mid-stream"),
"should flag a mid-stream drop: {msg}"
);
let aborted_after_output = eof_before_exit_error(true, 7, 4096);
assert!(aborted_after_output
.to_string()
.contains("aborted it locally"));
}
#[test]
fn exec_eof_is_structured_and_downcastable() {
let setup = eof_before_exit_error(false, 0, 0);
let eof = setup
.get_ref()
.and_then(|e| e.downcast_ref::<ExecEof>())
.expect("UnexpectedEof must wrap a downcastable ExecEof");
assert!(
eof.is_setup_failure(),
"no output + not aborted ⇒ retryable setup failure"
);
let mid = eof_before_exit_error(false, 3, 100);
let mid_eof = mid.get_ref().unwrap().downcast_ref::<ExecEof>().unwrap();
assert!(
!mid_eof.is_setup_failure(),
"mid-stream drop must never be retried"
);
let aborted = eof_before_exit_error(true, 0, 0);
let ab_eof = aborted
.get_ref()
.unwrap()
.downcast_ref::<ExecEof>()
.unwrap();
assert!(
!ab_eof.is_setup_failure(),
"local abort is not an agent setup failure"
);
}
#[test]
fn exec_signaler_is_send_sync_clone() {
fn assert_send<T: Send>() {}
fn assert_sync<T: Sync>() {}
fn assert_clone<T: Clone>() {}
assert_send::<ExecSignaler>();
assert_sync::<ExecSignaler>();
assert_clone::<ExecSignaler>();
}
#[test]
fn exec_child_is_send() {
fn assert_send<T: Send>() {}
assert_send::<ExecChild>();
}
fn short_sock_dir() -> PathBuf {
use std::sync::atomic::{AtomicU64, Ordering};
static SEQ: AtomicU64 = AtomicU64::new(0);
let dir = PathBuf::from("/tmp").join(format!(
"sm-exec-{}-{}",
std::process::id(),
SEQ.fetch_add(1, Ordering::Relaxed)
));
std::fs::create_dir_all(&dir).unwrap();
dir
}
#[test]
fn signaler_abort_unblocks_blocked_wait() {
let dir = short_sock_dir();
let sock_path: PathBuf = dir.join("exec.sock");
let listener = UnixListener::bind(&sock_path).unwrap();
let listener_handle = thread::spawn(move || {
if let Ok((mut conn, _addr)) = listener.accept() {
let mut hdr = [0u8; 5];
if conn.read_exact(&mut hdr).is_ok() {
let len = u32::from_be_bytes([hdr[1], hdr[2], hdr[3], hdr[4]]) as usize;
let mut payload = vec![0u8; len];
let _ = conn.read_exact(&mut payload);
}
thread::sleep(Duration::from_secs(60));
}
});
let builder = ExecBuilder::new(sock_path.clone()).argv(["true"]);
let child = builder.spawn().expect("spawn against listener");
let signaler = child.signaler();
let abort_handle = thread::spawn(move || {
thread::sleep(Duration::from_millis(50));
signaler.abort().expect("abort must succeed");
});
let t0 = Instant::now();
let res = child.wait();
let elapsed = t0.elapsed();
let _ = abort_handle.join();
assert!(
elapsed < Duration::from_secs(5),
"wait must unblock quickly after abort; took {:?}",
elapsed
);
assert!(
res.is_err(),
"wait after abort must return Err (the agent didn't send EXIT); got {:?}",
res.as_ref().ok()
);
let err = res.unwrap_err();
assert_eq!(
err.kind(),
io::ErrorKind::UnexpectedEof,
"expected UnexpectedEof, got {:?}: {err}",
err.kind()
);
let _ = std::fs::remove_file(&sock_path);
let _ = std::fs::remove_dir(&dir);
std::mem::forget(listener_handle);
}
#[test]
fn try_wait_returns_none_then_some_after_abort() {
let dir = short_sock_dir();
let sock_path: PathBuf = dir.join("exec.sock");
let listener = UnixListener::bind(&sock_path).unwrap();
let listener_handle = thread::spawn(move || {
if let Ok((mut conn, _)) = listener.accept() {
let mut hdr = [0u8; 5];
if conn.read_exact(&mut hdr).is_ok() {
let len = u32::from_be_bytes([hdr[1], hdr[2], hdr[3], hdr[4]]) as usize;
let mut payload = vec![0u8; len];
let _ = conn.read_exact(&mut payload);
}
thread::sleep(Duration::from_secs(60));
}
});
let builder = ExecBuilder::new(sock_path.clone()).argv(["true"]);
let child = builder.spawn().expect("spawn");
assert!(
child.try_wait().expect("try_wait pre-exit").is_none(),
"try_wait must return None while child is running"
);
child.abort().expect("abort");
let t0 = Instant::now();
let result = loop {
match child.try_wait() {
Ok(Some(_)) => break Ok(()), Ok(None) => {
if t0.elapsed() > Duration::from_secs(2) {
panic!("try_wait never observed post-abort completion");
}
thread::sleep(Duration::from_millis(5));
}
Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => {
break Err(e);
}
Err(e) => panic!("unexpected try_wait error: {e}"),
}
};
assert!(result.is_err(), "try_wait must surface UnexpectedEof");
let _ = std::fs::remove_file(&sock_path);
let _ = std::fs::remove_dir(&dir);
std::mem::forget(listener_handle);
}
#[test]
fn signaler_clone_shares_underlying_socket() {
let dir = short_sock_dir();
let sock_path: PathBuf = dir.join("exec.sock");
let listener = UnixListener::bind(&sock_path).unwrap();
let listener_handle = thread::spawn(move || {
if let Ok((mut conn, _)) = listener.accept() {
let mut hdr = [0u8; 5];
if conn.read_exact(&mut hdr).is_ok() {
let len = u32::from_be_bytes([hdr[1], hdr[2], hdr[3], hdr[4]]) as usize;
let mut payload = vec![0u8; len];
let _ = conn.read_exact(&mut payload);
}
thread::sleep(Duration::from_secs(60));
}
});
let builder = ExecBuilder::new(sock_path.clone()).argv(["true"]);
let child = builder.spawn().expect("spawn");
let s1 = child.signaler();
let s2 = s1.clone(); drop(s1);
thread::spawn(move || {
thread::sleep(Duration::from_millis(50));
let _ = s2.abort();
});
let res = child.wait();
assert!(res.is_err(), "cloned signaler's abort must unblock wait");
let _ = std::fs::remove_file(&sock_path);
let _ = std::fs::remove_dir(&dir);
std::mem::forget(listener_handle);
}
}
#[cfg(test)]
mod frame_proptests {
use super::{read_frame, send_frame};
use proptest::prelude::*;
use std::io::Write;
use std::os::unix::net::UnixStream;
use std::sync::{Arc, Mutex};
proptest! {
#![proptest_config(ProptestConfig::with_cases(512))]
#[test]
fn frame_roundtrips(kind in any::<u8>(), payload in prop::collection::vec(any::<u8>(), 0..8192)) {
let (tx, mut rx) = UnixStream::pair().unwrap();
let tx = Arc::new(Mutex::new(tx));
send_frame(&tx, kind, &payload).unwrap();
let (got_kind, got_payload) = read_frame(&mut rx).unwrap();
prop_assert_eq!(got_kind, kind);
prop_assert_eq!(got_payload, payload);
}
#[test]
fn read_frame_tolerates_garbage(raw in prop::collection::vec(any::<u8>(), 0..512)) {
let (mut tx, mut rx) = UnixStream::pair().unwrap();
tx.write_all(&raw).ok();
drop(tx); if let Ok((_kind, payload)) = read_frame(&mut rx) {
prop_assert!(payload.len() <= 16 * 1024 * 1024);
}
}
}
const CAP: usize = 16 * 1024 * 1024;
#[test]
fn read_frame_length_cap_boundary() {
use std::io::ErrorKind;
let (mut tx, mut rx) = UnixStream::pair().unwrap();
let mut hdr = [0u8; 5];
hdr[0] = 1; hdr[1..5].copy_from_slice(&((CAP + 1) as u32).to_be_bytes());
tx.write_all(&hdr).unwrap();
drop(tx);
let err = read_frame(&mut rx).unwrap_err();
assert_eq!(
err.kind(),
ErrorKind::InvalidData,
"over-cap must be InvalidData"
);
assert!(err.to_string().contains("16 MiB"), "got: {err}");
let (tx, mut rx) = UnixStream::pair().unwrap();
let tx = Arc::new(Mutex::new(tx));
let writer = std::thread::spawn(move || {
send_frame(&tx, 7, &vec![0xabu8; CAP]).unwrap();
});
let (kind, payload) = read_frame(&mut rx).unwrap();
writer.join().unwrap();
assert_eq!(kind, 7);
assert_eq!(payload.len(), CAP);
assert!(payload.iter().all(|&b| b == 0xab));
}
}