use std::{
collections::{BTreeMap, VecDeque},
convert::Infallible,
fs,
io::{Read, Write},
os::unix::{fs::PermissionsExt, process::CommandExt},
path::PathBuf,
process::{Child, Command, Stdio},
sync::{Arc, Condvar, Mutex},
time::Duration,
};
use anyhow::{Context, Result, bail};
use axum::{
Json, Router,
body::Body,
extract::{
Path as AxumPath, Query, State, WebSocketUpgrade,
ws::{Message, WebSocket},
},
http::StatusCode,
response::{IntoResponse, Response},
routing::{any, get, post},
};
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use bytes::Bytes;
use chrono::Utc;
use futures_util::{SinkExt, StreamExt};
use nix::{
sys::signal::{Signal, killpg},
unistd::Pid,
};
use portable_pty::{CommandBuilder, MasterPty, PtySize, native_pty_system};
use serde::Deserialize;
use serde_json::json;
use tokio::{
net::UnixListener,
sync::{broadcast, mpsc, watch},
};
use tracing::{Instrument, debug, error, info, info_span, warn};
use crate::{
logging::{LogConfig, install_process_log},
model::{
EncodedData, Group, IoMode, OutputEvent, Process, ProcessState, SignalRequest, StopRequest,
},
observability::{self, ProcessRole},
output::{OutputFiles, OutputLog, OutputState, RecordReader},
store::{atomic_json, read_json},
};
const REPLAY_LIMIT: usize = 4 * 1024 * 1024;
const ACKNOWLEDGED_INPUT_PROTOCOL: &str = "acked-v1";
const SEQUENCED_OUTPUT_PROTOCOL: &str = "sequenced-v1";
const MAX_ACKNOWLEDGED_INPUT_CLIENTS: usize = 64;
const OUTPUT_SYNC_INTERVAL: Duration = Duration::from_millis(25);
struct Inner {
process: Process,
input: Option<Box<dyn Write + Send>>,
pty: Option<Box<dyn MasterPty + Send>>,
controller: Option<Controller>,
next_attach: u64,
acknowledged_input: BTreeMap<String, u64>,
acknowledged_input_order: VecDeque<String>,
}
struct OutputRuntime {
state: OutputState,
files: OutputFiles,
replay: VecDeque<u8>,
sync_error_reported: bool,
}
#[derive(Clone)]
struct Controller {
attach_id: u64,
client_id: Option<String>,
}
#[derive(Clone)]
enum WireMessage {
Binary { sequence: u64, data: Vec<u8> },
Text(String),
}
#[derive(Clone)]
pub struct Shim {
dir: PathBuf,
socket: PathBuf,
output_log: OutputLog,
inner: Arc<Mutex<Inner>>,
output_runtime: Arc<Mutex<OutputRuntime>>,
output: broadcast::Sender<WireMessage>,
lifecycle: broadcast::Sender<Process>,
readers: Arc<(Mutex<usize>, Condvar)>,
shutdown: Arc<tokio::sync::Notify>,
}
#[derive(Clone)]
pub struct GroupShim {
workspace_dir: PathBuf,
socket: PathBuf,
processes: Arc<Mutex<BTreeMap<String, Shim>>>,
lifecycle: broadcast::Sender<Process>,
shutdown: watch::Sender<bool>,
log_config: LogConfig,
}
#[derive(serde::Deserialize)]
struct StartProcessRequest {
process_dir: PathBuf,
}
impl GroupShim {
pub fn load(workspace_dir: impl Into<PathBuf>, socket: impl Into<PathBuf>) -> Result<Self> {
let log_config = LogConfig::from_env()?;
let (lifecycle, _) = broadcast::channel(256);
Ok(Self {
workspace_dir: workspace_dir.into(),
socket: socket.into(),
processes: Arc::new(Mutex::new(BTreeMap::new())),
lifecycle,
shutdown: watch::channel(false).0,
log_config,
})
}
pub async fn serve(self, ready_fd: Option<i32>, lifecycle_fd: Option<i32>) -> Result<()> {
if lifecycle_fd.is_some() {
install_process_log(self.workspace_dir.join("shim.log"), self.log_config)?;
}
observability::init(ProcessRole::Shim)?;
let group = read_json::<Group>(self.workspace_dir.join("group.json")).ok();
let span = info_span!(
"group_shim",
component = "shim",
workspace_id = group
.as_ref()
.map_or("", |value| value.workspace_id.as_str()),
group_id = group.as_ref().map_or("", |value| value.id.as_str()),
pid = std::process::id()
);
self.serve_runtime(ready_fd, lifecycle_fd)
.instrument(span)
.await
}
async fn serve_runtime(self, ready_fd: Option<i32>, lifecycle_fd: Option<i32>) -> Result<()> {
if let Some(parent) = self.socket.parent() {
fs::create_dir_all(parent)?;
fs::set_permissions(parent, fs::Permissions::from_mode(0o700))?;
}
let _ = fs::remove_file(&self.socket);
let listener = UnixListener::bind(&self.socket)?;
fs::set_permissions(&self.socket, fs::Permissions::from_mode(0o600))?;
if let Some(fd) = lifecycle_fd {
unsafe {
libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC);
}
let group = self.clone();
std::thread::spawn(move || {
let mut byte = [0u8; 1];
let _ = unsafe { libc::read(fd, byte.as_mut_ptr().cast(), 1) };
unsafe {
libc::close(fd);
}
group.terminate_all();
group.shutdown.send_replace(true);
});
}
if let Some(fd) = ready_fd {
write_ready(fd);
}
info!(socket = %self.socket.display(), "group shim ready");
let idle_group = self.clone();
let mut lifecycle = self.lifecycle.subscribe();
tokio::spawn(async move {
loop {
match lifecycle.recv().await {
Ok(process) => {
if process.state.terminal() {
let mut processes = idle_group.processes.lock().unwrap();
if processes.get(&process.id).is_some_and(|shim| {
let inner = shim.inner.lock().unwrap();
inner.process.execution == process.execution
&& inner.process.state.terminal()
}) {
processes.remove(&process.id);
}
debug!(process_id = %process.id, "terminal supervisor unloaded");
if processes.is_empty() {
idle_group.shutdown.send_replace(true);
drop(processes);
break;
}
}
}
Err(broadcast::error::RecvError::Lagged(_)) => {
let mut processes = idle_group.processes.lock().unwrap();
processes
.retain(|_, shim| !shim.inner.lock().unwrap().process.state.terminal());
if processes.is_empty() {
idle_group.shutdown.send_replace(true);
drop(processes);
break;
}
}
Err(broadcast::error::RecvError::Closed) => break,
}
}
});
let app = Router::new()
.route("/v1/processes", post(group_start))
.route("/v1/events", get(group_events))
.route("/v1/processes/{id}/status", get(group_status))
.route("/v1/processes/{id}/signal", post(group_signal))
.route("/v1/processes/{id}/stop", post(group_stop))
.route("/v1/processes/{id}/logs", get(group_logs))
.route("/v1/processes/{id}/attach", any(group_attach))
.route("/v1/shutdown", post(group_shutdown))
.with_state(self.clone());
let mut stopping = self.shutdown.subscribe();
let socket = self.socket.clone();
let result = axum::serve(listener, app)
.with_graceful_shutdown(async move {
let _ = stopping.wait_for(|stopping| *stopping).await;
let _ = fs::remove_file(socket);
})
.await;
if !*self.shutdown.borrow() {
let _ = fs::remove_file(&self.socket);
}
result?;
info!("group shim stopped");
Ok(())
}
fn get(&self, id: &str) -> Option<Shim> {
self.processes.lock().unwrap().get(id).cloned()
}
fn terminate_all(&self) {
let shims = self
.processes
.lock()
.unwrap()
.values()
.cloned()
.collect::<Vec<_>>();
for shim in &shims {
let pgid = shim.inner.lock().unwrap().process.process_group_id;
if pgid > 0 && !shim.inner.lock().unwrap().process.state.terminal() {
warn!(process_id = %shim.inner.lock().unwrap().process.id, pgid, "terminating process after lifeline closed");
let _ = killpg(Pid::from_raw(pgid), Signal::SIGTERM);
}
}
std::thread::sleep(Duration::from_secs(10));
for shim in shims {
let inner = shim.inner.lock().unwrap();
if inner.process.process_group_id > 0 && !inner.process.state.terminal() {
warn!(process_id = %inner.process.id, pgid = inner.process.process_group_id, "forcing process termination");
let _ = killpg(
Pid::from_raw(inner.process.process_group_id),
Signal::SIGKILL,
);
}
}
}
}
async fn group_start(
State(group): State<GroupShim>,
Json(request): Json<StartProcessRequest>,
) -> Response {
if !request.process_dir.starts_with(&group.workspace_dir) {
return (StatusCode::BAD_REQUEST, "invalid process directory").into_response();
}
let shim = match Shim::load(
&request.process_dir,
group.socket.with_extension("unused"),
group.lifecycle.clone(),
group.log_config,
) {
Ok(v) => v,
Err(e) => return (StatusCode::BAD_REQUEST, e.to_string()).into_response(),
};
let id = shim.inner.lock().unwrap().process.id.clone();
{
let mut processes = group.processes.lock().unwrap();
if *group.shutdown.borrow() {
return (StatusCode::SERVICE_UNAVAILABLE, "group shim is stopping").into_response();
}
if let Some(existing) = processes.get(&id)
&& !existing.inner.lock().unwrap().process.state.terminal()
{
return Json(json!({"ok":true})).into_response();
}
processes.insert(id.clone(), shim.clone());
}
if let Err(error) = shim.start_process() {
shim.fail_wait(&error);
return (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response();
}
Json(json!({"ok":true})).into_response()
}
async fn group_events(State(group): State<GroupShim>) -> Response {
let mut receiver = group.lifecycle.subscribe();
let mut stopping = group.shutdown.subscribe();
let initial = group
.processes
.lock()
.unwrap()
.values()
.filter_map(|shim| {
let process = shim.inner.lock().unwrap().process.clone();
serde_json::to_vec(&process).ok().map(|mut line| {
line.push(b'\n');
Ok::<_, Infallible>(Bytes::from(line))
})
})
.collect::<Vec<_>>();
let follow = async_stream::stream! {
loop {
let process = if *stopping.borrow() {
match receiver.try_recv() {
Ok(process) => process,
Err(_) => break,
}
} else {
tokio::select! {
item = receiver.recv() => match item {
Ok(process) => process,
Err(_) => break, },
_ = stopping.changed() => continue,
}
};
if let Ok(mut line) = serde_json::to_vec(&process) {
line.push(b'\n');
yield Ok::<_, Infallible>(Bytes::from(line));
}
}
};
Body::from_stream(tokio_stream::StreamExt::chain(
tokio_stream::iter(initial),
follow,
))
.into_response()
}
async fn group_status(State(group): State<GroupShim>, AxumPath(id): AxumPath<String>) -> Response {
match group.get(&id) {
Some(shim) => status(State(shim)).await.into_response(),
None => StatusCode::NOT_FOUND.into_response(),
}
}
async fn group_signal(
State(group): State<GroupShim>,
AxumPath(id): AxumPath<String>,
Json(req): Json<SignalRequest>,
) -> Response {
match group.get(&id) {
Some(shim) => signal(State(shim), Json(req)).await,
None => StatusCode::NOT_FOUND.into_response(),
}
}
async fn group_stop(
State(group): State<GroupShim>,
AxumPath(id): AxumPath<String>,
Json(req): Json<StopRequest>,
) -> Response {
match group.get(&id) {
Some(shim) => stop(State(shim), Json(req)).await,
None => StatusCode::NOT_FOUND.into_response(),
}
}
async fn group_logs(
State(group): State<GroupShim>,
AxumPath(id): AxumPath<String>,
query: Query<LogsQuery>,
) -> Response {
match group.get(&id) {
Some(shim) => logs(State(shim), query).await,
None => StatusCode::NOT_FOUND.into_response(),
}
}
async fn group_attach(
State(group): State<GroupShim>,
AxumPath(id): AxumPath<String>,
query: Query<AttachQuery>,
ws: WebSocketUpgrade,
) -> Response {
match group.get(&id) {
Some(shim) => attach(State(shim), query, ws).await,
None => StatusCode::NOT_FOUND.into_response(),
}
}
async fn group_shutdown(State(group): State<GroupShim>) -> Response {
let processes = group.processes.lock().unwrap();
if processes
.values()
.any(|s| !s.inner.lock().unwrap().process.state.terminal())
{
return (StatusCode::CONFLICT, "group still has running processes").into_response();
}
group.shutdown.send_replace(true);
Json(json!({"ok":true})).into_response()
}
impl Shim {
pub fn load(
dir: impl Into<PathBuf>,
socket: impl Into<PathBuf>,
lifecycle: broadcast::Sender<Process>,
log_config: LogConfig,
) -> Result<Self> {
let dir = dir.into();
let socket = socket.into();
let process: Process = read_json(dir.join("metadata.json"))?;
let (output_log, output_state, output_files) = OutputLog::open(&dir, log_config)?;
let replay = output_log.replay_tail("tty", REPLAY_LIMIT)?;
let (output, _) = broadcast::channel(256);
let output_runtime = Arc::new(Mutex::new(OutputRuntime {
state: output_state,
files: output_files,
replay,
sync_error_reported: false,
}));
spawn_output_sync(&output_runtime, process.id.clone());
Ok(Self {
dir,
socket,
output_log,
inner: Arc::new(Mutex::new(Inner {
process,
input: None,
pty: None,
controller: None,
next_attach: 0,
acknowledged_input: BTreeMap::new(),
acknowledged_input_order: VecDeque::new(),
})),
output_runtime,
output,
lifecycle,
readers: Arc::new((Mutex::new(0), Condvar::new())),
shutdown: Arc::new(tokio::sync::Notify::new()),
})
}
fn publish_lifecycle(&self, process: &Process) {
let _ = self.lifecycle.send(process.clone());
}
pub async fn serve(self, ready_fd: Option<i32>) -> Result<()> {
if let Some(parent) = self.socket.parent() {
fs::create_dir_all(parent)?;
fs::set_permissions(parent, fs::Permissions::from_mode(0o700))?;
}
let _ = fs::remove_file(&self.socket);
if let Err(error) = self.start_process() {
let mut inner = self.inner.lock().unwrap();
inner.process.state = ProcessState::Failed;
inner.process.error = error.to_string();
inner.process.finished_at = Some(Utc::now());
if let Err(persist_error) = atomic_json(&self.dir.join("metadata.json"), &inner.process)
{
error!(error = %persist_error, process_id = %inner.process.id, "persist process startup failure failed");
}
error!(%error, process_id = %inner.process.id, "process startup failed");
return Err(error);
}
let listener = UnixListener::bind(&self.socket)?;
fs::set_permissions(&self.socket, fs::Permissions::from_mode(0o600))?;
if let Some(fd) = ready_fd {
write_ready(fd);
}
let app = Router::new()
.route("/v1/status", get(status))
.route("/v1/signal", post(signal))
.route("/v1/stop", post(stop))
.route("/v1/logs", get(logs))
.route("/v1/attach", any(attach))
.route("/v1/shutdown", post(shutdown))
.with_state(self.clone());
let stopping = self.shutdown.clone();
let result = axum::serve(listener, app)
.with_graceful_shutdown(async move { stopping.notified().await })
.await;
let _ = fs::remove_file(&self.socket);
result?;
Ok(())
}
fn start_process(&self) -> Result<()> {
let (process, env): (Process, BTreeMap<String, String>) = {
let inner = self.inner.lock().unwrap();
let env = read_json(self.dir.join("runtime-env.json"))
.unwrap_or_else(|_| inner.process.env.clone());
(inner.process.clone(), env)
};
if process.command.is_empty() {
bail!("empty command")
}
match process.io_mode {
IoMode::Pipe => self.start_pipe(process, env),
IoMode::Tty => self.start_tty(process, env),
}
}
fn start_pipe(&self, process: Process, env: BTreeMap<String, String>) -> Result<()> {
let mut command = Command::new(&process.command[0]);
command
.args(&process.command[1..])
.env_clear()
.envs(env)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
if !process.cwd.is_empty() {
command.current_dir(&process.cwd);
}
unsafe {
command.pre_exec(|| {
if libc::setpgid(0, 0) == -1 {
return Err(std::io::Error::last_os_error());
}
Ok(())
});
}
let mut child = command.spawn().context("spawn process")?;
let pid = child.id() as i32;
let stdin = child
.stdin
.take()
.map(|v| Box::new(v) as Box<dyn Write + Send>);
if let Some(stdout) = child.stdout.take() {
self.spawn_reader("stdout", stdout);
}
if let Some(stderr) = child.stderr.take() {
self.spawn_reader("stderr", stderr);
}
{
let mut inner = self.inner.lock().unwrap();
inner.input = stdin;
inner.process.pid = pid;
inner.process.process_group_id = pid;
inner.process.state = ProcessState::Running;
inner.process.started_at = Some(Utc::now());
atomic_json(&self.dir.join("metadata.json"), &inner.process)?;
self.publish_lifecycle(&inner.process);
};
info!(process_id = %process.id, execution = process.execution, pid, io_mode = "pipe", "process started");
self.spawn_wait_pipe(child);
Ok(())
}
fn start_tty(&self, process: Process, env: BTreeMap<String, String>) -> Result<()> {
let pair = native_pty_system().openpty(PtySize {
rows: process.initial_rows,
cols: process.initial_cols,
pixel_width: 0,
pixel_height: 0,
})?;
let mut command = CommandBuilder::new(&process.command[0]);
command.args(&process.command[1..]);
if !process.cwd.is_empty() {
command.cwd(&process.cwd);
}
for (key, value) in env {
command.env(key, value);
}
let child = pair.slave.spawn_command(command)?;
let pid = child.process_id().unwrap_or(0) as i32;
let reader = pair.master.try_clone_reader()?;
let writer = pair.master.take_writer()?;
{
let mut inner = self.inner.lock().unwrap();
inner.input = Some(writer);
inner.pty = Some(pair.master);
inner.process.pid = pid;
inner.process.process_group_id = pid;
inner.process.state = ProcessState::Running;
inner.process.started_at = Some(Utc::now());
atomic_json(&self.dir.join("metadata.json"), &inner.process)?;
self.publish_lifecycle(&inner.process);
};
info!(process_id = %process.id, execution = process.execution, pid, io_mode = "tty", "process started");
self.spawn_reader("tty", reader);
self.spawn_wait_pty(child);
Ok(())
}
fn spawn_reader(&self, stream: &'static str, mut reader: impl Read + Send + 'static) {
let shim = self.clone();
*shim.readers.0.lock().unwrap() += 1;
std::thread::spawn(move || {
let mut buffer = [0u8; 16 * 1024];
loop {
match reader.read(&mut buffer) {
Ok(0) => break,
Err(error) => {
warn!(%error, process_id = %shim.inner.lock().unwrap().process.id, stream, "process output reader failed");
break;
}
Ok(count) => shim.emit(stream, &buffer[..count]),
}
}
let mut count = shim.readers.0.lock().unwrap();
*count -= 1;
shim.readers.1.notify_all();
});
}
fn emit(&self, stream: &str, bytes: &[u8]) {
let (execution, process_id) = {
let inner = self.inner.lock().unwrap();
(inner.process.execution, inner.process.id.clone())
};
let mut output = self.output_runtime.lock().unwrap();
let append_result = {
let OutputRuntime { state, files, .. } = &mut *output;
self.output_log
.append(state, files, stream, bytes, execution)
};
let event = match append_result {
Ok(event) => event,
Err(error) => {
error!(%error, process_id = %process_id, execution, stream, "persist process output failed");
return;
}
};
output.replay.extend(bytes);
while output.replay.len() > REPLAY_LIMIT {
output.replay.pop_front();
}
drop(output);
let _ = self.output.send(WireMessage::Binary {
sequence: event.sequence,
data: bytes.to_vec(),
});
}
fn spawn_wait_pipe(&self, mut child: Child) {
let shim = self.clone();
std::thread::spawn(move || match child.wait() {
Ok(status) => shim.finish(
status.code(),
std::os::unix::process::ExitStatusExt::signal(&status),
),
Err(error) => shim.fail_wait(error),
});
}
fn spawn_wait_pty(&self, mut child: Box<dyn portable_pty::Child + Send + Sync>) {
let shim = self.clone();
std::thread::spawn(move || match child.wait() {
Ok(status) => shim.finish(Some(status.exit_code() as i32), None),
Err(error) => shim.fail_wait(error),
});
}
fn finish(&self, code: Option<i32>, signal: Option<i32>) {
let (lock, ready) = &*self.readers;
let mut count = lock.lock().unwrap();
while *count != 0 {
count = ready.wait(count).unwrap();
}
drop(count);
self.sync_output();
let process = {
let mut inner = self.inner.lock().unwrap();
inner.process.exit_code = Some(code.unwrap_or(-1));
inner.process.exit_signal = signal.and_then(signal_name).unwrap_or_default();
inner.process.state = ProcessState::Exited;
inner.process.finished_at = Some(Utc::now());
inner.input = None;
if let Err(error) = atomic_json(&self.dir.join("metadata.json"), &inner.process) {
error!(%error, process_id = %inner.process.id, "persist terminal process state failed");
}
let message = serde_json::to_string(&json!({"type":"exit","exit_code":inner.process.exit_code,"signal":inner.process.exit_signal})).unwrap_or_default();
let _ = self.output.send(WireMessage::Text(message));
self.publish_lifecycle(&inner.process);
inner.process.clone()
};
info!(process_id = %process.id, execution = process.execution, exit_code = ?process.exit_code, signal = %process.exit_signal, "process exited");
}
fn fail_wait(&self, error: impl std::fmt::Display) {
self.sync_output();
let process = {
let mut inner = self.inner.lock().unwrap();
inner.process.state = ProcessState::Failed;
inner.process.error = error.to_string();
inner.process.finished_at = Some(Utc::now());
if let Err(persist_error) = atomic_json(&self.dir.join("metadata.json"), &inner.process)
{
error!(error = %persist_error, process_id = %inner.process.id, "persist failed process state failed");
}
self.publish_lifecycle(&inner.process);
inner.process.clone()
};
error!(process_id = %process.id, execution = process.execution, error = %process.error, "process wait failed");
let _ = self.output.send(WireMessage::Text(
serde_json::to_string(&json!({"type":"exit","error":"process wait failed"}))
.unwrap_or_default(),
));
}
fn sync_output(&self) {
let process_id = self.inner.lock().unwrap().process.id.clone();
let mut output = self.output_runtime.lock().unwrap();
match output.files.sync_data() {
Ok(_) => output.sync_error_reported = false,
Err(error) if !output.sync_error_reported => {
output.sync_error_reported = true;
error!(%error, process_id = %process_id, "sync process output failed");
}
Err(_) => {}
}
}
}
fn spawn_output_sync(output_runtime: &Arc<Mutex<OutputRuntime>>, process_id: String) {
let output_runtime = Arc::downgrade(output_runtime);
std::thread::spawn(move || {
loop {
std::thread::sleep(OUTPUT_SYNC_INTERVAL);
let Some(output_runtime) = output_runtime.upgrade() else {
break;
};
let mut output = output_runtime.lock().unwrap();
match output.files.sync_data() {
Ok(_) => output.sync_error_reported = false,
Err(error) if !output.sync_error_reported => {
output.sync_error_reported = true;
error!(%error, process_id = %process_id, "sync process output failed");
}
Err(_) => {}
}
}
});
}
async fn status(State(shim): State<Shim>) -> Json<Process> {
Json(shim.inner.lock().unwrap().process.clone())
}
async fn signal(State(shim): State<Shim>, Json(request): Json<SignalRequest>) -> Response {
let (pgid, process_id) = {
let inner = shim.inner.lock().unwrap();
(inner.process.process_group_id, inner.process.id.clone())
};
match parse_signal(&request.signal)
.and_then(|sig| killpg(Pid::from_raw(pgid), sig).map_err(anyhow::Error::from))
{
Ok(()) => {
debug!(%process_id, signal = %request.signal, "signal delivered");
Json(json!({"ok":true})).into_response()
}
Err(error) => (StatusCode::BAD_REQUEST, error.to_string()).into_response(),
}
}
async fn stop(State(shim): State<Shim>, Json(request): Json<StopRequest>) -> Response {
let (pgid, process) = {
let mut inner = shim.inner.lock().unwrap();
if inner.process.state.terminal() {
return Json(json!({"ok":true})).into_response();
}
inner.process.state = ProcessState::Stopping;
if let Err(error) = atomic_json(&shim.dir.join("metadata.json"), &inner.process) {
error!(%error, process_id = %inner.process.id, "persist stopping process state failed");
}
shim.publish_lifecycle(&inner.process);
(inner.process.process_group_id, inner.process.clone())
};
info!(process_id = %process.id, grace_millis = request.grace_millis, "stopping process");
let _ = killpg(Pid::from_raw(pgid), Signal::SIGTERM);
let delayed = shim.clone();
tokio::spawn(async move {
let grace = if request.grace_millis <= 0 {
10_000
} else {
request.grace_millis
};
tokio::time::sleep(Duration::from_millis(grace as u64)).await;
if !delayed.inner.lock().unwrap().process.state.terminal() {
warn!(process_id = %delayed.inner.lock().unwrap().process.id, "grace period expired; sending SIGKILL");
let _ = killpg(Pid::from_raw(pgid), Signal::SIGKILL);
}
});
Json(json!({"ok":true})).into_response()
}
#[derive(Deserialize)]
struct LogsQuery {
#[serde(default)]
after: u64,
#[serde(default)]
follow: bool,
}
async fn logs(State(shim): State<Shim>, Query(query): Query<LogsQuery>) -> Response {
if !query.follow {
let boundary = shim.output_runtime.lock().unwrap().state.sequence;
let dir = shim.dir.clone();
let stream = async_stream::stream! {
if let Ok(records) = RecordReader::open(&dir, query.after, Some(boundary)) {
for record in records.flatten() {
if let Ok(line) = record.to_json_line() {
yield Ok::<_, Infallible>(Bytes::from(line));
}
}
}
};
return Body::from_stream(stream).into_response();
}
let mut receiver = shim.output.subscribe();
let boundary = shim.output_runtime.lock().unwrap().state.sequence;
let stream = async_stream::stream! {
if let Ok(records) = RecordReader::open(&shim.dir, query.after, Some(boundary)) {
for record in records.flatten() {
if let Ok(line) = record.to_json_line() {
yield Ok::<_, Infallible>(Bytes::from(line));
}
}
}
let mut cursor = boundary.max(query.after);
loop {
match receiver.recv().await {
Ok(WireMessage::Text(text)) => {
if serde_json::from_str::<serde_json::Value>(&text)
.ok()
.and_then(|value| value.get("type").and_then(|kind| kind.as_str()).map(str::to_owned))
.as_deref() == Some("exit")
{
let latest = shim.output_runtime.lock().unwrap().state.sequence;
if let Ok(records) = RecordReader::open(&shim.dir, cursor, Some(latest)) {
for record in records.flatten() {
if let Ok(line) = record.to_json_line() {
yield Ok::<_, Infallible>(Bytes::from(line));
}
}
}
break;
}
},
Ok(WireMessage::Binary { .. }) | Err(broadcast::error::RecvError::Lagged(_)) => {
let latest = shim.output_runtime.lock().unwrap().state.sequence;
if let Ok(records) = RecordReader::open(&shim.dir, cursor, Some(latest)) {
for record in records.flatten() {
if let Ok(line) = record.to_json_line() {
yield Ok::<_, Infallible>(Bytes::from(line));
}
}
}
cursor = latest;
}
Err(broadcast::error::RecvError::Closed) => break,
}
}
};
Body::from_stream(stream).into_response()
}
#[derive(Deserialize)]
struct AttachQuery {
#[serde(default)]
readonly: bool,
#[serde(default)]
takeover: bool,
#[serde(default = "default_replay")]
replay_bytes: usize,
input_protocol: Option<String>,
input_client_id: Option<String>,
control: Option<String>,
controller_client_id: Option<String>,
output_protocol: Option<String>,
after_output_sequence: Option<u64>,
}
fn default_replay() -> usize {
8192
}
async fn attach(
State(shim): State<Shim>,
Query(query): Query<AttachQuery>,
ws: WebSocketUpgrade,
) -> Response {
if shim.inner.lock().unwrap().process.io_mode != IoMode::Tty {
return (StatusCode::CONFLICT, "process has no TTY").into_response();
}
let valid_id = |value: &str| {
!value.is_empty()
&& value.len() <= 128
&& value
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_')
};
if query
.input_protocol
.as_deref()
.is_some_and(|value| value != ACKNOWLEDGED_INPUT_PROTOCOL)
|| query.input_protocol.is_some() != query.input_client_id.is_some()
|| query
.input_client_id
.as_deref()
.is_some_and(|value| !valid_id(value))
{
return (
StatusCode::BAD_REQUEST,
"invalid acknowledged input protocol",
)
.into_response();
}
if query
.control
.as_deref()
.is_some_and(|value| value != "preferred")
|| query.control.is_some() != query.controller_client_id.is_some()
|| query
.controller_client_id
.as_deref()
.is_some_and(|value| !valid_id(value))
|| (query.control.is_some() && (query.readonly || query.takeover))
|| query
.output_protocol
.as_deref()
.is_some_and(|value| value != SEQUENCED_OUTPUT_PROTOCOL)
|| (query.after_output_sequence.is_some()
&& query.output_protocol.as_deref() != Some(SEQUENCED_OUTPUT_PROTOCOL))
{
return (StatusCode::BAD_REQUEST, "invalid terminal attach protocol").into_response();
}
let (id, owns_input) = {
let mut inner = shim.inner.lock().unwrap();
inner.next_attach += 1;
let id = inner.next_attach;
let owns_input = if query.control.as_deref() == Some("preferred") {
let preferred = query.controller_client_id.as_deref();
if inner
.controller
.as_ref()
.is_none_or(|controller| controller.client_id.as_deref() == preferred)
{
inner.controller = Some(Controller {
attach_id: id,
client_id: query.controller_client_id.clone(),
});
true
} else {
false
}
} else if query.readonly {
false
} else {
if inner.controller.is_some() && !query.takeover {
return (StatusCode::CONFLICT, "controller already exists").into_response();
}
if inner.controller.is_some() && query.takeover {
info!(process_id = %inner.process.id, "TTY controller taken over");
}
inner.controller = Some(Controller {
attach_id: id,
client_id: None,
});
true
};
(id, owns_input)
};
ws.on_upgrade(move |socket| attach_socket(shim, socket, id, owns_input, query))
}
async fn attach_socket(
shim: Shim,
socket: WebSocket,
id: u64,
owns_input: bool,
query: AttachQuery,
) {
let (mut sink, mut source) = socket.split();
let mut output = shim.output.subscribe();
let boundary = shim.output_runtime.lock().unwrap().state.sequence;
let sequenced = query.output_protocol.as_deref() == Some(SEQUENCED_OUTPUT_PROTOCOL);
let mut output_cursor = query.after_output_sequence.unwrap_or(0);
if sequenced {
let records = if query.after_output_sequence.is_some() {
collect_output_records(&shim, output_cursor, boundary)
} else {
shim.output_log
.replay_event_tail("tty", query.replay_bytes, boundary)
.map(|events| events.into_iter().map(OutputReplay::Event).collect())
};
let records = match records {
Ok(records) => records,
Err(_) => {
let _ = sink
.send(Message::Text(
json!({"type":"error","code":"output_replay_failed"})
.to_string()
.into(),
))
.await;
release(&shim, id);
return;
}
};
let start = records
.iter()
.map(|record| match record {
OutputReplay::Event(event) => event.sequence.saturating_sub(1),
OutputReplay::Gap {
oldest_available, ..
} => oldest_available.saturating_sub(1),
})
.next()
.unwrap_or(boundary);
if sink
.send(Message::Text(
json!({"type":"output_cursor","sequence":start.to_string()})
.to_string()
.into(),
))
.await
.is_err()
{
release(&shim, id);
return;
}
output_cursor = start;
for record in records {
if send_output_replay(&mut sink, record, &mut output_cursor)
.await
.is_err()
{
release(&shim, id);
return;
}
}
} else {
let replay = {
let output = shim.output_runtime.lock().unwrap();
output
.replay
.iter()
.skip(output.replay.len().saturating_sub(query.replay_bytes))
.copied()
.collect::<Vec<_>>()
};
if !replay.is_empty() && sink.send(Message::Binary(replay.into())).await.is_err() {
release(&shim, id);
return;
}
}
let (reply_tx, mut reply_rx) = mpsc::channel::<String>(64);
let ownership = if owns_input { "controller" } else { "readonly" };
if query.control.is_some()
&& sink
.send(Message::Text(
json!({"type":"ownership_state","state":ownership})
.to_string()
.into(),
))
.await
.is_err()
{
release(&shim, id);
return;
}
if owns_input && query.control.is_none() {
let notice = serde_json::to_string(
&json!({"type":"ownership_changed","controller":id,"takeover":query.takeover}),
)
.unwrap();
let _ = shim.output.send(WireMessage::Text(notice));
}
let terminal = {
let inner = shim.inner.lock().unwrap();
if inner.process.state.terminal() {
Some(serde_json::to_string(&json!({"type":"exit","exit_code":inner.process.exit_code.unwrap_or(0),"signal":inner.process.exit_signal})).unwrap())
} else {
None
}
};
if let Some(message) = terminal
&& sink.send(Message::Text(message.into())).await.is_err()
{
release(&shim, id);
return;
}
let acknowledged_client = query.input_client_id.clone();
let send_output = async {
let mut cursor = output_cursor;
loop {
tokio::select! {
received = output.recv() => match received {
Ok(WireMessage::Binary { sequence, data }) => {
if sequenced {
if sequence <= cursor { continue; }
if sequence > cursor.saturating_add(1) {
let records = collect_output_records(&shim, cursor, sequence);
let Ok(records) = records else {
let _ = sink.send(Message::Text(json!({"type":"error","code":"output_replay_failed"}).to_string().into())).await;
break;
};
for record in records {
if send_output_replay(&mut sink, record, &mut cursor).await.is_err() { return; }
}
} else if sink.send(Message::Binary(sequenced_frame(sequence, &data).into())).await.is_err() { break; }
cursor = cursor.max(sequence);
} else if sink.send(Message::Binary(data.into())).await.is_err() { break; }
},
Ok(WireMessage::Text(text)) => if sink.send(Message::Text(text.into())).await.is_err() { break; },
Err(broadcast::error::RecvError::Lagged(_)) if sequenced => {
let latest = shim.output_runtime.lock().unwrap().state.sequence;
let Ok(records) = collect_output_records(&shim, cursor, latest) else {
let _ = sink.send(Message::Text(json!({"type":"error","code":"output_replay_failed"}).to_string().into())).await;
break;
};
for record in records {
if send_output_replay(&mut sink, record, &mut cursor).await.is_err() { return; }
}
},
Err(_) => break,
},
reply = reply_rx.recv() => match reply {
Some(text) => if sink.send(Message::Text(text.into())).await.is_err() { break; },
None => break,
},
}
}
};
let receive_input = async {
loop {
match futures_util::StreamExt::next(&mut source).await {
Some(Ok(Message::Binary(data))) if owns_input && acknowledged_client.is_none() => {
let result = write_unacknowledged_input(&shim, id, &data);
if let Some(message) = result {
let _ = reply_tx.send(message).await;
break;
}
}
Some(Ok(Message::Text(text))) if owns_input => {
let response = if let Some(client_id) = acknowledged_client.as_deref() {
handle_acknowledged_control(&shim, id, client_id, &text)
} else {
handle_control(&shim, id, &text);
None
};
if let Some(message) = response
&& reply_tx.send(message).await.is_err()
{
break;
}
}
Some(Ok(Message::Binary(_))) | Some(Ok(Message::Text(_))) if !owns_input => {
if reply_tx
.send(
serde_json::to_string(
&json!({"type":"input_rejected","code":"read_only"}),
)
.unwrap_or_default(),
)
.await
.is_err()
{
break;
}
}
Some(Ok(Message::Close(_))) | None | Some(Err(_)) => break,
_ => {}
}
}
};
tokio::select! {
_ = send_output => {},
_ = receive_input => {},
}
release(&shim, id);
}
fn write_unacknowledged_input(shim: &Shim, id: u64, data: &[u8]) -> Option<String> {
let mut inner = shim.inner.lock().unwrap();
if inner.controller.as_ref().map(|value| value.attach_id) != Some(id) {
return Some(
serde_json::to_string(&json!({"type":"input_error","code":"ownership_lost"}))
.unwrap_or_default(),
);
}
let failed = inner
.input
.as_mut()
.is_none_or(|input| input.write_all(data).is_err());
failed.then(|| {
serde_json::to_string(&json!({"type":"input_error","code":"pty_write_failed"}))
.unwrap_or_default()
})
}
fn handle_acknowledged_control(
shim: &Shim,
id: u64,
client_id: &str,
text: &str,
) -> Option<String> {
let Ok(value) = serde_json::from_str::<serde_json::Value>(text) else {
return Some(
serde_json::to_string(&json!({"type":"input_error","code":"invalid_control"}))
.unwrap_or_default(),
);
};
if value.get("type").and_then(|value| value.as_str()) != Some("input") {
handle_control_value(shim, id, &value);
return None;
}
let Some(sequence) = value.get("sequence").and_then(|value| value.as_u64()) else {
return Some(
serde_json::to_string(&json!({"type":"input_error","code":"invalid_sequence"}))
.unwrap_or_default(),
);
};
let Some(encoded) = value.get("data").and_then(|value| value.as_str()) else {
return Some(
serde_json::to_string(&json!({"type":"input_error","code":"invalid_input"}))
.unwrap_or_default(),
);
};
let Ok(data) = BASE64.decode(encoded) else {
return Some(
serde_json::to_string(&json!({"type":"input_error","code":"invalid_input"}))
.unwrap_or_default(),
);
};
let mut inner = shim.inner.lock().unwrap();
if inner.controller.as_ref().map(|value| value.attach_id) != Some(id) {
return Some(
serde_json::to_string(
&json!({"type":"input_rejected","code":"ownership_lost","sequence":sequence}),
)
.unwrap_or_default(),
);
}
let last_acknowledged = inner
.acknowledged_input
.get(client_id)
.copied()
.unwrap_or(0);
if sequence <= last_acknowledged {
return Some(input_ack(client_id, last_acknowledged));
}
if sequence != last_acknowledged + 1 {
return Some(serde_json::to_string(&json!({"type":"input_rejected","code":"sequence_gap","sequence":sequence,"expected":last_acknowledged + 1})).unwrap_or_default());
}
let failed = inner
.input
.as_mut()
.is_none_or(|input| input.write_all(&data).is_err());
if failed {
return Some(
serde_json::to_string(
&json!({"type":"input_error","code":"pty_write_failed","sequence":sequence}),
)
.unwrap_or_default(),
);
}
if !inner.acknowledged_input.contains_key(client_id) {
while inner.acknowledged_input.len() >= MAX_ACKNOWLEDGED_INPUT_CLIENTS {
if let Some(expired) = inner.acknowledged_input_order.pop_front() {
inner.acknowledged_input.remove(&expired);
}
}
inner
.acknowledged_input_order
.push_back(client_id.to_owned());
}
inner
.acknowledged_input
.insert(client_id.to_owned(), sequence);
Some(input_ack(client_id, sequence))
}
fn input_ack(client_id: &str, sequence: u64) -> String {
serde_json::to_string(&json!({"type":"input_ack","client_id":client_id,"sequence":sequence}))
.unwrap_or_default()
}
fn handle_control(shim: &Shim, id: u64, text: &str) {
let Ok(value) = serde_json::from_str::<serde_json::Value>(text) else {
return;
};
handle_control_value(shim, id, &value);
}
fn handle_control_value(shim: &Shim, id: u64, value: &serde_json::Value) {
let inner = shim.inner.lock().unwrap();
if inner.controller.as_ref().map(|value| value.attach_id) != Some(id) {
return;
}
match value.get("type").and_then(|v| v.as_str()) {
Some("resize") => {
let rows = value.get("rows").and_then(|v| v.as_u64()).unwrap_or(40) as u16;
let cols = value.get("cols").and_then(|v| v.as_u64()).unwrap_or(120) as u16;
if let Some(pty) = inner.pty.as_ref() {
let _ = pty.resize(PtySize {
rows,
cols,
pixel_width: 0,
pixel_height: 0,
});
}
}
Some("signal") => {
if let Some(name) = value.get("signal").and_then(|v| v.as_str())
&& let Ok(sig) = parse_signal(name)
{
let _ = killpg(Pid::from_raw(inner.process.process_group_id), sig);
}
}
_ => {}
}
}
fn release(shim: &Shim, id: u64) {
let mut inner = shim.inner.lock().unwrap();
if inner.controller.as_ref().map(|value| value.attach_id) == Some(id) {
inner.controller = None;
}
}
enum OutputReplay {
Gap {
requested_after: u64,
oldest_available: u64,
},
Event(OutputEvent),
}
fn collect_output_records(shim: &Shim, after: u64, through: u64) -> Result<Vec<OutputReplay>> {
RecordReader::open(&shim.dir, after, Some(through))?
.map(|record| match record? {
crate::output::LogRecord::Gap(gap) => Ok(OutputReplay::Gap {
requested_after: gap.requested_after,
oldest_available: gap.oldest_available,
}),
crate::output::LogRecord::Output(event) if event.stream == "tty" => {
Ok(OutputReplay::Event(event))
}
crate::output::LogRecord::Output(_) => {
Err(anyhow::anyhow!("unexpected non-TTY output"))
}
})
.collect()
}
async fn send_output_replay(
sink: &mut futures_util::stream::SplitSink<WebSocket, Message>,
record: OutputReplay,
cursor: &mut u64,
) -> Result<(), ()> {
match record {
OutputReplay::Gap {
requested_after,
oldest_available,
} => {
sink.send(Message::Text(
json!({
"type":"output_gap",
"requested_after":requested_after.to_string(),
"oldest_available":oldest_available.to_string()
})
.to_string()
.into(),
))
.await
.map_err(|_| ())?;
*cursor = oldest_available.saturating_sub(1);
}
OutputReplay::Event(event) => {
let data = output_data(event.data.as_ref()).ok_or(())?;
sink.send(Message::Binary(
sequenced_frame(event.sequence, &data).into(),
))
.await
.map_err(|_| ())?;
*cursor = (*cursor).max(event.sequence);
}
}
Ok(())
}
fn output_data(data: Option<&EncodedData>) -> Option<Vec<u8>> {
let data = data?;
match data.encoding.as_str() {
"utf8" => Some(data.text.as_bytes().to_vec()),
"base64" => BASE64.decode(&data.base64).ok(),
_ => None,
}
}
fn sequenced_frame(sequence: u64, data: &[u8]) -> Vec<u8> {
let mut frame = Vec::with_capacity(8 + data.len());
frame.extend_from_slice(&sequence.to_be_bytes());
frame.extend_from_slice(data);
frame
}
async fn shutdown(State(shim): State<Shim>) -> Response {
if !shim.inner.lock().unwrap().process.state.terminal() {
return (StatusCode::CONFLICT, "process still running").into_response();
}
shim.shutdown.notify_one();
Json(json!({"ok":true})).into_response()
}
fn parse_signal(value: &str) -> Result<Signal> {
match value
.trim_start_matches("SIG")
.to_ascii_uppercase()
.as_str()
{
"INT" => Ok(Signal::SIGINT),
"TERM" => Ok(Signal::SIGTERM),
"KILL" => Ok(Signal::SIGKILL),
"HUP" => Ok(Signal::SIGHUP),
"QUIT" => Ok(Signal::SIGQUIT),
"USR1" => Ok(Signal::SIGUSR1),
"USR2" => Ok(Signal::SIGUSR2),
"STOP" => Ok(Signal::SIGSTOP),
"CONT" => Ok(Signal::SIGCONT),
other => bail!("unsupported signal {other}"),
}
}
fn signal_name(value: i32) -> Option<String> {
Signal::try_from(value)
.ok()
.map(|s| format!("{s:?}").trim_start_matches("SIG").to_owned())
}
fn write_ready(fd: i32) {
unsafe {
let byte = [1u8];
libc::write(fd, byte.as_ptr().cast(), 1);
libc::close(fd);
}
}
#[cfg(test)]
mod tests;