use std::collections::VecDeque;
use std::fmt;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use futures::future::BoxFuture;
use graphrt_core::node::Clock;
use tokio::io::AsyncReadExt;
use crate::conn::{Connection, NotifySink};
use crate::discover::verify_checksums;
use crate::error::CallError;
use crate::manifest::{LoadedManifest, ProcessRuntime};
use crate::transport::{INITIALIZE_TIMEOUT, Transport, check_initialize, initialize_params, with_timeout};
pub const DEFAULT_PASS_ENV: &[&str] = &["PATH", "HOME", "LANG", "TMPDIR", "SYSTEMROOT", "USERPROFILE", "APPDATA"];
const STDERR_TAIL_BYTES: usize = 64 * 1024;
const MAX_BACKOFF: Duration = Duration::from_secs(5);
pub const MAX_CONSECUTIVE_TIMEOUTS: u64 = 3;
struct Live {
conn: Arc<Connection>,
init: serde_json::Value,
}
pub struct ProcessTransport {
plugin: LoadedManifest,
runtime: ProcessRuntime,
live: tokio::sync::Mutex<Option<Live>>,
failures: AtomicU64,
timeouts: AtomicU64,
kill_pending: Mutex<Option<String>>,
kill_requested: AtomicBool,
next_id: AtomicU64,
stderr: Arc<Mutex<VecDeque<u8>>>,
exit_status: Arc<Mutex<Option<String>>>,
}
impl fmt::Debug for ProcessTransport {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ProcessTransport")
.field("plugin", &self.plugin.manifest.name)
.field("command", &self.runtime.command)
.finish_non_exhaustive()
}
}
impl ProcessTransport {
pub fn new(plugin: LoadedManifest, runtime: ProcessRuntime) -> Self {
Self {
plugin,
runtime,
live: tokio::sync::Mutex::new(None),
failures: AtomicU64::new(0),
timeouts: AtomicU64::new(0),
kill_pending: Mutex::new(None),
kill_requested: AtomicBool::new(false),
next_id: AtomicU64::new(1),
stderr: Arc::new(Mutex::new(VecDeque::new())),
exit_status: Arc::new(Mutex::new(None)),
}
}
fn kill_current(&self, reason: &str) {
match self.live.try_lock() {
Ok(mut live) => {
if let Some(l) = live.take() {
l.conn.close(reason);
}
}
Err(_) => {
*self.kill_pending.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(reason.to_owned());
self.kill_requested.store(true, Ordering::SeqCst);
}
}
}
pub fn stderr_tail(&self) -> String {
stderr_text(&self.stderr)
}
pub async fn recent_logs(&self) -> Vec<String> {
self.live.lock().await.as_ref().map(|l| l.conn.recent_logs()).unwrap_or_default()
}
async fn connection(&self, clock: &Arc<dyn Clock>) -> Result<(Arc<Connection>, serde_json::Value), CallError> {
let mut live = self.live.lock().await;
if self.kill_requested.swap(false, Ordering::SeqCst) {
let reason = self.kill_pending.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take();
if let Some(l) = live.take() {
l.conn.close(reason.as_deref().unwrap_or("plugin restart requested"));
}
}
if let Some(l) = live.as_ref() {
if !l.conn.is_closed() {
return Ok((l.conn.clone(), l.init.clone()));
}
*live = None;
}
let failures = self.failures.load(Ordering::SeqCst);
if failures > 0 && clock.supports_timers() {
let exp = u32::try_from(failures.min(6)).unwrap_or(6);
let backoff = Duration::from_millis(100u64.saturating_mul(1u64 << exp)).min(MAX_BACKOFF);
clock.sleep(backoff).await;
}
let started = self.start(clock).await;
match started {
Ok(l) => {
let out = (l.conn.clone(), l.init.clone());
*live = Some(l);
Ok(out)
}
Err(e) => {
self.failures.fetch_add(1, Ordering::SeqCst);
Err(e)
}
}
}
async fn start(&self, clock: &Arc<dyn Clock>) -> Result<Live, CallError> {
let name = &self.plugin.manifest.name;
verify_checksums(&self.plugin).map_err(|e| CallError::Start(e.to_string()))?;
let command = std::path::Path::new(&self.runtime.command);
let in_plugin_dir = command.is_relative() && command.components().count() > 1;
let program = if in_plugin_dir { self.plugin.dir.join(command) } else { command.to_path_buf() };
if in_plugin_dir {
let meta = std::fs::symlink_metadata(&program)
.map_err(|e| CallError::Start(format!("cannot stat `{}`: {e}", self.runtime.command)))?;
if meta.file_type().is_symlink() {
return Err(CallError::Start(format!(
"`{}` is a symlink; plugin entrypoints must be files",
self.runtime.command
)));
}
}
let mut cmd = tokio::process::Command::new(&program);
cmd.args(&self.runtime.args);
let cwd = match &self.runtime.cwd {
Some(rel) => self.plugin.resolve(rel),
None => self.plugin.dir.clone(),
};
cmd.current_dir(&cwd);
if !self.runtime.inherit_env {
cmd.env_clear();
let names =
DEFAULT_PASS_ENV.iter().copied().chain(self.plugin.manifest.permissions.env.iter().map(String::as_str));
for var in names {
if let Some(value) = std::env::var_os(var) {
cmd.env(var, value);
}
}
}
cmd.envs(&self.runtime.env);
cmd.env("GRAPHRT_PLUGIN", name);
cmd.env("GRAPHRT_PLUGIN_PROTOCOL", crate::manifest::PROTOCOL_VERSION.to_string());
cmd.stdin(std::process::Stdio::piped());
cmd.stdout(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::piped());
cmd.kill_on_drop(true);
#[cfg(unix)]
cmd.process_group(0);
let mut child =
cmd.spawn().map_err(|e| CallError::Start(format!("cannot run `{}`: {e}", self.runtime.command)))?;
let pid = child.id();
if let Ok(mut status) = self.exit_status.lock() {
*status = None;
}
let stdin = child.stdin.take().ok_or_else(|| CallError::Start("no stdin pipe".into()))?;
let stdout = child.stdout.take().ok_or_else(|| CallError::Start("no stdout pipe".into()))?;
if let Some(mut stderr) = child.stderr.take() {
if let Ok(mut tail) = self.stderr.lock() {
tail.clear();
}
let sink = self.stderr.clone();
tokio::spawn(async move {
let mut buf = [0u8; 4096];
while let Ok(n) = stderr.read(&mut buf).await {
if n == 0 {
break;
}
if let Ok(mut tail) = sink.lock() {
tail.extend(buf.iter().take(n));
while tail.len() > STDERR_TAIL_BYTES {
tail.pop_front();
}
}
}
});
}
let (kill_tx, kill_rx) = tokio::sync::oneshot::channel::<()>();
let exit_status = self.exit_status.clone();
tokio::spawn(async move {
let status = tokio::select! {
status = child.wait() => status,
_ = kill_rx => {
kill_group(pid);
let _already_exited = child.start_kill();
child.wait().await
}
};
kill_group(pid);
if let Ok(mut slot) = exit_status.lock() {
*slot = Some(match status {
Ok(s) => describe_exit(&s),
Err(e) => format!("unknown exit status ({e})"),
});
}
});
let stderr_view = self.stderr.clone();
let status_view = self.exit_status.clone();
let detail: crate::conn::CrashDetail = Arc::new(move || {
let status = status_view.lock().ok().and_then(|s| s.clone());
let stderr = stderr_text(&stderr_view);
match (status, stderr.is_empty()) {
(Some(s), true) => s,
(Some(s), false) => format!("{s}; stderr: {stderr}"),
(None, false) => format!("stderr: {stderr}"),
(None, true) => String::new(),
}
});
let max_message_bytes = self.plugin.manifest.limits.max_message_bytes;
let conn =
Arc::new(Connection::with_guard(stdout, stdin, max_message_bytes, Some(Box::new(kill_tx)), Some(detail)));
conn.set_label(name);
let id = conn.next_id();
let params = initialize_params(max_message_bytes);
let init = with_timeout(clock, INITIALIZE_TIMEOUT, conn.call(id, "initialize", params, None, false))
.await
.map_err(|e| match e {
CallError::Crashed(m) => CallError::Start(m),
other => other,
})?;
check_initialize(&init)?;
Ok(Live { conn, init })
}
}
fn kill_group(pid: Option<u32>) {
#[cfg(unix)]
if let Some(pid) = pid {
let _no_group_left = std::process::Command::new("kill")
.args(["-s", "KILL", "--", &format!("-{pid}")])
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
}
#[cfg(not(unix))]
let _ = pid;
}
fn describe_exit(status: &std::process::ExitStatus) -> String {
if let Some(code) = status.code() {
return format!("process exited with code {code}");
}
#[cfg(unix)]
{
use std::os::unix::process::ExitStatusExt;
if let Some(signal) = status.signal() {
return format!("process killed by signal {signal}");
}
}
"process exited".to_owned()
}
fn stderr_text(buf: &Mutex<VecDeque<u8>>) -> String {
let bytes: Vec<u8> = buf.lock().map(|b| b.iter().copied().collect()).unwrap_or_default();
let text = String::from_utf8_lossy(&bytes);
let trimmed = text.trim();
let start = trimmed.len().saturating_sub(2048);
let start = (start..trimmed.len()).find(|i| trimmed.is_char_boundary(*i)).unwrap_or(trimmed.len());
trimmed.get(start..).unwrap_or_default().to_owned()
}
impl Transport for ProcessTransport {
fn next_id(&self) -> String {
format!("r-{}", self.next_id.fetch_add(1, Ordering::Relaxed))
}
fn call<'a>(
&'a self,
id: String,
method: &'a str,
params: serde_json::Value,
sink: Option<NotifySink>,
cancel_on_drop: bool,
clock: &'a Arc<dyn Clock>,
) -> BoxFuture<'a, Result<serde_json::Value, CallError>> {
Box::pin(async move {
let (conn, _init) = self.connection(clock).await?;
let result = conn.call(id, method, params, sink, cancel_on_drop).await;
match &result {
Err(CallError::Crashed(_)) => {
self.failures.fetch_add(1, Ordering::SeqCst);
}
_ => self.failures.store(0, Ordering::SeqCst),
}
result
})
}
fn initialize<'a>(&'a self, clock: &'a Arc<dyn Clock>) -> BoxFuture<'a, Result<serde_json::Value, CallError>> {
Box::pin(async move { self.connection(clock).await.map(|(_, init)| init) })
}
fn note_timeout(&self) {
let count = self.timeouts.fetch_add(1, Ordering::SeqCst).saturating_add(1);
if count >= MAX_CONSECUTIVE_TIMEOUTS {
self.timeouts.store(0, Ordering::SeqCst);
self.failures.fetch_add(1, Ordering::SeqCst);
self.kill_current(&format!(
"plugin `{}` stopped answering ({count} consecutive timeouts); restarting it",
self.plugin.manifest.name
));
}
}
fn note_answer(&self) {
self.timeouts.store(0, Ordering::SeqCst);
}
}