Skip to main content

graphrt_plugin/
process.rs

1//! Child-process plugins: lazy start, crash detection and restart with backoff.
2
3use std::collections::VecDeque;
4use std::fmt;
5use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
6use std::sync::{Arc, Mutex};
7use std::time::Duration;
8
9use futures::future::BoxFuture;
10use graphrt_core::node::Clock;
11use tokio::io::AsyncReadExt;
12
13use crate::conn::{Connection, NotifySink};
14use crate::discover::verify_checksums;
15use crate::error::CallError;
16use crate::manifest::{LoadedManifest, ProcessRuntime};
17use crate::transport::{INITIALIZE_TIMEOUT, Transport, check_initialize, initialize_params, with_timeout};
18
19/// Host variables always passed to plugin processes.
20pub const DEFAULT_PASS_ENV: &[&str] = &["PATH", "HOME", "LANG", "TMPDIR", "SYSTEMROOT", "USERPROFILE", "APPDATA"];
21
22/// How much plugin stderr is kept for diagnostics.
23const STDERR_TAIL_BYTES: usize = 64 * 1024;
24
25/// Upper bound of the restart backoff.
26const MAX_BACKOFF: Duration = Duration::from_secs(5);
27
28/// Consecutive request timeouts after which a plugin process is considered
29/// wedged and is killed; the next request starts a fresh process.
30pub const MAX_CONSECUTIVE_TIMEOUTS: u64 = 3;
31
32struct Live {
33    conn: Arc<Connection>,
34    init: serde_json::Value,
35}
36
37/// Supervises one plugin process.
38pub struct ProcessTransport {
39    plugin: LoadedManifest,
40    runtime: ProcessRuntime,
41    live: tokio::sync::Mutex<Option<Live>>,
42    /// Consecutive failed starts / crashes, used for backoff.
43    failures: AtomicU64,
44    /// Consecutive request timeouts of the current process.
45    timeouts: AtomicU64,
46    /// A kill was requested while the connection lock was busy; the next
47    /// `connection()` call performs it before handing out the process.
48    kill_pending: Mutex<Option<String>>,
49    /// Fast-path flag mirroring `kill_pending.is_some()`.
50    kill_requested: AtomicBool,
51    next_id: AtomicU64,
52    stderr: Arc<Mutex<VecDeque<u8>>>,
53    /// How the last process exited (code or signal), for crash reports.
54    exit_status: Arc<Mutex<Option<String>>>,
55}
56
57impl fmt::Debug for ProcessTransport {
58    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
59        f.debug_struct("ProcessTransport")
60            .field("plugin", &self.plugin.manifest.name)
61            .field("command", &self.runtime.command)
62            .finish_non_exhaustive()
63    }
64}
65
66impl ProcessTransport {
67    pub fn new(plugin: LoadedManifest, runtime: ProcessRuntime) -> Self {
68        Self {
69            plugin,
70            runtime,
71            live: tokio::sync::Mutex::new(None),
72            failures: AtomicU64::new(0),
73            timeouts: AtomicU64::new(0),
74            kill_pending: Mutex::new(None),
75            kill_requested: AtomicBool::new(false),
76            next_id: AtomicU64::new(1),
77            stderr: Arc::new(Mutex::new(VecDeque::new())),
78            exit_status: Arc::new(Mutex::new(None)),
79        }
80    }
81
82    /// Kill the current process (if any). The next request starts a new one.
83    ///
84    /// When the connection lock is busy the kill is recorded and performed by the
85    /// next `connection()` call, so a wedged process is never silently kept.
86    fn kill_current(&self, reason: &str) {
87        match self.live.try_lock() {
88            Ok(mut live) => {
89                if let Some(l) = live.take() {
90                    // Closing fails in-flight requests; once they release the connection
91                    // its guard kills the process group.
92                    l.conn.close(reason);
93                }
94            }
95            Err(_) => {
96                *self.kill_pending.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(reason.to_owned());
97                self.kill_requested.store(true, Ordering::SeqCst);
98            }
99        }
100    }
101
102    /// The last bytes the plugin wrote to stderr.
103    pub fn stderr_tail(&self) -> String {
104        stderr_text(&self.stderr)
105    }
106
107    /// Recent `log` notifications of the running process.
108    pub async fn recent_logs(&self) -> Vec<String> {
109        self.live.lock().await.as_ref().map(|l| l.conn.recent_logs()).unwrap_or_default()
110    }
111
112    /// Return a live, initialized connection, starting the process if needed.
113    async fn connection(&self, clock: &Arc<dyn Clock>) -> Result<(Arc<Connection>, serde_json::Value), CallError> {
114        let mut live = self.live.lock().await;
115        if self.kill_requested.swap(false, Ordering::SeqCst) {
116            let reason = self.kill_pending.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take();
117            if let Some(l) = live.take() {
118                l.conn.close(reason.as_deref().unwrap_or("plugin restart requested"));
119            }
120        }
121        if let Some(l) = live.as_ref() {
122            if !l.conn.is_closed() {
123                return Ok((l.conn.clone(), l.init.clone()));
124            }
125            *live = None;
126        }
127        let failures = self.failures.load(Ordering::SeqCst);
128        if failures > 0 && clock.supports_timers() {
129            let exp = u32::try_from(failures.min(6)).unwrap_or(6);
130            let backoff = Duration::from_millis(100u64.saturating_mul(1u64 << exp)).min(MAX_BACKOFF);
131            clock.sleep(backoff).await;
132        }
133        let started = self.start(clock).await;
134        match started {
135            Ok(l) => {
136                let out = (l.conn.clone(), l.init.clone());
137                *live = Some(l);
138                Ok(out)
139            }
140            Err(e) => {
141                self.failures.fetch_add(1, Ordering::SeqCst);
142                Err(e)
143            }
144        }
145    }
146
147    async fn start(&self, clock: &Arc<dyn Clock>) -> Result<Live, CallError> {
148        let name = &self.plugin.manifest.name;
149        verify_checksums(&self.plugin).map_err(|e| CallError::Start(e.to_string()))?;
150        // A relative command with a path separator (`./bin/plugin`) is relative to the
151        // plugin directory; a bare name (`python3`) is looked up on PATH.
152        let command = std::path::Path::new(&self.runtime.command);
153        let in_plugin_dir = command.is_relative() && command.components().count() > 1;
154        let program = if in_plugin_dir { self.plugin.dir.join(command) } else { command.to_path_buf() };
155        if in_plugin_dir {
156            // A symlinked entrypoint could point anywhere, bypassing checksums.
157            let meta = std::fs::symlink_metadata(&program)
158                .map_err(|e| CallError::Start(format!("cannot stat `{}`: {e}", self.runtime.command)))?;
159            if meta.file_type().is_symlink() {
160                return Err(CallError::Start(format!(
161                    "`{}` is a symlink; plugin entrypoints must be files",
162                    self.runtime.command
163                )));
164            }
165        }
166        let mut cmd = tokio::process::Command::new(&program);
167        cmd.args(&self.runtime.args);
168        let cwd = match &self.runtime.cwd {
169            Some(rel) => self.plugin.resolve(rel),
170            None => self.plugin.dir.clone(),
171        };
172        cmd.current_dir(&cwd);
173        if !self.runtime.inherit_env {
174            cmd.env_clear();
175            let names =
176                DEFAULT_PASS_ENV.iter().copied().chain(self.plugin.manifest.permissions.env.iter().map(String::as_str));
177            for var in names {
178                if let Some(value) = std::env::var_os(var) {
179                    cmd.env(var, value);
180                }
181            }
182        }
183        cmd.envs(&self.runtime.env);
184        cmd.env("GRAPHRT_PLUGIN", name);
185        cmd.env("GRAPHRT_PLUGIN_PROTOCOL", crate::manifest::PROTOCOL_VERSION.to_string());
186        cmd.stdin(std::process::Stdio::piped());
187        cmd.stdout(std::process::Stdio::piped());
188        cmd.stderr(std::process::Stdio::piped());
189        cmd.kill_on_drop(true);
190        // Own process group, so wrappers (`sh run.sh`, `npx`, `uv run`) and every
191        // process they start are killed together.
192        #[cfg(unix)]
193        cmd.process_group(0);
194
195        let mut child =
196            cmd.spawn().map_err(|e| CallError::Start(format!("cannot run `{}`: {e}", self.runtime.command)))?;
197        let pid = child.id();
198        if let Ok(mut status) = self.exit_status.lock() {
199            *status = None;
200        }
201        let stdin = child.stdin.take().ok_or_else(|| CallError::Start("no stdin pipe".into()))?;
202        let stdout = child.stdout.take().ok_or_else(|| CallError::Start("no stdout pipe".into()))?;
203        if let Some(mut stderr) = child.stderr.take() {
204            if let Ok(mut tail) = self.stderr.lock() {
205                tail.clear();
206            }
207            let sink = self.stderr.clone();
208            tokio::spawn(async move {
209                let mut buf = [0u8; 4096];
210                while let Ok(n) = stderr.read(&mut buf).await {
211                    if n == 0 {
212                        break;
213                    }
214                    if let Ok(mut tail) = sink.lock() {
215                        tail.extend(buf.iter().take(n));
216                        while tail.len() > STDERR_TAIL_BYTES {
217                            tail.pop_front();
218                        }
219                    }
220                }
221            });
222        }
223
224        // The guard owns the child: dropping the connection kills the process group.
225        let (kill_tx, kill_rx) = tokio::sync::oneshot::channel::<()>();
226        let exit_status = self.exit_status.clone();
227        tokio::spawn(async move {
228            let status = tokio::select! {
229                status = child.wait() => status,
230                _ = kill_rx => {
231                    kill_group(pid);
232                    let _already_exited = child.start_kill();
233                    child.wait().await
234                }
235            };
236            // The main process is gone; reap anything it left in its group.
237            kill_group(pid);
238            if let Ok(mut slot) = exit_status.lock() {
239                *slot = Some(match status {
240                    Ok(s) => describe_exit(&s),
241                    Err(e) => format!("unknown exit status ({e})"),
242                });
243            }
244        });
245        let stderr_view = self.stderr.clone();
246        let status_view = self.exit_status.clone();
247        let detail: crate::conn::CrashDetail = Arc::new(move || {
248            let status = status_view.lock().ok().and_then(|s| s.clone());
249            let stderr = stderr_text(&stderr_view);
250            match (status, stderr.is_empty()) {
251                (Some(s), true) => s,
252                (Some(s), false) => format!("{s}; stderr: {stderr}"),
253                (None, false) => format!("stderr: {stderr}"),
254                (None, true) => String::new(),
255            }
256        });
257        let max_message_bytes = self.plugin.manifest.limits.max_message_bytes;
258        let conn =
259            Arc::new(Connection::with_guard(stdout, stdin, max_message_bytes, Some(Box::new(kill_tx)), Some(detail)));
260        conn.set_label(name);
261
262        let id = conn.next_id();
263        let params = initialize_params(max_message_bytes);
264        let init = with_timeout(clock, INITIALIZE_TIMEOUT, conn.call(id, "initialize", params, None, false))
265            .await
266            .map_err(|e| match e {
267                CallError::Crashed(m) => CallError::Start(m),
268                other => other,
269            })?;
270        check_initialize(&init)?;
271        Ok(Live { conn, init })
272    }
273}
274
275/// Kill every process in the plugin's process group (Unix). Uses the `kill`
276/// utility, so no unsafe libc call is needed.
277fn kill_group(pid: Option<u32>) {
278    #[cfg(unix)]
279    if let Some(pid) = pid {
280        let _no_group_left = std::process::Command::new("kill")
281            .args(["-s", "KILL", "--", &format!("-{pid}")])
282            .stdin(std::process::Stdio::null())
283            .stdout(std::process::Stdio::null())
284            .stderr(std::process::Stdio::null())
285            .status();
286    }
287    #[cfg(not(unix))]
288    let _ = pid;
289}
290
291fn describe_exit(status: &std::process::ExitStatus) -> String {
292    if let Some(code) = status.code() {
293        return format!("process exited with code {code}");
294    }
295    #[cfg(unix)]
296    {
297        use std::os::unix::process::ExitStatusExt;
298        if let Some(signal) = status.signal() {
299            return format!("process killed by signal {signal}");
300        }
301    }
302    "process exited".to_owned()
303}
304
305fn stderr_text(buf: &Mutex<VecDeque<u8>>) -> String {
306    let bytes: Vec<u8> = buf.lock().map(|b| b.iter().copied().collect()).unwrap_or_default();
307    let text = String::from_utf8_lossy(&bytes);
308    let trimmed = text.trim();
309    // Keep error messages readable: the last ~2 KiB are enough to diagnose a crash.
310    let start = trimmed.len().saturating_sub(2048);
311    let start = (start..trimmed.len()).find(|i| trimmed.is_char_boundary(*i)).unwrap_or(trimmed.len());
312    trimmed.get(start..).unwrap_or_default().to_owned()
313}
314
315impl Transport for ProcessTransport {
316    fn next_id(&self) -> String {
317        format!("r-{}", self.next_id.fetch_add(1, Ordering::Relaxed))
318    }
319
320    fn call<'a>(
321        &'a self,
322        id: String,
323        method: &'a str,
324        params: serde_json::Value,
325        sink: Option<NotifySink>,
326        cancel_on_drop: bool,
327        clock: &'a Arc<dyn Clock>,
328    ) -> BoxFuture<'a, Result<serde_json::Value, CallError>> {
329        Box::pin(async move {
330            let (conn, _init) = self.connection(clock).await?;
331            let result = conn.call(id, method, params, sink, cancel_on_drop).await;
332            match &result {
333                Err(CallError::Crashed(_)) => {
334                    self.failures.fetch_add(1, Ordering::SeqCst);
335                }
336                _ => self.failures.store(0, Ordering::SeqCst),
337            }
338            result
339        })
340    }
341
342    fn initialize<'a>(&'a self, clock: &'a Arc<dyn Clock>) -> BoxFuture<'a, Result<serde_json::Value, CallError>> {
343        Box::pin(async move { self.connection(clock).await.map(|(_, init)| init) })
344    }
345
346    fn note_timeout(&self) {
347        let count = self.timeouts.fetch_add(1, Ordering::SeqCst).saturating_add(1);
348        if count >= MAX_CONSECUTIVE_TIMEOUTS {
349            self.timeouts.store(0, Ordering::SeqCst);
350            self.failures.fetch_add(1, Ordering::SeqCst);
351            self.kill_current(&format!(
352                "plugin `{}` stopped answering ({count} consecutive timeouts); restarting it",
353                self.plugin.manifest.name
354            ));
355        }
356    }
357
358    fn note_answer(&self) {
359        self.timeouts.store(0, Ordering::SeqCst);
360    }
361}