1use std::collections::VecDeque;
4use std::fmt;
5use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
6use std::sync::{Arc, Mutex};
7use std::time::Duration;
8
9use futures::future::BoxFuture;
10use graphrt_core::node::Clock;
11use tokio::io::AsyncReadExt;
12
13use crate::conn::{Connection, NotifySink};
14use crate::discover::verify_checksums;
15use crate::error::CallError;
16use crate::manifest::{LoadedManifest, ProcessRuntime};
17use crate::transport::{INITIALIZE_TIMEOUT, Transport, check_initialize, initialize_params, with_timeout};
18
19pub const DEFAULT_PASS_ENV: &[&str] = &["PATH", "HOME", "LANG", "TMPDIR", "SYSTEMROOT", "USERPROFILE", "APPDATA"];
21
22const STDERR_TAIL_BYTES: usize = 64 * 1024;
24
25const MAX_BACKOFF: Duration = Duration::from_secs(5);
27
28pub const MAX_CONSECUTIVE_TIMEOUTS: u64 = 3;
31
32struct Live {
33 conn: Arc<Connection>,
34 init: serde_json::Value,
35}
36
37pub struct ProcessTransport {
39 plugin: LoadedManifest,
40 runtime: ProcessRuntime,
41 live: tokio::sync::Mutex<Option<Live>>,
42 failures: AtomicU64,
44 timeouts: AtomicU64,
46 kill_pending: Mutex<Option<String>>,
49 kill_requested: AtomicBool,
51 next_id: AtomicU64,
52 stderr: Arc<Mutex<VecDeque<u8>>>,
53 exit_status: Arc<Mutex<Option<String>>>,
55}
56
57impl fmt::Debug for ProcessTransport {
58 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
59 f.debug_struct("ProcessTransport")
60 .field("plugin", &self.plugin.manifest.name)
61 .field("command", &self.runtime.command)
62 .finish_non_exhaustive()
63 }
64}
65
66impl ProcessTransport {
67 pub fn new(plugin: LoadedManifest, runtime: ProcessRuntime) -> Self {
68 Self {
69 plugin,
70 runtime,
71 live: tokio::sync::Mutex::new(None),
72 failures: AtomicU64::new(0),
73 timeouts: AtomicU64::new(0),
74 kill_pending: Mutex::new(None),
75 kill_requested: AtomicBool::new(false),
76 next_id: AtomicU64::new(1),
77 stderr: Arc::new(Mutex::new(VecDeque::new())),
78 exit_status: Arc::new(Mutex::new(None)),
79 }
80 }
81
82 fn kill_current(&self, reason: &str) {
87 match self.live.try_lock() {
88 Ok(mut live) => {
89 if let Some(l) = live.take() {
90 l.conn.close(reason);
93 }
94 }
95 Err(_) => {
96 *self.kill_pending.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(reason.to_owned());
97 self.kill_requested.store(true, Ordering::SeqCst);
98 }
99 }
100 }
101
102 pub fn stderr_tail(&self) -> String {
104 stderr_text(&self.stderr)
105 }
106
107 pub async fn recent_logs(&self) -> Vec<String> {
109 self.live.lock().await.as_ref().map(|l| l.conn.recent_logs()).unwrap_or_default()
110 }
111
112 async fn connection(&self, clock: &Arc<dyn Clock>) -> Result<(Arc<Connection>, serde_json::Value), CallError> {
114 let mut live = self.live.lock().await;
115 if self.kill_requested.swap(false, Ordering::SeqCst) {
116 let reason = self.kill_pending.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take();
117 if let Some(l) = live.take() {
118 l.conn.close(reason.as_deref().unwrap_or("plugin restart requested"));
119 }
120 }
121 if let Some(l) = live.as_ref() {
122 if !l.conn.is_closed() {
123 return Ok((l.conn.clone(), l.init.clone()));
124 }
125 *live = None;
126 }
127 let failures = self.failures.load(Ordering::SeqCst);
128 if failures > 0 && clock.supports_timers() {
129 let exp = u32::try_from(failures.min(6)).unwrap_or(6);
130 let backoff = Duration::from_millis(100u64.saturating_mul(1u64 << exp)).min(MAX_BACKOFF);
131 clock.sleep(backoff).await;
132 }
133 let started = self.start(clock).await;
134 match started {
135 Ok(l) => {
136 let out = (l.conn.clone(), l.init.clone());
137 *live = Some(l);
138 Ok(out)
139 }
140 Err(e) => {
141 self.failures.fetch_add(1, Ordering::SeqCst);
142 Err(e)
143 }
144 }
145 }
146
147 async fn start(&self, clock: &Arc<dyn Clock>) -> Result<Live, CallError> {
148 let name = &self.plugin.manifest.name;
149 verify_checksums(&self.plugin).map_err(|e| CallError::Start(e.to_string()))?;
150 let command = std::path::Path::new(&self.runtime.command);
153 let in_plugin_dir = command.is_relative() && command.components().count() > 1;
154 let program = if in_plugin_dir { self.plugin.dir.join(command) } else { command.to_path_buf() };
155 if in_plugin_dir {
156 let meta = std::fs::symlink_metadata(&program)
158 .map_err(|e| CallError::Start(format!("cannot stat `{}`: {e}", self.runtime.command)))?;
159 if meta.file_type().is_symlink() {
160 return Err(CallError::Start(format!(
161 "`{}` is a symlink; plugin entrypoints must be files",
162 self.runtime.command
163 )));
164 }
165 }
166 let mut cmd = tokio::process::Command::new(&program);
167 cmd.args(&self.runtime.args);
168 let cwd = match &self.runtime.cwd {
169 Some(rel) => self.plugin.resolve(rel),
170 None => self.plugin.dir.clone(),
171 };
172 cmd.current_dir(&cwd);
173 if !self.runtime.inherit_env {
174 cmd.env_clear();
175 let names =
176 DEFAULT_PASS_ENV.iter().copied().chain(self.plugin.manifest.permissions.env.iter().map(String::as_str));
177 for var in names {
178 if let Some(value) = std::env::var_os(var) {
179 cmd.env(var, value);
180 }
181 }
182 }
183 cmd.envs(&self.runtime.env);
184 cmd.env("GRAPHRT_PLUGIN", name);
185 cmd.env("GRAPHRT_PLUGIN_PROTOCOL", crate::manifest::PROTOCOL_VERSION.to_string());
186 cmd.stdin(std::process::Stdio::piped());
187 cmd.stdout(std::process::Stdio::piped());
188 cmd.stderr(std::process::Stdio::piped());
189 cmd.kill_on_drop(true);
190 #[cfg(unix)]
193 cmd.process_group(0);
194
195 let mut child =
196 cmd.spawn().map_err(|e| CallError::Start(format!("cannot run `{}`: {e}", self.runtime.command)))?;
197 let pid = child.id();
198 if let Ok(mut status) = self.exit_status.lock() {
199 *status = None;
200 }
201 let stdin = child.stdin.take().ok_or_else(|| CallError::Start("no stdin pipe".into()))?;
202 let stdout = child.stdout.take().ok_or_else(|| CallError::Start("no stdout pipe".into()))?;
203 if let Some(mut stderr) = child.stderr.take() {
204 if let Ok(mut tail) = self.stderr.lock() {
205 tail.clear();
206 }
207 let sink = self.stderr.clone();
208 tokio::spawn(async move {
209 let mut buf = [0u8; 4096];
210 while let Ok(n) = stderr.read(&mut buf).await {
211 if n == 0 {
212 break;
213 }
214 if let Ok(mut tail) = sink.lock() {
215 tail.extend(buf.iter().take(n));
216 while tail.len() > STDERR_TAIL_BYTES {
217 tail.pop_front();
218 }
219 }
220 }
221 });
222 }
223
224 let (kill_tx, kill_rx) = tokio::sync::oneshot::channel::<()>();
226 let exit_status = self.exit_status.clone();
227 tokio::spawn(async move {
228 let status = tokio::select! {
229 status = child.wait() => status,
230 _ = kill_rx => {
231 kill_group(pid);
232 let _already_exited = child.start_kill();
233 child.wait().await
234 }
235 };
236 kill_group(pid);
238 if let Ok(mut slot) = exit_status.lock() {
239 *slot = Some(match status {
240 Ok(s) => describe_exit(&s),
241 Err(e) => format!("unknown exit status ({e})"),
242 });
243 }
244 });
245 let stderr_view = self.stderr.clone();
246 let status_view = self.exit_status.clone();
247 let detail: crate::conn::CrashDetail = Arc::new(move || {
248 let status = status_view.lock().ok().and_then(|s| s.clone());
249 let stderr = stderr_text(&stderr_view);
250 match (status, stderr.is_empty()) {
251 (Some(s), true) => s,
252 (Some(s), false) => format!("{s}; stderr: {stderr}"),
253 (None, false) => format!("stderr: {stderr}"),
254 (None, true) => String::new(),
255 }
256 });
257 let max_message_bytes = self.plugin.manifest.limits.max_message_bytes;
258 let conn =
259 Arc::new(Connection::with_guard(stdout, stdin, max_message_bytes, Some(Box::new(kill_tx)), Some(detail)));
260 conn.set_label(name);
261
262 let id = conn.next_id();
263 let params = initialize_params(max_message_bytes);
264 let init = with_timeout(clock, INITIALIZE_TIMEOUT, conn.call(id, "initialize", params, None, false))
265 .await
266 .map_err(|e| match e {
267 CallError::Crashed(m) => CallError::Start(m),
268 other => other,
269 })?;
270 check_initialize(&init)?;
271 Ok(Live { conn, init })
272 }
273}
274
275fn kill_group(pid: Option<u32>) {
278 #[cfg(unix)]
279 if let Some(pid) = pid {
280 let _no_group_left = std::process::Command::new("kill")
281 .args(["-s", "KILL", "--", &format!("-{pid}")])
282 .stdin(std::process::Stdio::null())
283 .stdout(std::process::Stdio::null())
284 .stderr(std::process::Stdio::null())
285 .status();
286 }
287 #[cfg(not(unix))]
288 let _ = pid;
289}
290
291fn describe_exit(status: &std::process::ExitStatus) -> String {
292 if let Some(code) = status.code() {
293 return format!("process exited with code {code}");
294 }
295 #[cfg(unix)]
296 {
297 use std::os::unix::process::ExitStatusExt;
298 if let Some(signal) = status.signal() {
299 return format!("process killed by signal {signal}");
300 }
301 }
302 "process exited".to_owned()
303}
304
305fn stderr_text(buf: &Mutex<VecDeque<u8>>) -> String {
306 let bytes: Vec<u8> = buf.lock().map(|b| b.iter().copied().collect()).unwrap_or_default();
307 let text = String::from_utf8_lossy(&bytes);
308 let trimmed = text.trim();
309 let start = trimmed.len().saturating_sub(2048);
311 let start = (start..trimmed.len()).find(|i| trimmed.is_char_boundary(*i)).unwrap_or(trimmed.len());
312 trimmed.get(start..).unwrap_or_default().to_owned()
313}
314
315impl Transport for ProcessTransport {
316 fn next_id(&self) -> String {
317 format!("r-{}", self.next_id.fetch_add(1, Ordering::Relaxed))
318 }
319
320 fn call<'a>(
321 &'a self,
322 id: String,
323 method: &'a str,
324 params: serde_json::Value,
325 sink: Option<NotifySink>,
326 cancel_on_drop: bool,
327 clock: &'a Arc<dyn Clock>,
328 ) -> BoxFuture<'a, Result<serde_json::Value, CallError>> {
329 Box::pin(async move {
330 let (conn, _init) = self.connection(clock).await?;
331 let result = conn.call(id, method, params, sink, cancel_on_drop).await;
332 match &result {
333 Err(CallError::Crashed(_)) => {
334 self.failures.fetch_add(1, Ordering::SeqCst);
335 }
336 _ => self.failures.store(0, Ordering::SeqCst),
337 }
338 result
339 })
340 }
341
342 fn initialize<'a>(&'a self, clock: &'a Arc<dyn Clock>) -> BoxFuture<'a, Result<serde_json::Value, CallError>> {
343 Box::pin(async move { self.connection(clock).await.map(|(_, init)| init) })
344 }
345
346 fn note_timeout(&self) {
347 let count = self.timeouts.fetch_add(1, Ordering::SeqCst).saturating_add(1);
348 if count >= MAX_CONSECUTIVE_TIMEOUTS {
349 self.timeouts.store(0, Ordering::SeqCst);
350 self.failures.fetch_add(1, Ordering::SeqCst);
351 self.kill_current(&format!(
352 "plugin `{}` stopped answering ({count} consecutive timeouts); restarting it",
353 self.plugin.manifest.name
354 ));
355 }
356 }
357
358 fn note_answer(&self) {
359 self.timeouts.store(0, Ordering::SeqCst);
360 }
361}