Skip to main content

weirdnet_ext/
supervisor.rs

1//! The supervisor: starts an installed extension in the sandbox, serves the broker on a
2//! private socket pair, restarts it with backoff when it crashes, stops it when idle or
3//! asked, and passes events and grant changes to it.
4//!
5//! Every start checks the installed files again ([`Installer::verify`]), makes a new socket
6//! pair and a new one-time token, so a session never outlives its process.
7
8use crate::broker::{Broker, LogLine, Services, Shared, random_hex};
9use crate::error::{Error, Result};
10use crate::grants::Grants;
11use crate::package::Installer;
12use crate::sandbox::{self, Policy, Report};
13use std::path::PathBuf;
14use std::sync::Arc;
15use std::sync::atomic::{AtomicU64, Ordering};
16use std::time::{Duration, Instant};
17use tokio::sync::{mpsc, oneshot, watch};
18use weirdnet_ext_proto::v1;
19
20/// Restart backoff.
21#[derive(Debug, Clone, Copy)]
22pub struct Backoff {
23    /// The first wait.
24    pub initial: Duration,
25    /// The longest wait (it doubles up to this).
26    pub max: Duration,
27    /// A run this long resets the backoff.
28    pub reset_after: Duration,
29    /// Give up after this many crashes in a row (each shorter than `reset_after`).
30    pub max_failures: u32,
31}
32
33impl Default for Backoff {
34    fn default() -> Self {
35        Self { initial: Duration::from_secs(1), max: Duration::from_secs(60), reset_after: Duration::from_secs(60), max_failures: 8 }
36    }
37}
38
39/// How to run an extension.
40#[derive(Clone)]
41pub struct Config {
42    /// The host app and version, told to the extension ("morph 0.5.0").
43    pub host: String,
44    /// Where the extensions are installed.
45    pub installer: Installer,
46    /// The extension id.
47    pub id: String,
48    /// Its data directory (the only place it can write).
49    pub data_dir: PathBuf,
50    /// Extra arguments for the binary.
51    pub args: Vec<String>,
52    /// Extra paths or limits; `None` uses [`Policy::for_extension`].
53    pub policy: Option<Policy>,
54    /// What the user granted.
55    pub grants: Grants,
56    /// The app's services.
57    pub services: Arc<dyn Services>,
58    /// Restart backoff.
59    pub backoff: Backoff,
60    /// Stop after this long without calls or events.
61    pub idle: Option<Duration>,
62    /// How long a stop waits before killing.
63    pub stop_grace: Duration,
64    /// Where extension log lines go besides `tracing`.
65    pub logs: Option<mpsc::Sender<LogLine>>,
66}
67
68impl Config {
69    /// A config with defaults: no extra args, the default policy, default backoff, no idle
70    /// stop, 3 s stop grace.
71    pub fn new(
72        host: impl Into<String>,
73        installer: Installer,
74        id: impl Into<String>,
75        data_dir: impl Into<PathBuf>,
76        grants: Grants,
77        services: Arc<dyn Services>,
78    ) -> Self {
79        Self {
80            host: host.into(),
81            installer,
82            id: id.into(),
83            data_dir: data_dir.into(),
84            args: Vec::new(),
85            policy: None,
86            grants,
87            services,
88            backoff: Backoff::default(),
89            idle: None,
90            stop_grace: Duration::from_secs(3),
91            logs: None,
92        }
93    }
94}
95
96/// Where an extension is.
97#[derive(Debug, Clone, PartialEq, Eq)]
98pub enum Status {
99    /// Being checked and started.
100    Starting,
101    /// Running.
102    Running {
103        /// Its process id.
104        pid: u32,
105        /// How many times it was started.
106        starts: u32,
107        /// What the sandbox did.
108        sandbox: Report,
109    },
110    /// Crashed; restarting after a wait.
111    Restarting {
112        /// How it ended.
113        exit: String,
114        /// The wait.
115        after: Duration,
116    },
117    /// Stopped (asked, idle, or exited cleanly).
118    Stopped {
119        /// Why.
120        reason: String,
121    },
122    /// Gave up (it can't start, failed its checks, or crashed too often).
123    Failed {
124        /// Why.
125        error: String,
126    },
127}
128
129impl Status {
130    /// True once it won't run again.
131    pub fn is_final(&self) -> bool {
132        matches!(self, Status::Stopped { .. } | Status::Failed { .. })
133    }
134}
135
136enum Cmd {
137    Stop(String, oneshot::Sender<()>),
138    Event(v1::Event),
139}
140
141/// A running extension.
142#[derive(Clone)]
143pub struct Handle {
144    cmd: mpsc::Sender<Cmd>,
145    status: watch::Receiver<Status>,
146    health: watch::Receiver<Option<v1::HealthReport>>,
147    grants: Arc<watch::Sender<Grants>>,
148}
149
150impl Handle {
151    /// The current status.
152    pub fn status(&self) -> Status {
153        self.status.borrow().clone()
154    }
155
156    /// Watches the status.
157    pub fn watch(&self) -> watch::Receiver<Status> {
158        self.status.clone()
159    }
160
161    /// The last health report.
162    pub fn health(&self) -> Option<v1::HealthReport> {
163        self.health.borrow().clone()
164    }
165
166    /// Waits for a status matching `f`.
167    pub async fn wait_for(&self, f: impl Fn(&Status) -> bool) -> Status {
168        let mut w = self.status.clone();
169        w.wait_for(|s| f(s)).await.map(|s| s.clone()).unwrap_or_else(|_| self.status())
170    }
171
172    /// Sends an app event (dropped if the extension isn't running).
173    pub async fn send(&self, kind: impl Into<String>, payload: Vec<u8>) {
174        let e = v1::Event { kind: Some(v1::event::Kind::App(v1::AppEvent { kind: kind.into(), payload })) };
175        let _ = self.cmd.send(Cmd::Event(e)).await;
176    }
177
178    /// Asks for a health report.
179    pub async fn ping(&self, seq: u64) {
180        let _ = self.cmd.send(Cmd::Event(v1::Event { kind: Some(v1::event::Kind::Ping(v1::Ping { seq })) })).await;
181    }
182
183    /// Replaces the grants: checked on the next call, and told to the extension.
184    pub async fn set_grants(&self, g: Grants) {
185        let proto = g.to_proto();
186        self.grants.send_replace(g);
187        let _ = self.cmd.send(Cmd::Event(v1::Event { kind: Some(v1::event::Kind::Grants(v1::Grants { granted: proto })) })).await;
188    }
189
190    /// Stops the extension: a Stop event, then a kill after the grace period.
191    pub async fn stop(&self, reason: impl Into<String>) {
192        let (tx, rx) = oneshot::channel();
193        if self.cmd.send(Cmd::Stop(reason.into(), tx)).await.is_ok() {
194            let _ = rx.await;
195        }
196        self.wait_for(Status::is_final).await;
197    }
198}
199
200/// Starts an extension under supervision.
201pub fn start(config: Config) -> Handle {
202    let (cmd_tx, cmd_rx) = mpsc::channel(64);
203    let (status_tx, status_rx) = watch::channel(Status::Starting);
204    let (health_tx, health_rx) = watch::channel(None);
205    let grants = Arc::new(watch::channel(config.grants.clone()).0);
206    let h = Handle { cmd: cmd_tx, status: status_rx, health: health_rx, grants: grants.clone() };
207    tokio::spawn(run(config, cmd_rx, status_tx, health_tx, grants));
208    h
209}
210
211enum Ended {
212    Stopped(String),
213    Exited(std::process::ExitStatus, Duration),
214}
215
216async fn run(
217    config: Config,
218    mut cmd: mpsc::Receiver<Cmd>,
219    status: watch::Sender<Status>,
220    health: watch::Sender<Option<v1::HealthReport>>,
221    grants: Arc<watch::Sender<Grants>>,
222) {
223    let mut starts = 0u32;
224    let mut failures = 0u32;
225    let mut wait = config.backoff.initial;
226    loop {
227        status.send_replace(Status::Starting);
228        starts += 1;
229        let ended = match once(&config, &mut cmd, &status, &health, &grants, starts).await {
230            Ok(e) => e,
231            Err(e) => {
232                status.send_replace(Status::Failed { error: e.to_string() });
233                return;
234            }
235        };
236        match ended {
237            Ended::Stopped(reason) => {
238                status.send_replace(Status::Stopped { reason });
239                return;
240            }
241            Ended::Exited(st, ran) => {
242                if st.success() {
243                    status.send_replace(Status::Stopped { reason: "exited".into() });
244                    return;
245                }
246                if ran >= config.backoff.reset_after {
247                    failures = 0;
248                    wait = config.backoff.initial;
249                }
250                failures += 1;
251                if failures >= config.backoff.max_failures {
252                    status.send_replace(Status::Failed { error: format!("crashed {failures} times in a row (last: {})", describe(&st)) });
253                    return;
254                }
255                status.send_replace(Status::Restarting { exit: describe(&st), after: wait });
256                let sleep = tokio::time::sleep(wait);
257                tokio::pin!(sleep);
258                loop {
259                    tokio::select! {
260                        _ = &mut sleep => break,
261                        c = cmd.recv() => match c {
262                            Some(Cmd::Stop(reason, done)) => {
263                                let _ = done.send(());
264                                status.send_replace(Status::Stopped { reason });
265                                return;
266                            }
267                            Some(Cmd::Event(_)) => {}
268                            None => return,
269                        }
270                    }
271                }
272                wait = (wait * 2).min(config.backoff.max);
273            }
274        }
275    }
276}
277
278fn describe(st: &std::process::ExitStatus) -> String {
279    #[cfg(unix)]
280    {
281        use std::os::unix::process::ExitStatusExt;
282        if let Some(s) = st.signal() {
283            return format!("killed by signal {s}");
284        }
285    }
286    match st.code() {
287        Some(c) => format!("exit code {c}"),
288        None => "ended".into(),
289    }
290}
291
292#[cfg(unix)]
293fn kill_group(pid: u32) {
294    // SAFETY: a signal to the process group the child leads.
295    unsafe {
296        libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
297    }
298}
299
300#[cfg(unix)]
301async fn once(
302    config: &Config,
303    cmd: &mut mpsc::Receiver<Cmd>,
304    status: &watch::Sender<Status>,
305    health: &watch::Sender<Option<v1::HealthReport>>,
306    grants: &Arc<watch::Sender<Grants>>,
307    starts: u32,
308) -> Result<Ended> {
309    use std::os::fd::AsRawFd;
310    let installed = config.installer.verify(&config.id)?;
311    let binary =
312        installed.native().ok_or_else(|| Error::Unsupported(format!("{} has no binary for {}", config.id, crate::manifest::current_target())))?;
313    let policy = config.policy.clone().unwrap_or_else(|| Policy::for_extension(&installed.dir, &config.data_dir));
314    let (host_end, ext_end) = std::os::unix::net::UnixStream::pair()?;
315    host_end.set_nonblocking(true)?;
316    let token = random_hex(32);
317    let fd = ext_end.as_raw_fd();
318    let env = vec![
319        (weirdnet_ext_proto::ENV_FD.to_string(), fd.to_string()),
320        (weirdnet_ext_proto::ENV_TOKEN.to_string(), token.clone()),
321        (weirdnet_ext_proto::ENV_ID.to_string(), config.id.clone()),
322        ("HOME".to_string(), config.data_dir.display().to_string()),
323        ("TMPDIR".to_string(), config.data_dir.display().to_string()),
324        ("RUST_BACKTRACE".to_string(), "1".to_string()),
325    ];
326    let (mut command, report) = sandbox::command(&binary, &config.args, &policy, fd, &env)?;
327    let mut child = command.spawn().map_err(|e| Error::Process(format!("starting {}: {e}", binary.display())))?;
328    drop(ext_end);
329    let pid = child.id().unwrap_or(0);
330    let started = Instant::now();
331
332    let (ev_tx, ev_rx) = mpsc::channel(64);
333    let shared = Arc::new(Shared { started, last_activity: AtomicU64::new(0), health: health.clone(), logs: config.logs.clone() });
334    let broker = Broker::new(
335        config.id.clone(),
336        installed.manifest.version.clone(),
337        config.host.clone(),
338        token,
339        grants.subscribe(),
340        ev_rx,
341        config.services.clone(),
342        shared.clone(),
343    );
344    let io = tokio::net::UnixStream::from_std(host_end)?;
345    let (shut_tx, shut_rx) = oneshot::channel::<()>();
346    let server = tokio::spawn(async move {
347        // One connection; the stream then stays open, because a server whose incoming
348        // stream ends shuts its connections down.
349        let incoming = tokio_stream::StreamExt::chain(tokio_stream::once(Ok::<_, std::io::Error>(io)), tokio_stream::pending());
350        let _ = tonic::transport::Server::builder()
351            .add_service(v1::host_server::HostServer::new(broker))
352            .serve_with_incoming_shutdown(incoming, async {
353                let _ = shut_rx.await;
354            })
355            .await;
356    });
357    status.send_replace(Status::Running { pid, starts, sandbox: report });
358
359    let idle_tick = config.idle.map(|d| d.min(Duration::from_secs(1)).max(Duration::from_millis(50)));
360    let mut tick = tokio::time::interval(idle_tick.unwrap_or(Duration::from_secs(3600)));
361    let stop = |reason: String| async move { reason };
362    let ended = loop {
363        tokio::select! {
364            st = child.wait() => {
365                let st = st?;
366                break Ended::Exited(st, started.elapsed());
367            }
368            c = cmd.recv() => match c {
369                Some(Cmd::Event(e)) => {
370                    shared.touch();
371                    let _ = ev_tx.try_send(e);
372                }
373                Some(Cmd::Stop(reason, done)) => {
374                    graceful(&mut child, &ev_tx, pid, config.stop_grace, &reason).await;
375                    let _ = done.send(());
376                    break Ended::Stopped(stop(reason).await);
377                }
378                None => {
379                    graceful(&mut child, &ev_tx, pid, config.stop_grace, "host gone").await;
380                    break Ended::Stopped("host gone".into());
381                }
382            },
383            _ = tick.tick(), if config.idle.is_some() => {
384                let idle = config.idle.unwrap();
385                let last = Duration::from_millis(shared.last_activity.load(Ordering::Relaxed));
386                if started.elapsed().saturating_sub(last) >= idle {
387                    graceful(&mut child, &ev_tx, pid, config.stop_grace, "idle").await;
388                    break Ended::Stopped("idle".into());
389                }
390            }
391        }
392    };
393    kill_group(pid);
394    let _ = shut_tx.send(());
395    server.abort();
396    Ok(ended)
397}
398
399#[cfg(unix)]
400async fn graceful(child: &mut tokio::process::Child, ev: &mpsc::Sender<v1::Event>, pid: u32, grace: Duration, reason: &str) {
401    let _ = ev.try_send(v1::Event { kind: Some(v1::event::Kind::Stop(v1::Stop { reason: reason.into(), grace_ms: grace.as_millis() as u32 })) });
402    if tokio::time::timeout(grace, child.wait()).await.is_err() {
403        kill_group(pid);
404        let _ = child.wait().await;
405    }
406}
407
408#[cfg(not(unix))]
409async fn once(
410    config: &Config,
411    _: &mut mpsc::Receiver<Cmd>,
412    _: &watch::Sender<Status>,
413    _: &watch::Sender<Option<v1::HealthReport>>,
414    _: &Arc<watch::Sender<Grants>>,
415    _: u32,
416) -> Result<Ended> {
417    config.installer.verify(&config.id)?;
418    Err(Error::Unsupported("native extensions don't run on this OS yet; use the WASM build".into()))
419}