1use crate::broker::{Broker, LogLine, Services, Shared, random_hex};
9use crate::error::{Error, Result};
10use crate::grants::Grants;
11use crate::package::Installer;
12use crate::sandbox::{self, Policy, Report};
13use std::path::PathBuf;
14use std::sync::Arc;
15use std::sync::atomic::{AtomicU64, Ordering};
16use std::time::{Duration, Instant};
17use tokio::sync::{mpsc, oneshot, watch};
18use weirdnet_ext_proto::v1;
19
20#[derive(Debug, Clone, Copy)]
22pub struct Backoff {
23 pub initial: Duration,
25 pub max: Duration,
27 pub reset_after: Duration,
29 pub max_failures: u32,
31}
32
33impl Default for Backoff {
34 fn default() -> Self {
35 Self { initial: Duration::from_secs(1), max: Duration::from_secs(60), reset_after: Duration::from_secs(60), max_failures: 8 }
36 }
37}
38
39#[derive(Clone)]
41pub struct Config {
42 pub host: String,
44 pub installer: Installer,
46 pub id: String,
48 pub data_dir: PathBuf,
50 pub args: Vec<String>,
52 pub policy: Option<Policy>,
54 pub grants: Grants,
56 pub services: Arc<dyn Services>,
58 pub backoff: Backoff,
60 pub idle: Option<Duration>,
62 pub stop_grace: Duration,
64 pub logs: Option<mpsc::Sender<LogLine>>,
66}
67
68impl Config {
69 pub fn new(
72 host: impl Into<String>,
73 installer: Installer,
74 id: impl Into<String>,
75 data_dir: impl Into<PathBuf>,
76 grants: Grants,
77 services: Arc<dyn Services>,
78 ) -> Self {
79 Self {
80 host: host.into(),
81 installer,
82 id: id.into(),
83 data_dir: data_dir.into(),
84 args: Vec::new(),
85 policy: None,
86 grants,
87 services,
88 backoff: Backoff::default(),
89 idle: None,
90 stop_grace: Duration::from_secs(3),
91 logs: None,
92 }
93 }
94}
95
96#[derive(Debug, Clone, PartialEq, Eq)]
98pub enum Status {
99 Starting,
101 Running {
103 pid: u32,
105 starts: u32,
107 sandbox: Report,
109 },
110 Restarting {
112 exit: String,
114 after: Duration,
116 },
117 Stopped {
119 reason: String,
121 },
122 Failed {
124 error: String,
126 },
127}
128
129impl Status {
130 pub fn is_final(&self) -> bool {
132 matches!(self, Status::Stopped { .. } | Status::Failed { .. })
133 }
134}
135
136enum Cmd {
137 Stop(String, oneshot::Sender<()>),
138 Event(v1::Event),
139}
140
141#[derive(Clone)]
143pub struct Handle {
144 cmd: mpsc::Sender<Cmd>,
145 status: watch::Receiver<Status>,
146 health: watch::Receiver<Option<v1::HealthReport>>,
147 grants: Arc<watch::Sender<Grants>>,
148}
149
150impl Handle {
151 pub fn status(&self) -> Status {
153 self.status.borrow().clone()
154 }
155
156 pub fn watch(&self) -> watch::Receiver<Status> {
158 self.status.clone()
159 }
160
161 pub fn health(&self) -> Option<v1::HealthReport> {
163 self.health.borrow().clone()
164 }
165
166 pub async fn wait_for(&self, f: impl Fn(&Status) -> bool) -> Status {
168 let mut w = self.status.clone();
169 w.wait_for(|s| f(s)).await.map(|s| s.clone()).unwrap_or_else(|_| self.status())
170 }
171
172 pub async fn send(&self, kind: impl Into<String>, payload: Vec<u8>) {
174 let e = v1::Event { kind: Some(v1::event::Kind::App(v1::AppEvent { kind: kind.into(), payload })) };
175 let _ = self.cmd.send(Cmd::Event(e)).await;
176 }
177
178 pub async fn ping(&self, seq: u64) {
180 let _ = self.cmd.send(Cmd::Event(v1::Event { kind: Some(v1::event::Kind::Ping(v1::Ping { seq })) })).await;
181 }
182
183 pub async fn set_grants(&self, g: Grants) {
185 let proto = g.to_proto();
186 self.grants.send_replace(g);
187 let _ = self.cmd.send(Cmd::Event(v1::Event { kind: Some(v1::event::Kind::Grants(v1::Grants { granted: proto })) })).await;
188 }
189
190 pub async fn stop(&self, reason: impl Into<String>) {
192 let (tx, rx) = oneshot::channel();
193 if self.cmd.send(Cmd::Stop(reason.into(), tx)).await.is_ok() {
194 let _ = rx.await;
195 }
196 self.wait_for(Status::is_final).await;
197 }
198}
199
200pub fn start(config: Config) -> Handle {
202 let (cmd_tx, cmd_rx) = mpsc::channel(64);
203 let (status_tx, status_rx) = watch::channel(Status::Starting);
204 let (health_tx, health_rx) = watch::channel(None);
205 let grants = Arc::new(watch::channel(config.grants.clone()).0);
206 let h = Handle { cmd: cmd_tx, status: status_rx, health: health_rx, grants: grants.clone() };
207 tokio::spawn(run(config, cmd_rx, status_tx, health_tx, grants));
208 h
209}
210
211enum Ended {
212 Stopped(String),
213 Exited(std::process::ExitStatus, Duration),
214}
215
216async fn run(
217 config: Config,
218 mut cmd: mpsc::Receiver<Cmd>,
219 status: watch::Sender<Status>,
220 health: watch::Sender<Option<v1::HealthReport>>,
221 grants: Arc<watch::Sender<Grants>>,
222) {
223 let mut starts = 0u32;
224 let mut failures = 0u32;
225 let mut wait = config.backoff.initial;
226 loop {
227 status.send_replace(Status::Starting);
228 starts += 1;
229 let ended = match once(&config, &mut cmd, &status, &health, &grants, starts).await {
230 Ok(e) => e,
231 Err(e) => {
232 status.send_replace(Status::Failed { error: e.to_string() });
233 return;
234 }
235 };
236 match ended {
237 Ended::Stopped(reason) => {
238 status.send_replace(Status::Stopped { reason });
239 return;
240 }
241 Ended::Exited(st, ran) => {
242 if st.success() {
243 status.send_replace(Status::Stopped { reason: "exited".into() });
244 return;
245 }
246 if ran >= config.backoff.reset_after {
247 failures = 0;
248 wait = config.backoff.initial;
249 }
250 failures += 1;
251 if failures >= config.backoff.max_failures {
252 status.send_replace(Status::Failed { error: format!("crashed {failures} times in a row (last: {})", describe(&st)) });
253 return;
254 }
255 status.send_replace(Status::Restarting { exit: describe(&st), after: wait });
256 let sleep = tokio::time::sleep(wait);
257 tokio::pin!(sleep);
258 loop {
259 tokio::select! {
260 _ = &mut sleep => break,
261 c = cmd.recv() => match c {
262 Some(Cmd::Stop(reason, done)) => {
263 let _ = done.send(());
264 status.send_replace(Status::Stopped { reason });
265 return;
266 }
267 Some(Cmd::Event(_)) => {}
268 None => return,
269 }
270 }
271 }
272 wait = (wait * 2).min(config.backoff.max);
273 }
274 }
275 }
276}
277
278fn describe(st: &std::process::ExitStatus) -> String {
279 #[cfg(unix)]
280 {
281 use std::os::unix::process::ExitStatusExt;
282 if let Some(s) = st.signal() {
283 return format!("killed by signal {s}");
284 }
285 }
286 match st.code() {
287 Some(c) => format!("exit code {c}"),
288 None => "ended".into(),
289 }
290}
291
292#[cfg(unix)]
293fn kill_group(pid: u32) {
294 unsafe {
296 libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
297 }
298}
299
300#[cfg(unix)]
301async fn once(
302 config: &Config,
303 cmd: &mut mpsc::Receiver<Cmd>,
304 status: &watch::Sender<Status>,
305 health: &watch::Sender<Option<v1::HealthReport>>,
306 grants: &Arc<watch::Sender<Grants>>,
307 starts: u32,
308) -> Result<Ended> {
309 use std::os::fd::AsRawFd;
310 let installed = config.installer.verify(&config.id)?;
311 let binary =
312 installed.native().ok_or_else(|| Error::Unsupported(format!("{} has no binary for {}", config.id, crate::manifest::current_target())))?;
313 let policy = config.policy.clone().unwrap_or_else(|| Policy::for_extension(&installed.dir, &config.data_dir));
314 let (host_end, ext_end) = std::os::unix::net::UnixStream::pair()?;
315 host_end.set_nonblocking(true)?;
316 let token = random_hex(32);
317 let fd = ext_end.as_raw_fd();
318 let env = vec![
319 (weirdnet_ext_proto::ENV_FD.to_string(), fd.to_string()),
320 (weirdnet_ext_proto::ENV_TOKEN.to_string(), token.clone()),
321 (weirdnet_ext_proto::ENV_ID.to_string(), config.id.clone()),
322 ("HOME".to_string(), config.data_dir.display().to_string()),
323 ("TMPDIR".to_string(), config.data_dir.display().to_string()),
324 ("RUST_BACKTRACE".to_string(), "1".to_string()),
325 ];
326 let (mut command, report) = sandbox::command(&binary, &config.args, &policy, fd, &env)?;
327 let mut child = command.spawn().map_err(|e| Error::Process(format!("starting {}: {e}", binary.display())))?;
328 drop(ext_end);
329 let pid = child.id().unwrap_or(0);
330 let started = Instant::now();
331
332 let (ev_tx, ev_rx) = mpsc::channel(64);
333 let shared = Arc::new(Shared { started, last_activity: AtomicU64::new(0), health: health.clone(), logs: config.logs.clone() });
334 let broker = Broker::new(
335 config.id.clone(),
336 installed.manifest.version.clone(),
337 config.host.clone(),
338 token,
339 grants.subscribe(),
340 ev_rx,
341 config.services.clone(),
342 shared.clone(),
343 );
344 let io = tokio::net::UnixStream::from_std(host_end)?;
345 let (shut_tx, shut_rx) = oneshot::channel::<()>();
346 let server = tokio::spawn(async move {
347 let incoming = tokio_stream::StreamExt::chain(tokio_stream::once(Ok::<_, std::io::Error>(io)), tokio_stream::pending());
350 let _ = tonic::transport::Server::builder()
351 .add_service(v1::host_server::HostServer::new(broker))
352 .serve_with_incoming_shutdown(incoming, async {
353 let _ = shut_rx.await;
354 })
355 .await;
356 });
357 status.send_replace(Status::Running { pid, starts, sandbox: report });
358
359 let idle_tick = config.idle.map(|d| d.min(Duration::from_secs(1)).max(Duration::from_millis(50)));
360 let mut tick = tokio::time::interval(idle_tick.unwrap_or(Duration::from_secs(3600)));
361 let stop = |reason: String| async move { reason };
362 let ended = loop {
363 tokio::select! {
364 st = child.wait() => {
365 let st = st?;
366 break Ended::Exited(st, started.elapsed());
367 }
368 c = cmd.recv() => match c {
369 Some(Cmd::Event(e)) => {
370 shared.touch();
371 let _ = ev_tx.try_send(e);
372 }
373 Some(Cmd::Stop(reason, done)) => {
374 graceful(&mut child, &ev_tx, pid, config.stop_grace, &reason).await;
375 let _ = done.send(());
376 break Ended::Stopped(stop(reason).await);
377 }
378 None => {
379 graceful(&mut child, &ev_tx, pid, config.stop_grace, "host gone").await;
380 break Ended::Stopped("host gone".into());
381 }
382 },
383 _ = tick.tick(), if config.idle.is_some() => {
384 let idle = config.idle.unwrap();
385 let last = Duration::from_millis(shared.last_activity.load(Ordering::Relaxed));
386 if started.elapsed().saturating_sub(last) >= idle {
387 graceful(&mut child, &ev_tx, pid, config.stop_grace, "idle").await;
388 break Ended::Stopped("idle".into());
389 }
390 }
391 }
392 };
393 kill_group(pid);
394 let _ = shut_tx.send(());
395 server.abort();
396 Ok(ended)
397}
398
399#[cfg(unix)]
400async fn graceful(child: &mut tokio::process::Child, ev: &mpsc::Sender<v1::Event>, pid: u32, grace: Duration, reason: &str) {
401 let _ = ev.try_send(v1::Event { kind: Some(v1::event::Kind::Stop(v1::Stop { reason: reason.into(), grace_ms: grace.as_millis() as u32 })) });
402 if tokio::time::timeout(grace, child.wait()).await.is_err() {
403 kill_group(pid);
404 let _ = child.wait().await;
405 }
406}
407
408#[cfg(not(unix))]
409async fn once(
410 config: &Config,
411 _: &mut mpsc::Receiver<Cmd>,
412 _: &watch::Sender<Status>,
413 _: &watch::Sender<Option<v1::HealthReport>>,
414 _: &Arc<watch::Sender<Grants>>,
415 _: u32,
416) -> Result<Ended> {
417 config.installer.verify(&config.id)?;
418 Err(Error::Unsupported("native extensions don't run on this OS yet; use the WASM build".into()))
419}