capsule_runtime/child_launch.rs
1//! Launching a delegated child as an operating-system process of the parent's own runtime.
2//!
3//! The daemon referees and hands back an approval; this is what turns that approval into a
4//! running capsule. The parent's runtime composes a directory for the child beneath its own
5//! accessible workdir, builds the child's environment from the child's declaration rather than
6//! its own, and starts the `mur` binary as a subprocess. The child's runtime registers with the
7//! daemon for itself (see [`crate::registration`]), so the parent never states what the child
8//! holds — it only names which artifact is running.
9//!
10//! Three properties fall out of the child being a process rather than a thread:
11//!
12//! * A daemon crash takes no child with it. Nothing the child needs to keep running lives in the
13//! daemon's address space.
14//! * Each child has its own process environment and working directory, so a native subprocess
15//! started by one child inherits nothing a sibling shares.
16//! * A child that declares the `sealed` containment floor enters a mount namespace of its own,
17//! because the containment machinery installs per process and the child *is* a process.
18//!
19//! **The approval travels on the child's standard input.** Not on the argument vector and not in
20//! the environment: both are readable from `/proc/<pid>` by any process running as the same user,
21//! which is exactly what a sibling capsule's shell tool is.
22
23use std::io::{BufRead, BufReader, Write};
24use std::path::{Path, PathBuf};
25use std::process::{Child, Command, ExitStatus, Stdio};
26use std::sync::{mpsc, Arc, Condvar, Mutex};
27use std::time::{Duration, Instant};
28
29use crate::delegation::{
30 self, CompletionAddress, DelegationOutcome, DelegationStatus, Reporter, Spawner, SpawnerHandle,
31 SPAWNER_ENV,
32};
33use crate::errors::RuntimeError;
34use crate::mac_token;
35use crate::spawn_credential::SpawnApproval;
36
37/// Environment variable naming the `mur` binary to launch children with.
38///
39/// Defaults to [`std::env::current_exe`], which is correct in production: the process doing the
40/// launching *is* `mur`. A test harness that runs inside its own test binary sets this to the
41/// built `mur` instead.
42pub const MUR_BINARY_ENV: &str = "MURMUR_MUR_BINARY";
43
44/// How long the parent waits for a child to print its `--json` launch line.
45///
46/// Generous, because it bounds more than a handshake: a script capsule prints the line when its
47/// run has *finished*, and an agent capsule compiles its driver component before it binds a port,
48/// which on a cold host is the slowest thing in a launch.
49const CHILD_READY_TIMEOUT: Duration = Duration::from_secs(180);
50
51/// Owner-only, so a sibling capsule running as the same user cannot list or read a child's
52/// directory even though both sit under one parent.
53#[cfg(unix)]
54const CHILD_DIR_MODE: u32 = 0o700;
55
56/// How many of the child's last stderr lines are kept to explain a launch that never reported.
57///
58/// The child's diagnostics are the operator's, so every line is echoed to this process's stderr as
59/// it arrives; this bound is only what a *failure* quotes back, so a child that logged for an hour
60/// before dying does not turn into an unbounded error string.
61const CHILD_STDERR_TAIL_LINES: usize = 20;
62
63/// How long a post-mortem read of the tail waits for the drain to reach the end of the pipe.
64///
65/// Bounds a thread being scheduled, not the child: by the time anything asks for a dead child's
66/// tail the exit has already closed the write end, so the drain is at most one wake-up away.
67/// A pipe some surviving grandchild still holds open would never reach EOF at all, and the bound
68/// is what stops that inheriting process from wedging the parent's launch or its watcher.
69const CHILD_STDERR_DRAIN_TIMEOUT: Duration = Duration::from_secs(5);
70
71/// How often the completion watcher asks whether the child is still running.
72///
73/// It polls rather than blocking on `wait`, because the process handle is shared with
74/// [`LaunchedChild::shutdown`] and [`Drop`]: a watcher blocked inside `wait` would hold the lock
75/// those need, and a second `wait` on one child is not a thing two owners can both do.
76const CHILD_WATCH_INTERVAL: Duration = Duration::from_millis(100);
77
78/// What the parent's runtime needs in order to start one approved child.
79///
80/// Carries no manifest and no capability declaration: what the child holds is decided by the
81/// daemon from the child's own registry manifest, both at the `POST /spawn` that granted `grant`
82/// and again at the `POST /register` the child performs for itself.
83#[derive(Debug)]
84pub struct ChildLaunchRequest {
85 /// The parent's own accessible workdir. The child's directory is composed beneath it, which
86 /// is what keeps the child inside the single preopen the parent's WASI layer already has.
87 pub parent_accessible_workdir: PathBuf,
88 pub capsule_name: String,
89 pub capsule_version: String,
90 /// The approval `POST /spawn` returned, spent by the child's own registration.
91 pub grant: SpawnApproval,
92 /// The child's `capabilities.env.allow`. Names listed here are copied from the parent's
93 /// process environment into the child's; every other host variable the parent holds is
94 /// absent from the child, because the child's environment is built from a cleared one.
95 pub child_env_allow: Vec<String>,
96 /// Base URL of the daemon the child registers with.
97 pub roost_url: String,
98 /// The session this child belongs to, or `None` for a launch that is not a delegation.
99 ///
100 /// `Some` is what turns the launch into a delegation: the launcher mints a delegation id and
101 /// injects a [`SpawnerHandle`] as [`SPAWNER_ENV`], which is where the child reads the lineage
102 /// it records in its own `session_start`. The watcher that reports for a child which could
103 /// not report for itself starts only when that spawner also names a
104 /// [`CompletionAddress`] — a parent waiting on its own connection wants no completion.
105 /// `None` injects nothing and starts nothing.
106 pub spawner: Option<Spawner>,
107 /// How long the completion watcher observes this child before ending it, or `None` to observe
108 /// it for as long as it runs.
109 ///
110 /// Meaningful only alongside a [`CompletionAddress`], because the watcher that enforces it
111 /// starts only for a spawner that names one. On expiry the watcher stops the process and
112 /// posts a [`DelegationStatus::Terminated`] outcome naming the bound, which is the only thing
113 /// that tells a parent about a child that never ends.
114 pub completion_deadline: Option<Duration>,
115}
116
117/// How a child's process ended, as the watcher sees it.
118enum Ending {
119 /// The process exited on its own. `None` when it was reaped by someone who did not keep the
120 /// status.
121 Exited(Option<ExitStatus>),
122 /// The delegation was ended on purpose rather than by the child: by the parent, through
123 /// [`LaunchedChild::shutdown`] or `Drop`, or by the completion watcher at its deadline.
124 Deliberate,
125}
126
127/// The one owner of the child's process handle.
128///
129/// Shared behind a lock between [`LaunchedChild`] — which kills and reaps — and the completion
130/// watcher, which only ever asks whether the process is still there. Single ownership of the
131/// `Child` is what keeps `pid()`, `shutdown()` and `Drop` behaving as they did: nothing else
132/// takes the handle, and nothing else waits on it.
133struct ChildProcess {
134 /// `None` once [`LaunchedChild::shutdown`] or `Drop` has reaped the process.
135 child: Option<Child>,
136 /// Set before the kill by whoever ended the delegation on purpose, so a crash is never read
137 /// as a termination. It does not say *who*: the completion watcher sets it at its own
138 /// deadline too, and tells the two apart by a flag of its own.
139 deliberate: bool,
140 /// The exit status, once anyone has observed it.
141 status: Option<ExitStatus>,
142}
143
144impl ChildProcess {
145 /// How the process ended, or `None` while it is still running.
146 ///
147 /// Reaps a process that exited on its own, which is what makes a later `wait` in `shutdown`
148 /// return the cached status rather than block.
149 fn poll(&mut self) -> Option<Ending> {
150 if self.deliberate {
151 return Some(Ending::Deliberate);
152 }
153 let Some(child) = self.child.as_mut() else {
154 // Reaped by neither `shutdown` nor `Drop`, which both set `deliberate` first: this
155 // is unreachable, and reporting the exit is the harmless reading of it.
156 return Some(Ending::Exited(self.status));
157 };
158 match child.try_wait() {
159 Ok(Some(status)) => {
160 self.status = Some(status);
161 Some(Ending::Exited(Some(status)))
162 }
163 Ok(None) => None,
164 // The handle is unusable; treating the child as gone is the only outcome that lets
165 // the delegation be reported at all.
166 Err(_) => Some(Ending::Exited(self.status)),
167 }
168 }
169
170 /// Kill and reap. Idempotent.
171 fn end(&mut self) -> Result<(), std::io::Error> {
172 let Some(mut child) = self.child.take() else {
173 return Ok(());
174 };
175 // `kill` on an already-exited process is not an error worth surfacing — the wait below is
176 // what actually retires the entry in the process table.
177 let _ = child.kill();
178 let status = child.wait()?;
179 self.status = Some(status);
180 Ok(())
181 }
182}
183
184/// A running child capsule and the handles to end it.
185pub struct LaunchedChild {
186 /// The directory the parent created for this child, and passed as its `--workdir`. The
187 /// child's own session artifacts land at `<workdir>/.murmur/<session_id>/`.
188 pub workdir: PathBuf,
189 /// The session id the child's runtime minted for itself.
190 pub session_id: String,
191 /// The child's A2A endpoint, `http://host:port`. Empty for a script capsule, which binds no
192 /// port and has already finished by the time it reports.
193 pub capsule_url: String,
194 /// The argument vector the parent built, in order, starting with the binary.
195 ///
196 /// Exposed so a caller — a test, notably — can assert what was passed without reading
197 /// `/proc`. It carries no token: the approval goes in on standard input.
198 pub argv: Vec<String>,
199 /// The complete environment the parent built for this child, in insertion order. Also carries
200 /// no token.
201 pub env: Vec<(String, String)>,
202 /// The id this launch's completion reports under, `None` when no spawner was supplied.
203 ///
204 /// Minted here, injected into the child as part of [`SPAWNER_ENV`], and echoed back on the
205 /// completion — the one value that joins a delegation to the task it produces at the parent.
206 pub delegation_id: Option<String>,
207 process: Arc<Mutex<ChildProcess>>,
208 /// The child's last [`CHILD_STDERR_TAIL_LINES`] lines, retained so a crash can say why.
209 stderr_tail: Arc<StderrTail>,
210 /// When the child process was started, for the completion's `duration_ms`.
211 started: Instant,
212 /// Set by [`LaunchedChild::release`]. The one thing that stops [`Drop`] signalling the child:
213 /// a released process is no longer this handle's to end.
214 released: bool,
215}
216
217impl std::fmt::Debug for LaunchedChild {
218 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
219 f.debug_struct("LaunchedChild")
220 .field("workdir", &self.workdir)
221 .field("session_id", &self.session_id)
222 .field("capsule_url", &self.capsule_url)
223 .field("delegation_id", &self.delegation_id)
224 .field("pid", &self.pid())
225 .finish()
226 }
227}
228
229impl LaunchedChild {
230 /// The child's operating-system process id. `0` once it has been reaped.
231 pub fn pid(&self) -> u32 {
232 lock(&self.process)
233 .child
234 .as_ref()
235 .map(Child::id)
236 .unwrap_or(0)
237 }
238
239 /// The child's last [`CHILD_STDERR_TAIL_LINES`] lines, oldest first.
240 ///
241 /// The same lines a crash completion quotes back, exposed so a caller can read what the child
242 /// said without scraping the stderr they were already echoed to.
243 pub fn stderr_tail(&self) -> Vec<String> {
244 self.stderr_tail.lines()
245 }
246
247 /// Terminate the child and reap it. Idempotent: a second call, or a call after `Drop` has
248 /// already run, does nothing.
249 ///
250 /// Marks the ending as deliberate, so the watcher records the delegation as `terminated` and
251 /// posts nothing: the only party that would be told is the party that did it.
252 pub fn shutdown(&mut self) -> Result<(), RuntimeError> {
253 let mut process = lock(&self.process);
254 process.deliberate = true;
255 process.end().map_err(|error| {
256 RuntimeError::Runtime(format!("failed to reap child capsule: {error}"))
257 })
258 }
259
260 /// Stop owning the child's lifetime without signalling it, and let this handle go.
261 ///
262 /// The one escape from [`Drop`]'s kill. After this the process keeps running and **only the
263 /// completion watcher started by [`launch_child_capsule`] will ever observe or reap it** — a
264 /// child released from a launch that named no [`CompletionAddress`] is a process nothing will
265 /// ever wait on.
266 ///
267 /// This is how a delegation outlives the call that made it: `delegate-task` returns as soon as
268 /// the child is running and holding its task, and what the child eventually did arrives at the
269 /// parent as a completion the watcher either forwards or writes itself.
270 pub fn release(mut self) {
271 self.released = true;
272 }
273}
274
275impl Drop for LaunchedChild {
276 /// Terminates and reaps, so a parent that returns early — including by panicking — leaves no
277 /// orphaned capsule process behind holding a port and a directory. Deliberate on the same
278 /// terms as [`LaunchedChild::shutdown`].
279 ///
280 /// A released child is left alone: its lifetime stopped being this handle's the moment
281 /// [`LaunchedChild::release`] was called.
282 fn drop(&mut self) {
283 if self.released {
284 return;
285 }
286 let mut process = lock(&self.process);
287 process.deliberate = true;
288 let _ = process.end();
289 }
290}
291
292/// `workdir` named from `root`, falling back to the absolute path when it sits outside it.
293///
294/// One rule, used by the delegating plane wherever it names a child's directory to the parent's
295/// agent or to the parent's trace.
296pub(crate) fn workdir_relative_to(workdir: &Path, root: &Path) -> String {
297 workdir
298 .strip_prefix(root)
299 .unwrap_or(workdir)
300 .to_string_lossy()
301 .replace('\\', "/")
302}
303
304/// A mutex this crate holds only for the length of one field access, so a poisoned lock is
305/// recovered from rather than turned into a panic in a `Drop`.
306fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
307 mutex
308 .lock()
309 .unwrap_or_else(|poisoned| poisoned.into_inner())
310}
311
312/// Where a child of this parent goes: `<parent accessible workdir>/.murmur/children/<name>-<16 hex>`.
313///
314/// The 16 hex characters are fresh per call, so delegating the same capsule name and version twice
315/// yields two directories rather than one shared one. Beneath `.murmur/`, alongside the parent's
316/// own session directories, so a child's tree is inside the parent's single preopen and is pruned
317/// with it.
318pub fn child_workdir_for(parent_accessible_workdir: &Path, capsule_name: &str) -> PathBuf {
319 let suffix = mac_token::random_hex(8).unwrap_or_else(|_| {
320 // The OS CSPRNG is not expected to fail; a nanosecond clock reading still distinguishes
321 // two delegations rather than collapsing them onto one directory.
322 format!(
323 "{:016x}",
324 std::time::SystemTime::now()
325 .duration_since(std::time::UNIX_EPOCH)
326 .map(|elapsed| elapsed.as_nanos() as u64)
327 .unwrap_or(0)
328 )
329 });
330 parent_accessible_workdir
331 .join(".murmur")
332 .join("children")
333 .join(format!("{capsule_name}-{suffix}"))
334}
335
336/// Create the child's directory, start `mur` on it, and wait for the child to report itself.
337pub fn launch_child_capsule(request: ChildLaunchRequest) -> Result<LaunchedChild, RuntimeError> {
338 let workdir = child_workdir_for(&request.parent_accessible_workdir, &request.capsule_name);
339 create_child_dir(&workdir)?;
340
341 let binary = mur_binary()?;
342 let argv: Vec<String> = vec![
343 binary.display().to_string(),
344 "run".to_string(),
345 "--capsule".to_string(),
346 request.capsule_name.clone(),
347 "--capsule-version".to_string(),
348 request.capsule_version.clone(),
349 "--workdir".to_string(),
350 workdir.display().to_string(),
351 "--json".to_string(),
352 "--no-env-file".to_string(),
353 "--spawn-grant-stdin".to_string(),
354 ];
355 // Minted here, once per launch: a caller holding one `Spawner` that delegates twice gets two
356 // delegation ids, and neither child can report under the other's.
357 let handle = request
358 .spawner
359 .as_ref()
360 .map(|spawner| SpawnerHandle::for_delegation(spawner, delegation::new_delegation_id()));
361 let env = child_environment(&request, handle.as_ref());
362 let started = Instant::now();
363
364 let mut command = Command::new(&binary);
365 command
366 .args(&argv[1..])
367 .current_dir(&workdir)
368 .env_clear()
369 .stdin(Stdio::piped())
370 .stdout(Stdio::piped())
371 .stderr(Stdio::piped());
372 for (key, value) in &env {
373 command.env(key, value);
374 }
375
376 let mut child = command.spawn().map_err(|error| {
377 RuntimeError::Runtime(format!(
378 "failed to start the child capsule process '{}': {error}",
379 binary.display()
380 ))
381 })?;
382
383 // The grant's one and only appearance outside the parent's memory: one line on a pipe that is
384 // closed immediately afterwards. Failing to write it is fatal — the child would otherwise sit
385 // waiting on a line that will never come.
386 let write_result = child
387 .stdin
388 .take()
389 .ok_or_else(|| "the child process exposed no standard input".to_string())
390 .and_then(|mut stdin| {
391 writeln!(stdin, "{}", request.grant.expose())
392 .map_err(|error| format!("failed to hand the child its launch grant: {error}"))
393 });
394 let stderr_tail = drain_stderr(&mut child);
395 let mut launched = LaunchedChild {
396 workdir,
397 session_id: String::new(),
398 capsule_url: String::new(),
399 argv,
400 env,
401 delegation_id: handle.as_ref().map(|handle| handle.delegation_id.clone()),
402 process: Arc::new(Mutex::new(ChildProcess {
403 child: Some(child),
404 deliberate: false,
405 status: None,
406 })),
407 stderr_tail: Arc::clone(&stderr_tail),
408 started,
409 released: false,
410 };
411 if let Err(reason) = write_result {
412 return Err(RuntimeError::Runtime(reason));
413 }
414
415 let line = first_json_line(&mut launched, &stderr_tail)?;
416 let report: serde_json::Value = serde_json::from_str(&line).map_err(|error| {
417 RuntimeError::Runtime(format!(
418 "the child capsule's first --json line did not parse: {error}"
419 ))
420 })?;
421 let session_id = report
422 .get("session_id")
423 .and_then(serde_json::Value::as_str)
424 .ok_or_else(|| {
425 RuntimeError::Runtime(
426 "the child capsule's --json line carried no session_id".to_string(),
427 )
428 })?;
429 let url = report
430 .get("url")
431 .and_then(serde_json::Value::as_str)
432 .unwrap_or_default();
433
434 launched.session_id = session_id.to_string();
435 // A script capsule binds no port and reports an empty url; promoting that to `http://` would
436 // manufacture an address nothing answers on.
437 launched.capsule_url = if url.is_empty() {
438 String::new()
439 } else {
440 format!("http://{url}")
441 };
442
443 // Started only for a launch somebody wants told about. A lineage-only handle names no
444 // address, so a child whose parent waits on the connection it already holds is launched with
445 // no watcher behind it and posts nothing anywhere.
446 if let Some(handle) = handle {
447 if let Some(address) = handle.report_to.clone() {
448 watch_for_completion(
449 &launched,
450 handle,
451 address,
452 &request.capsule_name,
453 &request.capsule_version,
454 request.completion_deadline,
455 );
456 }
457 }
458 Ok(launched)
459}
460
461/// Report for a child that ended without reporting for itself, and stop one that never ends.
462///
463/// Polls the shared process handle rather than blocking on `wait`, so it never contends with
464/// [`LaunchedChild::shutdown`] or `Drop` for ownership of the `Child`. For a released child this
465/// thread is the process's only remaining observer, and the only thing that will ever reap it.
466///
467/// `deadline` bounds how long the child is watched, from the instant this watcher starts — which
468/// is the instant the child reported itself ready, not the instant its process was spawned.
469/// Getting to that point is bounded separately by [`CHILD_READY_TIMEOUT`], and folding a cold
470/// host's staging time into the delegation deadline would end a child for being slow to start
471/// rather than slow to work. On expiry the watcher stops the child itself and posts a
472/// [`DelegationStatus::Terminated`] outcome naming the bound — the deadline is the watcher's own,
473/// held in a local flag rather than read back off [`ChildProcess::deliberate`], so an ending the
474/// *parent* chose stays distinguishable from one this thread imposed. `None` watches for as long
475/// as the child runs.
476///
477/// What it does when the process ends on its own is decided by what the child left behind:
478///
479/// * A completion the child already delivered is left alone — exactly one completion per
480/// delegation reaches the parent.
481/// * A completion the child recorded but could not deliver is retried once, and the file is
482/// rewritten with the result. Nothing is retried after that.
483/// * No completion at all means the child died without a word: the watcher builds one with
484/// `status: crashed`, carrying the exit status and the child's bounded stderr tail, and posts
485/// it in the child's place.
486/// * An ending the parent chose is recorded as `terminated` and posted to nobody, because the
487/// only party that would be told is the party that did it.
488fn watch_for_completion(
489 launched: &LaunchedChild,
490 handle: SpawnerHandle,
491 address: CompletionAddress,
492 capsule_name: &str,
493 capsule_version: &str,
494 deadline: Option<Duration>,
495) {
496 let process = Arc::clone(&launched.process);
497 let stderr_tail = Arc::clone(&launched.stderr_tail);
498 let workdir = launched.workdir.clone();
499 let session_id = launched.session_id.clone();
500 let capsule_name = capsule_name.to_string();
501 let capsule_version = capsule_version.to_string();
502 let started = launched.started;
503
504 let watching_from = Instant::now();
505 std::thread::spawn(move || {
506 let expires_at = deadline.map(|bound| watching_from + bound);
507 // Set here and read nowhere else: `ChildProcess::deliberate` is about to be set too, by
508 // this thread, so it can no longer say who chose the ending.
509 let mut expired = false;
510 let ending = loop {
511 if expires_at.is_some_and(|at| Instant::now() >= at) {
512 let mut process = lock(&process);
513 process.deliberate = true;
514 let _ = process.end();
515 expired = true;
516 break Ending::Deliberate;
517 }
518 if let Some(ending) = lock(&process).poll() {
519 break ending;
520 }
521 std::thread::sleep(CHILD_WATCH_INTERVAL);
522 };
523 let duration_ms = started.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
524
525 if let Some(recorded) = delegation::read_completion(&workdir) {
526 if recorded.delivered {
527 return;
528 }
529 let retried = delegation::report_completion(&handle, &address, recorded, &workdir);
530 if !retried.delivered {
531 // One retry, then the file and the line above it are the record.
532 crate::runtime_err!(
533 "[capsule-runtime] delegation {}: the child's completion is undelivered after one retry",
534 handle.delegation_id
535 );
536 }
537 return;
538 }
539
540 let outcome = |status: DelegationStatus, detail: String| DelegationOutcome {
541 delegation_id: handle.delegation_id.clone(),
542 capsule_name: capsule_name.clone(),
543 capsule_version: capsule_version.clone(),
544 session_id: session_id.clone(),
545 status,
546 // The launcher reports for a child that recorded nothing, so it makes no claim about
547 // a result file it never saw named.
548 result_path: None,
549 workdir: workdir.display().to_string(),
550 duration_ms,
551 detail: Some(detail),
552 reported_by: Reporter::Launcher,
553 delivered: false,
554 delivery_error: None,
555 };
556
557 match ending {
558 // The deadline is the parent's runtime acting on the parent's behalf while the parent
559 // is elsewhere, so unlike an ending the parent chose by hand this one is posted: the
560 // delegating agent asked for an outcome and this is the outcome.
561 Ending::Deliberate if expired => {
562 let bound = deadline.map(|bound| bound.as_secs()).unwrap_or_default();
563 delegation::report_completion(
564 &handle,
565 &address,
566 outcome(
567 DelegationStatus::Terminated,
568 format!(
569 "the capsule was still running after {bound}s and was ended at the \
570 delegation deadline"
571 ),
572 ),
573 &workdir,
574 );
575 }
576 Ending::Deliberate => {
577 delegation::record_terminated(
578 &workdir,
579 outcome(
580 DelegationStatus::Terminated,
581 "the parent ended this delegation".to_string(),
582 ),
583 );
584 }
585 Ending::Exited(status) => {
586 let status_text = match status {
587 Some(status) => status.to_string(),
588 None => "unknown exit status".to_string(),
589 };
590 let tail = stderr_tail.lines_at_end().join("\n");
591 let detail = if tail.is_empty() {
592 format!(
593 "the child process ended without recording a completion ({status_text})"
594 )
595 } else {
596 format!(
597 "the child process ended without recording a completion ({status_text}); \
598 its last stderr lines were:\n{tail}"
599 )
600 };
601 delegation::report_completion(
602 &handle,
603 &address,
604 outcome(DelegationStatus::Crashed, detail),
605 &workdir,
606 );
607 }
608 }
609 });
610}
611
612/// The child's complete environment, built from a cleared one.
613///
614/// `capabilities.env.allow` is the child's own, not the parent's: a sibling's declaration reaches
615/// nothing here, and a variable the parent holds but the child did not declare is simply absent.
616/// The runtime-owned names — `PATH`, `HOME`, `MURMUR_ROOST_URL`, and [`SPAWNER_ENV`] on a
617/// delegated launch — are applied last, so a child cannot displace the daemon URL it is required
618/// to register with, or the handle it reports its outcome to, by allowlisting the name.
619fn child_environment(
620 request: &ChildLaunchRequest,
621 handle: Option<&SpawnerHandle>,
622) -> Vec<(String, String)> {
623 let mut env: Vec<(String, String)> = Vec::new();
624 for key in &request.child_env_allow {
625 if let Ok(value) = std::env::var(key) {
626 env.push((key.clone(), value));
627 }
628 }
629 env.retain(|(key, _)| {
630 !matches!(key.as_str(), "PATH" | "HOME" | "MURMUR_ROOST_URL") && key != SPAWNER_ENV
631 });
632
633 if let Ok(path) = std::env::var("PATH") {
634 env.push(("PATH".to_string(), path));
635 }
636 if let Ok(home) = std::env::var("HOME") {
637 env.push(("HOME".to_string(), home));
638 }
639 env.push(("MURMUR_ROOST_URL".to_string(), request.roost_url.clone()));
640 // Last, with the other runtime-owned names, and only for a launch that has a spawner: a child
641 // nobody wants told holds no handle at all, whatever this process's own environment carries
642 // and whatever the child declared.
643 if let Some(handle) = handle {
644 env.push((SPAWNER_ENV.to_string(), handle.to_env_value()));
645 }
646 env
647}
648
649/// The binary a child is started from: [`MUR_BINARY_ENV`] when set, else this process's own image.
650fn mur_binary() -> Result<PathBuf, RuntimeError> {
651 if let Some(path) = std::env::var_os(MUR_BINARY_ENV) {
652 if !path.is_empty() {
653 return Ok(PathBuf::from(path));
654 }
655 }
656 std::env::current_exe().map_err(|error| {
657 RuntimeError::Runtime(format!(
658 "cannot locate the mur binary to launch a child capsule with: {error}"
659 ))
660 })
661}
662
663fn create_child_dir(workdir: &Path) -> Result<(), RuntimeError> {
664 std::fs::create_dir_all(workdir).map_err(|source| RuntimeError::CreateWorkdir {
665 path: workdir.display().to_string(),
666 source,
667 })?;
668 #[cfg(unix)]
669 {
670 use std::os::unix::fs::PermissionsExt;
671 std::fs::set_permissions(workdir, std::fs::Permissions::from_mode(CHILD_DIR_MODE))
672 .map_err(|source| RuntimeError::CreateWorkdir {
673 path: workdir.display().to_string(),
674 source,
675 })?;
676 }
677 Ok(())
678}
679
680/// The child's stderr as the parent keeps it: every line echoed to this process's stderr as it
681/// arrives, with the last [`CHILD_STDERR_TAIL_LINES`] retained so a child that dies before
682/// reporting can say why.
683struct StderrTail {
684 state: Mutex<StderrState>,
685 /// Notified once [`StderrState::at_eof`] is set.
686 drained: Condvar,
687}
688
689#[derive(Default)]
690struct StderrState {
691 lines: Vec<String>,
692 /// Whether the pipe has been read to its end, which the child's exit is what causes.
693 at_eof: bool,
694}
695
696impl StderrTail {
697 /// What the child has said so far, without waiting for anything. For a child that is still
698 /// running, which is the only state in which "so far" is the question being asked.
699 fn lines(&self) -> Vec<String> {
700 lock(&self.state).lines.clone()
701 }
702
703 /// What a child that has ended left behind, read only once the pipe is at its end.
704 ///
705 /// The exit that closes the child's stdout closes its stderr in the same breath, so the thread
706 /// that notices the ending and the thread still draining stderr are racing, and the tail is
707 /// empty for as long as the drain has not caught up. Reading it unsynchronised turns "refused
708 /// because the declared floor is unmeetable here" into a refusal that names no reason —
709 /// the one case the tail is kept for. Bounded by [`CHILD_STDERR_DRAIN_TIMEOUT`].
710 fn lines_at_end(&self) -> Vec<String> {
711 let (state, _) = self
712 .drained
713 .wait_timeout_while(lock(&self.state), CHILD_STDERR_DRAIN_TIMEOUT, |state| {
714 !state.at_eof
715 })
716 .unwrap_or_else(|poisoned| poisoned.into_inner());
717 state.lines.clone()
718 }
719
720 /// Mark the pipe finished and wake every post-mortem reader waiting on it.
721 fn finish(&self) {
722 lock(&self.state).at_eof = true;
723 self.drained.notify_all();
724 }
725}
726
727/// Start draining the child's stderr into a [`StderrTail`].
728///
729/// The child's diagnostics belong to the operator running the parent, so nothing is swallowed —
730/// this only remembers, in addition to printing.
731fn drain_stderr(child: &mut Child) -> Arc<StderrTail> {
732 let tail = Arc::new(StderrTail {
733 state: Mutex::new(StderrState::default()),
734 drained: Condvar::new(),
735 });
736 let Some(stderr) = child.stderr.take() else {
737 // No pipe is an end already reached: a reader that waited here would wait out the whole
738 // bound for lines that can never arrive.
739 tail.finish();
740 return tail;
741 };
742 let collector = Arc::clone(&tail);
743 std::thread::spawn(move || {
744 for line in BufReader::new(stderr).lines().map_while(Result::ok) {
745 crate::runtime_err!("{line}");
746 let mut state = lock(&collector.state);
747 if state.lines.len() == CHILD_STDERR_TAIL_LINES {
748 state.lines.remove(0);
749 }
750 state.lines.push(line);
751 }
752 collector.finish();
753 });
754 tail
755}
756
757/// Read the child's first standard-output line, then keep draining the pipe.
758///
759/// The drain matters: a child whose stdout pipe fills blocks forever, and a capsule that logs
760/// after its launch line would otherwise deadlock against a parent that had stopped reading.
761fn first_json_line(
762 launched: &mut LaunchedChild,
763 stderr_tail: &StderrTail,
764) -> Result<String, RuntimeError> {
765 let stdout = lock(&launched.process)
766 .child
767 .as_mut()
768 .and_then(|child| child.stdout.take())
769 .ok_or_else(|| {
770 RuntimeError::Runtime("the child process exposed no standard output".to_string())
771 })?;
772
773 let (tx, rx) = mpsc::channel::<Option<String>>();
774 std::thread::spawn(move || {
775 let mut reader = BufReader::new(stdout);
776 let mut line = String::new();
777 let first = match reader.read_line(&mut line) {
778 Ok(0) | Err(_) => None,
779 Ok(_) => Some(line.trim_end().to_string()),
780 };
781 let _ = tx.send(first);
782 // Everything the child says after its launch line goes to this process's own stdout, so
783 // the pipe stays drained for as long as the child lives. This thread outlives the parent's
784 // own readiness line by the whole life of the child, so it is the one relay most likely to
785 // meet a reader that has already gone: it relays the line and keeps draining either way.
786 let mut rest = String::new();
787 while let Ok(read) = reader.read_line(&mut rest) {
788 if read == 0 {
789 break;
790 }
791 crate::diagnostic::raw_to_stdout(&rest);
792 rest.clear();
793 }
794 });
795
796 match rx.recv_timeout(CHILD_READY_TIMEOUT) {
797 Ok(Some(line)) => Ok(line),
798 Ok(None) => {
799 let status = lock(&launched.process)
800 .child
801 .as_mut()
802 .and_then(|child| child.wait().ok())
803 .map(|status| status.to_string())
804 .unwrap_or_else(|| "unknown".to_string());
805 // The child's own refusal — an unmeetable containment floor, a registration the
806 // daemon declined — is written to its stderr and is the only thing that explains this
807 // failure, so it is quoted rather than replaced by a generic "did not start".
808 let reason = stderr_tail.lines_at_end().join("\n");
809 Err(RuntimeError::Runtime(format!(
810 "the child capsule '{}' exited without reporting a launch line ({status}): {reason}",
811 launched.workdir.display()
812 )))
813 }
814 Err(_) => Err(RuntimeError::Runtime(format!(
815 "the child capsule '{}' did not report a launch line within {}s",
816 launched.workdir.display(),
817 CHILD_READY_TIMEOUT.as_secs()
818 ))),
819 }
820}
821
822#[cfg(test)]
823mod tests {
824 use super::*;
825
826 #[test]
827 fn a_child_directory_is_unique_per_delegation() {
828 let parent = PathBuf::from("/tmp/parent");
829 let first = child_workdir_for(&parent, "worker");
830 let second = child_workdir_for(&parent, "worker");
831
832 assert_ne!(first, second);
833 assert_eq!(
834 first.parent(),
835 Some(parent.join(".murmur").join("children").as_path())
836 );
837 assert!(first
838 .file_name()
839 .unwrap()
840 .to_string_lossy()
841 .starts_with("worker-"));
842 // `<name>-<16 hex>`
843 assert_eq!(
844 first.file_name().unwrap().to_string_lossy().len(),
845 "worker-".len() + 16
846 );
847 }
848
849 fn request(env_allow: &[&str], spawner: Option<Spawner>) -> ChildLaunchRequest {
850 ChildLaunchRequest {
851 parent_accessible_workdir: PathBuf::from("/tmp/parent"),
852 capsule_name: "worker".to_string(),
853 capsule_version: "0.1.0".to_string(),
854 grant: SpawnApproval::new("msa1.token".to_string()),
855 child_env_allow: env_allow.iter().map(|name| name.to_string()).collect(),
856 roost_url: "http://127.0.0.1:7700".to_string(),
857 spawner,
858 completion_deadline: None,
859 }
860 }
861
862 /// A handle naming some *other* delegation, for the two cases that prove a child cannot be
863 /// handed this process's own.
864 ///
865 /// Readable rather than nonsense, because `SPAWNER_ENV` is process-wide and every other test
866 /// in this binary shares it: an unreadable value here refuses an unrelated `stage_session`
867 /// running beside it.
868 fn decoy() -> String {
869 SpawnerHandle {
870 session_id: "ses_decoy".to_string(),
871 context_id: "ctx_decoy".to_string(),
872 delegation_id: "dlg_decoy".to_string(),
873 report_to: None,
874 }
875 .to_env_value()
876 }
877
878 fn spawner() -> Spawner {
879 Spawner {
880 session_id: "ses_parent".to_string(),
881 context_id: "ctx_parent".to_string(),
882 report_to: Some(CompletionAddress {
883 url: "http://127.0.0.1:7000".to_string(),
884 trust: crate::origin::TrustClass::Trusted,
885 }),
886 }
887 }
888
889 #[test]
890 fn a_childs_environment_is_built_from_its_own_declaration() {
891 std::env::set_var("MURMUR_CHILD_LAUNCH_TEST_A", "a");
892 std::env::set_var("MURMUR_CHILD_LAUNCH_TEST_B", "b");
893
894 let env = child_environment(&request(&["MURMUR_CHILD_LAUNCH_TEST_A"], None), None);
895 let names: Vec<&str> = env.iter().map(|(key, _)| key.as_str()).collect();
896
897 assert!(names.contains(&"MURMUR_CHILD_LAUNCH_TEST_A"));
898 assert!(!names.contains(&"MURMUR_CHILD_LAUNCH_TEST_B"));
899 assert!(names.contains(&"MURMUR_ROOST_URL"));
900 assert!(!env.iter().any(|(_, value)| value.contains("msa1.token")));
901 }
902
903 #[test]
904 fn a_runtime_owned_name_cannot_be_displaced_by_a_declaration() {
905 std::env::set_var("MURMUR_ROOST_URL", "http://127.0.0.1:1");
906
907 let env = child_environment(&request(&["MURMUR_ROOST_URL"], None), None);
908 let urls: Vec<&String> = env
909 .iter()
910 .filter(|(key, _)| key == "MURMUR_ROOST_URL")
911 .map(|(_, value)| value)
912 .collect();
913
914 assert_eq!(urls, vec!["http://127.0.0.1:7700"]);
915 }
916
917 /// The injected handle is the one the parent composed, whatever this process's own
918 /// environment holds and whatever the child declared.
919 #[test]
920 fn the_spawner_handle_is_injected_last_and_cannot_be_displaced() {
921 std::env::set_var(SPAWNER_ENV, decoy());
922 let handle = SpawnerHandle::for_delegation(&spawner(), "dlg_0001".to_string());
923
924 let env = child_environment(&request(&[SPAWNER_ENV], Some(spawner())), Some(&handle));
925 let injected: Vec<&String> = env
926 .iter()
927 .filter(|(key, _)| key == SPAWNER_ENV)
928 .map(|(_, value)| value)
929 .collect();
930
931 assert_eq!(injected.len(), 1, "{env:?}");
932 assert_eq!(
933 SpawnerHandle::parse(injected[0]).expect("the injected value is a handle"),
934 handle
935 );
936 assert_eq!(
937 env.last().map(|(key, _)| key.as_str()),
938 Some(SPAWNER_ENV),
939 "the handle is applied in the runtime-owned tail: {env:?}"
940 );
941 std::env::remove_var(SPAWNER_ENV);
942 }
943
944 /// A launch with no spawner injects nothing, even from a decoy the launching process holds.
945 #[test]
946 fn a_launch_without_a_spawner_injects_no_handle() {
947 std::env::set_var(SPAWNER_ENV, decoy());
948
949 let env = child_environment(&request(&[SPAWNER_ENV], None), None);
950
951 assert!(!env.iter().any(|(key, _)| key == SPAWNER_ENV), "{env:?}");
952 std::env::remove_var(SPAWNER_ENV);
953 }
954
955 /// A child's exit closes its stdout and its stderr together, so the reader that reports the
956 /// ending can reach the tail before the drain has appended to it. The post-mortem read is the
957 /// one that must not: an empty tail is a refusal that names no reason.
958 #[test]
959 fn a_tail_read_after_the_end_waits_for_the_drain_to_catch_up() {
960 let tail = Arc::new(StderrTail {
961 state: Mutex::new(StderrState::default()),
962 drained: Condvar::new(),
963 });
964
965 let writer = Arc::clone(&tail);
966 std::thread::spawn(move || {
967 std::thread::sleep(Duration::from_millis(200));
968 lock(&writer.state)
969 .lines
970 .push("error[E-CAP-003]: not achievable on this host".to_string());
971 writer.finish();
972 });
973
974 assert!(tail.lines().is_empty());
975 assert_eq!(
976 tail.lines_at_end(),
977 vec!["error[E-CAP-003]: not achievable on this host".to_string()]
978 );
979 }
980
981 /// A child that exposed no stderr pipe has nothing to wait for, so the bound is never spent on
982 /// lines that cannot arrive.
983 #[test]
984 fn a_tail_with_no_pipe_behind_it_is_at_its_end_already() {
985 let tail = StderrTail {
986 state: Mutex::new(StderrState::default()),
987 drained: Condvar::new(),
988 };
989 tail.finish();
990
991 let started = Instant::now();
992 assert!(tail.lines_at_end().is_empty());
993 assert!(
994 started.elapsed() < CHILD_STDERR_DRAIN_TIMEOUT,
995 "{started:?}"
996 );
997 }
998}