capsule_runtime/child_launch.rs
1//! Launching a delegated child as an operating-system process of the parent's own runtime.
2//!
3//! The daemon referees and hands back an approval; this is what turns that approval into a
4//! running capsule. The parent's runtime composes a directory for the child beneath its own
5//! accessible workdir, builds the child's environment from the child's declaration rather than
6//! its own, and starts the `mur` binary as a subprocess. The child's runtime registers with the
7//! daemon for itself (see [`crate::registration`]), so the parent never states what the child
8//! holds — it only names which artifact is running.
9//!
10//! Three properties fall out of the child being a process rather than a thread:
11//!
12//! * A daemon crash takes no child with it. Nothing the child needs to keep running lives in the
13//! daemon's address space.
14//! * Each child has its own process environment and working directory, so a native subprocess
15//! started by one child inherits nothing a sibling shares.
16//! * A child that declares the `sealed` containment floor enters a mount namespace of its own,
17//! because the containment machinery installs per process and the child *is* a process.
18//!
19//! **The approval travels on the child's standard input.** Not on the argument vector and not in
20//! the environment: both are readable from `/proc/<pid>` by any process running as the same user,
21//! which is exactly what a sibling capsule's shell tool is.
22
23use std::io::{BufRead, BufReader, Write};
24use std::path::{Path, PathBuf};
25use std::process::{Child, Command, ExitStatus, Stdio};
26use std::sync::{mpsc, Arc, Condvar, Mutex};
27use std::time::{Duration, Instant};
28
29use crate::delegation::{
30 self, CompletionAddress, DelegationOutcome, DelegationStatus, Reporter, Spawner, SpawnerHandle,
31 SPAWNER_ENV,
32};
33use crate::errors::RuntimeError;
34use crate::mac_token;
35use crate::spawn_credential::SpawnApproval;
36
37/// Environment variable naming the `mur` binary to launch children with.
38///
39/// Defaults to [`std::env::current_exe`], which is correct in production: the process doing the
40/// launching *is* `mur`. A test harness that runs inside its own test binary sets this to the
41/// built `mur` instead.
42pub const MUR_BINARY_ENV: &str = "MURMUR_MUR_BINARY";
43
44/// How long the parent waits for a child to print its `--json` launch line.
45///
46/// Generous, because it bounds more than a handshake: a script capsule prints the line when its
47/// run has *finished*, and an agent capsule compiles its driver component before it binds a port,
48/// which on a cold host is the slowest thing in a launch.
49const CHILD_READY_TIMEOUT: Duration = Duration::from_secs(180);
50
51/// Owner-only, so a sibling capsule running as the same user cannot list or read a child's
52/// directory even though both sit under one parent.
53#[cfg(unix)]
54const CHILD_DIR_MODE: u32 = 0o700;
55
56/// How many of the child's last stderr lines are kept to explain a launch that never reported.
57///
58/// The child's diagnostics are the operator's, so every line is echoed to this process's stderr as
59/// it arrives; this bound is only what a *failure* quotes back, so a child that logged for an hour
60/// before dying does not turn into an unbounded error string.
61const CHILD_STDERR_TAIL_LINES: usize = 20;
62
63/// How long a post-mortem read of the tail waits for the drain to reach the end of the pipe.
64///
65/// Bounds a thread being scheduled, not the child: by the time anything asks for a dead child's
66/// tail the exit has already closed the write end, so the drain is at most one wake-up away.
67/// A pipe some surviving grandchild still holds open would never reach EOF at all, and the bound
68/// is what stops that inheriting process from wedging the parent's launch or its watcher.
69const CHILD_STDERR_DRAIN_TIMEOUT: Duration = Duration::from_secs(5);
70
71/// How often the completion watcher asks whether the child is still running.
72///
73/// It polls rather than blocking on `wait`, because the process handle is shared with
74/// [`LaunchedChild::shutdown`] and [`Drop`]: a watcher blocked inside `wait` would hold the lock
75/// those need, and a second `wait` on one child is not a thing two owners can both do.
76const CHILD_WATCH_INTERVAL: Duration = Duration::from_millis(100);
77
78/// What the parent's runtime needs in order to start one approved child.
79///
80/// Carries no manifest and no capability declaration: what the child holds is decided by the
81/// daemon from the child's own registry manifest, both at the `POST /spawn` that granted `grant`
82/// and again at the `POST /register` the child performs for itself.
83#[derive(Debug)]
84pub struct ChildLaunchRequest {
85 /// The parent's own accessible workdir. The child's directory is composed beneath it, which
86 /// is what keeps the child inside the single preopen the parent's WASI layer already has.
87 pub parent_accessible_workdir: PathBuf,
88 pub capsule_name: String,
89 pub capsule_version: String,
90 /// The approval `POST /spawn` returned, spent by the child's own registration.
91 pub grant: SpawnApproval,
92 /// The child's `capabilities.env.allow`. Names listed here are copied from the parent's
93 /// process environment into the child's; every other host variable the parent holds is
94 /// absent from the child, because the child's environment is built from a cleared one.
95 pub child_env_allow: Vec<String>,
96 /// Base URL of the daemon the child registers with.
97 pub roost_url: String,
98 /// The session this child belongs to, or `None` for a launch that is not a delegation.
99 ///
100 /// `Some` is what turns the launch into a delegation: the launcher mints a delegation id and
101 /// injects a [`SpawnerHandle`] as [`SPAWNER_ENV`], which is where the child reads the lineage
102 /// it records in its own `session_start`. The watcher that reports for a child which could
103 /// not report for itself starts only when that spawner also names a
104 /// [`CompletionAddress`] — a parent waiting on its own connection wants no completion.
105 /// `None` injects nothing and starts nothing.
106 pub spawner: Option<Spawner>,
107 /// How long the completion watcher observes this child before ending it, or `None` to observe
108 /// it for as long as it runs.
109 ///
110 /// Meaningful only alongside a [`CompletionAddress`], because the watcher that enforces it
111 /// starts only for a spawner that names one. On expiry the watcher stops the process and
112 /// posts a [`DelegationStatus::Terminated`] outcome naming the bound, which is the only thing
113 /// that tells a parent about a child that never ends.
114 pub completion_deadline: Option<Duration>,
115}
116
117/// How a child's process ended, as the watcher sees it.
118enum Ending {
119 /// The process exited on its own. `None` when it was reaped by someone who did not keep the
120 /// status.
121 Exited(Option<ExitStatus>),
122 /// The delegation was ended on purpose rather than by the child: by the parent, through
123 /// [`LaunchedChild::shutdown`] or `Drop`, or by the completion watcher at its deadline.
124 Deliberate,
125}
126
127/// The one owner of the child's process handle.
128///
129/// Shared behind a lock between [`LaunchedChild`] — which kills and reaps — and the completion
130/// watcher, which only ever asks whether the process is still there. Single ownership of the
131/// `Child` is what keeps `pid()`, `shutdown()` and `Drop` behaving as they did: nothing else
132/// takes the handle, and nothing else waits on it.
133struct ChildProcess {
134 /// `None` once [`LaunchedChild::shutdown`] or `Drop` has reaped the process.
135 child: Option<Child>,
136 /// Set before the kill by whoever ended the delegation on purpose, so a crash is never read
137 /// as a termination. It does not say *who*: the completion watcher sets it at its own
138 /// deadline too, and tells the two apart by a flag of its own.
139 deliberate: bool,
140 /// The exit status, once anyone has observed it.
141 status: Option<ExitStatus>,
142}
143
144impl ChildProcess {
145 /// How the process ended, or `None` while it is still running.
146 ///
147 /// Reaps a process that exited on its own, which is what makes a later `wait` in `shutdown`
148 /// return the cached status rather than block.
149 fn poll(&mut self) -> Option<Ending> {
150 if self.deliberate {
151 return Some(Ending::Deliberate);
152 }
153 let Some(child) = self.child.as_mut() else {
154 // Reaped by neither `shutdown` nor `Drop`, which both set `deliberate` first: this
155 // is unreachable, and reporting the exit is the harmless reading of it.
156 return Some(Ending::Exited(self.status));
157 };
158 match child.try_wait() {
159 Ok(Some(status)) => {
160 self.status = Some(status);
161 Some(Ending::Exited(Some(status)))
162 }
163 Ok(None) => None,
164 // The handle is unusable; treating the child as gone is the only outcome that lets
165 // the delegation be reported at all.
166 Err(_) => Some(Ending::Exited(self.status)),
167 }
168 }
169
170 /// Kill and reap. Idempotent.
171 fn end(&mut self) -> Result<(), std::io::Error> {
172 let Some(mut child) = self.child.take() else {
173 return Ok(());
174 };
175 // `kill` on an already-exited process is not an error worth surfacing — the wait below is
176 // what actually retires the entry in the process table.
177 let _ = child.kill();
178 let status = child.wait()?;
179 self.status = Some(status);
180 Ok(())
181 }
182}
183
184/// A running child capsule and the handles to end it.
185pub struct LaunchedChild {
186 /// The directory the parent created for this child, and passed as its `--workdir`. The
187 /// child's own session artifacts land at `<workdir>/.murmur/<session_id>/`.
188 pub workdir: PathBuf,
189 /// The session id the child's runtime minted for itself.
190 pub session_id: String,
191 /// The child's A2A endpoint, `http://host:port`. Empty for a script capsule, which binds no
192 /// port and has already finished by the time it reports.
193 pub capsule_url: String,
194 /// The argument vector the parent built, in order, starting with the binary.
195 ///
196 /// Exposed so a caller — a test, notably — can assert what was passed without reading
197 /// `/proc`. It carries no token: the approval goes in on standard input.
198 pub argv: Vec<String>,
199 /// The complete environment the parent built for this child, in insertion order. Also carries
200 /// no token.
201 pub env: Vec<(String, String)>,
202 /// The id this launch's completion reports under, `None` when no spawner was supplied.
203 ///
204 /// Minted here, injected into the child as part of [`SPAWNER_ENV`], and echoed back on the
205 /// completion — the one value that joins a delegation to the task it produces at the parent.
206 pub delegation_id: Option<String>,
207 /// The operator token of a child that declares `network.authentication`, read from
208 /// `tokens.operator` on its readiness line. The parent presents it on every call to the
209 /// child's door. Never printed by `Debug`.
210 pub door_token: Option<crate::door_auth::DoorToken>,
211 process: Arc<Mutex<ChildProcess>>,
212 /// The child's last [`CHILD_STDERR_TAIL_LINES`] lines, retained so a crash can say why.
213 stderr_tail: Arc<StderrTail>,
214 /// When the child process was started, for the completion's `duration_ms`.
215 started: Instant,
216 /// Set by [`LaunchedChild::release`]. The one thing that stops [`Drop`] signalling the child:
217 /// a released process is no longer this handle's to end.
218 released: bool,
219}
220
221impl std::fmt::Debug for LaunchedChild {
222 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
223 f.debug_struct("LaunchedChild")
224 .field("workdir", &self.workdir)
225 .field("session_id", &self.session_id)
226 .field("capsule_url", &self.capsule_url)
227 .field("delegation_id", &self.delegation_id)
228 .field(
229 "door_token",
230 &self.door_token.as_ref().map(|_| "<redacted>"),
231 )
232 .field("pid", &self.pid())
233 .finish()
234 }
235}
236
237impl LaunchedChild {
238 /// The child's operating-system process id. `0` once it has been reaped.
239 pub fn pid(&self) -> u32 {
240 lock(&self.process)
241 .child
242 .as_ref()
243 .map(Child::id)
244 .unwrap_or(0)
245 }
246
247 /// The child's last [`CHILD_STDERR_TAIL_LINES`] lines, oldest first.
248 ///
249 /// The same lines a crash completion quotes back, exposed so a caller can read what the child
250 /// said without scraping the stderr they were already echoed to.
251 pub fn stderr_tail(&self) -> Vec<String> {
252 self.stderr_tail.lines()
253 }
254
255 /// Terminate the child and reap it. Idempotent: a second call, or a call after `Drop` has
256 /// already run, does nothing.
257 ///
258 /// Marks the ending as deliberate, so the watcher records the delegation as `terminated` and
259 /// posts nothing: the only party that would be told is the party that did it.
260 pub fn shutdown(&mut self) -> Result<(), RuntimeError> {
261 let mut process = lock(&self.process);
262 process.deliberate = true;
263 process.end().map_err(|error| {
264 RuntimeError::Runtime(format!("failed to reap child capsule: {error}"))
265 })
266 }
267
268 /// Stop owning the child's lifetime without signalling it, and let this handle go.
269 ///
270 /// The one escape from [`Drop`]'s kill. After this the process keeps running and **only the
271 /// completion watcher started by [`launch_child_capsule`] will ever observe or reap it** — a
272 /// child released from a launch that named no [`CompletionAddress`] is a process nothing will
273 /// ever wait on.
274 ///
275 /// This is how a delegation outlives the call that made it: `delegate-task` returns as soon as
276 /// the child is running and holding its task, and what the child eventually did arrives at the
277 /// parent as a completion the watcher either forwards or writes itself.
278 pub fn release(mut self) {
279 self.released = true;
280 }
281}
282
283impl Drop for LaunchedChild {
284 /// Terminates and reaps, so a parent that returns early — including by panicking — leaves no
285 /// orphaned capsule process behind holding a port and a directory. Deliberate on the same
286 /// terms as [`LaunchedChild::shutdown`].
287 ///
288 /// A released child is left alone: its lifetime stopped being this handle's the moment
289 /// [`LaunchedChild::release`] was called.
290 fn drop(&mut self) {
291 if self.released {
292 return;
293 }
294 let mut process = lock(&self.process);
295 process.deliberate = true;
296 let _ = process.end();
297 }
298}
299
300/// `workdir` named from `root`, falling back to the absolute path when it sits outside it.
301///
302/// One rule, used by the delegating plane wherever it names a child's directory to the parent's
303/// agent or to the parent's trace.
304pub(crate) fn workdir_relative_to(workdir: &Path, root: &Path) -> String {
305 workdir
306 .strip_prefix(root)
307 .unwrap_or(workdir)
308 .to_string_lossy()
309 .replace('\\', "/")
310}
311
312/// A mutex this crate holds only for the length of one field access, so a poisoned lock is
313/// recovered from rather than turned into a panic in a `Drop`.
314fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
315 mutex
316 .lock()
317 .unwrap_or_else(|poisoned| poisoned.into_inner())
318}
319
320/// Where a child of this parent goes: `<parent accessible workdir>/.murmur/children/<name>-<16 hex>`.
321///
322/// The 16 hex characters are fresh per call, so delegating the same capsule name and version twice
323/// yields two directories rather than one shared one. Beneath `.murmur/`, alongside the parent's
324/// own session directories, so a child's tree is inside the parent's single preopen and is pruned
325/// with it.
326pub fn child_workdir_for(parent_accessible_workdir: &Path, capsule_name: &str) -> PathBuf {
327 let suffix = mac_token::random_hex(8).unwrap_or_else(|_| {
328 // The OS CSPRNG is not expected to fail; a nanosecond clock reading still distinguishes
329 // two delegations rather than collapsing them onto one directory.
330 format!(
331 "{:016x}",
332 std::time::SystemTime::now()
333 .duration_since(std::time::UNIX_EPOCH)
334 .map(|elapsed| elapsed.as_nanos() as u64)
335 .unwrap_or(0)
336 )
337 });
338 parent_accessible_workdir
339 .join(".murmur")
340 .join("children")
341 .join(format!("{capsule_name}-{suffix}"))
342}
343
344/// Create the child's directory, start `mur` on it, and wait for the child to report itself.
345pub fn launch_child_capsule(request: ChildLaunchRequest) -> Result<LaunchedChild, RuntimeError> {
346 let workdir = child_workdir_for(&request.parent_accessible_workdir, &request.capsule_name);
347 create_child_dir(&workdir)?;
348
349 let binary = mur_binary()?;
350 let argv: Vec<String> = vec![
351 binary.display().to_string(),
352 "run".to_string(),
353 "--capsule".to_string(),
354 request.capsule_name.clone(),
355 "--capsule-version".to_string(),
356 request.capsule_version.clone(),
357 "--workdir".to_string(),
358 workdir.display().to_string(),
359 "--json".to_string(),
360 "--no-env-file".to_string(),
361 "--spawn-grant-stdin".to_string(),
362 ];
363 // Minted here, once per launch: a caller holding one `Spawner` that delegates twice gets two
364 // delegation ids, and neither child can report under the other's.
365 let handle = request
366 .spawner
367 .as_ref()
368 .map(|spawner| SpawnerHandle::for_delegation(spawner, delegation::new_delegation_id()));
369 let env = child_environment(&request, handle.as_ref());
370 let started = Instant::now();
371
372 let mut command = Command::new(&binary);
373 command
374 .args(&argv[1..])
375 .current_dir(&workdir)
376 .env_clear()
377 .stdin(Stdio::piped())
378 .stdout(Stdio::piped())
379 .stderr(Stdio::piped());
380 for (key, value) in &env {
381 command.env(key, value);
382 }
383
384 let mut child = command.spawn().map_err(|error| {
385 RuntimeError::Runtime(format!(
386 "failed to start the child capsule process '{}': {error}",
387 binary.display()
388 ))
389 })?;
390
391 // The grant's one and only appearance outside the parent's memory: one line on a pipe that is
392 // closed immediately afterwards. Failing to write it is fatal — the child would otherwise sit
393 // waiting on a line that will never come.
394 let write_result = child
395 .stdin
396 .take()
397 .ok_or_else(|| "the child process exposed no standard input".to_string())
398 .and_then(|mut stdin| {
399 writeln!(stdin, "{}", request.grant.expose())
400 .map_err(|error| format!("failed to hand the child its launch grant: {error}"))
401 });
402 let stderr_tail = drain_stderr(&mut child);
403 let mut launched = LaunchedChild {
404 workdir,
405 session_id: String::new(),
406 capsule_url: String::new(),
407 argv,
408 env,
409 delegation_id: handle.as_ref().map(|handle| handle.delegation_id.clone()),
410 door_token: None,
411 process: Arc::new(Mutex::new(ChildProcess {
412 child: Some(child),
413 deliberate: false,
414 status: None,
415 })),
416 stderr_tail: Arc::clone(&stderr_tail),
417 started,
418 released: false,
419 };
420 if let Err(reason) = write_result {
421 return Err(RuntimeError::Runtime(reason));
422 }
423
424 let line = first_json_line(&mut launched, &stderr_tail)?;
425 let report: serde_json::Value = serde_json::from_str(&line).map_err(|error| {
426 RuntimeError::Runtime(format!(
427 "the child capsule's first --json line did not parse: {error}"
428 ))
429 })?;
430 let session_id = report
431 .get("session_id")
432 .and_then(serde_json::Value::as_str)
433 .ok_or_else(|| {
434 RuntimeError::Runtime(
435 "the child capsule's --json line carried no session_id".to_string(),
436 )
437 })?;
438 let url = report
439 .get("url")
440 .and_then(serde_json::Value::as_str)
441 .unwrap_or_default();
442
443 launched.session_id = session_id.to_string();
444 launched.door_token = readiness_door_token(&report);
445 // A script capsule binds no port and reports an empty url; promoting that to `http://` would
446 // manufacture an address nothing answers on.
447 launched.capsule_url = if url.is_empty() {
448 String::new()
449 } else {
450 format!("http://{url}")
451 };
452
453 // Started only for a launch somebody wants told about. A lineage-only handle names no
454 // address, so a child whose parent waits on the connection it already holds is launched with
455 // no watcher behind it and posts nothing anywhere.
456 if let Some(handle) = handle {
457 if let Some(address) = handle.report_to.clone() {
458 watch_for_completion(
459 &launched,
460 handle,
461 address,
462 &request.capsule_name,
463 &request.capsule_version,
464 request.completion_deadline,
465 );
466 }
467 }
468 Ok(launched)
469}
470
471/// The operator token a child's `--json` readiness line carries under `tokens.operator`, present
472/// only when the child declares `network.authentication`.
473fn readiness_door_token(report: &serde_json::Value) -> Option<crate::door_auth::DoorToken> {
474 report
475 .pointer("/tokens/operator")
476 .and_then(serde_json::Value::as_str)
477 .filter(|token| !token.is_empty())
478 .map(|token| crate::door_auth::DoorToken::new(token.to_string()))
479}
480
481/// Report for a child that ended without reporting for itself, and stop one that never ends.
482///
483/// Polls the shared process handle rather than blocking on `wait`, so it never contends with
484/// [`LaunchedChild::shutdown`] or `Drop` for ownership of the `Child`. For a released child this
485/// thread is the process's only remaining observer, and the only thing that will ever reap it.
486///
487/// `deadline` bounds how long the child is watched, from the instant this watcher starts — which
488/// is the instant the child reported itself ready, not the instant its process was spawned.
489/// Getting to that point is bounded separately by [`CHILD_READY_TIMEOUT`], and folding a cold
490/// host's staging time into the delegation deadline would end a child for being slow to start
491/// rather than slow to work. On expiry the watcher stops the child itself and posts a
492/// [`DelegationStatus::Terminated`] outcome naming the bound — the deadline is the watcher's own,
493/// held in a local flag rather than read back off [`ChildProcess::deliberate`], so an ending the
494/// *parent* chose stays distinguishable from one this thread imposed. `None` watches for as long
495/// as the child runs.
496///
497/// What it does when the process ends on its own is decided by what the child left behind:
498///
499/// * A completion the child already delivered is left alone — exactly one completion per
500/// delegation reaches the parent.
501/// * A completion the child recorded but could not deliver is retried once, and the file is
502/// rewritten with the result. Nothing is retried after that.
503/// * No completion at all means the child died without a word: the watcher builds one with
504/// `status: crashed`, carrying the exit status and the child's bounded stderr tail, and posts
505/// it in the child's place.
506/// * An ending the parent chose is recorded as `terminated` and posted to nobody, because the
507/// only party that would be told is the party that did it.
508fn watch_for_completion(
509 launched: &LaunchedChild,
510 handle: SpawnerHandle,
511 address: CompletionAddress,
512 capsule_name: &str,
513 capsule_version: &str,
514 deadline: Option<Duration>,
515) {
516 let process = Arc::clone(&launched.process);
517 let stderr_tail = Arc::clone(&launched.stderr_tail);
518 let workdir = launched.workdir.clone();
519 let session_id = launched.session_id.clone();
520 let capsule_name = capsule_name.to_string();
521 let capsule_version = capsule_version.to_string();
522 let started = launched.started;
523
524 let watching_from = Instant::now();
525 std::thread::spawn(move || {
526 let expires_at = deadline.map(|bound| watching_from + bound);
527 // Set here and read nowhere else: `ChildProcess::deliberate` is about to be set too, by
528 // this thread, so it can no longer say who chose the ending.
529 let mut expired = false;
530 let ending = loop {
531 if expires_at.is_some_and(|at| Instant::now() >= at) {
532 let mut process = lock(&process);
533 process.deliberate = true;
534 let _ = process.end();
535 expired = true;
536 break Ending::Deliberate;
537 }
538 if let Some(ending) = lock(&process).poll() {
539 break ending;
540 }
541 std::thread::sleep(CHILD_WATCH_INTERVAL);
542 };
543 let duration_ms = started.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
544
545 if let Some(recorded) = delegation::read_completion(&workdir) {
546 if recorded.delivered {
547 return;
548 }
549 let retried = delegation::report_completion(&handle, &address, recorded, &workdir);
550 if !retried.delivered {
551 // One retry, then the file and the line above it are the record.
552 crate::runtime_err!(
553 "[capsule-runtime] delegation {}: the child's completion is undelivered after one retry",
554 handle.delegation_id
555 );
556 }
557 return;
558 }
559
560 let outcome = |status: DelegationStatus, detail: String| DelegationOutcome {
561 delegation_id: handle.delegation_id.clone(),
562 capsule_name: capsule_name.clone(),
563 capsule_version: capsule_version.clone(),
564 session_id: session_id.clone(),
565 status,
566 // The launcher reports for a child that recorded nothing, so it makes no claim about
567 // a result file it never saw named.
568 result_path: None,
569 workdir: workdir.display().to_string(),
570 duration_ms,
571 detail: Some(detail),
572 reported_by: Reporter::Launcher,
573 delivered: false,
574 delivery_error: None,
575 };
576
577 match ending {
578 // The deadline is the parent's runtime acting on the parent's behalf while the parent
579 // is elsewhere, so unlike an ending the parent chose by hand this one is posted: the
580 // delegating agent asked for an outcome and this is the outcome.
581 Ending::Deliberate if expired => {
582 let bound = deadline.map(|bound| bound.as_secs()).unwrap_or_default();
583 delegation::report_completion(
584 &handle,
585 &address,
586 outcome(
587 DelegationStatus::Terminated,
588 format!(
589 "the capsule was still running after {bound}s and was ended at the \
590 delegation deadline"
591 ),
592 ),
593 &workdir,
594 );
595 }
596 Ending::Deliberate => {
597 delegation::record_terminated(
598 &workdir,
599 outcome(
600 DelegationStatus::Terminated,
601 "the parent ended this delegation".to_string(),
602 ),
603 );
604 }
605 Ending::Exited(status) => {
606 let status_text = match status {
607 Some(status) => status.to_string(),
608 None => "unknown exit status".to_string(),
609 };
610 let tail = stderr_tail.lines_at_end().join("\n");
611 let detail = if tail.is_empty() {
612 format!(
613 "the child process ended without recording a completion ({status_text})"
614 )
615 } else {
616 format!(
617 "the child process ended without recording a completion ({status_text}); \
618 its last stderr lines were:\n{tail}"
619 )
620 };
621 delegation::report_completion(
622 &handle,
623 &address,
624 outcome(DelegationStatus::Crashed, detail),
625 &workdir,
626 );
627 }
628 }
629 });
630}
631
632/// The child's complete environment, built from a cleared one.
633///
634/// `capabilities.env.allow` is the child's own, not the parent's: a sibling's declaration reaches
635/// nothing here, and a variable the parent holds but the child did not declare is simply absent.
636/// The runtime-owned names — `PATH`, `HOME`, `MURMUR_ROOST_URL`, and [`SPAWNER_ENV`] on a
637/// delegated launch — are applied last, so a child cannot displace the daemon URL it is required
638/// to register with, or the handle it reports its outcome to, by allowlisting the name.
639fn child_environment(
640 request: &ChildLaunchRequest,
641 handle: Option<&SpawnerHandle>,
642) -> Vec<(String, String)> {
643 let mut env: Vec<(String, String)> = Vec::new();
644 for key in &request.child_env_allow {
645 if let Ok(value) = std::env::var(key) {
646 env.push((key.clone(), value));
647 }
648 }
649 env.retain(|(key, _)| {
650 !matches!(key.as_str(), "PATH" | "HOME" | "MURMUR_ROOST_URL") && key != SPAWNER_ENV
651 });
652
653 if let Ok(path) = std::env::var("PATH") {
654 env.push(("PATH".to_string(), path));
655 }
656 if let Ok(home) = std::env::var("HOME") {
657 env.push(("HOME".to_string(), home));
658 }
659 env.push(("MURMUR_ROOST_URL".to_string(), request.roost_url.clone()));
660 // Last, with the other runtime-owned names, and only for a launch that has a spawner: a child
661 // nobody wants told holds no handle at all, whatever this process's own environment carries
662 // and whatever the child declared.
663 if let Some(handle) = handle {
664 env.push((SPAWNER_ENV.to_string(), handle.to_env_value()));
665 }
666 env
667}
668
669/// The binary a child is started from: [`MUR_BINARY_ENV`] when set, else this process's own image.
670fn mur_binary() -> Result<PathBuf, RuntimeError> {
671 if let Some(path) = std::env::var_os(MUR_BINARY_ENV) {
672 if !path.is_empty() {
673 return Ok(PathBuf::from(path));
674 }
675 }
676 std::env::current_exe().map_err(|error| {
677 RuntimeError::Runtime(format!(
678 "cannot locate the mur binary to launch a child capsule with: {error}"
679 ))
680 })
681}
682
683fn create_child_dir(workdir: &Path) -> Result<(), RuntimeError> {
684 std::fs::create_dir_all(workdir).map_err(|source| RuntimeError::CreateWorkdir {
685 path: workdir.display().to_string(),
686 source,
687 })?;
688 #[cfg(unix)]
689 {
690 use std::os::unix::fs::PermissionsExt;
691 std::fs::set_permissions(workdir, std::fs::Permissions::from_mode(CHILD_DIR_MODE))
692 .map_err(|source| RuntimeError::CreateWorkdir {
693 path: workdir.display().to_string(),
694 source,
695 })?;
696 }
697 Ok(())
698}
699
700/// The child's stderr as the parent keeps it: every line echoed to this process's stderr as it
701/// arrives, with the last [`CHILD_STDERR_TAIL_LINES`] retained so a child that dies before
702/// reporting can say why.
703struct StderrTail {
704 state: Mutex<StderrState>,
705 /// Notified once [`StderrState::at_eof`] is set.
706 drained: Condvar,
707}
708
709#[derive(Default)]
710struct StderrState {
711 lines: Vec<String>,
712 /// Whether the pipe has been read to its end, which the child's exit is what causes.
713 at_eof: bool,
714}
715
716impl StderrTail {
717 /// What the child has said so far, without waiting for anything. For a child that is still
718 /// running, which is the only state in which "so far" is the question being asked.
719 fn lines(&self) -> Vec<String> {
720 lock(&self.state).lines.clone()
721 }
722
723 /// What a child that has ended left behind, read only once the pipe is at its end.
724 ///
725 /// The exit that closes the child's stdout closes its stderr in the same breath, so the thread
726 /// that notices the ending and the thread still draining stderr are racing, and the tail is
727 /// empty for as long as the drain has not caught up. Reading it unsynchronised turns "refused
728 /// because the declared floor is unmeetable here" into a refusal that names no reason —
729 /// the one case the tail is kept for. Bounded by [`CHILD_STDERR_DRAIN_TIMEOUT`].
730 fn lines_at_end(&self) -> Vec<String> {
731 let (state, _) = self
732 .drained
733 .wait_timeout_while(lock(&self.state), CHILD_STDERR_DRAIN_TIMEOUT, |state| {
734 !state.at_eof
735 })
736 .unwrap_or_else(|poisoned| poisoned.into_inner());
737 state.lines.clone()
738 }
739
740 /// Mark the pipe finished and wake every post-mortem reader waiting on it.
741 fn finish(&self) {
742 lock(&self.state).at_eof = true;
743 self.drained.notify_all();
744 }
745}
746
747/// Start draining the child's stderr into a [`StderrTail`].
748///
749/// The child's diagnostics belong to the operator running the parent, so nothing is swallowed —
750/// this only remembers, in addition to printing.
751fn drain_stderr(child: &mut Child) -> Arc<StderrTail> {
752 let tail = Arc::new(StderrTail {
753 state: Mutex::new(StderrState::default()),
754 drained: Condvar::new(),
755 });
756 let Some(stderr) = child.stderr.take() else {
757 // No pipe is an end already reached: a reader that waited here would wait out the whole
758 // bound for lines that can never arrive.
759 tail.finish();
760 return tail;
761 };
762 let collector = Arc::clone(&tail);
763 std::thread::spawn(move || {
764 for line in BufReader::new(stderr).lines().map_while(Result::ok) {
765 crate::runtime_err!("{line}");
766 let mut state = lock(&collector.state);
767 if state.lines.len() == CHILD_STDERR_TAIL_LINES {
768 state.lines.remove(0);
769 }
770 state.lines.push(line);
771 }
772 collector.finish();
773 });
774 tail
775}
776
777/// Read the child's first standard-output line, then keep draining the pipe.
778///
779/// The drain matters: a child whose stdout pipe fills blocks forever, and a capsule that logs
780/// after its launch line would otherwise deadlock against a parent that had stopped reading.
781fn first_json_line(
782 launched: &mut LaunchedChild,
783 stderr_tail: &StderrTail,
784) -> Result<String, RuntimeError> {
785 let stdout = lock(&launched.process)
786 .child
787 .as_mut()
788 .and_then(|child| child.stdout.take())
789 .ok_or_else(|| {
790 RuntimeError::Runtime("the child process exposed no standard output".to_string())
791 })?;
792
793 let (tx, rx) = mpsc::channel::<Option<String>>();
794 std::thread::spawn(move || {
795 let mut reader = BufReader::new(stdout);
796 let mut line = String::new();
797 let first = match reader.read_line(&mut line) {
798 Ok(0) | Err(_) => None,
799 Ok(_) => Some(line.trim_end().to_string()),
800 };
801 let _ = tx.send(first);
802 // Everything the child says after its launch line goes to this process's own stdout, so
803 // the pipe stays drained for as long as the child lives. This thread outlives the parent's
804 // own readiness line by the whole life of the child, so it is the one relay most likely to
805 // meet a reader that has already gone: it relays the line and keeps draining either way.
806 let mut rest = String::new();
807 while let Ok(read) = reader.read_line(&mut rest) {
808 if read == 0 {
809 break;
810 }
811 crate::diagnostic::raw_to_stdout(&rest);
812 rest.clear();
813 }
814 });
815
816 match rx.recv_timeout(CHILD_READY_TIMEOUT) {
817 Ok(Some(line)) => Ok(line),
818 Ok(None) => {
819 let status = lock(&launched.process)
820 .child
821 .as_mut()
822 .and_then(|child| child.wait().ok())
823 .map(|status| status.to_string())
824 .unwrap_or_else(|| "unknown".to_string());
825 // The child's own refusal — an unmeetable containment floor, a registration the
826 // daemon declined — is written to its stderr and is the only thing that explains this
827 // failure, so it is quoted rather than replaced by a generic "did not start".
828 let reason = stderr_tail.lines_at_end().join("\n");
829 Err(RuntimeError::Runtime(format!(
830 "the child capsule '{}' exited without reporting a launch line ({status}): {reason}",
831 launched.workdir.display()
832 )))
833 }
834 Err(_) => Err(RuntimeError::Runtime(format!(
835 "the child capsule '{}' did not report a launch line within {}s",
836 launched.workdir.display(),
837 CHILD_READY_TIMEOUT.as_secs()
838 ))),
839 }
840}
841
842#[cfg(test)]
843mod tests {
844 use super::*;
845
846 #[test]
847 fn a_child_directory_is_unique_per_delegation() {
848 let parent = PathBuf::from("/tmp/parent");
849 let first = child_workdir_for(&parent, "worker");
850 let second = child_workdir_for(&parent, "worker");
851
852 assert_ne!(first, second);
853 assert_eq!(
854 first.parent(),
855 Some(parent.join(".murmur").join("children").as_path())
856 );
857 assert!(first
858 .file_name()
859 .unwrap()
860 .to_string_lossy()
861 .starts_with("worker-"));
862 // `<name>-<16 hex>`
863 assert_eq!(
864 first.file_name().unwrap().to_string_lossy().len(),
865 "worker-".len() + 16
866 );
867 }
868
869 fn request(env_allow: &[&str], spawner: Option<Spawner>) -> ChildLaunchRequest {
870 ChildLaunchRequest {
871 parent_accessible_workdir: PathBuf::from("/tmp/parent"),
872 capsule_name: "worker".to_string(),
873 capsule_version: "0.1.0".to_string(),
874 grant: SpawnApproval::new("msa1.token".to_string()),
875 child_env_allow: env_allow.iter().map(|name| name.to_string()).collect(),
876 roost_url: "http://127.0.0.1:7700".to_string(),
877 spawner,
878 completion_deadline: None,
879 }
880 }
881
882 /// A handle naming some *other* delegation, for the two cases that prove a child cannot be
883 /// handed this process's own.
884 ///
885 /// Readable rather than nonsense, because `SPAWNER_ENV` is process-wide and every other test
886 /// in this binary shares it: an unreadable value here refuses an unrelated `stage_session`
887 /// running beside it.
888 fn decoy() -> String {
889 SpawnerHandle {
890 session_id: "ses_decoy".to_string(),
891 context_id: "ctx_decoy".to_string(),
892 delegation_id: "dlg_decoy".to_string(),
893 report_to: None,
894 }
895 .to_env_value()
896 }
897
898 fn spawner() -> Spawner {
899 Spawner {
900 session_id: "ses_parent".to_string(),
901 context_id: "ctx_parent".to_string(),
902 report_to: Some(CompletionAddress {
903 url: "http://127.0.0.1:7000".to_string(),
904 trust: crate::origin::TrustClass::Trusted,
905 }),
906 }
907 }
908
909 #[test]
910 fn a_childs_environment_is_built_from_its_own_declaration() {
911 std::env::set_var("MURMUR_CHILD_LAUNCH_TEST_A", "a");
912 std::env::set_var("MURMUR_CHILD_LAUNCH_TEST_B", "b");
913
914 let env = child_environment(&request(&["MURMUR_CHILD_LAUNCH_TEST_A"], None), None);
915 let names: Vec<&str> = env.iter().map(|(key, _)| key.as_str()).collect();
916
917 assert!(names.contains(&"MURMUR_CHILD_LAUNCH_TEST_A"));
918 assert!(!names.contains(&"MURMUR_CHILD_LAUNCH_TEST_B"));
919 assert!(names.contains(&"MURMUR_ROOST_URL"));
920 assert!(!env.iter().any(|(_, value)| value.contains("msa1.token")));
921 }
922
923 #[test]
924 fn a_runtime_owned_name_cannot_be_displaced_by_a_declaration() {
925 std::env::set_var("MURMUR_ROOST_URL", "http://127.0.0.1:1");
926
927 let env = child_environment(&request(&["MURMUR_ROOST_URL"], None), None);
928 let urls: Vec<&String> = env
929 .iter()
930 .filter(|(key, _)| key == "MURMUR_ROOST_URL")
931 .map(|(_, value)| value)
932 .collect();
933
934 assert_eq!(urls, vec!["http://127.0.0.1:7700"]);
935 }
936
937 /// The injected handle is the one the parent composed, whatever this process's own
938 /// environment holds and whatever the child declared.
939 #[test]
940 fn the_spawner_handle_is_injected_last_and_cannot_be_displaced() {
941 std::env::set_var(SPAWNER_ENV, decoy());
942 let handle = SpawnerHandle::for_delegation(&spawner(), "dlg_0001".to_string());
943
944 let env = child_environment(&request(&[SPAWNER_ENV], Some(spawner())), Some(&handle));
945 let injected: Vec<&String> = env
946 .iter()
947 .filter(|(key, _)| key == SPAWNER_ENV)
948 .map(|(_, value)| value)
949 .collect();
950
951 assert_eq!(injected.len(), 1, "{env:?}");
952 assert_eq!(
953 SpawnerHandle::parse(injected[0]).expect("the injected value is a handle"),
954 handle
955 );
956 assert_eq!(
957 env.last().map(|(key, _)| key.as_str()),
958 Some(SPAWNER_ENV),
959 "the handle is applied in the runtime-owned tail: {env:?}"
960 );
961 std::env::remove_var(SPAWNER_ENV);
962 }
963
964 /// A launch with no spawner injects nothing, even from a decoy the launching process holds.
965 #[test]
966 fn a_launch_without_a_spawner_injects_no_handle() {
967 std::env::set_var(SPAWNER_ENV, decoy());
968
969 let env = child_environment(&request(&[SPAWNER_ENV], None), None);
970
971 assert!(!env.iter().any(|(key, _)| key == SPAWNER_ENV), "{env:?}");
972 std::env::remove_var(SPAWNER_ENV);
973 }
974
975 /// A child's exit closes its stdout and its stderr together, so the reader that reports the
976 /// ending can reach the tail before the drain has appended to it. The post-mortem read is the
977 /// one that must not: an empty tail is a refusal that names no reason.
978 #[test]
979 fn a_tail_read_after_the_end_waits_for_the_drain_to_catch_up() {
980 let tail = Arc::new(StderrTail {
981 state: Mutex::new(StderrState::default()),
982 drained: Condvar::new(),
983 });
984
985 let writer = Arc::clone(&tail);
986 std::thread::spawn(move || {
987 std::thread::sleep(Duration::from_millis(200));
988 lock(&writer.state)
989 .lines
990 .push("error[E-CAP-003]: not achievable on this host".to_string());
991 writer.finish();
992 });
993
994 assert!(tail.lines().is_empty());
995 assert_eq!(
996 tail.lines_at_end(),
997 vec!["error[E-CAP-003]: not achievable on this host".to_string()]
998 );
999 }
1000
1001 /// A child that exposed no stderr pipe has nothing to wait for, so the bound is never spent on
1002 /// lines that cannot arrive.
1003 #[test]
1004 fn a_tail_with_no_pipe_behind_it_is_at_its_end_already() {
1005 let tail = StderrTail {
1006 state: Mutex::new(StderrState::default()),
1007 drained: Condvar::new(),
1008 };
1009 tail.finish();
1010
1011 let started = Instant::now();
1012 assert!(tail.lines_at_end().is_empty());
1013 assert!(
1014 started.elapsed() < CHILD_STDERR_DRAIN_TIMEOUT,
1015 "{started:?}"
1016 );
1017 }
1018
1019 #[test]
1020 fn child_launch_reads_the_operator_token_from_the_readiness_line() {
1021 let report = serde_json::json!({
1022 "url": "localhost:1",
1023 "session_id": "ses_x",
1024 "tokens": {"operator": "mdt1.a.b", "watcher": "mdt1.c.d"},
1025 });
1026 assert_eq!(
1027 readiness_door_token(&report).map(|token| token.expose().to_string()),
1028 Some("mdt1.a.b".to_string())
1029 );
1030 let public = serde_json::json!({"url": "localhost:1", "session_id": "ses_x"});
1031 assert_eq!(readiness_door_token(&public), None);
1032 }
1033
1034 #[test]
1035 fn child_launch_debug_prints_no_door_token() {
1036 let child = LaunchedChild {
1037 workdir: PathBuf::from("/tmp/child"),
1038 session_id: "ses_child".to_string(),
1039 capsule_url: "http://localhost:1".to_string(),
1040 argv: Vec::new(),
1041 env: Vec::new(),
1042 delegation_id: None,
1043 door_token: Some(crate::door_auth::DoorToken::new(
1044 "mdt1.secretpayload.secretmac".to_string(),
1045 )),
1046 process: Arc::new(Mutex::new(ChildProcess {
1047 child: None,
1048 deliberate: false,
1049 status: None,
1050 })),
1051 stderr_tail: Arc::new(StderrTail {
1052 state: Mutex::new(StderrState::default()),
1053 drained: Condvar::new(),
1054 }),
1055 started: Instant::now(),
1056 released: true,
1057 };
1058 let debug = format!("{child:?}");
1059 assert!(!debug.contains("secretpayload"), "{debug}");
1060 assert!(debug.contains("door_token"), "{debug}");
1061 }
1062}