Skip to main content

capsule_runtime/
child_launch.rs

1//! Launching a delegated child as an operating-system process of the parent's own runtime.
2//!
3//! The daemon referees and hands back an approval; this is what turns that approval into a
4//! running capsule. The parent's runtime composes a directory for the child beneath its own
5//! accessible workdir, builds the child's environment from the child's declaration rather than
6//! its own, and starts the `mur` binary as a subprocess. The child's runtime registers with the
7//! daemon for itself (see [`crate::registration`]), so the parent never states what the child
8//! holds — it only names which artifact is running.
9//!
10//! Three properties fall out of the child being a process rather than a thread:
11//!
12//! * A daemon crash takes no child with it. Nothing the child needs to keep running lives in the
13//!   daemon's address space.
14//! * Each child has its own process environment and working directory, so a native subprocess
15//!   started by one child inherits nothing a sibling shares.
16//! * A child that declares the `sealed` containment floor enters a mount namespace of its own,
17//!   because the containment machinery installs per process and the child *is* a process.
18//!
19//! **The approval travels on the child's standard input.** Not on the argument vector and not in
20//! the environment: both are readable from `/proc/<pid>` by any process running as the same user,
21//! which is exactly what a sibling capsule's shell tool is.
22
23use std::io::{BufRead, BufReader, Write};
24use std::path::{Path, PathBuf};
25use std::process::{Child, Command, ExitStatus, Stdio};
26use std::sync::{mpsc, Arc, Condvar, Mutex};
27use std::time::{Duration, Instant};
28
29use crate::delegation::{
30    self, CompletionAddress, DelegationOutcome, DelegationStatus, Reporter, Spawner, SpawnerHandle,
31    SPAWNER_ENV,
32};
33use crate::errors::RuntimeError;
34use crate::mac_token;
35use crate::spawn_credential::SpawnApproval;
36
37/// Environment variable naming the `mur` binary to launch children with.
38///
39/// Defaults to [`std::env::current_exe`], which is correct in production: the process doing the
40/// launching *is* `mur`. A test harness that runs inside its own test binary sets this to the
41/// built `mur` instead.
42pub const MUR_BINARY_ENV: &str = "MURMUR_MUR_BINARY";
43
44/// How long the parent waits for a child to print its `--json` launch line.
45///
46/// Generous, because it bounds more than a handshake: a script capsule prints the line when its
47/// run has *finished*, and an agent capsule compiles its driver component before it binds a port,
48/// which on a cold host is the slowest thing in a launch.
49const CHILD_READY_TIMEOUT: Duration = Duration::from_secs(180);
50
51/// Owner-only, so a sibling capsule running as the same user cannot list or read a child's
52/// directory even though both sit under one parent.
53#[cfg(unix)]
54const CHILD_DIR_MODE: u32 = 0o700;
55
56/// How many of the child's last stderr lines are kept to explain a launch that never reported.
57///
58/// The child's diagnostics are the operator's, so every line is echoed to this process's stderr as
59/// it arrives; this bound is only what a *failure* quotes back, so a child that logged for an hour
60/// before dying does not turn into an unbounded error string.
61const CHILD_STDERR_TAIL_LINES: usize = 20;
62
63/// How long a post-mortem read of the tail waits for the drain to reach the end of the pipe.
64///
65/// Bounds a thread being scheduled, not the child: by the time anything asks for a dead child's
66/// tail the exit has already closed the write end, so the drain is at most one wake-up away.
67/// A pipe some surviving grandchild still holds open would never reach EOF at all, and the bound
68/// is what stops that inheriting process from wedging the parent's launch or its watcher.
69const CHILD_STDERR_DRAIN_TIMEOUT: Duration = Duration::from_secs(5);
70
71/// How often the completion watcher asks whether the child is still running.
72///
73/// It polls rather than blocking on `wait`, because the process handle is shared with
74/// [`LaunchedChild::shutdown`] and [`Drop`]: a watcher blocked inside `wait` would hold the lock
75/// those need, and a second `wait` on one child is not a thing two owners can both do.
76const CHILD_WATCH_INTERVAL: Duration = Duration::from_millis(100);
77
78/// What the parent's runtime needs in order to start one approved child.
79///
80/// Carries no manifest and no capability declaration: what the child holds is decided by the
81/// daemon from the child's own registry manifest, both at the `POST /spawn` that granted `grant`
82/// and again at the `POST /register` the child performs for itself.
83#[derive(Debug)]
84pub struct ChildLaunchRequest {
85    /// The parent's own accessible workdir. The child's directory is composed beneath it, which
86    /// is what keeps the child inside the single preopen the parent's WASI layer already has.
87    pub parent_accessible_workdir: PathBuf,
88    pub capsule_name: String,
89    pub capsule_version: String,
90    /// The approval `POST /spawn` returned, spent by the child's own registration.
91    pub grant: SpawnApproval,
92    /// The child's `capabilities.env.allow`. Names listed here are copied from the parent's
93    /// process environment into the child's; every other host variable the parent holds is
94    /// absent from the child, because the child's environment is built from a cleared one.
95    pub child_env_allow: Vec<String>,
96    /// Base URL of the daemon the child registers with.
97    pub roost_url: String,
98    /// The session this child belongs to, or `None` for a launch that is not a delegation.
99    ///
100    /// `Some` is what turns the launch into a delegation: the launcher mints a delegation id and
101    /// injects a [`SpawnerHandle`] as [`SPAWNER_ENV`], which is where the child reads the lineage
102    /// it records in its own `session_start`. The watcher that reports for a child which could
103    /// not report for itself starts only when that spawner also names a
104    /// [`CompletionAddress`] — a parent waiting on its own connection wants no completion.
105    /// `None` injects nothing and starts nothing.
106    pub spawner: Option<Spawner>,
107    /// How long the completion watcher observes this child before ending it, or `None` to observe
108    /// it for as long as it runs.
109    ///
110    /// Meaningful only alongside a [`CompletionAddress`], because the watcher that enforces it
111    /// starts only for a spawner that names one. On expiry the watcher stops the process and
112    /// posts a [`DelegationStatus::Terminated`] outcome naming the bound, which is the only thing
113    /// that tells a parent about a child that never ends.
114    pub completion_deadline: Option<Duration>,
115}
116
117/// How a child's process ended, as the watcher sees it.
118enum Ending {
119    /// The process exited on its own. `None` when it was reaped by someone who did not keep the
120    /// status.
121    Exited(Option<ExitStatus>),
122    /// The delegation was ended on purpose rather than by the child: by the parent, through
123    /// [`LaunchedChild::shutdown`] or `Drop`, or by the completion watcher at its deadline.
124    Deliberate,
125}
126
127/// The one owner of the child's process handle.
128///
129/// Shared behind a lock between [`LaunchedChild`] — which kills and reaps — and the completion
130/// watcher, which only ever asks whether the process is still there. Single ownership of the
131/// `Child` is what keeps `pid()`, `shutdown()` and `Drop` behaving as they did: nothing else
132/// takes the handle, and nothing else waits on it.
133struct ChildProcess {
134    /// `None` once [`LaunchedChild::shutdown`] or `Drop` has reaped the process.
135    child: Option<Child>,
136    /// Set before the kill by whoever ended the delegation on purpose, so a crash is never read
137    /// as a termination. It does not say *who*: the completion watcher sets it at its own
138    /// deadline too, and tells the two apart by a flag of its own.
139    deliberate: bool,
140    /// The exit status, once anyone has observed it.
141    status: Option<ExitStatus>,
142}
143
144impl ChildProcess {
145    /// How the process ended, or `None` while it is still running.
146    ///
147    /// Reaps a process that exited on its own, which is what makes a later `wait` in `shutdown`
148    /// return the cached status rather than block.
149    fn poll(&mut self) -> Option<Ending> {
150        if self.deliberate {
151            return Some(Ending::Deliberate);
152        }
153        let Some(child) = self.child.as_mut() else {
154            // Reaped by neither `shutdown` nor `Drop`, which both set `deliberate` first: this
155            // is unreachable, and reporting the exit is the harmless reading of it.
156            return Some(Ending::Exited(self.status));
157        };
158        match child.try_wait() {
159            Ok(Some(status)) => {
160                self.status = Some(status);
161                Some(Ending::Exited(Some(status)))
162            }
163            Ok(None) => None,
164            // The handle is unusable; treating the child as gone is the only outcome that lets
165            // the delegation be reported at all.
166            Err(_) => Some(Ending::Exited(self.status)),
167        }
168    }
169
170    /// Kill and reap. Idempotent.
171    fn end(&mut self) -> Result<(), std::io::Error> {
172        let Some(mut child) = self.child.take() else {
173            return Ok(());
174        };
175        // `kill` on an already-exited process is not an error worth surfacing — the wait below is
176        // what actually retires the entry in the process table.
177        let _ = child.kill();
178        let status = child.wait()?;
179        self.status = Some(status);
180        Ok(())
181    }
182}
183
184/// A running child capsule and the handles to end it.
185pub struct LaunchedChild {
186    /// The directory the parent created for this child, and passed as its `--workdir`. The
187    /// child's own session artifacts land at `<workdir>/.murmur/<session_id>/`.
188    pub workdir: PathBuf,
189    /// The session id the child's runtime minted for itself.
190    pub session_id: String,
191    /// The child's A2A endpoint, `http://host:port`. Empty for a script capsule, which binds no
192    /// port and has already finished by the time it reports.
193    pub capsule_url: String,
194    /// The argument vector the parent built, in order, starting with the binary.
195    ///
196    /// Exposed so a caller — a test, notably — can assert what was passed without reading
197    /// `/proc`. It carries no token: the approval goes in on standard input.
198    pub argv: Vec<String>,
199    /// The complete environment the parent built for this child, in insertion order. Also carries
200    /// no token.
201    pub env: Vec<(String, String)>,
202    /// The id this launch's completion reports under, `None` when no spawner was supplied.
203    ///
204    /// Minted here, injected into the child as part of [`SPAWNER_ENV`], and echoed back on the
205    /// completion — the one value that joins a delegation to the task it produces at the parent.
206    pub delegation_id: Option<String>,
207    /// The operator token of a child that declares `network.authentication`, read from
208    /// `tokens.operator` on its readiness line. The parent presents it on every call to the
209    /// child's door. Never printed by `Debug`.
210    pub door_token: Option<crate::door_auth::DoorToken>,
211    process: Arc<Mutex<ChildProcess>>,
212    /// The child's last [`CHILD_STDERR_TAIL_LINES`] lines, retained so a crash can say why.
213    stderr_tail: Arc<StderrTail>,
214    /// When the child process was started, for the completion's `duration_ms`.
215    started: Instant,
216    /// Set by [`LaunchedChild::release`]. The one thing that stops [`Drop`] signalling the child:
217    /// a released process is no longer this handle's to end.
218    released: bool,
219}
220
221impl std::fmt::Debug for LaunchedChild {
222    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
223        f.debug_struct("LaunchedChild")
224            .field("workdir", &self.workdir)
225            .field("session_id", &self.session_id)
226            .field("capsule_url", &self.capsule_url)
227            .field("delegation_id", &self.delegation_id)
228            .field(
229                "door_token",
230                &self.door_token.as_ref().map(|_| "<redacted>"),
231            )
232            .field("pid", &self.pid())
233            .finish()
234    }
235}
236
237impl LaunchedChild {
238    /// The child's operating-system process id. `0` once it has been reaped.
239    pub fn pid(&self) -> u32 {
240        lock(&self.process)
241            .child
242            .as_ref()
243            .map(Child::id)
244            .unwrap_or(0)
245    }
246
247    /// The child's last [`CHILD_STDERR_TAIL_LINES`] lines, oldest first.
248    ///
249    /// The same lines a crash completion quotes back, exposed so a caller can read what the child
250    /// said without scraping the stderr they were already echoed to.
251    pub fn stderr_tail(&self) -> Vec<String> {
252        self.stderr_tail.lines()
253    }
254
255    /// Terminate the child and reap it. Idempotent: a second call, or a call after `Drop` has
256    /// already run, does nothing.
257    ///
258    /// Marks the ending as deliberate, so the watcher records the delegation as `terminated` and
259    /// posts nothing: the only party that would be told is the party that did it.
260    pub fn shutdown(&mut self) -> Result<(), RuntimeError> {
261        let mut process = lock(&self.process);
262        process.deliberate = true;
263        process.end().map_err(|error| {
264            RuntimeError::Runtime(format!("failed to reap child capsule: {error}"))
265        })
266    }
267
268    /// Stop owning the child's lifetime without signalling it, and let this handle go.
269    ///
270    /// The one escape from [`Drop`]'s kill. After this the process keeps running and **only the
271    /// completion watcher started by [`launch_child_capsule`] will ever observe or reap it** — a
272    /// child released from a launch that named no [`CompletionAddress`] is a process nothing will
273    /// ever wait on.
274    ///
275    /// This is how a delegation outlives the call that made it: `delegate-task` returns as soon as
276    /// the child is running and holding its task, and what the child eventually did arrives at the
277    /// parent as a completion the watcher either forwards or writes itself.
278    pub fn release(mut self) {
279        self.released = true;
280    }
281}
282
283impl Drop for LaunchedChild {
284    /// Terminates and reaps, so a parent that returns early — including by panicking — leaves no
285    /// orphaned capsule process behind holding a port and a directory. Deliberate on the same
286    /// terms as [`LaunchedChild::shutdown`].
287    ///
288    /// A released child is left alone: its lifetime stopped being this handle's the moment
289    /// [`LaunchedChild::release`] was called.
290    fn drop(&mut self) {
291        if self.released {
292            return;
293        }
294        let mut process = lock(&self.process);
295        process.deliberate = true;
296        let _ = process.end();
297    }
298}
299
300/// `workdir` named from `root`, falling back to the absolute path when it sits outside it.
301///
302/// One rule, used by the delegating plane wherever it names a child's directory to the parent's
303/// agent or to the parent's trace.
304pub(crate) fn workdir_relative_to(workdir: &Path, root: &Path) -> String {
305    workdir
306        .strip_prefix(root)
307        .unwrap_or(workdir)
308        .to_string_lossy()
309        .replace('\\', "/")
310}
311
312/// A mutex this crate holds only for the length of one field access, so a poisoned lock is
313/// recovered from rather than turned into a panic in a `Drop`.
314fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
315    mutex
316        .lock()
317        .unwrap_or_else(|poisoned| poisoned.into_inner())
318}
319
320/// Where a child of this parent goes: `<parent accessible workdir>/.murmur/children/<name>-<16 hex>`.
321///
322/// The 16 hex characters are fresh per call, so delegating the same capsule name and version twice
323/// yields two directories rather than one shared one. Beneath `.murmur/`, alongside the parent's
324/// own session directories, so a child's tree is inside the parent's single preopen and is pruned
325/// with it.
326pub fn child_workdir_for(parent_accessible_workdir: &Path, capsule_name: &str) -> PathBuf {
327    let suffix = mac_token::random_hex(8).unwrap_or_else(|_| {
328        // The OS CSPRNG is not expected to fail; a nanosecond clock reading still distinguishes
329        // two delegations rather than collapsing them onto one directory.
330        format!(
331            "{:016x}",
332            std::time::SystemTime::now()
333                .duration_since(std::time::UNIX_EPOCH)
334                .map(|elapsed| elapsed.as_nanos() as u64)
335                .unwrap_or(0)
336        )
337    });
338    parent_accessible_workdir
339        .join(".murmur")
340        .join("children")
341        .join(format!("{capsule_name}-{suffix}"))
342}
343
344/// Create the child's directory, start `mur` on it, and wait for the child to report itself.
345pub fn launch_child_capsule(request: ChildLaunchRequest) -> Result<LaunchedChild, RuntimeError> {
346    let workdir = child_workdir_for(&request.parent_accessible_workdir, &request.capsule_name);
347    create_child_dir(&workdir)?;
348
349    let binary = mur_binary()?;
350    let argv: Vec<String> = vec![
351        binary.display().to_string(),
352        "run".to_string(),
353        "--capsule".to_string(),
354        request.capsule_name.clone(),
355        "--capsule-version".to_string(),
356        request.capsule_version.clone(),
357        "--workdir".to_string(),
358        workdir.display().to_string(),
359        "--json".to_string(),
360        "--no-env-file".to_string(),
361        "--spawn-grant-stdin".to_string(),
362    ];
363    // Minted here, once per launch: a caller holding one `Spawner` that delegates twice gets two
364    // delegation ids, and neither child can report under the other's.
365    let handle = request
366        .spawner
367        .as_ref()
368        .map(|spawner| SpawnerHandle::for_delegation(spawner, delegation::new_delegation_id()));
369    let env = child_environment(&request, handle.as_ref());
370    let started = Instant::now();
371
372    let mut command = Command::new(&binary);
373    command
374        .args(&argv[1..])
375        .current_dir(&workdir)
376        .env_clear()
377        .stdin(Stdio::piped())
378        .stdout(Stdio::piped())
379        .stderr(Stdio::piped());
380    for (key, value) in &env {
381        command.env(key, value);
382    }
383
384    let mut child = command.spawn().map_err(|error| {
385        RuntimeError::Runtime(format!(
386            "failed to start the child capsule process '{}': {error}",
387            binary.display()
388        ))
389    })?;
390
391    // The grant's one and only appearance outside the parent's memory: one line on a pipe that is
392    // closed immediately afterwards. Failing to write it is fatal — the child would otherwise sit
393    // waiting on a line that will never come.
394    let write_result = child
395        .stdin
396        .take()
397        .ok_or_else(|| "the child process exposed no standard input".to_string())
398        .and_then(|mut stdin| {
399            writeln!(stdin, "{}", request.grant.expose())
400                .map_err(|error| format!("failed to hand the child its launch grant: {error}"))
401        });
402    let stderr_tail = drain_stderr(&mut child);
403    let mut launched = LaunchedChild {
404        workdir,
405        session_id: String::new(),
406        capsule_url: String::new(),
407        argv,
408        env,
409        delegation_id: handle.as_ref().map(|handle| handle.delegation_id.clone()),
410        door_token: None,
411        process: Arc::new(Mutex::new(ChildProcess {
412            child: Some(child),
413            deliberate: false,
414            status: None,
415        })),
416        stderr_tail: Arc::clone(&stderr_tail),
417        started,
418        released: false,
419    };
420    if let Err(reason) = write_result {
421        return Err(RuntimeError::Runtime(reason));
422    }
423
424    let line = first_json_line(&mut launched, &stderr_tail)?;
425    let report: serde_json::Value = serde_json::from_str(&line).map_err(|error| {
426        RuntimeError::Runtime(format!(
427            "the child capsule's first --json line did not parse: {error}"
428        ))
429    })?;
430    let session_id = report
431        .get("session_id")
432        .and_then(serde_json::Value::as_str)
433        .ok_or_else(|| {
434            RuntimeError::Runtime(
435                "the child capsule's --json line carried no session_id".to_string(),
436            )
437        })?;
438    let url = report
439        .get("url")
440        .and_then(serde_json::Value::as_str)
441        .unwrap_or_default();
442
443    launched.session_id = session_id.to_string();
444    launched.door_token = readiness_door_token(&report);
445    // A script capsule binds no port and reports an empty url; promoting that to `http://` would
446    // manufacture an address nothing answers on.
447    launched.capsule_url = if url.is_empty() {
448        String::new()
449    } else {
450        format!("http://{url}")
451    };
452
453    // Started only for a launch somebody wants told about. A lineage-only handle names no
454    // address, so a child whose parent waits on the connection it already holds is launched with
455    // no watcher behind it and posts nothing anywhere.
456    if let Some(handle) = handle {
457        if let Some(address) = handle.report_to.clone() {
458            watch_for_completion(
459                &launched,
460                handle,
461                address,
462                &request.capsule_name,
463                &request.capsule_version,
464                request.completion_deadline,
465            );
466        }
467    }
468    Ok(launched)
469}
470
471/// The operator token a child's `--json` readiness line carries under `tokens.operator`, present
472/// only when the child declares `network.authentication`.
473fn readiness_door_token(report: &serde_json::Value) -> Option<crate::door_auth::DoorToken> {
474    report
475        .pointer("/tokens/operator")
476        .and_then(serde_json::Value::as_str)
477        .filter(|token| !token.is_empty())
478        .map(|token| crate::door_auth::DoorToken::new(token.to_string()))
479}
480
481/// Report for a child that ended without reporting for itself, and stop one that never ends.
482///
483/// Polls the shared process handle rather than blocking on `wait`, so it never contends with
484/// [`LaunchedChild::shutdown`] or `Drop` for ownership of the `Child`. For a released child this
485/// thread is the process's only remaining observer, and the only thing that will ever reap it.
486///
487/// `deadline` bounds how long the child is watched, from the instant this watcher starts — which
488/// is the instant the child reported itself ready, not the instant its process was spawned.
489/// Getting to that point is bounded separately by [`CHILD_READY_TIMEOUT`], and folding a cold
490/// host's staging time into the delegation deadline would end a child for being slow to start
491/// rather than slow to work. On expiry the watcher stops the child itself and posts a
492/// [`DelegationStatus::Terminated`] outcome naming the bound — the deadline is the watcher's own,
493/// held in a local flag rather than read back off [`ChildProcess::deliberate`], so an ending the
494/// *parent* chose stays distinguishable from one this thread imposed. `None` watches for as long
495/// as the child runs.
496///
497/// What it does when the process ends on its own is decided by what the child left behind:
498///
499/// * A completion the child already delivered is left alone — exactly one completion per
500///   delegation reaches the parent.
501/// * A completion the child recorded but could not deliver is retried once, and the file is
502///   rewritten with the result. Nothing is retried after that.
503/// * No completion at all means the child died without a word: the watcher builds one with
504///   `status: crashed`, carrying the exit status and the child's bounded stderr tail, and posts
505///   it in the child's place.
506/// * An ending the parent chose is recorded as `terminated` and posted to nobody, because the
507///   only party that would be told is the party that did it.
508fn watch_for_completion(
509    launched: &LaunchedChild,
510    handle: SpawnerHandle,
511    address: CompletionAddress,
512    capsule_name: &str,
513    capsule_version: &str,
514    deadline: Option<Duration>,
515) {
516    let process = Arc::clone(&launched.process);
517    let stderr_tail = Arc::clone(&launched.stderr_tail);
518    let workdir = launched.workdir.clone();
519    let session_id = launched.session_id.clone();
520    let capsule_name = capsule_name.to_string();
521    let capsule_version = capsule_version.to_string();
522    let started = launched.started;
523
524    let watching_from = Instant::now();
525    std::thread::spawn(move || {
526        let expires_at = deadline.map(|bound| watching_from + bound);
527        // Set here and read nowhere else: `ChildProcess::deliberate` is about to be set too, by
528        // this thread, so it can no longer say who chose the ending.
529        let mut expired = false;
530        let ending = loop {
531            if expires_at.is_some_and(|at| Instant::now() >= at) {
532                let mut process = lock(&process);
533                process.deliberate = true;
534                let _ = process.end();
535                expired = true;
536                break Ending::Deliberate;
537            }
538            if let Some(ending) = lock(&process).poll() {
539                break ending;
540            }
541            std::thread::sleep(CHILD_WATCH_INTERVAL);
542        };
543        let duration_ms = started.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
544
545        if let Some(recorded) = delegation::read_completion(&workdir) {
546            if recorded.delivered {
547                return;
548            }
549            let retried = delegation::report_completion(&handle, &address, recorded, &workdir);
550            if !retried.delivered {
551                // One retry, then the file and the line above it are the record.
552                crate::runtime_err!(
553                    "[capsule-runtime] delegation {}: the child's completion is undelivered after one retry",
554                    handle.delegation_id
555                );
556            }
557            return;
558        }
559
560        let outcome = |status: DelegationStatus, detail: String| DelegationOutcome {
561            delegation_id: handle.delegation_id.clone(),
562            capsule_name: capsule_name.clone(),
563            capsule_version: capsule_version.clone(),
564            session_id: session_id.clone(),
565            status,
566            // The launcher reports for a child that recorded nothing, so it makes no claim about
567            // a result file it never saw named.
568            result_path: None,
569            workdir: workdir.display().to_string(),
570            duration_ms,
571            detail: Some(detail),
572            reported_by: Reporter::Launcher,
573            delivered: false,
574            delivery_error: None,
575        };
576
577        match ending {
578            // The deadline is the parent's runtime acting on the parent's behalf while the parent
579            // is elsewhere, so unlike an ending the parent chose by hand this one is posted: the
580            // delegating agent asked for an outcome and this is the outcome.
581            Ending::Deliberate if expired => {
582                let bound = deadline.map(|bound| bound.as_secs()).unwrap_or_default();
583                delegation::report_completion(
584                    &handle,
585                    &address,
586                    outcome(
587                        DelegationStatus::Terminated,
588                        format!(
589                            "the capsule was still running after {bound}s and was ended at the \
590                             delegation deadline"
591                        ),
592                    ),
593                    &workdir,
594                );
595            }
596            Ending::Deliberate => {
597                delegation::record_terminated(
598                    &workdir,
599                    outcome(
600                        DelegationStatus::Terminated,
601                        "the parent ended this delegation".to_string(),
602                    ),
603                );
604            }
605            Ending::Exited(status) => {
606                let status_text = match status {
607                    Some(status) => status.to_string(),
608                    None => "unknown exit status".to_string(),
609                };
610                let tail = stderr_tail.lines_at_end().join("\n");
611                let detail = if tail.is_empty() {
612                    format!(
613                        "the child process ended without recording a completion ({status_text})"
614                    )
615                } else {
616                    format!(
617                        "the child process ended without recording a completion ({status_text}); \
618                         its last stderr lines were:\n{tail}"
619                    )
620                };
621                delegation::report_completion(
622                    &handle,
623                    &address,
624                    outcome(DelegationStatus::Crashed, detail),
625                    &workdir,
626                );
627            }
628        }
629    });
630}
631
632/// The child's complete environment, built from a cleared one.
633///
634/// `capabilities.env.allow` is the child's own, not the parent's: a sibling's declaration reaches
635/// nothing here, and a variable the parent holds but the child did not declare is simply absent.
636/// The runtime-owned names — `PATH`, `HOME`, `MURMUR_ROOST_URL`, and [`SPAWNER_ENV`] on a
637/// delegated launch — are applied last, so a child cannot displace the daemon URL it is required
638/// to register with, or the handle it reports its outcome to, by allowlisting the name.
639fn child_environment(
640    request: &ChildLaunchRequest,
641    handle: Option<&SpawnerHandle>,
642) -> Vec<(String, String)> {
643    let mut env: Vec<(String, String)> = Vec::new();
644    for key in &request.child_env_allow {
645        if let Ok(value) = std::env::var(key) {
646            env.push((key.clone(), value));
647        }
648    }
649    env.retain(|(key, _)| {
650        !matches!(key.as_str(), "PATH" | "HOME" | "MURMUR_ROOST_URL") && key != SPAWNER_ENV
651    });
652
653    if let Ok(path) = std::env::var("PATH") {
654        env.push(("PATH".to_string(), path));
655    }
656    if let Ok(home) = std::env::var("HOME") {
657        env.push(("HOME".to_string(), home));
658    }
659    env.push(("MURMUR_ROOST_URL".to_string(), request.roost_url.clone()));
660    // Last, with the other runtime-owned names, and only for a launch that has a spawner: a child
661    // nobody wants told holds no handle at all, whatever this process's own environment carries
662    // and whatever the child declared.
663    if let Some(handle) = handle {
664        env.push((SPAWNER_ENV.to_string(), handle.to_env_value()));
665    }
666    env
667}
668
669/// The binary a child is started from: [`MUR_BINARY_ENV`] when set, else this process's own image.
670fn mur_binary() -> Result<PathBuf, RuntimeError> {
671    if let Some(path) = std::env::var_os(MUR_BINARY_ENV) {
672        if !path.is_empty() {
673            return Ok(PathBuf::from(path));
674        }
675    }
676    std::env::current_exe().map_err(|error| {
677        RuntimeError::Runtime(format!(
678            "cannot locate the mur binary to launch a child capsule with: {error}"
679        ))
680    })
681}
682
683fn create_child_dir(workdir: &Path) -> Result<(), RuntimeError> {
684    std::fs::create_dir_all(workdir).map_err(|source| RuntimeError::CreateWorkdir {
685        path: workdir.display().to_string(),
686        source,
687    })?;
688    #[cfg(unix)]
689    {
690        use std::os::unix::fs::PermissionsExt;
691        std::fs::set_permissions(workdir, std::fs::Permissions::from_mode(CHILD_DIR_MODE))
692            .map_err(|source| RuntimeError::CreateWorkdir {
693                path: workdir.display().to_string(),
694                source,
695            })?;
696    }
697    Ok(())
698}
699
700/// The child's stderr as the parent keeps it: every line echoed to this process's stderr as it
701/// arrives, with the last [`CHILD_STDERR_TAIL_LINES`] retained so a child that dies before
702/// reporting can say why.
703struct StderrTail {
704    state: Mutex<StderrState>,
705    /// Notified once [`StderrState::at_eof`] is set.
706    drained: Condvar,
707}
708
709#[derive(Default)]
710struct StderrState {
711    lines: Vec<String>,
712    /// Whether the pipe has been read to its end, which the child's exit is what causes.
713    at_eof: bool,
714}
715
716impl StderrTail {
717    /// What the child has said so far, without waiting for anything. For a child that is still
718    /// running, which is the only state in which "so far" is the question being asked.
719    fn lines(&self) -> Vec<String> {
720        lock(&self.state).lines.clone()
721    }
722
723    /// What a child that has ended left behind, read only once the pipe is at its end.
724    ///
725    /// The exit that closes the child's stdout closes its stderr in the same breath, so the thread
726    /// that notices the ending and the thread still draining stderr are racing, and the tail is
727    /// empty for as long as the drain has not caught up. Reading it unsynchronised turns "refused
728    /// because the declared floor is unmeetable here" into a refusal that names no reason —
729    /// the one case the tail is kept for. Bounded by [`CHILD_STDERR_DRAIN_TIMEOUT`].
730    fn lines_at_end(&self) -> Vec<String> {
731        let (state, _) = self
732            .drained
733            .wait_timeout_while(lock(&self.state), CHILD_STDERR_DRAIN_TIMEOUT, |state| {
734                !state.at_eof
735            })
736            .unwrap_or_else(|poisoned| poisoned.into_inner());
737        state.lines.clone()
738    }
739
740    /// Mark the pipe finished and wake every post-mortem reader waiting on it.
741    fn finish(&self) {
742        lock(&self.state).at_eof = true;
743        self.drained.notify_all();
744    }
745}
746
747/// Start draining the child's stderr into a [`StderrTail`].
748///
749/// The child's diagnostics belong to the operator running the parent, so nothing is swallowed —
750/// this only remembers, in addition to printing.
751fn drain_stderr(child: &mut Child) -> Arc<StderrTail> {
752    let tail = Arc::new(StderrTail {
753        state: Mutex::new(StderrState::default()),
754        drained: Condvar::new(),
755    });
756    let Some(stderr) = child.stderr.take() else {
757        // No pipe is an end already reached: a reader that waited here would wait out the whole
758        // bound for lines that can never arrive.
759        tail.finish();
760        return tail;
761    };
762    let collector = Arc::clone(&tail);
763    std::thread::spawn(move || {
764        for line in BufReader::new(stderr).lines().map_while(Result::ok) {
765            crate::runtime_err!("{line}");
766            let mut state = lock(&collector.state);
767            if state.lines.len() == CHILD_STDERR_TAIL_LINES {
768                state.lines.remove(0);
769            }
770            state.lines.push(line);
771        }
772        collector.finish();
773    });
774    tail
775}
776
777/// Read the child's first standard-output line, then keep draining the pipe.
778///
779/// The drain matters: a child whose stdout pipe fills blocks forever, and a capsule that logs
780/// after its launch line would otherwise deadlock against a parent that had stopped reading.
781fn first_json_line(
782    launched: &mut LaunchedChild,
783    stderr_tail: &StderrTail,
784) -> Result<String, RuntimeError> {
785    let stdout = lock(&launched.process)
786        .child
787        .as_mut()
788        .and_then(|child| child.stdout.take())
789        .ok_or_else(|| {
790            RuntimeError::Runtime("the child process exposed no standard output".to_string())
791        })?;
792
793    let (tx, rx) = mpsc::channel::<Option<String>>();
794    std::thread::spawn(move || {
795        let mut reader = BufReader::new(stdout);
796        let mut line = String::new();
797        let first = match reader.read_line(&mut line) {
798            Ok(0) | Err(_) => None,
799            Ok(_) => Some(line.trim_end().to_string()),
800        };
801        let _ = tx.send(first);
802        // Everything the child says after its launch line goes to this process's own stdout, so
803        // the pipe stays drained for as long as the child lives. This thread outlives the parent's
804        // own readiness line by the whole life of the child, so it is the one relay most likely to
805        // meet a reader that has already gone: it relays the line and keeps draining either way.
806        let mut rest = String::new();
807        while let Ok(read) = reader.read_line(&mut rest) {
808            if read == 0 {
809                break;
810            }
811            crate::diagnostic::raw_to_stdout(&rest);
812            rest.clear();
813        }
814    });
815
816    match rx.recv_timeout(CHILD_READY_TIMEOUT) {
817        Ok(Some(line)) => Ok(line),
818        Ok(None) => {
819            let status = lock(&launched.process)
820                .child
821                .as_mut()
822                .and_then(|child| child.wait().ok())
823                .map(|status| status.to_string())
824                .unwrap_or_else(|| "unknown".to_string());
825            // The child's own refusal — an unmeetable containment floor, a registration the
826            // daemon declined — is written to its stderr and is the only thing that explains this
827            // failure, so it is quoted rather than replaced by a generic "did not start".
828            let reason = stderr_tail.lines_at_end().join("\n");
829            Err(RuntimeError::Runtime(format!(
830                "the child capsule '{}' exited without reporting a launch line ({status}): {reason}",
831                launched.workdir.display()
832            )))
833        }
834        Err(_) => Err(RuntimeError::Runtime(format!(
835            "the child capsule '{}' did not report a launch line within {}s",
836            launched.workdir.display(),
837            CHILD_READY_TIMEOUT.as_secs()
838        ))),
839    }
840}
841
842#[cfg(test)]
843mod tests {
844    use super::*;
845
846    #[test]
847    fn a_child_directory_is_unique_per_delegation() {
848        let parent = PathBuf::from("/tmp/parent");
849        let first = child_workdir_for(&parent, "worker");
850        let second = child_workdir_for(&parent, "worker");
851
852        assert_ne!(first, second);
853        assert_eq!(
854            first.parent(),
855            Some(parent.join(".murmur").join("children").as_path())
856        );
857        assert!(first
858            .file_name()
859            .unwrap()
860            .to_string_lossy()
861            .starts_with("worker-"));
862        // `<name>-<16 hex>`
863        assert_eq!(
864            first.file_name().unwrap().to_string_lossy().len(),
865            "worker-".len() + 16
866        );
867    }
868
869    fn request(env_allow: &[&str], spawner: Option<Spawner>) -> ChildLaunchRequest {
870        ChildLaunchRequest {
871            parent_accessible_workdir: PathBuf::from("/tmp/parent"),
872            capsule_name: "worker".to_string(),
873            capsule_version: "0.1.0".to_string(),
874            grant: SpawnApproval::new("msa1.token".to_string()),
875            child_env_allow: env_allow.iter().map(|name| name.to_string()).collect(),
876            roost_url: "http://127.0.0.1:7700".to_string(),
877            spawner,
878            completion_deadline: None,
879        }
880    }
881
882    /// A handle naming some *other* delegation, for the two cases that prove a child cannot be
883    /// handed this process's own.
884    ///
885    /// Readable rather than nonsense, because `SPAWNER_ENV` is process-wide and every other test
886    /// in this binary shares it: an unreadable value here refuses an unrelated `stage_session`
887    /// running beside it.
888    fn decoy() -> String {
889        SpawnerHandle {
890            session_id: "ses_decoy".to_string(),
891            context_id: "ctx_decoy".to_string(),
892            delegation_id: "dlg_decoy".to_string(),
893            report_to: None,
894        }
895        .to_env_value()
896    }
897
898    fn spawner() -> Spawner {
899        Spawner {
900            session_id: "ses_parent".to_string(),
901            context_id: "ctx_parent".to_string(),
902            report_to: Some(CompletionAddress {
903                url: "http://127.0.0.1:7000".to_string(),
904                trust: crate::origin::TrustClass::Trusted,
905            }),
906        }
907    }
908
909    #[test]
910    fn a_childs_environment_is_built_from_its_own_declaration() {
911        std::env::set_var("MURMUR_CHILD_LAUNCH_TEST_A", "a");
912        std::env::set_var("MURMUR_CHILD_LAUNCH_TEST_B", "b");
913
914        let env = child_environment(&request(&["MURMUR_CHILD_LAUNCH_TEST_A"], None), None);
915        let names: Vec<&str> = env.iter().map(|(key, _)| key.as_str()).collect();
916
917        assert!(names.contains(&"MURMUR_CHILD_LAUNCH_TEST_A"));
918        assert!(!names.contains(&"MURMUR_CHILD_LAUNCH_TEST_B"));
919        assert!(names.contains(&"MURMUR_ROOST_URL"));
920        assert!(!env.iter().any(|(_, value)| value.contains("msa1.token")));
921    }
922
923    #[test]
924    fn a_runtime_owned_name_cannot_be_displaced_by_a_declaration() {
925        std::env::set_var("MURMUR_ROOST_URL", "http://127.0.0.1:1");
926
927        let env = child_environment(&request(&["MURMUR_ROOST_URL"], None), None);
928        let urls: Vec<&String> = env
929            .iter()
930            .filter(|(key, _)| key == "MURMUR_ROOST_URL")
931            .map(|(_, value)| value)
932            .collect();
933
934        assert_eq!(urls, vec!["http://127.0.0.1:7700"]);
935    }
936
937    /// The injected handle is the one the parent composed, whatever this process's own
938    /// environment holds and whatever the child declared.
939    #[test]
940    fn the_spawner_handle_is_injected_last_and_cannot_be_displaced() {
941        std::env::set_var(SPAWNER_ENV, decoy());
942        let handle = SpawnerHandle::for_delegation(&spawner(), "dlg_0001".to_string());
943
944        let env = child_environment(&request(&[SPAWNER_ENV], Some(spawner())), Some(&handle));
945        let injected: Vec<&String> = env
946            .iter()
947            .filter(|(key, _)| key == SPAWNER_ENV)
948            .map(|(_, value)| value)
949            .collect();
950
951        assert_eq!(injected.len(), 1, "{env:?}");
952        assert_eq!(
953            SpawnerHandle::parse(injected[0]).expect("the injected value is a handle"),
954            handle
955        );
956        assert_eq!(
957            env.last().map(|(key, _)| key.as_str()),
958            Some(SPAWNER_ENV),
959            "the handle is applied in the runtime-owned tail: {env:?}"
960        );
961        std::env::remove_var(SPAWNER_ENV);
962    }
963
964    /// A launch with no spawner injects nothing, even from a decoy the launching process holds.
965    #[test]
966    fn a_launch_without_a_spawner_injects_no_handle() {
967        std::env::set_var(SPAWNER_ENV, decoy());
968
969        let env = child_environment(&request(&[SPAWNER_ENV], None), None);
970
971        assert!(!env.iter().any(|(key, _)| key == SPAWNER_ENV), "{env:?}");
972        std::env::remove_var(SPAWNER_ENV);
973    }
974
975    /// A child's exit closes its stdout and its stderr together, so the reader that reports the
976    /// ending can reach the tail before the drain has appended to it. The post-mortem read is the
977    /// one that must not: an empty tail is a refusal that names no reason.
978    #[test]
979    fn a_tail_read_after_the_end_waits_for_the_drain_to_catch_up() {
980        let tail = Arc::new(StderrTail {
981            state: Mutex::new(StderrState::default()),
982            drained: Condvar::new(),
983        });
984
985        let writer = Arc::clone(&tail);
986        std::thread::spawn(move || {
987            std::thread::sleep(Duration::from_millis(200));
988            lock(&writer.state)
989                .lines
990                .push("error[E-CAP-003]: not achievable on this host".to_string());
991            writer.finish();
992        });
993
994        assert!(tail.lines().is_empty());
995        assert_eq!(
996            tail.lines_at_end(),
997            vec!["error[E-CAP-003]: not achievable on this host".to_string()]
998        );
999    }
1000
1001    /// A child that exposed no stderr pipe has nothing to wait for, so the bound is never spent on
1002    /// lines that cannot arrive.
1003    #[test]
1004    fn a_tail_with_no_pipe_behind_it_is_at_its_end_already() {
1005        let tail = StderrTail {
1006            state: Mutex::new(StderrState::default()),
1007            drained: Condvar::new(),
1008        };
1009        tail.finish();
1010
1011        let started = Instant::now();
1012        assert!(tail.lines_at_end().is_empty());
1013        assert!(
1014            started.elapsed() < CHILD_STDERR_DRAIN_TIMEOUT,
1015            "{started:?}"
1016        );
1017    }
1018
1019    #[test]
1020    fn child_launch_reads_the_operator_token_from_the_readiness_line() {
1021        let report = serde_json::json!({
1022            "url": "localhost:1",
1023            "session_id": "ses_x",
1024            "tokens": {"operator": "mdt1.a.b", "watcher": "mdt1.c.d"},
1025        });
1026        assert_eq!(
1027            readiness_door_token(&report).map(|token| token.expose().to_string()),
1028            Some("mdt1.a.b".to_string())
1029        );
1030        let public = serde_json::json!({"url": "localhost:1", "session_id": "ses_x"});
1031        assert_eq!(readiness_door_token(&public), None);
1032    }
1033
1034    #[test]
1035    fn child_launch_debug_prints_no_door_token() {
1036        let child = LaunchedChild {
1037            workdir: PathBuf::from("/tmp/child"),
1038            session_id: "ses_child".to_string(),
1039            capsule_url: "http://localhost:1".to_string(),
1040            argv: Vec::new(),
1041            env: Vec::new(),
1042            delegation_id: None,
1043            door_token: Some(crate::door_auth::DoorToken::new(
1044                "mdt1.secretpayload.secretmac".to_string(),
1045            )),
1046            process: Arc::new(Mutex::new(ChildProcess {
1047                child: None,
1048                deliberate: false,
1049                status: None,
1050            })),
1051            stderr_tail: Arc::new(StderrTail {
1052                state: Mutex::new(StderrState::default()),
1053                drained: Condvar::new(),
1054            }),
1055            started: Instant::now(),
1056            released: true,
1057        };
1058        let debug = format!("{child:?}");
1059        assert!(!debug.contains("secretpayload"), "{debug}");
1060        assert!(debug.contains("door_token"), "{debug}");
1061    }
1062}