Skip to main content

capsule_runtime/
plan.rs

1use std::{
2    collections::{HashMap, HashSet},
3    path::Path,
4    sync::Arc,
5    thread,
6    time::Instant,
7};
8
9use serde::Deserialize;
10use serde_json::Value;
11
12use murmur_artifact::DEFAULT_EXPORT_MAX_BYTES;
13
14use crate::{
15    bindings::host::murmur::tool::run::{Status as ToolStatus, ToolInput, ToolResult},
16    delegation_plane::{
17        DelegationLaunch, DelegationOrigin, DelegationPlane, DelegationRequest, DelegationStatus,
18    },
19    errors::RuntimeError,
20    network_namespace::SessionNetns,
21    resource_plane::{self, SymlinkPolicy},
22    sandbox,
23    shell::{execute_shell, split_shell_words},
24    spawn_credential::SpawnCredential,
25    trace::{PlanEndRecord, PlanStepRecord, PlanStepShape, PlanTraceAppender},
26    types::CapabilityPolicy,
27    workdir_fs::WorkdirRoot,
28};
29
30pub mod condition;
31
32#[derive(Debug, Deserialize)]
33struct PlanFile {
34    id: String,
35    steps: Vec<StepDef>,
36}
37
38#[derive(Debug, Clone, Deserialize)]
39struct StepDef {
40    id: String,
41    tool: Option<String>,
42    shell: Option<String>,
43    capsule: Option<String>,
44    #[serde(default)]
45    input: Value,
46    #[serde(default)]
47    depends_on: Vec<String>,
48    #[serde(rename = "if")]
49    condition: Option<String>,
50    #[serde(default = "default_on_error")]
51    on_error: String,
52    #[serde(default)]
53    retries: u32,
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq)]
57enum StepKind {
58    Tool,
59    Shell,
60    Spawn,
61}
62
63impl StepKind {
64    /// The name a trace line carries for this kind. `"capsule"` rather than `"spawn"`: a trace
65    /// names the plan key its author wrote, not the host's word for what it does.
66    fn as_str(self) -> &'static str {
67        match self {
68            StepKind::Tool => "tool",
69            StepKind::Shell => "shell",
70            StepKind::Spawn => "capsule",
71        }
72    }
73}
74
75#[derive(Debug, Clone, PartialEq, Eq)]
76pub enum StepStatus {
77    Success,
78    Failed,
79    Skipped,
80}
81
82#[derive(Debug, Clone, PartialEq, Eq)]
83pub struct StepResult {
84    pub step_id: String,
85    pub status: StepStatus,
86    pub output: Option<String>,
87    pub error: Option<String>,
88}
89
90#[derive(Debug, Clone, PartialEq, Eq)]
91pub struct ExecutionReport {
92    pub results: Vec<StepResult>,
93    pub completed: bool,
94    /// `None` on a cancelled run: no step's failure is what stopped it.
95    pub failed_step: Option<String>,
96    /// Whether the run stopped because [`SchedulerContext::task`] reported its task cancelled.
97    /// `completed` is `false` whenever this is `true`.
98    pub canceled: bool,
99}
100
101/// The error a `capsule` step settles with when the task running its plan is cancelled while
102/// its sub-capsule is running.
103pub const CANCELED_STEP_ERROR: &str =
104    "the task running this plan was cancelled; this step's sub-capsule was ended with it";
105
106/// The error every step a cancelled run never dispatched settles `skipped` with.
107pub const CANCELED_BEFORE_STEP_ERROR: &str =
108    "the task running this plan was cancelled before this step ran";
109
110/// The `reason` on a cancelled run's `plan_end`.
111pub const PLAN_CANCELED_REASON: &str = "the task running this plan was cancelled";
112
113/// The task a plan runs for, as the scheduler sees it: whether it has been cancelled, and the set
114/// that accounts for every sub-capsule the task starts.
115///
116/// One rule keeps a `capsule` step's terminal `delegation` line single: whoever takes the step's
117/// row out of the task's set writes it. A step that ends on its own takes the row through
118/// [`PlanTask::delegation_settled`]; a step whose task was cancelled leaves it, and the task
119/// closes it after its `task_canceled`.
120pub trait PlanTask: Sync {
121    /// Whether the task this plan runs for has been cancelled. Polled; cheap, never blocks.
122    fn is_canceled(&self) -> bool;
123    /// A capsule step's child is up: account for it in the task's delegation set before it is
124    /// handed its task.
125    fn delegation_started(&self, launch: &DelegationLaunch);
126    /// The step is closing its row. `true` when the row was still the step's and the step now
127    /// writes the terminal `delegation` line; `false` when the task already holds it.
128    #[must_use]
129    fn delegation_settled(&self, delegation_id: &str) -> bool;
130}
131
132pub struct SchedulerContext<'a> {
133    /// The directory a `shell` step runs in and a `tool` step resolves a `data_path` against —
134    /// the capsule's own workdir, which under `--workdir` is the operator's directory.
135    pub accessible_workdir: WorkdirRoot,
136    /// The session's runtime directory and its siblings: the plan file is read from
137    /// `runtime/plans/`, and every `shell` step is told the directory through
138    /// `$MURMUR_RUNTIME_DIR` and may read it and not change it, as the session's own commands may.
139    /// A `shell` step's `$HOME` and `$TMPDIR` are its records directory's `home/` and `tmp/`.
140    pub runtime_dir: crate::runtime_dir::RuntimeDirRoots,
141    pub capability_policy: CapabilityPolicy,
142    pub installed_tools: HashSet<String>,
143    pub capsule_versions: HashMap<String, String>,
144    pub current_session_id: Option<String>,
145    /// The conversation the task that started this plan is running in, named on the
146    /// `MURMUR_SPAWNER` handle every `capsule` step injects into its child.
147    ///
148    /// Nothing is minted here. `None` names no conversation and injects no handle at all, which
149    /// is the existing behaviour for a caller that has none, and not an error: a child told it
150    /// was spawned from a conversation that does not exist is worse than one told nothing.
151    pub current_context_id: Option<String>,
152    /// Where a `capsule` step reads a sub-capsule's manifest from, to learn which host variables
153    /// that sub-capsule declares. The session's own store, so a step resolves the artifact its
154    /// child will run.
155    pub registry: std::sync::Arc<dyn murmur_artifact::Registry>,
156    /// The credential this session presents to `mur-roost` when a `capsule` step delegates.
157    ///
158    /// `None` for every session that was not granted `capabilities.spawn.allow`, and a `capsule`
159    /// step in such a session fails before any request is made. The token is held here and read
160    /// exactly twice — into the two request headers below — so nothing that formats this context,
161    /// a step result or a trace record can carry it.
162    pub spawn_credential: Option<SpawnCredential>,
163    /// The session's formation, which every `capsule` step's child joins. `None` for a session
164    /// nobody placed in a formation, whose children carry none either.
165    pub formation_id: Option<crate::formation::FormationId>,
166    /// The session's deadline, which every `capsule` step's child inherits as its own session
167    /// deadline. `None` bounds no child.
168    pub deadline: Option<crate::session_status::SessionDeadline>,
169    /// Where this run's per-step lifecycle records and its `capsule` steps' delegation records
170    /// go, or `None` to record nothing.
171    ///
172    /// Emission is opt-in at the call site: with `None` no file is opened and no line is
173    /// written, and the run's `ExecutionReport` is identical either way.
174    ///
175    /// A `capsule` step writes into it twice over, and the two pairs describe different things:
176    /// the plan-step pair is the scheduler's unit of work — its dependencies, its attempts, its
177    /// status after `on_error` — and the delegation pair is one child launch, named by its `dlg_`
178    /// id. A retried step is one plan step and several delegations.
179    pub trace: Option<&'a PlanTraceAppender>,
180    /// The session's own pre-dispatch decision point, asked about every `tool` and `shell` step
181    /// before it runs. `Some(reason)` fails that step with `reason` as its error and dispatches
182    /// nothing.
183    ///
184    /// A plan step is a call the model made, one indirection removed, so it is subject to the
185    /// same two refusals a direct call is: `capabilities.filesystem.read_only` and a
186    /// `commit_policy: deny` policy hook. Without this the scheduler would reach the session's
187    /// executors underneath both, and a command the session refuses would run as a plan step.
188    ///
189    /// A `capsule` step is not asked about: it runs nothing in this session, and the child it
190    /// delegates to applies its own manifest to whatever it does.
191    pub gate_step: &'a (dyn Fn(&PlannedCall<'_>) -> Option<String> + Sync),
192    pub invoke_tool: &'a (dyn Fn(&str, ToolInput) -> Result<ToolResult, String> + Sync),
193    /// The task this plan runs for, or `None` for a run that cannot be cancelled and whose
194    /// `capsule` steps account for their children in no set beyond their own trace lines.
195    ///
196    /// Once it reports cancelled, a running `capsule` step ends its sub-capsule, no step is
197    /// retried or dispatched, and the run ends `canceled`. A `tool` or `shell` step already
198    /// running finishes on its own bound.
199    pub task: Option<&'a dyn PlanTask>,
200}
201
202/// What a step is about to dispatch, as [`SchedulerContext::gate_step`] is shown it.
203///
204/// The scheduler holds no session state and resolves nothing itself: it hands over the same
205/// values it is about to hand the executor, and the gate turns them into whatever its own
206/// decision point reads.
207pub enum PlannedCall<'a> {
208    Tool {
209        name: &'a str,
210        /// The step's `input`, interpolated, exactly as the tool will receive it.
211        input_json: &'a str,
212    },
213    Shell {
214        /// The first word of the step's `shell` command.
215        binary: &'a str,
216        /// The step's `shell` command as written, untruncated.
217        command: &'a str,
218        /// The argument list the executable will receive, already split. No shell expands it, so
219        /// what is here is literally what runs.
220        argv: &'a [String],
221    },
222}
223
224/// Run the plan at `plan_path`. Its shell steps get a network namespace of their own, released
225/// when the plan ends.
226pub fn execute(plan_path: &Path, ctx: &SchedulerContext<'_>) -> ExecutionReport {
227    execute_with_netns(plan_path, ctx, None)
228}
229
230/// [`execute`] for a plan submitted inside a session: its shell steps run in that session's
231/// network namespace, so they reach a server the session's own commands started, and the plan
232/// leaves the namespace to the session when it ends.
233pub(crate) fn execute_in_session(
234    plan_path: &Path,
235    ctx: &SchedulerContext<'_>,
236    session_netns: Arc<SessionNetns>,
237) -> ExecutionReport {
238    execute_with_netns(plan_path, ctx, Some(session_netns))
239}
240
241fn execute_with_netns(
242    plan_path: &Path,
243    ctx: &SchedulerContext<'_>,
244    session_netns: Option<Arc<SessionNetns>>,
245) -> ExecutionReport {
246    let started = Instant::now();
247    match execute_inner(plan_path, ctx, started, session_netns) {
248        Ok(report) => report,
249        // The only failures that reach here are a plan file that could not be read or parsed, so
250        // there is no DAG to have described and no plan node for the summary to hang off.
251        Err((failed_step, error)) => {
252            if let Some(appender) = ctx.trace {
253                appender.write_plan_end(
254                    None,
255                    PlanEndRecord {
256                        plan_id: String::new(),
257                        outcome: "failed",
258                        failed_step: Some(failed_step.clone()),
259                        steps_total: 0,
260                        steps_succeeded: 0,
261                        steps_failed: 0,
262                        steps_skipped: 0,
263                        duration_ms: elapsed_ms(started),
264                        reason: Some(error.clone()),
265                    },
266                );
267            }
268            ExecutionReport {
269                results: vec![StepResult {
270                    step_id: failed_step.clone(),
271                    status: StepStatus::Failed,
272                    output: None,
273                    error: Some(error),
274                }],
275                completed: false,
276                failed_step: Some(failed_step),
277                canceled: false,
278            }
279        }
280    }
281}
282
283fn execute_inner(
284    plan_path: &Path,
285    ctx: &SchedulerContext<'_>,
286    started: Instant,
287    session_netns: Option<Arc<SessionNetns>>,
288) -> Result<ExecutionReport, (String, String)> {
289    // Read through the runtime directory's root, never by path, from the directory no capsule
290    // command can write: the plan that runs is the plan that was validated.
291    let runtime_root = &ctx.runtime_dir.runtime;
292    let relative = plan_path
293        .strip_prefix(runtime_root.path())
294        .unwrap_or(plan_path);
295    let raw = match runtime_root.read_to_string(relative) {
296        Ok(Some(raw)) => raw,
297        Ok(None) => {
298            return Err((
299                "plan".to_string(),
300                format!(
301                    "failed to read plan file {}: it does not exist",
302                    plan_path.display()
303                ),
304            ))
305        }
306        Err(error) => {
307            return Err((
308                "plan".to_string(),
309                format!("failed to read plan file: {error}"),
310            ))
311        }
312    };
313    let plan: PlanFile = serde_json::from_str(&raw).map_err(|error| {
314        (
315            "plan".to_string(),
316            format!("failed to parse plan JSON: {error}"),
317        )
318    })?;
319
320    // The DAG is described the moment the file parses, before it is validated: a plan the
321    // validator refuses still has a structure worth reading, and the refusal is only legible
322    // beside the steps it names.
323    let plan_node = ctx
324        .trace
325        .map(|appender| appender.write_plan_start(&plan.id, plan_shape(&plan)));
326
327    if let Err((step_id, error)) = validate_plan(&plan, ctx) {
328        trace_plan_end(
329            ctx,
330            plan_node.as_deref(),
331            PlanEndRecord {
332                plan_id: plan.id.clone(),
333                outcome: "failed",
334                failed_step: Some(step_id.clone()),
335                steps_total: plan.steps.len(),
336                steps_succeeded: 0,
337                steps_failed: 0,
338                steps_skipped: 0,
339                duration_ms: elapsed_ms(started),
340                reason: Some(error.clone()),
341            },
342        );
343        return Ok(ExecutionReport {
344            results: vec![StepResult {
345                step_id: step_id.clone(),
346                status: StepStatus::Failed,
347                output: None,
348                error: Some(error),
349            }],
350            completed: false,
351            failed_step: Some(step_id.clone()),
352            canceled: false,
353        });
354    }
355
356    // Aggregate process bounding for this plan's shell steps, on the same fail-closed terms as
357    // `runtime::launch_session`: a plan that can run a subprocess on Linux does not start at all
358    // unless its tree can be put in a cgroup scope. `has_native_artifact` is `false` here — a
359    // plan step reaches native binaries only through `invoke_tool`, which routes back into the
360    // session that already established its own scope.
361    let cgroup_scope = match crate::cgroup::prepare_policy_scope(
362        &ctx.capability_policy,
363        false,
364        &plan.id,
365        ctx.accessible_workdir.path(),
366    ) {
367        // The plan's own `io.max` outcome is dropped here rather than reported: this scope is a
368        // second one, created for the plan's shell steps inside a session whose `session_start`
369        // already carries the launch scope's `io_max` and whose launch already fired `W-SEC-021`
370        // if the ceiling did not apply. Both scopes sit on the same delegated base over the same
371        // workdir device, so a per-plan repetition would say nothing new.
372        Ok(prepared) => prepared.scope,
373        Err(reason) => {
374            let error =
375                crate::errors::RuntimeError::CgroupDelegationUnavailable { reason }.to_string();
376            return Ok(refused_before_any_step(
377                &plan,
378                ctx,
379                plan_node.as_deref(),
380                started,
381                error,
382            ));
383        }
384    };
385    let workdir_guard =
386        crate::resources::workdir_guard_max_bytes(&ctx.capability_policy).map(|max_bytes| {
387            crate::resources::WorkdirGuard::spawn(
388                &ctx.accessible_workdir,
389                ctx.runtime_dir.temp.as_ref(),
390                ctx.runtime_dir.home.as_ref(),
391                ctx.runtime_dir.records.as_ref(),
392                max_bytes,
393            )
394        });
395
396    // Host-probed kernel enforcement tier + resolved network allowlist for this plan's shell
397    // steps — resolved once, up front, same cadence as `runtime::launch_session`. The `/tmp`
398    // mapping is decided from the same reading, for the tier this enforcement applies.
399    let probe = sandbox::HostProbe::probe();
400    let mount_namespace_usable = probe.mount_namespace_usable();
401    // The runtime directory's placement caps the class and never the tier, and only the tier is
402    // installed here, so `false` stands in for a placement the plan context does not carry.
403    let decision = crate::containment::ContainmentDecision::decide(
404        probe.tier(),
405        ctx.capability_policy.containment_floor,
406        ctx.capability_policy.workdir_exec_allowed,
407        false,
408    );
409    let shell_enforcement =
410        match sandbox::ShellEnforcement::resolve(&ctx.capability_policy, &decision, probe) {
411            Ok(enforcement) => {
412                let temp_mapping = crate::temp_dir::temp_mapping(
413                    enforcement.tier,
414                    mount_namespace_usable,
415                    ctx.accessible_workdir.path(),
416                );
417                enforcement
418                    .with_host_bounding(cgroup_scope, workdir_guard)
419                    .with_runtime_dir(ctx.runtime_dir.clone(), None)
420                    .with_temp_mapping(temp_mapping)
421            }
422            Err(error) => {
423                let error =
424                    format!("failed to resolve shell subprocess sandbox enforcement: {error}");
425                return Ok(refused_before_any_step(
426                    &plan,
427                    ctx,
428                    plan_node.as_deref(),
429                    started,
430                    error,
431                ));
432            }
433        };
434    let owns_netns = session_netns.is_none();
435    let shell_enforcement = match session_netns {
436        Some(session_netns) => shell_enforcement.with_session_netns(session_netns),
437        None => shell_enforcement,
438    };
439    // Held to the end of this function, so every return from here on ends what the plan started.
440    let _plan_end = PlanEnd {
441        enforcement: &shell_enforcement,
442        owns_netns,
443    };
444
445    let mut results = HashMap::<String, StepResult>::new();
446    let mut result_order = Vec::<String>::new();
447    let mut failed_step = None;
448    let mut canceled = false;
449
450    loop {
451        if canceled {
452            // Nothing is dispatched once the task is cancelled, and nothing is left unsettled:
453            // every step without a result is recorded as never having run.
454            for step in &plan.steps {
455                if results.contains_key(&step.id) {
456                    continue;
457                }
458                trace_undispatched(
459                    ctx,
460                    plan_node.as_deref(),
461                    &plan.id,
462                    step,
463                    "skipped",
464                    Some(CANCELED_BEFORE_STEP_ERROR.to_string()),
465                );
466                insert_result(
467                    &mut results,
468                    &mut result_order,
469                    StepResult {
470                        step_id: step.id.clone(),
471                        status: StepStatus::Skipped,
472                        output: None,
473                        error: Some(CANCELED_BEFORE_STEP_ERROR.to_string()),
474                    },
475                );
476            }
477            break;
478        }
479        if results.len() == plan.steps.len() || failed_step.is_some() {
480            break;
481        }
482        if task_canceled(ctx) {
483            canceled = true;
484            continue;
485        }
486
487        let mut ready = Vec::new();
488        let mut progressed = false;
489
490        for step in &plan.steps {
491            if results.contains_key(&step.id) {
492                continue;
493            }
494            if !step.depends_on.iter().all(|dep| results.contains_key(dep)) {
495                continue;
496            }
497
498            match condition::evaluate(step.condition.as_deref(), &results) {
499                Ok(true) => ready.push(step.clone()),
500                Ok(false) => {
501                    // Never dispatched, so it writes no `plan_step_start` and no attempt: a step
502                    // that did not run must not read like one that did.
503                    trace_undispatched(ctx, plan_node.as_deref(), &plan.id, step, "skipped", None);
504                    insert_result(
505                        &mut results,
506                        &mut result_order,
507                        StepResult {
508                            step_id: step.id.clone(),
509                            status: StepStatus::Skipped,
510                            output: None,
511                            error: None,
512                        },
513                    );
514                    progressed = true;
515                }
516                Err(error) => {
517                    trace_undispatched(
518                        ctx,
519                        plan_node.as_deref(),
520                        &plan.id,
521                        step,
522                        "failed",
523                        Some(error.clone()),
524                    );
525                    insert_result(
526                        &mut results,
527                        &mut result_order,
528                        StepResult {
529                            step_id: step.id.clone(),
530                            status: StepStatus::Failed,
531                            output: None,
532                            error: Some(error),
533                        },
534                    );
535                    failed_step = Some(step.id.clone());
536                    progressed = true;
537                    break;
538                }
539            }
540        }
541
542        if failed_step.is_some() {
543            break;
544        }
545
546        if ready.is_empty() {
547            if progressed {
548                continue;
549            }
550            const BLOCKED: &str = "plan is blocked by a dependency cycle or missing dependency";
551            let blocked_step = plan
552                .steps
553                .iter()
554                .find(|step| !results.contains_key(&step.id));
555            let blocked = blocked_step
556                .map(|step| step.id.clone())
557                .unwrap_or_else(|| "plan".to_string());
558            if let Some(step) = blocked_step {
559                trace_undispatched(
560                    ctx,
561                    plan_node.as_deref(),
562                    &plan.id,
563                    step,
564                    "failed",
565                    Some(BLOCKED.to_string()),
566                );
567            }
568            insert_result(
569                &mut results,
570                &mut result_order,
571                StepResult {
572                    step_id: blocked.clone(),
573                    status: StepStatus::Failed,
574                    output: None,
575                    error: Some(BLOCKED.to_string()),
576                },
577            );
578            failed_step = Some(blocked);
579            break;
580        }
581
582        for step in &ready {
583            trace_step_start(ctx, plan_node.as_deref(), &plan.id, step);
584        }
585
586        let mut completed = Vec::new();
587        thread::scope(|scope| {
588            let mut handles = Vec::new();
589            for step in ready {
590                let snapshot = results.clone();
591                let shell_enforcement = &shell_enforcement;
592                handles.push(scope.spawn(move || {
593                    execute_step_with_retries(&step, ctx, &snapshot, shell_enforcement)
594                }));
595            }
596            for handle in handles {
597                completed.push(handle.join().unwrap_or_else(|_| {
598                    StepOutcome::undispatched(failed("unknown", "step dispatch thread panicked"))
599                }));
600            }
601        });
602        // A wave that ends with its task cancelled settles each step as it returned. No
603        // `on_error` policy applies: the run stops whatever the steps say, and a policy that
604        // turned a failure into `fail` would name a step as the reason it stopped.
605        canceled = task_canceled(ctx);
606
607        for outcome in completed {
608            let Some(step) = plan
609                .steps
610                .iter()
611                .find(|step| step.id == outcome.result.step_id)
612            else {
613                // A dispatch thread that panicked names no step this plan declared.
614                trace_step(
615                    ctx,
616                    plan_node.as_deref(),
617                    PlanStepRecord {
618                        plan_id: plan.id.clone(),
619                        step_id: outcome.result.step_id.clone(),
620                        kind: "unknown",
621                        status: outcome.result.status.as_str(),
622                        attempts: outcome.attempts,
623                        duration_ms: outcome.duration_ms,
624                        error: outcome.result.error.clone(),
625                        input: None,
626                        state_effect: None,
627                        resource_id: None,
628                    },
629                );
630                insert_result(&mut results, &mut result_order, outcome.result);
631                if !canceled {
632                    failed_step = Some("unknown".to_string());
633                    break;
634                }
635                continue;
636            };
637
638            let kind = step_kind_name(step);
639            let StepOutcome {
640                result,
641                attempts,
642                duration_ms,
643                input,
644                state_effect,
645                resource_id,
646            } = outcome;
647
648            // The `on_error` policy is applied here, before the step is recorded either way, so
649            // the trace and the returned report can never disagree about what a step settled as.
650            let (settled, stops) = if result.status == StepStatus::Failed && !canceled {
651                match step.on_error.as_str() {
652                    "fail" => (result, true),
653                    "skip" => (
654                        StepResult {
655                            step_id: result.step_id,
656                            status: StepStatus::Skipped,
657                            output: None,
658                            error: result.error,
659                        },
660                        false,
661                    ),
662                    "continue" => (result, false),
663                    policy => (
664                        StepResult {
665                            error: Some(format!("invalid on_error policy '{policy}'")),
666                            ..result
667                        },
668                        true,
669                    ),
670                }
671            } else {
672                (result, false)
673            };
674
675            trace_step(
676                ctx,
677                plan_node.as_deref(),
678                PlanStepRecord {
679                    plan_id: plan.id.clone(),
680                    step_id: settled.step_id.clone(),
681                    kind,
682                    status: settled.status.as_str(),
683                    attempts,
684                    duration_ms,
685                    error: settled.error.clone(),
686                    input,
687                    state_effect,
688                    resource_id,
689                },
690            );
691
692            let id = settled.step_id.clone();
693            insert_result(&mut results, &mut result_order, settled);
694            if stops {
695                failed_step = Some(id);
696                break;
697            }
698        }
699    }
700
701    let ordered_results = result_order
702        .into_iter()
703        .filter_map(|id| results.remove(&id))
704        .collect::<Vec<_>>();
705
706    if canceled {
707        failed_step = None;
708    }
709    trace_plan_end(
710        ctx,
711        plan_node.as_deref(),
712        PlanEndRecord {
713            plan_id: plan.id.clone(),
714            outcome: if canceled {
715                "canceled"
716            } else if failed_step.is_none() {
717                "completed"
718            } else {
719                "failed"
720            },
721            failed_step: failed_step.clone(),
722            steps_total: plan.steps.len(),
723            steps_succeeded: count_status(&ordered_results, StepStatus::Success),
724            steps_failed: count_status(&ordered_results, StepStatus::Failed),
725            steps_skipped: count_status(&ordered_results, StepStatus::Skipped),
726            duration_ms: elapsed_ms(started),
727            reason: canceled.then(|| PLAN_CANCELED_REASON.to_string()),
728        },
729    );
730
731    Ok(ExecutionReport {
732        results: ordered_results,
733        completed: !canceled && failed_step.is_none(),
734        failed_step,
735        canceled,
736    })
737}
738
739/// Ends what a plan's shell steps started, on whichever path the plan returns by.
740///
741/// Kills the plan's own cgroup scope, so nothing a step backgrounded outlives the plan. Releases
742/// the network namespace only when the plan created it: a session's namespace belongs to the
743/// session, whose own commands may still be using it.
744struct PlanEnd<'a> {
745    enforcement: &'a sandbox::ShellEnforcement,
746    owns_netns: bool,
747}
748
749impl Drop for PlanEnd<'_> {
750    fn drop(&mut self) {
751        self.enforcement.kill_session_processes();
752        if self.owns_netns {
753            self.enforcement.session_netns.end();
754        }
755    }
756}
757
758// ── Trace emission ───────────────────────────────────────────────────────────
759
760/// The name a step's trace lines carry for its kind. `"unknown"` only for a step declaring
761/// none or several of `tool`/`shell`/`capsule`, which `validate_plan` refuses — it can reach a
762/// line only through `plan_start`, which is written before validation.
763fn step_kind_name(step: &StepDef) -> &'static str {
764    infer_kind(step).map(StepKind::as_str).unwrap_or("unknown")
765}
766
767/// The DAG as authored, for `plan_start`.
768fn plan_shape(plan: &PlanFile) -> Vec<PlanStepShape> {
769    plan.steps
770        .iter()
771        .map(|step| PlanStepShape {
772            step_id: step.id.clone(),
773            kind: step_kind_name(step),
774            depends_on: step.depends_on.clone(),
775            has_condition: step.condition.is_some(),
776        })
777        .collect()
778}
779
780fn trace_step_start(
781    ctx: &SchedulerContext<'_>,
782    plan_node: Option<&str>,
783    plan_id: &str,
784    step: &StepDef,
785) {
786    if let (Some(appender), Some(node)) = (ctx.trace, plan_node) {
787        appender.write_plan_step_start(
788            node,
789            plan_id,
790            &step.id,
791            step_kind_name(step),
792            step.depends_on.clone(),
793        );
794    }
795}
796
797fn trace_step(ctx: &SchedulerContext<'_>, plan_node: Option<&str>, record: PlanStepRecord) {
798    if let (Some(appender), Some(node)) = (ctx.trace, plan_node) {
799        appender.write_plan_step(node, record);
800    }
801}
802
803/// The terminal line for a step that settled without ever being dispatched: a false `if`, a
804/// condition that would not evaluate, a step the DAG left unreachable. No attempt was made, so
805/// none is claimed.
806fn trace_undispatched(
807    ctx: &SchedulerContext<'_>,
808    plan_node: Option<&str>,
809    plan_id: &str,
810    step: &StepDef,
811    status: &'static str,
812    error: Option<String>,
813) {
814    trace_step(
815        ctx,
816        plan_node,
817        PlanStepRecord {
818            plan_id: plan_id.to_string(),
819            step_id: step.id.clone(),
820            kind: step_kind_name(step),
821            status,
822            attempts: 0,
823            duration_ms: 0,
824            error,
825            input: None,
826            state_effect: None,
827            resource_id: None,
828        },
829    );
830}
831
832fn trace_plan_end(ctx: &SchedulerContext<'_>, plan_node: Option<&str>, record: PlanEndRecord) {
833    if let Some(appender) = ctx.trace {
834        appender.write_plan_end(plan_node, record);
835    }
836}
837
838/// The report and the trace for a plan the host refused before any step ran: a cgroup scope it
839/// could not delegate, a shell sandbox it could not resolve. The DAG is already described, so
840/// the summary hangs off the plan node and names the plan — only the reason is new.
841fn refused_before_any_step(
842    plan: &PlanFile,
843    ctx: &SchedulerContext<'_>,
844    plan_node: Option<&str>,
845    started: Instant,
846    error: String,
847) -> ExecutionReport {
848    trace_plan_end(
849        ctx,
850        plan_node,
851        PlanEndRecord {
852            plan_id: plan.id.clone(),
853            outcome: "failed",
854            failed_step: Some("plan".to_string()),
855            steps_total: plan.steps.len(),
856            steps_succeeded: 0,
857            steps_failed: 0,
858            steps_skipped: 0,
859            duration_ms: elapsed_ms(started),
860            reason: Some(error.clone()),
861        },
862    );
863    ExecutionReport {
864        results: vec![StepResult {
865            step_id: "plan".to_string(),
866            status: StepStatus::Failed,
867            output: None,
868            error: Some(error),
869        }],
870        completed: false,
871        failed_step: Some("plan".to_string()),
872        canceled: false,
873    }
874}
875
876/// Whether the task this run is for has been cancelled. `false` for a run with no task.
877fn task_canceled(ctx: &SchedulerContext<'_>) -> bool {
878    ctx.task.is_some_and(PlanTask::is_canceled)
879}
880
881fn count_status(results: &[StepResult], status: StepStatus) -> usize {
882    results
883        .iter()
884        .filter(|result| result.status == status)
885        .count()
886}
887
888fn elapsed_ms(started: Instant) -> u64 {
889    started.elapsed().as_millis() as u64
890}
891
892fn validate_plan(plan: &PlanFile, ctx: &SchedulerContext<'_>) -> Result<(), (String, String)> {
893    let mut ids = HashSet::new();
894    for step in &plan.steps {
895        if !ids.insert(step.id.clone()) {
896            return Err((step.id.clone(), format!("duplicate step id '{}'", step.id)));
897        }
898
899        match infer_kind(step) {
900            Ok(StepKind::Tool) => {
901                let tool = step.tool.as_ref().unwrap();
902                // Refused here rather than bounded by a depth counter: a plan that could submit a
903                // plan would nest one scheduler inside another's worker thread, and the outer
904                // report would carry the inner one as a step's output text.
905                if tool == crate::runtime::SUBMIT_PLAN_TOOL {
906                    return Err((
907                        step.id.clone(),
908                        format!(
909                            "a plan step cannot submit another plan: '{tool}' is not callable \
910                             from inside a plan"
911                        ),
912                    ));
913                }
914                // The agent's own switch, ending its task without an answer, and pulling or
915                // removing an artifact are decisions it makes in a turn, which a plan step is
916                // not: the step would change the model, end the task, or change the session's
917                // tools under the turn that submitted the plan.
918                if tool == crate::runtime::SWITCH_DRIVER_TOOL
919                    || tool == crate::runtime::END_WITHOUT_ANSWER_TOOL
920                    || tool == crate::runtime::PULL_ARTIFACT_TOOL
921                    || tool == crate::runtime::REMOVE_ARTIFACT_TOOL
922                {
923                    return Err((
924                        step.id.clone(),
925                        format!("'{tool}' is not callable from inside a plan"),
926                    ));
927                }
928                if !ctx.installed_tools.contains(tool) {
929                    return Err((step.id.clone(), format!("tool '{tool}' is not installed")));
930                }
931            }
932            Ok(StepKind::Shell) => {
933                let Some(binary) = shell_binary(step.shell.as_deref().unwrap_or_default()) else {
934                    return Err((step.id.clone(), "shell command is empty".to_string()));
935                };
936                if !ctx.capability_policy.shell_allow.contains(&binary) {
937                    return Err((
938                        step.id.clone(),
939                        format!("binary '{binary}' is not in capabilities.shell.allow"),
940                    ));
941                }
942            }
943            Ok(StepKind::Spawn) => {
944                // spawn_allow is enforced by mur-roost from the parent's CapabilityPolicy.
945                // No local check here — the capsule cannot self-authorize its own spawn rights.
946
947                // The input shape is settled before any step runs, so a plan that could only
948                // hand its child an envelope is refused without taking a cgroup scope, without
949                // spawning anything, and without the earlier steps having already had effects.
950                if let Err(error) = capsule_task_text(&step.input) {
951                    return Err((step.id.clone(), error));
952                }
953            }
954            Err(error) => return Err((step.id.clone(), error)),
955        }
956
957        if !matches!(step.on_error.as_str(), "fail" | "skip" | "continue") {
958            return Err((
959                step.id.clone(),
960                format!("invalid on_error policy '{}'", step.on_error),
961            ));
962        }
963    }
964
965    for step in &plan.steps {
966        for dep in &step.depends_on {
967            if !ids.contains(dep) {
968                return Err((step.id.clone(), format!("unknown dependency '{dep}'")));
969            }
970        }
971        validate_references(step, &ids)?;
972    }
973
974    Ok(())
975}
976
977fn validate_references(step: &StepDef, ids: &HashSet<String>) -> Result<(), (String, String)> {
978    let mut references = Vec::new();
979    collect_references(&step.input, &mut references);
980    if let Some(condition) = &step.condition {
981        for token in condition.split_whitespace() {
982            if token.starts_with('$') {
983                references.push(token.trim_matches(|c| c == '(' || c == ')').to_string());
984            }
985        }
986    }
987
988    for reference in references {
989        if let Some((step_id, field)) = parse_reference(&reference) {
990            if !ids.contains(step_id) {
991                return Err((step.id.clone(), format!("unknown reference '{reference}'")));
992            }
993            if field != "output" && field != "status" {
994                return Err((
995                    step.id.clone(),
996                    format!("unknown reference field '{field}'"),
997                ));
998            }
999        }
1000    }
1001
1002    Ok(())
1003}
1004
1005fn collect_references(value: &Value, references: &mut Vec<String>) {
1006    match value {
1007        Value::String(value) if value.starts_with('$') => references.push(value.clone()),
1008        Value::Array(values) => values
1009            .iter()
1010            .for_each(|value| collect_references(value, references)),
1011        Value::Object(map) => map
1012            .values()
1013            .for_each(|value| collect_references(value, references)),
1014        _ => {}
1015    }
1016}
1017
1018/// What a worker thread reports back to the scheduler loop about the step it ran: the verdict,
1019/// plus the facts only the worker observed and the terminal `plan_step` line needs.
1020///
1021/// [`execute_step_once`] returns one attempt's own `attempts: 1` and duration;
1022/// [`execute_step_with_retries`] overwrites both with the totals for the step.
1023struct StepOutcome {
1024    result: StepResult,
1025    attempts: u32,
1026    duration_ms: u64,
1027    /// The interpolated input the attempt dispatched. `Some` for a tool step; `None` for every
1028    /// other kind, and for a failure that never reached a dispatch.
1029    input: Option<Value>,
1030    /// What the tool declared about this call in `tool-result.metadata`. Both `None` for every
1031    /// kind but a tool step, and for a tool that declared nothing.
1032    state_effect: Option<String>,
1033    resource_id: Option<String>,
1034}
1035
1036impl StepOutcome {
1037    /// One attempt of a step that declares nothing about itself: every kind but a tool step, and
1038    /// every failure that never reached a dispatch.
1039    fn plain(result: StepResult, started: Instant) -> Self {
1040        Self {
1041            result,
1042            attempts: 1,
1043            duration_ms: elapsed_ms(started),
1044            input: None,
1045            state_effect: None,
1046            resource_id: None,
1047        }
1048    }
1049
1050    /// A verdict reached without a dispatch, and so without an attempt to count or time.
1051    fn undispatched(result: StepResult) -> Self {
1052        Self {
1053            result,
1054            attempts: 0,
1055            duration_ms: 0,
1056            input: None,
1057            state_effect: None,
1058            resource_id: None,
1059        }
1060    }
1061}
1062
1063fn execute_step_with_retries(
1064    step: &StepDef,
1065    ctx: &SchedulerContext<'_>,
1066    results: &HashMap<String, StepResult>,
1067    enforcement: &sandbox::ShellEnforcement,
1068) -> StepOutcome {
1069    let started = Instant::now();
1070    let attempts = step.retries.saturating_add(1);
1071    let mut last = None;
1072    for attempt in 1..=attempts {
1073        let mut outcome = execute_step_once(step, ctx, results, enforcement);
1074        outcome.attempts = attempt;
1075        outcome.duration_ms = elapsed_ms(started);
1076        // A cancelled task's step is not tried again; the attempt that saw the cancel stands.
1077        if outcome.result.status != StepStatus::Failed || task_canceled(ctx) {
1078            return outcome;
1079        }
1080        last = Some(outcome);
1081    }
1082
1083    last.unwrap_or_else(|| StepOutcome::undispatched(failed(&step.id, "step was not attempted")))
1084}
1085
1086fn execute_step_once(
1087    step: &StepDef,
1088    ctx: &SchedulerContext<'_>,
1089    results: &HashMap<String, StepResult>,
1090    enforcement: &sandbox::ShellEnforcement,
1091) -> StepOutcome {
1092    let started = Instant::now();
1093    let mut input = step.input.clone();
1094    if let Err(error) = interpolate_value(&mut input, results) {
1095        return StepOutcome::plain(failed(&step.id, error), started);
1096    }
1097    let input_json = serde_json::to_string(&input).unwrap_or_else(|_| "{}".to_string());
1098
1099    match infer_kind(step) {
1100        Ok(StepKind::Tool) => {
1101            let (result, state_effect, resource_id) = dispatch_tool_step(step, ctx, input_json);
1102            StepOutcome {
1103                result,
1104                attempts: 1,
1105                duration_ms: elapsed_ms(started),
1106                // A step declaring no `input` interpolates to JSON null, which would write
1107                // `"input": null` rather than omitting the key. Absent means absent.
1108                input: (!input.is_null()).then_some(input),
1109                state_effect,
1110                resource_id,
1111            }
1112        }
1113        Ok(StepKind::Shell) => {
1114            StepOutcome::plain(dispatch_shell_step(step, ctx, enforcement), started)
1115        }
1116        Ok(StepKind::Spawn) => StepOutcome::plain(dispatch_capsule_step(step, ctx, input), started),
1117        Err(error) => StepOutcome::plain(failed(&step.id, error), started),
1118    }
1119}
1120
1121/// Dispatches one tool step, and returns the verdict beside the `state_effect` and `resource_id`
1122/// the tool declared about the call. Both are read through the agent loop's own extractors, so
1123/// a plan step and an agent turn read one tool's self-description the same way.
1124fn dispatch_tool_step(
1125    step: &StepDef,
1126    ctx: &SchedulerContext<'_>,
1127    input_json: String,
1128) -> (StepResult, Option<String>, Option<String>) {
1129    let name = step.tool.as_deref().unwrap_or_default();
1130    if let Some(refusal) = (ctx.gate_step)(&PlannedCall::Tool {
1131        name,
1132        input_json: &input_json,
1133    }) {
1134        return (failed(&step.id, refusal), None, None);
1135    }
1136    match (ctx.invoke_tool)(
1137        name,
1138        ToolInput {
1139            data: Some(input_json),
1140            log_path: None,
1141        },
1142    ) {
1143        Ok(result) => {
1144            let state_effect = crate::agent::extract_state_effect(&result.metadata);
1145            let resource_id = crate::agent::extract_resource_id(&result.metadata);
1146            let verdict = if matches!(result.status, ToolStatus::Passed) {
1147                match tool_step_output(
1148                    name,
1149                    ctx.accessible_workdir.path(),
1150                    result.data,
1151                    result.data_path.as_deref(),
1152                ) {
1153                    Ok(output) => StepResult {
1154                        step_id: step.id.clone(),
1155                        status: StepStatus::Success,
1156                        output,
1157                        error: None,
1158                    },
1159                    Err(error) => failed(&step.id, error),
1160                }
1161            } else {
1162                failed(
1163                    &step.id,
1164                    result
1165                        .summary
1166                        .or(result.data)
1167                        .unwrap_or_else(|| "tool step failed".to_string()),
1168                )
1169            };
1170            (verdict, state_effect, resource_id)
1171        }
1172        Err(error) => (failed(&step.id, error), None, None),
1173    }
1174}
1175
1176fn dispatch_shell_step(
1177    step: &StepDef,
1178    ctx: &SchedulerContext<'_>,
1179    enforcement: &sandbox::ShellEnforcement,
1180) -> StepResult {
1181    let command = step.shell.as_deref().unwrap_or_default();
1182    let Some((binary, args)) = parse_shell_command(command) else {
1183        return failed(&step.id, "shell command is empty");
1184    };
1185    if let Some(refusal) = (ctx.gate_step)(&PlannedCall::Shell {
1186        binary: &binary,
1187        command,
1188        argv: &args,
1189    }) {
1190        return failed(&step.id, refusal);
1191    }
1192    let arg_refs = args.iter().map(String::as_str).collect::<Vec<_>>();
1193
1194    match execute_shell(
1195        &binary,
1196        &arg_refs,
1197        &[],
1198        &ctx.accessible_workdir,
1199        &ctx.capability_policy,
1200        enforcement,
1201    ) {
1202        Ok(result) if result.exit_code == 0 => StepResult {
1203            step_id: step.id.clone(),
1204            status: StepStatus::Success,
1205            output: Some(result.stdout),
1206            error: None,
1207        },
1208        // A step killed for exceeding a host resource limit fails with the limit named, not with
1209        // whatever the dying process managed to write to stderr — for `SIGXCPU`/`SIGXFSZ` and the
1210        // cgroup kills that is usually nothing at all, which would otherwise report an empty
1211        // error for a step that was very deliberately terminated.
1212        Ok(result) => match result.resource_limit_hit {
1213            Some(limit) => failed(
1214                &step.id,
1215                crate::errors::RuntimeError::ShellResourceLimitExceeded {
1216                    binary: result.binary,
1217                    limit,
1218                    detail: if result.stderr.trim().is_empty() {
1219                        format!("exit code {}", result.exit_code)
1220                    } else {
1221                        result.stderr
1222                    },
1223                }
1224                .to_string(),
1225            ),
1226            // A command that fails without writing to stderr is the common case, not the odd
1227            // one — `test`, `grep -q` and `diff` all report only through their exit status. The
1228            // code is the whole of what failed, so the step carries it rather than an empty
1229            // error the model has to guess at.
1230            None if result.stderr.trim().is_empty() => {
1231                failed(&step.id, format!("exit code {}", result.exit_code))
1232            }
1233            None => failed(&step.id, result.stderr),
1234        },
1235        // Including a sealed composed-root failure, which fails the step carrying its full
1236        // named text. Unlike the agent turn loop (which owns a session and ends it — see
1237        // `agent::run_agent_loop`), this scheduler owns nothing above the step it is running,
1238        // so naming the cause in the step's error is the whole of what it can do.
1239        Err(error) => failed(&step.id, error.to_string()),
1240    }
1241}
1242
1243/// The session's `capabilities.spawn.workdir_snapshot`, copying its work folder and never its
1244/// runtime or records directory: what a `capsule` step's plane starts a listed child with, exactly
1245/// as the session's own `delegate-task` plane does.
1246fn snapshot_grant(
1247    ctx: &SchedulerContext<'_>,
1248    records_root: &WorkdirRoot,
1249) -> crate::delegation_plane::WorkdirSnapshotGrant {
1250    crate::delegation_plane::WorkdirSnapshotGrant::new(
1251        ctx.capability_policy.spawn_workdir_snapshot.clone(),
1252        ctx.accessible_workdir.clone(),
1253        &[&ctx.runtime_dir.runtime, records_root],
1254    )
1255}
1256
1257fn dispatch_capsule_step(step: &StepDef, ctx: &SchedulerContext<'_>, input: Value) -> StepResult {
1258    let named = step.capsule.as_deref().unwrap_or_default();
1259    // A step names a capsule by the bare part, as `delegate-task` does; mur-roost is asked about
1260    // the full `capabilities.spawn.allow` entry it stands for.
1261    let full = crate::tool_names::spawn_target(&ctx.capability_policy.spawn_allow, named);
1262    let capsule = full.as_str();
1263    // `validate_plan` already accepted this step's un-interpolated input, and interpolation
1264    // replaces a string with a string, so the refusal here is unreachable through `execute`.
1265    let task = match capsule_task_text(&input) {
1266        Ok(task) => task,
1267        Err(error) => return failed(&step.id, error),
1268    };
1269    let roost_url = match std::env::var("MURMUR_ROOST_URL") {
1270        Ok(value) if !value.trim().is_empty() => value,
1271        _ => {
1272            return failed(
1273                &step.id,
1274                "MURMUR_ROOST_URL is not set; capsule steps require mur-roost",
1275            )
1276        }
1277    };
1278    let Some(credential) = ctx.spawn_credential.as_ref() else {
1279        return failed(
1280            &step.id,
1281            "this session holds no spawn credential; a capsule step requires one, and mur-roost \
1282             mints it only for a session whose manifest declares capabilities.spawn.allow",
1283        );
1284    };
1285
1286    // Everything a delegation is — the referee's approval, the child's process, the task delivery
1287    // and the bounded wait — belongs to `DelegationPlane`, and a capsule step is input and output
1288    // marshalling around it. The agent-facing `delegate-task` tool calls the same code, so there
1289    // is one delegation mechanism in this crate rather than two that drift.
1290    // A plan step reads no manifest of its own here, so it delegates under the default bound
1291    // rather than under a declared one.
1292    let Some(records_root) = ctx.runtime_dir.records.clone() else {
1293        return failed(
1294            &step.id,
1295            "this plan runs outside a session's records directory, where a capsule step's child \
1296             work folder is created",
1297        );
1298    };
1299    let snapshot_grant = snapshot_grant(ctx, &records_root);
1300    let plane = DelegationPlane::new(
1301        roost_url,
1302        credential.clone(),
1303        records_root,
1304        ctx.current_session_id.clone().unwrap_or_default(),
1305        crate::delegation_plane::DELEGATION_RESULT_TIMEOUT,
1306        std::sync::Arc::clone(&ctx.registry),
1307        ctx.capability_policy.env_allow.clone(),
1308        ctx.formation_id.clone(),
1309    )
1310    .bounded_by(ctx.deadline)
1311    .snapshotting_workdir(snapshot_grant);
1312    // The launch notice is written where it arrives, on this thread, while the child is still
1313    // holding its task: `delegate` blocks until the child answers, so a notice parked for later
1314    // would only reach disk once there was already a terminal line to write beside it. The task
1315    // accounts for the child in the same callback, before the child is handed its task, so a
1316    // cancel from then on finds it in the task's set.
1317    let (trace, plan_task) = (ctx.trace, ctx.task);
1318    let launched = (trace.is_some() || plan_task.is_some()).then(|| {
1319        std::sync::Arc::new(move |notice: DelegationLaunch| {
1320            if let Some(trace) = trace {
1321                trace.write_delegation_start(
1322                    &notice.delegation_id,
1323                    &notice.capsule,
1324                    &notice.version,
1325                    &notice.child_session_id,
1326                    &notice.child_workdir,
1327                    &notice.child_records_dir,
1328                    notice.workdir_snapshot.as_ref(),
1329                );
1330            }
1331            if let Some(plan_task) = plan_task {
1332                plan_task.delegation_started(&notice);
1333            }
1334        }) as std::sync::Arc<dyn Fn(_) + Send + Sync>
1335    });
1336    let stop = plan_task.map(|plan_task| {
1337        std::sync::Arc::new(move || plan_task.is_canceled())
1338            as std::sync::Arc<dyn Fn() -> bool + Send + Sync>
1339    });
1340    let started = Instant::now();
1341    let result = plane.delegate(
1342        &DelegationRequest {
1343            capsule: capsule.to_string(),
1344            // A plan names a capsule; the version comes from the context the plan was validated
1345            // against, and a context that names none means `0.1.0`.
1346            version: ctx
1347                .capsule_versions
1348                .get(named)
1349                .cloned()
1350                .unwrap_or_else(|| "0.1.0".to_string()),
1351            task,
1352        },
1353        &DelegationOrigin {
1354            context_id: ctx.current_context_id.clone().unwrap_or_default(),
1355            launched,
1356            stop,
1357        },
1358        &ctx.runtime_dir.runtime,
1359    );
1360
1361    // The child is ended and its row is the cancelled task's: the task writes the terminal
1362    // `delegation` line after its `task_canceled`, so the step writes none.
1363    if result.status == DelegationStatus::Canceled {
1364        return failed(&step.id, CANCELED_STEP_ERROR);
1365    }
1366    // Whoever takes the row out of the task's set writes the terminal line. A step with no task,
1367    // or a delegation that named no child, has no row to take.
1368    let closes_row = match plan_task {
1369        Some(plan_task) if !result.delegation_id.is_empty() => {
1370            plan_task.delegation_settled(&result.delegation_id)
1371        }
1372        _ => true,
1373    };
1374    if let Some(trace) = ctx.trace.filter(|_| closes_row) {
1375        trace.write_delegation(
1376            &result.capsule,
1377            &result.version,
1378            // Empty rather than absent is how a result names no child; the record says `null`,
1379            // so nothing reading it can join against an id that was never minted.
1380            Some(result.delegation_id.clone()).filter(|id| !id.is_empty()),
1381            Some(result.session_id.clone()).filter(|id| !id.is_empty()),
1382            elapsed_ms(started),
1383            result.status.as_str(),
1384            match result.status {
1385                DelegationStatus::Completed => None,
1386                _ => Some(result.output.clone()),
1387            },
1388        );
1389    }
1390
1391    match result.status {
1392        DelegationStatus::Completed => StepResult {
1393            step_id: step.id.clone(),
1394            status: StepStatus::Success,
1395            output: Some(result.output),
1396            error: None,
1397        },
1398        _ => failed(&step.id, result.output),
1399    }
1400}
1401
1402/// The task text a capsule step hands to the child capsule, or the reason the step's `input`
1403/// cannot become task text.
1404///
1405/// Whatever this returns becomes the child's first user message verbatim: `DelegationPlane`
1406/// sends it as the sole text part of an A2A `SendMessage`, the receiving runtime writes that
1407/// text to `task.md`, and `agent::read_task` pushes it at the model. Nothing on the receiving
1408/// side parses a task envelope, so an envelope is not structure — it is prose that happens to be
1409/// JSON, and it spends the child's first turn on decoding. A capsule step's `input` is therefore
1410/// the task text and nothing else, in one of two shapes:
1411///
1412/// | `input` | task text |
1413/// | --- | --- |
1414/// | a JSON string | that string, verbatim |
1415/// | `{ "objective": "<text>" }`, and no other key | `<text>`, verbatim |
1416///
1417/// Every other shape is refused rather than serialized into the message. Interpolation replaces
1418/// a string with a string and so cannot change the shape, which is what lets `validate_plan`
1419/// refuse the whole plan before a cgroup scope is taken or a child is spawned.
1420fn capsule_task_text(input: &Value) -> Result<String, String> {
1421    const ACCEPTS: &str = "capsule step input must be a string or { \"objective\": \"<text>\" }";
1422    const NO_INPUT: &str = "capsule step has no input; its input is the task text the child \
1423                            receives, given as a string or as { \"objective\": \"<text>\" }";
1424
1425    match input {
1426        Value::String(text) => Ok(text.clone()),
1427        Value::Object(map) => {
1428            // serde_json's map is a BTreeMap or an IndexMap depending on the `preserve_order`
1429            // feature, so the listed keys are sorted here rather than taken in iteration order.
1430            let mut unknown = map
1431                .keys()
1432                .filter(|key| key.as_str() != "objective")
1433                .map(String::as_str)
1434                .collect::<Vec<_>>();
1435            unknown.sort_unstable();
1436            if !unknown.is_empty() {
1437                return Err(format!(
1438                    "capsule step input has keys the child cannot receive: {}; nothing on the \
1439                     receiving side parses a task envelope, so put the whole instruction in \
1440                     \"objective\"",
1441                    unknown.join(", ")
1442                ));
1443            }
1444
1445            match map.get("objective") {
1446                Some(Value::String(text)) => Ok(text.clone()),
1447                Some(_) => Err("capsule step input \"objective\" must be a string".to_string()),
1448                None => Err(ACCEPTS.to_string()),
1449            }
1450        }
1451        Value::Null => Err(NO_INPUT.to_string()),
1452        Value::Bool(_) => Err(format!("{ACCEPTS}, not a boolean")),
1453        Value::Number(_) => Err(format!("{ACCEPTS}, not a number")),
1454        Value::Array(_) => Err(format!("{ACCEPTS}, not an array")),
1455    }
1456}
1457
1458fn interpolate_value(
1459    value: &mut Value,
1460    results: &HashMap<String, StepResult>,
1461) -> Result<(), String> {
1462    match value {
1463        Value::String(text) if text.starts_with('$') => {
1464            *text = condition::resolve_reference(text, results)?;
1465        }
1466        Value::Array(values) => {
1467            for value in values {
1468                interpolate_value(value, results)?;
1469            }
1470        }
1471        Value::Object(map) => {
1472            for value in map.values_mut() {
1473                interpolate_value(value, results)?;
1474            }
1475        }
1476        _ => {}
1477    }
1478
1479    Ok(())
1480}
1481
1482fn infer_kind(step: &StepDef) -> Result<StepKind, String> {
1483    let count =
1484        step.tool.is_some() as u8 + step.shell.is_some() as u8 + step.capsule.is_some() as u8;
1485    match count {
1486        1 if step.tool.is_some() => Ok(StepKind::Tool),
1487        1 if step.shell.is_some() => Ok(StepKind::Shell),
1488        1 if step.capsule.is_some() => Ok(StepKind::Spawn),
1489        0 => Err("step must declare exactly one of tool, shell, or capsule".to_string()),
1490        _ => Err("step declares more than one of tool, shell, or capsule".to_string()),
1491    }
1492}
1493
1494fn default_on_error() -> String {
1495    "fail".to_string()
1496}
1497
1498fn failed(step_id: &str, error: impl Into<String>) -> StepResult {
1499    StepResult {
1500        step_id: step_id.to_string(),
1501        status: StepStatus::Failed,
1502        output: None,
1503        error: Some(error.into()),
1504    }
1505}
1506
1507fn insert_result(
1508    results: &mut HashMap<String, StepResult>,
1509    order: &mut Vec<String>,
1510    result: StepResult,
1511) {
1512    order.push(result.step_id.clone());
1513    results.insert(result.step_id.clone(), result);
1514}
1515
1516fn shell_binary(command: &str) -> Option<String> {
1517    parse_shell_command(command).map(|(binary, _)| binary)
1518}
1519
1520fn parse_shell_command(command: &str) -> Option<(String, Vec<String>)> {
1521    let parts = split_shell_words(command);
1522    let (binary, args) = parts.split_first()?;
1523    Some((binary.clone(), args.to_vec()))
1524}
1525
1526fn parse_reference(reference: &str) -> Option<(&str, &str)> {
1527    reference.strip_prefix('$')?.rsplit_once('.')
1528}
1529
1530/// The output of a tool step that passed: in-band `data` when the tool sent it, otherwise the
1531/// file the tool named in `data_path`.
1532///
1533/// `data_path` is the escape hatch for a result too large to carry in band — the system prompt
1534/// tells the model to read it when `truncated` is set — so a path the host cannot honour is
1535/// resolved against the guest's own boundary rather than dropped from the interface. The
1536/// resolution is [`resource_plane::read_file_beneath_root`] against `workdir`, the directory the
1537/// tool's `.` means, and a refusal fails the step: a step whose declared output could not be read
1538/// did not succeed.
1539///
1540/// The symlink policy is [`SymlinkPolicy::Refuse`] whatever containment class the session
1541/// achieved. `resource_plane::symlink_policy` follows symlinks under `sealed` and `advisory`
1542/// because a capsule reading its own export root has no outside to name; this read runs in the
1543/// host process, which sits outside every class's boundary with ambient filesystem authority, so
1544/// a symlink under the workdir pointing at `/etc/shadow` would be followed successfully at any
1545/// class.
1546fn tool_step_output(
1547    tool: &str,
1548    workdir: &Path,
1549    data: Option<String>,
1550    data_path: Option<&str>,
1551) -> Result<Option<String>, String> {
1552    if let Some(data) = data {
1553        return Ok(Some(data));
1554    }
1555    let Some(relpath) = data_path else {
1556        return Ok(None);
1557    };
1558    match resource_plane::read_file_beneath_root(
1559        workdir,
1560        relpath,
1561        DEFAULT_EXPORT_MAX_BYTES,
1562        SymlinkPolicy::Refuse,
1563    ) {
1564        Ok(response) => Ok(Some(String::from_utf8_lossy(&response.bytes).into_owned())),
1565        Err(error) => Err(RuntimeError::ToolDataPathRefused {
1566            tool: tool.to_string(),
1567            path: relpath.to_string(),
1568            code: error.code().to_string(),
1569            detail: error.message(),
1570        }
1571        .to_string()),
1572    }
1573}
1574
1575#[cfg(test)]
1576mod tests {
1577    use std::{fs, path::PathBuf};
1578    use std::{
1579        sync::{
1580            atomic::{AtomicUsize, Ordering},
1581            Arc, Condvar, Mutex,
1582        },
1583        time::Duration,
1584    };
1585
1586    use serde_json::json;
1587    use tempfile::tempdir;
1588
1589    use super::*;
1590
1591    /// The `shell_allow` grant below is what makes every test built on this context need a
1592    /// delegated cgroup v2 scope: `execute` bounds the plan's whole subprocess tree before
1593    /// running a step, and fails closed when it cannot.
1594    fn test_ctx<'a>(
1595        workdir: PathBuf,
1596        invoke_tool: &'a (dyn Fn(&str, ToolInput) -> Result<ToolResult, String> + Sync),
1597    ) -> SchedulerContext<'a> {
1598        test_ctx_gated(workdir, invoke_tool, &UNGATED)
1599    }
1600
1601    /// A decision point that refuses nothing, which is what a session with neither a policy hook
1602    /// nor a `capabilities.filesystem.read_only` declaration presents.
1603    const UNGATED: fn(&PlannedCall<'_>) -> Option<String> = |_| None;
1604
1605    fn test_ctx_gated<'a>(
1606        workdir: PathBuf,
1607        invoke_tool: &'a (dyn Fn(&str, ToolInput) -> Result<ToolResult, String> + Sync),
1608        gate_step: &'a (dyn Fn(&PlannedCall<'_>) -> Option<String> + Sync),
1609    ) -> SchedulerContext<'a> {
1610        SchedulerContext {
1611            accessible_workdir: WorkdirRoot::open(&workdir).unwrap(),
1612            runtime_dir: crate::runtime_dir::RuntimeDirRoots::new(
1613                WorkdirRoot::open(&workdir).unwrap(),
1614                WorkdirRoot::open(&workdir).unwrap(),
1615                WorkdirRoot::open(&workdir).unwrap(),
1616            )
1617            // Kept past the test: the context's clones may outlive it.
1618            .with_home(WorkdirRoot::open(&tempfile::tempdir().unwrap().keep()).unwrap()),
1619            capability_policy: CapabilityPolicy {
1620                shell_allow: vec!["bash".to_string()],
1621                spawn_allow: vec!["worker".to_string()],
1622                ..CapabilityPolicy::default()
1623            },
1624            installed_tools: HashSet::from([
1625                "ok".to_string(),
1626                "fail".to_string(),
1627                "slow".to_string(),
1628                "a".to_string(),
1629                "b".to_string(),
1630            ]),
1631            capsule_versions: HashMap::new(),
1632            current_session_id: None,
1633            current_context_id: None,
1634            // Nothing in these cases delegates, so no artifact is ever resolved through it.
1635            registry: std::sync::Arc::new(murmur_artifact::LocalRegistry::new(
1636                tempdir().unwrap().keep(),
1637            )),
1638            spawn_credential: None,
1639            formation_id: None,
1640            deadline: None,
1641            trace: None,
1642            gate_step,
1643            invoke_tool,
1644            task: None,
1645        }
1646    }
1647
1648    fn tool_result(status: ToolStatus, data: Option<String>) -> ToolResult {
1649        ToolResult {
1650            status,
1651            summary: None,
1652            data,
1653            data_path: None,
1654            truncated: false,
1655            metadata: Vec::new(),
1656            attachments: Vec::new(),
1657        }
1658    }
1659
1660    /// A `capsule` step's plane copies what the session's own `delegate-task` plane copies: the
1661    /// session's `capabilities.spawn.workdir_snapshot`, its work folder, and never its runtime or
1662    /// records directory.
1663    #[test]
1664    fn plan_capsule_step_snapshot_grant_is_the_sessions() {
1665        let work = tempdir().unwrap();
1666        let records = tempdir().unwrap();
1667        let invoke = |_: &str, _: ToolInput| -> Result<ToolResult, String> { unreachable!() };
1668        let mut ctx = test_ctx(work.path().to_path_buf(), &invoke);
1669        ctx.capability_policy.spawn_workdir_snapshot = vec!["local/worker".to_string()];
1670        let records_root = WorkdirRoot::open(records.path()).unwrap();
1671
1672        let grant = snapshot_grant(&ctx, &records_root);
1673
1674        assert_eq!(grant.capsules, vec!["local/worker".to_string()]);
1675        assert_eq!(grant.source.path(), work.path());
1676        assert_eq!(
1677            grant.excluded,
1678            vec![
1679                ctx.runtime_dir.runtime.identity().unwrap(),
1680                records_root.identity().unwrap()
1681            ]
1682        );
1683    }
1684
1685    fn write_plan(workdir: &Path, plan: Value) -> PathBuf {
1686        let path = workdir.join("plan.json");
1687        fs::write(&path, serde_json::to_string(&plan).unwrap()).unwrap();
1688        path
1689    }
1690
1691    /// The tools that are a turn's own decisions — switching the model, ending the task without
1692    /// an answer, pulling or removing an artifact — are refused as plan steps, before any step
1693    /// runs.
1694    #[test]
1695    fn a_turn_s_own_decisions_are_not_callable_from_a_plan() {
1696        let dir = tempdir().unwrap();
1697        let invoke = |_: &str, _: ToolInput| -> Result<ToolResult, String> {
1698            panic!("a refused plan ran a step")
1699        };
1700        let ctx = test_ctx(dir.path().to_path_buf(), &invoke);
1701        for tool in [
1702            crate::runtime::SWITCH_DRIVER_TOOL,
1703            crate::runtime::END_WITHOUT_ANSWER_TOOL,
1704            crate::runtime::PULL_ARTIFACT_TOOL,
1705            crate::runtime::REMOVE_ARTIFACT_TOOL,
1706        ] {
1707            let plan: PlanFile = serde_json::from_value(json!({
1708                "id": "p", "steps": [{"id": "s1", "tool": tool, "input": {"reason": "none"}}],
1709            }))
1710            .unwrap();
1711            assert_eq!(
1712                validate_plan(&plan, &ctx).unwrap_err(),
1713                (
1714                    "s1".to_string(),
1715                    format!("'{tool}' is not callable from inside a plan")
1716                )
1717            );
1718        }
1719    }
1720
1721    #[test]
1722    fn capsule_task_text_delivers_a_bare_string_verbatim() {
1723        assert_eq!(
1724            capsule_task_text(&json!("do the thing")).unwrap(),
1725            "do the thing"
1726        );
1727    }
1728
1729    #[test]
1730    fn capsule_task_text_delivers_a_lone_objective_verbatim() {
1731        assert_eq!(
1732            capsule_task_text(&json!({"objective": "do the thing"})).unwrap(),
1733            "do the thing"
1734        );
1735    }
1736
1737    /// An empty task is a well-shaped one. What a child does with nothing to do is the child's
1738    /// business, not this contract's.
1739    #[test]
1740    fn capsule_task_text_accepts_empty_task_text() {
1741        assert_eq!(capsule_task_text(&json!("")).unwrap(), "");
1742        assert_eq!(capsule_task_text(&json!({"objective": ""})).unwrap(), "");
1743    }
1744
1745    #[test]
1746    fn capsule_task_text_refuses_a_task_envelope() {
1747        let error = capsule_task_text(&json!({"task": "hello"})).unwrap_err();
1748        assert!(
1749            error.contains("keys the child cannot receive: task"),
1750            "{error}"
1751        );
1752        assert!(error.contains("\"objective\""), "{error}");
1753    }
1754
1755    #[test]
1756    fn capsule_task_text_lists_unaccepted_keys_in_sorted_order() {
1757        let error =
1758            capsule_task_text(&json!({"render_as": "json", "objective": "x", "instructions": "y"}))
1759                .unwrap_err();
1760        assert!(
1761            error.contains("keys the child cannot receive: instructions, render_as"),
1762            "{error}"
1763        );
1764    }
1765
1766    #[test]
1767    fn capsule_task_text_refuses_a_non_string_objective() {
1768        for input in [json!({"objective": 7}), json!({"objective": null})] {
1769            assert_eq!(
1770                capsule_task_text(&input).unwrap_err(),
1771                "capsule step input \"objective\" must be a string"
1772            );
1773        }
1774    }
1775
1776    #[test]
1777    fn capsule_task_text_refuses_an_object_without_an_objective() {
1778        assert_eq!(
1779            capsule_task_text(&json!({})).unwrap_err(),
1780            "capsule step input must be a string or { \"objective\": \"<text>\" }"
1781        );
1782    }
1783
1784    #[test]
1785    fn capsule_task_text_names_the_type_it_was_handed() {
1786        assert!(capsule_task_text(&json!(7))
1787            .unwrap_err()
1788            .ends_with("not a number"));
1789        assert!(capsule_task_text(&json!(true))
1790            .unwrap_err()
1791            .ends_with("not a boolean"));
1792        assert!(capsule_task_text(&json!(["a"]))
1793            .unwrap_err()
1794            .ends_with("not an array"));
1795    }
1796
1797    /// `StepDef::input` defaults to `Value::Null`, so an omitted `input` and an explicit null
1798    /// are the same value and get the same refusal.
1799    #[test]
1800    fn capsule_task_text_refuses_an_absent_input() {
1801        let error = capsule_task_text(&Value::Null).unwrap_err();
1802        assert_eq!(
1803            error,
1804            "capsule step has no input; its input is the task text the child receives, given as \
1805             a string or as { \"objective\": \"<text>\" }"
1806        );
1807    }
1808
1809    /// A `$reference` is a string, so both accepted shapes survive load-time validation and are
1810    /// resolved at dispatch.
1811    #[test]
1812    fn capsule_task_text_accepts_an_uninterpolated_reference() {
1813        assert_eq!(
1814            capsule_task_text(&json!("$analyse.output")).unwrap(),
1815            "$analyse.output"
1816        );
1817        assert_eq!(
1818            capsule_task_text(&json!({"objective": "$analyse.output"})).unwrap(),
1819            "$analyse.output"
1820        );
1821    }
1822
1823    /// [`test_ctx_gated`] with no `shell` or `spawn` grant, so the run needs no delegated cgroup
1824    /// scope and holds on any host.
1825    fn test_ctx_unbounded<'a>(
1826        workdir: PathBuf,
1827        invoke_tool: &'a (dyn Fn(&str, ToolInput) -> Result<ToolResult, String> + Sync),
1828        gate_step: &'a (dyn Fn(&PlannedCall<'_>) -> Option<String> + Sync),
1829    ) -> SchedulerContext<'a> {
1830        SchedulerContext {
1831            capability_policy: CapabilityPolicy::default(),
1832            ..test_ctx_gated(workdir, invoke_tool, gate_step)
1833        }
1834    }
1835
1836    /// The decision point refuses, and the step it refused never reaches the executor.
1837    ///
1838    /// The refusal has to arrive as the step's own error rather than as an empty success, because
1839    /// a plan whose steps quietly did nothing is indistinguishable from one that worked.
1840    #[test]
1841    fn a_refused_step_fails_with_the_refusal_and_is_never_dispatched() {
1842        let dir = tempdir().unwrap();
1843        let dispatched = Arc::new(AtomicUsize::new(0));
1844        let counted = Arc::clone(&dispatched);
1845        let invoke = move |_name: &str, _input: ToolInput| {
1846            counted.fetch_add(1, Ordering::SeqCst);
1847            Ok(tool_result(ToolStatus::Passed, Some("ran".to_string())))
1848        };
1849        let gate = |call: &PlannedCall<'_>| match call {
1850            PlannedCall::Tool { name: "a", .. } => Some("refused by the manifest".to_string()),
1851            _ => None,
1852        };
1853        let plan = write_plan(
1854            dir.path(),
1855            json!({
1856                "id": "p",
1857                "steps": [
1858                    {"id": "refused", "tool": "a", "on_error": "continue"},
1859                    {"id": "allowed", "tool": "b"}
1860                ]
1861            }),
1862        );
1863
1864        let report = execute(
1865            &plan,
1866            &test_ctx_unbounded(dir.path().to_path_buf(), &invoke, &gate),
1867        );
1868        let refused = report
1869            .results
1870            .iter()
1871            .find(|result| result.step_id == "refused")
1872            .expect("the refused step settled");
1873        assert_eq!(refused.status, StepStatus::Failed);
1874        assert_eq!(refused.error.as_deref(), Some("refused by the manifest"));
1875        assert_eq!(refused.output, None);
1876        // The one dispatch is the step the gate let through.
1877        assert_eq!(dispatched.load(Ordering::SeqCst), 1);
1878    }
1879
1880    /// A command that fails silently still says why. The model is handed the step's `error` as
1881    /// the whole account of what went wrong, so an empty one reports a failure with no reason.
1882    #[test]
1883    fn a_shell_step_that_fails_without_stderr_reports_its_exit_code() {
1884        if crate::cgroup::skip_without_host_support(
1885            "plan::tests::a_shell_step_that_fails_without_stderr_reports_its_exit_code",
1886        ) {
1887            return;
1888        }
1889        let dir = tempdir().unwrap();
1890        let invoke = |_name: &str, _input: ToolInput| Ok(tool_result(ToolStatus::Passed, None));
1891        let plan = write_plan(
1892            dir.path(),
1893            json!({
1894                "id": "p",
1895                "steps": [{"id": "quiet", "shell": "bash -c 'exit 3'"}]
1896            }),
1897        );
1898
1899        let report = execute(&plan, &test_ctx(dir.path().to_path_buf(), &invoke));
1900        assert!(!report.completed);
1901        assert_eq!(report.results[0].status, StepStatus::Failed);
1902        assert_eq!(report.results[0].error.as_deref(), Some("exit code 3"));
1903    }
1904
1905    /// A plan submitted inside a session runs its shell steps in that session's network
1906    /// namespace, so a step reaches a server one of the session's own commands started — and the
1907    /// plan leaves the namespace to the session when it ends.
1908    #[cfg(target_os = "linux")]
1909    #[test]
1910    #[allow(clippy::print_stdout)]
1911    fn a_plan_shell_step_joins_the_session_namespace() {
1912        if crate::cgroup::skip_without_host_support(
1913            "plan::tests::a_plan_shell_step_joins_the_session_namespace",
1914        ) {
1915            return;
1916        }
1917        let dir = tempdir().unwrap();
1918        let policy = CapabilityPolicy {
1919            shell_allow: vec!["bash".to_string(), "nc".to_string(), "sleep".to_string()],
1920            ..CapabilityPolicy::default()
1921        };
1922        let session = sandbox::ShellEnforcement::resolve_declared(
1923            &policy,
1924            murmur_artifact::ContainmentClass::Scoped,
1925            sandbox::HostProbe::probe(),
1926        )
1927        .unwrap()
1928        .with_test_session_dirs();
1929        let server = execute_shell(
1930            "bash",
1931            &[
1932                "-c",
1933                "(printf \"pong\\n\" | nc -N -l 127.0.0.1 8080) >/dev/null 2>&1 & echo $! > server.pid",
1934            ],
1935            &[],
1936            &crate::workdir_fs::WorkdirRoot::open(dir.path()).unwrap(),
1937            &policy,
1938            &session,
1939        )
1940        .expect("the session's server command runs");
1941        assert_eq!(server.exit_code, 0, "stderr: {}", server.stderr);
1942
1943        let invoke = |_name: &str, _input: ToolInput| Ok(tool_result(ToolStatus::Passed, None));
1944        let mut ctx = test_ctx(dir.path().to_path_buf(), &invoke);
1945        ctx.capability_policy.shell_allow = policy.shell_allow.clone();
1946        let plan = write_plan(
1947            dir.path(),
1948            json!({
1949                "id": "p",
1950                "steps": [{
1951                    "id": "client",
1952                    "shell": "bash -c 'for i in {1..50}; do exec 3<>/dev/tcp/127.0.0.1/8080 \
1953                              2>/dev/null && break; sleep 0.1; done; read -r reply <&3; \
1954                              echo \"$reply\"'",
1955                }]
1956            }),
1957        );
1958
1959        let report = execute_in_session(&plan, &ctx, Arc::clone(&session.session_netns));
1960        println!(
1961            "plan completed: {}; step {} {:?} output {:?} error {:?}; session namespaces \
1962             created: {}",
1963            report.completed,
1964            report.results[0].step_id,
1965            report.results[0].status,
1966            report.results[0].output.as_deref().map(str::trim),
1967            report.results[0].error,
1968            session.session_netns.creations()
1969        );
1970        assert!(report.completed, "{:?}", report.results);
1971        assert_eq!(report.results[0].status, StepStatus::Success);
1972        assert_eq!(
1973            report.results[0].output.as_deref().map(str::trim),
1974            Some("pong")
1975        );
1976        assert_eq!(session.session_netns.creations(), 1);
1977
1978        // The plan did not end the session's namespace: the session's next command still runs.
1979        let after = execute_shell(
1980            "bash",
1981            &["-c", "true"],
1982            &[],
1983            &crate::workdir_fs::WorkdirRoot::open(dir.path()).unwrap(),
1984            &policy,
1985            &session,
1986        )
1987        .expect("the session outlives the plan it submitted");
1988        assert_eq!(after.exit_code, 0);
1989        session.end_session();
1990    }
1991
1992    /// A `shell` step is put to the same decision point, and a refused one starts no subprocess.
1993    #[test]
1994    fn a_refused_shell_step_starts_no_subprocess() {
1995        if crate::cgroup::skip_without_host_support(
1996            "plan::tests::a_refused_shell_step_starts_no_subprocess",
1997        ) {
1998            return;
1999        }
2000        let dir = tempdir().unwrap();
2001        let invoke = |_name: &str, _input: ToolInput| Ok(tool_result(ToolStatus::Passed, None));
2002        let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
2003        let recorded = Arc::clone(&seen);
2004        let gate = move |call: &PlannedCall<'_>| match call {
2005            PlannedCall::Shell { binary, argv, .. } => {
2006                recorded
2007                    .lock()
2008                    .unwrap()
2009                    .push((binary.to_string(), argv.to_vec()));
2010                Some("refused by the manifest".to_string())
2011            }
2012            _ => None,
2013        };
2014        let marker = dir.path().join("written-by-a-refused-step");
2015        let plan = write_plan(
2016            dir.path(),
2017            json!({
2018                "id": "p",
2019                "steps": [{
2020                    "id": "w",
2021                    "shell": format!("bash -c 'printf x > {}'", marker.display()),
2022                }]
2023            }),
2024        );
2025
2026        let report = execute(
2027            &plan,
2028            &test_ctx_gated(dir.path().to_path_buf(), &invoke, &gate),
2029        );
2030        assert!(!report.completed);
2031        assert_eq!(report.failed_step.as_deref(), Some("w"));
2032        assert_eq!(
2033            report.results[0].error.as_deref(),
2034            Some("refused by the manifest")
2035        );
2036        assert!(!marker.exists(), "a refused step must not have run");
2037        // The gate is shown the argv the executable would have received, not the raw command.
2038        let seen = seen.lock().unwrap();
2039        assert_eq!(seen.len(), 1);
2040        assert_eq!(seen[0].0, "bash");
2041        assert_eq!(seen[0].1[0], "-c");
2042    }
2043
2044    #[test]
2045    fn dependent_step_receives_upstream_output() {
2046        if crate::cgroup::skip_without_host_support(
2047            "plan::tests::dependent_step_receives_upstream_output",
2048        ) {
2049            return;
2050        }
2051        let dir = tempdir().unwrap();
2052        let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
2053        let seen_tool = Arc::clone(&seen);
2054        let invoke = move |_name: &str, input: ToolInput| {
2055            seen_tool
2056                .lock()
2057                .unwrap()
2058                .push(input.data.unwrap_or_default());
2059            Ok(tool_result(
2060                ToolStatus::Passed,
2061                Some("upstream".to_string()),
2062            ))
2063        };
2064        let plan = write_plan(
2065            dir.path(),
2066            json!({
2067                "id": "p",
2068                "steps": [
2069                    {"id": "a", "tool": "ok"},
2070                    {"id": "b", "tool": "ok", "depends_on": ["a"], "input": {"value": "$a.output"}}
2071                ]
2072            }),
2073        );
2074
2075        let report = execute(&plan, &test_ctx(dir.path().to_path_buf(), &invoke));
2076        assert!(report.completed);
2077        assert_eq!(report.results.len(), 2);
2078        assert!(seen.lock().unwrap()[1].contains("upstream"));
2079    }
2080
2081    #[test]
2082    fn parallel_tool_steps_run_concurrently() {
2083        if crate::cgroup::skip_without_host_support(
2084            "plan::tests::parallel_tool_steps_run_concurrently",
2085        ) {
2086            return;
2087        }
2088        // How long a step waits for its sibling to be in flight. Run sequentially, the first
2089        // step waits this out alone; run concurrently, the steps meet as soon as both start.
2090        const RENDEZVOUS_GUARD: Duration = Duration::from_secs(30);
2091
2092        /// `met` latches once both steps are in flight together, so the first step to leave the
2093        /// rendezvous cannot strand the other.
2094        #[derive(Default)]
2095        struct Rendezvous {
2096            in_flight: usize,
2097            met: bool,
2098        }
2099
2100        let dir = tempdir().unwrap();
2101        let rendezvous = Arc::new((Mutex::new(Rendezvous::default()), Condvar::new()));
2102        let met_sibling = Arc::new(AtomicUsize::new(0));
2103        let invoke_rendezvous = Arc::clone(&rendezvous);
2104        let invoke_met = Arc::clone(&met_sibling);
2105        let invoke = move |_name: &str, _input: ToolInput| {
2106            let (state, changed) = &*invoke_rendezvous;
2107            let mut state = state.lock().unwrap();
2108            state.in_flight += 1;
2109            if state.in_flight == 2 {
2110                state.met = true;
2111            }
2112            changed.notify_all();
2113            let (mut state, _) = changed
2114                .wait_timeout_while(state, RENDEZVOUS_GUARD, |state| !state.met)
2115                .unwrap();
2116            if state.met {
2117                invoke_met.fetch_add(1, Ordering::SeqCst);
2118            }
2119            state.in_flight -= 1;
2120            Ok(tool_result(ToolStatus::Passed, Some("ok".to_string())))
2121        };
2122        let plan = write_plan(
2123            dir.path(),
2124            json!({
2125                "id": "p",
2126                "steps": [
2127                    {"id": "a", "tool": "slow"},
2128                    {"id": "b", "tool": "slow"}
2129                ]
2130            }),
2131        );
2132
2133        let report = execute(&plan, &test_ctx(dir.path().to_path_buf(), &invoke));
2134        assert!(report.completed);
2135        assert_eq!(
2136            met_sibling.load(Ordering::SeqCst),
2137            2,
2138            "the two independent steps never ran at the same time"
2139        );
2140    }
2141
2142    #[test]
2143    fn on_error_and_retries_are_applied() {
2144        if crate::cgroup::skip_without_host_support("plan::tests::on_error_and_retries_are_applied")
2145        {
2146            return;
2147        }
2148        let dir = tempdir().unwrap();
2149        let calls = Arc::new(AtomicUsize::new(0));
2150        let invoke_calls = Arc::clone(&calls);
2151        let invoke = move |name: &str, _input: ToolInput| {
2152            invoke_calls.fetch_add(1, Ordering::SeqCst);
2153            if name == "a" {
2154                Ok(tool_result(ToolStatus::Failed, None))
2155            } else {
2156                Ok(tool_result(ToolStatus::Passed, Some("ok".to_string())))
2157            }
2158        };
2159        let plan = write_plan(
2160            dir.path(),
2161            json!({
2162                "id": "p",
2163                "steps": [
2164                    {"id": "a", "tool": "a", "retries": 2, "on_error": "continue"},
2165                    {"id": "b", "tool": "b", "depends_on": ["a"]}
2166                ]
2167            }),
2168        );
2169
2170        let report = execute(&plan, &test_ctx(dir.path().to_path_buf(), &invoke));
2171        assert!(report.completed);
2172        assert_eq!(calls.load(Ordering::SeqCst), 4);
2173    }
2174
2175    #[test]
2176    fn condition_false_skips_step() {
2177        if crate::cgroup::skip_without_host_support("plan::tests::condition_false_skips_step") {
2178            return;
2179        }
2180        let dir = tempdir().unwrap();
2181        let invoke = |_name: &str, _input: ToolInput| {
2182            Ok(tool_result(ToolStatus::Passed, Some("ok".to_string())))
2183        };
2184        let plan = write_plan(
2185            dir.path(),
2186            json!({
2187                "id": "p",
2188                "steps": [
2189                    {"id": "a", "tool": "ok"},
2190                    {"id": "b", "tool": "ok", "depends_on": ["a"], "if": "$a.status == 'failed'"}
2191                ]
2192            }),
2193        );
2194
2195        let report = execute(&plan, &test_ctx(dir.path().to_path_buf(), &invoke));
2196        assert!(report.completed);
2197        assert_eq!(report.results[1].status, StepStatus::Skipped);
2198    }
2199
2200    /// A task whose cancel flag a case raises by hand. It holds no delegation set: these cases
2201    /// launch no sub-capsule.
2202    #[derive(Default)]
2203    struct FlaggedTask(std::sync::atomic::AtomicBool);
2204
2205    impl PlanTask for FlaggedTask {
2206        fn is_canceled(&self) -> bool {
2207            self.0.load(Ordering::SeqCst)
2208        }
2209        fn delegation_started(&self, _launch: &DelegationLaunch) {}
2210        fn delegation_settled(&self, _delegation_id: &str) -> bool {
2211            true
2212        }
2213    }
2214
2215    /// A step that fails after its task is cancelled is not tried again, and its `on_error:
2216    /// fail` names no failed step: the run stopped because the task did, whatever the step said.
2217    #[test]
2218    fn a_cancelled_task_tries_no_step_again() {
2219        let dir = tempdir().unwrap();
2220        let task = FlaggedTask::default();
2221        let calls = AtomicUsize::new(0);
2222        let invoke = |_name: &str, _input: ToolInput| {
2223            calls.fetch_add(1, Ordering::SeqCst);
2224            task.0.store(true, Ordering::SeqCst);
2225            Ok(tool_result(ToolStatus::Failed, None))
2226        };
2227        let plan = write_plan(
2228            dir.path(),
2229            json!({
2230                "id": "p",
2231                "steps": [
2232                    {"id": "a", "tool": "a", "retries": 2, "on_error": "fail"},
2233                    {"id": "b", "tool": "b", "depends_on": ["a"]}
2234                ]
2235            }),
2236        );
2237
2238        let ctx = SchedulerContext {
2239            task: Some(&task),
2240            ..test_ctx_unbounded(dir.path().to_path_buf(), &invoke, &UNGATED)
2241        };
2242        let report = execute(&plan, &ctx);
2243
2244        assert_eq!(calls.load(Ordering::SeqCst), 1, "{report:?}");
2245        assert!(report.canceled, "{report:?}");
2246        assert!(!report.completed);
2247        assert_eq!(report.failed_step, None);
2248        assert_eq!(report.results[0].status, StepStatus::Failed);
2249        assert_eq!(report.results[1].status, StepStatus::Skipped);
2250        assert_eq!(
2251            report.results[1].error.as_deref(),
2252            Some(CANCELED_BEFORE_STEP_ERROR)
2253        );
2254    }
2255
2256    /// A task cancelled before the first wave dispatches nothing at all.
2257    #[test]
2258    fn a_task_cancelled_before_the_plan_ran_dispatches_nothing() {
2259        let dir = tempdir().unwrap();
2260        let task = FlaggedTask::default();
2261        task.0.store(true, Ordering::SeqCst);
2262        let calls = AtomicUsize::new(0);
2263        let invoke = |_name: &str, _input: ToolInput| {
2264            calls.fetch_add(1, Ordering::SeqCst);
2265            Ok(tool_result(ToolStatus::Passed, Some("ran".to_string())))
2266        };
2267        let plan = write_plan(
2268            dir.path(),
2269            json!({"id": "p", "steps": [{"id": "a", "tool": "a"}, {"id": "b", "tool": "b"}]}),
2270        );
2271
2272        let ctx = SchedulerContext {
2273            task: Some(&task),
2274            ..test_ctx_unbounded(dir.path().to_path_buf(), &invoke, &UNGATED)
2275        };
2276        let report = execute(&plan, &ctx);
2277
2278        assert_eq!(calls.load(Ordering::SeqCst), 0);
2279        assert!(report.canceled && !report.completed, "{report:?}");
2280        assert!(report
2281            .results
2282            .iter()
2283            .all(|result| result.status == StepStatus::Skipped));
2284    }
2285
2286    /// A run whose task is never cancelled reports exactly what a run with no task does.
2287    #[test]
2288    fn a_task_nobody_cancels_changes_nothing() {
2289        let dir = tempdir().unwrap();
2290        let task = FlaggedTask::default();
2291        let invoke = |_name: &str, _input: ToolInput| {
2292            Ok(tool_result(ToolStatus::Passed, Some("ran".to_string())))
2293        };
2294        let plan = write_plan(
2295            dir.path(),
2296            json!({"id": "p", "steps": [{"id": "a", "tool": "a"}, {"id": "b", "tool": "b"}]}),
2297        );
2298
2299        let without = execute(
2300            &plan,
2301            &test_ctx_unbounded(dir.path().to_path_buf(), &invoke, &UNGATED),
2302        );
2303        let with = execute(
2304            &plan,
2305            &SchedulerContext {
2306                task: Some(&task),
2307                ..test_ctx_unbounded(dir.path().to_path_buf(), &invoke, &UNGATED)
2308            },
2309        );
2310        assert_eq!(with, without);
2311        assert!(with.completed && !with.canceled);
2312    }
2313}