1use std::{
2 collections::{HashMap, HashSet},
3 path::Path,
4 sync::Arc,
5 thread,
6 time::Instant,
7};
8
9use serde::Deserialize;
10use serde_json::Value;
11
12use murmur_artifact::DEFAULT_EXPORT_MAX_BYTES;
13
14use crate::{
15 bindings::host::murmur::tool::run::{Status as ToolStatus, ToolInput, ToolResult},
16 delegation_plane::{
17 DelegationLaunch, DelegationOrigin, DelegationPlane, DelegationRequest, DelegationStatus,
18 },
19 errors::RuntimeError,
20 network_namespace::SessionNetns,
21 resource_plane::{self, SymlinkPolicy},
22 sandbox,
23 shell::{execute_shell, split_shell_words},
24 spawn_credential::SpawnCredential,
25 trace::{PlanEndRecord, PlanStepRecord, PlanStepShape, PlanTraceAppender},
26 types::CapabilityPolicy,
27 workdir_fs::WorkdirRoot,
28};
29
30pub mod condition;
31
32#[derive(Debug, Deserialize)]
33struct PlanFile {
34 id: String,
35 steps: Vec<StepDef>,
36}
37
38#[derive(Debug, Clone, Deserialize)]
39struct StepDef {
40 id: String,
41 tool: Option<String>,
42 shell: Option<String>,
43 capsule: Option<String>,
44 #[serde(default)]
45 input: Value,
46 #[serde(default)]
47 depends_on: Vec<String>,
48 #[serde(rename = "if")]
49 condition: Option<String>,
50 #[serde(default = "default_on_error")]
51 on_error: String,
52 #[serde(default)]
53 retries: u32,
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq)]
57enum StepKind {
58 Tool,
59 Shell,
60 Spawn,
61}
62
63impl StepKind {
64 fn as_str(self) -> &'static str {
67 match self {
68 StepKind::Tool => "tool",
69 StepKind::Shell => "shell",
70 StepKind::Spawn => "capsule",
71 }
72 }
73}
74
75#[derive(Debug, Clone, PartialEq, Eq)]
76pub enum StepStatus {
77 Success,
78 Failed,
79 Skipped,
80}
81
82#[derive(Debug, Clone, PartialEq, Eq)]
83pub struct StepResult {
84 pub step_id: String,
85 pub status: StepStatus,
86 pub output: Option<String>,
87 pub error: Option<String>,
88}
89
90#[derive(Debug, Clone, PartialEq, Eq)]
91pub struct ExecutionReport {
92 pub results: Vec<StepResult>,
93 pub completed: bool,
94 pub failed_step: Option<String>,
96 pub canceled: bool,
99}
100
101pub const CANCELED_STEP_ERROR: &str =
104 "the task running this plan was cancelled; this step's sub-capsule was ended with it";
105
106pub const CANCELED_BEFORE_STEP_ERROR: &str =
108 "the task running this plan was cancelled before this step ran";
109
110pub const PLAN_CANCELED_REASON: &str = "the task running this plan was cancelled";
112
113pub trait PlanTask: Sync {
121 fn is_canceled(&self) -> bool;
123 fn delegation_started(&self, launch: &DelegationLaunch);
126 #[must_use]
129 fn delegation_settled(&self, delegation_id: &str) -> bool;
130}
131
132pub struct SchedulerContext<'a> {
133 pub accessible_workdir: WorkdirRoot,
136 pub runtime_dir: crate::runtime_dir::RuntimeDirRoots,
141 pub capability_policy: CapabilityPolicy,
142 pub installed_tools: HashSet<String>,
143 pub capsule_versions: HashMap<String, String>,
144 pub current_session_id: Option<String>,
145 pub current_context_id: Option<String>,
152 pub registry: std::sync::Arc<dyn murmur_artifact::Registry>,
156 pub spawn_credential: Option<SpawnCredential>,
163 pub formation_id: Option<crate::formation::FormationId>,
166 pub deadline: Option<crate::session_status::SessionDeadline>,
169 pub trace: Option<&'a PlanTraceAppender>,
180 pub gate_step: &'a (dyn Fn(&PlannedCall<'_>) -> Option<String> + Sync),
192 pub invoke_tool: &'a (dyn Fn(&str, ToolInput) -> Result<ToolResult, String> + Sync),
193 pub task: Option<&'a dyn PlanTask>,
200}
201
202pub enum PlannedCall<'a> {
208 Tool {
209 name: &'a str,
210 input_json: &'a str,
212 },
213 Shell {
214 binary: &'a str,
216 command: &'a str,
218 argv: &'a [String],
221 },
222}
223
224pub fn execute(plan_path: &Path, ctx: &SchedulerContext<'_>) -> ExecutionReport {
227 execute_with_netns(plan_path, ctx, None)
228}
229
230pub(crate) fn execute_in_session(
234 plan_path: &Path,
235 ctx: &SchedulerContext<'_>,
236 session_netns: Arc<SessionNetns>,
237) -> ExecutionReport {
238 execute_with_netns(plan_path, ctx, Some(session_netns))
239}
240
241fn execute_with_netns(
242 plan_path: &Path,
243 ctx: &SchedulerContext<'_>,
244 session_netns: Option<Arc<SessionNetns>>,
245) -> ExecutionReport {
246 let started = Instant::now();
247 match execute_inner(plan_path, ctx, started, session_netns) {
248 Ok(report) => report,
249 Err((failed_step, error)) => {
252 if let Some(appender) = ctx.trace {
253 appender.write_plan_end(
254 None,
255 PlanEndRecord {
256 plan_id: String::new(),
257 outcome: "failed",
258 failed_step: Some(failed_step.clone()),
259 steps_total: 0,
260 steps_succeeded: 0,
261 steps_failed: 0,
262 steps_skipped: 0,
263 duration_ms: elapsed_ms(started),
264 reason: Some(error.clone()),
265 },
266 );
267 }
268 ExecutionReport {
269 results: vec![StepResult {
270 step_id: failed_step.clone(),
271 status: StepStatus::Failed,
272 output: None,
273 error: Some(error),
274 }],
275 completed: false,
276 failed_step: Some(failed_step),
277 canceled: false,
278 }
279 }
280 }
281}
282
283fn execute_inner(
284 plan_path: &Path,
285 ctx: &SchedulerContext<'_>,
286 started: Instant,
287 session_netns: Option<Arc<SessionNetns>>,
288) -> Result<ExecutionReport, (String, String)> {
289 let runtime_root = &ctx.runtime_dir.runtime;
292 let relative = plan_path
293 .strip_prefix(runtime_root.path())
294 .unwrap_or(plan_path);
295 let raw = match runtime_root.read_to_string(relative) {
296 Ok(Some(raw)) => raw,
297 Ok(None) => {
298 return Err((
299 "plan".to_string(),
300 format!(
301 "failed to read plan file {}: it does not exist",
302 plan_path.display()
303 ),
304 ))
305 }
306 Err(error) => {
307 return Err((
308 "plan".to_string(),
309 format!("failed to read plan file: {error}"),
310 ))
311 }
312 };
313 let plan: PlanFile = serde_json::from_str(&raw).map_err(|error| {
314 (
315 "plan".to_string(),
316 format!("failed to parse plan JSON: {error}"),
317 )
318 })?;
319
320 let plan_node = ctx
324 .trace
325 .map(|appender| appender.write_plan_start(&plan.id, plan_shape(&plan)));
326
327 if let Err((step_id, error)) = validate_plan(&plan, ctx) {
328 trace_plan_end(
329 ctx,
330 plan_node.as_deref(),
331 PlanEndRecord {
332 plan_id: plan.id.clone(),
333 outcome: "failed",
334 failed_step: Some(step_id.clone()),
335 steps_total: plan.steps.len(),
336 steps_succeeded: 0,
337 steps_failed: 0,
338 steps_skipped: 0,
339 duration_ms: elapsed_ms(started),
340 reason: Some(error.clone()),
341 },
342 );
343 return Ok(ExecutionReport {
344 results: vec![StepResult {
345 step_id: step_id.clone(),
346 status: StepStatus::Failed,
347 output: None,
348 error: Some(error),
349 }],
350 completed: false,
351 failed_step: Some(step_id.clone()),
352 canceled: false,
353 });
354 }
355
356 let cgroup_scope = match crate::cgroup::prepare_policy_scope(
362 &ctx.capability_policy,
363 false,
364 &plan.id,
365 ctx.accessible_workdir.path(),
366 ) {
367 Ok(prepared) => prepared.scope,
373 Err(reason) => {
374 let error =
375 crate::errors::RuntimeError::CgroupDelegationUnavailable { reason }.to_string();
376 return Ok(refused_before_any_step(
377 &plan,
378 ctx,
379 plan_node.as_deref(),
380 started,
381 error,
382 ));
383 }
384 };
385 let workdir_guard =
386 crate::resources::workdir_guard_max_bytes(&ctx.capability_policy).map(|max_bytes| {
387 crate::resources::WorkdirGuard::spawn(
388 &ctx.accessible_workdir,
389 ctx.runtime_dir.temp.as_ref(),
390 ctx.runtime_dir.home.as_ref(),
391 ctx.runtime_dir.records.as_ref(),
392 max_bytes,
393 )
394 });
395
396 let probe = sandbox::HostProbe::probe();
400 let mount_namespace_usable = probe.mount_namespace_usable();
401 let decision = crate::containment::ContainmentDecision::decide(
404 probe.tier(),
405 ctx.capability_policy.containment_floor,
406 ctx.capability_policy.workdir_exec_allowed,
407 false,
408 );
409 let shell_enforcement =
410 match sandbox::ShellEnforcement::resolve(&ctx.capability_policy, &decision, probe) {
411 Ok(enforcement) => {
412 let temp_mapping = crate::temp_dir::temp_mapping(
413 enforcement.tier,
414 mount_namespace_usable,
415 ctx.accessible_workdir.path(),
416 );
417 enforcement
418 .with_host_bounding(cgroup_scope, workdir_guard)
419 .with_runtime_dir(ctx.runtime_dir.clone(), None)
420 .with_temp_mapping(temp_mapping)
421 }
422 Err(error) => {
423 let error =
424 format!("failed to resolve shell subprocess sandbox enforcement: {error}");
425 return Ok(refused_before_any_step(
426 &plan,
427 ctx,
428 plan_node.as_deref(),
429 started,
430 error,
431 ));
432 }
433 };
434 let owns_netns = session_netns.is_none();
435 let shell_enforcement = match session_netns {
436 Some(session_netns) => shell_enforcement.with_session_netns(session_netns),
437 None => shell_enforcement,
438 };
439 let _plan_end = PlanEnd {
441 enforcement: &shell_enforcement,
442 owns_netns,
443 };
444
445 let mut results = HashMap::<String, StepResult>::new();
446 let mut result_order = Vec::<String>::new();
447 let mut failed_step = None;
448 let mut canceled = false;
449
450 loop {
451 if canceled {
452 for step in &plan.steps {
455 if results.contains_key(&step.id) {
456 continue;
457 }
458 trace_undispatched(
459 ctx,
460 plan_node.as_deref(),
461 &plan.id,
462 step,
463 "skipped",
464 Some(CANCELED_BEFORE_STEP_ERROR.to_string()),
465 );
466 insert_result(
467 &mut results,
468 &mut result_order,
469 StepResult {
470 step_id: step.id.clone(),
471 status: StepStatus::Skipped,
472 output: None,
473 error: Some(CANCELED_BEFORE_STEP_ERROR.to_string()),
474 },
475 );
476 }
477 break;
478 }
479 if results.len() == plan.steps.len() || failed_step.is_some() {
480 break;
481 }
482 if task_canceled(ctx) {
483 canceled = true;
484 continue;
485 }
486
487 let mut ready = Vec::new();
488 let mut progressed = false;
489
490 for step in &plan.steps {
491 if results.contains_key(&step.id) {
492 continue;
493 }
494 if !step.depends_on.iter().all(|dep| results.contains_key(dep)) {
495 continue;
496 }
497
498 match condition::evaluate(step.condition.as_deref(), &results) {
499 Ok(true) => ready.push(step.clone()),
500 Ok(false) => {
501 trace_undispatched(ctx, plan_node.as_deref(), &plan.id, step, "skipped", None);
504 insert_result(
505 &mut results,
506 &mut result_order,
507 StepResult {
508 step_id: step.id.clone(),
509 status: StepStatus::Skipped,
510 output: None,
511 error: None,
512 },
513 );
514 progressed = true;
515 }
516 Err(error) => {
517 trace_undispatched(
518 ctx,
519 plan_node.as_deref(),
520 &plan.id,
521 step,
522 "failed",
523 Some(error.clone()),
524 );
525 insert_result(
526 &mut results,
527 &mut result_order,
528 StepResult {
529 step_id: step.id.clone(),
530 status: StepStatus::Failed,
531 output: None,
532 error: Some(error),
533 },
534 );
535 failed_step = Some(step.id.clone());
536 progressed = true;
537 break;
538 }
539 }
540 }
541
542 if failed_step.is_some() {
543 break;
544 }
545
546 if ready.is_empty() {
547 if progressed {
548 continue;
549 }
550 const BLOCKED: &str = "plan is blocked by a dependency cycle or missing dependency";
551 let blocked_step = plan
552 .steps
553 .iter()
554 .find(|step| !results.contains_key(&step.id));
555 let blocked = blocked_step
556 .map(|step| step.id.clone())
557 .unwrap_or_else(|| "plan".to_string());
558 if let Some(step) = blocked_step {
559 trace_undispatched(
560 ctx,
561 plan_node.as_deref(),
562 &plan.id,
563 step,
564 "failed",
565 Some(BLOCKED.to_string()),
566 );
567 }
568 insert_result(
569 &mut results,
570 &mut result_order,
571 StepResult {
572 step_id: blocked.clone(),
573 status: StepStatus::Failed,
574 output: None,
575 error: Some(BLOCKED.to_string()),
576 },
577 );
578 failed_step = Some(blocked);
579 break;
580 }
581
582 for step in &ready {
583 trace_step_start(ctx, plan_node.as_deref(), &plan.id, step);
584 }
585
586 let mut completed = Vec::new();
587 thread::scope(|scope| {
588 let mut handles = Vec::new();
589 for step in ready {
590 let snapshot = results.clone();
591 let shell_enforcement = &shell_enforcement;
592 handles.push(scope.spawn(move || {
593 execute_step_with_retries(&step, ctx, &snapshot, shell_enforcement)
594 }));
595 }
596 for handle in handles {
597 completed.push(handle.join().unwrap_or_else(|_| {
598 StepOutcome::undispatched(failed("unknown", "step dispatch thread panicked"))
599 }));
600 }
601 });
602 canceled = task_canceled(ctx);
606
607 for outcome in completed {
608 let Some(step) = plan
609 .steps
610 .iter()
611 .find(|step| step.id == outcome.result.step_id)
612 else {
613 trace_step(
615 ctx,
616 plan_node.as_deref(),
617 PlanStepRecord {
618 plan_id: plan.id.clone(),
619 step_id: outcome.result.step_id.clone(),
620 kind: "unknown",
621 status: outcome.result.status.as_str(),
622 attempts: outcome.attempts,
623 duration_ms: outcome.duration_ms,
624 error: outcome.result.error.clone(),
625 input: None,
626 state_effect: None,
627 resource_id: None,
628 },
629 );
630 insert_result(&mut results, &mut result_order, outcome.result);
631 if !canceled {
632 failed_step = Some("unknown".to_string());
633 break;
634 }
635 continue;
636 };
637
638 let kind = step_kind_name(step);
639 let StepOutcome {
640 result,
641 attempts,
642 duration_ms,
643 input,
644 state_effect,
645 resource_id,
646 } = outcome;
647
648 let (settled, stops) = if result.status == StepStatus::Failed && !canceled {
651 match step.on_error.as_str() {
652 "fail" => (result, true),
653 "skip" => (
654 StepResult {
655 step_id: result.step_id,
656 status: StepStatus::Skipped,
657 output: None,
658 error: result.error,
659 },
660 false,
661 ),
662 "continue" => (result, false),
663 policy => (
664 StepResult {
665 error: Some(format!("invalid on_error policy '{policy}'")),
666 ..result
667 },
668 true,
669 ),
670 }
671 } else {
672 (result, false)
673 };
674
675 trace_step(
676 ctx,
677 plan_node.as_deref(),
678 PlanStepRecord {
679 plan_id: plan.id.clone(),
680 step_id: settled.step_id.clone(),
681 kind,
682 status: settled.status.as_str(),
683 attempts,
684 duration_ms,
685 error: settled.error.clone(),
686 input,
687 state_effect,
688 resource_id,
689 },
690 );
691
692 let id = settled.step_id.clone();
693 insert_result(&mut results, &mut result_order, settled);
694 if stops {
695 failed_step = Some(id);
696 break;
697 }
698 }
699 }
700
701 let ordered_results = result_order
702 .into_iter()
703 .filter_map(|id| results.remove(&id))
704 .collect::<Vec<_>>();
705
706 if canceled {
707 failed_step = None;
708 }
709 trace_plan_end(
710 ctx,
711 plan_node.as_deref(),
712 PlanEndRecord {
713 plan_id: plan.id.clone(),
714 outcome: if canceled {
715 "canceled"
716 } else if failed_step.is_none() {
717 "completed"
718 } else {
719 "failed"
720 },
721 failed_step: failed_step.clone(),
722 steps_total: plan.steps.len(),
723 steps_succeeded: count_status(&ordered_results, StepStatus::Success),
724 steps_failed: count_status(&ordered_results, StepStatus::Failed),
725 steps_skipped: count_status(&ordered_results, StepStatus::Skipped),
726 duration_ms: elapsed_ms(started),
727 reason: canceled.then(|| PLAN_CANCELED_REASON.to_string()),
728 },
729 );
730
731 Ok(ExecutionReport {
732 results: ordered_results,
733 completed: !canceled && failed_step.is_none(),
734 failed_step,
735 canceled,
736 })
737}
738
739struct PlanEnd<'a> {
745 enforcement: &'a sandbox::ShellEnforcement,
746 owns_netns: bool,
747}
748
749impl Drop for PlanEnd<'_> {
750 fn drop(&mut self) {
751 self.enforcement.kill_session_processes();
752 if self.owns_netns {
753 self.enforcement.session_netns.end();
754 }
755 }
756}
757
758fn step_kind_name(step: &StepDef) -> &'static str {
764 infer_kind(step).map(StepKind::as_str).unwrap_or("unknown")
765}
766
767fn plan_shape(plan: &PlanFile) -> Vec<PlanStepShape> {
769 plan.steps
770 .iter()
771 .map(|step| PlanStepShape {
772 step_id: step.id.clone(),
773 kind: step_kind_name(step),
774 depends_on: step.depends_on.clone(),
775 has_condition: step.condition.is_some(),
776 })
777 .collect()
778}
779
780fn trace_step_start(
781 ctx: &SchedulerContext<'_>,
782 plan_node: Option<&str>,
783 plan_id: &str,
784 step: &StepDef,
785) {
786 if let (Some(appender), Some(node)) = (ctx.trace, plan_node) {
787 appender.write_plan_step_start(
788 node,
789 plan_id,
790 &step.id,
791 step_kind_name(step),
792 step.depends_on.clone(),
793 );
794 }
795}
796
797fn trace_step(ctx: &SchedulerContext<'_>, plan_node: Option<&str>, record: PlanStepRecord) {
798 if let (Some(appender), Some(node)) = (ctx.trace, plan_node) {
799 appender.write_plan_step(node, record);
800 }
801}
802
803fn trace_undispatched(
807 ctx: &SchedulerContext<'_>,
808 plan_node: Option<&str>,
809 plan_id: &str,
810 step: &StepDef,
811 status: &'static str,
812 error: Option<String>,
813) {
814 trace_step(
815 ctx,
816 plan_node,
817 PlanStepRecord {
818 plan_id: plan_id.to_string(),
819 step_id: step.id.clone(),
820 kind: step_kind_name(step),
821 status,
822 attempts: 0,
823 duration_ms: 0,
824 error,
825 input: None,
826 state_effect: None,
827 resource_id: None,
828 },
829 );
830}
831
832fn trace_plan_end(ctx: &SchedulerContext<'_>, plan_node: Option<&str>, record: PlanEndRecord) {
833 if let Some(appender) = ctx.trace {
834 appender.write_plan_end(plan_node, record);
835 }
836}
837
838fn refused_before_any_step(
842 plan: &PlanFile,
843 ctx: &SchedulerContext<'_>,
844 plan_node: Option<&str>,
845 started: Instant,
846 error: String,
847) -> ExecutionReport {
848 trace_plan_end(
849 ctx,
850 plan_node,
851 PlanEndRecord {
852 plan_id: plan.id.clone(),
853 outcome: "failed",
854 failed_step: Some("plan".to_string()),
855 steps_total: plan.steps.len(),
856 steps_succeeded: 0,
857 steps_failed: 0,
858 steps_skipped: 0,
859 duration_ms: elapsed_ms(started),
860 reason: Some(error.clone()),
861 },
862 );
863 ExecutionReport {
864 results: vec![StepResult {
865 step_id: "plan".to_string(),
866 status: StepStatus::Failed,
867 output: None,
868 error: Some(error),
869 }],
870 completed: false,
871 failed_step: Some("plan".to_string()),
872 canceled: false,
873 }
874}
875
876fn task_canceled(ctx: &SchedulerContext<'_>) -> bool {
878 ctx.task.is_some_and(PlanTask::is_canceled)
879}
880
881fn count_status(results: &[StepResult], status: StepStatus) -> usize {
882 results
883 .iter()
884 .filter(|result| result.status == status)
885 .count()
886}
887
888fn elapsed_ms(started: Instant) -> u64 {
889 started.elapsed().as_millis() as u64
890}
891
892fn validate_plan(plan: &PlanFile, ctx: &SchedulerContext<'_>) -> Result<(), (String, String)> {
893 let mut ids = HashSet::new();
894 for step in &plan.steps {
895 if !ids.insert(step.id.clone()) {
896 return Err((step.id.clone(), format!("duplicate step id '{}'", step.id)));
897 }
898
899 match infer_kind(step) {
900 Ok(StepKind::Tool) => {
901 let tool = step.tool.as_ref().unwrap();
902 if tool == crate::runtime::SUBMIT_PLAN_TOOL {
906 return Err((
907 step.id.clone(),
908 format!(
909 "a plan step cannot submit another plan: '{tool}' is not callable \
910 from inside a plan"
911 ),
912 ));
913 }
914 if tool == crate::runtime::SWITCH_DRIVER_TOOL
919 || tool == crate::runtime::END_WITHOUT_ANSWER_TOOL
920 || tool == crate::runtime::PULL_ARTIFACT_TOOL
921 || tool == crate::runtime::REMOVE_ARTIFACT_TOOL
922 {
923 return Err((
924 step.id.clone(),
925 format!("'{tool}' is not callable from inside a plan"),
926 ));
927 }
928 if !ctx.installed_tools.contains(tool) {
929 return Err((step.id.clone(), format!("tool '{tool}' is not installed")));
930 }
931 }
932 Ok(StepKind::Shell) => {
933 let Some(binary) = shell_binary(step.shell.as_deref().unwrap_or_default()) else {
934 return Err((step.id.clone(), "shell command is empty".to_string()));
935 };
936 if !ctx.capability_policy.shell_allow.contains(&binary) {
937 return Err((
938 step.id.clone(),
939 format!("binary '{binary}' is not in capabilities.shell.allow"),
940 ));
941 }
942 }
943 Ok(StepKind::Spawn) => {
944 if let Err(error) = capsule_task_text(&step.input) {
951 return Err((step.id.clone(), error));
952 }
953 }
954 Err(error) => return Err((step.id.clone(), error)),
955 }
956
957 if !matches!(step.on_error.as_str(), "fail" | "skip" | "continue") {
958 return Err((
959 step.id.clone(),
960 format!("invalid on_error policy '{}'", step.on_error),
961 ));
962 }
963 }
964
965 for step in &plan.steps {
966 for dep in &step.depends_on {
967 if !ids.contains(dep) {
968 return Err((step.id.clone(), format!("unknown dependency '{dep}'")));
969 }
970 }
971 validate_references(step, &ids)?;
972 }
973
974 Ok(())
975}
976
977fn validate_references(step: &StepDef, ids: &HashSet<String>) -> Result<(), (String, String)> {
978 let mut references = Vec::new();
979 collect_references(&step.input, &mut references);
980 if let Some(condition) = &step.condition {
981 for token in condition.split_whitespace() {
982 if token.starts_with('$') {
983 references.push(token.trim_matches(|c| c == '(' || c == ')').to_string());
984 }
985 }
986 }
987
988 for reference in references {
989 if let Some((step_id, field)) = parse_reference(&reference) {
990 if !ids.contains(step_id) {
991 return Err((step.id.clone(), format!("unknown reference '{reference}'")));
992 }
993 if field != "output" && field != "status" {
994 return Err((
995 step.id.clone(),
996 format!("unknown reference field '{field}'"),
997 ));
998 }
999 }
1000 }
1001
1002 Ok(())
1003}
1004
1005fn collect_references(value: &Value, references: &mut Vec<String>) {
1006 match value {
1007 Value::String(value) if value.starts_with('$') => references.push(value.clone()),
1008 Value::Array(values) => values
1009 .iter()
1010 .for_each(|value| collect_references(value, references)),
1011 Value::Object(map) => map
1012 .values()
1013 .for_each(|value| collect_references(value, references)),
1014 _ => {}
1015 }
1016}
1017
1018struct StepOutcome {
1024 result: StepResult,
1025 attempts: u32,
1026 duration_ms: u64,
1027 input: Option<Value>,
1030 state_effect: Option<String>,
1033 resource_id: Option<String>,
1034}
1035
1036impl StepOutcome {
1037 fn plain(result: StepResult, started: Instant) -> Self {
1040 Self {
1041 result,
1042 attempts: 1,
1043 duration_ms: elapsed_ms(started),
1044 input: None,
1045 state_effect: None,
1046 resource_id: None,
1047 }
1048 }
1049
1050 fn undispatched(result: StepResult) -> Self {
1052 Self {
1053 result,
1054 attempts: 0,
1055 duration_ms: 0,
1056 input: None,
1057 state_effect: None,
1058 resource_id: None,
1059 }
1060 }
1061}
1062
1063fn execute_step_with_retries(
1064 step: &StepDef,
1065 ctx: &SchedulerContext<'_>,
1066 results: &HashMap<String, StepResult>,
1067 enforcement: &sandbox::ShellEnforcement,
1068) -> StepOutcome {
1069 let started = Instant::now();
1070 let attempts = step.retries.saturating_add(1);
1071 let mut last = None;
1072 for attempt in 1..=attempts {
1073 let mut outcome = execute_step_once(step, ctx, results, enforcement);
1074 outcome.attempts = attempt;
1075 outcome.duration_ms = elapsed_ms(started);
1076 if outcome.result.status != StepStatus::Failed || task_canceled(ctx) {
1078 return outcome;
1079 }
1080 last = Some(outcome);
1081 }
1082
1083 last.unwrap_or_else(|| StepOutcome::undispatched(failed(&step.id, "step was not attempted")))
1084}
1085
1086fn execute_step_once(
1087 step: &StepDef,
1088 ctx: &SchedulerContext<'_>,
1089 results: &HashMap<String, StepResult>,
1090 enforcement: &sandbox::ShellEnforcement,
1091) -> StepOutcome {
1092 let started = Instant::now();
1093 let mut input = step.input.clone();
1094 if let Err(error) = interpolate_value(&mut input, results) {
1095 return StepOutcome::plain(failed(&step.id, error), started);
1096 }
1097 let input_json = serde_json::to_string(&input).unwrap_or_else(|_| "{}".to_string());
1098
1099 match infer_kind(step) {
1100 Ok(StepKind::Tool) => {
1101 let (result, state_effect, resource_id) = dispatch_tool_step(step, ctx, input_json);
1102 StepOutcome {
1103 result,
1104 attempts: 1,
1105 duration_ms: elapsed_ms(started),
1106 input: (!input.is_null()).then_some(input),
1109 state_effect,
1110 resource_id,
1111 }
1112 }
1113 Ok(StepKind::Shell) => {
1114 StepOutcome::plain(dispatch_shell_step(step, ctx, enforcement), started)
1115 }
1116 Ok(StepKind::Spawn) => StepOutcome::plain(dispatch_capsule_step(step, ctx, input), started),
1117 Err(error) => StepOutcome::plain(failed(&step.id, error), started),
1118 }
1119}
1120
1121fn dispatch_tool_step(
1125 step: &StepDef,
1126 ctx: &SchedulerContext<'_>,
1127 input_json: String,
1128) -> (StepResult, Option<String>, Option<String>) {
1129 let name = step.tool.as_deref().unwrap_or_default();
1130 if let Some(refusal) = (ctx.gate_step)(&PlannedCall::Tool {
1131 name,
1132 input_json: &input_json,
1133 }) {
1134 return (failed(&step.id, refusal), None, None);
1135 }
1136 match (ctx.invoke_tool)(
1137 name,
1138 ToolInput {
1139 data: Some(input_json),
1140 log_path: None,
1141 },
1142 ) {
1143 Ok(result) => {
1144 let state_effect = crate::agent::extract_state_effect(&result.metadata);
1145 let resource_id = crate::agent::extract_resource_id(&result.metadata);
1146 let verdict = if matches!(result.status, ToolStatus::Passed) {
1147 match tool_step_output(
1148 name,
1149 ctx.accessible_workdir.path(),
1150 result.data,
1151 result.data_path.as_deref(),
1152 ) {
1153 Ok(output) => StepResult {
1154 step_id: step.id.clone(),
1155 status: StepStatus::Success,
1156 output,
1157 error: None,
1158 },
1159 Err(error) => failed(&step.id, error),
1160 }
1161 } else {
1162 failed(
1163 &step.id,
1164 result
1165 .summary
1166 .or(result.data)
1167 .unwrap_or_else(|| "tool step failed".to_string()),
1168 )
1169 };
1170 (verdict, state_effect, resource_id)
1171 }
1172 Err(error) => (failed(&step.id, error), None, None),
1173 }
1174}
1175
1176fn dispatch_shell_step(
1177 step: &StepDef,
1178 ctx: &SchedulerContext<'_>,
1179 enforcement: &sandbox::ShellEnforcement,
1180) -> StepResult {
1181 let command = step.shell.as_deref().unwrap_or_default();
1182 let Some((binary, args)) = parse_shell_command(command) else {
1183 return failed(&step.id, "shell command is empty");
1184 };
1185 if let Some(refusal) = (ctx.gate_step)(&PlannedCall::Shell {
1186 binary: &binary,
1187 command,
1188 argv: &args,
1189 }) {
1190 return failed(&step.id, refusal);
1191 }
1192 let arg_refs = args.iter().map(String::as_str).collect::<Vec<_>>();
1193
1194 match execute_shell(
1195 &binary,
1196 &arg_refs,
1197 &[],
1198 &ctx.accessible_workdir,
1199 &ctx.capability_policy,
1200 enforcement,
1201 ) {
1202 Ok(result) if result.exit_code == 0 => StepResult {
1203 step_id: step.id.clone(),
1204 status: StepStatus::Success,
1205 output: Some(result.stdout),
1206 error: None,
1207 },
1208 Ok(result) => match result.resource_limit_hit {
1213 Some(limit) => failed(
1214 &step.id,
1215 crate::errors::RuntimeError::ShellResourceLimitExceeded {
1216 binary: result.binary,
1217 limit,
1218 detail: if result.stderr.trim().is_empty() {
1219 format!("exit code {}", result.exit_code)
1220 } else {
1221 result.stderr
1222 },
1223 }
1224 .to_string(),
1225 ),
1226 None if result.stderr.trim().is_empty() => {
1231 failed(&step.id, format!("exit code {}", result.exit_code))
1232 }
1233 None => failed(&step.id, result.stderr),
1234 },
1235 Err(error) => failed(&step.id, error.to_string()),
1240 }
1241}
1242
1243fn snapshot_grant(
1247 ctx: &SchedulerContext<'_>,
1248 records_root: &WorkdirRoot,
1249) -> crate::delegation_plane::WorkdirSnapshotGrant {
1250 crate::delegation_plane::WorkdirSnapshotGrant::new(
1251 ctx.capability_policy.spawn_workdir_snapshot.clone(),
1252 ctx.accessible_workdir.clone(),
1253 &[&ctx.runtime_dir.runtime, records_root],
1254 )
1255}
1256
1257fn dispatch_capsule_step(step: &StepDef, ctx: &SchedulerContext<'_>, input: Value) -> StepResult {
1258 let named = step.capsule.as_deref().unwrap_or_default();
1259 let full = crate::tool_names::spawn_target(&ctx.capability_policy.spawn_allow, named);
1262 let capsule = full.as_str();
1263 let task = match capsule_task_text(&input) {
1266 Ok(task) => task,
1267 Err(error) => return failed(&step.id, error),
1268 };
1269 let roost_url = match std::env::var("MURMUR_ROOST_URL") {
1270 Ok(value) if !value.trim().is_empty() => value,
1271 _ => {
1272 return failed(
1273 &step.id,
1274 "MURMUR_ROOST_URL is not set; capsule steps require mur-roost",
1275 )
1276 }
1277 };
1278 let Some(credential) = ctx.spawn_credential.as_ref() else {
1279 return failed(
1280 &step.id,
1281 "this session holds no spawn credential; a capsule step requires one, and mur-roost \
1282 mints it only for a session whose manifest declares capabilities.spawn.allow",
1283 );
1284 };
1285
1286 let Some(records_root) = ctx.runtime_dir.records.clone() else {
1293 return failed(
1294 &step.id,
1295 "this plan runs outside a session's records directory, where a capsule step's child \
1296 work folder is created",
1297 );
1298 };
1299 let snapshot_grant = snapshot_grant(ctx, &records_root);
1300 let plane = DelegationPlane::new(
1301 roost_url,
1302 credential.clone(),
1303 records_root,
1304 ctx.current_session_id.clone().unwrap_or_default(),
1305 crate::delegation_plane::DELEGATION_RESULT_TIMEOUT,
1306 std::sync::Arc::clone(&ctx.registry),
1307 ctx.capability_policy.env_allow.clone(),
1308 ctx.formation_id.clone(),
1309 )
1310 .bounded_by(ctx.deadline)
1311 .snapshotting_workdir(snapshot_grant);
1312 let (trace, plan_task) = (ctx.trace, ctx.task);
1318 let launched = (trace.is_some() || plan_task.is_some()).then(|| {
1319 std::sync::Arc::new(move |notice: DelegationLaunch| {
1320 if let Some(trace) = trace {
1321 trace.write_delegation_start(
1322 ¬ice.delegation_id,
1323 ¬ice.capsule,
1324 ¬ice.version,
1325 ¬ice.child_session_id,
1326 ¬ice.child_workdir,
1327 ¬ice.child_records_dir,
1328 notice.workdir_snapshot.as_ref(),
1329 );
1330 }
1331 if let Some(plan_task) = plan_task {
1332 plan_task.delegation_started(¬ice);
1333 }
1334 }) as std::sync::Arc<dyn Fn(_) + Send + Sync>
1335 });
1336 let stop = plan_task.map(|plan_task| {
1337 std::sync::Arc::new(move || plan_task.is_canceled())
1338 as std::sync::Arc<dyn Fn() -> bool + Send + Sync>
1339 });
1340 let started = Instant::now();
1341 let result = plane.delegate(
1342 &DelegationRequest {
1343 capsule: capsule.to_string(),
1344 version: ctx
1347 .capsule_versions
1348 .get(named)
1349 .cloned()
1350 .unwrap_or_else(|| "0.1.0".to_string()),
1351 task,
1352 },
1353 &DelegationOrigin {
1354 context_id: ctx.current_context_id.clone().unwrap_or_default(),
1355 launched,
1356 stop,
1357 },
1358 &ctx.runtime_dir.runtime,
1359 );
1360
1361 if result.status == DelegationStatus::Canceled {
1364 return failed(&step.id, CANCELED_STEP_ERROR);
1365 }
1366 let closes_row = match plan_task {
1369 Some(plan_task) if !result.delegation_id.is_empty() => {
1370 plan_task.delegation_settled(&result.delegation_id)
1371 }
1372 _ => true,
1373 };
1374 if let Some(trace) = ctx.trace.filter(|_| closes_row) {
1375 trace.write_delegation(
1376 &result.capsule,
1377 &result.version,
1378 Some(result.delegation_id.clone()).filter(|id| !id.is_empty()),
1381 Some(result.session_id.clone()).filter(|id| !id.is_empty()),
1382 elapsed_ms(started),
1383 result.status.as_str(),
1384 match result.status {
1385 DelegationStatus::Completed => None,
1386 _ => Some(result.output.clone()),
1387 },
1388 );
1389 }
1390
1391 match result.status {
1392 DelegationStatus::Completed => StepResult {
1393 step_id: step.id.clone(),
1394 status: StepStatus::Success,
1395 output: Some(result.output),
1396 error: None,
1397 },
1398 _ => failed(&step.id, result.output),
1399 }
1400}
1401
1402fn capsule_task_text(input: &Value) -> Result<String, String> {
1421 const ACCEPTS: &str = "capsule step input must be a string or { \"objective\": \"<text>\" }";
1422 const NO_INPUT: &str = "capsule step has no input; its input is the task text the child \
1423 receives, given as a string or as { \"objective\": \"<text>\" }";
1424
1425 match input {
1426 Value::String(text) => Ok(text.clone()),
1427 Value::Object(map) => {
1428 let mut unknown = map
1431 .keys()
1432 .filter(|key| key.as_str() != "objective")
1433 .map(String::as_str)
1434 .collect::<Vec<_>>();
1435 unknown.sort_unstable();
1436 if !unknown.is_empty() {
1437 return Err(format!(
1438 "capsule step input has keys the child cannot receive: {}; nothing on the \
1439 receiving side parses a task envelope, so put the whole instruction in \
1440 \"objective\"",
1441 unknown.join(", ")
1442 ));
1443 }
1444
1445 match map.get("objective") {
1446 Some(Value::String(text)) => Ok(text.clone()),
1447 Some(_) => Err("capsule step input \"objective\" must be a string".to_string()),
1448 None => Err(ACCEPTS.to_string()),
1449 }
1450 }
1451 Value::Null => Err(NO_INPUT.to_string()),
1452 Value::Bool(_) => Err(format!("{ACCEPTS}, not a boolean")),
1453 Value::Number(_) => Err(format!("{ACCEPTS}, not a number")),
1454 Value::Array(_) => Err(format!("{ACCEPTS}, not an array")),
1455 }
1456}
1457
1458fn interpolate_value(
1459 value: &mut Value,
1460 results: &HashMap<String, StepResult>,
1461) -> Result<(), String> {
1462 match value {
1463 Value::String(text) if text.starts_with('$') => {
1464 *text = condition::resolve_reference(text, results)?;
1465 }
1466 Value::Array(values) => {
1467 for value in values {
1468 interpolate_value(value, results)?;
1469 }
1470 }
1471 Value::Object(map) => {
1472 for value in map.values_mut() {
1473 interpolate_value(value, results)?;
1474 }
1475 }
1476 _ => {}
1477 }
1478
1479 Ok(())
1480}
1481
1482fn infer_kind(step: &StepDef) -> Result<StepKind, String> {
1483 let count =
1484 step.tool.is_some() as u8 + step.shell.is_some() as u8 + step.capsule.is_some() as u8;
1485 match count {
1486 1 if step.tool.is_some() => Ok(StepKind::Tool),
1487 1 if step.shell.is_some() => Ok(StepKind::Shell),
1488 1 if step.capsule.is_some() => Ok(StepKind::Spawn),
1489 0 => Err("step must declare exactly one of tool, shell, or capsule".to_string()),
1490 _ => Err("step declares more than one of tool, shell, or capsule".to_string()),
1491 }
1492}
1493
1494fn default_on_error() -> String {
1495 "fail".to_string()
1496}
1497
1498fn failed(step_id: &str, error: impl Into<String>) -> StepResult {
1499 StepResult {
1500 step_id: step_id.to_string(),
1501 status: StepStatus::Failed,
1502 output: None,
1503 error: Some(error.into()),
1504 }
1505}
1506
1507fn insert_result(
1508 results: &mut HashMap<String, StepResult>,
1509 order: &mut Vec<String>,
1510 result: StepResult,
1511) {
1512 order.push(result.step_id.clone());
1513 results.insert(result.step_id.clone(), result);
1514}
1515
1516fn shell_binary(command: &str) -> Option<String> {
1517 parse_shell_command(command).map(|(binary, _)| binary)
1518}
1519
1520fn parse_shell_command(command: &str) -> Option<(String, Vec<String>)> {
1521 let parts = split_shell_words(command);
1522 let (binary, args) = parts.split_first()?;
1523 Some((binary.clone(), args.to_vec()))
1524}
1525
1526fn parse_reference(reference: &str) -> Option<(&str, &str)> {
1527 reference.strip_prefix('$')?.rsplit_once('.')
1528}
1529
1530fn tool_step_output(
1547 tool: &str,
1548 workdir: &Path,
1549 data: Option<String>,
1550 data_path: Option<&str>,
1551) -> Result<Option<String>, String> {
1552 if let Some(data) = data {
1553 return Ok(Some(data));
1554 }
1555 let Some(relpath) = data_path else {
1556 return Ok(None);
1557 };
1558 match resource_plane::read_file_beneath_root(
1559 workdir,
1560 relpath,
1561 DEFAULT_EXPORT_MAX_BYTES,
1562 SymlinkPolicy::Refuse,
1563 ) {
1564 Ok(response) => Ok(Some(String::from_utf8_lossy(&response.bytes).into_owned())),
1565 Err(error) => Err(RuntimeError::ToolDataPathRefused {
1566 tool: tool.to_string(),
1567 path: relpath.to_string(),
1568 code: error.code().to_string(),
1569 detail: error.message(),
1570 }
1571 .to_string()),
1572 }
1573}
1574
1575#[cfg(test)]
1576mod tests {
1577 use std::{fs, path::PathBuf};
1578 use std::{
1579 sync::{
1580 atomic::{AtomicUsize, Ordering},
1581 Arc, Condvar, Mutex,
1582 },
1583 time::Duration,
1584 };
1585
1586 use serde_json::json;
1587 use tempfile::tempdir;
1588
1589 use super::*;
1590
1591 fn test_ctx<'a>(
1595 workdir: PathBuf,
1596 invoke_tool: &'a (dyn Fn(&str, ToolInput) -> Result<ToolResult, String> + Sync),
1597 ) -> SchedulerContext<'a> {
1598 test_ctx_gated(workdir, invoke_tool, &UNGATED)
1599 }
1600
1601 const UNGATED: fn(&PlannedCall<'_>) -> Option<String> = |_| None;
1604
1605 fn test_ctx_gated<'a>(
1606 workdir: PathBuf,
1607 invoke_tool: &'a (dyn Fn(&str, ToolInput) -> Result<ToolResult, String> + Sync),
1608 gate_step: &'a (dyn Fn(&PlannedCall<'_>) -> Option<String> + Sync),
1609 ) -> SchedulerContext<'a> {
1610 SchedulerContext {
1611 accessible_workdir: WorkdirRoot::open(&workdir).unwrap(),
1612 runtime_dir: crate::runtime_dir::RuntimeDirRoots::new(
1613 WorkdirRoot::open(&workdir).unwrap(),
1614 WorkdirRoot::open(&workdir).unwrap(),
1615 WorkdirRoot::open(&workdir).unwrap(),
1616 )
1617 .with_home(WorkdirRoot::open(&tempfile::tempdir().unwrap().keep()).unwrap()),
1619 capability_policy: CapabilityPolicy {
1620 shell_allow: vec!["bash".to_string()],
1621 spawn_allow: vec!["worker".to_string()],
1622 ..CapabilityPolicy::default()
1623 },
1624 installed_tools: HashSet::from([
1625 "ok".to_string(),
1626 "fail".to_string(),
1627 "slow".to_string(),
1628 "a".to_string(),
1629 "b".to_string(),
1630 ]),
1631 capsule_versions: HashMap::new(),
1632 current_session_id: None,
1633 current_context_id: None,
1634 registry: std::sync::Arc::new(murmur_artifact::LocalRegistry::new(
1636 tempdir().unwrap().keep(),
1637 )),
1638 spawn_credential: None,
1639 formation_id: None,
1640 deadline: None,
1641 trace: None,
1642 gate_step,
1643 invoke_tool,
1644 task: None,
1645 }
1646 }
1647
1648 fn tool_result(status: ToolStatus, data: Option<String>) -> ToolResult {
1649 ToolResult {
1650 status,
1651 summary: None,
1652 data,
1653 data_path: None,
1654 truncated: false,
1655 metadata: Vec::new(),
1656 attachments: Vec::new(),
1657 }
1658 }
1659
1660 #[test]
1664 fn plan_capsule_step_snapshot_grant_is_the_sessions() {
1665 let work = tempdir().unwrap();
1666 let records = tempdir().unwrap();
1667 let invoke = |_: &str, _: ToolInput| -> Result<ToolResult, String> { unreachable!() };
1668 let mut ctx = test_ctx(work.path().to_path_buf(), &invoke);
1669 ctx.capability_policy.spawn_workdir_snapshot = vec!["local/worker".to_string()];
1670 let records_root = WorkdirRoot::open(records.path()).unwrap();
1671
1672 let grant = snapshot_grant(&ctx, &records_root);
1673
1674 assert_eq!(grant.capsules, vec!["local/worker".to_string()]);
1675 assert_eq!(grant.source.path(), work.path());
1676 assert_eq!(
1677 grant.excluded,
1678 vec![
1679 ctx.runtime_dir.runtime.identity().unwrap(),
1680 records_root.identity().unwrap()
1681 ]
1682 );
1683 }
1684
1685 fn write_plan(workdir: &Path, plan: Value) -> PathBuf {
1686 let path = workdir.join("plan.json");
1687 fs::write(&path, serde_json::to_string(&plan).unwrap()).unwrap();
1688 path
1689 }
1690
1691 #[test]
1695 fn a_turn_s_own_decisions_are_not_callable_from_a_plan() {
1696 let dir = tempdir().unwrap();
1697 let invoke = |_: &str, _: ToolInput| -> Result<ToolResult, String> {
1698 panic!("a refused plan ran a step")
1699 };
1700 let ctx = test_ctx(dir.path().to_path_buf(), &invoke);
1701 for tool in [
1702 crate::runtime::SWITCH_DRIVER_TOOL,
1703 crate::runtime::END_WITHOUT_ANSWER_TOOL,
1704 crate::runtime::PULL_ARTIFACT_TOOL,
1705 crate::runtime::REMOVE_ARTIFACT_TOOL,
1706 ] {
1707 let plan: PlanFile = serde_json::from_value(json!({
1708 "id": "p", "steps": [{"id": "s1", "tool": tool, "input": {"reason": "none"}}],
1709 }))
1710 .unwrap();
1711 assert_eq!(
1712 validate_plan(&plan, &ctx).unwrap_err(),
1713 (
1714 "s1".to_string(),
1715 format!("'{tool}' is not callable from inside a plan")
1716 )
1717 );
1718 }
1719 }
1720
1721 #[test]
1722 fn capsule_task_text_delivers_a_bare_string_verbatim() {
1723 assert_eq!(
1724 capsule_task_text(&json!("do the thing")).unwrap(),
1725 "do the thing"
1726 );
1727 }
1728
1729 #[test]
1730 fn capsule_task_text_delivers_a_lone_objective_verbatim() {
1731 assert_eq!(
1732 capsule_task_text(&json!({"objective": "do the thing"})).unwrap(),
1733 "do the thing"
1734 );
1735 }
1736
1737 #[test]
1740 fn capsule_task_text_accepts_empty_task_text() {
1741 assert_eq!(capsule_task_text(&json!("")).unwrap(), "");
1742 assert_eq!(capsule_task_text(&json!({"objective": ""})).unwrap(), "");
1743 }
1744
1745 #[test]
1746 fn capsule_task_text_refuses_a_task_envelope() {
1747 let error = capsule_task_text(&json!({"task": "hello"})).unwrap_err();
1748 assert!(
1749 error.contains("keys the child cannot receive: task"),
1750 "{error}"
1751 );
1752 assert!(error.contains("\"objective\""), "{error}");
1753 }
1754
1755 #[test]
1756 fn capsule_task_text_lists_unaccepted_keys_in_sorted_order() {
1757 let error =
1758 capsule_task_text(&json!({"render_as": "json", "objective": "x", "instructions": "y"}))
1759 .unwrap_err();
1760 assert!(
1761 error.contains("keys the child cannot receive: instructions, render_as"),
1762 "{error}"
1763 );
1764 }
1765
1766 #[test]
1767 fn capsule_task_text_refuses_a_non_string_objective() {
1768 for input in [json!({"objective": 7}), json!({"objective": null})] {
1769 assert_eq!(
1770 capsule_task_text(&input).unwrap_err(),
1771 "capsule step input \"objective\" must be a string"
1772 );
1773 }
1774 }
1775
1776 #[test]
1777 fn capsule_task_text_refuses_an_object_without_an_objective() {
1778 assert_eq!(
1779 capsule_task_text(&json!({})).unwrap_err(),
1780 "capsule step input must be a string or { \"objective\": \"<text>\" }"
1781 );
1782 }
1783
1784 #[test]
1785 fn capsule_task_text_names_the_type_it_was_handed() {
1786 assert!(capsule_task_text(&json!(7))
1787 .unwrap_err()
1788 .ends_with("not a number"));
1789 assert!(capsule_task_text(&json!(true))
1790 .unwrap_err()
1791 .ends_with("not a boolean"));
1792 assert!(capsule_task_text(&json!(["a"]))
1793 .unwrap_err()
1794 .ends_with("not an array"));
1795 }
1796
1797 #[test]
1800 fn capsule_task_text_refuses_an_absent_input() {
1801 let error = capsule_task_text(&Value::Null).unwrap_err();
1802 assert_eq!(
1803 error,
1804 "capsule step has no input; its input is the task text the child receives, given as \
1805 a string or as { \"objective\": \"<text>\" }"
1806 );
1807 }
1808
1809 #[test]
1812 fn capsule_task_text_accepts_an_uninterpolated_reference() {
1813 assert_eq!(
1814 capsule_task_text(&json!("$analyse.output")).unwrap(),
1815 "$analyse.output"
1816 );
1817 assert_eq!(
1818 capsule_task_text(&json!({"objective": "$analyse.output"})).unwrap(),
1819 "$analyse.output"
1820 );
1821 }
1822
1823 fn test_ctx_unbounded<'a>(
1826 workdir: PathBuf,
1827 invoke_tool: &'a (dyn Fn(&str, ToolInput) -> Result<ToolResult, String> + Sync),
1828 gate_step: &'a (dyn Fn(&PlannedCall<'_>) -> Option<String> + Sync),
1829 ) -> SchedulerContext<'a> {
1830 SchedulerContext {
1831 capability_policy: CapabilityPolicy::default(),
1832 ..test_ctx_gated(workdir, invoke_tool, gate_step)
1833 }
1834 }
1835
1836 #[test]
1841 fn a_refused_step_fails_with_the_refusal_and_is_never_dispatched() {
1842 let dir = tempdir().unwrap();
1843 let dispatched = Arc::new(AtomicUsize::new(0));
1844 let counted = Arc::clone(&dispatched);
1845 let invoke = move |_name: &str, _input: ToolInput| {
1846 counted.fetch_add(1, Ordering::SeqCst);
1847 Ok(tool_result(ToolStatus::Passed, Some("ran".to_string())))
1848 };
1849 let gate = |call: &PlannedCall<'_>| match call {
1850 PlannedCall::Tool { name: "a", .. } => Some("refused by the manifest".to_string()),
1851 _ => None,
1852 };
1853 let plan = write_plan(
1854 dir.path(),
1855 json!({
1856 "id": "p",
1857 "steps": [
1858 {"id": "refused", "tool": "a", "on_error": "continue"},
1859 {"id": "allowed", "tool": "b"}
1860 ]
1861 }),
1862 );
1863
1864 let report = execute(
1865 &plan,
1866 &test_ctx_unbounded(dir.path().to_path_buf(), &invoke, &gate),
1867 );
1868 let refused = report
1869 .results
1870 .iter()
1871 .find(|result| result.step_id == "refused")
1872 .expect("the refused step settled");
1873 assert_eq!(refused.status, StepStatus::Failed);
1874 assert_eq!(refused.error.as_deref(), Some("refused by the manifest"));
1875 assert_eq!(refused.output, None);
1876 assert_eq!(dispatched.load(Ordering::SeqCst), 1);
1878 }
1879
1880 #[test]
1883 fn a_shell_step_that_fails_without_stderr_reports_its_exit_code() {
1884 if crate::cgroup::skip_without_host_support(
1885 "plan::tests::a_shell_step_that_fails_without_stderr_reports_its_exit_code",
1886 ) {
1887 return;
1888 }
1889 let dir = tempdir().unwrap();
1890 let invoke = |_name: &str, _input: ToolInput| Ok(tool_result(ToolStatus::Passed, None));
1891 let plan = write_plan(
1892 dir.path(),
1893 json!({
1894 "id": "p",
1895 "steps": [{"id": "quiet", "shell": "bash -c 'exit 3'"}]
1896 }),
1897 );
1898
1899 let report = execute(&plan, &test_ctx(dir.path().to_path_buf(), &invoke));
1900 assert!(!report.completed);
1901 assert_eq!(report.results[0].status, StepStatus::Failed);
1902 assert_eq!(report.results[0].error.as_deref(), Some("exit code 3"));
1903 }
1904
1905 #[cfg(target_os = "linux")]
1909 #[test]
1910 #[allow(clippy::print_stdout)]
1911 fn a_plan_shell_step_joins_the_session_namespace() {
1912 if crate::cgroup::skip_without_host_support(
1913 "plan::tests::a_plan_shell_step_joins_the_session_namespace",
1914 ) {
1915 return;
1916 }
1917 let dir = tempdir().unwrap();
1918 let policy = CapabilityPolicy {
1919 shell_allow: vec!["bash".to_string(), "nc".to_string(), "sleep".to_string()],
1920 ..CapabilityPolicy::default()
1921 };
1922 let session = sandbox::ShellEnforcement::resolve_declared(
1923 &policy,
1924 murmur_artifact::ContainmentClass::Scoped,
1925 sandbox::HostProbe::probe(),
1926 )
1927 .unwrap()
1928 .with_test_session_dirs();
1929 let server = execute_shell(
1930 "bash",
1931 &[
1932 "-c",
1933 "(printf \"pong\\n\" | nc -N -l 127.0.0.1 8080) >/dev/null 2>&1 & echo $! > server.pid",
1934 ],
1935 &[],
1936 &crate::workdir_fs::WorkdirRoot::open(dir.path()).unwrap(),
1937 &policy,
1938 &session,
1939 )
1940 .expect("the session's server command runs");
1941 assert_eq!(server.exit_code, 0, "stderr: {}", server.stderr);
1942
1943 let invoke = |_name: &str, _input: ToolInput| Ok(tool_result(ToolStatus::Passed, None));
1944 let mut ctx = test_ctx(dir.path().to_path_buf(), &invoke);
1945 ctx.capability_policy.shell_allow = policy.shell_allow.clone();
1946 let plan = write_plan(
1947 dir.path(),
1948 json!({
1949 "id": "p",
1950 "steps": [{
1951 "id": "client",
1952 "shell": "bash -c 'for i in {1..50}; do exec 3<>/dev/tcp/127.0.0.1/8080 \
1953 2>/dev/null && break; sleep 0.1; done; read -r reply <&3; \
1954 echo \"$reply\"'",
1955 }]
1956 }),
1957 );
1958
1959 let report = execute_in_session(&plan, &ctx, Arc::clone(&session.session_netns));
1960 println!(
1961 "plan completed: {}; step {} {:?} output {:?} error {:?}; session namespaces \
1962 created: {}",
1963 report.completed,
1964 report.results[0].step_id,
1965 report.results[0].status,
1966 report.results[0].output.as_deref().map(str::trim),
1967 report.results[0].error,
1968 session.session_netns.creations()
1969 );
1970 assert!(report.completed, "{:?}", report.results);
1971 assert_eq!(report.results[0].status, StepStatus::Success);
1972 assert_eq!(
1973 report.results[0].output.as_deref().map(str::trim),
1974 Some("pong")
1975 );
1976 assert_eq!(session.session_netns.creations(), 1);
1977
1978 let after = execute_shell(
1980 "bash",
1981 &["-c", "true"],
1982 &[],
1983 &crate::workdir_fs::WorkdirRoot::open(dir.path()).unwrap(),
1984 &policy,
1985 &session,
1986 )
1987 .expect("the session outlives the plan it submitted");
1988 assert_eq!(after.exit_code, 0);
1989 session.end_session();
1990 }
1991
1992 #[test]
1994 fn a_refused_shell_step_starts_no_subprocess() {
1995 if crate::cgroup::skip_without_host_support(
1996 "plan::tests::a_refused_shell_step_starts_no_subprocess",
1997 ) {
1998 return;
1999 }
2000 let dir = tempdir().unwrap();
2001 let invoke = |_name: &str, _input: ToolInput| Ok(tool_result(ToolStatus::Passed, None));
2002 let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
2003 let recorded = Arc::clone(&seen);
2004 let gate = move |call: &PlannedCall<'_>| match call {
2005 PlannedCall::Shell { binary, argv, .. } => {
2006 recorded
2007 .lock()
2008 .unwrap()
2009 .push((binary.to_string(), argv.to_vec()));
2010 Some("refused by the manifest".to_string())
2011 }
2012 _ => None,
2013 };
2014 let marker = dir.path().join("written-by-a-refused-step");
2015 let plan = write_plan(
2016 dir.path(),
2017 json!({
2018 "id": "p",
2019 "steps": [{
2020 "id": "w",
2021 "shell": format!("bash -c 'printf x > {}'", marker.display()),
2022 }]
2023 }),
2024 );
2025
2026 let report = execute(
2027 &plan,
2028 &test_ctx_gated(dir.path().to_path_buf(), &invoke, &gate),
2029 );
2030 assert!(!report.completed);
2031 assert_eq!(report.failed_step.as_deref(), Some("w"));
2032 assert_eq!(
2033 report.results[0].error.as_deref(),
2034 Some("refused by the manifest")
2035 );
2036 assert!(!marker.exists(), "a refused step must not have run");
2037 let seen = seen.lock().unwrap();
2039 assert_eq!(seen.len(), 1);
2040 assert_eq!(seen[0].0, "bash");
2041 assert_eq!(seen[0].1[0], "-c");
2042 }
2043
2044 #[test]
2045 fn dependent_step_receives_upstream_output() {
2046 if crate::cgroup::skip_without_host_support(
2047 "plan::tests::dependent_step_receives_upstream_output",
2048 ) {
2049 return;
2050 }
2051 let dir = tempdir().unwrap();
2052 let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
2053 let seen_tool = Arc::clone(&seen);
2054 let invoke = move |_name: &str, input: ToolInput| {
2055 seen_tool
2056 .lock()
2057 .unwrap()
2058 .push(input.data.unwrap_or_default());
2059 Ok(tool_result(
2060 ToolStatus::Passed,
2061 Some("upstream".to_string()),
2062 ))
2063 };
2064 let plan = write_plan(
2065 dir.path(),
2066 json!({
2067 "id": "p",
2068 "steps": [
2069 {"id": "a", "tool": "ok"},
2070 {"id": "b", "tool": "ok", "depends_on": ["a"], "input": {"value": "$a.output"}}
2071 ]
2072 }),
2073 );
2074
2075 let report = execute(&plan, &test_ctx(dir.path().to_path_buf(), &invoke));
2076 assert!(report.completed);
2077 assert_eq!(report.results.len(), 2);
2078 assert!(seen.lock().unwrap()[1].contains("upstream"));
2079 }
2080
2081 #[test]
2082 fn parallel_tool_steps_run_concurrently() {
2083 if crate::cgroup::skip_without_host_support(
2084 "plan::tests::parallel_tool_steps_run_concurrently",
2085 ) {
2086 return;
2087 }
2088 const RENDEZVOUS_GUARD: Duration = Duration::from_secs(30);
2091
2092 #[derive(Default)]
2095 struct Rendezvous {
2096 in_flight: usize,
2097 met: bool,
2098 }
2099
2100 let dir = tempdir().unwrap();
2101 let rendezvous = Arc::new((Mutex::new(Rendezvous::default()), Condvar::new()));
2102 let met_sibling = Arc::new(AtomicUsize::new(0));
2103 let invoke_rendezvous = Arc::clone(&rendezvous);
2104 let invoke_met = Arc::clone(&met_sibling);
2105 let invoke = move |_name: &str, _input: ToolInput| {
2106 let (state, changed) = &*invoke_rendezvous;
2107 let mut state = state.lock().unwrap();
2108 state.in_flight += 1;
2109 if state.in_flight == 2 {
2110 state.met = true;
2111 }
2112 changed.notify_all();
2113 let (mut state, _) = changed
2114 .wait_timeout_while(state, RENDEZVOUS_GUARD, |state| !state.met)
2115 .unwrap();
2116 if state.met {
2117 invoke_met.fetch_add(1, Ordering::SeqCst);
2118 }
2119 state.in_flight -= 1;
2120 Ok(tool_result(ToolStatus::Passed, Some("ok".to_string())))
2121 };
2122 let plan = write_plan(
2123 dir.path(),
2124 json!({
2125 "id": "p",
2126 "steps": [
2127 {"id": "a", "tool": "slow"},
2128 {"id": "b", "tool": "slow"}
2129 ]
2130 }),
2131 );
2132
2133 let report = execute(&plan, &test_ctx(dir.path().to_path_buf(), &invoke));
2134 assert!(report.completed);
2135 assert_eq!(
2136 met_sibling.load(Ordering::SeqCst),
2137 2,
2138 "the two independent steps never ran at the same time"
2139 );
2140 }
2141
2142 #[test]
2143 fn on_error_and_retries_are_applied() {
2144 if crate::cgroup::skip_without_host_support("plan::tests::on_error_and_retries_are_applied")
2145 {
2146 return;
2147 }
2148 let dir = tempdir().unwrap();
2149 let calls = Arc::new(AtomicUsize::new(0));
2150 let invoke_calls = Arc::clone(&calls);
2151 let invoke = move |name: &str, _input: ToolInput| {
2152 invoke_calls.fetch_add(1, Ordering::SeqCst);
2153 if name == "a" {
2154 Ok(tool_result(ToolStatus::Failed, None))
2155 } else {
2156 Ok(tool_result(ToolStatus::Passed, Some("ok".to_string())))
2157 }
2158 };
2159 let plan = write_plan(
2160 dir.path(),
2161 json!({
2162 "id": "p",
2163 "steps": [
2164 {"id": "a", "tool": "a", "retries": 2, "on_error": "continue"},
2165 {"id": "b", "tool": "b", "depends_on": ["a"]}
2166 ]
2167 }),
2168 );
2169
2170 let report = execute(&plan, &test_ctx(dir.path().to_path_buf(), &invoke));
2171 assert!(report.completed);
2172 assert_eq!(calls.load(Ordering::SeqCst), 4);
2173 }
2174
2175 #[test]
2176 fn condition_false_skips_step() {
2177 if crate::cgroup::skip_without_host_support("plan::tests::condition_false_skips_step") {
2178 return;
2179 }
2180 let dir = tempdir().unwrap();
2181 let invoke = |_name: &str, _input: ToolInput| {
2182 Ok(tool_result(ToolStatus::Passed, Some("ok".to_string())))
2183 };
2184 let plan = write_plan(
2185 dir.path(),
2186 json!({
2187 "id": "p",
2188 "steps": [
2189 {"id": "a", "tool": "ok"},
2190 {"id": "b", "tool": "ok", "depends_on": ["a"], "if": "$a.status == 'failed'"}
2191 ]
2192 }),
2193 );
2194
2195 let report = execute(&plan, &test_ctx(dir.path().to_path_buf(), &invoke));
2196 assert!(report.completed);
2197 assert_eq!(report.results[1].status, StepStatus::Skipped);
2198 }
2199
2200 #[derive(Default)]
2203 struct FlaggedTask(std::sync::atomic::AtomicBool);
2204
2205 impl PlanTask for FlaggedTask {
2206 fn is_canceled(&self) -> bool {
2207 self.0.load(Ordering::SeqCst)
2208 }
2209 fn delegation_started(&self, _launch: &DelegationLaunch) {}
2210 fn delegation_settled(&self, _delegation_id: &str) -> bool {
2211 true
2212 }
2213 }
2214
2215 #[test]
2218 fn a_cancelled_task_tries_no_step_again() {
2219 let dir = tempdir().unwrap();
2220 let task = FlaggedTask::default();
2221 let calls = AtomicUsize::new(0);
2222 let invoke = |_name: &str, _input: ToolInput| {
2223 calls.fetch_add(1, Ordering::SeqCst);
2224 task.0.store(true, Ordering::SeqCst);
2225 Ok(tool_result(ToolStatus::Failed, None))
2226 };
2227 let plan = write_plan(
2228 dir.path(),
2229 json!({
2230 "id": "p",
2231 "steps": [
2232 {"id": "a", "tool": "a", "retries": 2, "on_error": "fail"},
2233 {"id": "b", "tool": "b", "depends_on": ["a"]}
2234 ]
2235 }),
2236 );
2237
2238 let ctx = SchedulerContext {
2239 task: Some(&task),
2240 ..test_ctx_unbounded(dir.path().to_path_buf(), &invoke, &UNGATED)
2241 };
2242 let report = execute(&plan, &ctx);
2243
2244 assert_eq!(calls.load(Ordering::SeqCst), 1, "{report:?}");
2245 assert!(report.canceled, "{report:?}");
2246 assert!(!report.completed);
2247 assert_eq!(report.failed_step, None);
2248 assert_eq!(report.results[0].status, StepStatus::Failed);
2249 assert_eq!(report.results[1].status, StepStatus::Skipped);
2250 assert_eq!(
2251 report.results[1].error.as_deref(),
2252 Some(CANCELED_BEFORE_STEP_ERROR)
2253 );
2254 }
2255
2256 #[test]
2258 fn a_task_cancelled_before_the_plan_ran_dispatches_nothing() {
2259 let dir = tempdir().unwrap();
2260 let task = FlaggedTask::default();
2261 task.0.store(true, Ordering::SeqCst);
2262 let calls = AtomicUsize::new(0);
2263 let invoke = |_name: &str, _input: ToolInput| {
2264 calls.fetch_add(1, Ordering::SeqCst);
2265 Ok(tool_result(ToolStatus::Passed, Some("ran".to_string())))
2266 };
2267 let plan = write_plan(
2268 dir.path(),
2269 json!({"id": "p", "steps": [{"id": "a", "tool": "a"}, {"id": "b", "tool": "b"}]}),
2270 );
2271
2272 let ctx = SchedulerContext {
2273 task: Some(&task),
2274 ..test_ctx_unbounded(dir.path().to_path_buf(), &invoke, &UNGATED)
2275 };
2276 let report = execute(&plan, &ctx);
2277
2278 assert_eq!(calls.load(Ordering::SeqCst), 0);
2279 assert!(report.canceled && !report.completed, "{report:?}");
2280 assert!(report
2281 .results
2282 .iter()
2283 .all(|result| result.status == StepStatus::Skipped));
2284 }
2285
2286 #[test]
2288 fn a_task_nobody_cancels_changes_nothing() {
2289 let dir = tempdir().unwrap();
2290 let task = FlaggedTask::default();
2291 let invoke = |_name: &str, _input: ToolInput| {
2292 Ok(tool_result(ToolStatus::Passed, Some("ran".to_string())))
2293 };
2294 let plan = write_plan(
2295 dir.path(),
2296 json!({"id": "p", "steps": [{"id": "a", "tool": "a"}, {"id": "b", "tool": "b"}]}),
2297 );
2298
2299 let without = execute(
2300 &plan,
2301 &test_ctx_unbounded(dir.path().to_path_buf(), &invoke, &UNGATED),
2302 );
2303 let with = execute(
2304 &plan,
2305 &SchedulerContext {
2306 task: Some(&task),
2307 ..test_ctx_unbounded(dir.path().to_path_buf(), &invoke, &UNGATED)
2308 },
2309 );
2310 assert_eq!(with, without);
2311 assert!(with.completed && !with.canceled);
2312 }
2313}