1use std::collections::HashMap;
13
14use serde::{Deserialize, Serialize};
15
16use super::{DependencyPolicy, NodeKind, NodeTrust, WorkflowNode, WorkflowSpec};
17use crate::orchestration::task_graph::{TaskGraph, TaskStatus};
18use crate::orchestration::tournament::{EntrantId, Match, Tournament, TournamentAction};
19use crate::types::agent::{AgentIsolation, AgentRole, ContextInheritance, IsolationManifest};
20use crate::types::error::DeepStrikeError;
21use crate::types::error::Result;
22use crate::types::result::{LoopResult, TerminationReason};
23
24#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
25pub struct WorkflowSubmissionError {
26 pub node_index: usize,
27 pub reason: String,
28}
29
30impl std::fmt::Display for WorkflowSubmissionError {
31 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
32 write!(f, "node {}: {}", self.node_index, self.reason)
33 }
34}
35
36impl std::error::Error for WorkflowSubmissionError {}
37
38#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
40#[serde(rename_all = "snake_case")]
41pub enum WorkflowNodeStatus {
42 Completed,
43 CompletedPartial,
44 Failed,
45 SkippedUpstreamFailed,
46}
47
48#[derive(Debug, Clone, Serialize, Deserialize)]
49pub struct WorkflowNodeOutcome {
50 pub node_id: String,
51 pub status: WorkflowNodeStatus,
52 #[serde(skip_serializing_if = "Option::is_none")]
53 pub termination: Option<TerminationReason>,
54 #[serde(skip_serializing_if = "Option::is_none")]
55 pub output: Option<crate::types::message::Message>,
56}
57
58pub fn node_agent_id(node: usize) -> String {
60 format!("wf-node{node}")
61}
62
63#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
68pub struct WorkflowSpawnInfo {
69 pub agent_id: String,
70 pub goal: String,
71 pub role: String,
72 pub isolation: String,
73 pub context_inheritance: String,
74 #[serde(default, skip_serializing_if = "Option::is_none")]
75 pub model_hint: Option<String>,
76 #[serde(default = "default_trust")]
79 pub trust: String,
80 #[serde(default, skip_serializing_if = "Option::is_none")]
84 pub output_schema: Option<serde_json::Value>,
85 #[serde(default, skip_serializing_if = "Option::is_none")]
89 pub reducer: Option<String>,
90 #[serde(default, skip_serializing_if = "Vec::is_empty")]
93 pub input_agent_ids: Vec<String>,
94 #[serde(default, skip_serializing_if = "Option::is_none")]
99 pub judge_match: Option<JudgeMatch>,
100 #[serde(default, skip_serializing_if = "Option::is_none")]
106 pub loop_max_iters: Option<usize>,
107 #[serde(default, skip_serializing_if = "Vec::is_empty")]
112 pub classify_labels: Vec<String>,
113 #[serde(default, skip_serializing_if = "Option::is_none")]
117 pub token_budget: Option<u64>,
118 #[serde(default, skip_serializing_if = "Option::is_none")]
120 pub max_turns: Option<u32>,
121 #[serde(default, skip_serializing_if = "Option::is_none")]
123 pub max_wall_ms: Option<u64>,
124}
125
126fn default_trust() -> String {
127 "trusted".to_string()
128}
129
130#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
134pub struct JudgeMatch {
135 pub left: String,
136 pub right: String,
137}
138
139#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
145pub struct WorkflowBudget {
146 pub nodes_used: usize,
148 #[serde(default, skip_serializing_if = "Option::is_none")]
150 pub nodes_max: Option<usize>,
151 #[serde(default, skip_serializing_if = "Option::is_none")]
154 pub nodes_remaining: Option<usize>,
155 pub running_subagents: usize,
157 #[serde(default, skip_serializing_if = "Option::is_none")]
159 pub max_concurrent_subagents: Option<usize>,
160 #[serde(default, skip_serializing_if = "Option::is_none")]
163 pub concurrency_remaining: Option<usize>,
164 #[serde(default)]
167 pub tokens_used: u64,
168 #[serde(default, skip_serializing_if = "Option::is_none")]
170 pub tokens_max: Option<u64>,
171 #[serde(default, skip_serializing_if = "Option::is_none")]
175 pub tokens_remaining: Option<u64>,
176}
177
178fn role_label(role: AgentRole) -> &'static str {
179 match role {
180 AgentRole::Explore => "explore",
181 AgentRole::Plan => "plan",
182 AgentRole::Implement => "implement",
183 AgentRole::Verify => "verify",
184 AgentRole::Custom => "custom",
185 }
186}
187
188fn isolation_label(isolation: AgentIsolation) -> &'static str {
189 match isolation {
190 AgentIsolation::Shared => "shared",
191 AgentIsolation::ReadOnly => "read_only",
192 AgentIsolation::Worktree => "worktree",
193 AgentIsolation::Remote => "remote",
194 }
195}
196
197fn inheritance_label(inheritance: ContextInheritance) -> &'static str {
198 match inheritance {
199 ContextInheritance::None => "none",
200 ContextInheritance::SystemOnly => "system_only",
201 ContextInheritance::Full => "full",
202 }
203}
204
205fn trust_label(trust: NodeTrust) -> &'static str {
206 match trust {
207 NodeTrust::Trusted => "trusted",
208 NodeTrust::Quarantined => "quarantined",
209 }
210}
211
212struct TournamentState {
216 entrant_nodes: Vec<usize>,
218 entrants_remaining: usize,
220 bracket: Option<Tournament>,
222 judge_nodes: Vec<usize>,
224 judge_winners: Vec<Option<EntrantId>>,
226 judges_remaining: usize,
228}
229
230pub struct WorkflowRun {
232 graph: TaskGraph,
233 nodes: Vec<WorkflowNode>,
234 scheduler_policy: crate::scheduler::policy::SchedulerPolicyConfig,
235 node_of_agent: HashMap<String, usize>,
237 iter_counts: HashMap<usize, usize>,
240 tournaments: HashMap<usize, TournamentState>,
242 child_controller: HashMap<usize, usize>,
244 judge_matches: HashMap<usize, JudgeMatch>,
246}
247
248#[derive(Debug, Clone)]
253pub(crate) struct WorkflowRuntimeNodeState {
254 pub node: WorkflowNode,
255 pub status: TaskStatus,
256 pub result: Option<LoopResult>,
257 pub active_agent_id: Option<String>,
258 pub iterations_completed: usize,
259}
260
261impl WorkflowRun {
262 pub fn new(spec: &WorkflowSpec) -> Result<Self> {
264 let mut run = Self {
265 graph: spec.validate()?,
266 nodes: spec.nodes.clone(),
267 scheduler_policy: crate::scheduler::policy::SchedulerPolicyConfig::default(),
268 node_of_agent: HashMap::new(),
269 iter_counts: HashMap::new(),
270 tournaments: HashMap::new(),
271 child_controller: HashMap::new(),
272 judge_matches: HashMap::new(),
273 };
274 run.refresh_scheduling();
275 run.resolve_dependency_outcomes();
276 Ok(run)
277 }
278
279 pub(crate) fn restore_from_checkpoint(
281 spec: &WorkflowSpec,
282 states: &[WorkflowRuntimeNodeState],
283 ) -> Result<Self> {
284 let mut run = Self::new(spec)?;
285 let graph_states: Vec<(TaskStatus, Option<LoopResult>)> = states
286 .iter()
287 .map(|state| (state.status, state.result.clone()))
288 .collect();
289 run.graph
290 .restore_runtime_state(&graph_states)
291 .map_err(DeepStrikeError::InvalidConfig)?;
292
293 for (node, state) in states.iter().enumerate() {
294 if state.iterations_completed > 0 {
295 run.iter_counts.insert(node, state.iterations_completed);
296 }
297 match (state.status, state.active_agent_id.as_deref()) {
298 (TaskStatus::Running, Some(agent_id)) => {
299 run.node_of_agent.insert(agent_id.to_string(), node);
300 }
301 (TaskStatus::Running, None) => {
302 return Err(DeepStrikeError::InvalidConfig(format!(
303 "running workflow node {node} carries no active agent id"
304 )));
305 }
306 (_, Some(agent_id)) => {
307 return Err(DeepStrikeError::InvalidConfig(format!(
308 "non-running workflow node {node} carries active agent id {agent_id:?}"
309 )));
310 }
311 (_, None) => {}
312 }
313 }
314 run.refresh_scheduling();
315 Ok(run)
316 }
317
318 pub fn ready_batch(&mut self) -> Vec<usize> {
320 self.graph.ready_tasks()
321 }
322
323 pub fn set_scheduler_policy(
324 &mut self,
325 policy: crate::scheduler::policy::SchedulerPolicyConfig,
326 ) {
327 self.scheduler_policy = policy;
328 self.refresh_scheduling();
329 }
330
331 fn refresh_scheduling(&mut self) {
332 let token_costs: Vec<u64> = self
333 .nodes
334 .iter()
335 .map(|node| u64::from(node.token_budget.unwrap_or(0)))
336 .collect();
337 self.graph
338 .configure_scheduling(self.scheduler_policy, &token_costs);
339 }
340
341 pub fn current_agent_id(&self, node: usize) -> String {
346 match self.nodes[node].kind {
347 NodeKind::Loop { .. } => {
348 let k = self.iter_counts.get(&node).copied().unwrap_or(0);
349 format!("{}-i{k}", node_agent_id(node))
350 }
351 NodeKind::Spawn
355 | NodeKind::Classify { .. }
356 | NodeKind::Tournament { .. }
357 | NodeKind::Reduce { .. } => node_agent_id(node),
358 }
359 }
360
361 pub fn manifest_for(&self, node: usize) -> IsolationManifest {
365 let n = &self.nodes[node];
366 IsolationManifest {
367 agent_id: self.current_agent_id(node).into(),
368 role: n.role,
369 isolation: n.isolation,
370 context_inheritance: n.context_inheritance,
371 permitted_capability_ids: Vec::new(),
372 }
373 }
374
375 pub fn quarantine_violation(&self, node: usize) -> bool {
381 let n = &self.nodes[node];
382 matches!(n.trust, NodeTrust::Quarantined)
383 && !matches!(n.isolation, AgentIsolation::ReadOnly)
384 }
385
386 pub fn spawn_info(&self, node: usize) -> WorkflowSpawnInfo {
390 let n = &self.nodes[node];
391 let reducer = match &n.kind {
396 NodeKind::Reduce { reducer } => Some(reducer.clone()),
397 _ => None,
398 };
399 let input_agent_ids: Vec<String> = n.depends_on.iter().map(|&d| node_agent_id(d)).collect();
400 let loop_max_iters = match &n.kind {
404 NodeKind::Loop { max_iters } => Some(*max_iters),
405 _ => None,
406 };
407 let classify_labels = match &n.kind {
408 NodeKind::Classify { branches } => branches.iter().map(|b| b.label.clone()).collect(),
409 _ => Vec::new(),
410 };
411 WorkflowSpawnInfo {
412 agent_id: self.current_agent_id(node),
413 goal: n.task.goal.clone(),
414 role: role_label(n.role).to_string(),
415 isolation: isolation_label(n.isolation).to_string(),
416 context_inheritance: inheritance_label(n.context_inheritance).to_string(),
417 model_hint: n.model_hint.clone(),
418 trust: trust_label(n.trust).to_string(),
419 output_schema: n.output_schema.clone(),
420 reducer,
421 input_agent_ids,
422 judge_match: self.judge_matches.get(&node).cloned(),
423 loop_max_iters,
424 classify_labels,
425 token_budget: n.token_budget,
426 max_turns: n.max_turns,
427 max_wall_ms: n.max_wall_ms,
428 }
429 }
430
431 pub fn mark_spawned(&mut self, node: usize, agent_id: &str) {
435 self.graph.start(node);
436 self.node_of_agent.insert(agent_id.to_string(), node);
437 }
438
439 pub fn mark_denied(&mut self, node: usize) {
442 self.graph.fail(node);
443 self.resolve_dependency_outcomes();
444 }
445
446 pub fn mark_spawn_failed(&mut self, agent_id: &str) -> Option<usize> {
450 let node = self.node_of_agent.remove(agent_id)?;
451 self.graph.fail(node);
452 self.resolve_dependency_outcomes();
453 Some(node)
454 }
455
456 pub fn record_completion(&mut self, agent_id: &str, result: LoopResult) -> Option<usize> {
464 let node = *self.node_of_agent.get(agent_id)?;
465
466 if let Some(&controller) = self.child_controller.get(&node) {
469 return self.advance_tournament(controller, node, result);
470 }
471
472 if matches!(self.nodes[node].kind, NodeKind::Loop { .. })
475 && result.termination != TerminationReason::Completed
476 {
477 self.settle_result(node, result);
478 return Some(node);
479 }
480
481 match &self.nodes[node].kind {
482 NodeKind::Loop { max_iters } => {
483 let max_iters = *max_iters;
486 let stop_requested = result.loop_continue == Some(false);
487 let done = self.iter_counts.entry(node).or_insert(0);
488 *done += 1;
489 if *done < max_iters && !stop_requested {
490 self.graph.set_ready(node);
492 return Some(node);
493 }
494 }
495 NodeKind::Classify { branches } => {
496 let chosen = result.classify_branch.clone();
500 let prune: Vec<usize> = branches
501 .iter()
502 .filter(|b| Some(&b.label) != chosen.as_ref())
503 .flat_map(|b| b.nodes.iter().copied())
504 .collect();
505 for bn in prune {
506 self.graph.fail(bn);
507 }
508 }
509 NodeKind::Spawn | NodeKind::Tournament { .. } | NodeKind::Reduce { .. } => {}
513 }
514
515 self.settle_result(node, result);
518 Some(node)
519 }
520
521 fn settle_result(&mut self, node: usize, result: LoopResult) {
522 match result.termination {
523 TerminationReason::Completed => self.graph.complete(node, result),
524 TerminationReason::MaxTurns
525 | TerminationReason::TokenBudget
526 | TerminationReason::Timeout
527 | TerminationReason::MilestoneExceeded
528 | TerminationReason::ContextOverflow
529 | TerminationReason::NoProgress => self.graph.complete_partial(node, result),
530 TerminationReason::Error | TerminationReason::UserAbort => {
531 self.graph.fail_with_result(node, result)
532 }
533 }
534 self.resolve_dependency_outcomes();
535 }
536
537 fn resolve_dependency_outcomes(&mut self) {
540 loop {
541 let mut changed = false;
542 for node in 0..self.nodes.len() {
543 if self.graph.get(node).map(|n| n.status) != Some(TaskStatus::Pending) {
544 continue;
545 }
546 let policy = self.nodes[node].dep_policy;
547 if policy == DependencyPolicy::Optional {
548 self.graph.set_ready(node);
549 changed = true;
550 continue;
551 }
552 let statuses: Vec<TaskStatus> = self.nodes[node]
553 .depends_on
554 .iter()
555 .filter_map(|&dep| self.graph.get(dep).map(|n| n.status))
556 .collect();
557 let all_terminal = statuses.iter().all(|status| status.is_terminal());
558 let impossible = match policy {
559 DependencyPolicy::AllSuccess => statuses.iter().any(|status| {
560 matches!(
561 status,
562 TaskStatus::CompletedPartial
563 | TaskStatus::Failed
564 | TaskStatus::SkippedUpstreamFailed
565 )
566 }),
567 DependencyPolicy::AcceptPartial => statuses.iter().any(|status| {
568 matches!(
569 status,
570 TaskStatus::Failed | TaskStatus::SkippedUpstreamFailed
571 )
572 }),
573 DependencyPolicy::AllTerminal | DependencyPolicy::Optional => false,
574 };
575 if impossible {
576 self.graph.skip_upstream_failed(node);
577 changed = true;
578 } else if all_terminal {
579 self.graph.set_ready(node);
580 changed = true;
581 }
582 }
583 if !changed {
584 break;
585 }
586 }
587 }
588
589 fn append_child(&mut self, node: WorkflowNode) -> usize {
595 let idx = self.graph.add(node.task.clone(), Vec::new());
596 debug_assert_eq!(idx, self.nodes.len(), "graph/nodes index drift");
597 self.nodes.push(node);
598 self.refresh_scheduling();
599 idx
600 }
601
602 pub fn expand_ready_controllers(&mut self) {
607 let pending: Vec<usize> = (0..self.nodes.len())
608 .filter(|i| !self.tournaments.contains_key(i))
609 .filter(|&i| matches!(self.nodes[i].kind, NodeKind::Tournament { .. }))
610 .filter(|&i| self.graph.get(i).map(|n| n.status) == Some(TaskStatus::Ready))
611 .collect();
612 for c in pending {
613 self.expand_tournament(c);
614 }
615 }
616
617 fn expand_tournament(&mut self, c: usize) {
620 let entrants = match &self.nodes[c].kind {
621 NodeKind::Tournament { entrants } => entrants.clone(),
622 _ => return,
623 };
624 let trust = self.nodes[c].trust;
625 self.graph.start(c);
627 if entrants.len() < 2 {
631 self.complete_tournament(c, None);
632 return;
633 }
634 let mut entrant_nodes = Vec::with_capacity(entrants.len());
635 for task in entrants {
636 let child = WorkflowNode::new(task, AgentRole::Custom)
637 .with_isolation(AgentIsolation::ReadOnly)
638 .with_trust(trust);
639 let idx = self.append_child(child);
640 self.child_controller.insert(idx, c);
641 entrant_nodes.push(idx);
642 }
643 let entrants_remaining = entrant_nodes.len();
644 self.tournaments.insert(
645 c,
646 TournamentState {
647 entrant_nodes,
648 entrants_remaining,
649 bracket: None,
650 judge_nodes: Vec::new(),
651 judge_winners: Vec::new(),
652 judges_remaining: 0,
653 },
654 );
655 }
656
657 fn advance_tournament(
660 &mut self,
661 controller: usize,
662 child: usize,
663 result: LoopResult,
664 ) -> Option<usize> {
665 match result.termination {
671 TerminationReason::Completed => self.graph.complete(child, result.clone()),
672 TerminationReason::MaxTurns
673 | TerminationReason::TokenBudget
674 | TerminationReason::Timeout
675 | TerminationReason::MilestoneExceeded
676 | TerminationReason::ContextOverflow
677 | TerminationReason::NoProgress => self.graph.complete_partial(child, result.clone()),
678 TerminationReason::Error | TerminationReason::UserAbort => {
679 self.graph.fail_with_result(child, result.clone())
680 }
681 }
682
683 let in_entrant_phase = self.tournaments.get(&controller)?.bracket.is_none();
684 if in_entrant_phase {
685 let all_in = {
686 let st = self.tournaments.get_mut(&controller)?;
687 st.entrants_remaining = st.entrants_remaining.saturating_sub(1);
688 st.entrants_remaining == 0
689 };
690 if all_in {
691 self.begin_bracket(controller);
692 }
693 } else {
694 let round_done = {
695 let st = self.tournaments.get_mut(&controller)?;
696 if let Some(pos) = st.judge_nodes.iter().position(|&n| n == child) {
697 st.judge_winners[pos] = result.tournament_winner.clone();
698 }
699 st.judges_remaining = st.judges_remaining.saturating_sub(1);
700 st.judges_remaining == 0
701 };
702 if round_done {
703 self.finish_round(controller);
704 }
705 }
706 Some(controller)
707 }
708
709 fn begin_bracket(&mut self, controller: usize) {
711 let entrant_ids: Vec<EntrantId> = self
712 .tournaments
713 .get(&controller)
714 .map(|st| st.entrant_nodes.iter().map(|&n| node_agent_id(n)).collect())
715 .unwrap_or_default();
716 let mut bracket = match Tournament::new(entrant_ids) {
718 Ok(b) => b,
719 Err(_) => return self.complete_tournament(controller, None),
720 };
721 let action = bracket.start();
722 if let Some(st) = self.tournaments.get_mut(&controller) {
723 st.bracket = Some(bracket);
724 }
725 self.apply_action(controller, action);
726 }
727
728 fn finish_round(&mut self, controller: usize) {
730 let winners: Vec<EntrantId> = self
731 .tournaments
732 .get(&controller)
733 .map(|st| st.judge_winners.iter().filter_map(|w| w.clone()).collect())
734 .unwrap_or_default();
735 let action = {
736 let st = match self.tournaments.get_mut(&controller) {
737 Some(st) => st,
738 None => return,
739 };
740 match st.bracket.as_mut() {
741 Some(b) => b.feed_round(winners),
744 None => return,
745 }
746 };
747 match action {
748 Ok(act) => self.apply_action(controller, act),
749 Err(_) => self.complete_tournament(controller, None),
750 }
751 }
752
753 fn apply_action(&mut self, controller: usize, action: TournamentAction) {
755 match action {
756 TournamentAction::JudgeRound { matches, .. } => self.emit_judges(controller, matches),
757 TournamentAction::Done { winner, .. } => {
758 self.complete_tournament(controller, Some(winner))
759 }
760 }
761 }
762
763 fn emit_judges(&mut self, controller: usize, matches: Vec<Match>) {
766 let criterion = self.nodes[controller].task.clone();
767 let trust = self.nodes[controller].trust;
768 let mut judge_nodes = Vec::with_capacity(matches.len());
769 for m in &matches {
770 let judge = WorkflowNode::new(criterion.clone(), AgentRole::Verify).with_trust(trust);
771 let idx = self.append_child(judge);
772 self.child_controller.insert(idx, controller);
773 self.judge_matches.insert(
774 idx,
775 JudgeMatch {
776 left: m.left.clone(),
777 right: m.right.clone(),
778 },
779 );
780 judge_nodes.push(idx);
781 }
782 if let Some(st) = self.tournaments.get_mut(&controller) {
783 st.judge_winners = vec![None; judge_nodes.len()];
784 st.judges_remaining = judge_nodes.len();
785 st.judge_nodes = judge_nodes;
786 }
787 }
788
789 fn complete_tournament(&mut self, controller: usize, winner: Option<EntrantId>) {
795 self.tournaments.remove(&controller);
796 let Some(winner) = winner else {
797 self.graph.fail(controller);
798 self.resolve_dependency_outcomes();
799 return;
800 };
801 let result = LoopResult {
802 termination: TerminationReason::Completed,
803 final_message: None,
804 turns_used: 0,
805 total_tokens_used: 0,
806 loop_continue: None,
807 classify_branch: None,
808 tournament_winner: Some(winner),
809 pace_decision: None,
810 };
811 self.graph.complete(controller, result);
812 self.resolve_dependency_outcomes();
813 }
814
815 pub fn submit_nodes_from(
841 &mut self,
842 submitter: Option<&str>,
843 mut nodes: Vec<WorkflowNode>,
844 ) -> std::result::Result<Vec<usize>, WorkflowSubmissionError> {
845 let submitter_quarantined = submitter.is_some_and(|s| self.is_agent_quarantined(s));
846 if submitter_quarantined {
847 for node in &mut nodes {
848 node.trust = NodeTrust::Quarantined;
849 }
850 }
851 self.submit_nodes(nodes)
852 }
853
854 pub fn submit_nodes(
855 &mut self,
856 mut nodes: Vec<WorkflowNode>,
857 ) -> std::result::Result<Vec<usize>, WorkflowSubmissionError> {
858 let base = self.nodes.len();
859 let batch_len = nodes.len();
860 for (node_index, node) in nodes.iter().enumerate() {
861 if matches!(node.kind, NodeKind::Loop { max_iters: 0 }) {
862 return Err(WorkflowSubmissionError {
863 node_index,
864 reason: "loop max_iters must be greater than zero".to_string(),
865 });
866 }
867 if matches!(&node.kind, NodeKind::Tournament { entrants } if entrants.len() < 2) {
868 return Err(WorkflowSubmissionError {
869 node_index,
870 reason: "tournament requires at least two entrants".to_string(),
871 });
872 }
873 for &dependency in &node.depends_on {
874 if dependency >= batch_len {
875 return Err(WorkflowSubmissionError {
876 node_index,
877 reason: format!(
878 "dependency {dependency} out of range for batch of {batch_len} nodes"
879 ),
880 });
881 }
882 if dependency == node_index {
883 return Err(WorkflowSubmissionError {
884 node_index,
885 reason: "node depends on itself".to_string(),
886 });
887 }
888 }
889 if let NodeKind::Classify { branches } = &node.kind {
890 for branch_node in branches.iter().flat_map(|br| br.nodes.iter().copied()) {
891 if branch_node >= batch_len {
892 return Err(WorkflowSubmissionError {
893 node_index,
894 reason: format!("classify branch node {branch_node} out of range"),
895 });
896 }
897 if branch_node == node_index {
898 return Err(WorkflowSubmissionError {
899 node_index,
900 reason: "classifier cannot select itself as a branch node".to_string(),
901 });
902 }
903 if !nodes[branch_node].depends_on.contains(&node_index) {
904 return Err(WorkflowSubmissionError {
905 node_index: branch_node,
906 reason: format!(
907 "classify branch node must depend on classifier {node_index}"
908 ),
909 });
910 }
911 }
912 }
913 }
914
915 if WorkflowSpec::new(nodes.clone()).validate().is_err() {
916 return Err(WorkflowSubmissionError {
917 node_index: 0,
918 reason: "submission introduces a dependency cycle".to_string(),
919 });
920 }
921
922 for node in &mut nodes {
923 node.depends_on = node.depends_on.iter().map(|dep| base + dep).collect();
924 if let NodeKind::Classify { branches } = &mut node.kind {
925 for branch in branches {
926 branch.nodes = branch.nodes.iter().map(|node| base + node).collect();
927 }
928 }
929 }
930
931 let mut ids = Vec::with_capacity(nodes.len());
932 for node in nodes {
933 let deps = node.depends_on.clone();
934 let idx = self.graph.add(node.task.clone(), deps);
935 debug_assert_eq!(idx, self.nodes.len(), "graph/nodes index drift");
936 self.nodes.push(node);
937 ids.push(idx);
938 }
939 self.refresh_scheduling();
940 self.resolve_dependency_outcomes();
941 Ok(ids)
942 }
943
944 pub fn owns_agent(&self, agent_id: &str) -> bool {
946 self.node_of_agent.contains_key(agent_id)
947 }
948
949 pub fn is_agent_quarantined(&self, agent_id: &str) -> bool {
954 self.node_of_agent
955 .get(agent_id)
956 .is_some_and(|&node| matches!(self.nodes[node].trust, NodeTrust::Quarantined))
957 }
958
959 #[cfg(test)]
965 pub(crate) fn quarantine_agent(&mut self, agent_id: &str) -> bool {
966 match self.node_of_agent.get(agent_id).copied() {
967 Some(node) => {
968 self.nodes[node].trust = NodeTrust::Quarantined;
969 true
970 }
971 None => false,
972 }
973 }
974
975 #[cfg(test)]
979 pub(crate) fn batch_drained(&self) -> bool {
980 !(0..self.graph.len()).any(|i| {
981 matches!(
982 self.graph.get(i).map(|n| &n.status),
983 Some(crate::orchestration::task_graph::TaskStatus::Running)
984 )
985 })
986 }
987
988 #[cfg(test)]
990 pub(crate) fn is_complete(&self) -> bool {
991 self.graph.all_done()
992 }
993
994 pub fn finish(&mut self) -> Vec<WorkflowNodeOutcome> {
996 for node in 0..self.graph.len() {
997 match self.graph.get(node).map(|node| node.status) {
998 Some(TaskStatus::Pending | TaskStatus::Ready) => {
999 self.graph.skip_upstream_failed(node)
1000 }
1001 Some(TaskStatus::Running) => self.graph.fail(node),
1002 _ => {}
1003 }
1004 }
1005 self.node_outcomes()
1006 }
1007
1008 pub fn node_outcomes(&self) -> Vec<WorkflowNodeOutcome> {
1009 (0..self.graph.len())
1010 .filter_map(|node| {
1011 let graph_node = self.graph.get(node)?;
1012 let status = match graph_node.status {
1013 TaskStatus::Completed => WorkflowNodeStatus::Completed,
1014 TaskStatus::CompletedPartial => WorkflowNodeStatus::CompletedPartial,
1015 TaskStatus::Failed => WorkflowNodeStatus::Failed,
1016 TaskStatus::SkippedUpstreamFailed => WorkflowNodeStatus::SkippedUpstreamFailed,
1017 TaskStatus::Pending | TaskStatus::Ready | TaskStatus::Running => return None,
1018 };
1019 Some(WorkflowNodeOutcome {
1020 node_id: node_agent_id(node),
1021 status,
1022 termination: graph_node.result.as_ref().map(|result| result.termination),
1023 output: graph_node
1024 .result
1025 .as_ref()
1026 .and_then(|result| result.final_message.clone()),
1027 })
1028 })
1029 .collect()
1030 }
1031
1032 pub fn abort_outcomes(&self) -> Vec<WorkflowNodeOutcome> {
1035 (0..self.graph.len())
1036 .filter_map(|node| {
1037 let graph_node = self.graph.get(node)?;
1038 let (status, termination) = match graph_node.status {
1039 TaskStatus::Completed => (
1040 WorkflowNodeStatus::Completed,
1041 graph_node.result.as_ref().map(|result| result.termination),
1042 ),
1043 TaskStatus::CompletedPartial => (
1044 WorkflowNodeStatus::CompletedPartial,
1045 graph_node.result.as_ref().map(|result| result.termination),
1046 ),
1047 TaskStatus::Pending | TaskStatus::SkippedUpstreamFailed => {
1048 (WorkflowNodeStatus::SkippedUpstreamFailed, None)
1049 }
1050 TaskStatus::Ready | TaskStatus::Running | TaskStatus::Failed => (
1051 WorkflowNodeStatus::Failed,
1052 Some(
1053 graph_node
1054 .result
1055 .as_ref()
1056 .map_or(TerminationReason::UserAbort, |result| result.termination),
1057 ),
1058 ),
1059 };
1060 Some(WorkflowNodeOutcome {
1061 node_id: node_agent_id(node),
1062 status,
1063 termination,
1064 output: graph_node
1065 .result
1066 .as_ref()
1067 .and_then(|result| result.final_message.clone()),
1068 })
1069 })
1070 .collect()
1071 }
1072
1073 pub fn len(&self) -> usize {
1075 self.graph.len()
1076 }
1077
1078 pub(crate) fn checkpoint_nodes(&self) -> Vec<WorkflowRuntimeNodeState> {
1080 (0..self.graph.len())
1081 .filter_map(|node| {
1082 let graph_node = self.graph.get(node)?;
1083 let active_agent_id = (graph_node.status == TaskStatus::Running)
1084 .then(|| {
1085 self.node_of_agent.iter().find_map(|(agent_id, &owner)| {
1086 (owner == node).then(|| agent_id.clone())
1087 })
1088 })
1089 .flatten();
1090 Some(WorkflowRuntimeNodeState {
1091 node: self.nodes[node].clone(),
1092 status: graph_node.status,
1093 result: graph_node.result.clone(),
1094 active_agent_id,
1095 iterations_completed: self.iter_counts.get(&node).copied().unwrap_or(0),
1096 })
1097 })
1098 .collect()
1099 }
1100}
1101
1102#[cfg(test)]
1103mod tests {
1104 use super::*;
1105 use crate::orchestration::workflow::{ClassifyBranch, fanout_synthesize};
1106 use crate::types::result::{LoopResult, TerminationReason};
1107 use crate::types::task::RuntimeTask;
1108
1109 fn done() -> LoopResult {
1110 LoopResult {
1111 termination: TerminationReason::Completed,
1112 final_message: None,
1113 turns_used: 1,
1114 total_tokens_used: 0,
1115 loop_continue: None,
1116 classify_branch: None,
1117 tournament_winner: None,
1118 pace_decision: None,
1119 }
1120 }
1121
1122 fn terminated(termination: TerminationReason) -> LoopResult {
1123 LoopResult {
1124 termination,
1125 ..done()
1126 }
1127 }
1128
1129 fn fanout2() -> WorkflowRun {
1130 let spec = fanout_synthesize(
1132 vec![RuntimeTask::new("w0"), RuntimeTask::new("w1")],
1133 RuntimeTask::new("synth"),
1134 );
1135 WorkflowRun::new(&spec).unwrap()
1136 }
1137
1138 fn judge_done(winner: &str) -> LoopResult {
1140 LoopResult {
1141 tournament_winner: Some(winner.to_string()),
1142 ..done()
1143 }
1144 }
1145
1146 fn spawn_round(run: &mut WorkflowRun) -> Vec<(usize, String)> {
1149 run.expand_ready_controllers();
1150 let ready = run.ready_batch();
1151 let mut out = Vec::new();
1152 for node in ready {
1153 let id = run.current_agent_id(node);
1154 run.mark_spawned(node, &id);
1155 out.push((node, id));
1156 }
1157 out
1158 }
1159
1160 fn outcome_ids(run: &WorkflowRun, status: WorkflowNodeStatus) -> Vec<String> {
1161 run.node_outcomes()
1162 .into_iter()
1163 .filter(|outcome| outcome.status == status)
1164 .map(|outcome| outcome.node_id)
1165 .collect()
1166 }
1167
1168 #[test]
1169 fn first_batch_is_the_workers() {
1170 let mut run = fanout2();
1171 assert_eq!(run.ready_batch(), vec![0, 1]);
1172 assert_eq!(run.len(), 3);
1173 assert!(!run.is_complete());
1174 }
1175
1176 #[test]
1179 fn submit_nodes_appends_independent_nodes_ready_immediately() {
1180 use crate::orchestration::workflow::WorkflowNode;
1181 use crate::types::agent::AgentRole;
1182
1183 let mut run = fanout2(); assert_eq!(run.len(), 3);
1185 let ids = run
1186 .submit_nodes(vec![
1187 WorkflowNode::new(RuntimeTask::new("extra-a"), AgentRole::Implement),
1188 WorkflowNode::new(RuntimeTask::new("extra-b"), AgentRole::Implement),
1189 ])
1190 .unwrap();
1191 assert_eq!(ids, vec![3, 4], "appended after the existing 3 nodes");
1192 assert_eq!(run.len(), 5);
1193 let ready = run.ready_batch();
1194 assert!(
1195 ready.contains(&3) && ready.contains(&4),
1196 "submitted independent nodes are immediately ready: {ready:?}"
1197 );
1198 }
1199
1200 #[test]
1201 fn submitted_nodes_must_complete_before_workflow_is_done() {
1202 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1203 use crate::types::agent::AgentRole;
1204
1205 let spec = WorkflowSpec::new(vec![WorkflowNode::new(
1207 RuntimeTask::new("root"),
1208 AgentRole::Implement,
1209 )]);
1210 let mut run = WorkflowRun::new(&spec).unwrap();
1211 let id0 = run.current_agent_id(0);
1212 run.mark_spawned(0, &id0);
1213 run.record_completion(&id0, done());
1214 let ids = run
1215 .submit_nodes(vec![WorkflowNode::new(
1216 RuntimeTask::new("more"),
1217 AgentRole::Implement,
1218 )])
1219 .unwrap();
1220 assert_eq!(ids, vec![1]);
1221 assert!(
1222 !run.is_complete(),
1223 "not complete while the submitted node is pending"
1224 );
1225 let spawned = spawn_round(&mut run);
1226 assert_eq!(spawned, vec![(1usize, "wf-node1".to_string())]);
1227 run.record_completion("wf-node1", done());
1228 assert!(
1229 run.is_complete(),
1230 "complete once the submitted node finishes"
1231 );
1232 }
1233
1234 #[test]
1235 fn reduce_node_carries_reducer_and_inputs_then_completes_like_a_spawn() {
1236 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1237 use crate::types::agent::AgentRole;
1238
1239 let spec = WorkflowSpec::new(vec![
1242 WorkflowNode::new(RuntimeTask::new("worker-a"), AgentRole::Explore),
1243 WorkflowNode::new(RuntimeTask::new("worker-b"), AgentRole::Explore),
1244 WorkflowNode::new(RuntimeTask::new("merge"), AgentRole::Implement)
1245 .with_reduce("dedupe_lines")
1246 .with_depends_on(vec![0, 1]),
1247 ]);
1248 let mut run = WorkflowRun::new(&spec).unwrap();
1249
1250 assert_eq!(run.ready_batch(), vec![0, 1]);
1252 for i in [0usize, 1] {
1253 let id = run.current_agent_id(i);
1254 run.mark_spawned(i, &id);
1255 run.record_completion(&id, done());
1256 }
1257
1258 assert_eq!(run.ready_batch(), vec![2]);
1260 let info = run.spawn_info(2);
1261 assert_eq!(info.reducer.as_deref(), Some("dedupe_lines"));
1262 assert_eq!(
1263 info.input_agent_ids,
1264 vec!["wf-node0".to_string(), "wf-node1".to_string()]
1265 );
1266
1267 run.mark_spawned(2, "wf-node2");
1269 run.record_completion("wf-node2", done());
1270 assert!(run.is_complete());
1271 let completed = outcome_ids(&run, WorkflowNodeStatus::Completed);
1272 assert_eq!(completed, vec!["wf-node0", "wf-node1", "wf-node2"]);
1273 assert_eq!(run.node_outcomes().len(), completed.len());
1274 }
1275
1276 #[test]
1277 fn output_schema_reaches_the_spawn_descriptor() {
1278 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1279 use crate::types::agent::AgentRole;
1280
1281 let schema = serde_json::json!({
1283 "type": "object",
1284 "required": ["verdict"],
1285 "properties": { "verdict": { "type": "string" } }
1286 });
1287 let spec = WorkflowSpec::new(vec![
1288 WorkflowNode::new(RuntimeTask::new("judge"), AgentRole::Verify)
1289 .with_output_schema(schema.clone()),
1290 ]);
1291 let run = WorkflowRun::new(&spec).unwrap();
1292 let info = run.spawn_info(0);
1293 assert_eq!(info.output_schema.as_ref(), Some(&schema));
1294
1295 let json = serde_json::to_string(&info).unwrap();
1297 let back: WorkflowSpawnInfo = serde_json::from_str(&json).unwrap();
1298 assert_eq!(back.output_schema, Some(schema));
1299
1300 let plain = WorkflowSpec::new(vec![WorkflowNode::new(
1302 RuntimeTask::new("x"),
1303 AgentRole::Implement,
1304 )]);
1305 let plain_info = WorkflowRun::new(&plain).unwrap().spawn_info(0);
1306 assert!(plain_info.output_schema.is_none());
1307 assert!(
1308 !serde_json::to_string(&plain_info)
1309 .unwrap()
1310 .contains("output_schema")
1311 );
1312 }
1313
1314 #[test]
1315 fn quarantined_submitter_taints_submitted_nodes() {
1316 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1317 use crate::types::agent::AgentRole;
1318
1319 let spec = WorkflowSpec::new(vec![
1323 WorkflowNode::new(RuntimeTask::new("read-untrusted"), AgentRole::Explore).quarantined(),
1324 ]);
1325 let mut run = WorkflowRun::new(&spec).unwrap();
1326 let id0 = run.current_agent_id(0);
1327 run.mark_spawned(0, &id0);
1328 run.record_completion(&id0, done());
1329
1330 let ids = run
1332 .submit_nodes_from(
1333 Some(&id0),
1334 vec![WorkflowNode::new(
1335 RuntimeTask::new("act"),
1336 AgentRole::Implement,
1337 )],
1338 )
1339 .unwrap();
1340 assert_eq!(ids, vec![1]);
1341 let id1 = run.current_agent_id(1);
1342 run.mark_spawned(1, &id1);
1343 assert!(
1344 run.is_agent_quarantined(&id1),
1345 "submitted node inherits the submitter's quarantine (no escalation)"
1346 );
1347
1348 let ids2 = run
1350 .submit_nodes_from(
1351 None,
1352 vec![WorkflowNode::new(
1353 RuntimeTask::new("trusted-work"),
1354 AgentRole::Implement,
1355 )],
1356 )
1357 .unwrap();
1358 let id2 = run.current_agent_id(ids2[0]);
1359 run.mark_spawned(ids2[0], &id2);
1360 assert!(
1361 !run.is_agent_quarantined(&id2),
1362 "no quarantined submitter ⇒ no coercion"
1363 );
1364 }
1365
1366 #[test]
1367 fn submit_nodes_honors_batch_relative_backward_deps() {
1368 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1369 use crate::types::agent::AgentRole;
1370
1371 let spec = WorkflowSpec::new(vec![WorkflowNode::new(
1372 RuntimeTask::new("root"),
1373 AgentRole::Implement,
1374 )]);
1375 let mut run = WorkflowRun::new(&spec).unwrap();
1376 let id0 = run.current_agent_id(0);
1377 run.mark_spawned(0, &id0);
1378 run.record_completion(&id0, done());
1379 let ids = run
1381 .submit_nodes(vec![
1382 WorkflowNode::new(RuntimeTask::new("extractor"), AgentRole::Implement),
1383 WorkflowNode::new(RuntimeTask::new("dependent"), AgentRole::Implement)
1384 .with_depends_on(vec![0]),
1385 ])
1386 .unwrap();
1387 assert_eq!(ids, vec![1, 2]);
1388 assert_eq!(
1389 run.ready_batch(),
1390 vec![1],
1391 "backward dep keeps the dependent pending"
1392 );
1393 run.mark_spawned(1, "wf-node1");
1394 run.record_completion("wf-node1", done());
1395 assert_eq!(
1396 run.ready_batch(),
1397 vec![2],
1398 "dependent unblocks after the extractor"
1399 );
1400 }
1401
1402 #[test]
1403 fn submit_nodes_accepts_acyclic_forward_dependencies() {
1404 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1405 use crate::types::agent::AgentRole;
1406
1407 let spec = WorkflowSpec::new(vec![WorkflowNode::new(
1408 RuntimeTask::new("root"),
1409 AgentRole::Implement,
1410 )]);
1411 let mut run = WorkflowRun::new(&spec).unwrap();
1412 let ids = run
1414 .submit_nodes(vec![
1415 WorkflowNode::new(RuntimeTask::new("consumer"), AgentRole::Implement)
1416 .with_depends_on(vec![1]),
1417 WorkflowNode::new(RuntimeTask::new("producer"), AgentRole::Implement),
1418 ])
1419 .unwrap();
1420 assert_eq!(ids, vec![1, 2]);
1421 assert_eq!(
1422 run.ready_batch(),
1423 vec![2, 0],
1424 "the producer is on the longer critical path and runs before the independent root"
1425 );
1426 run.mark_spawned(2, "wf-node2");
1427 run.record_completion("wf-node2", done());
1428 assert!(run.ready_batch().contains(&1));
1429 }
1430
1431 #[test]
1432 fn submit_nodes_rejects_malformed_batches_atomically() {
1433 use crate::orchestration::workflow::WorkflowNode;
1434 use crate::types::agent::AgentRole;
1435
1436 for nodes in [
1437 vec![
1438 WorkflowNode::new(RuntimeTask::new("bad-range"), AgentRole::Implement)
1439 .with_depends_on(vec![1]),
1440 ],
1441 vec![
1442 WorkflowNode::new(RuntimeTask::new("self"), AgentRole::Implement)
1443 .with_depends_on(vec![0]),
1444 ],
1445 vec![
1446 WorkflowNode::new(RuntimeTask::new("a"), AgentRole::Implement)
1447 .with_depends_on(vec![1]),
1448 WorkflowNode::new(RuntimeTask::new("b"), AgentRole::Implement)
1449 .with_depends_on(vec![0]),
1450 ],
1451 ] {
1452 let mut run = fanout2();
1453 let before = run.len();
1454 assert!(run.submit_nodes(nodes).is_err());
1455 assert_eq!(run.len(), before, "rejection must precede every mutation");
1456 }
1457 }
1458
1459 #[test]
1460 fn submitted_node_can_itself_be_a_loop_control_flow() {
1461 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1466 use crate::types::agent::AgentRole;
1467
1468 let spec = WorkflowSpec::new(vec![WorkflowNode::new(
1469 RuntimeTask::new("root"),
1470 AgentRole::Implement,
1471 )]);
1472 let mut run = WorkflowRun::new(&spec).unwrap();
1473 let id0 = run.current_agent_id(0);
1474 run.mark_spawned(0, &id0);
1475 run.record_completion(&id0, done());
1476
1477 let ids = run
1479 .submit_nodes(vec![
1480 WorkflowNode::new(RuntimeTask::new("refine"), AgentRole::Implement).with_loop(2),
1481 ])
1482 .unwrap();
1483 assert_eq!(ids, vec![1]);
1484
1485 for k in 0..2 {
1487 assert_eq!(
1488 run.ready_batch(),
1489 vec![1],
1490 "submitted loop ready for iteration {k}"
1491 );
1492 let id = run.current_agent_id(1);
1493 assert_eq!(
1494 id,
1495 format!("wf-node1-i{k}"),
1496 "submitted loop gets per-iteration ids"
1497 );
1498 run.mark_spawned(1, &id);
1499 run.record_completion(&id, done());
1500 }
1501 assert!(
1502 run.is_complete(),
1503 "submitted loop ran its 2 iterations then finished"
1504 );
1505 }
1506
1507 #[test]
1508 fn submitted_tournament_runs_bracket_then_promotes_submitted_dependent() {
1509 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1513 use crate::types::agent::AgentRole;
1514
1515 let spec = WorkflowSpec::new(vec![WorkflowNode::new(
1516 RuntimeTask::new("root"),
1517 AgentRole::Implement,
1518 )]);
1519 let mut run = WorkflowRun::new(&spec).unwrap();
1520 let id0 = run.current_agent_id(0);
1521 run.mark_spawned(0, &id0);
1522 run.record_completion(&id0, done());
1523
1524 let ids = run
1526 .submit_nodes(vec![
1527 WorkflowNode::new(RuntimeTask::new("pick best"), AgentRole::Plan)
1528 .with_tournament(vec![RuntimeTask::new("x"), RuntimeTask::new("y")]),
1529 WorkflowNode::new(RuntimeTask::new("use winner"), AgentRole::Implement)
1530 .with_depends_on(vec![0]),
1531 ])
1532 .unwrap();
1533 assert_eq!(ids, vec![1, 2], "appended controller=1, dependent=2");
1534
1535 let entrants = spawn_round(&mut run);
1537 let entrant_nodes: Vec<usize> = entrants.iter().map(|(n, _)| *n).collect();
1538 assert_eq!(
1539 entrant_nodes,
1540 vec![3, 4],
1541 "two entrant children appended after the dependent"
1542 );
1543 for (_, id) in &entrants {
1544 run.record_completion(id, done());
1545 }
1546
1547 let r1 = spawn_round(&mut run);
1549 assert_eq!(r1.len(), 1, "one judge for two entrants");
1550 let jm = run
1551 .spawn_info(r1[0].0)
1552 .judge_match
1553 .expect("judge carries a match");
1554 assert_eq!(
1555 jm,
1556 JudgeMatch {
1557 left: node_agent_id(3),
1558 right: node_agent_id(4)
1559 }
1560 );
1561
1562 run.record_completion(&r1[0].1, judge_done(&node_agent_id(3)));
1564 assert_eq!(
1565 run.ready_batch(),
1566 vec![2],
1567 "submitted dependent unblocks after the bracket"
1568 );
1569 let last = spawn_round(&mut run);
1570 assert_eq!(last, vec![(2, node_agent_id(2))]);
1571 run.record_completion(&last[0].1, done());
1572 assert!(run.is_complete());
1573 }
1574
1575 #[test]
1576 fn submitted_classify_remaps_branch_indices_and_prunes() {
1577 use crate::orchestration::workflow::{
1581 ClassifyBranch, NodeKind, WorkflowNode, WorkflowSpec,
1582 };
1583 use crate::types::agent::AgentRole;
1584
1585 let spec = WorkflowSpec::new(vec![WorkflowNode::new(
1586 RuntimeTask::new("root"),
1587 AgentRole::Implement,
1588 )]);
1589 let mut run = WorkflowRun::new(&spec).unwrap();
1590 let id0 = run.current_agent_id(0);
1591 run.mark_spawned(0, &id0);
1592 run.record_completion(&id0, done());
1593
1594 let ids = run
1596 .submit_nodes(vec![
1597 WorkflowNode::new(RuntimeTask::new("route"), AgentRole::Plan).with_classify(vec![
1598 ClassifyBranch {
1599 label: "a".into(),
1600 nodes: vec![1],
1601 },
1602 ClassifyBranch {
1603 label: "b".into(),
1604 nodes: vec![2],
1605 },
1606 ]),
1607 WorkflowNode::new(RuntimeTask::new("branch-a"), AgentRole::Implement)
1608 .with_depends_on(vec![0]),
1609 WorkflowNode::new(RuntimeTask::new("branch-b"), AgentRole::Implement)
1610 .with_depends_on(vec![0]),
1611 ])
1612 .unwrap();
1613 assert_eq!(ids, vec![1, 2, 3], "classify=1, branchA=2, branchB=3");
1614
1615 if let NodeKind::Classify { branches } = &run.nodes[1].kind {
1617 assert_eq!(
1618 branches[0].nodes,
1619 vec![2],
1620 "branch a remapped to absolute node 2"
1621 );
1622 assert_eq!(
1623 branches[1].nodes,
1624 vec![3],
1625 "branch b remapped to absolute node 3"
1626 );
1627 } else {
1628 panic!("node 1 should be a classify node");
1629 }
1630
1631 let r = spawn_round(&mut run);
1633 assert_eq!(r, vec![(1, node_agent_id(1))], "classifier runs first");
1634 run.record_completion(
1635 &r[0].1,
1636 LoopResult {
1637 classify_branch: Some("a".into()),
1638 ..done()
1639 },
1640 );
1641
1642 assert_eq!(run.ready_batch(), vec![2], "only branch a is enabled");
1643 let failed = outcome_ids(&run, WorkflowNodeStatus::Failed);
1644 assert!(
1645 failed.contains(&node_agent_id(3)),
1646 "branch b is explicitly failed by routing"
1647 );
1648
1649 let last = spawn_round(&mut run);
1650 assert_eq!(last, vec![(2, node_agent_id(2))]);
1651 run.record_completion(&last[0].1, done());
1652 assert!(run.is_complete());
1653 let completed = outcome_ids(&run, WorkflowNodeStatus::Completed);
1654 assert!(completed.contains(&node_agent_id(1)) && completed.contains(&node_agent_id(2)));
1655 }
1656
1657 #[test]
1658 fn loop_node_iterates_with_distinct_ids_then_promotes_dependent() {
1659 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1660 use crate::types::agent::AgentRole;
1661
1662 let spec = WorkflowSpec::new(vec![
1664 WorkflowNode::new(RuntimeTask::new("refine"), AgentRole::Implement).with_loop(3),
1665 WorkflowNode::new(RuntimeTask::new("finalize"), AgentRole::Implement)
1666 .with_depends_on(vec![0]),
1667 ]);
1668 let mut run = WorkflowRun::new(&spec).unwrap();
1669
1670 for k in 0..3 {
1672 assert_eq!(
1673 run.ready_batch(),
1674 vec![0],
1675 "loop node ready for iteration {k}"
1676 );
1677 let id = run.current_agent_id(0);
1678 assert_eq!(id, format!("wf-node0-i{k}"), "distinct per-iteration id");
1679 run.mark_spawned(0, &id);
1680 assert!(!run.is_complete());
1681 let node = run.record_completion(&id, done()).unwrap();
1682 assert_eq!(node, 0);
1683 if k < 2 {
1684 assert_eq!(run.ready_batch(), vec![0]);
1686 }
1687 }
1688
1689 assert_eq!(
1691 run.ready_batch(),
1692 vec![1],
1693 "dependent unblocks only after the loop ends"
1694 );
1695 let id1 = run.current_agent_id(1);
1696 assert_eq!(id1, "wf-node1", "spawn node keeps the plain id");
1697 run.mark_spawned(1, &id1);
1698 run.record_completion(&id1, done());
1699 assert!(run.is_complete());
1700 }
1701
1702 #[test]
1703 fn synth_becomes_ready_only_after_both_workers() {
1704 let mut run = fanout2();
1705 for &n in &[0usize, 1usize] {
1706 let id = node_agent_id(n);
1707 run.mark_spawned(n, &id);
1708 }
1709 assert!(!run.batch_drained());
1710 assert_eq!(run.record_completion(&node_agent_id(0), done()), Some(0));
1712 assert!(!run.batch_drained());
1713 assert!(run.ready_batch().is_empty());
1714 assert_eq!(run.record_completion(&node_agent_id(1), done()), Some(1));
1716 assert!(run.batch_drained());
1717 assert_eq!(run.ready_batch(), vec![2]);
1718 assert!(!run.is_complete());
1719 run.mark_spawned(2, &node_agent_id(2));
1721 run.record_completion(&node_agent_id(2), done());
1722 assert!(run.is_complete());
1723 }
1724
1725 #[test]
1726 fn denied_node_skips_dependents_and_closes_outcome() {
1727 let mut run = fanout2();
1728 run.mark_spawned(0, &node_agent_id(0));
1730 run.mark_denied(1);
1731 run.record_completion(&node_agent_id(0), done());
1732 assert!(run.batch_drained());
1733 assert!(run.ready_batch().is_empty());
1734 assert!(run.is_complete());
1735 let outcomes = run.finish();
1736 assert_eq!(outcomes.len(), 3);
1737 assert_eq!(outcomes[0].status, WorkflowNodeStatus::Completed);
1738 assert_eq!(outcomes[1].status, WorkflowNodeStatus::Failed);
1739 assert_eq!(
1740 outcomes[2].status,
1741 WorkflowNodeStatus::SkippedUpstreamFailed
1742 );
1743 }
1744
1745 #[test]
1746 fn terminal_mapping_and_dependency_policies_are_explicit() {
1747 use crate::orchestration::workflow::{DependencyPolicy, WorkflowNode, WorkflowSpec};
1748 use crate::types::agent::AgentRole;
1749
1750 let cases = [
1751 (TerminationReason::Completed, WorkflowNodeStatus::Completed),
1752 (
1753 TerminationReason::MaxTurns,
1754 WorkflowNodeStatus::CompletedPartial,
1755 ),
1756 (
1757 TerminationReason::TokenBudget,
1758 WorkflowNodeStatus::CompletedPartial,
1759 ),
1760 (
1761 TerminationReason::Timeout,
1762 WorkflowNodeStatus::CompletedPartial,
1763 ),
1764 (
1765 TerminationReason::ContextOverflow,
1766 WorkflowNodeStatus::CompletedPartial,
1767 ),
1768 (
1769 TerminationReason::NoProgress,
1770 WorkflowNodeStatus::CompletedPartial,
1771 ),
1772 (
1773 TerminationReason::MilestoneExceeded,
1774 WorkflowNodeStatus::CompletedPartial,
1775 ),
1776 (TerminationReason::Error, WorkflowNodeStatus::Failed),
1777 (TerminationReason::UserAbort, WorkflowNodeStatus::Failed),
1778 ];
1779 for (termination, expected) in cases {
1780 let spec = WorkflowSpec::new(vec![WorkflowNode::new(
1781 RuntimeTask::new("node"),
1782 AgentRole::Implement,
1783 )]);
1784 let mut run = WorkflowRun::new(&spec).unwrap();
1785 run.mark_spawned(0, "wf-node0");
1786 run.record_completion("wf-node0", terminated(termination));
1787 let outcome = run.finish().remove(0);
1788 assert_eq!(outcome.status, expected);
1789 assert_eq!(outcome.termination, Some(termination));
1790 }
1791
1792 let spec = WorkflowSpec::new(vec![
1793 WorkflowNode::new(RuntimeTask::new("upstream"), AgentRole::Implement),
1794 WorkflowNode::new(RuntimeTask::new("strict"), AgentRole::Implement)
1795 .with_depends_on(vec![0]),
1796 WorkflowNode::new(RuntimeTask::new("partial-ok"), AgentRole::Implement)
1797 .with_depends_on(vec![0])
1798 .with_dependency_policy(DependencyPolicy::AcceptPartial),
1799 ]);
1800 let mut run = WorkflowRun::new(&spec).unwrap();
1801 run.mark_spawned(0, "wf-node0");
1802 run.record_completion("wf-node0", terminated(TerminationReason::Timeout));
1803 assert_eq!(run.ready_batch(), vec![2]);
1804 assert_eq!(
1805 run.node_outcomes()[1].status,
1806 WorkflowNodeStatus::SkippedUpstreamFailed
1807 );
1808 }
1809
1810 #[test]
1811 fn all_terminal_and_optional_have_distinct_waiting_semantics() {
1812 use crate::orchestration::workflow::{DependencyPolicy, WorkflowNode, WorkflowSpec};
1813 use crate::types::agent::AgentRole;
1814
1815 let spec = WorkflowSpec::new(vec![
1816 WorkflowNode::new(RuntimeTask::new("upstream"), AgentRole::Implement),
1817 WorkflowNode::new(RuntimeTask::new("cleanup"), AgentRole::Implement)
1818 .with_depends_on(vec![0])
1819 .with_dependency_policy(DependencyPolicy::AllTerminal),
1820 WorkflowNode::new(RuntimeTask::new("best-effort"), AgentRole::Implement)
1821 .with_depends_on(vec![0])
1822 .with_dependency_policy(DependencyPolicy::Optional),
1823 ]);
1824 let mut run = WorkflowRun::new(&spec).unwrap();
1825 assert_eq!(run.ready_batch(), vec![0, 2]);
1826 run.mark_spawned(0, "wf-node0");
1827 run.record_completion("wf-node0", terminated(TerminationReason::Error));
1828 assert!(run.ready_batch().contains(&1));
1829 }
1830
1831 #[test]
1832 fn loop_terminal_result_is_not_retried() {
1833 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1834 use crate::types::agent::AgentRole;
1835
1836 let spec = WorkflowSpec::new(vec![
1837 WorkflowNode::new(RuntimeTask::new("loop"), AgentRole::Implement).with_loop(3),
1838 ]);
1839 let mut run = WorkflowRun::new(&spec).unwrap();
1840 run.mark_spawned(0, "wf-node0-i0");
1841 run.record_completion("wf-node0-i0", terminated(TerminationReason::Timeout));
1842 assert!(run.ready_batch().is_empty());
1843 assert_eq!(run.finish()[0].status, WorkflowNodeStatus::CompletedPartial);
1844 }
1845
1846 #[test]
1847 fn manifest_preserves_node_isolation_and_inheritance() {
1848 let run = fanout2();
1849 let m = run.manifest_for(0);
1850 assert_eq!(m.agent_id.as_str(), "wf-node0");
1851 assert_eq!(m.isolation, crate::types::agent::AgentIsolation::ReadOnly);
1853 assert_eq!(
1854 m.context_inheritance,
1855 crate::types::agent::ContextInheritance::SystemOnly
1856 );
1857 }
1858
1859 #[test]
1860 fn unknown_agent_completion_is_none() {
1861 let mut run = fanout2();
1862 assert_eq!(run.record_completion("not-a-node", done()), None);
1863 }
1864
1865 #[test]
1866 fn spawn_info_carries_model_hint_and_trust() {
1867 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1868 use crate::types::agent::AgentRole;
1869
1870 let spec = WorkflowSpec::new(vec![
1871 WorkflowNode::new(RuntimeTask::new("read tickets"), AgentRole::Explore)
1872 .quarantined()
1873 .with_model_hint("haiku"),
1874 WorkflowNode::new(RuntimeTask::new("act"), AgentRole::Implement),
1875 ]);
1876 let run = WorkflowRun::new(&spec).unwrap();
1877
1878 let q = run.spawn_info(0);
1880 assert_eq!(q.trust, "quarantined");
1881 assert_eq!(q.model_hint.as_deref(), Some("haiku"));
1882 let t = run.spawn_info(1);
1884 assert_eq!(t.trust, "trusted");
1885 assert_eq!(t.model_hint, None);
1886 }
1887
1888 #[test]
1889 fn spawn_info_carries_loop_and_classify_hints() {
1890 use crate::orchestration::workflow::{ClassifyBranch, WorkflowNode, WorkflowSpec};
1891 use crate::types::agent::AgentRole;
1892
1893 let spec = WorkflowSpec::new(vec![
1894 WorkflowNode::new(RuntimeTask::new("refine"), AgentRole::Implement).with_loop(3),
1896 WorkflowNode::new(RuntimeTask::new("route"), AgentRole::Plan).with_classify(vec![
1898 ClassifyBranch {
1899 label: "bug".into(),
1900 nodes: vec![],
1901 },
1902 ClassifyBranch {
1903 label: "feature".into(),
1904 nodes: vec![],
1905 },
1906 ]),
1907 WorkflowNode::new(RuntimeTask::new("act"), AgentRole::Implement),
1909 ]);
1910 let run = WorkflowRun::new(&spec).unwrap();
1911
1912 let l = run.spawn_info(0);
1913 assert_eq!(l.loop_max_iters, Some(3));
1914 assert!(l.classify_labels.is_empty());
1915 assert_eq!(l.token_budget, None, "no token budget unless set");
1916
1917 let c = run.spawn_info(1);
1918 assert_eq!(
1919 c.classify_labels,
1920 vec!["bug".to_string(), "feature".to_string()]
1921 );
1922 assert_eq!(c.loop_max_iters, None);
1923
1924 let s = run.spawn_info(2);
1925 assert_eq!(s.loop_max_iters, None);
1926 assert!(s.classify_labels.is_empty());
1927 }
1928
1929 #[test]
1930 fn spawn_info_carries_token_budget() {
1931 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1932 use crate::types::agent::AgentRole;
1933
1934 let spec = WorkflowSpec::new(vec![
1935 WorkflowNode::new(RuntimeTask::new("expensive"), AgentRole::Implement)
1936 .with_token_budget(10_000),
1937 WorkflowNode::new(RuntimeTask::new("plain"), AgentRole::Implement),
1938 ]);
1939 let run = WorkflowRun::new(&spec).unwrap();
1940 assert_eq!(run.spawn_info(0).token_budget, Some(10_000));
1941 assert_eq!(run.spawn_info(1).token_budget, None);
1942 }
1943
1944 use crate::orchestration::workflow::{NodeKind, WorkflowNode, WorkflowSpec};
1947 use crate::types::agent::AgentRole;
1948
1949 #[test]
1953 fn tournament_runs_bracket_then_promotes_dependent() {
1954 let spec = WorkflowSpec::new(vec![
1955 WorkflowNode::new(RuntimeTask::new("pick the best ad"), AgentRole::Plan)
1956 .with_tournament(vec![
1957 RuntimeTask::new("ad A"),
1958 RuntimeTask::new("ad B"),
1959 RuntimeTask::new("ad C"),
1960 RuntimeTask::new("ad D"),
1961 ]),
1962 WorkflowNode::new(RuntimeTask::new("ship the winner"), AgentRole::Implement)
1963 .with_depends_on(vec![0]),
1964 ]);
1965 let mut run = WorkflowRun::new(&spec).unwrap();
1966
1967 let entrants = spawn_round(&mut run);
1970 let entrant_nodes: Vec<usize> = entrants.iter().map(|(n, _)| *n).collect();
1971 assert_eq!(
1972 entrant_nodes,
1973 vec![2, 3, 4, 5],
1974 "4 entrant children, no controller spawn"
1975 );
1976 assert!(
1977 run.spawn_info(2).judge_match.is_none(),
1978 "entrants are not judges"
1979 );
1980 assert!(!run.is_complete());
1981
1982 for (i, (node, id)) in entrants.iter().enumerate() {
1984 run.record_completion(id, done());
1985 if i < 3 {
1986 assert!(
1987 run.ready_batch().is_empty(),
1988 "no judges until every entrant is in"
1989 );
1990 }
1991 let _ = node;
1992 }
1993
1994 let r1 = spawn_round(&mut run);
1996 assert_eq!(r1.len(), 2, "two round-1 judges");
1997 let jm0 = run
1998 .spawn_info(r1[0].0)
1999 .judge_match
2000 .expect("judge carries a match");
2001 assert_eq!(
2002 jm0,
2003 JudgeMatch {
2004 left: node_agent_id(2),
2005 right: node_agent_id(3)
2006 }
2007 );
2008 let jm1 = run
2009 .spawn_info(r1[1].0)
2010 .judge_match
2011 .expect("judge carries a match");
2012 assert_eq!(
2013 jm1,
2014 JudgeMatch {
2015 left: node_agent_id(4),
2016 right: node_agent_id(5)
2017 }
2018 );
2019
2020 run.record_completion(&r1[0].1, judge_done(&node_agent_id(2)));
2022 run.record_completion(&r1[1].1, judge_done(&node_agent_id(4)));
2023 assert!(
2024 run.ready_batch().iter().all(|&n| n != 1),
2025 "dependent gated until the final"
2026 );
2027
2028 let r2 = spawn_round(&mut run);
2030 assert_eq!(r2.len(), 1, "one final judge");
2031 let jmf = run
2032 .spawn_info(r2[0].0)
2033 .judge_match
2034 .expect("final judge carries a match");
2035 assert_eq!(
2036 jmf,
2037 JudgeMatch {
2038 left: node_agent_id(2),
2039 right: node_agent_id(4)
2040 }
2041 );
2042
2043 run.record_completion(&r2[0].1, judge_done(&node_agent_id(4)));
2045 let winner = run
2046 .graph
2047 .get(0)
2048 .and_then(|n| n.result.as_ref())
2049 .and_then(|r| r.tournament_winner.clone());
2050 assert_eq!(
2051 winner.as_deref(),
2052 Some(node_agent_id(4).as_str()),
2053 "champion recorded"
2054 );
2055 assert_eq!(
2056 run.ready_batch(),
2057 vec![1],
2058 "dependent unblocks only after the bracket resolves"
2059 );
2060
2061 let last = spawn_round(&mut run);
2063 assert_eq!(last, vec![(1, node_agent_id(1))]);
2064 run.record_completion(&last[0].1, done());
2065 assert!(run.is_complete());
2066 }
2067
2068 #[test]
2071 fn tournament_with_bye_resolves() {
2072 let spec = WorkflowSpec::new(vec![
2073 WorkflowNode::new(RuntimeTask::new("rank"), AgentRole::Plan).with_tournament(vec![
2074 RuntimeTask::new("x"),
2075 RuntimeTask::new("y"),
2076 RuntimeTask::new("z"),
2077 ]),
2078 ]);
2079 let mut run = WorkflowRun::new(&spec).unwrap();
2080
2081 let entrants = spawn_round(&mut run); assert_eq!(entrants.len(), 3);
2083 for (_, id) in &entrants {
2084 run.record_completion(id, done());
2085 }
2086 let r1 = spawn_round(&mut run);
2088 assert_eq!(r1.len(), 1, "one match, one bye");
2089 run.record_completion(&r1[0].1, judge_done(&node_agent_id(1)));
2090 let r2 = spawn_round(&mut run);
2092 assert_eq!(r2.len(), 1);
2093 let jm = run.spawn_info(r2[0].0).judge_match.unwrap();
2094 assert_eq!(
2095 jm,
2096 JudgeMatch {
2097 left: node_agent_id(1),
2098 right: node_agent_id(3)
2099 }
2100 );
2101 run.record_completion(&r2[0].1, judge_done(&node_agent_id(3)));
2102 let winner = run
2103 .graph
2104 .get(0)
2105 .and_then(|n| n.result.as_ref())
2106 .and_then(|r| r.tournament_winner.clone());
2107 assert_eq!(winner.as_deref(), Some(node_agent_id(3).as_str()));
2108 assert!(run.is_complete());
2109 }
2110
2111 #[test]
2114 fn tournament_children_inherit_controller_trust() {
2115 let spec = WorkflowSpec::new(vec![
2116 WorkflowNode::new(RuntimeTask::new("judge untrusted inputs"), AgentRole::Plan)
2117 .quarantined()
2118 .with_tournament(vec![RuntimeTask::new("a"), RuntimeTask::new("b")]),
2119 ]);
2120 let mut run = WorkflowRun::new(&spec).unwrap();
2121
2122 let entrants = spawn_round(&mut run);
2123 for (node, _) in &entrants {
2124 assert_eq!(
2125 run.spawn_info(*node).trust,
2126 "quarantined",
2127 "entrant inherits quarantine"
2128 );
2129 assert!(
2130 !run.quarantine_violation(*node),
2131 "read-only entrant is quarantine-clean"
2132 );
2133 }
2134 for (_, id) in &entrants {
2135 run.record_completion(id, done());
2136 }
2137 let r1 = spawn_round(&mut run);
2138 assert_eq!(
2139 run.spawn_info(r1[0].0).trust,
2140 "quarantined",
2141 "judge inherits quarantine"
2142 );
2143 assert!(!run.quarantine_violation(r1[0].0));
2144 }
2145
2146 #[test]
2149 fn tournament_controller_never_spawns_itself() {
2150 let spec = WorkflowSpec::new(vec![
2151 WorkflowNode::new(RuntimeTask::new("c"), AgentRole::Plan)
2152 .with_tournament(vec![RuntimeTask::new("a"), RuntimeTask::new("b")]),
2153 ]);
2154 let mut run = WorkflowRun::new(&spec).unwrap();
2155 assert!(matches!(run.nodes[0].kind, NodeKind::Tournament { .. }));
2156 let first = spawn_round(&mut run);
2157 assert!(
2158 first.iter().all(|(n, _)| *n != 0),
2159 "controller node 0 never spawns directly"
2160 );
2161 }
2162
2163 #[test]
2164 fn errored_tournament_child_is_failed_and_no_champion_fails_controller() {
2165 let spec = WorkflowSpec::new(vec![
2168 WorkflowNode::new(RuntimeTask::new("pick"), AgentRole::Plan)
2169 .with_tournament(vec![RuntimeTask::new("x"), RuntimeTask::new("y")]),
2170 WorkflowNode::new(RuntimeTask::new("use winner"), AgentRole::Implement)
2171 .with_depends_on(vec![0]),
2172 ]);
2173 let mut run = WorkflowRun::new(&spec).unwrap();
2174 let entrants = spawn_round(&mut run);
2175 assert_eq!(entrants.len(), 2);
2176 run.record_completion(&entrants[0].1, done());
2177 run.record_completion(
2178 &entrants[1].1,
2179 LoopResult {
2180 termination: TerminationReason::Error,
2181 ..done()
2182 },
2183 );
2184 let judges = spawn_round(&mut run);
2186 assert_eq!(judges.len(), 1, "one match for two entrants");
2187 run.record_completion(&judges[0].1, done()); let failed = outcome_ids(&run, WorkflowNodeStatus::Failed);
2189 assert!(
2190 failed.contains(&entrants[1].1),
2191 "errored entrant reported failed"
2192 );
2193 assert!(
2194 failed.contains(&"wf-node0".to_string()),
2195 "no-champion controller failed"
2196 );
2197 assert!(
2198 !run.ready_batch().contains(&1),
2199 "dependent of the failed controller starves"
2200 );
2201 }
2202
2203 #[test]
2204 fn submitted_tournament_with_one_entrant_is_rejected_atomically() {
2205 let mut run = fanout2();
2206 let before = run.len();
2207 let controller = WorkflowNode::new(RuntimeTask::new("pick"), AgentRole::Plan)
2208 .with_tournament(vec![RuntimeTask::new("only")]);
2209 assert!(run.submit_nodes(vec![controller]).is_err());
2210 assert_eq!(run.len(), before);
2211 }
2212
2213 #[test]
2214 fn submitted_classify_branch_without_classifier_dependency_is_rejected() {
2215 let mut run = fanout2();
2216 let before = run.len();
2217 let classifier = WorkflowNode::new(RuntimeTask::new("route"), AgentRole::Plan)
2218 .with_classify(vec![ClassifyBranch {
2219 label: "a".to_string(),
2220 nodes: vec![1],
2221 }]);
2222 let branch = WorkflowNode::new(RuntimeTask::new("on a"), AgentRole::Implement);
2223 assert!(run.submit_nodes(vec![classifier, branch]).is_err());
2224 assert_eq!(run.len(), before);
2225 }
2226
2227 #[test]
2228 fn submitted_zero_iter_loop_is_rejected() {
2229 let mut run = fanout2();
2230 let before = run.len();
2231 let mut node = WorkflowNode::new(RuntimeTask::new("once"), AgentRole::Implement);
2232 node.kind = NodeKind::Loop { max_iters: 0 };
2233 assert!(run.submit_nodes(vec![node]).is_err());
2234 assert_eq!(run.len(), before);
2235 }
2236
2237 #[test]
2238 fn spawn_info_carries_dep_ids_and_per_node_caps() {
2239 let spec = WorkflowSpec::new(vec![
2242 WorkflowNode::new(RuntimeTask::new("w"), AgentRole::Explore),
2243 WorkflowNode::new(RuntimeTask::new("synth"), AgentRole::Plan)
2244 .with_depends_on(vec![0])
2245 .with_max_turns(4)
2246 .with_max_wall_ms(30_000),
2247 ]);
2248 let run = WorkflowRun::new(&spec).unwrap();
2249 let info = run.spawn_info(1);
2250 assert_eq!(info.input_agent_ids, vec!["wf-node0"]);
2251 assert_eq!(info.max_turns, Some(4));
2252 assert_eq!(info.max_wall_ms, Some(30_000));
2253 assert!(info.reducer.is_none(), "plain node stays non-reduce");
2254 let root = run.spawn_info(0);
2255 assert!(root.input_agent_ids.is_empty());
2256 assert_eq!(root.max_turns, None);
2257 }
2258
2259 #[test]
2268 fn f1_critical_path_node_is_scheduled_before_a_lower_id_leaf() {
2269 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
2270 use crate::scheduler::policy::SchedulerPolicyConfig;
2271 use crate::types::agent::AgentRole;
2272
2273 let spec = WorkflowSpec::new(vec![
2276 WorkflowNode::new(RuntimeTask::new("leaf"), AgentRole::Implement),
2277 WorkflowNode::new(RuntimeTask::new("chain-root"), AgentRole::Implement),
2278 WorkflowNode::new(RuntimeTask::new("mid"), AgentRole::Implement)
2279 .with_depends_on(vec![1]),
2280 WorkflowNode::new(RuntimeTask::new("tail"), AgentRole::Implement)
2281 .with_depends_on(vec![2]),
2282 ]);
2283 let mut run = WorkflowRun::new(&spec).unwrap();
2284 run.set_scheduler_policy(SchedulerPolicyConfig::default());
2285
2286 assert_eq!(
2287 run.ready_batch(),
2288 vec![1, 0],
2289 "the deeper critical path (node 1) outranks the lower-id leaf (node 0)"
2290 );
2291 }
2292
2293 #[test]
2297 fn f2_rearming_loop_does_not_starve_an_independent_node() {
2298 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
2299 use crate::scheduler::policy::SchedulerPolicyConfig;
2300 use crate::types::agent::AgentRole;
2301
2302 let spec = WorkflowSpec::new(vec![
2304 WorkflowNode::new(RuntimeTask::new("loop"), AgentRole::Implement).with_loop(5),
2305 WorkflowNode::new(RuntimeTask::new("independent"), AgentRole::Implement),
2306 ]);
2307 let mut run = WorkflowRun::new(&spec).unwrap();
2308 run.set_scheduler_policy(SchedulerPolicyConfig::default());
2309
2310 let first = run.ready_batch();
2314 assert_eq!(first[0], 0, "loop takes the first slot on the initial tie");
2315 let id = run.current_agent_id(0);
2316 run.mark_spawned(0, &id);
2317 run.record_completion(&id, done()); assert_eq!(
2320 run.ready_batch()[0],
2321 1,
2322 "the independent node runs before the loop's second iteration (no starvation)"
2323 );
2324 }
2325
2326 #[test]
2329 fn f3_failure_and_partial_propagate_transitively_by_policy() {
2330 use crate::orchestration::workflow::{DependencyPolicy, WorkflowNode, WorkflowSpec};
2331 use crate::types::agent::AgentRole;
2332
2333 let spec = WorkflowSpec::new(vec![
2336 WorkflowNode::new(RuntimeTask::new("a"), AgentRole::Implement),
2337 WorkflowNode::new(RuntimeTask::new("b"), AgentRole::Implement).with_depends_on(vec![0]),
2338 WorkflowNode::new(RuntimeTask::new("c"), AgentRole::Implement).with_depends_on(vec![1]),
2339 ]);
2340 let mut run = WorkflowRun::new(&spec).unwrap();
2341 run.mark_spawned(0, "wf-node0");
2342 run.record_completion("wf-node0", terminated(TerminationReason::Error));
2343 let outcomes = run.finish();
2344 assert_eq!(outcomes[0].status, WorkflowNodeStatus::Failed);
2345 assert_eq!(
2346 outcomes[1].status,
2347 WorkflowNodeStatus::SkippedUpstreamFailed
2348 );
2349 assert_eq!(
2350 outcomes[2].status,
2351 WorkflowNodeStatus::SkippedUpstreamFailed,
2352 "the failure propagates through the whole chain"
2353 );
2354
2355 let spec = WorkflowSpec::new(vec![
2358 WorkflowNode::new(RuntimeTask::new("up"), AgentRole::Implement),
2359 WorkflowNode::new(RuntimeTask::new("strict"), AgentRole::Implement)
2360 .with_depends_on(vec![0]),
2361 WorkflowNode::new(RuntimeTask::new("lenient"), AgentRole::Implement)
2362 .with_depends_on(vec![0])
2363 .with_dependency_policy(DependencyPolicy::AcceptPartial),
2364 ]);
2365 let mut run = WorkflowRun::new(&spec).unwrap();
2366 run.mark_spawned(0, "wf-node0");
2367 run.record_completion("wf-node0", terminated(TerminationReason::Timeout)); assert_eq!(
2369 run.ready_batch(),
2370 vec![2],
2371 "only the AcceptPartial dependent runs"
2372 );
2373 assert_eq!(
2374 run.node_outcomes()[1].status,
2375 WorkflowNodeStatus::SkippedUpstreamFailed,
2376 "the AllSuccess dependent is skipped behind the partial upstream"
2377 );
2378 }
2379
2380 struct Lcg(u64);
2390 impl Lcg {
2391 fn below(&mut self, n: u64) -> u64 {
2392 self.0 = self
2393 .0
2394 .wrapping_mul(6364136223846793005)
2395 .wrapping_add(1442695040888963407);
2396 (self.0 ^ (self.0 >> 33)) % n.max(1)
2397 }
2398 }
2399
2400 #[test]
2401 fn finish_closes_every_node_into_exactly_one_terminal_state_over_random_dags() {
2402 use crate::orchestration::workflow::{DependencyPolicy, WorkflowNode, WorkflowSpec};
2403 use crate::types::agent::AgentRole;
2404 use std::collections::BTreeSet;
2405
2406 let terminations = [
2407 TerminationReason::Completed,
2408 TerminationReason::MaxTurns,
2409 TerminationReason::TokenBudget,
2410 TerminationReason::Timeout,
2411 TerminationReason::ContextOverflow,
2412 TerminationReason::NoProgress,
2413 TerminationReason::MilestoneExceeded,
2414 TerminationReason::Error,
2415 TerminationReason::UserAbort,
2416 ];
2417 let policies = [
2418 DependencyPolicy::AllSuccess,
2419 DependencyPolicy::AcceptPartial,
2420 DependencyPolicy::AllTerminal,
2421 DependencyPolicy::Optional,
2422 ];
2423
2424 for seed in 0..300u64 {
2425 let mut rng = Lcg(seed.wrapping_mul(0x9E3779B97F4A7C15).wrapping_add(1));
2426 let n = 2 + rng.below(7) as usize;
2427
2428 let mut nodes = Vec::new();
2431 for i in 0..n {
2432 let mut deps = Vec::new();
2433 for j in 0..i {
2434 if rng.below(3) == 0 {
2435 deps.push(j);
2436 }
2437 }
2438 let policy = policies[rng.below(policies.len() as u64) as usize];
2439 nodes.push(
2440 WorkflowNode::new(RuntimeTask::new(format!("n{i}")), AgentRole::Implement)
2441 .with_depends_on(deps)
2442 .with_dependency_policy(policy),
2443 );
2444 }
2445 let spec = WorkflowSpec::new(nodes);
2446 let mut run = WorkflowRun::new(&spec).unwrap();
2447
2448 for _ in 0..(n * 4 + 4) {
2451 let ready = run.ready_batch();
2452 if ready.is_empty() {
2453 break;
2454 }
2455 for node in ready {
2456 let agent = node_agent_id(node);
2457 run.mark_spawned(node, &agent);
2458 if rng.below(5) == 0 {
2459 run.mark_denied(node);
2460 } else {
2461 let termination =
2462 terminations[rng.below(terminations.len() as u64) as usize];
2463 run.record_completion(&agent, terminated(termination));
2464 }
2465 }
2466 }
2467
2468 let outcomes = run.finish();
2469 assert_eq!(outcomes.len(), n, "seed {seed}: every node has an outcome");
2471 let ids: BTreeSet<String> = outcomes.iter().map(|o| o.node_id.clone()).collect();
2472 assert_eq!(ids.len(), n, "seed {seed}: node ids are unique");
2473 for node in 0..n {
2474 assert!(
2475 ids.contains(&node_agent_id(node)),
2476 "seed {seed}: node {node} is in the closed outcome set"
2477 );
2478 }
2479 for outcome in &outcomes {
2480 assert!(
2481 matches!(
2482 outcome.status,
2483 WorkflowNodeStatus::Completed
2484 | WorkflowNodeStatus::CompletedPartial
2485 | WorkflowNodeStatus::Failed
2486 | WorkflowNodeStatus::SkippedUpstreamFailed
2487 ),
2488 "seed {seed}: {} is terminal",
2489 outcome.node_id
2490 );
2491 }
2492 }
2493 }
2494}