1use std::collections::HashMap;
13
14use serde::{Deserialize, Serialize};
15
16use super::{DependencyPolicy, NodeKind, NodeTrust, WorkflowNode, WorkflowSpec};
17use crate::orchestration::task_graph::{SchedulingFactors, TaskGraph, TaskStatus};
18use crate::orchestration::tournament::{EntrantId, Match, Tournament, TournamentAction};
19use crate::types::agent::{AgentIsolation, AgentRole, ContextInheritance, IsolationManifest};
20use crate::types::error::DeepStrikeError;
21use crate::types::error::Result;
22use crate::types::result::{LoopResult, TerminationReason};
23
24#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
25pub struct WorkflowSubmissionError {
26 pub node_index: usize,
27 pub reason: String,
28}
29
30impl std::fmt::Display for WorkflowSubmissionError {
31 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
32 write!(f, "node {}: {}", self.node_index, self.reason)
33 }
34}
35
36impl std::error::Error for WorkflowSubmissionError {}
37
38#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
40#[serde(rename_all = "snake_case")]
41pub enum WorkflowNodeStatus {
42 Completed,
43 CompletedPartial,
44 Failed,
45 SkippedUpstreamFailed,
46}
47
48#[derive(Debug, Clone, Serialize, Deserialize)]
49pub struct WorkflowNodeOutcome {
50 pub node_id: String,
51 pub status: WorkflowNodeStatus,
52 #[serde(skip_serializing_if = "Option::is_none")]
53 pub termination: Option<TerminationReason>,
54 #[serde(skip_serializing_if = "Option::is_none")]
55 pub output: Option<crate::types::message::Message>,
56}
57
58pub fn node_agent_id(node: usize) -> String {
60 format!("wf-node{node}")
61}
62
63#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
68pub struct WorkflowSpawnInfo {
69 pub agent_id: String,
70 pub goal: String,
71 pub role: String,
72 pub isolation: String,
73 pub context_inheritance: String,
74 #[serde(default, skip_serializing_if = "Option::is_none")]
75 pub model_hint: Option<String>,
76 #[serde(default = "default_trust")]
79 pub trust: String,
80 #[serde(default, skip_serializing_if = "Option::is_none")]
84 pub output_schema: Option<serde_json::Value>,
85 #[serde(default, skip_serializing_if = "Option::is_none")]
89 pub reducer: Option<String>,
90 #[serde(default, skip_serializing_if = "Vec::is_empty")]
93 pub input_agent_ids: Vec<String>,
94 #[serde(default, skip_serializing_if = "Option::is_none")]
99 pub judge_match: Option<JudgeMatch>,
100 #[serde(default, skip_serializing_if = "Option::is_none")]
106 pub loop_max_iters: Option<usize>,
107 #[serde(default, skip_serializing_if = "Vec::is_empty")]
112 pub classify_labels: Vec<String>,
113 #[serde(default, skip_serializing_if = "Option::is_none")]
117 pub token_budget: Option<u64>,
118 #[serde(default, skip_serializing_if = "Option::is_none")]
120 pub max_turns: Option<u32>,
121 #[serde(default, skip_serializing_if = "Option::is_none")]
123 pub max_wall_ms: Option<u64>,
124}
125
126fn default_trust() -> String {
127 "trusted".to_string()
128}
129
130#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
134pub struct JudgeMatch {
135 pub left: String,
136 pub right: String,
137}
138
139#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
145pub struct WorkflowBudget {
146 pub nodes_used: usize,
148 #[serde(default, skip_serializing_if = "Option::is_none")]
150 pub nodes_max: Option<usize>,
151 #[serde(default, skip_serializing_if = "Option::is_none")]
154 pub nodes_remaining: Option<usize>,
155 pub running_subagents: usize,
157 #[serde(default, skip_serializing_if = "Option::is_none")]
159 pub max_concurrent_subagents: Option<usize>,
160 #[serde(default, skip_serializing_if = "Option::is_none")]
163 pub concurrency_remaining: Option<usize>,
164 #[serde(default)]
167 pub tokens_used: u64,
168 #[serde(default, skip_serializing_if = "Option::is_none")]
170 pub tokens_max: Option<u64>,
171 #[serde(default, skip_serializing_if = "Option::is_none")]
175 pub tokens_remaining: Option<u64>,
176}
177
178fn role_label(role: AgentRole) -> &'static str {
179 match role {
180 AgentRole::Explore => "explore",
181 AgentRole::Plan => "plan",
182 AgentRole::Implement => "implement",
183 AgentRole::Verify => "verify",
184 AgentRole::Custom => "custom",
185 }
186}
187
188fn isolation_label(isolation: AgentIsolation) -> &'static str {
189 match isolation {
190 AgentIsolation::Shared => "shared",
191 AgentIsolation::ReadOnly => "read_only",
192 AgentIsolation::Worktree => "worktree",
193 AgentIsolation::Remote => "remote",
194 }
195}
196
197fn inheritance_label(inheritance: ContextInheritance) -> &'static str {
198 match inheritance {
199 ContextInheritance::None => "none",
200 ContextInheritance::SystemOnly => "system_only",
201 ContextInheritance::Full => "full",
202 }
203}
204
205fn trust_label(trust: NodeTrust) -> &'static str {
206 match trust {
207 NodeTrust::Trusted => "trusted",
208 NodeTrust::Quarantined => "quarantined",
209 }
210}
211
212struct TournamentState {
216 entrant_nodes: Vec<usize>,
218 entrants_remaining: usize,
220 bracket: Option<Tournament>,
222 judge_nodes: Vec<usize>,
224 judge_winners: Vec<Option<EntrantId>>,
226 judges_remaining: usize,
228}
229
230pub struct WorkflowRun {
232 graph: TaskGraph,
233 nodes: Vec<WorkflowNode>,
234 scheduler_policy: crate::scheduler::policy::SchedulerPolicyConfig,
235 node_of_agent: HashMap<String, usize>,
237 iter_counts: HashMap<usize, usize>,
240 tournaments: HashMap<usize, TournamentState>,
242 child_controller: HashMap<usize, usize>,
244 judge_matches: HashMap<usize, JudgeMatch>,
246}
247
248#[derive(Debug, Clone)]
253pub(crate) struct WorkflowRuntimeNodeState {
254 pub node: WorkflowNode,
255 pub status: TaskStatus,
256 pub result: Option<LoopResult>,
257 pub active_agent_id: Option<String>,
258 pub iterations_completed: usize,
259}
260
261impl WorkflowRun {
262 pub fn new(spec: &WorkflowSpec) -> Result<Self> {
264 let mut run = Self {
265 graph: spec.validate()?,
266 nodes: spec.nodes.clone(),
267 scheduler_policy: crate::scheduler::policy::SchedulerPolicyConfig::default(),
268 node_of_agent: HashMap::new(),
269 iter_counts: HashMap::new(),
270 tournaments: HashMap::new(),
271 child_controller: HashMap::new(),
272 judge_matches: HashMap::new(),
273 };
274 run.refresh_scheduling();
275 run.resolve_dependency_outcomes();
276 Ok(run)
277 }
278
279 pub(crate) fn restore_from_checkpoint(
281 spec: &WorkflowSpec,
282 states: &[WorkflowRuntimeNodeState],
283 ) -> Result<Self> {
284 let mut run = Self::new(spec)?;
285 let graph_states: Vec<(TaskStatus, Option<LoopResult>)> = states
286 .iter()
287 .map(|state| (state.status, state.result.clone()))
288 .collect();
289 run.graph
290 .restore_runtime_state(&graph_states)
291 .map_err(DeepStrikeError::InvalidConfig)?;
292
293 for (node, state) in states.iter().enumerate() {
294 if state.iterations_completed > 0 {
295 run.iter_counts.insert(node, state.iterations_completed);
296 }
297 match (state.status, state.active_agent_id.as_deref()) {
298 (TaskStatus::Running, Some(agent_id)) => {
299 run.node_of_agent.insert(agent_id.to_string(), node);
300 }
301 (TaskStatus::Running, None) => {
302 return Err(DeepStrikeError::InvalidConfig(format!(
303 "running workflow node {node} carries no active agent id"
304 )));
305 }
306 (_, Some(agent_id)) => {
307 return Err(DeepStrikeError::InvalidConfig(format!(
308 "non-running workflow node {node} carries active agent id {agent_id:?}"
309 )));
310 }
311 (_, None) => {}
312 }
313 }
314 run.refresh_scheduling();
315 Ok(run)
316 }
317
318 pub fn ready_batch(&mut self) -> Vec<usize> {
320 self.graph.ready_tasks()
321 }
322
323 pub fn set_scheduler_policy(
324 &mut self,
325 policy: crate::scheduler::policy::SchedulerPolicyConfig,
326 ) {
327 self.scheduler_policy = policy;
328 self.refresh_scheduling();
329 }
330
331 fn refresh_scheduling(&mut self) {
332 let token_costs: Vec<u64> = self
333 .nodes
334 .iter()
335 .map(|node| u64::from(node.token_budget.unwrap_or(0)))
336 .collect();
337 let factors: Vec<SchedulingFactors> = self
338 .nodes
339 .iter()
340 .map(|node| node.scheduling_factors)
341 .collect();
342 self.graph
343 .configure_scheduling_with_factors(self.scheduler_policy, &token_costs, &factors);
344 }
345
346 pub fn current_agent_id(&self, node: usize) -> String {
351 match self.nodes[node].kind {
352 NodeKind::Loop { .. } => {
353 let k = self.iter_counts.get(&node).copied().unwrap_or(0);
354 format!("{}-i{k}", node_agent_id(node))
355 }
356 NodeKind::Spawn
360 | NodeKind::Classify { .. }
361 | NodeKind::Tournament { .. }
362 | NodeKind::Reduce { .. } => node_agent_id(node),
363 }
364 }
365
366 pub fn manifest_for(&self, node: usize) -> IsolationManifest {
372 let n = &self.nodes[node];
373 IsolationManifest {
374 agent_id: self.current_agent_id(node).into(),
375 role: n.role,
376 isolation: n.isolation,
377 context_inheritance: n.context_inheritance,
378 permitted_capability_ids: Vec::new(),
379 requested_capabilities: n.requested_capabilities.clone(),
380 requested_budget: n.requested_budget,
381 }
382 }
383
384 pub fn quarantine_violation(&self, node: usize) -> bool {
390 let n = &self.nodes[node];
391 matches!(n.trust, NodeTrust::Quarantined)
392 && !matches!(n.isolation, AgentIsolation::ReadOnly)
393 }
394
395 pub fn spawn_info(&self, node: usize) -> WorkflowSpawnInfo {
399 let n = &self.nodes[node];
400 let reducer = match &n.kind {
405 NodeKind::Reduce { reducer } => Some(reducer.clone()),
406 _ => None,
407 };
408 let input_agent_ids: Vec<String> = n.depends_on.iter().map(|&d| node_agent_id(d)).collect();
409 let loop_max_iters = match &n.kind {
413 NodeKind::Loop { max_iters } => Some(*max_iters),
414 _ => None,
415 };
416 let classify_labels = match &n.kind {
417 NodeKind::Classify { branches } => branches.iter().map(|b| b.label.clone()).collect(),
418 _ => Vec::new(),
419 };
420 WorkflowSpawnInfo {
421 agent_id: self.current_agent_id(node),
422 goal: n.task.goal.clone(),
423 role: role_label(n.role).to_string(),
424 isolation: isolation_label(n.isolation).to_string(),
425 context_inheritance: inheritance_label(n.context_inheritance).to_string(),
426 model_hint: n.model_hint.clone(),
427 trust: trust_label(n.trust).to_string(),
428 output_schema: n.output_schema.clone(),
429 reducer,
430 input_agent_ids,
431 judge_match: self.judge_matches.get(&node).cloned(),
432 loop_max_iters,
433 classify_labels,
434 token_budget: n.token_budget,
435 max_turns: n.max_turns,
436 max_wall_ms: n.max_wall_ms,
437 }
438 }
439
440 pub fn mark_spawned(&mut self, node: usize, agent_id: &str) {
444 self.graph.start(node);
445 self.node_of_agent.insert(agent_id.to_string(), node);
446 }
447
448 pub fn mark_denied(&mut self, node: usize) {
451 self.graph.fail(node);
452 self.resolve_dependency_outcomes();
453 }
454
455 pub fn mark_spawn_failed(&mut self, agent_id: &str) -> Option<usize> {
459 let node = self.node_of_agent.remove(agent_id)?;
460 self.graph.fail(node);
461 self.resolve_dependency_outcomes();
462 Some(node)
463 }
464
465 pub fn record_completion(&mut self, agent_id: &str, result: LoopResult) -> Option<usize> {
473 let node = *self.node_of_agent.get(agent_id)?;
474
475 if let Some(&controller) = self.child_controller.get(&node) {
478 return self.advance_tournament(controller, node, result);
479 }
480
481 if matches!(self.nodes[node].kind, NodeKind::Loop { .. })
484 && result.termination != TerminationReason::Completed
485 {
486 self.settle_result(node, result);
487 return Some(node);
488 }
489
490 match &self.nodes[node].kind {
491 NodeKind::Loop { max_iters } => {
492 let max_iters = *max_iters;
495 let stop_requested = result.loop_continue == Some(false);
496 let done = self.iter_counts.entry(node).or_insert(0);
497 *done += 1;
498 if *done < max_iters && !stop_requested {
499 self.graph.set_ready(node);
501 return Some(node);
502 }
503 }
504 NodeKind::Classify { branches } => {
505 let chosen = result.classify_branch.clone();
509 let prune: Vec<usize> = branches
510 .iter()
511 .filter(|b| Some(&b.label) != chosen.as_ref())
512 .flat_map(|b| b.nodes.iter().copied())
513 .collect();
514 for bn in prune {
515 self.graph.fail(bn);
516 }
517 }
518 NodeKind::Spawn | NodeKind::Tournament { .. } | NodeKind::Reduce { .. } => {}
522 }
523
524 self.settle_result(node, result);
527 Some(node)
528 }
529
530 fn settle_result(&mut self, node: usize, result: LoopResult) {
531 match result.termination {
532 TerminationReason::Completed => self.graph.complete(node, result),
533 TerminationReason::MaxTurns
534 | TerminationReason::TokenBudget
535 | TerminationReason::Timeout
536 | TerminationReason::MilestoneExceeded
537 | TerminationReason::ContextOverflow
538 | TerminationReason::NoProgress => self.graph.complete_partial(node, result),
539 TerminationReason::Error | TerminationReason::UserAbort => {
540 self.graph.fail_with_result(node, result)
541 }
542 }
543 self.resolve_dependency_outcomes();
544 }
545
546 fn resolve_dependency_outcomes(&mut self) {
549 loop {
550 let mut changed = false;
551 for node in 0..self.nodes.len() {
552 if self.graph.get(node).map(|n| n.status) != Some(TaskStatus::Pending) {
553 continue;
554 }
555 let policy = self.nodes[node].dep_policy;
556 if policy == DependencyPolicy::Optional {
557 self.graph.set_ready(node);
558 changed = true;
559 continue;
560 }
561 let statuses: Vec<TaskStatus> = self.nodes[node]
562 .depends_on
563 .iter()
564 .filter_map(|&dep| self.graph.get(dep).map(|n| n.status))
565 .collect();
566 let all_terminal = statuses.iter().all(|status| status.is_terminal());
567 let impossible = match policy {
568 DependencyPolicy::AllSuccess => statuses.iter().any(|status| {
569 matches!(
570 status,
571 TaskStatus::CompletedPartial
572 | TaskStatus::Failed
573 | TaskStatus::SkippedUpstreamFailed
574 )
575 }),
576 DependencyPolicy::AcceptPartial => statuses.iter().any(|status| {
577 matches!(
578 status,
579 TaskStatus::Failed | TaskStatus::SkippedUpstreamFailed
580 )
581 }),
582 DependencyPolicy::AllTerminal | DependencyPolicy::Optional => false,
583 };
584 if impossible {
585 self.graph.skip_upstream_failed(node);
586 changed = true;
587 } else if all_terminal {
588 self.graph.set_ready(node);
589 changed = true;
590 }
591 }
592 if !changed {
593 break;
594 }
595 }
596 }
597
598 fn append_child(&mut self, node: WorkflowNode) -> usize {
604 let idx = self.graph.add(node.task.clone(), Vec::new());
605 debug_assert_eq!(idx, self.nodes.len(), "graph/nodes index drift");
606 self.nodes.push(node);
607 self.refresh_scheduling();
608 idx
609 }
610
611 pub fn expand_ready_controllers(&mut self) {
616 let pending: Vec<usize> = (0..self.nodes.len())
617 .filter(|i| !self.tournaments.contains_key(i))
618 .filter(|&i| matches!(self.nodes[i].kind, NodeKind::Tournament { .. }))
619 .filter(|&i| self.graph.get(i).map(|n| n.status) == Some(TaskStatus::Ready))
620 .collect();
621 for c in pending {
622 self.expand_tournament(c);
623 }
624 }
625
626 fn expand_tournament(&mut self, c: usize) {
629 let entrants = match &self.nodes[c].kind {
630 NodeKind::Tournament { entrants } => entrants.clone(),
631 _ => return,
632 };
633 let trust = self.nodes[c].trust;
634 self.graph.start(c);
636 if entrants.len() < 2 {
640 self.complete_tournament(c, None);
641 return;
642 }
643 let mut entrant_nodes = Vec::with_capacity(entrants.len());
644 for task in entrants {
645 let child = WorkflowNode::new(task, AgentRole::Custom)
646 .with_isolation(AgentIsolation::ReadOnly)
647 .with_trust(trust);
648 let idx = self.append_child(child);
649 self.child_controller.insert(idx, c);
650 entrant_nodes.push(idx);
651 }
652 let entrants_remaining = entrant_nodes.len();
653 self.tournaments.insert(
654 c,
655 TournamentState {
656 entrant_nodes,
657 entrants_remaining,
658 bracket: None,
659 judge_nodes: Vec::new(),
660 judge_winners: Vec::new(),
661 judges_remaining: 0,
662 },
663 );
664 }
665
666 fn advance_tournament(
669 &mut self,
670 controller: usize,
671 child: usize,
672 result: LoopResult,
673 ) -> Option<usize> {
674 match result.termination {
680 TerminationReason::Completed => self.graph.complete(child, result.clone()),
681 TerminationReason::MaxTurns
682 | TerminationReason::TokenBudget
683 | TerminationReason::Timeout
684 | TerminationReason::MilestoneExceeded
685 | TerminationReason::ContextOverflow
686 | TerminationReason::NoProgress => self.graph.complete_partial(child, result.clone()),
687 TerminationReason::Error | TerminationReason::UserAbort => {
688 self.graph.fail_with_result(child, result.clone())
689 }
690 }
691
692 let in_entrant_phase = self.tournaments.get(&controller)?.bracket.is_none();
693 if in_entrant_phase {
694 let all_in = {
695 let st = self.tournaments.get_mut(&controller)?;
696 st.entrants_remaining = st.entrants_remaining.saturating_sub(1);
697 st.entrants_remaining == 0
698 };
699 if all_in {
700 self.begin_bracket(controller);
701 }
702 } else {
703 let round_done = {
704 let st = self.tournaments.get_mut(&controller)?;
705 if let Some(pos) = st.judge_nodes.iter().position(|&n| n == child) {
706 st.judge_winners[pos] = result.tournament_winner.clone();
707 }
708 st.judges_remaining = st.judges_remaining.saturating_sub(1);
709 st.judges_remaining == 0
710 };
711 if round_done {
712 self.finish_round(controller);
713 }
714 }
715 Some(controller)
716 }
717
718 fn begin_bracket(&mut self, controller: usize) {
720 let entrant_ids: Vec<EntrantId> = self
721 .tournaments
722 .get(&controller)
723 .map(|st| st.entrant_nodes.iter().map(|&n| node_agent_id(n)).collect())
724 .unwrap_or_default();
725 let mut bracket = match Tournament::new(entrant_ids) {
727 Ok(b) => b,
728 Err(_) => return self.complete_tournament(controller, None),
729 };
730 let action = bracket.start();
731 if let Some(st) = self.tournaments.get_mut(&controller) {
732 st.bracket = Some(bracket);
733 }
734 self.apply_action(controller, action);
735 }
736
737 fn finish_round(&mut self, controller: usize) {
739 let winners: Vec<EntrantId> = self
740 .tournaments
741 .get(&controller)
742 .map(|st| st.judge_winners.iter().filter_map(|w| w.clone()).collect())
743 .unwrap_or_default();
744 let action = {
745 let st = match self.tournaments.get_mut(&controller) {
746 Some(st) => st,
747 None => return,
748 };
749 match st.bracket.as_mut() {
750 Some(b) => b.feed_round(winners),
753 None => return,
754 }
755 };
756 match action {
757 Ok(act) => self.apply_action(controller, act),
758 Err(_) => self.complete_tournament(controller, None),
759 }
760 }
761
762 fn apply_action(&mut self, controller: usize, action: TournamentAction) {
764 match action {
765 TournamentAction::JudgeRound { matches, .. } => self.emit_judges(controller, matches),
766 TournamentAction::Done { winner, .. } => {
767 self.complete_tournament(controller, Some(winner))
768 }
769 }
770 }
771
772 fn emit_judges(&mut self, controller: usize, matches: Vec<Match>) {
775 let criterion = self.nodes[controller].task.clone();
776 let trust = self.nodes[controller].trust;
777 let mut judge_nodes = Vec::with_capacity(matches.len());
778 for m in &matches {
779 let judge = WorkflowNode::new(criterion.clone(), AgentRole::Verify).with_trust(trust);
780 let idx = self.append_child(judge);
781 self.child_controller.insert(idx, controller);
782 self.judge_matches.insert(
783 idx,
784 JudgeMatch {
785 left: m.left.clone(),
786 right: m.right.clone(),
787 },
788 );
789 judge_nodes.push(idx);
790 }
791 if let Some(st) = self.tournaments.get_mut(&controller) {
792 st.judge_winners = vec![None; judge_nodes.len()];
793 st.judges_remaining = judge_nodes.len();
794 st.judge_nodes = judge_nodes;
795 }
796 }
797
798 fn complete_tournament(&mut self, controller: usize, winner: Option<EntrantId>) {
804 self.tournaments.remove(&controller);
805 let Some(winner) = winner else {
806 self.graph.fail(controller);
807 self.resolve_dependency_outcomes();
808 return;
809 };
810 let result = LoopResult {
811 termination: TerminationReason::Completed,
812 final_message: None,
813 turns_used: 0,
814 total_tokens_used: 0,
815 loop_continue: None,
816 classify_branch: None,
817 tournament_winner: Some(winner),
818 pace_decision: None,
819 };
820 self.graph.complete(controller, result);
821 self.resolve_dependency_outcomes();
822 }
823
824 pub fn submit_nodes_from(
850 &mut self,
851 submitter: Option<&str>,
852 mut nodes: Vec<WorkflowNode>,
853 ) -> std::result::Result<Vec<usize>, WorkflowSubmissionError> {
854 let submitter_quarantined = submitter.is_some_and(|s| self.is_agent_quarantined(s));
855 if submitter_quarantined {
856 for node in &mut nodes {
857 node.trust = NodeTrust::Quarantined;
858 }
859 }
860 self.submit_nodes(nodes)
861 }
862
863 pub fn submit_nodes(
864 &mut self,
865 mut nodes: Vec<WorkflowNode>,
866 ) -> std::result::Result<Vec<usize>, WorkflowSubmissionError> {
867 let base = self.nodes.len();
868 let batch_len = nodes.len();
869 for (node_index, node) in nodes.iter().enumerate() {
870 if matches!(node.kind, NodeKind::Loop { max_iters: 0 }) {
871 return Err(WorkflowSubmissionError {
872 node_index,
873 reason: "loop max_iters must be greater than zero".to_string(),
874 });
875 }
876 if matches!(&node.kind, NodeKind::Tournament { entrants } if entrants.len() < 2) {
877 return Err(WorkflowSubmissionError {
878 node_index,
879 reason: "tournament requires at least two entrants".to_string(),
880 });
881 }
882 for &dependency in &node.depends_on {
883 if dependency >= batch_len {
884 return Err(WorkflowSubmissionError {
885 node_index,
886 reason: format!(
887 "dependency {dependency} out of range for batch of {batch_len} nodes"
888 ),
889 });
890 }
891 if dependency == node_index {
892 return Err(WorkflowSubmissionError {
893 node_index,
894 reason: "node depends on itself".to_string(),
895 });
896 }
897 }
898 if let NodeKind::Classify { branches } = &node.kind {
899 for branch_node in branches.iter().flat_map(|br| br.nodes.iter().copied()) {
900 if branch_node >= batch_len {
901 return Err(WorkflowSubmissionError {
902 node_index,
903 reason: format!("classify branch node {branch_node} out of range"),
904 });
905 }
906 if branch_node == node_index {
907 return Err(WorkflowSubmissionError {
908 node_index,
909 reason: "classifier cannot select itself as a branch node".to_string(),
910 });
911 }
912 if !nodes[branch_node].depends_on.contains(&node_index) {
913 return Err(WorkflowSubmissionError {
914 node_index: branch_node,
915 reason: format!(
916 "classify branch node must depend on classifier {node_index}"
917 ),
918 });
919 }
920 }
921 }
922 }
923
924 if WorkflowSpec::new(nodes.clone()).validate().is_err() {
925 return Err(WorkflowSubmissionError {
926 node_index: 0,
927 reason: "submission introduces a dependency cycle".to_string(),
928 });
929 }
930
931 for node in &mut nodes {
932 node.depends_on = node.depends_on.iter().map(|dep| base + dep).collect();
933 if let NodeKind::Classify { branches } = &mut node.kind {
934 for branch in branches {
935 branch.nodes = branch.nodes.iter().map(|node| base + node).collect();
936 }
937 }
938 }
939
940 let mut ids = Vec::with_capacity(nodes.len());
941 for node in nodes {
942 let deps = node.depends_on.clone();
943 let idx = self.graph.add(node.task.clone(), deps);
944 debug_assert_eq!(idx, self.nodes.len(), "graph/nodes index drift");
945 self.nodes.push(node);
946 ids.push(idx);
947 }
948 self.refresh_scheduling();
949 self.resolve_dependency_outcomes();
950 Ok(ids)
951 }
952
953 pub fn owns_agent(&self, agent_id: &str) -> bool {
955 self.node_of_agent.contains_key(agent_id)
956 }
957
958 pub(crate) fn spawn_info_for_agent(&self, agent_id: &str) -> Option<WorkflowSpawnInfo> {
960 self.node_of_agent
961 .get(agent_id)
962 .copied()
963 .map(|node| self.spawn_info(node))
964 }
965
966 pub fn is_agent_quarantined(&self, agent_id: &str) -> bool {
971 self.node_of_agent
972 .get(agent_id)
973 .is_some_and(|&node| matches!(self.nodes[node].trust, NodeTrust::Quarantined))
974 }
975
976 #[cfg(test)]
982 pub(crate) fn quarantine_agent(&mut self, agent_id: &str) -> bool {
983 match self.node_of_agent.get(agent_id).copied() {
984 Some(node) => {
985 self.nodes[node].trust = NodeTrust::Quarantined;
986 true
987 }
988 None => false,
989 }
990 }
991
992 #[cfg(test)]
996 pub(crate) fn batch_drained(&self) -> bool {
997 !(0..self.graph.len()).any(|i| {
998 matches!(
999 self.graph.get(i).map(|n| &n.status),
1000 Some(crate::orchestration::task_graph::TaskStatus::Running)
1001 )
1002 })
1003 }
1004
1005 #[cfg(test)]
1007 pub(crate) fn is_complete(&self) -> bool {
1008 self.graph.all_done()
1009 }
1010
1011 pub fn finish(&mut self) -> Vec<WorkflowNodeOutcome> {
1013 for node in 0..self.graph.len() {
1014 match self.graph.get(node).map(|node| node.status) {
1015 Some(TaskStatus::Pending | TaskStatus::Ready) => {
1016 self.graph.skip_upstream_failed(node)
1017 }
1018 Some(TaskStatus::Running) => self.graph.fail(node),
1019 _ => {}
1020 }
1021 }
1022 self.node_outcomes()
1023 }
1024
1025 pub fn node_outcomes(&self) -> Vec<WorkflowNodeOutcome> {
1026 (0..self.graph.len())
1027 .filter_map(|node| {
1028 let graph_node = self.graph.get(node)?;
1029 let status = match graph_node.status {
1030 TaskStatus::Completed => WorkflowNodeStatus::Completed,
1031 TaskStatus::CompletedPartial => WorkflowNodeStatus::CompletedPartial,
1032 TaskStatus::Failed => WorkflowNodeStatus::Failed,
1033 TaskStatus::SkippedUpstreamFailed => WorkflowNodeStatus::SkippedUpstreamFailed,
1034 TaskStatus::Pending | TaskStatus::Ready | TaskStatus::Running => return None,
1035 };
1036 Some(WorkflowNodeOutcome {
1037 node_id: node_agent_id(node),
1038 status,
1039 termination: graph_node.result.as_ref().map(|result| result.termination),
1040 output: graph_node
1041 .result
1042 .as_ref()
1043 .and_then(|result| result.final_message.clone()),
1044 })
1045 })
1046 .collect()
1047 }
1048
1049 pub fn abort_outcomes(&self) -> Vec<WorkflowNodeOutcome> {
1052 (0..self.graph.len())
1053 .filter_map(|node| {
1054 let graph_node = self.graph.get(node)?;
1055 let (status, termination) = match graph_node.status {
1056 TaskStatus::Completed => (
1057 WorkflowNodeStatus::Completed,
1058 graph_node.result.as_ref().map(|result| result.termination),
1059 ),
1060 TaskStatus::CompletedPartial => (
1061 WorkflowNodeStatus::CompletedPartial,
1062 graph_node.result.as_ref().map(|result| result.termination),
1063 ),
1064 TaskStatus::Pending | TaskStatus::SkippedUpstreamFailed => {
1065 (WorkflowNodeStatus::SkippedUpstreamFailed, None)
1066 }
1067 TaskStatus::Ready | TaskStatus::Running | TaskStatus::Failed => (
1068 WorkflowNodeStatus::Failed,
1069 Some(
1070 graph_node
1071 .result
1072 .as_ref()
1073 .map_or(TerminationReason::UserAbort, |result| result.termination),
1074 ),
1075 ),
1076 };
1077 Some(WorkflowNodeOutcome {
1078 node_id: node_agent_id(node),
1079 status,
1080 termination,
1081 output: graph_node
1082 .result
1083 .as_ref()
1084 .and_then(|result| result.final_message.clone()),
1085 })
1086 })
1087 .collect()
1088 }
1089
1090 pub fn len(&self) -> usize {
1092 self.graph.len()
1093 }
1094
1095 pub(crate) fn checkpoint_nodes(&self) -> Vec<WorkflowRuntimeNodeState> {
1097 (0..self.graph.len())
1098 .filter_map(|node| {
1099 let graph_node = self.graph.get(node)?;
1100 let active_agent_id = (graph_node.status == TaskStatus::Running)
1101 .then(|| {
1102 self.node_of_agent.iter().find_map(|(agent_id, &owner)| {
1103 (owner == node).then(|| agent_id.clone())
1104 })
1105 })
1106 .flatten();
1107 Some(WorkflowRuntimeNodeState {
1108 node: self.nodes[node].clone(),
1109 status: graph_node.status,
1110 result: graph_node.result.clone(),
1111 active_agent_id,
1112 iterations_completed: self.iter_counts.get(&node).copied().unwrap_or(0),
1113 })
1114 })
1115 .collect()
1116 }
1117}
1118
1119#[cfg(test)]
1120mod tests {
1121 use super::*;
1122 use crate::orchestration::workflow::{ClassifyBranch, fanout_synthesize};
1123 use crate::types::result::{LoopResult, TerminationReason};
1124 use crate::types::task::RuntimeTask;
1125
1126 fn done() -> LoopResult {
1127 LoopResult {
1128 termination: TerminationReason::Completed,
1129 final_message: None,
1130 turns_used: 1,
1131 total_tokens_used: 0,
1132 loop_continue: None,
1133 classify_branch: None,
1134 tournament_winner: None,
1135 pace_decision: None,
1136 }
1137 }
1138
1139 fn terminated(termination: TerminationReason) -> LoopResult {
1140 LoopResult {
1141 termination,
1142 ..done()
1143 }
1144 }
1145
1146 fn fanout2() -> WorkflowRun {
1147 let spec = fanout_synthesize(
1149 vec![RuntimeTask::new("w0"), RuntimeTask::new("w1")],
1150 RuntimeTask::new("synth"),
1151 );
1152 WorkflowRun::new(&spec).unwrap()
1153 }
1154
1155 fn judge_done(winner: &str) -> LoopResult {
1157 LoopResult {
1158 tournament_winner: Some(winner.to_string()),
1159 ..done()
1160 }
1161 }
1162
1163 fn spawn_round(run: &mut WorkflowRun) -> Vec<(usize, String)> {
1166 run.expand_ready_controllers();
1167 let ready = run.ready_batch();
1168 let mut out = Vec::new();
1169 for node in ready {
1170 let id = run.current_agent_id(node);
1171 run.mark_spawned(node, &id);
1172 out.push((node, id));
1173 }
1174 out
1175 }
1176
1177 fn outcome_ids(run: &WorkflowRun, status: WorkflowNodeStatus) -> Vec<String> {
1178 run.node_outcomes()
1179 .into_iter()
1180 .filter(|outcome| outcome.status == status)
1181 .map(|outcome| outcome.node_id)
1182 .collect()
1183 }
1184
1185 #[test]
1186 fn first_batch_is_the_workers() {
1187 let mut run = fanout2();
1188 assert_eq!(run.ready_batch(), vec![0, 1]);
1189 assert_eq!(run.len(), 3);
1190 assert!(!run.is_complete());
1191 }
1192
1193 #[test]
1196 fn submit_nodes_appends_independent_nodes_ready_immediately() {
1197 use crate::orchestration::workflow::WorkflowNode;
1198 use crate::types::agent::AgentRole;
1199
1200 let mut run = fanout2(); assert_eq!(run.len(), 3);
1202 let ids = run
1203 .submit_nodes(vec![
1204 WorkflowNode::new(RuntimeTask::new("extra-a"), AgentRole::Implement),
1205 WorkflowNode::new(RuntimeTask::new("extra-b"), AgentRole::Implement),
1206 ])
1207 .unwrap();
1208 assert_eq!(ids, vec![3, 4], "appended after the existing 3 nodes");
1209 assert_eq!(run.len(), 5);
1210 let ready = run.ready_batch();
1211 assert!(
1212 ready.contains(&3) && ready.contains(&4),
1213 "submitted independent nodes are immediately ready: {ready:?}"
1214 );
1215 }
1216
1217 #[test]
1218 fn submitted_nodes_must_complete_before_workflow_is_done() {
1219 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1220 use crate::types::agent::AgentRole;
1221
1222 let spec = WorkflowSpec::new(vec![WorkflowNode::new(
1224 RuntimeTask::new("root"),
1225 AgentRole::Implement,
1226 )]);
1227 let mut run = WorkflowRun::new(&spec).unwrap();
1228 let id0 = run.current_agent_id(0);
1229 run.mark_spawned(0, &id0);
1230 run.record_completion(&id0, done());
1231 let ids = run
1232 .submit_nodes(vec![WorkflowNode::new(
1233 RuntimeTask::new("more"),
1234 AgentRole::Implement,
1235 )])
1236 .unwrap();
1237 assert_eq!(ids, vec![1]);
1238 assert!(
1239 !run.is_complete(),
1240 "not complete while the submitted node is pending"
1241 );
1242 let spawned = spawn_round(&mut run);
1243 assert_eq!(spawned, vec![(1usize, "wf-node1".to_string())]);
1244 run.record_completion("wf-node1", done());
1245 assert!(
1246 run.is_complete(),
1247 "complete once the submitted node finishes"
1248 );
1249 }
1250
1251 #[test]
1252 fn reduce_node_carries_reducer_and_inputs_then_completes_like_a_spawn() {
1253 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1254 use crate::types::agent::AgentRole;
1255
1256 let spec = WorkflowSpec::new(vec![
1259 WorkflowNode::new(RuntimeTask::new("worker-a"), AgentRole::Explore),
1260 WorkflowNode::new(RuntimeTask::new("worker-b"), AgentRole::Explore),
1261 WorkflowNode::new(RuntimeTask::new("merge"), AgentRole::Implement)
1262 .with_reduce("dedupe_lines")
1263 .with_depends_on(vec![0, 1]),
1264 ]);
1265 let mut run = WorkflowRun::new(&spec).unwrap();
1266
1267 assert_eq!(run.ready_batch(), vec![0, 1]);
1269 for i in [0usize, 1] {
1270 let id = run.current_agent_id(i);
1271 run.mark_spawned(i, &id);
1272 run.record_completion(&id, done());
1273 }
1274
1275 assert_eq!(run.ready_batch(), vec![2]);
1277 let info = run.spawn_info(2);
1278 assert_eq!(info.reducer.as_deref(), Some("dedupe_lines"));
1279 assert_eq!(
1280 info.input_agent_ids,
1281 vec!["wf-node0".to_string(), "wf-node1".to_string()]
1282 );
1283
1284 run.mark_spawned(2, "wf-node2");
1286 run.record_completion("wf-node2", done());
1287 assert!(run.is_complete());
1288 let completed = outcome_ids(&run, WorkflowNodeStatus::Completed);
1289 assert_eq!(completed, vec!["wf-node0", "wf-node1", "wf-node2"]);
1290 assert_eq!(run.node_outcomes().len(), completed.len());
1291 }
1292
1293 #[test]
1294 fn output_schema_reaches_the_spawn_descriptor() {
1295 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1296 use crate::types::agent::AgentRole;
1297
1298 let schema = serde_json::json!({
1300 "type": "object",
1301 "required": ["verdict"],
1302 "properties": { "verdict": { "type": "string" } }
1303 });
1304 let spec = WorkflowSpec::new(vec![
1305 WorkflowNode::new(RuntimeTask::new("judge"), AgentRole::Verify)
1306 .with_output_schema(schema.clone()),
1307 ]);
1308 let run = WorkflowRun::new(&spec).unwrap();
1309 let info = run.spawn_info(0);
1310 assert_eq!(info.output_schema.as_ref(), Some(&schema));
1311
1312 let json = serde_json::to_string(&info).unwrap();
1314 let back: WorkflowSpawnInfo = serde_json::from_str(&json).unwrap();
1315 assert_eq!(back.output_schema, Some(schema));
1316
1317 let plain = WorkflowSpec::new(vec![WorkflowNode::new(
1319 RuntimeTask::new("x"),
1320 AgentRole::Implement,
1321 )]);
1322 let plain_info = WorkflowRun::new(&plain).unwrap().spawn_info(0);
1323 assert!(plain_info.output_schema.is_none());
1324 assert!(
1325 !serde_json::to_string(&plain_info)
1326 .unwrap()
1327 .contains("output_schema")
1328 );
1329 }
1330
1331 #[test]
1332 fn quarantined_submitter_taints_submitted_nodes() {
1333 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1334 use crate::types::agent::AgentRole;
1335
1336 let spec = WorkflowSpec::new(vec![
1340 WorkflowNode::new(RuntimeTask::new("read-untrusted"), AgentRole::Explore).quarantined(),
1341 ]);
1342 let mut run = WorkflowRun::new(&spec).unwrap();
1343 let id0 = run.current_agent_id(0);
1344 run.mark_spawned(0, &id0);
1345 run.record_completion(&id0, done());
1346
1347 let ids = run
1349 .submit_nodes_from(
1350 Some(&id0),
1351 vec![WorkflowNode::new(
1352 RuntimeTask::new("act"),
1353 AgentRole::Implement,
1354 )],
1355 )
1356 .unwrap();
1357 assert_eq!(ids, vec![1]);
1358 let id1 = run.current_agent_id(1);
1359 run.mark_spawned(1, &id1);
1360 assert!(
1361 run.is_agent_quarantined(&id1),
1362 "submitted node inherits the submitter's quarantine (no escalation)"
1363 );
1364
1365 let ids2 = run
1367 .submit_nodes_from(
1368 None,
1369 vec![WorkflowNode::new(
1370 RuntimeTask::new("trusted-work"),
1371 AgentRole::Implement,
1372 )],
1373 )
1374 .unwrap();
1375 let id2 = run.current_agent_id(ids2[0]);
1376 run.mark_spawned(ids2[0], &id2);
1377 assert!(
1378 !run.is_agent_quarantined(&id2),
1379 "no quarantined submitter ⇒ no coercion"
1380 );
1381 }
1382
1383 #[test]
1384 fn submit_nodes_honors_batch_relative_backward_deps() {
1385 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1386 use crate::types::agent::AgentRole;
1387
1388 let spec = WorkflowSpec::new(vec![WorkflowNode::new(
1389 RuntimeTask::new("root"),
1390 AgentRole::Implement,
1391 )]);
1392 let mut run = WorkflowRun::new(&spec).unwrap();
1393 let id0 = run.current_agent_id(0);
1394 run.mark_spawned(0, &id0);
1395 run.record_completion(&id0, done());
1396 let ids = run
1398 .submit_nodes(vec![
1399 WorkflowNode::new(RuntimeTask::new("extractor"), AgentRole::Implement),
1400 WorkflowNode::new(RuntimeTask::new("dependent"), AgentRole::Implement)
1401 .with_depends_on(vec![0]),
1402 ])
1403 .unwrap();
1404 assert_eq!(ids, vec![1, 2]);
1405 assert_eq!(
1406 run.ready_batch(),
1407 vec![1],
1408 "backward dep keeps the dependent pending"
1409 );
1410 run.mark_spawned(1, "wf-node1");
1411 run.record_completion("wf-node1", done());
1412 assert_eq!(
1413 run.ready_batch(),
1414 vec![2],
1415 "dependent unblocks after the extractor"
1416 );
1417 }
1418
1419 #[test]
1420 fn submit_nodes_accepts_acyclic_forward_dependencies() {
1421 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1422 use crate::types::agent::AgentRole;
1423
1424 let spec = WorkflowSpec::new(vec![WorkflowNode::new(
1425 RuntimeTask::new("root"),
1426 AgentRole::Implement,
1427 )]);
1428 let mut run = WorkflowRun::new(&spec).unwrap();
1429 let ids = run
1431 .submit_nodes(vec![
1432 WorkflowNode::new(RuntimeTask::new("consumer"), AgentRole::Implement)
1433 .with_depends_on(vec![1]),
1434 WorkflowNode::new(RuntimeTask::new("producer"), AgentRole::Implement),
1435 ])
1436 .unwrap();
1437 assert_eq!(ids, vec![1, 2]);
1438 assert_eq!(
1439 run.ready_batch(),
1440 vec![2, 0],
1441 "the producer is on the longer critical path and runs before the independent root"
1442 );
1443 run.mark_spawned(2, "wf-node2");
1444 run.record_completion("wf-node2", done());
1445 assert!(run.ready_batch().contains(&1));
1446 }
1447
1448 #[test]
1449 fn submit_nodes_rejects_malformed_batches_atomically() {
1450 use crate::orchestration::workflow::WorkflowNode;
1451 use crate::types::agent::AgentRole;
1452
1453 for nodes in [
1454 vec![
1455 WorkflowNode::new(RuntimeTask::new("bad-range"), AgentRole::Implement)
1456 .with_depends_on(vec![1]),
1457 ],
1458 vec![
1459 WorkflowNode::new(RuntimeTask::new("self"), AgentRole::Implement)
1460 .with_depends_on(vec![0]),
1461 ],
1462 vec![
1463 WorkflowNode::new(RuntimeTask::new("a"), AgentRole::Implement)
1464 .with_depends_on(vec![1]),
1465 WorkflowNode::new(RuntimeTask::new("b"), AgentRole::Implement)
1466 .with_depends_on(vec![0]),
1467 ],
1468 ] {
1469 let mut run = fanout2();
1470 let before = run.len();
1471 assert!(run.submit_nodes(nodes).is_err());
1472 assert_eq!(run.len(), before, "rejection must precede every mutation");
1473 }
1474 }
1475
1476 #[test]
1477 fn submitted_node_can_itself_be_a_loop_control_flow() {
1478 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1483 use crate::types::agent::AgentRole;
1484
1485 let spec = WorkflowSpec::new(vec![WorkflowNode::new(
1486 RuntimeTask::new("root"),
1487 AgentRole::Implement,
1488 )]);
1489 let mut run = WorkflowRun::new(&spec).unwrap();
1490 let id0 = run.current_agent_id(0);
1491 run.mark_spawned(0, &id0);
1492 run.record_completion(&id0, done());
1493
1494 let ids = run
1496 .submit_nodes(vec![
1497 WorkflowNode::new(RuntimeTask::new("refine"), AgentRole::Implement).with_loop(2),
1498 ])
1499 .unwrap();
1500 assert_eq!(ids, vec![1]);
1501
1502 for k in 0..2 {
1504 assert_eq!(
1505 run.ready_batch(),
1506 vec![1],
1507 "submitted loop ready for iteration {k}"
1508 );
1509 let id = run.current_agent_id(1);
1510 assert_eq!(
1511 id,
1512 format!("wf-node1-i{k}"),
1513 "submitted loop gets per-iteration ids"
1514 );
1515 run.mark_spawned(1, &id);
1516 run.record_completion(&id, done());
1517 }
1518 assert!(
1519 run.is_complete(),
1520 "submitted loop ran its 2 iterations then finished"
1521 );
1522 }
1523
1524 #[test]
1525 fn submitted_tournament_runs_bracket_then_promotes_submitted_dependent() {
1526 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1530 use crate::types::agent::AgentRole;
1531
1532 let spec = WorkflowSpec::new(vec![WorkflowNode::new(
1533 RuntimeTask::new("root"),
1534 AgentRole::Implement,
1535 )]);
1536 let mut run = WorkflowRun::new(&spec).unwrap();
1537 let id0 = run.current_agent_id(0);
1538 run.mark_spawned(0, &id0);
1539 run.record_completion(&id0, done());
1540
1541 let ids = run
1543 .submit_nodes(vec![
1544 WorkflowNode::new(RuntimeTask::new("pick best"), AgentRole::Plan)
1545 .with_tournament(vec![RuntimeTask::new("x"), RuntimeTask::new("y")]),
1546 WorkflowNode::new(RuntimeTask::new("use winner"), AgentRole::Implement)
1547 .with_depends_on(vec![0]),
1548 ])
1549 .unwrap();
1550 assert_eq!(ids, vec![1, 2], "appended controller=1, dependent=2");
1551
1552 let entrants = spawn_round(&mut run);
1554 let entrant_nodes: Vec<usize> = entrants.iter().map(|(n, _)| *n).collect();
1555 assert_eq!(
1556 entrant_nodes,
1557 vec![3, 4],
1558 "two entrant children appended after the dependent"
1559 );
1560 for (_, id) in &entrants {
1561 run.record_completion(id, done());
1562 }
1563
1564 let r1 = spawn_round(&mut run);
1566 assert_eq!(r1.len(), 1, "one judge for two entrants");
1567 let jm = run
1568 .spawn_info(r1[0].0)
1569 .judge_match
1570 .expect("judge carries a match");
1571 assert_eq!(
1572 jm,
1573 JudgeMatch {
1574 left: node_agent_id(3),
1575 right: node_agent_id(4)
1576 }
1577 );
1578
1579 run.record_completion(&r1[0].1, judge_done(&node_agent_id(3)));
1581 assert_eq!(
1582 run.ready_batch(),
1583 vec![2],
1584 "submitted dependent unblocks after the bracket"
1585 );
1586 let last = spawn_round(&mut run);
1587 assert_eq!(last, vec![(2, node_agent_id(2))]);
1588 run.record_completion(&last[0].1, done());
1589 assert!(run.is_complete());
1590 }
1591
1592 #[test]
1593 fn submitted_classify_remaps_branch_indices_and_prunes() {
1594 use crate::orchestration::workflow::{
1598 ClassifyBranch, NodeKind, WorkflowNode, WorkflowSpec,
1599 };
1600 use crate::types::agent::AgentRole;
1601
1602 let spec = WorkflowSpec::new(vec![WorkflowNode::new(
1603 RuntimeTask::new("root"),
1604 AgentRole::Implement,
1605 )]);
1606 let mut run = WorkflowRun::new(&spec).unwrap();
1607 let id0 = run.current_agent_id(0);
1608 run.mark_spawned(0, &id0);
1609 run.record_completion(&id0, done());
1610
1611 let ids = run
1613 .submit_nodes(vec![
1614 WorkflowNode::new(RuntimeTask::new("route"), AgentRole::Plan).with_classify(vec![
1615 ClassifyBranch {
1616 label: "a".into(),
1617 nodes: vec![1],
1618 },
1619 ClassifyBranch {
1620 label: "b".into(),
1621 nodes: vec![2],
1622 },
1623 ]),
1624 WorkflowNode::new(RuntimeTask::new("branch-a"), AgentRole::Implement)
1625 .with_depends_on(vec![0]),
1626 WorkflowNode::new(RuntimeTask::new("branch-b"), AgentRole::Implement)
1627 .with_depends_on(vec![0]),
1628 ])
1629 .unwrap();
1630 assert_eq!(ids, vec![1, 2, 3], "classify=1, branchA=2, branchB=3");
1631
1632 if let NodeKind::Classify { branches } = &run.nodes[1].kind {
1634 assert_eq!(
1635 branches[0].nodes,
1636 vec![2],
1637 "branch a remapped to absolute node 2"
1638 );
1639 assert_eq!(
1640 branches[1].nodes,
1641 vec![3],
1642 "branch b remapped to absolute node 3"
1643 );
1644 } else {
1645 panic!("node 1 should be a classify node");
1646 }
1647
1648 let r = spawn_round(&mut run);
1650 assert_eq!(r, vec![(1, node_agent_id(1))], "classifier runs first");
1651 run.record_completion(
1652 &r[0].1,
1653 LoopResult {
1654 classify_branch: Some("a".into()),
1655 ..done()
1656 },
1657 );
1658
1659 assert_eq!(run.ready_batch(), vec![2], "only branch a is enabled");
1660 let failed = outcome_ids(&run, WorkflowNodeStatus::Failed);
1661 assert!(
1662 failed.contains(&node_agent_id(3)),
1663 "branch b is explicitly failed by routing"
1664 );
1665
1666 let last = spawn_round(&mut run);
1667 assert_eq!(last, vec![(2, node_agent_id(2))]);
1668 run.record_completion(&last[0].1, done());
1669 assert!(run.is_complete());
1670 let completed = outcome_ids(&run, WorkflowNodeStatus::Completed);
1671 assert!(completed.contains(&node_agent_id(1)) && completed.contains(&node_agent_id(2)));
1672 }
1673
1674 #[test]
1675 fn loop_node_iterates_with_distinct_ids_then_promotes_dependent() {
1676 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1677 use crate::types::agent::AgentRole;
1678
1679 let spec = WorkflowSpec::new(vec![
1681 WorkflowNode::new(RuntimeTask::new("refine"), AgentRole::Implement).with_loop(3),
1682 WorkflowNode::new(RuntimeTask::new("finalize"), AgentRole::Implement)
1683 .with_depends_on(vec![0]),
1684 ]);
1685 let mut run = WorkflowRun::new(&spec).unwrap();
1686
1687 for k in 0..3 {
1689 assert_eq!(
1690 run.ready_batch(),
1691 vec![0],
1692 "loop node ready for iteration {k}"
1693 );
1694 let id = run.current_agent_id(0);
1695 assert_eq!(id, format!("wf-node0-i{k}"), "distinct per-iteration id");
1696 run.mark_spawned(0, &id);
1697 assert!(!run.is_complete());
1698 let node = run.record_completion(&id, done()).unwrap();
1699 assert_eq!(node, 0);
1700 if k < 2 {
1701 assert_eq!(run.ready_batch(), vec![0]);
1703 }
1704 }
1705
1706 assert_eq!(
1708 run.ready_batch(),
1709 vec![1],
1710 "dependent unblocks only after the loop ends"
1711 );
1712 let id1 = run.current_agent_id(1);
1713 assert_eq!(id1, "wf-node1", "spawn node keeps the plain id");
1714 run.mark_spawned(1, &id1);
1715 run.record_completion(&id1, done());
1716 assert!(run.is_complete());
1717 }
1718
1719 #[test]
1720 fn synth_becomes_ready_only_after_both_workers() {
1721 let mut run = fanout2();
1722 for &n in &[0usize, 1usize] {
1723 let id = node_agent_id(n);
1724 run.mark_spawned(n, &id);
1725 }
1726 assert!(!run.batch_drained());
1727 assert_eq!(run.record_completion(&node_agent_id(0), done()), Some(0));
1729 assert!(!run.batch_drained());
1730 assert!(run.ready_batch().is_empty());
1731 assert_eq!(run.record_completion(&node_agent_id(1), done()), Some(1));
1733 assert!(run.batch_drained());
1734 assert_eq!(run.ready_batch(), vec![2]);
1735 assert!(!run.is_complete());
1736 run.mark_spawned(2, &node_agent_id(2));
1738 run.record_completion(&node_agent_id(2), done());
1739 assert!(run.is_complete());
1740 }
1741
1742 #[test]
1743 fn denied_node_skips_dependents_and_closes_outcome() {
1744 let mut run = fanout2();
1745 run.mark_spawned(0, &node_agent_id(0));
1747 run.mark_denied(1);
1748 run.record_completion(&node_agent_id(0), done());
1749 assert!(run.batch_drained());
1750 assert!(run.ready_batch().is_empty());
1751 assert!(run.is_complete());
1752 let outcomes = run.finish();
1753 assert_eq!(outcomes.len(), 3);
1754 assert_eq!(outcomes[0].status, WorkflowNodeStatus::Completed);
1755 assert_eq!(outcomes[1].status, WorkflowNodeStatus::Failed);
1756 assert_eq!(
1757 outcomes[2].status,
1758 WorkflowNodeStatus::SkippedUpstreamFailed
1759 );
1760 }
1761
1762 #[test]
1763 fn terminal_mapping_and_dependency_policies_are_explicit() {
1764 use crate::orchestration::workflow::{DependencyPolicy, WorkflowNode, WorkflowSpec};
1765 use crate::types::agent::AgentRole;
1766
1767 let cases = [
1768 (TerminationReason::Completed, WorkflowNodeStatus::Completed),
1769 (
1770 TerminationReason::MaxTurns,
1771 WorkflowNodeStatus::CompletedPartial,
1772 ),
1773 (
1774 TerminationReason::TokenBudget,
1775 WorkflowNodeStatus::CompletedPartial,
1776 ),
1777 (
1778 TerminationReason::Timeout,
1779 WorkflowNodeStatus::CompletedPartial,
1780 ),
1781 (
1782 TerminationReason::ContextOverflow,
1783 WorkflowNodeStatus::CompletedPartial,
1784 ),
1785 (
1786 TerminationReason::NoProgress,
1787 WorkflowNodeStatus::CompletedPartial,
1788 ),
1789 (
1790 TerminationReason::MilestoneExceeded,
1791 WorkflowNodeStatus::CompletedPartial,
1792 ),
1793 (TerminationReason::Error, WorkflowNodeStatus::Failed),
1794 (TerminationReason::UserAbort, WorkflowNodeStatus::Failed),
1795 ];
1796 for (termination, expected) in cases {
1797 let spec = WorkflowSpec::new(vec![WorkflowNode::new(
1798 RuntimeTask::new("node"),
1799 AgentRole::Implement,
1800 )]);
1801 let mut run = WorkflowRun::new(&spec).unwrap();
1802 run.mark_spawned(0, "wf-node0");
1803 run.record_completion("wf-node0", terminated(termination));
1804 let outcome = run.finish().remove(0);
1805 assert_eq!(outcome.status, expected);
1806 assert_eq!(outcome.termination, Some(termination));
1807 }
1808
1809 let spec = WorkflowSpec::new(vec![
1810 WorkflowNode::new(RuntimeTask::new("upstream"), AgentRole::Implement),
1811 WorkflowNode::new(RuntimeTask::new("strict"), AgentRole::Implement)
1812 .with_depends_on(vec![0]),
1813 WorkflowNode::new(RuntimeTask::new("partial-ok"), AgentRole::Implement)
1814 .with_depends_on(vec![0])
1815 .with_dependency_policy(DependencyPolicy::AcceptPartial),
1816 ]);
1817 let mut run = WorkflowRun::new(&spec).unwrap();
1818 run.mark_spawned(0, "wf-node0");
1819 run.record_completion("wf-node0", terminated(TerminationReason::Timeout));
1820 assert_eq!(run.ready_batch(), vec![2]);
1821 assert_eq!(
1822 run.node_outcomes()[1].status,
1823 WorkflowNodeStatus::SkippedUpstreamFailed
1824 );
1825 }
1826
1827 #[test]
1828 fn all_terminal_and_optional_have_distinct_waiting_semantics() {
1829 use crate::orchestration::workflow::{DependencyPolicy, WorkflowNode, WorkflowSpec};
1830 use crate::types::agent::AgentRole;
1831
1832 let spec = WorkflowSpec::new(vec![
1833 WorkflowNode::new(RuntimeTask::new("upstream"), AgentRole::Implement),
1834 WorkflowNode::new(RuntimeTask::new("cleanup"), AgentRole::Implement)
1835 .with_depends_on(vec![0])
1836 .with_dependency_policy(DependencyPolicy::AllTerminal),
1837 WorkflowNode::new(RuntimeTask::new("best-effort"), AgentRole::Implement)
1838 .with_depends_on(vec![0])
1839 .with_dependency_policy(DependencyPolicy::Optional),
1840 ]);
1841 let mut run = WorkflowRun::new(&spec).unwrap();
1842 assert_eq!(run.ready_batch(), vec![0, 2]);
1843 run.mark_spawned(0, "wf-node0");
1844 run.record_completion("wf-node0", terminated(TerminationReason::Error));
1845 assert!(run.ready_batch().contains(&1));
1846 }
1847
1848 #[test]
1849 fn loop_terminal_result_is_not_retried() {
1850 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1851 use crate::types::agent::AgentRole;
1852
1853 let spec = WorkflowSpec::new(vec![
1854 WorkflowNode::new(RuntimeTask::new("loop"), AgentRole::Implement).with_loop(3),
1855 ]);
1856 let mut run = WorkflowRun::new(&spec).unwrap();
1857 run.mark_spawned(0, "wf-node0-i0");
1858 run.record_completion("wf-node0-i0", terminated(TerminationReason::Timeout));
1859 assert!(run.ready_batch().is_empty());
1860 assert_eq!(run.finish()[0].status, WorkflowNodeStatus::CompletedPartial);
1861 }
1862
1863 #[test]
1864 fn manifest_preserves_node_isolation_and_inheritance() {
1865 let run = fanout2();
1866 let m = run.manifest_for(0);
1867 assert_eq!(m.agent_id.as_str(), "wf-node0");
1868 assert_eq!(m.isolation, crate::types::agent::AgentIsolation::ReadOnly);
1870 assert_eq!(
1871 m.context_inheritance,
1872 crate::types::agent::ContextInheritance::SystemOnly
1873 );
1874 }
1875
1876 #[test]
1877 fn unknown_agent_completion_is_none() {
1878 let mut run = fanout2();
1879 assert_eq!(run.record_completion("not-a-node", done()), None);
1880 }
1881
1882 #[test]
1883 fn spawn_info_carries_model_hint_and_trust() {
1884 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1885 use crate::types::agent::AgentRole;
1886
1887 let spec = WorkflowSpec::new(vec![
1888 WorkflowNode::new(RuntimeTask::new("read tickets"), AgentRole::Explore)
1889 .quarantined()
1890 .with_model_hint("haiku"),
1891 WorkflowNode::new(RuntimeTask::new("act"), AgentRole::Implement),
1892 ]);
1893 let run = WorkflowRun::new(&spec).unwrap();
1894
1895 let q = run.spawn_info(0);
1897 assert_eq!(q.trust, "quarantined");
1898 assert_eq!(q.model_hint.as_deref(), Some("haiku"));
1899 let t = run.spawn_info(1);
1901 assert_eq!(t.trust, "trusted");
1902 assert_eq!(t.model_hint, None);
1903 }
1904
1905 #[test]
1906 fn spawn_info_carries_loop_and_classify_hints() {
1907 use crate::orchestration::workflow::{ClassifyBranch, WorkflowNode, WorkflowSpec};
1908 use crate::types::agent::AgentRole;
1909
1910 let spec = WorkflowSpec::new(vec![
1911 WorkflowNode::new(RuntimeTask::new("refine"), AgentRole::Implement).with_loop(3),
1913 WorkflowNode::new(RuntimeTask::new("route"), AgentRole::Plan).with_classify(vec![
1915 ClassifyBranch {
1916 label: "bug".into(),
1917 nodes: vec![],
1918 },
1919 ClassifyBranch {
1920 label: "feature".into(),
1921 nodes: vec![],
1922 },
1923 ]),
1924 WorkflowNode::new(RuntimeTask::new("act"), AgentRole::Implement),
1926 ]);
1927 let run = WorkflowRun::new(&spec).unwrap();
1928
1929 let l = run.spawn_info(0);
1930 assert_eq!(l.loop_max_iters, Some(3));
1931 assert!(l.classify_labels.is_empty());
1932 assert_eq!(l.token_budget, None, "no token budget unless set");
1933
1934 let c = run.spawn_info(1);
1935 assert_eq!(
1936 c.classify_labels,
1937 vec!["bug".to_string(), "feature".to_string()]
1938 );
1939 assert_eq!(c.loop_max_iters, None);
1940
1941 let s = run.spawn_info(2);
1942 assert_eq!(s.loop_max_iters, None);
1943 assert!(s.classify_labels.is_empty());
1944 }
1945
1946 #[test]
1947 fn spawn_info_carries_token_budget() {
1948 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
1949 use crate::types::agent::AgentRole;
1950
1951 let spec = WorkflowSpec::new(vec![
1952 WorkflowNode::new(RuntimeTask::new("expensive"), AgentRole::Implement)
1953 .with_token_budget(10_000),
1954 WorkflowNode::new(RuntimeTask::new("plain"), AgentRole::Implement),
1955 ]);
1956 let run = WorkflowRun::new(&spec).unwrap();
1957 assert_eq!(run.spawn_info(0).token_budget, Some(10_000));
1958 assert_eq!(run.spawn_info(1).token_budget, None);
1959 }
1960
1961 use crate::orchestration::workflow::{NodeKind, WorkflowNode, WorkflowSpec};
1964 use crate::types::agent::AgentRole;
1965
1966 #[test]
1970 fn tournament_runs_bracket_then_promotes_dependent() {
1971 let spec = WorkflowSpec::new(vec![
1972 WorkflowNode::new(RuntimeTask::new("pick the best ad"), AgentRole::Plan)
1973 .with_tournament(vec![
1974 RuntimeTask::new("ad A"),
1975 RuntimeTask::new("ad B"),
1976 RuntimeTask::new("ad C"),
1977 RuntimeTask::new("ad D"),
1978 ]),
1979 WorkflowNode::new(RuntimeTask::new("ship the winner"), AgentRole::Implement)
1980 .with_depends_on(vec![0]),
1981 ]);
1982 let mut run = WorkflowRun::new(&spec).unwrap();
1983
1984 let entrants = spawn_round(&mut run);
1987 let entrant_nodes: Vec<usize> = entrants.iter().map(|(n, _)| *n).collect();
1988 assert_eq!(
1989 entrant_nodes,
1990 vec![2, 3, 4, 5],
1991 "4 entrant children, no controller spawn"
1992 );
1993 assert!(
1994 run.spawn_info(2).judge_match.is_none(),
1995 "entrants are not judges"
1996 );
1997 assert!(!run.is_complete());
1998
1999 for (i, (node, id)) in entrants.iter().enumerate() {
2001 run.record_completion(id, done());
2002 if i < 3 {
2003 assert!(
2004 run.ready_batch().is_empty(),
2005 "no judges until every entrant is in"
2006 );
2007 }
2008 let _ = node;
2009 }
2010
2011 let r1 = spawn_round(&mut run);
2013 assert_eq!(r1.len(), 2, "two round-1 judges");
2014 let jm0 = run
2015 .spawn_info(r1[0].0)
2016 .judge_match
2017 .expect("judge carries a match");
2018 assert_eq!(
2019 jm0,
2020 JudgeMatch {
2021 left: node_agent_id(2),
2022 right: node_agent_id(3)
2023 }
2024 );
2025 let jm1 = run
2026 .spawn_info(r1[1].0)
2027 .judge_match
2028 .expect("judge carries a match");
2029 assert_eq!(
2030 jm1,
2031 JudgeMatch {
2032 left: node_agent_id(4),
2033 right: node_agent_id(5)
2034 }
2035 );
2036
2037 run.record_completion(&r1[0].1, judge_done(&node_agent_id(2)));
2039 run.record_completion(&r1[1].1, judge_done(&node_agent_id(4)));
2040 assert!(
2041 run.ready_batch().iter().all(|&n| n != 1),
2042 "dependent gated until the final"
2043 );
2044
2045 let r2 = spawn_round(&mut run);
2047 assert_eq!(r2.len(), 1, "one final judge");
2048 let jmf = run
2049 .spawn_info(r2[0].0)
2050 .judge_match
2051 .expect("final judge carries a match");
2052 assert_eq!(
2053 jmf,
2054 JudgeMatch {
2055 left: node_agent_id(2),
2056 right: node_agent_id(4)
2057 }
2058 );
2059
2060 run.record_completion(&r2[0].1, judge_done(&node_agent_id(4)));
2062 let winner = run
2063 .graph
2064 .get(0)
2065 .and_then(|n| n.result.as_ref())
2066 .and_then(|r| r.tournament_winner.clone());
2067 assert_eq!(
2068 winner.as_deref(),
2069 Some(node_agent_id(4).as_str()),
2070 "champion recorded"
2071 );
2072 assert_eq!(
2073 run.ready_batch(),
2074 vec![1],
2075 "dependent unblocks only after the bracket resolves"
2076 );
2077
2078 let last = spawn_round(&mut run);
2080 assert_eq!(last, vec![(1, node_agent_id(1))]);
2081 run.record_completion(&last[0].1, done());
2082 assert!(run.is_complete());
2083 }
2084
2085 #[test]
2088 fn tournament_with_bye_resolves() {
2089 let spec = WorkflowSpec::new(vec![
2090 WorkflowNode::new(RuntimeTask::new("rank"), AgentRole::Plan).with_tournament(vec![
2091 RuntimeTask::new("x"),
2092 RuntimeTask::new("y"),
2093 RuntimeTask::new("z"),
2094 ]),
2095 ]);
2096 let mut run = WorkflowRun::new(&spec).unwrap();
2097
2098 let entrants = spawn_round(&mut run); assert_eq!(entrants.len(), 3);
2100 for (_, id) in &entrants {
2101 run.record_completion(id, done());
2102 }
2103 let r1 = spawn_round(&mut run);
2105 assert_eq!(r1.len(), 1, "one match, one bye");
2106 run.record_completion(&r1[0].1, judge_done(&node_agent_id(1)));
2107 let r2 = spawn_round(&mut run);
2109 assert_eq!(r2.len(), 1);
2110 let jm = run.spawn_info(r2[0].0).judge_match.unwrap();
2111 assert_eq!(
2112 jm,
2113 JudgeMatch {
2114 left: node_agent_id(1),
2115 right: node_agent_id(3)
2116 }
2117 );
2118 run.record_completion(&r2[0].1, judge_done(&node_agent_id(3)));
2119 let winner = run
2120 .graph
2121 .get(0)
2122 .and_then(|n| n.result.as_ref())
2123 .and_then(|r| r.tournament_winner.clone());
2124 assert_eq!(winner.as_deref(), Some(node_agent_id(3).as_str()));
2125 assert!(run.is_complete());
2126 }
2127
2128 #[test]
2131 fn tournament_children_inherit_controller_trust() {
2132 let spec = WorkflowSpec::new(vec![
2133 WorkflowNode::new(RuntimeTask::new("judge untrusted inputs"), AgentRole::Plan)
2134 .quarantined()
2135 .with_tournament(vec![RuntimeTask::new("a"), RuntimeTask::new("b")]),
2136 ]);
2137 let mut run = WorkflowRun::new(&spec).unwrap();
2138
2139 let entrants = spawn_round(&mut run);
2140 for (node, _) in &entrants {
2141 assert_eq!(
2142 run.spawn_info(*node).trust,
2143 "quarantined",
2144 "entrant inherits quarantine"
2145 );
2146 assert!(
2147 !run.quarantine_violation(*node),
2148 "read-only entrant is quarantine-clean"
2149 );
2150 }
2151 for (_, id) in &entrants {
2152 run.record_completion(id, done());
2153 }
2154 let r1 = spawn_round(&mut run);
2155 assert_eq!(
2156 run.spawn_info(r1[0].0).trust,
2157 "quarantined",
2158 "judge inherits quarantine"
2159 );
2160 assert!(!run.quarantine_violation(r1[0].0));
2161 }
2162
2163 #[test]
2166 fn tournament_controller_never_spawns_itself() {
2167 let spec = WorkflowSpec::new(vec![
2168 WorkflowNode::new(RuntimeTask::new("c"), AgentRole::Plan)
2169 .with_tournament(vec![RuntimeTask::new("a"), RuntimeTask::new("b")]),
2170 ]);
2171 let mut run = WorkflowRun::new(&spec).unwrap();
2172 assert!(matches!(run.nodes[0].kind, NodeKind::Tournament { .. }));
2173 let first = spawn_round(&mut run);
2174 assert!(
2175 first.iter().all(|(n, _)| *n != 0),
2176 "controller node 0 never spawns directly"
2177 );
2178 }
2179
2180 #[test]
2181 fn errored_tournament_child_is_failed_and_no_champion_fails_controller() {
2182 let spec = WorkflowSpec::new(vec![
2185 WorkflowNode::new(RuntimeTask::new("pick"), AgentRole::Plan)
2186 .with_tournament(vec![RuntimeTask::new("x"), RuntimeTask::new("y")]),
2187 WorkflowNode::new(RuntimeTask::new("use winner"), AgentRole::Implement)
2188 .with_depends_on(vec![0]),
2189 ]);
2190 let mut run = WorkflowRun::new(&spec).unwrap();
2191 let entrants = spawn_round(&mut run);
2192 assert_eq!(entrants.len(), 2);
2193 run.record_completion(&entrants[0].1, done());
2194 run.record_completion(
2195 &entrants[1].1,
2196 LoopResult {
2197 termination: TerminationReason::Error,
2198 ..done()
2199 },
2200 );
2201 let judges = spawn_round(&mut run);
2203 assert_eq!(judges.len(), 1, "one match for two entrants");
2204 run.record_completion(&judges[0].1, done()); let failed = outcome_ids(&run, WorkflowNodeStatus::Failed);
2206 assert!(
2207 failed.contains(&entrants[1].1),
2208 "errored entrant reported failed"
2209 );
2210 assert!(
2211 failed.contains(&"wf-node0".to_string()),
2212 "no-champion controller failed"
2213 );
2214 assert!(
2215 !run.ready_batch().contains(&1),
2216 "dependent of the failed controller starves"
2217 );
2218 }
2219
2220 #[test]
2221 fn submitted_tournament_with_one_entrant_is_rejected_atomically() {
2222 let mut run = fanout2();
2223 let before = run.len();
2224 let controller = WorkflowNode::new(RuntimeTask::new("pick"), AgentRole::Plan)
2225 .with_tournament(vec![RuntimeTask::new("only")]);
2226 assert!(run.submit_nodes(vec![controller]).is_err());
2227 assert_eq!(run.len(), before);
2228 }
2229
2230 #[test]
2231 fn submitted_classify_branch_without_classifier_dependency_is_rejected() {
2232 let mut run = fanout2();
2233 let before = run.len();
2234 let classifier = WorkflowNode::new(RuntimeTask::new("route"), AgentRole::Plan)
2235 .with_classify(vec![ClassifyBranch {
2236 label: "a".to_string(),
2237 nodes: vec![1],
2238 }]);
2239 let branch = WorkflowNode::new(RuntimeTask::new("on a"), AgentRole::Implement);
2240 assert!(run.submit_nodes(vec![classifier, branch]).is_err());
2241 assert_eq!(run.len(), before);
2242 }
2243
2244 #[test]
2245 fn submitted_zero_iter_loop_is_rejected() {
2246 let mut run = fanout2();
2247 let before = run.len();
2248 let mut node = WorkflowNode::new(RuntimeTask::new("once"), AgentRole::Implement);
2249 node.kind = NodeKind::Loop { max_iters: 0 };
2250 assert!(run.submit_nodes(vec![node]).is_err());
2251 assert_eq!(run.len(), before);
2252 }
2253
2254 #[test]
2255 fn spawn_info_carries_dep_ids_and_per_node_caps() {
2256 let spec = WorkflowSpec::new(vec![
2259 WorkflowNode::new(RuntimeTask::new("w"), AgentRole::Explore),
2260 WorkflowNode::new(RuntimeTask::new("synth"), AgentRole::Plan)
2261 .with_depends_on(vec![0])
2262 .with_max_turns(4)
2263 .with_max_wall_ms(30_000),
2264 ]);
2265 let run = WorkflowRun::new(&spec).unwrap();
2266 let info = run.spawn_info(1);
2267 assert_eq!(info.input_agent_ids, vec!["wf-node0"]);
2268 assert_eq!(info.max_turns, Some(4));
2269 assert_eq!(info.max_wall_ms, Some(30_000));
2270 assert!(info.reducer.is_none(), "plain node stays non-reduce");
2271 let root = run.spawn_info(0);
2272 assert!(root.input_agent_ids.is_empty());
2273 assert_eq!(root.max_turns, None);
2274 }
2275
2276 #[test]
2285 fn f1_critical_path_node_is_scheduled_before_a_lower_id_leaf() {
2286 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
2287 use crate::scheduler::policy::SchedulerPolicyConfig;
2288 use crate::types::agent::AgentRole;
2289
2290 let spec = WorkflowSpec::new(vec![
2293 WorkflowNode::new(RuntimeTask::new("leaf"), AgentRole::Implement),
2294 WorkflowNode::new(RuntimeTask::new("chain-root"), AgentRole::Implement),
2295 WorkflowNode::new(RuntimeTask::new("mid"), AgentRole::Implement)
2296 .with_depends_on(vec![1]),
2297 WorkflowNode::new(RuntimeTask::new("tail"), AgentRole::Implement)
2298 .with_depends_on(vec![2]),
2299 ]);
2300 let mut run = WorkflowRun::new(&spec).unwrap();
2301 run.set_scheduler_policy(SchedulerPolicyConfig::default());
2302
2303 assert_eq!(
2304 run.ready_batch(),
2305 vec![1, 0],
2306 "the deeper critical path (node 1) outranks the lower-id leaf (node 0)"
2307 );
2308 }
2309
2310 #[test]
2314 fn f2_rearming_loop_does_not_starve_an_independent_node() {
2315 use crate::orchestration::workflow::{WorkflowNode, WorkflowSpec};
2316 use crate::scheduler::policy::SchedulerPolicyConfig;
2317 use crate::types::agent::AgentRole;
2318
2319 let spec = WorkflowSpec::new(vec![
2321 WorkflowNode::new(RuntimeTask::new("loop"), AgentRole::Implement).with_loop(5),
2322 WorkflowNode::new(RuntimeTask::new("independent"), AgentRole::Implement),
2323 ]);
2324 let mut run = WorkflowRun::new(&spec).unwrap();
2325 run.set_scheduler_policy(SchedulerPolicyConfig::default());
2326
2327 let first = run.ready_batch();
2331 assert_eq!(first[0], 0, "loop takes the first slot on the initial tie");
2332 let id = run.current_agent_id(0);
2333 run.mark_spawned(0, &id);
2334 run.record_completion(&id, done()); assert_eq!(
2337 run.ready_batch()[0],
2338 1,
2339 "the independent node runs before the loop's second iteration (no starvation)"
2340 );
2341 }
2342
2343 #[test]
2344 fn workflow_source_factors_survive_checkpoint_rebuild_and_preserve_ready_order() {
2345 use crate::orchestration::task_graph::SchedulingFactors;
2346 use crate::scheduler::policy::SchedulerPolicyConfig;
2347
2348 let spec = WorkflowSpec::new(vec![
2349 WorkflowNode::new(RuntimeTask::new("ordinary"), AgentRole::Implement),
2350 WorkflowNode::new(RuntimeTask::new("urgent"), AgentRole::Implement)
2351 .with_scheduling_factors(SchedulingFactors {
2352 deadline_urgency: 1,
2353 process_priority: 0,
2354 resource_pressure: 0,
2355 budget_pressure: 0,
2356 }),
2357 ]);
2358 let policy = SchedulerPolicyConfig {
2359 critical_path_weight: 0,
2360 fanout_weight: 0,
2361 age_weight: 0,
2362 token_cost_weight: 0,
2363 deadline_weight: 1,
2364 process_priority_weight: 0,
2365 resource_pressure_weight: 0,
2366 budget_pressure_weight: 0,
2367 ..SchedulerPolicyConfig::default()
2368 };
2369 let mut original = WorkflowRun::new(&spec).unwrap();
2370 original.set_scheduler_policy(policy);
2371 assert_eq!(original.ready_batch(), vec![1, 0]);
2372
2373 let states = original.checkpoint_nodes();
2374 let mut restored = WorkflowRun::restore_from_checkpoint(&spec, &states).unwrap();
2375 restored.set_scheduler_policy(policy);
2376 assert_eq!(restored.ready_batch(), vec![1, 0]);
2377 }
2378
2379 #[test]
2382 fn f3_failure_and_partial_propagate_transitively_by_policy() {
2383 use crate::orchestration::workflow::{DependencyPolicy, WorkflowNode, WorkflowSpec};
2384 use crate::types::agent::AgentRole;
2385
2386 let spec = WorkflowSpec::new(vec![
2389 WorkflowNode::new(RuntimeTask::new("a"), AgentRole::Implement),
2390 WorkflowNode::new(RuntimeTask::new("b"), AgentRole::Implement).with_depends_on(vec![0]),
2391 WorkflowNode::new(RuntimeTask::new("c"), AgentRole::Implement).with_depends_on(vec![1]),
2392 ]);
2393 let mut run = WorkflowRun::new(&spec).unwrap();
2394 run.mark_spawned(0, "wf-node0");
2395 run.record_completion("wf-node0", terminated(TerminationReason::Error));
2396 let outcomes = run.finish();
2397 assert_eq!(outcomes[0].status, WorkflowNodeStatus::Failed);
2398 assert_eq!(
2399 outcomes[1].status,
2400 WorkflowNodeStatus::SkippedUpstreamFailed
2401 );
2402 assert_eq!(
2403 outcomes[2].status,
2404 WorkflowNodeStatus::SkippedUpstreamFailed,
2405 "the failure propagates through the whole chain"
2406 );
2407
2408 let spec = WorkflowSpec::new(vec![
2411 WorkflowNode::new(RuntimeTask::new("up"), AgentRole::Implement),
2412 WorkflowNode::new(RuntimeTask::new("strict"), AgentRole::Implement)
2413 .with_depends_on(vec![0]),
2414 WorkflowNode::new(RuntimeTask::new("lenient"), AgentRole::Implement)
2415 .with_depends_on(vec![0])
2416 .with_dependency_policy(DependencyPolicy::AcceptPartial),
2417 ]);
2418 let mut run = WorkflowRun::new(&spec).unwrap();
2419 run.mark_spawned(0, "wf-node0");
2420 run.record_completion("wf-node0", terminated(TerminationReason::Timeout)); assert_eq!(
2422 run.ready_batch(),
2423 vec![2],
2424 "only the AcceptPartial dependent runs"
2425 );
2426 assert_eq!(
2427 run.node_outcomes()[1].status,
2428 WorkflowNodeStatus::SkippedUpstreamFailed,
2429 "the AllSuccess dependent is skipped behind the partial upstream"
2430 );
2431 }
2432
2433 struct Lcg(u64);
2443 impl Lcg {
2444 fn below(&mut self, n: u64) -> u64 {
2445 self.0 = self
2446 .0
2447 .wrapping_mul(6364136223846793005)
2448 .wrapping_add(1442695040888963407);
2449 (self.0 ^ (self.0 >> 33)) % n.max(1)
2450 }
2451 }
2452
2453 #[test]
2454 fn finish_closes_every_node_into_exactly_one_terminal_state_over_random_dags() {
2455 use crate::orchestration::workflow::{DependencyPolicy, WorkflowNode, WorkflowSpec};
2456 use crate::types::agent::AgentRole;
2457 use std::collections::BTreeSet;
2458
2459 let terminations = [
2460 TerminationReason::Completed,
2461 TerminationReason::MaxTurns,
2462 TerminationReason::TokenBudget,
2463 TerminationReason::Timeout,
2464 TerminationReason::ContextOverflow,
2465 TerminationReason::NoProgress,
2466 TerminationReason::MilestoneExceeded,
2467 TerminationReason::Error,
2468 TerminationReason::UserAbort,
2469 ];
2470 let policies = [
2471 DependencyPolicy::AllSuccess,
2472 DependencyPolicy::AcceptPartial,
2473 DependencyPolicy::AllTerminal,
2474 DependencyPolicy::Optional,
2475 ];
2476
2477 for seed in 0..300u64 {
2478 let mut rng = Lcg(seed.wrapping_mul(0x9E3779B97F4A7C15).wrapping_add(1));
2479 let n = 2 + rng.below(7) as usize;
2480
2481 let mut nodes = Vec::new();
2484 for i in 0..n {
2485 let mut deps = Vec::new();
2486 for j in 0..i {
2487 if rng.below(3) == 0 {
2488 deps.push(j);
2489 }
2490 }
2491 let policy = policies[rng.below(policies.len() as u64) as usize];
2492 nodes.push(
2493 WorkflowNode::new(RuntimeTask::new(format!("n{i}")), AgentRole::Implement)
2494 .with_depends_on(deps)
2495 .with_dependency_policy(policy),
2496 );
2497 }
2498 let spec = WorkflowSpec::new(nodes);
2499 let mut run = WorkflowRun::new(&spec).unwrap();
2500
2501 for _ in 0..(n * 4 + 4) {
2504 let ready = run.ready_batch();
2505 if ready.is_empty() {
2506 break;
2507 }
2508 for node in ready {
2509 let agent = node_agent_id(node);
2510 run.mark_spawned(node, &agent);
2511 if rng.below(5) == 0 {
2512 run.mark_denied(node);
2513 } else {
2514 let termination =
2515 terminations[rng.below(terminations.len() as u64) as usize];
2516 run.record_completion(&agent, terminated(termination));
2517 }
2518 }
2519 }
2520
2521 let outcomes = run.finish();
2522 assert_eq!(outcomes.len(), n, "seed {seed}: every node has an outcome");
2524 let ids: BTreeSet<String> = outcomes.iter().map(|o| o.node_id.clone()).collect();
2525 assert_eq!(ids.len(), n, "seed {seed}: node ids are unique");
2526 for node in 0..n {
2527 assert!(
2528 ids.contains(&node_agent_id(node)),
2529 "seed {seed}: node {node} is in the closed outcome set"
2530 );
2531 }
2532 for outcome in &outcomes {
2533 assert!(
2534 matches!(
2535 outcome.status,
2536 WorkflowNodeStatus::Completed
2537 | WorkflowNodeStatus::CompletedPartial
2538 | WorkflowNodeStatus::Failed
2539 | WorkflowNodeStatus::SkippedUpstreamFailed
2540 ),
2541 "seed {seed}: {} is terminal",
2542 outcome.node_id
2543 );
2544 }
2545 }
2546 }
2547}