Skip to main content

clm_rust_core/layer1/
workflow.rs

1//! Layer 1: Pocket Flow Monadic Workflow DAG Orchestrator
2//! Canonical Path: kernel/clm_rust_core/src/layer1/workflow.rs
3
4use std::collections::{HashMap, HashSet};
5use std::path::{Path, PathBuf};
6use serde::{Deserialize, Serialize};
7use serde_json::{json, Value};
8use regex::Regex;
9
10use crate::layer1::action::{ActionManifest, HoareSandwichEngine, HoareSandwichResult};
11
12#[derive(Debug, Clone, Serialize, Deserialize)]
13pub struct WorkflowMetadata {
14    #[serde(default = "default_workflow_type")]
15    pub r#type: String,
16    pub name: String,
17    pub description: Option<String>,
18    #[serde(default = "default_author")]
19    pub author: String,
20}
21
22fn default_workflow_type() -> String { "workflow".to_string() }
23fn default_author() -> String { "did:clm:system".to_string() }
24
25#[derive(Debug, Clone, Serialize, Deserialize)]
26pub struct WorkflowInput {
27    pub name: String,
28    pub format: Option<String>,
29    pub description: Option<String>,
30}
31
32#[derive(Debug, Clone, Serialize, Deserialize)]
33pub struct WorkflowStep {
34    pub id: String,
35    pub action: String,
36    #[serde(default)]
37    pub inputs: HashMap<String, String>,
38    #[serde(default)]
39    pub depends_on: Vec<String>,
40}
41
42impl WorkflowStep {
43    pub fn extract_implicit_dependencies(&self) -> HashSet<String> {
44        let mut deps = HashSet::new();
45        let re = Regex::new(r"^\$([a-z0-9_\-]+)\.output").unwrap();
46        for val in self.inputs.values() {
47            if let Some(caps) = re.captures(val.trim()) {
48                if let Some(m) = caps.get(1) {
49                    let ref_name = m.as_str();
50                    if ref_name != "workflow" {
51                        deps.insert(ref_name.to_string());
52                    }
53                }
54            }
55        }
56        deps
57    }
58
59    pub fn all_dependencies(&self) -> HashSet<String> {
60        let mut all: HashSet<String> = self.depends_on.iter().cloned().collect();
61        for d in self.extract_implicit_dependencies() {
62            all.insert(d);
63        }
64        all
65    }
66}
67
68#[derive(Debug, Clone, Serialize, Deserialize)]
69pub struct WorkflowOutput {
70    pub name: String,
71    pub source: String,
72    pub format: Option<String>,
73    pub description: Option<String>,
74}
75
76#[derive(Debug, Clone, Serialize, Deserialize)]
77pub struct WorkflowManifest {
78    #[serde(default = "default_version")]
79    pub version: String,
80    pub clm: WorkflowMetadata,
81    #[serde(default)]
82    pub inputs: Vec<WorkflowInput>,
83    pub steps: Vec<WorkflowStep>,
84    #[serde(default)]
85    pub outputs: Vec<WorkflowOutput>,
86}
87
88fn default_version() -> String { "1.0".to_string() }
89
90impl WorkflowManifest {
91    pub fn from_yaml_str(yaml_str: &str) -> Result<Self, String> {
92        let manifest: WorkflowManifest = serde_yaml::from_str(yaml_str)
93            .map_err(|e| format!("Failed to parse YAML WorkflowManifest: {}", e))?;
94        manifest.validate_dag()?;
95        Ok(manifest)
96    }
97
98    pub fn from_yaml_file<P: AsRef<Path>>(path: P) -> Result<Self, String> {
99        let content = std::fs::read_to_string(path.as_ref())
100            .map_err(|e| format!("Cannot read file {}: {}", path.as_ref().display(), e))?;
101        Self::from_yaml_str(&content)
102    }
103
104    pub fn validate_dag(&self) -> Result<(), String> {
105        let mut step_ids = HashSet::new();
106        for s in &self.steps {
107            if step_ids.contains(&s.id) {
108                return Err(format!("Duplicate step id '{}' in workflow '{}'", s.id, self.clm.name));
109            }
110            step_ids.insert(s.id.clone());
111        }
112
113        let mut adj: HashMap<String, HashSet<String>> = HashMap::new();
114        let mut in_degree: HashMap<String, usize> = HashMap::new();
115        for sid in &step_ids {
116            adj.insert(sid.clone(), HashSet::new());
117            in_degree.insert(sid.clone(), 0);
118        }
119
120        for s in &self.steps {
121            for dep in s.all_dependencies() {
122                if !step_ids.contains(&dep) {
123                    return Err(format!("Step '{}' references unknown dependency '{}'", s.id, dep));
124                }
125                if dep == s.id {
126                    return Err(format!("Step '{}' has self-referential dependency", s.id));
127                }
128                if let Some(neighbors) = adj.get_mut(&dep) {
129                    neighbors.insert(s.id.clone());
130                }
131            }
132        }
133
134        for (_, neighbors) in &adj {
135            for neighbor in neighbors {
136                if let Some(deg) = in_degree.get_mut(neighbor) {
137                    *deg += 1;
138                }
139            }
140        }
141
142        // Kahn's algorithm
143        let mut queue: Vec<String> = in_degree
144            .iter()
145            .filter(|(_, deg)| **deg == 0)
146            .map(|(sid, _)| sid.clone())
147            .collect();
148        queue.sort(); // deterministic ordering
149
150        let mut sorted_order = Vec::new();
151        while !queue.is_empty() {
152            let curr = queue.remove(0);
153            sorted_order.push(curr.clone());
154            if let Some(neighbors) = adj.get(&curr) {
155                let mut next_neighbors: Vec<String> = neighbors.iter().cloned().collect();
156                next_neighbors.sort();
157                for n in next_neighbors {
158                    if let Some(deg) = in_degree.get_mut(&n) {
159                        *deg -= 1;
160                        if *deg == 0 {
161                            queue.push(n);
162                        }
163                    }
164                }
165            }
166        }
167
168        if sorted_order.len() != step_ids.len() {
169            let cycle_nodes: Vec<String> = in_degree
170                .iter()
171                .filter(|(_, deg)| **deg > 0)
172                .map(|(sid, _)| sid.clone())
173                .collect();
174            return Err(format!(
175                "Workflow '{}' contains a circular dependency involving steps: {:?}",
176                self.clm.name, cycle_nodes
177            ));
178        }
179
180        Ok(())
181    }
182
183    pub fn get_topological_order(&self) -> Result<Vec<String>, String> {
184        self.validate_dag()?;
185        let step_ids: HashSet<String> = self.steps.iter().map(|s| s.id.clone()).collect();
186        let mut adj: HashMap<String, HashSet<String>> = HashMap::new();
187        let mut in_degree: HashMap<String, usize> = HashMap::new();
188        for sid in &step_ids {
189            adj.insert(sid.clone(), HashSet::new());
190            in_degree.insert(sid.clone(), 0);
191        }
192        for s in &self.steps {
193            for dep in s.all_dependencies() {
194                if let Some(neighbors) = adj.get_mut(&dep) {
195                    neighbors.insert(s.id.clone());
196                }
197            }
198        }
199        for (_, neighbors) in &adj {
200            for neighbor in neighbors {
201                if let Some(deg) = in_degree.get_mut(neighbor) {
202                    *deg += 1;
203                }
204            }
205        }
206
207        let mut queue: Vec<String> = in_degree
208            .iter()
209            .filter(|(_, deg)| **deg == 0)
210            .map(|(sid, _)| sid.clone())
211            .collect();
212        queue.sort();
213
214        let mut order = Vec::new();
215        while !queue.is_empty() {
216            let curr = queue.remove(0);
217            order.push(curr.clone());
218            if let Some(neighbors) = adj.get(&curr) {
219                let mut next_neighbors: Vec<String> = neighbors.iter().cloned().collect();
220                next_neighbors.sort();
221                for n in next_neighbors {
222                    if let Some(deg) = in_degree.get_mut(&n) {
223                        *deg -= 1;
224                        if *deg == 0 {
225                            queue.push(n);
226                        }
227                    }
228                }
229            }
230        }
231        Ok(order)
232    }
233
234    pub fn get_execution_stages(&self) -> Result<Vec<Vec<String>>, String> {
235        let order = self.get_topological_order()?;
236        let step_map: HashMap<String, &WorkflowStep> = self.steps.iter().map(|s| (s.id.clone(), s)).collect();
237        let mut depths: HashMap<String, usize> = HashMap::new();
238
239        for sid in &order {
240            let step = step_map.get(sid).unwrap();
241            let deps = step.all_dependencies();
242            if deps.is_empty() {
243                depths.insert(sid.clone(), 0);
244            } else {
245                let max_d = deps.iter().map(|d| depths.get(d).cloned().unwrap_or(0)).max().unwrap_or(0);
246                depths.insert(sid.clone(), max_d + 1);
247            }
248        }
249
250        let max_depth = depths.values().cloned().max().unwrap_or(0);
251        let mut stages: Vec<Vec<String>> = vec![vec![]; max_depth + 1];
252        for sid in order {
253            let d = depths[&sid];
254            stages[d].push(sid);
255        }
256        Ok(stages)
257    }
258}
259
260#[derive(Debug, Clone, Serialize, Deserialize)]
261pub struct WorkflowExecutionResult {
262    pub status: String,
263    pub workflow_name: String,
264    pub outputs: HashMap<String, String>,
265    pub step_results: HashMap<String, HoareSandwichResult>,
266    pub receipts: HashMap<String, String>,
267    pub failed_step: Option<String>,
268    pub error: Option<String>,
269    pub metrics: HashMap<String, Value>,
270}
271
272pub struct PocketFlowWorkflowOrchestrator {
273    pub base_dir: PathBuf,
274}
275
276impl PocketFlowWorkflowOrchestrator {
277    pub fn new<P: AsRef<Path>>(base_dir: P) -> Self {
278        Self {
279            base_dir: base_dir.as_ref().to_path_buf(),
280        }
281    }
282
283    pub fn execute(
284        &self,
285        manifest: &WorkflowManifest,
286        inputs: &HashMap<String, String>,
287    ) -> WorkflowExecutionResult {
288        let t_start = std::time::Instant::now();
289
290        // 1. Initialize scope
291        let mut scope: HashMap<String, String> = HashMap::new();
292        for (k, v) in inputs {
293            scope.insert(format!("workflow.input.{}", k), v.clone());
294            scope.insert(format!("$workflow.input.{}", k), v.clone());
295            scope.insert(k.clone(), v.clone());
296        }
297        if let Some((_, first_val)) = inputs.iter().next() {
298            scope.insert("workflow.input".to_string(), first_val.clone());
299            scope.insert("$workflow.input".to_string(), first_val.clone());
300        }
301
302        // 2. Validate DAG and get topological order
303        let order = match manifest.get_topological_order() {
304            Ok(ord) => ord,
305            Err(e) => {
306                let mut metrics = HashMap::new();
307                metrics.insert("wall_time_ms".to_string(), json!(0.0));
308                return WorkflowExecutionResult {
309                    status: "failed".to_string(),
310                    workflow_name: manifest.clm.name.clone(),
311                    outputs: HashMap::new(),
312                    step_results: HashMap::new(),
313                    receipts: HashMap::new(),
314                    failed_step: None,
315                    error: Some(e),
316                    metrics,
317                };
318            }
319        };
320
321        let step_map: HashMap<String, &WorkflowStep> = manifest.steps.iter().map(|s| (s.id.clone(), s)).collect();
322        let mut step_results: HashMap<String, HoareSandwichResult> = HashMap::new();
323        let mut receipts: HashMap<String, String> = HashMap::new();
324
325        for sid in &order {
326            let step = step_map[sid];
327
328            // Resolve action path with multi-fallback
329            let action_p = Path::new(&step.action);
330            let mut resolved_action_path = None;
331
332            if action_p.is_file() {
333                resolved_action_path = Some(action_p.to_path_buf());
334            } else if self.base_dir.join(action_p).is_file() {
335                resolved_action_path = Some(self.base_dir.join(action_p));
336            } else if let Some(fname) = action_p.file_name() {
337                if self.base_dir.join(fname).is_file() {
338                    resolved_action_path = Some(self.base_dir.join(fname));
339                }
340            }
341
342            if resolved_action_path.is_none() {
343                if let Ok(mut dir) = std::env::current_dir() {
344                    for _ in 0..5 {
345                        if dir.join(action_p).is_file() {
346                            resolved_action_path = Some(dir.join(action_p));
347                            break;
348                        }
349                        if !dir.pop() {
350                            break;
351                        }
352                    }
353                }
354            }
355
356            let action_path = resolved_action_path.unwrap_or_else(|| self.base_dir.join(&step.action));
357
358            let action_manifest = match ActionManifest::from_yaml_file(&action_path) {
359                Ok(m) => m,
360                Err(e) => {
361                    let elapsed_ms = t_start.elapsed().as_secs_f64() * 1000.0;
362                    let mut metrics = HashMap::new();
363                    metrics.insert("wall_time_ms".to_string(), json!((elapsed_ms * 1000.0).round() / 1000.0));
364                    return WorkflowExecutionResult {
365                        status: "failed".to_string(),
366                        workflow_name: manifest.clm.name.clone(),
367                        outputs: HashMap::new(),
368                        step_results,
369                        receipts,
370                        failed_step: Some(sid.clone()),
371                        error: Some(format!("Failed to load action '{}': {}", step.action, e)),
372                        metrics,
373                    };
374                }
375            };
376
377            // Bind step inputs from scope
378            let mut step_bindings = HashMap::new();
379            for (inp_name, binding_expr) in &step.inputs {
380                let expr = binding_expr.trim();
381                let resolved = scope.get(expr)
382                    .or_else(|| scope.get(expr.trim_start_matches('$')))
383                    .cloned()
384                    .unwrap_or_else(|| {
385                        // If binding_expr is a path to a file or literal
386                        if Path::new(expr).is_file() {
387                            std::fs::read_to_string(expr).unwrap_or_else(|_| expr.to_string())
388                        } else {
389                            expr.to_string()
390                        }
391                    });
392                step_bindings.insert(inp_name.clone(), resolved);
393            }
394
395            // Execute Hoare Sandwich for step
396            let step_res = HoareSandwichEngine::execute(&action_manifest, &step_bindings);
397
398            if step_res.status != "success" {
399                let elapsed_ms = t_start.elapsed().as_secs_f64() * 1000.0;
400                let mut metrics = HashMap::new();
401                metrics.insert("wall_time_ms".to_string(), json!((elapsed_ms * 1000.0).round() / 1000.0));
402                step_results.insert(sid.clone(), step_res.clone());
403                return WorkflowExecutionResult {
404                    status: "failed".to_string(),
405                    workflow_name: manifest.clm.name.clone(),
406                    outputs: HashMap::new(),
407                    step_results,
408                    receipts,
409                    failed_step: Some(sid.clone()),
410                    error: step_res.error,
411                    metrics,
412                };
413            }
414
415            // Record receipt
416            if let Some(r) = &step_res.receipt {
417                receipts.insert(sid.clone(), r.uri.clone());
418            }
419
420            // Record outputs into scope
421            if let Some((_, first_out)) = step_res.outputs.iter().next() {
422                scope.insert(format!("{}.output", sid), first_out.clone());
423                scope.insert(format!("${}.output", sid), first_out.clone());
424            }
425            for (out_name, out_payload) in &step_res.outputs {
426                scope.insert(format!("{}.output.{}", sid, out_name), out_payload.clone());
427                scope.insert(format!("${}.output.{}", sid, out_name), out_payload.clone());
428            }
429
430            step_results.insert(sid.clone(), step_res);
431        }
432
433        // Map declared workflow outputs
434        let mut workflow_outputs = HashMap::new();
435        for wf_out in &manifest.outputs {
436            let source_expr = wf_out.source.trim();
437            let clean_expr = source_expr.trim_start_matches('$');
438            let resolved = scope.get(source_expr)
439                .or_else(|| scope.get(clean_expr))
440                .cloned()
441                .unwrap_or_default();
442            workflow_outputs.insert(wf_out.name.clone(), resolved);
443        }
444
445        let elapsed_ms = t_start.elapsed().as_secs_f64() * 1000.0;
446        let mut metrics = HashMap::new();
447        metrics.insert("wall_time_ms".to_string(), json!((elapsed_ms * 1000.0).round() / 1000.0));
448        metrics.insert("completed_steps".to_string(), json!(step_results.len()));
449
450        WorkflowExecutionResult {
451            status: "success".to_string(),
452            workflow_name: manifest.clm.name.clone(),
453            outputs: workflow_outputs,
454            step_results,
455            receipts,
456            failed_step: None,
457            error: None,
458            metrics,
459        }
460    }
461}