Skip to main content

clm_rust_core/layer1/
workflow.rs

1//! Layer 1: Pocket Flow Monadic Workflow DAG Orchestrator
2//! Canonical Path: kernel/clm_rust_core/src/layer1/workflow.rs
3
4use std::collections::{HashMap, HashSet};
5use std::path::{Path, PathBuf};
6use serde::{Deserialize, Serialize};
7use serde_json::{json, Value};
8use regex::Regex;
9
10use crate::layer1::action::{ActionManifest, HoareSandwichEngine, HoareSandwichResult};
11
12#[derive(Debug, Clone, Serialize, Deserialize)]
13pub struct WorkflowMetadata {
14    #[serde(default = "default_workflow_type")]
15    pub r#type: String,
16    pub name: String,
17    pub description: Option<String>,
18    #[serde(default = "default_author")]
19    pub author: String,
20}
21
22fn default_workflow_type() -> String { "workflow".to_string() }
23fn default_author() -> String { crate::layer0::vocabulary_identities::DID_CLM_SYSTEM.to_string() }
24
25#[derive(Debug, Clone, Serialize, Deserialize)]
26pub struct WorkflowInput {
27    pub name: String,
28    pub format: Option<String>,
29    pub description: Option<String>,
30}
31
32#[derive(Debug, Clone, Serialize, Deserialize)]
33pub struct WorkflowStep {
34    pub id: String,
35    pub action: String,
36    #[serde(default)]
37    pub inputs: HashMap<String, String>,
38    #[serde(default)]
39    pub depends_on: Vec<String>,
40}
41
42impl WorkflowStep {
43    pub fn extract_implicit_dependencies(&self) -> HashSet<String> {
44        let mut deps = HashSet::new();
45        let re = Regex::new(r"^\$([a-z0-9_\-]+)\.output").unwrap();
46        for val in self.inputs.values() {
47            if let Some(caps) = re.captures(val.trim())
48                && let Some(m) = caps.get(1) {
49                    let ref_name = m.as_str();
50                    if ref_name != "workflow" {
51                        deps.insert(ref_name.to_string());
52                    }
53                }
54        }
55        deps
56    }
57
58    pub fn all_dependencies(&self) -> HashSet<String> {
59        let mut all: HashSet<String> = self.depends_on.iter().cloned().collect();
60        for d in self.extract_implicit_dependencies() {
61            all.insert(d);
62        }
63        all
64    }
65}
66
67#[derive(Debug, Clone, Serialize, Deserialize)]
68pub struct WorkflowOutput {
69    pub name: String,
70    pub source: String,
71    pub format: Option<String>,
72    pub description: Option<String>,
73}
74
75#[derive(Debug, Clone, Serialize, Deserialize)]
76pub struct WorkflowManifest {
77    #[serde(default = "default_version")]
78    pub version: String,
79    pub clm: WorkflowMetadata,
80    #[serde(default)]
81    pub inputs: Vec<WorkflowInput>,
82    pub steps: Vec<WorkflowStep>,
83    #[serde(default)]
84    pub outputs: Vec<WorkflowOutput>,
85}
86
87fn default_version() -> String { "1.0".to_string() }
88
89impl WorkflowManifest {
90    pub fn from_yaml_str(yaml_str: &str) -> Result<Self, String> {
91        let manifest: WorkflowManifest = serde_yaml::from_str(yaml_str)
92            .map_err(|e| format!("Failed to parse YAML WorkflowManifest: {}", e))?;
93        manifest.validate_dag()?;
94        Ok(manifest)
95    }
96
97    pub fn from_yaml_file<P: AsRef<Path>>(path: P) -> Result<Self, String> {
98        let content = std::fs::read_to_string(path.as_ref())
99            .map_err(|e| format!("Cannot read file {}: {}", path.as_ref().display(), e))?;
100        Self::from_yaml_str(&content)
101    }
102
103    pub fn validate_dag(&self) -> Result<(), String> {
104        let mut step_ids = HashSet::new();
105        for s in &self.steps {
106            if step_ids.contains(&s.id) {
107                return Err(format!("Duplicate step id '{}' in workflow '{}'", s.id, self.clm.name));
108            }
109            step_ids.insert(s.id.clone());
110        }
111
112        let mut adj: HashMap<String, HashSet<String>> = HashMap::new();
113        let mut in_degree: HashMap<String, usize> = HashMap::new();
114        for sid in &step_ids {
115            adj.insert(sid.clone(), HashSet::new());
116            in_degree.insert(sid.clone(), 0);
117        }
118
119        for s in &self.steps {
120            for dep in s.all_dependencies() {
121                if !step_ids.contains(&dep) {
122                    return Err(format!("Step '{}' references unknown dependency '{}'", s.id, dep));
123                }
124                if dep == s.id {
125                    return Err(format!("Step '{}' has self-referential dependency", s.id));
126                }
127                if let Some(neighbors) = adj.get_mut(&dep) {
128                    neighbors.insert(s.id.clone());
129                }
130            }
131        }
132
133        for neighbors in adj.values() {
134            for neighbor in neighbors {
135                if let Some(deg) = in_degree.get_mut(neighbor) {
136                    *deg += 1;
137                }
138            }
139        }
140
141        // Kahn's algorithm
142        let mut queue: Vec<String> = in_degree
143            .iter()
144            .filter(|(_, deg)| **deg == 0)
145            .map(|(sid, _)| sid.clone())
146            .collect();
147        queue.sort(); // deterministic ordering
148
149        let mut sorted_order = Vec::new();
150        while !queue.is_empty() {
151            let curr = queue.remove(0);
152            sorted_order.push(curr.clone());
153            if let Some(neighbors) = adj.get(&curr) {
154                let mut next_neighbors: Vec<String> = neighbors.iter().cloned().collect();
155                next_neighbors.sort();
156                for n in next_neighbors {
157                    if let Some(deg) = in_degree.get_mut(&n) {
158                        *deg -= 1;
159                        if *deg == 0 {
160                            queue.push(n);
161                        }
162                    }
163                }
164            }
165        }
166
167        if sorted_order.len() != step_ids.len() {
168            let cycle_nodes: Vec<String> = in_degree
169                .iter()
170                .filter(|(_, deg)| **deg > 0)
171                .map(|(sid, _)| sid.clone())
172                .collect();
173            return Err(format!(
174                "Workflow '{}' contains a circular dependency involving steps: {:?}",
175                self.clm.name, cycle_nodes
176            ));
177        }
178
179        Ok(())
180    }
181
182    pub fn get_topological_order(&self) -> Result<Vec<String>, String> {
183        self.validate_dag()?;
184        let step_ids: HashSet<String> = self.steps.iter().map(|s| s.id.clone()).collect();
185        let mut adj: HashMap<String, HashSet<String>> = HashMap::new();
186        let mut in_degree: HashMap<String, usize> = HashMap::new();
187        for sid in &step_ids {
188            adj.insert(sid.clone(), HashSet::new());
189            in_degree.insert(sid.clone(), 0);
190        }
191        for s in &self.steps {
192            for dep in s.all_dependencies() {
193                if let Some(neighbors) = adj.get_mut(&dep) {
194                    neighbors.insert(s.id.clone());
195                }
196            }
197        }
198        for neighbors in adj.values() {
199            for neighbor in neighbors {
200                if let Some(deg) = in_degree.get_mut(neighbor) {
201                    *deg += 1;
202                }
203            }
204        }
205
206        let mut queue: Vec<String> = in_degree
207            .iter()
208            .filter(|(_, deg)| **deg == 0)
209            .map(|(sid, _)| sid.clone())
210            .collect();
211        queue.sort();
212
213        let mut order = Vec::new();
214        while !queue.is_empty() {
215            let curr = queue.remove(0);
216            order.push(curr.clone());
217            if let Some(neighbors) = adj.get(&curr) {
218                let mut next_neighbors: Vec<String> = neighbors.iter().cloned().collect();
219                next_neighbors.sort();
220                for n in next_neighbors {
221                    if let Some(deg) = in_degree.get_mut(&n) {
222                        *deg -= 1;
223                        if *deg == 0 {
224                            queue.push(n);
225                        }
226                    }
227                }
228            }
229        }
230        Ok(order)
231    }
232
233    pub fn get_execution_stages(&self) -> Result<Vec<Vec<String>>, String> {
234        let order = self.get_topological_order()?;
235        let step_map: HashMap<String, &WorkflowStep> = self.steps.iter().map(|s| (s.id.clone(), s)).collect();
236        let mut depths: HashMap<String, usize> = HashMap::new();
237
238        for sid in &order {
239            let step = step_map.get(sid).unwrap();
240            let deps = step.all_dependencies();
241            if deps.is_empty() {
242                depths.insert(sid.clone(), 0);
243            } else {
244                let max_d = deps.iter().map(|d| depths.get(d).cloned().unwrap_or(0)).max().unwrap_or(0);
245                depths.insert(sid.clone(), max_d + 1);
246            }
247        }
248
249        let max_depth = depths.values().cloned().max().unwrap_or(0);
250        let mut stages: Vec<Vec<String>> = vec![vec![]; max_depth + 1];
251        for sid in order {
252            let d = depths[&sid];
253            stages[d].push(sid);
254        }
255        Ok(stages)
256    }
257}
258
259#[derive(Debug, Clone, Serialize, Deserialize)]
260pub struct WorkflowExecutionResult {
261    pub status: String,
262    pub workflow_name: String,
263    pub outputs: HashMap<String, String>,
264    pub step_results: HashMap<String, HoareSandwichResult>,
265    pub receipts: HashMap<String, String>,
266    pub failed_step: Option<String>,
267    pub error: Option<String>,
268    pub metrics: HashMap<String, Value>,
269}
270
271pub struct PocketFlowWorkflowOrchestrator {
272    pub base_dir: PathBuf,
273}
274
275impl PocketFlowWorkflowOrchestrator {
276    pub fn new<P: AsRef<Path>>(base_dir: P) -> Self {
277        Self {
278            base_dir: base_dir.as_ref().to_path_buf(),
279        }
280    }
281
282    pub fn execute(
283        &self,
284        manifest: &WorkflowManifest,
285        inputs: &HashMap<String, String>,
286    ) -> WorkflowExecutionResult {
287        let t_start = std::time::Instant::now();
288
289        // 1. Initialize scope
290        let mut scope: HashMap<String, String> = HashMap::new();
291        for (k, v) in inputs {
292            scope.insert(format!("workflow.input.{}", k), v.clone());
293            scope.insert(format!("$workflow.input.{}", k), v.clone());
294            scope.insert(k.clone(), v.clone());
295        }
296        if let Some((_, first_val)) = inputs.iter().next() {
297            scope.insert("workflow.input".to_string(), first_val.clone());
298            scope.insert("$workflow.input".to_string(), first_val.clone());
299        }
300
301        // 2. Validate DAG and get topological order
302        let order = match manifest.get_topological_order() {
303            Ok(ord) => ord,
304            Err(e) => {
305                let mut metrics = HashMap::new();
306                metrics.insert("wall_time_ms".to_string(), json!(0.0));
307                return WorkflowExecutionResult {
308                    status: "failed".to_string(),
309                    workflow_name: manifest.clm.name.clone(),
310                    outputs: HashMap::new(),
311                    step_results: HashMap::new(),
312                    receipts: HashMap::new(),
313                    failed_step: None,
314                    error: Some(e),
315                    metrics,
316                };
317            }
318        };
319
320        let step_map: HashMap<String, &WorkflowStep> = manifest.steps.iter().map(|s| (s.id.clone(), s)).collect();
321        let mut step_results: HashMap<String, HoareSandwichResult> = HashMap::new();
322        let mut receipts: HashMap<String, String> = HashMap::new();
323
324        for sid in &order {
325            let step = step_map[sid];
326
327            // Resolve action path with multi-fallback
328            let action_p = Path::new(&step.action);
329            let mut resolved_action_path = None;
330
331            if action_p.is_file() {
332                resolved_action_path = Some(action_p.to_path_buf());
333            } else if self.base_dir.join(action_p).is_file() {
334                resolved_action_path = Some(self.base_dir.join(action_p));
335            } else if let Some(fname) = action_p.file_name()
336                && self.base_dir.join(fname).is_file() {
337                    resolved_action_path = Some(self.base_dir.join(fname));
338                }
339
340            if resolved_action_path.is_none()
341                && let Ok(mut dir) = std::env::current_dir() {
342                    for _ in 0..5 {
343                        if dir.join(action_p).is_file() {
344                            resolved_action_path = Some(dir.join(action_p));
345                            break;
346                        }
347                        if !dir.pop() {
348                            break;
349                        }
350                    }
351                }
352
353            let action_path = resolved_action_path.unwrap_or_else(|| self.base_dir.join(&step.action));
354
355            let action_manifest = match ActionManifest::from_yaml_file(&action_path) {
356                Ok(m) => m,
357                Err(e) => {
358                    let elapsed_ms = t_start.elapsed().as_secs_f64() * 1000.0;
359                    let mut metrics = HashMap::new();
360                    metrics.insert("wall_time_ms".to_string(), json!((elapsed_ms * 1000.0).round() / 1000.0));
361                    return WorkflowExecutionResult {
362                        status: "failed".to_string(),
363                        workflow_name: manifest.clm.name.clone(),
364                        outputs: HashMap::new(),
365                        step_results,
366                        receipts,
367                        failed_step: Some(sid.clone()),
368                        error: Some(format!("Failed to load action '{}': {}", step.action, e)),
369                        metrics,
370                    };
371                }
372            };
373
374            // Bind step inputs from scope
375            let mut step_bindings = HashMap::new();
376            for (inp_name, binding_expr) in &step.inputs {
377                let expr = binding_expr.trim();
378                let resolved = scope.get(expr)
379                    .or_else(|| scope.get(expr.trim_start_matches('$')))
380                    .cloned()
381                    .unwrap_or_else(|| {
382                        // If binding_expr is a path to a file or literal
383                        if Path::new(expr).is_file() {
384                            std::fs::read_to_string(expr).unwrap_or_else(|_| expr.to_string())
385                        } else {
386                            expr.to_string()
387                        }
388                    });
389                step_bindings.insert(inp_name.clone(), resolved);
390            }
391
392            // Execute Hoare Sandwich for step
393            let step_res = HoareSandwichEngine::execute(&action_manifest, &step_bindings);
394
395            if step_res.status != "success" {
396                let elapsed_ms = t_start.elapsed().as_secs_f64() * 1000.0;
397                let mut metrics = HashMap::new();
398                metrics.insert("wall_time_ms".to_string(), json!((elapsed_ms * 1000.0).round() / 1000.0));
399                step_results.insert(sid.clone(), step_res.clone());
400                return WorkflowExecutionResult {
401                    status: "failed".to_string(),
402                    workflow_name: manifest.clm.name.clone(),
403                    outputs: HashMap::new(),
404                    step_results,
405                    receipts,
406                    failed_step: Some(sid.clone()),
407                    error: step_res.error,
408                    metrics,
409                };
410            }
411
412            // Record receipt
413            if let Some(r) = &step_res.receipt {
414                receipts.insert(sid.clone(), r.uri.clone());
415            }
416
417            // Record outputs into scope
418            if let Some((_, first_out)) = step_res.outputs.iter().next() {
419                scope.insert(format!("{}.output", sid), first_out.clone());
420                scope.insert(format!("${}.output", sid), first_out.clone());
421            }
422            for (out_name, out_payload) in &step_res.outputs {
423                scope.insert(format!("{}.output.{}", sid, out_name), out_payload.clone());
424                scope.insert(format!("${}.output.{}", sid, out_name), out_payload.clone());
425            }
426
427            step_results.insert(sid.clone(), step_res);
428        }
429
430        // Map declared workflow outputs
431        let mut workflow_outputs = HashMap::new();
432        for wf_out in &manifest.outputs {
433            let source_expr = wf_out.source.trim();
434            let clean_expr = source_expr.trim_start_matches('$');
435            let resolved = scope.get(source_expr)
436                .or_else(|| scope.get(clean_expr))
437                .cloned()
438                .unwrap_or_default();
439            workflow_outputs.insert(wf_out.name.clone(), resolved);
440        }
441
442        let elapsed_ms = t_start.elapsed().as_secs_f64() * 1000.0;
443        let mut metrics = HashMap::new();
444        metrics.insert("wall_time_ms".to_string(), json!((elapsed_ms * 1000.0).round() / 1000.0));
445        metrics.insert("completed_steps".to_string(), json!(step_results.len()));
446
447        WorkflowExecutionResult {
448            status: "success".to_string(),
449            workflow_name: manifest.clm.name.clone(),
450            outputs: workflow_outputs,
451            step_results,
452            receipts,
453            failed_step: None,
454            error: None,
455            metrics,
456        }
457    }
458}