use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use regex::Regex;
use crate::layer1::action::{ActionManifest, HoareSandwichEngine, HoareSandwichResult};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WorkflowMetadata {
#[serde(default = "default_workflow_type")]
pub r#type: String,
pub name: String,
pub description: Option<String>,
#[serde(default = "default_author")]
pub author: String,
}
fn default_workflow_type() -> String { "workflow".to_string() }
fn default_author() -> String { crate::layer0::vocabulary_identities::DID_CLM_SYSTEM.to_string() }
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WorkflowInput {
pub name: String,
pub format: Option<String>,
pub description: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WorkflowStep {
pub id: String,
pub action: String,
#[serde(default)]
pub inputs: HashMap<String, String>,
#[serde(default)]
pub depends_on: Vec<String>,
}
impl WorkflowStep {
pub fn extract_implicit_dependencies(&self) -> HashSet<String> {
let mut deps = HashSet::new();
let re = Regex::new(r"^\$([a-z0-9_\-]+)\.output").unwrap();
for val in self.inputs.values() {
if let Some(caps) = re.captures(val.trim())
&& let Some(m) = caps.get(1) {
let ref_name = m.as_str();
if ref_name != "workflow" {
deps.insert(ref_name.to_string());
}
}
}
deps
}
pub fn all_dependencies(&self) -> HashSet<String> {
let mut all: HashSet<String> = self.depends_on.iter().cloned().collect();
for d in self.extract_implicit_dependencies() {
all.insert(d);
}
all
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WorkflowOutput {
pub name: String,
pub source: String,
pub format: Option<String>,
pub description: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WorkflowManifest {
#[serde(default = "default_version")]
pub version: String,
pub clm: WorkflowMetadata,
#[serde(default)]
pub inputs: Vec<WorkflowInput>,
pub steps: Vec<WorkflowStep>,
#[serde(default)]
pub outputs: Vec<WorkflowOutput>,
}
fn default_version() -> String { "1.0".to_string() }
impl WorkflowManifest {
pub fn from_yaml_str(yaml_str: &str) -> Result<Self, String> {
let manifest: WorkflowManifest = serde_yaml::from_str(yaml_str)
.map_err(|e| format!("Failed to parse YAML WorkflowManifest: {}", e))?;
manifest.validate_dag()?;
Ok(manifest)
}
pub fn from_yaml_file<P: AsRef<Path>>(path: P) -> Result<Self, String> {
let content = std::fs::read_to_string(path.as_ref())
.map_err(|e| format!("Cannot read file {}: {}", path.as_ref().display(), e))?;
Self::from_yaml_str(&content)
}
pub fn validate_dag(&self) -> Result<(), String> {
let mut step_ids = HashSet::new();
for s in &self.steps {
if step_ids.contains(&s.id) {
return Err(format!("Duplicate step id '{}' in workflow '{}'", s.id, self.clm.name));
}
step_ids.insert(s.id.clone());
}
let mut adj: HashMap<String, HashSet<String>> = HashMap::new();
let mut in_degree: HashMap<String, usize> = HashMap::new();
for sid in &step_ids {
adj.insert(sid.clone(), HashSet::new());
in_degree.insert(sid.clone(), 0);
}
for s in &self.steps {
for dep in s.all_dependencies() {
if !step_ids.contains(&dep) {
return Err(format!("Step '{}' references unknown dependency '{}'", s.id, dep));
}
if dep == s.id {
return Err(format!("Step '{}' has self-referential dependency", s.id));
}
if let Some(neighbors) = adj.get_mut(&dep) {
neighbors.insert(s.id.clone());
}
}
}
for neighbors in adj.values() {
for neighbor in neighbors {
if let Some(deg) = in_degree.get_mut(neighbor) {
*deg += 1;
}
}
}
let mut queue: Vec<String> = in_degree
.iter()
.filter(|(_, deg)| **deg == 0)
.map(|(sid, _)| sid.clone())
.collect();
queue.sort();
let mut sorted_order = Vec::new();
while !queue.is_empty() {
let curr = queue.remove(0);
sorted_order.push(curr.clone());
if let Some(neighbors) = adj.get(&curr) {
let mut next_neighbors: Vec<String> = neighbors.iter().cloned().collect();
next_neighbors.sort();
for n in next_neighbors {
if let Some(deg) = in_degree.get_mut(&n) {
*deg -= 1;
if *deg == 0 {
queue.push(n);
}
}
}
}
}
if sorted_order.len() != step_ids.len() {
let cycle_nodes: Vec<String> = in_degree
.iter()
.filter(|(_, deg)| **deg > 0)
.map(|(sid, _)| sid.clone())
.collect();
return Err(format!(
"Workflow '{}' contains a circular dependency involving steps: {:?}",
self.clm.name, cycle_nodes
));
}
Ok(())
}
pub fn get_topological_order(&self) -> Result<Vec<String>, String> {
self.validate_dag()?;
let step_ids: HashSet<String> = self.steps.iter().map(|s| s.id.clone()).collect();
let mut adj: HashMap<String, HashSet<String>> = HashMap::new();
let mut in_degree: HashMap<String, usize> = HashMap::new();
for sid in &step_ids {
adj.insert(sid.clone(), HashSet::new());
in_degree.insert(sid.clone(), 0);
}
for s in &self.steps {
for dep in s.all_dependencies() {
if let Some(neighbors) = adj.get_mut(&dep) {
neighbors.insert(s.id.clone());
}
}
}
for neighbors in adj.values() {
for neighbor in neighbors {
if let Some(deg) = in_degree.get_mut(neighbor) {
*deg += 1;
}
}
}
let mut queue: Vec<String> = in_degree
.iter()
.filter(|(_, deg)| **deg == 0)
.map(|(sid, _)| sid.clone())
.collect();
queue.sort();
let mut order = Vec::new();
while !queue.is_empty() {
let curr = queue.remove(0);
order.push(curr.clone());
if let Some(neighbors) = adj.get(&curr) {
let mut next_neighbors: Vec<String> = neighbors.iter().cloned().collect();
next_neighbors.sort();
for n in next_neighbors {
if let Some(deg) = in_degree.get_mut(&n) {
*deg -= 1;
if *deg == 0 {
queue.push(n);
}
}
}
}
}
Ok(order)
}
pub fn get_execution_stages(&self) -> Result<Vec<Vec<String>>, String> {
let order = self.get_topological_order()?;
let step_map: HashMap<String, &WorkflowStep> = self.steps.iter().map(|s| (s.id.clone(), s)).collect();
let mut depths: HashMap<String, usize> = HashMap::new();
for sid in &order {
let step = step_map.get(sid).unwrap();
let deps = step.all_dependencies();
if deps.is_empty() {
depths.insert(sid.clone(), 0);
} else {
let max_d = deps.iter().map(|d| depths.get(d).cloned().unwrap_or(0)).max().unwrap_or(0);
depths.insert(sid.clone(), max_d + 1);
}
}
let max_depth = depths.values().cloned().max().unwrap_or(0);
let mut stages: Vec<Vec<String>> = vec![vec![]; max_depth + 1];
for sid in order {
let d = depths[&sid];
stages[d].push(sid);
}
Ok(stages)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WorkflowExecutionResult {
pub status: String,
pub workflow_name: String,
pub outputs: HashMap<String, String>,
pub step_results: HashMap<String, HoareSandwichResult>,
pub receipts: HashMap<String, String>,
pub failed_step: Option<String>,
pub error: Option<String>,
pub metrics: HashMap<String, Value>,
}
pub struct PocketFlowWorkflowOrchestrator {
pub base_dir: PathBuf,
}
impl PocketFlowWorkflowOrchestrator {
pub fn new<P: AsRef<Path>>(base_dir: P) -> Self {
Self {
base_dir: base_dir.as_ref().to_path_buf(),
}
}
pub fn execute(
&self,
manifest: &WorkflowManifest,
inputs: &HashMap<String, String>,
) -> WorkflowExecutionResult {
let t_start = std::time::Instant::now();
let mut scope: HashMap<String, String> = HashMap::new();
for (k, v) in inputs {
scope.insert(format!("workflow.input.{}", k), v.clone());
scope.insert(format!("$workflow.input.{}", k), v.clone());
scope.insert(k.clone(), v.clone());
}
if let Some((_, first_val)) = inputs.iter().next() {
scope.insert("workflow.input".to_string(), first_val.clone());
scope.insert("$workflow.input".to_string(), first_val.clone());
}
let order = match manifest.get_topological_order() {
Ok(ord) => ord,
Err(e) => {
let mut metrics = HashMap::new();
metrics.insert("wall_time_ms".to_string(), json!(0.0));
return WorkflowExecutionResult {
status: "failed".to_string(),
workflow_name: manifest.clm.name.clone(),
outputs: HashMap::new(),
step_results: HashMap::new(),
receipts: HashMap::new(),
failed_step: None,
error: Some(e),
metrics,
};
}
};
let step_map: HashMap<String, &WorkflowStep> = manifest.steps.iter().map(|s| (s.id.clone(), s)).collect();
let mut step_results: HashMap<String, HoareSandwichResult> = HashMap::new();
let mut receipts: HashMap<String, String> = HashMap::new();
for sid in &order {
let step = step_map[sid];
let action_p = Path::new(&step.action);
let mut resolved_action_path = None;
if action_p.is_file() {
resolved_action_path = Some(action_p.to_path_buf());
} else if self.base_dir.join(action_p).is_file() {
resolved_action_path = Some(self.base_dir.join(action_p));
} else if let Some(fname) = action_p.file_name()
&& self.base_dir.join(fname).is_file() {
resolved_action_path = Some(self.base_dir.join(fname));
}
if resolved_action_path.is_none()
&& let Ok(mut dir) = std::env::current_dir() {
for _ in 0..5 {
if dir.join(action_p).is_file() {
resolved_action_path = Some(dir.join(action_p));
break;
}
if !dir.pop() {
break;
}
}
}
let action_path = resolved_action_path.unwrap_or_else(|| self.base_dir.join(&step.action));
let action_manifest = match ActionManifest::from_yaml_file(&action_path) {
Ok(m) => m,
Err(e) => {
let elapsed_ms = t_start.elapsed().as_secs_f64() * 1000.0;
let mut metrics = HashMap::new();
metrics.insert("wall_time_ms".to_string(), json!((elapsed_ms * 1000.0).round() / 1000.0));
return WorkflowExecutionResult {
status: "failed".to_string(),
workflow_name: manifest.clm.name.clone(),
outputs: HashMap::new(),
step_results,
receipts,
failed_step: Some(sid.clone()),
error: Some(format!("Failed to load action '{}': {}", step.action, e)),
metrics,
};
}
};
let mut step_bindings = HashMap::new();
for (inp_name, binding_expr) in &step.inputs {
let expr = binding_expr.trim();
let resolved = scope.get(expr)
.or_else(|| scope.get(expr.trim_start_matches('$')))
.cloned()
.unwrap_or_else(|| {
if Path::new(expr).is_file() {
std::fs::read_to_string(expr).unwrap_or_else(|_| expr.to_string())
} else {
expr.to_string()
}
});
step_bindings.insert(inp_name.clone(), resolved);
}
let step_res = HoareSandwichEngine::execute(&action_manifest, &step_bindings);
if step_res.status != "success" {
let elapsed_ms = t_start.elapsed().as_secs_f64() * 1000.0;
let mut metrics = HashMap::new();
metrics.insert("wall_time_ms".to_string(), json!((elapsed_ms * 1000.0).round() / 1000.0));
step_results.insert(sid.clone(), step_res.clone());
return WorkflowExecutionResult {
status: "failed".to_string(),
workflow_name: manifest.clm.name.clone(),
outputs: HashMap::new(),
step_results,
receipts,
failed_step: Some(sid.clone()),
error: step_res.error,
metrics,
};
}
if let Some(r) = &step_res.receipt {
receipts.insert(sid.clone(), r.uri.clone());
}
if let Some((_, first_out)) = step_res.outputs.iter().next() {
scope.insert(format!("{}.output", sid), first_out.clone());
scope.insert(format!("${}.output", sid), first_out.clone());
}
for (out_name, out_payload) in &step_res.outputs {
scope.insert(format!("{}.output.{}", sid, out_name), out_payload.clone());
scope.insert(format!("${}.output.{}", sid, out_name), out_payload.clone());
}
step_results.insert(sid.clone(), step_res);
}
let mut workflow_outputs = HashMap::new();
for wf_out in &manifest.outputs {
let source_expr = wf_out.source.trim();
let clean_expr = source_expr.trim_start_matches('$');
let resolved = scope.get(source_expr)
.or_else(|| scope.get(clean_expr))
.cloned()
.unwrap_or_default();
workflow_outputs.insert(wf_out.name.clone(), resolved);
}
let elapsed_ms = t_start.elapsed().as_secs_f64() * 1000.0;
let mut metrics = HashMap::new();
metrics.insert("wall_time_ms".to_string(), json!((elapsed_ms * 1000.0).round() / 1000.0));
metrics.insert("completed_steps".to_string(), json!(step_results.len()));
WorkflowExecutionResult {
status: "success".to_string(),
workflow_name: manifest.clm.name.clone(),
outputs: workflow_outputs,
step_results,
receipts,
failed_step: None,
error: None,
metrics,
}
}
}