mod child;
mod child_run;
mod wave;
use chrono::Utc;
use rust_decimal::Decimal;
use serde_json::{Value, from_value, json, to_value};
use tracing::{error, info, warn};
use uuid::Uuid;
use ironflow_store::models::{
NewStep, RunStatus, Step, StepKind, StepStatus, StepUpdate, normalize_worker_tags,
step_trace_id,
};
use crate::config::{WorkflowOptions, WorkflowStepConfig};
use crate::context::WorkflowContext;
use crate::context::lifecycle::check_replay_identity;
use crate::error::EngineError;
use crate::executor::{
ConcurrencyConflict, RecordedWorkflowStep, SubWorkflowOutcome, SubWorkflowOutput,
};
use crate::guard::WorkflowRejection;
use crate::handler::{TypedWorkflow, WorkflowHandler};
use crate::plan::{SharedPlanRecorder, lock_plan};
const CHILD_RUN_ID_KEY: &str = "child_run_id";
pub(crate) fn recorded_child_run_id(step: &Step) -> Option<Uuid> {
let raw = step.output.as_ref()?.get(CHILD_RUN_ID_KEY)?.as_str()?;
match Uuid::parse_str(raw) {
Ok(id) => Some(id),
Err(err) => {
warn!(
step_id = %step.id,
value = %raw,
error = %err,
"open workflow step records an invalid child run id"
);
None
}
}
}
#[derive(Clone, Copy)]
enum ChildResume {
Suspended(Uuid),
Interrupted(Uuid),
}
impl ChildResume {
fn run_id(self) -> Uuid {
match self {
Self::Suspended(id) | Self::Interrupted(id) => id,
}
}
}
enum ChildOutcome {
Finished(SubWorkflowOutput, bool),
Conflict(ConcurrencyConflict),
}
fn leaves_step_open(err: &EngineError) -> bool {
err.is_suspension() || matches!(err, EngineError::RunPaused { .. })
}
fn expect_completed(
outcome: SubWorkflowOutcome,
position: u32,
) -> Result<SubWorkflowOutput, EngineError> {
match outcome {
SubWorkflowOutcome::Completed(output) => Ok(output),
SubWorkflowOutcome::Conflict(_) => Err(EngineError::StepConfig(format!(
"workflow step at position {position} recorded a concurrency conflict; \
call workflow_with to read it"
))),
}
}
impl WorkflowContext {
pub async fn workflow<W: TypedWorkflow>(
&mut self,
handler: &W,
input: W::Input,
) -> Result<SubWorkflowOutput, EngineError> {
let payload = to_value(&input)?;
let position = self.position;
let outcome = self
.run_sub_workflow(handler, payload, WorkflowOptions::default())
.await?;
expect_completed(outcome, position)
}
pub async fn workflow_with<W: TypedWorkflow>(
&mut self,
handler: &W,
input: W::Input,
options: WorkflowOptions,
) -> Result<SubWorkflowOutcome, EngineError> {
let payload = to_value(&input)?;
self.run_sub_workflow(handler, payload, options).await
}
#[deprecated(
note = "implement `TypedWorkflow` on the child and call `workflow`: its payload is then checked at compile time"
)]
pub async fn workflow_dyn(
&mut self,
handler: &dyn WorkflowHandler,
payload: Value,
) -> Result<SubWorkflowOutput, EngineError> {
let position = self.position;
let outcome = self
.run_sub_workflow(handler, payload, WorkflowOptions::default())
.await?;
expect_completed(outcome, position)
}
async fn run_sub_workflow(
&mut self,
handler: &dyn WorkflowHandler,
payload: Value,
options: WorkflowOptions,
) -> Result<SubWorkflowOutcome, EngineError> {
let (step_name, config) = options.into_step_config(handler.name(), payload);
if let Some(plan) = self.plan().cloned() {
let planned = self
.plan_sub_workflow(&plan, handler, &step_name, config.payload)
.await?;
return Ok(SubWorkflowOutcome::Completed(planned));
}
let position = self.position;
let existing = self.replay_steps.get(&position).cloned();
if let Some(existing) = &existing {
check_replay_identity(existing, position, &step_name, &StepKind::Workflow)?;
if existing.status.state == StepStatus::Completed {
return self.replay_sub_workflow(existing);
}
}
let interrupted = self
.interrupted_children
.get(&(position, step_name.clone()))
.cloned();
if let Some(interrupted) = &interrupted {
check_replay_identity(interrupted, position, &step_name, &StepKind::Workflow)?;
}
self.check_child_worker_tags(handler)?;
if let (Some(guard_config), Some(guard_state)) = (&self.guard_config, &self.guard_state) {
guard_state
.lock()
.map_err(|_| WorkflowRejection::GuardUnavailable)?
.enter(guard_config, handler.name())?;
}
if let Err(err) = self.check_not_paused().await {
self.guard_record_return();
return Err(err);
}
self.position += 1;
let (step_id, resume) = match existing.filter(|s| s.status.state == StepStatus::Running) {
Some(step) => (
step.id,
recorded_child_run_id(&step).map(ChildResume::Suspended),
),
None => {
let recorded = self
.record_workflow_step(&step_name, &config, position)
.await;
let step_id = match recorded {
Ok(step_id) => step_id,
Err(err) => {
self.guard_record_return();
return Err(err);
}
};
let resume = interrupted
.as_ref()
.and_then(recorded_child_run_id)
.map(ChildResume::Interrupted);
(step_id, resume)
}
};
let run = self.execute_child_workflow(&config, step_id, resume).await;
if matches!(&run.result, Err(err) if err.is_suspension()) {
self.sibling_suspensions.extend(run.sibling_suspensions);
}
let settled = self.settle_child_step(step_id, &config, run.result).await;
self.guard_record_return();
if settled.is_ok() {
self.last_step_ids = vec![step_id];
}
settled
}
async fn record_workflow_step(
&self,
step_name: &str,
config: &WorkflowStepConfig,
position: u32,
) -> Result<Uuid, EngineError> {
let step = self
.store
.create_step(NewStep {
run_id: self.run_id,
trace_id: step_trace_id(self.run_id, step_name, position),
name: step_name.to_string(),
kind: StepKind::Workflow,
position,
input: Some(to_value(config)?),
is_error_handler: false,
})
.await?;
self.start_step(step.id, Utc::now()).await?;
Ok(step.id)
}
async fn settle_child_step(
&mut self,
step_id: Uuid,
config: &WorkflowStepConfig,
result: Result<ChildOutcome, EngineError>,
) -> Result<SubWorkflowOutcome, EngineError> {
match result {
Ok(ChildOutcome::Conflict(conflict)) => {
self.store
.update_step(
step_id,
StepUpdate {
status: Some(StepStatus::Completed),
output: Some(json!({ "concurrency_conflict": conflict })),
duration_ms: Some(0),
cost_usd: Some(Decimal::ZERO),
completed_at: Some(Utc::now()),
..StepUpdate::default()
},
)
.await?;
info!(
run_id = %self.run_id,
child_workflow = %config.workflow_name,
key = %conflict.key(),
holder = %conflict.run_id(),
"workflow step skipped: concurrency conflict"
);
Ok(SubWorkflowOutcome::Conflict(conflict))
}
Ok(ChildOutcome::Finished(output, child_had_allowed_failure)) => {
self.total_cost_usd += output.cost_usd();
self.total_duration_ms += output.duration_ms();
if child_had_allowed_failure {
self.has_allowed_failure = true;
}
self.store
.update_step(
step_id,
StepUpdate {
status: Some(StepStatus::Completed),
output: Some(to_value(&output)?),
duration_ms: Some(output.duration_ms()),
cost_usd: Some(output.cost_usd()),
completed_at: Some(Utc::now()),
..StepUpdate::default()
},
)
.await?;
info!(
run_id = %self.run_id,
child_workflow = %config.workflow_name,
duration_ms = output.duration_ms(),
"workflow step completed"
);
Ok(SubWorkflowOutcome::Completed(output))
}
Err(err) if leaves_step_open(&err) => Err(err),
Err(err) => {
if let Err(store_err) = self
.store
.update_step(
step_id,
StepUpdate {
status: Some(StepStatus::Failed),
error: Some(err.to_string()),
completed_at: Some(Utc::now()),
..StepUpdate::default()
},
)
.await
{
error!(step_id = %step_id, error = %store_err, "failed to persist step failure");
}
Err(err)
}
}
}
fn replay_sub_workflow(&mut self, step: &Step) -> Result<SubWorkflowOutcome, EngineError> {
let outcome = self.replay_recorded_child(step)?;
self.position += 1;
self.last_step_ids = vec![step.id];
Ok(outcome)
}
fn replay_recorded_child(&mut self, step: &Step) -> Result<SubWorkflowOutcome, EngineError> {
let recorded = step.output.clone().ok_or_else(|| {
EngineError::StepConfig(format!(
"completed workflow step {} has no recorded output",
step.id
))
})?;
let recorded: RecordedWorkflowStep = from_value(recorded)?;
let output = match SubWorkflowOutcome::from(recorded) {
SubWorkflowOutcome::Completed(output) => output,
SubWorkflowOutcome::Conflict(conflict) => {
info!(
run_id = %self.run_id,
step = %step.name,
key = %conflict.key(),
holder = %conflict.run_id(),
"workflow step replayed: concurrency conflict"
);
return Ok(SubWorkflowOutcome::Conflict(conflict));
}
};
self.total_duration_ms += output.duration_ms();
if matches!(
output.status(),
RunStatus::Warning | RunStatus::Failed | RunStatus::Cancelled
) {
self.has_allowed_failure = true;
}
info!(
run_id = %self.run_id,
child_run_id = %output.run_id(),
step = %step.name,
"workflow step replayed from previous execution"
);
Ok(SubWorkflowOutcome::Completed(output))
}
fn check_child_worker_tags(&self, handler: &dyn WorkflowHandler) -> Result<(), EngineError> {
let Some(carried) = &self.worker_tags else {
return Ok(());
};
let missing: Vec<String> = normalize_worker_tags(handler.required_worker_tags())
.into_iter()
.filter(|tag| !carried.contains(tag))
.collect();
if missing.is_empty() {
return Ok(());
}
Err(EngineError::InvalidWorkflow(format!(
"sub-workflow '{}' requires worker tags [{}] that this worker does not carry",
handler.name(),
missing.join(", ")
)))
}
async fn plan_sub_workflow(
&mut self,
plan: &SharedPlanRecorder,
handler: &dyn WorkflowHandler,
step_name: &str,
payload: Value,
) -> Result<SubWorkflowOutput, EngineError> {
self.position += 1;
{
let mut recorder = lock_plan(plan);
recorder.record_or_stop(step_name, StepKind::Workflow, &self.workflow_name, None)?;
recorder.set_last(vec![step_name.to_string()]);
}
Ok(self.expand_planned_child(plan, handler, payload).await)
}
async fn expand_planned_child(
&self,
plan: &SharedPlanRecorder,
handler: &dyn WorkflowHandler,
payload: Value,
) -> SubWorkflowOutput {
let sub_name = handler.name().to_string();
let planned = SubWorkflowOutput::new(
Uuid::nil(),
&sub_name,
RunStatus::Completed,
Decimal::ZERO,
0,
);
let expand = lock_plan(plan).enter_workflow();
if expand {
let previous_payload = lock_plan(plan).swap_payload(payload.clone());
let mut child = WorkflowContext::new(
Uuid::now_v7(),
sub_name.clone(),
self.store.clone(),
self.provider.clone(),
);
child.handler_resolver = self.handler_resolver.clone();
child.set_plan(plan.clone());
if let Err(err) = handler.execute(&mut child).await {
lock_plan(plan).fail(format!(
"sub-workflow {sub_name} could not be planned: {err}"
));
}
let mut recorder = lock_plan(plan);
recorder.swap_payload(previous_payload);
recorder.leave_workflow();
}
planned
}
}