use aion_core::{ActivityError, Event, Payload, RunId, WorkflowError, WorkflowId};
use chrono::{DateTime, Utc};
use crate::durability::{
Command, CorrelationKey, DurabilityError, NON_DETERMINISM_WORKFLOW_ERROR_PREFIX,
NonDeterminismError, Replay, ReplayStep, ReplayTerminal, Resolution,
correlation::correlation_keys_for_history, current_run_segment,
};
use crate::runtime::nif_determinism::{deterministic_float, deterministic_i64};
#[derive(Clone, Debug, PartialEq)]
pub struct RunInspection {
pub workflow_id: WorkflowId,
pub run_id: RunId,
pub steps: Vec<InspectStep>,
pub random: RandomDrawProjection,
pub divergence: Option<DivergentCommand>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RandomDrawProjection {
workflow_id: WorkflowId,
run_id: RunId,
}
impl RandomDrawProjection {
#[must_use]
const fn new(workflow_id: WorkflowId, run_id: RunId) -> Self {
Self {
workflow_id,
run_id,
}
}
#[must_use]
pub fn random_at(&self, ordinal: u64) -> f64 {
deterministic_float(&self.workflow_id, &self.run_id, ordinal)
}
pub fn random_int_at(&self, ordinal: u64, min: i64, max: i64) -> Result<i64, DurabilityError> {
if min > max {
return Err(DurabilityError::HistoryShape {
reason: format!(
"random_int_at range is inverted: min {min} is greater than max {max}"
),
});
}
Ok(deterministic_i64(
&self.workflow_id,
&self.run_id,
ordinal,
min,
max,
))
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct InspectStep {
pub seq: u64,
pub event_kind: &'static str,
pub correlation_key: Option<CorrelationKey>,
pub now: DateTime<Utc>,
pub projection: StepProjection,
}
#[derive(Clone, Debug, PartialEq)]
pub enum StepProjection {
Started {
workflow_type: String,
input: Payload,
},
Resolved(Resolution),
Terminal(ReplayTerminal),
AsyncArrival {
kind: &'static str,
},
NonReplay,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DivergentCommand {
pub seq: u64,
pub expected: String,
pub found: String,
}
impl From<&NonDeterminismError> for DivergentCommand {
fn from(error: &NonDeterminismError) -> Self {
Self {
seq: error.seq,
expected: error.expected.clone(),
found: error.found.clone(),
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub enum MockOutcome {
ActivityCompleted(Payload),
ActivityFailed(ActivityError),
ChildCompleted(Payload),
ChildFailed(WorkflowError),
SignalDelivered(Payload),
TimerFired,
}
#[derive(Clone, Debug, PartialEq)]
pub enum WhatIfOutcome {
Resolved {
from_seq: u64,
resolution: Resolution,
},
Terminal(ReplayTerminal),
Diverged(DivergentCommand),
}
pub fn inspect_run(history: Vec<Event>, run_id: &RunId) -> Result<RunInspection, DurabilityError> {
let segment = current_run_segment(history, run_id)?;
if segment.is_empty() {
return Err(empty_segment_error(run_id));
}
let workflow_id = run_workflow_id(&segment)?;
let keys = correlation_keys_for_history(&segment);
let mut replay = Replay::new(&workflow_id, run_id, segment.clone())?;
let commands = reconstruct_commands(&segment);
let mut command_index = 0;
let mut steps = Vec::with_capacity(segment.len());
for (event, correlation_key) in segment.iter().zip(keys.into_iter()) {
let projection = match command_for_event(event) {
CommandSlot::Issues => {
let Some(command) = commands.get(command_index).cloned() else {
return Err(DurabilityError::HistoryShape {
reason: format!(
"reconstructed command stream is shorter than history at seq {}",
event.seq()
),
});
};
command_index += 1;
match replay.step(&command) {
Ok(ReplayStep::Recorded(resolution)) => StepProjection::Resolved(resolution),
Ok(ReplayStep::Terminal(terminal)) => StepProjection::Terminal(terminal),
Ok(ReplayStep::ResumeLive) | Err(DurabilityError::NonDeterminism(_)) => {
StepProjection::NonReplay
}
Err(other) => return Err(other),
}
}
CommandSlot::Started {
workflow_type,
input,
} => StepProjection::Started {
workflow_type,
input,
},
CommandSlot::Terminal => StepProjection::Terminal(terminal_projection(event)?),
CommandSlot::AsyncArrival => StepProjection::AsyncArrival {
kind: event_kind(event),
},
CommandSlot::NonReplay => StepProjection::NonReplay,
};
steps.push(InspectStep {
seq: event.seq(),
event_kind: event_kind(event),
correlation_key,
now: *event.recorded_at(),
projection,
});
}
let divergence = recorded_divergence(&segment);
Ok(RunInspection {
workflow_id: workflow_id.clone(),
run_id: run_id.clone(),
steps,
random: RandomDrawProjection::new(workflow_id, run_id.clone()),
divergence,
})
}
pub fn what_if_from(
history: Vec<Event>,
run_id: &RunId,
from_seq: u64,
mocked: &MockOutcome,
) -> Result<WhatIfOutcome, DurabilityError> {
let segment = current_run_segment(history, run_id)?;
let workflow_id = run_workflow_id(&segment)?;
let fork_index = segment
.iter()
.position(|event| event.seq() == from_seq)
.ok_or_else(|| DurabilityError::HistoryShape {
reason: format!("run segment has no event at seq {from_seq} to fork from"),
})?;
let forked = forked_history(&segment, fork_index, &workflow_id, mocked)?;
let mut replay = Replay::new(&workflow_id, run_id, forked.clone())?;
let commands = reconstruct_commands(&forked);
let mut last_resolution = None;
for command in commands {
match replay.step(&command) {
Ok(ReplayStep::Recorded(resolution)) => last_resolution = Some(resolution),
Ok(ReplayStep::Terminal(terminal)) => {
return Ok(WhatIfOutcome::Terminal(terminal));
}
Ok(ReplayStep::ResumeLive) => break,
Err(DurabilityError::NonDeterminism(error)) => {
return Ok(WhatIfOutcome::Diverged(DivergentCommand::from(&error)));
}
Err(other) => return Err(other),
}
}
match last_resolution {
Some(resolution) => Ok(WhatIfOutcome::Resolved {
from_seq,
resolution,
}),
None => Err(DurabilityError::HistoryShape {
reason: format!(
"what-if from seq {from_seq} produced no resolution before history end"
),
}),
}
}
enum CommandSlot {
Issues,
Started {
workflow_type: String,
input: Payload,
},
Terminal,
AsyncArrival,
NonReplay,
}
fn command_for_event(event: &Event) -> CommandSlot {
match event {
Event::WorkflowStarted {
workflow_type,
input,
..
} => CommandSlot::Started {
workflow_type: workflow_type.clone(),
input: input.clone(),
},
Event::ActivityScheduled { .. }
| Event::TimerStarted { .. }
| Event::SignalReceived { .. }
| Event::ChildWorkflowStarted { .. } => CommandSlot::Issues,
Event::WorkflowCompleted { .. }
| Event::WorkflowFailed { .. }
| Event::WorkflowCancelled { .. }
| Event::WorkflowTimedOut { .. }
| Event::WorkflowContinuedAsNew { .. } => CommandSlot::Terminal,
Event::TimerFired { .. }
| Event::ActivityCompleted { .. }
| Event::ActivityFailed { .. }
| Event::ChildWorkflowCompleted { .. }
| Event::ChildWorkflowFailed { .. } => CommandSlot::AsyncArrival,
_ => CommandSlot::NonReplay,
}
}
fn reconstruct_commands(segment: &[Event]) -> Vec<Command> {
let keys = correlation_keys_for_history(segment);
let mut commands = Vec::new();
for (event, key) in segment.iter().zip(keys.into_iter()) {
match event {
Event::ActivityScheduled {
activity_type,
input,
..
} => {
if let Some(key) = key {
commands.push(Command::RunActivity {
key,
activity_type: activity_type.clone(),
input: input.clone(),
});
}
}
Event::TimerStarted { fire_at, .. } => {
if let Some(key) = key {
commands.push(Command::StartTimer {
key,
fire_at: *fire_at,
});
}
}
Event::SignalReceived { .. } => {
if let Some(key) = key {
commands.push(Command::AwaitSignal { key });
}
}
Event::ChildWorkflowStarted {
child_workflow_id,
workflow_type,
input,
..
} => {
if let Some(key) = key {
commands.push(Command::SpawnChild {
key,
workflow_type: workflow_type.clone(),
input: input.clone(),
});
commands.push(Command::AwaitChild {
child_workflow_id: child_workflow_id.clone(),
});
}
}
Event::WorkflowCompleted { result, .. } => {
commands.push(Command::CompleteWorkflow {
result: result.clone(),
});
}
_ => {}
}
}
commands
}
fn forked_history(
segment: &[Event],
fork_index: usize,
workflow_id: &WorkflowId,
mocked: &MockOutcome,
) -> Result<Vec<Event>, DurabilityError> {
let anchor = &segment[fork_index];
let mocked_outcome = mocked_outcome_event(anchor, workflow_id, mocked)?;
let mut forked: Vec<Event> = segment[..=fork_index].to_vec();
forked.push(mocked_outcome);
Ok(forked)
}
fn mocked_outcome_event(
anchor: &Event,
workflow_id: &WorkflowId,
mocked: &MockOutcome,
) -> Result<Event, DurabilityError> {
let envelope = aion_core::EventEnvelope {
seq: anchor.seq().saturating_add(1),
recorded_at: *anchor.recorded_at(),
workflow_id: workflow_id.clone(),
};
match (anchor, mocked) {
(Event::ActivityScheduled { activity_id, .. }, MockOutcome::ActivityCompleted(result)) => {
Ok(Event::ActivityCompleted {
envelope,
activity_id: activity_id.clone(),
result: result.clone(),
attempt: 1,
})
}
(Event::ActivityScheduled { activity_id, .. }, MockOutcome::ActivityFailed(error)) => {
ensure_terminal_activity_error(error)?;
Ok(Event::ActivityFailed {
envelope,
activity_id: activity_id.clone(),
error: error.clone(),
attempt: 1,
})
}
(Event::TimerStarted { timer_id, .. }, MockOutcome::TimerFired) => Ok(Event::TimerFired {
envelope,
timer_id: timer_id.clone(),
}),
(Event::SignalReceived { name, .. }, MockOutcome::SignalDelivered(payload)) => {
Ok(Event::SignalReceived {
envelope,
name: name.clone(),
payload: payload.clone(),
})
}
(
Event::ChildWorkflowStarted {
child_workflow_id, ..
},
MockOutcome::ChildCompleted(result),
) => Ok(Event::ChildWorkflowCompleted {
envelope,
child_workflow_id: child_workflow_id.clone(),
result: result.clone(),
}),
(
Event::ChildWorkflowStarted {
child_workflow_id, ..
},
MockOutcome::ChildFailed(error),
) => Ok(Event::ChildWorkflowFailed {
envelope,
child_workflow_id: child_workflow_id.clone(),
error: error.clone(),
}),
(anchor, mocked) => Err(DurabilityError::HistoryShape {
reason: format!(
"mocked outcome {mocked:?} does not match the {} anchor at the fork",
event_kind(anchor)
),
}),
}
}
fn ensure_terminal_activity_error(error: &ActivityError) -> Result<(), DurabilityError> {
if error.is_retryable() {
return Err(DurabilityError::HistoryShape {
reason: "mocked activity failure must be terminal to resolve at the fork".to_owned(),
});
}
Ok(())
}
fn terminal_projection(event: &Event) -> Result<ReplayTerminal, DurabilityError> {
match event {
Event::WorkflowCompleted { result, .. } => Ok(ReplayTerminal::Completed(result.clone())),
Event::WorkflowFailed { error, .. } => Ok(ReplayTerminal::Failed(error.clone())),
Event::WorkflowCancelled { reason, .. } => Ok(ReplayTerminal::Cancelled(reason.clone())),
Event::WorkflowTimedOut { timeout, .. } => Ok(ReplayTerminal::TimedOut(timeout.clone())),
Event::WorkflowContinuedAsNew { input, .. } => {
Ok(ReplayTerminal::ContinuedAsNew(input.clone()))
}
other => Err(DurabilityError::HistoryShape {
reason: format!(
"terminal projection requested for non-terminal event {}",
event_kind(other)
),
}),
}
}
fn recorded_divergence(segment: &[Event]) -> Option<DivergentCommand> {
let message = segment.iter().find_map(|event| match event {
Event::WorkflowFailed { error, .. }
if error
.message
.starts_with(NON_DETERMINISM_WORKFLOW_ERROR_PREFIX) =>
{
Some((event.seq(), error.message.as_str()))
}
_ => None,
})?;
parse_recorded_divergence(message.0, message.1)
}
fn parse_recorded_divergence(terminal_seq: u64, message: &str) -> Option<DivergentCommand> {
let after_sequence = message.split_once(" at sequence ")?.1;
let (seq_text, remainder) = after_sequence.split_once(": expected ")?;
let (expected, found) = remainder.split_once(", found ")?;
let seq = seq_text.trim().parse::<u64>().unwrap_or(terminal_seq);
Some(DivergentCommand {
seq,
expected: expected.to_owned(),
found: found.to_owned(),
})
}
fn run_workflow_id(segment: &[Event]) -> Result<WorkflowId, DurabilityError> {
segment
.first()
.map(|event| event.workflow_id().clone())
.ok_or_else(|| DurabilityError::HistoryShape {
reason: "run segment is empty".to_owned(),
})
}
fn empty_segment_error(run_id: &RunId) -> DurabilityError {
DurabilityError::HistoryShape {
reason: format!("run segment for {run_id} is empty"),
}
}
fn event_kind(event: &Event) -> &'static str {
match event {
Event::WorkflowStarted { .. } => "WorkflowStarted",
Event::WorkflowCompleted { .. } => "WorkflowCompleted",
Event::WorkflowFailed { .. } => "WorkflowFailed",
Event::WorkflowCancelled { .. } => "WorkflowCancelled",
Event::WorkflowTimedOut { .. } => "WorkflowTimedOut",
Event::WorkflowContinuedAsNew { .. } => "WorkflowContinuedAsNew",
Event::WorkflowReopened { .. } => "WorkflowReopened",
Event::WorkflowPaused { .. } => "WorkflowPaused",
Event::WorkflowResumed { .. } => "WorkflowResumed",
Event::SearchAttributesUpdated { .. } => "SearchAttributesUpdated",
Event::ActivityScheduled { .. } => "ActivityScheduled",
Event::ActivityStarted { .. } => "ActivityStarted",
Event::ActivityCompleted { .. } => "ActivityCompleted",
Event::ActivityFailed { .. } => "ActivityFailed",
Event::ActivityCancelled { .. } => "ActivityCancelled",
Event::TimerStarted { .. } => "TimerStarted",
Event::TimerFired { .. } => "TimerFired",
Event::TimerCancelled { .. } => "TimerCancelled",
Event::WithTimeoutCompleted { .. } => "WithTimeoutCompleted",
Event::SignalReceived { .. } => "SignalReceived",
Event::SignalSent { .. } => "SignalSent",
Event::ChildWorkflowStarted { .. } => "ChildWorkflowStarted",
Event::ChildWorkflowCompleted { .. } => "ChildWorkflowCompleted",
Event::ChildWorkflowFailed { .. } => "ChildWorkflowFailed",
Event::ChildWorkflowCancelled { .. } => "ChildWorkflowCancelled",
Event::ScheduleCreated { .. } => "ScheduleCreated",
Event::ScheduleUpdated { .. } => "ScheduleUpdated",
Event::SchedulePaused { .. } => "SchedulePaused",
Event::ScheduleResumed { .. } => "ScheduleResumed",
Event::ScheduleDeleted { .. } => "ScheduleDeleted",
Event::ScheduleTriggered { .. } => "ScheduleTriggered",
}
}
#[cfg(test)]
#[path = "replay_inspect_tests.rs"]
mod replay_inspect_tests;