use aion_core::{
ActivityError, ActivityErrorKind, ActivityId, Event, EventEnvelope, Payload, RunId, TimerId,
WorkflowError, WorkflowId,
};
use chrono::{DateTime, TimeZone, Utc};
use serde_json::json;
use uuid::Uuid;
use super::{
DivergentCommand, MockOutcome, StepProjection, WhatIfOutcome, inspect_run, what_if_from,
};
use crate::durability::{ReplayTerminal, Resolution};
use crate::runtime::nif_determinism::{deterministic_float, deterministic_i64};
type TestResult<T = ()> = Result<T, Box<dyn std::error::Error>>;
fn workflow_id() -> WorkflowId {
WorkflowId::new(Uuid::from_u128(1))
}
fn run_id() -> RunId {
RunId::new(Uuid::from_u128(2))
}
fn timestamp(seconds: i64) -> TestResult<DateTime<Utc>> {
Utc.timestamp_opt(seconds, 0)
.single()
.ok_or_else(|| format!("invalid timestamp {seconds}").into())
}
fn envelope(seq: u64, seconds: i64) -> TestResult<EventEnvelope> {
Ok(EventEnvelope {
seq,
recorded_at: timestamp(seconds)?,
workflow_id: workflow_id(),
})
}
fn payload(label: &str) -> TestResult<Payload> {
Ok(Payload::from_json(&json!({ "label": label }))?)
}
fn history() -> TestResult<Vec<Event>> {
let activity_id = ActivityId::from_sequence_position(0);
let timer_id = TimerId::anonymous(3);
Ok(vec![
Event::WorkflowStarted {
envelope: envelope(1, 10)?,
workflow_type: "workflow".to_owned(),
input: payload("input")?,
run_id: run_id(),
parent_run_id: None,
package_version: aion_core::PackageVersion::new("a".repeat(64)),
},
Event::ActivityScheduled {
envelope: envelope(2, 20)?,
activity_id: activity_id.clone(),
activity_type: "activity".to_owned(),
input: payload("activity-input")?,
task_queue: String::from("default"),
node: None,
},
Event::ActivityCompleted {
envelope: envelope(3, 30)?,
activity_id,
result: payload("activity-result")?,
},
Event::TimerStarted {
envelope: envelope(4, 40)?,
timer_id: timer_id.clone(),
fire_at: timestamp(100)?,
},
Event::TimerFired {
envelope: envelope(5, 50)?,
timer_id,
},
Event::WorkflowCompleted {
envelope: envelope(6, 60)?,
result: payload("workflow-result")?,
},
])
}
#[test]
fn inspect_run_projects_one_step_per_event_with_resolutions() -> TestResult {
let inspection = inspect_run(history()?, &run_id())?;
assert_eq!(inspection.workflow_id, workflow_id());
assert_eq!(inspection.run_id, run_id());
assert_eq!(inspection.steps.len(), 6, "one step per recorded event");
assert!(inspection.divergence.is_none());
assert!(matches!(
inspection.steps[0].projection,
StepProjection::Started { .. }
));
assert_eq!(
inspection.steps[1].projection,
StepProjection::Resolved(Resolution::ActivityCompleted(payload("activity-result")?))
);
assert_eq!(
inspection.steps[3].projection,
StepProjection::Resolved(Resolution::TimerFired)
);
assert_eq!(
inspection.steps[5].projection,
StepProjection::Terminal(ReplayTerminal::Completed(payload("workflow-result")?))
);
Ok(())
}
#[test]
fn inspect_run_now_equals_each_event_recorded_at() -> TestResult {
let inspection = inspect_run(history()?, &run_id())?;
let nows: Vec<i64> = inspection
.steps
.iter()
.map(|step| step.now.timestamp())
.collect();
assert_eq!(nows, vec![10, 20, 30, 40, 50, 60]);
Ok(())
}
#[test]
fn inspect_run_random_projection_equals_the_production_formula() -> TestResult {
let inspection = inspect_run(history()?, &run_id())?;
let workflow_id = workflow_id();
let run = run_id();
for ordinal in 0..32 {
let projected = inspection.random.random_at(ordinal);
let production = deterministic_float(&workflow_id, &run, ordinal);
assert!(
(projected - production).abs() < f64::EPSILON,
"random_at({ordinal}) must equal the production deterministic_float",
);
assert!((0.0..1.0).contains(&projected));
let projected_int = inspection.random.random_int_at(ordinal, 1, 100)?;
let production_int = deterministic_i64(&workflow_id, &run, ordinal, 1, 100);
assert_eq!(projected_int, production_int);
assert!((1..=100).contains(&projected_int));
}
Ok(())
}
#[test]
fn inspect_run_random_projection_is_deterministic_and_run_distinct() -> TestResult {
let first = inspect_run(history()?, &run_id())?;
let second = inspect_run(history()?, &run_id())?;
let other_run = RunId::new(Uuid::from_u128(999));
let mut other_history = history()?;
if let Some(Event::WorkflowStarted { run_id, .. }) = other_history.first_mut() {
*run_id = other_run.clone();
} else {
return Err("expected WorkflowStarted first".into());
}
let other = inspect_run(other_history, &other_run)?;
let mut differs = false;
for ordinal in 0..16 {
let first_value = first.random.random_at(ordinal);
let second_value = second.random.random_at(ordinal);
assert!(
(first_value - second_value).abs() < f64::EPSILON,
"same run must project an identical draw at ordinal {ordinal}",
);
if (first_value - other.random.random_at(ordinal)).abs() > f64::EPSILON {
differs = true;
}
}
assert!(
differs,
"a different RunId must project a different random draw at some ordinal",
);
Ok(())
}
#[test]
fn random_int_at_rejects_an_inverted_range() -> TestResult {
let inspection = inspect_run(history()?, &run_id())?;
let error = inspection.random.random_int_at(0, 10, 1).err();
assert!(matches!(
error,
Some(crate::durability::DurabilityError::HistoryShape { .. })
));
Ok(())
}
fn faulted_history() -> TestResult<(Vec<Event>, crate::durability::NonDeterminismError)> {
let violation = crate::durability::NonDeterminismError {
workflow_id: workflow_id(),
seq: 3,
expected: "Activity Activity(1)".to_owned(),
found: "ActivityCompleted family Some(Activity) key Some(Activity(0))".to_owned(),
};
let message = format!(
"{}: {violation}",
crate::durability::NON_DETERMINISM_WORKFLOW_ERROR_PREFIX
);
let history = vec![
Event::WorkflowStarted {
envelope: envelope(1, 10)?,
workflow_type: "workflow".to_owned(),
input: payload("input")?,
run_id: run_id(),
parent_run_id: None,
package_version: aion_core::PackageVersion::new("a".repeat(64)),
},
Event::ActivityScheduled {
envelope: envelope(2, 20)?,
activity_id: ActivityId::from_sequence_position(0),
activity_type: "activity".to_owned(),
input: payload("activity-input")?,
task_queue: String::from("default"),
node: None,
},
Event::WorkflowFailed {
envelope: envelope(3, 30)?,
error: aion_core::WorkflowError {
message,
details: None,
},
},
];
Ok((history, violation))
}
#[test]
fn inspect_run_surfaces_the_divergent_command() -> TestResult {
let (history, violation) = faulted_history()?;
let inspection = inspect_run(history, &run_id())?;
let divergence = inspection
.divergence
.ok_or("a faulted run must surface its recorded divergence")?;
assert_eq!(divergence.seq, violation.seq);
assert_eq!(divergence.expected, violation.expected);
assert_eq!(divergence.found, violation.found);
Ok(())
}
#[test]
fn recorded_divergence_round_trips_through_the_real_failure_format() -> TestResult {
let violation = crate::durability::NonDeterminismError {
workflow_id: workflow_id(),
seq: 42,
expected: "Activity Activity(7)".to_owned(),
found: "TimerFired family Some(Timer) key Some(Timer(TimerId(3)))".to_owned(),
};
let message = format!(
"{}: {violation}",
crate::durability::NON_DETERMINISM_WORKFLOW_ERROR_PREFIX
);
let history = vec![
Event::WorkflowStarted {
envelope: envelope(1, 10)?,
workflow_type: "workflow".to_owned(),
input: payload("input")?,
run_id: run_id(),
parent_run_id: None,
package_version: aion_core::PackageVersion::new("a".repeat(64)),
},
Event::WorkflowFailed {
envelope: envelope(2, 20)?,
error: aion_core::WorkflowError {
message,
details: None,
},
},
];
let inspection = inspect_run(history, &run_id())?;
let divergence = inspection
.divergence
.ok_or("expected a recorded divergence")?;
assert_eq!(divergence.seq, 42);
assert_eq!(divergence.expected, violation.expected);
assert_eq!(divergence.found, violation.found);
Ok(())
}
#[test]
fn divergent_command_is_built_from_the_resolver_error() {
let error = crate::durability::NonDeterminismError {
workflow_id: workflow_id(),
seq: 7,
expected: "Activity Activity(1)".to_owned(),
found: "ActivityCompleted family Activity key Activity(0)".to_owned(),
};
let divergence = DivergentCommand::from(&error);
assert_eq!(divergence.seq, 7);
assert_eq!(divergence.expected, error.expected);
assert_eq!(divergence.found, error.found);
}
fn child_signal_timer_history() -> TestResult<Vec<Event>> {
let child_id = WorkflowId::new(Uuid::from_u128(0xC417));
let timer_id = TimerId::anonymous(5);
Ok(vec![
Event::WorkflowStarted {
envelope: envelope(1, 10)?,
workflow_type: "workflow".to_owned(),
input: payload("input")?,
run_id: run_id(),
parent_run_id: None,
package_version: aion_core::PackageVersion::new("a".repeat(64)),
},
Event::ChildWorkflowStarted {
envelope: envelope(2, 20)?,
child_workflow_id: child_id.clone(),
workflow_type: "child".to_owned(),
input: payload("child-input")?,
package_version: aion_core::PackageVersion::new("b".repeat(64)),
},
Event::ChildWorkflowCompleted {
envelope: envelope(3, 30)?,
child_workflow_id: child_id,
result: payload("child-result")?,
},
Event::SignalReceived {
envelope: envelope(4, 40)?,
name: "ready".to_owned(),
payload: payload("signal-payload")?,
},
Event::TimerStarted {
envelope: envelope(5, 50)?,
timer_id: timer_id.clone(),
fire_at: timestamp(100)?,
},
Event::TimerFired {
envelope: envelope(6, 60)?,
timer_id,
},
Event::WorkflowCompleted {
envelope: envelope(7, 70)?,
result: payload("workflow-result")?,
},
])
}
#[test]
fn what_if_child_completion_resolves_through_the_real_replay() -> TestResult {
let outcome = what_if_from(
child_signal_timer_history()?,
&run_id(),
2, &MockOutcome::ChildCompleted(payload("mocked-child-result")?),
)?;
assert_eq!(
outcome,
WhatIfOutcome::Resolved {
from_seq: 2,
resolution: Resolution::ChildCompleted(payload("mocked-child-result")?),
}
);
Ok(())
}
#[test]
fn what_if_child_failure_resolves_through_the_real_replay() -> TestResult {
let error = WorkflowError {
message: "mocked child failure".to_owned(),
details: None,
};
let outcome = what_if_from(
child_signal_timer_history()?,
&run_id(),
2,
&MockOutcome::ChildFailed(error.clone()),
)?;
assert_eq!(
outcome,
WhatIfOutcome::Resolved {
from_seq: 2,
resolution: Resolution::ChildFailed(error),
}
);
Ok(())
}
#[test]
fn what_if_signal_delivery_resolves_through_the_real_replay() -> TestResult {
let outcome = what_if_from(
child_signal_timer_history()?,
&run_id(),
4, &MockOutcome::SignalDelivered(payload("mocked-signal")?),
)?;
assert_eq!(
outcome,
WhatIfOutcome::Resolved {
from_seq: 4,
resolution: Resolution::SignalDelivered(payload("mocked-signal")?),
}
);
Ok(())
}
#[test]
fn what_if_timer_firing_resolves_through_the_real_replay() -> TestResult {
let outcome = what_if_from(
child_signal_timer_history()?,
&run_id(),
5, &MockOutcome::TimerFired,
)?;
assert_eq!(
outcome,
WhatIfOutcome::Resolved {
from_seq: 5,
resolution: Resolution::TimerFired,
}
);
Ok(())
}
#[test]
fn what_if_activity_failure_diverges_from_the_recorded_completion() -> TestResult {
let failure = ActivityError {
kind: ActivityErrorKind::Terminal,
message: "mocked failure".to_owned(),
details: None,
};
let outcome = what_if_from(
history()?,
&run_id(),
2, &MockOutcome::ActivityFailed(failure.clone()),
)?;
match outcome {
WhatIfOutcome::Resolved {
from_seq,
resolution,
} => {
assert_eq!(from_seq, 2);
assert_eq!(resolution, Resolution::ActivityFailedTerminal(failure));
}
other => return Err(format!("expected a resolved fork, got {other:?}").into()),
}
Ok(())
}
#[test]
fn what_if_activity_completion_reproduces_the_recorded_path() -> TestResult {
let outcome = what_if_from(
history()?,
&run_id(),
2,
&MockOutcome::ActivityCompleted(payload("activity-result")?),
)?;
assert_eq!(
outcome,
WhatIfOutcome::Resolved {
from_seq: 2,
resolution: Resolution::ActivityCompleted(payload("activity-result")?),
}
);
Ok(())
}
#[test]
fn what_if_rejects_a_mismatched_mock_family() -> TestResult {
let error = what_if_from(history()?, &run_id(), 2, &MockOutcome::TimerFired).err();
assert!(matches!(
error,
Some(crate::durability::DurabilityError::HistoryShape { .. })
));
Ok(())
}
#[test]
fn what_if_rejects_an_absent_fork_sequence() -> TestResult {
let error = what_if_from(history()?, &run_id(), 9999, &MockOutcome::TimerFired).err();
assert!(matches!(
error,
Some(crate::durability::DurabilityError::HistoryShape { .. })
));
Ok(())
}
#[test]
fn inspect_run_errors_when_run_segment_is_absent() -> TestResult {
let absent = RunId::new(Uuid::from_u128(404));
let error = inspect_run(history()?, &absent).err();
assert!(matches!(
error,
Some(crate::durability::DurabilityError::HistoryShape { .. })
));
Ok(())
}
#[test]
fn inspect_run_scopes_to_the_current_run_segment() -> TestResult {
let first_run = RunId::new(Uuid::from_u128(100));
let second_run = RunId::new(Uuid::from_u128(200));
let history = vec![
Event::WorkflowStarted {
envelope: envelope(1, 10)?,
workflow_type: "workflow".to_owned(),
input: payload("first")?,
run_id: first_run.clone(),
parent_run_id: None,
package_version: aion_core::PackageVersion::new("a".repeat(64)),
},
Event::WorkflowContinuedAsNew {
envelope: envelope(2, 20)?,
input: payload("again")?,
workflow_type: None,
parent_run_id: first_run.clone(),
},
Event::WorkflowStarted {
envelope: envelope(3, 30)?,
workflow_type: "workflow".to_owned(),
input: payload("second")?,
run_id: second_run.clone(),
parent_run_id: Some(first_run),
package_version: aion_core::PackageVersion::new("a".repeat(64)),
},
Event::WorkflowCompleted {
envelope: envelope(4, 40)?,
result: payload("second-result")?,
},
];
let inspection = inspect_run(history, &second_run)?;
assert_eq!(inspection.steps.len(), 2, "only the second run's segment");
assert_eq!(inspection.steps[0].seq, 3);
assert_eq!(inspection.steps[1].seq, 4);
Ok(())
}