use std::sync::{Arc, Mutex};
use std::time::Duration;
use orcher_proto::kernel::v1::{command, RunActivation};
use orcher_proto::orcher::v1::journal_entry::Attributes;
use orcher_proto::orcher::v1::{
ChildWorkflowExecutionCompletedEventAttributes, ChildWorkflowExecutionStartedEventAttributes,
EntryType, JournalEntry, PollWorkflowExecutionResponse, StepCompletedEventAttributes,
TaskScheduledEventAttributes, TimerFiredEventAttributes, TimerStartedEventAttributes,
WorkflowExecutionStartedEventAttributes,
};
use orcher_proto::prost::Message;
use orcher_sdk_core::kernel::encoded;
use orcher_sdk_core::{Command, DeterminismViolation, ExecutionResult};
use crate::payload::Payload;
use crate::worker::registration::WorkflowHandlerFn;
use crate::worker::runtime::{ExecutionRuntime, RuntimeConfig};
use crate::workflow::WorkflowContext;
const ENGINE_EPOCH_MS: i64 = 1_600_000_000_000;
const ENTRY_GAP_MS: i64 = 7_000;
struct FakeEngine {
journal: Vec<JournalEntry>,
clock_ms: i64,
}
impl FakeEngine {
fn new() -> Self {
let mut engine = Self {
journal: Vec::new(),
clock_ms: ENGINE_EPOCH_MS,
};
engine.append(
EntryType::WorkflowExecutionStarted,
Attributes::WorkflowExecutionStarted(WorkflowExecutionStartedEventAttributes {
workflow_type: "under_test".into(),
..Default::default()
}),
);
engine
}
fn append(&mut self, entry_type: EntryType, attributes: Attributes) {
let at = self.clock_ms;
self.clock_ms += ENTRY_GAP_MS;
self.journal.push(JournalEntry {
entry_id: self.journal.len() as i64 + 1,
timestamp: Some(orcher_proto::prost_types::Timestamp {
seconds: at / 1000,
nanos: ((at % 1000) * 1_000_000) as i32,
}),
entry_type: entry_type as i32,
attributes: Some(attributes),
..Default::default()
});
}
fn entry_secs(&self, index: usize) -> i64 {
self.journal[index].timestamp.as_ref().unwrap().seconds
}
fn activation(&self) -> Vec<u8> {
PollWorkflowExecutionResponse {
workflow_id: "wf-replay".into(),
execution_id: "run-replay".into(),
workflow_type: "under_test".into(),
task_queue: "replay".into(),
input: b"null".to_vec(),
journal: self.journal.clone(),
attempt: 1,
..Default::default()
}
.encode_to_vec()
}
fn apply(&mut self, commands: &[Command]) {
for command in commands {
match command {
Command::ScheduleTask(cmd) => {
self.append(
EntryType::TaskScheduled,
Attributes::TaskScheduled(TaskScheduledEventAttributes {
task_id: cmd.task_id.clone(),
task_type: cmd.task_type.clone(),
..Default::default()
}),
);
self.append(
EntryType::StepCompleted,
Attributes::StepCompleted(StepCompletedEventAttributes {
step_name: cmd.task_id.clone(),
result: serde_json::to_vec(&cmd.task_id).unwrap(),
..Default::default()
}),
);
}
Command::StartTimer(cmd) => {
self.append(
EntryType::TimerStarted,
Attributes::TimerStarted(TimerStartedEventAttributes {
timer_id: cmd.timer_id.clone(),
..Default::default()
}),
);
self.append(
EntryType::TimerFired,
Attributes::TimerFired(TimerFiredEventAttributes {
timer_id: cmd.timer_id.clone(),
..Default::default()
}),
);
}
Command::StartChildWorkflow(cmd) => {
self.append(
EntryType::ChildWorkflowExecutionStarted,
Attributes::ChildWorkflowExecutionStarted(
ChildWorkflowExecutionStartedEventAttributes {
workflow_id: cmd.workflow_id.clone(),
workflow_type: cmd.workflow_type.clone(),
..Default::default()
},
),
);
self.append(
EntryType::ChildWorkflowExecutionCompleted,
Attributes::ChildWorkflowExecutionCompleted(
ChildWorkflowExecutionCompletedEventAttributes {
workflow_id: cmd.workflow_id.clone(),
result: serde_json::to_vec(&cmd.workflow_id).unwrap(),
..Default::default()
},
),
);
}
Command::RecordStepResult(cmd) => {
self.append(
EntryType::StepCompleted,
Attributes::StepCompleted(StepCompletedEventAttributes {
step_name: cmd.step_name.clone(),
step_type: cmd.step_type,
result: cmd.result.clone(),
..Default::default()
}),
);
}
_ => {}
}
}
}
}
type Issued = (&'static str, String, String);
fn issued_steps(commands: &[Command]) -> Vec<Issued> {
commands
.iter()
.filter_map(|command| match command {
Command::ScheduleTask(c) => Some(("task", c.task_id.clone(), c.task_type.clone())),
Command::StartTimer(c) => Some(("timer", c.timer_id.clone(), String::new())),
Command::StartChildWorkflow(c) => {
Some(("child", c.workflow_id.clone(), c.workflow_type.clone()))
}
Command::RecordStepResult(c) => Some(("closure", c.step_name.clone(), String::new())),
_ => None,
})
.collect()
}
fn completion(result: &ExecutionResult) -> Option<serde_json::Value> {
result.commands.iter().find_map(|command| match command {
Command::CompleteWorkflow(c) => Some(serde_json::from_slice(&c.result.data).unwrap()),
_ => None,
})
}
struct Answered {
result: ExecutionResult,
recorded: Vec<u8>,
answer: Vec<u8>,
}
impl std::ops::Deref for Answered {
type Target = ExecutionResult;
fn deref(&self) -> &ExecutionResult {
&self.result
}
}
impl Answered {
fn violations(&self) -> Vec<DeterminismViolation> {
encoded::check_recorded(&self.recorded, &self.answer).expect("core checks the answer")
}
}
fn resolved(recorded: &[u8]) -> RunActivation {
let bytes = encoded::resolve_recorded(recorded, "default").expect("the activation resolves");
RunActivation::decode(bytes.as_slice()).expect("a RunActivation")
}
async fn answer(runtime: &ExecutionRuntime, recorded: Vec<u8>) -> Answered {
let answer = runtime
.run_activation(resolved(&recorded))
.await
.encode_to_vec();
let result =
encoded::execution_result("run-replay", &answer).expect("the kernel takes the answer");
Answered {
result,
recorded,
answer,
}
}
async fn activate(runtime: &ExecutionRuntime, engine: &FakeEngine) -> ExecutionResult {
let answered = answer(runtime, engine.activation()).await;
let result = &answered.result;
if let Some(error) = &result.error {
panic!(
"the activation failed ({:?}): {}",
error.error_type, error.message
);
}
assert!(result.successful);
assert!(
result.reached_steps.is_some(),
"the activation does not report the steps it reached"
);
let violations = answered.violations();
assert!(
violations.is_empty(),
"the activation does not replay its journal: {violations:?}"
);
answered.result
}
async fn run_to_completion(
runtime: &ExecutionRuntime,
engine: &mut FakeEngine,
) -> (Vec<Issued>, serde_json::Value) {
let mut issued = Vec::new();
for _ in 0..16 {
let result = activate(runtime, engine).await;
issued.extend(issued_steps(&result.commands));
engine.apply(&result.commands);
if let Some(output) = completion(&result) {
return (issued, output);
}
}
panic!("the workflow did not complete; it issued {issued:?}");
}
async fn runtime_with(handler: WorkflowHandlerFn) -> ExecutionRuntime {
let runtime = ExecutionRuntime::new(RuntimeConfig::default());
runtime
.register_workflow_handler("under_test", handler)
.await;
runtime
}
fn output_of<T: serde::Serialize>(value: T) -> crate::Result<Payload> {
Ok(Payload::new_data(serde_json::to_vec(&value).unwrap()))
}
#[tokio::test]
async fn a_replayed_workflow_issues_every_step_under_the_id_it_first_had() {
let handler: WorkflowHandlerFn = Arc::new(|ctx: WorkflowContext, _input| {
Box::pin(async move {
let (a, slept) = futures::join!(
ctx.execute_task_by_name::<u32, String>("step_a", 1),
ctx.sleep(Duration::from_secs(5)),
);
let a = a?;
slept?;
let child: String = ctx.execute_child_workflow("child_wf", ()).await?;
let closure: String = ctx
.execute("closure_c", || async { Ok("closure ran".to_string()) })
.await?;
let b: String = ctx.execute_task_by_name("step_b", 2).await?;
output_of(vec![a, child, closure, b])
})
});
let runtime = runtime_with(handler).await;
let mut engine = FakeEngine::new();
let (issued, output) = run_to_completion(&runtime, &mut engine).await;
assert_eq!(
issued,
vec![
("task", "step_a_0".to_string(), "step_a".to_string()),
("timer", "timer_1".to_string(), String::new()),
(
"child",
"wf-replay-child-2".to_string(),
"child_wf".to_string()
),
("closure", "closure_c_3".to_string(), String::new()),
("task", "step_b_4".to_string(), "step_b".to_string()),
],
"each step is issued once, under one id"
);
assert_eq!(
output,
serde_json::json!(["step_a_0", "wf-replay-child-2", "closure ran", "step_b_4"])
);
let replay = activate(&runtime, &engine).await;
assert_eq!(issued_steps(&replay.commands), Vec::<Issued>::new());
assert_eq!(completion(&replay), Some(output));
}
#[tokio::test]
async fn a_healthy_saga_completes_without_compensating() {
let handler: WorkflowHandlerFn = Arc::new(|ctx: WorkflowContext, _input| {
Box::pin(async move {
let mut saga = crate::workflow::Saga::new();
let charged: String = saga
.add_step(&ctx, "charge_payment", 10u32)
.with_compensation("refund_payment")
.execute()
.await?;
let reserved: String = saga
.add_step(&ctx, "reserve_stock", 1u32)
.with_compensation("release_stock")
.execute()
.await?;
saga.commit();
output_of(vec![charged, reserved])
})
});
let runtime = runtime_with(handler).await;
let mut engine = FakeEngine::new();
let (issued, _output) = run_to_completion(&runtime, &mut engine).await;
let task_types: Vec<&str> = issued
.iter()
.filter(|(kind, _, _)| *kind == "task")
.map(|(_, _, task_type)| task_type.as_str())
.collect();
assert_eq!(
task_types,
["charge_payment", "reserve_stock"],
"a healthy saga ran a compensation"
);
}
#[tokio::test]
async fn workflow_time_comes_from_the_journal_and_is_the_same_on_every_replay() {
let readings: Arc<Mutex<Vec<(&'static str, i64)>>> = Arc::default();
let seen = Arc::clone(&readings);
let handler: WorkflowHandlerFn = Arc::new(move |ctx: WorkflowContext, _input| {
let seen = Arc::clone(&seen);
Box::pin(async move {
let read = |label: &'static str, value: i64| {
seen.lock().unwrap().push((label, value));
value
};
let started = read("started", ctx.time().now());
ctx.execute_task_by_name::<u32, String>("step_a", 1).await?;
let after_task = read("after task", ctx.time().now());
ctx.sleep(Duration::from_secs(60)).await?;
let after_timer = read("after timer", ctx.time().now());
ctx.execute("closure_c", || async { Ok(1u32) }).await?;
let after_closure = read("after closure", ctx.time().now());
let elapsed = read("elapsed", ctx.time().elapsed_secs() as i64);
output_of(vec![
started,
after_task,
after_timer,
after_closure,
elapsed,
])
})
});
let runtime = runtime_with(handler).await;
let mut engine = FakeEngine::new();
let (_, output) = run_to_completion(&runtime, &mut engine).await;
let started = engine.entry_secs(0);
let task_completed = engine.entry_secs(2);
let timer_fired = engine.entry_secs(4);
let expected = serde_json::json!([
started,
task_completed,
timer_fired,
timer_fired,
timer_fired - started
]);
assert_eq!(output, expected, "time is read from the journal");
let readings = readings.lock().unwrap().clone();
assert!(readings.len() > 5, "the workflow ran more than once");
for (label, value) in &readings {
let first = readings.iter().find(|(l, _)| l == label).map(|(_, v)| v);
assert_eq!(Some(value), first, "{label} differs between activations");
}
tokio::time::sleep(Duration::from_millis(1_100)).await;
let replay = activate(&runtime, &engine).await;
assert_eq!(completion(&replay), Some(expected));
}
#[tokio::test]
async fn a_closure_before_a_suspension_runs_once_and_is_journaled_before_it() {
let runs = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let counted = Arc::clone(&runs);
let handler: WorkflowHandlerFn = Arc::new(move |ctx: WorkflowContext, _input| {
let counted = Arc::clone(&counted);
Box::pin(async move {
let value: u32 = ctx
.execute("closure_c", move || async move {
Ok(counted.fetch_add(1, std::sync::atomic::Ordering::SeqCst) as u32 + 7)
})
.await?;
ctx.sleep(Duration::from_secs(5)).await?;
output_of(value)
})
});
let runtime = runtime_with(handler).await;
let mut engine = FakeEngine::new();
let (issued, output) = run_to_completion(&runtime, &mut engine).await;
assert_eq!(
runs.load(std::sync::atomic::Ordering::SeqCst),
1,
"the closure ran once"
);
assert_eq!(
output,
serde_json::json!(7),
"its first result is the one kept"
);
assert_eq!(
issued,
vec![
("closure", "closure_c_0".to_string(), String::new()),
("timer", "timer_1".to_string(), String::new()),
]
);
let replay = activate(&runtime, &engine).await;
assert_eq!(issued_steps(&replay.commands), Vec::<Issued>::new());
assert_eq!(completion(&replay), Some(output));
assert_eq!(
runs.load(std::sync::atomic::Ordering::SeqCst),
1,
"not run on replay"
);
}
fn status_watch(quiet_cap: u32) -> WorkflowHandlerFn {
Arc::new(move |ctx: WorkflowContext, _input| {
Box::pin(async move {
let mut quiet = 0;
for round in 1u32.. {
let ev: Option<String> = ctx
.wait_for_event_with_timeout("status", Duration::from_secs(4))
.await?;
let _: String = ctx.execute_task_by_name("applyStatus", round).await?;
quiet = if ev.is_some() { 0 } else { quiet + 1 };
if quiet >= quiet_cap {
let _: String = ctx.execute_task_by_name("escalate", round).await?;
return output_of(format!("escalated at round {round}"));
}
}
unreachable!()
})
})
}
#[tokio::test]
async fn a_loop_cut_short_is_refused_where_the_journal_goes_on() {
let old = runtime_with(status_watch(5)).await;
let mut engine = FakeEngine::new();
for _ in 0..6 {
let result = activate(&old, &engine).await;
engine.apply(&result.commands);
}
let new = runtime_with(status_watch(1)).await;
let result = answer(&new, engine.activation()).await;
assert_eq!(
result.reached_steps.as_deref(),
Some(
&[
"event_timeout_status_1".to_string(),
"applyStatus_0".to_string(),
"escalate_1".to_string(),
][..]
)
);
let violations = result.violations();
let [violation] = violations.as_slice() else {
panic!("expected one violation, got {violations:?}");
};
assert_eq!(violation.step_id.as_deref(), Some("event_timeout_status_2"));
assert_eq!(
violation.actual.as_deref(),
Some(r#"task "escalate_1" of type "escalate""#)
);
let result = activate(&old, &engine).await;
assert_eq!(
issued_steps(&result.commands),
vec![("timer", "event_timeout_status_4".to_string(), String::new())]
);
}
fn two_children_then_a_task() -> WorkflowHandlerFn {
Arc::new(|ctx: WorkflowContext, _input| {
Box::pin(async move {
let first: String = ctx.execute_child_workflow("child_wf", ()).await?;
let second: String = ctx.execute_child_workflow("child_wf", ()).await?;
let task: String = ctx.execute_task_by_name("step_b", 2).await?;
output_of(vec![first, second, task])
})
})
}
#[tokio::test]
async fn a_run_started_under_the_old_default_child_id_replays_to_its_child() {
let runtime = runtime_with(two_children_then_a_task()).await;
let mut engine = FakeEngine::new();
engine.append(
EntryType::ChildWorkflowExecutionStarted,
Attributes::ChildWorkflowExecutionStarted(ChildWorkflowExecutionStartedEventAttributes {
workflow_id: "child_0".into(),
workflow_type: "child_wf".into(),
..Default::default()
}),
);
let result = activate(&runtime, &engine).await;
assert_eq!(
result.reached_steps.as_deref(),
Some(&["child_0".to_string()][..])
);
for step in issued_steps(&result.commands) {
assert_eq!(
step,
("child", "child_0".to_string(), "child_wf".to_string())
);
}
engine.append(
EntryType::ChildWorkflowExecutionCompleted,
Attributes::ChildWorkflowExecutionCompleted(
ChildWorkflowExecutionCompletedEventAttributes {
workflow_id: "child_0".into(),
result: serde_json::to_vec("child_0").unwrap(),
..Default::default()
},
),
);
let (issued, output) = run_to_completion(&runtime, &mut engine).await;
assert_eq!(
issued,
vec![
(
"child",
"wf-replay-child-1".to_string(),
"child_wf".to_string()
),
("task", "step_b_2".to_string(), "step_b".to_string()),
]
);
assert_eq!(
output,
serde_json::json!(["child_0", "wf-replay-child-1", "step_b_2"])
);
let replay = activate(&runtime, &engine).await;
assert_eq!(issued_steps(&replay.commands), Vec::<Issued>::new());
assert_eq!(completion(&replay), Some(output));
}
fn charge_then_fail(skip_charge: bool) -> WorkflowHandlerFn {
Arc::new(move |ctx: WorkflowContext, _input| {
Box::pin(async move {
if !skip_charge {
let _: String = ctx.execute_task_by_name("charge", 1).await?;
}
let _: u32 = ctx.execute("count", || async { Ok(3) }).await?;
Err(crate::Error::Other("out of stock".into()))
})
})
}
#[tokio::test]
async fn a_failure_that_left_a_recorded_step_behind_is_held() {
let mut engine = FakeEngine::new();
let first = activate(&runtime_with(charge_then_fail(false)).await, &engine).await;
engine.apply(&first.commands);
let genuine = answer(
&runtime_with(charge_then_fail(false)).await,
engine.activation(),
)
.await;
assert!(!genuine.successful);
assert!(genuine.commands.is_empty(), "{:?}", genuine.commands);
assert_eq!(
genuine.reached_steps.as_deref(),
Some(&["charge_0".to_string()][..])
);
let violations = genuine.violations();
assert!(violations.is_empty(), "{violations:?}");
let activation = resolved(&engine.activation());
let calls = runtime_with(charge_then_fail(false))
.await
.run_activation(activation)
.await
.calls_before_failure
.expect("the calls before the failure")
.commands;
let kinds: Vec<(bool, &str)> = calls
.iter()
.map(|c| match &c.what {
Some(command::What::RunTask(_)) => (c.satisfied, "task"),
Some(command::What::RecordValue(_)) => (c.satisfied, "value"),
_ => (c.satisfied, "other"),
})
.collect();
assert_eq!(kinds, [(false, "value"), (true, "task")]);
let changed = answer(
&runtime_with(charge_then_fail(true)).await,
engine.activation(),
)
.await;
assert!(!changed.successful);
assert_eq!(changed.reached_steps, Some(vec![]));
let violations = changed.violations();
let [violation] = violations.as_slice() else {
panic!("expected one violation, got {violations:?}");
};
assert_eq!(violation.step_id.as_deref(), Some("charge_0"));
}
#[tokio::test]
async fn is_replaying_reports_each_point_as_new_exactly_once() {
let readings: Arc<Mutex<Vec<(&'static str, bool)>>> = Arc::default();
let seen = Arc::clone(&readings);
let handler: WorkflowHandlerFn = Arc::new(move |ctx: WorkflowContext, _input| {
let seen = Arc::clone(&seen);
Box::pin(async move {
let read = |label: &'static str| seen.lock().unwrap().push((label, ctx.is_replaying()));
read("start");
ctx.execute_task_by_name::<u32, String>("step_a", 1).await?;
read("after task");
ctx.sleep(Duration::from_secs(60)).await?;
read("after timer");
ctx.execute("closure_c", || async { Ok(1u32) }).await?;
read("after closure");
output_of(true)
})
});
let runtime = runtime_with(handler).await;
let mut engine = FakeEngine::new();
run_to_completion(&runtime, &mut engine).await;
let readings = readings.lock().unwrap().clone();
assert_eq!(
readings,
vec![
("start", false),
("start", true),
("after task", false),
("start", true),
("after task", true),
("after timer", false),
("after closure", false),
],
"a point reads as new on the activation that first reaches it, and as a replay after"
);
}