use std::collections::HashMap;
use std::sync::Arc;
use std::sync::RwLock;
use std::sync::atomic::{AtomicU64, Ordering};
use sha2::{Digest, Sha256};
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct OperationId {
positional: String,
wire: String,
}
impl OperationId {
pub(crate) fn new(positional: String) -> Self {
let wire = compute_wire_id(&positional);
Self { positional, wire }
}
pub(crate) fn positional(&self) -> &str {
&self.positional
}
pub(crate) fn wire(&self) -> &str {
&self.wire
}
}
fn compute_wire_id(positional: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(positional.as_bytes());
let digest = hasher.finalize();
hex::encode_sha256(digest.as_slice())
}
pub(crate) fn compute_wire_id_public(positional: &str) -> String {
compute_wire_id(positional)
}
mod hex {
const HEX_CHARS: &[u8; 16] = b"0123456789abcdef";
pub(super) fn encode_sha256(bytes: &[u8]) -> String {
let mut out = String::with_capacity(bytes.len() * 2);
for &b in bytes {
#[expect(clippy::indexing_slicing)] {
out.push(HEX_CHARS[(b >> 4) as usize] as char);
out.push(HEX_CHARS[(b & 0x0f) as usize] as char);
}
}
out
}
}
#[derive(Debug)]
pub(crate) struct IdCounter {
prefix: String,
counter: AtomicU64,
}
impl IdCounter {
pub(crate) fn new(prefix: String) -> Self {
Self {
prefix,
counter: AtomicU64::new(0),
}
}
pub(crate) fn mint(&self) -> OperationId {
let n = self.counter.fetch_add(1, Ordering::SeqCst) + 1;
OperationId::new(self.format(n))
}
pub(crate) fn child(positional_id: &str) -> Self {
Self::new(positional_id.to_owned())
}
fn format(&self, n: u64) -> String {
if self.prefix.is_empty() {
n.to_string()
} else {
format!("{}-{n}", self.prefix)
}
}
pub(crate) fn prefix(&self) -> &str {
&self.prefix
}
pub(crate) fn peek_next(&self) -> String {
let next_n = self.counter.load(Ordering::SeqCst) + 1;
self.format(next_n)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum CheckpointStatus {
Started,
Pending,
Ready,
Succeeded,
Failed,
Cancelled,
TimedOut,
Stopped,
Unknown(String),
}
impl CheckpointStatus {
pub(crate) fn is_terminal(&self) -> bool {
matches!(
self,
Self::Succeeded
| Self::Failed
| Self::Cancelled
| Self::TimedOut
| Self::Stopped
| Self::Unknown(_)
)
}
pub(crate) fn wire_str(&self) -> &str {
match self {
Self::Started => "STARTED",
Self::Pending => "PENDING",
Self::Ready => "READY",
Self::Succeeded => "SUCCEEDED",
Self::Failed => "FAILED",
Self::Cancelled => "CANCELLED",
Self::TimedOut => "TIMED_OUT",
Self::Stopped => "STOPPED",
Self::Unknown(raw) => raw,
}
}
}
#[derive(Debug, Clone)]
pub(crate) struct CheckpointRecord {
#[expect(dead_code)]
pub(crate) id: String,
pub(crate) status: CheckpointStatus,
pub(crate) result: Option<String>,
pub(crate) error_type: Option<String>,
pub(crate) error_message: Option<String>,
pub(crate) error_data: Option<String>,
pub(crate) stack_trace: Option<Vec<String>>,
pub(crate) attempt: u32,
pub(crate) invoke_result: Option<String>,
pub(crate) invoke_error_type: Option<String>,
pub(crate) invoke_error_message: Option<String>,
pub(crate) invoke_error_data: Option<String>,
pub(crate) invoke_stack_trace: Option<Vec<String>>,
pub(crate) replay_children: bool,
pub(crate) callback_id: Option<String>,
pub(crate) op_type: Option<String>,
pub(crate) sub_type: Option<String>,
pub(crate) op_name: Option<String>,
pub(crate) due_at: Option<std::time::SystemTime>,
pub(crate) parent_id: Option<String>,
pub(crate) start_timestamp: Option<std::time::SystemTime>,
pub(crate) end_timestamp: Option<std::time::SystemTime>,
}
#[cfg(test)]
impl CheckpointRecord {
pub(crate) fn with_status(id: &str, status: CheckpointStatus) -> Self {
Self {
id: id.to_owned(),
status,
result: None,
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
op_name: None,
due_at: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct CheckpointStatusView {
pub(crate) status: CheckpointStatus,
pub(crate) attempt: u32,
pub(crate) replay_children: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct TerminalReplaySnapshot {
pub(crate) status: CheckpointStatus,
pub(crate) replay_children: bool,
pub(crate) result: Option<String>,
pub(crate) error_message: Option<String>,
pub(crate) error_type: Option<String>,
pub(crate) error_data: Option<String>,
pub(crate) stack_trace: Option<Vec<String>>,
}
#[derive(Debug)]
pub(crate) struct CheckpointLog {
records: RwLock<HashMap<String, CheckpointRecord>>,
}
impl CheckpointLog {
#[cfg(test)]
pub(crate) fn empty() -> Self {
Self {
records: RwLock::new(HashMap::new()),
}
}
pub(crate) fn from_records(records: Vec<(String, CheckpointRecord)>) -> Self {
Self {
records: RwLock::new(records.into_iter().collect()),
}
}
#[cfg(test)]
pub(crate) fn get(&self, wire_id: &str) -> Option<CheckpointRecord> {
self.with_record(wire_id, Clone::clone)
}
pub(crate) fn with_record<R>(
&self,
wire_id: &str,
f: impl FnOnce(&CheckpointRecord) -> R,
) -> Option<R> {
let guard = self
.records
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
guard.get(wire_id).map(f)
}
pub(crate) fn status_view(&self, wire_id: &str) -> Option<CheckpointStatusView> {
self.with_record(wire_id, |record| CheckpointStatusView {
status: record.status.clone(),
attempt: record.attempt,
replay_children: record.replay_children,
})
}
pub(crate) fn contains(&self, wire_id: &str) -> bool {
let guard = self
.records
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
guard.contains_key(wire_id)
}
pub(crate) fn update(&self, wire_id: &str, f: impl FnOnce(&mut CheckpointRecord)) {
let mut guard = self
.records
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(record) = guard.get_mut(wire_id) {
f(record);
}
}
pub(crate) fn insert(
&self,
wire_id: String,
record: CheckpointRecord,
) -> Option<CheckpointStatus> {
let mut guard = self
.records
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
guard
.insert(wire_id, record)
.map(|previous| previous.status)
}
pub(crate) fn project_records<R>(
&self,
mut f: impl FnMut(&str, &CheckpointRecord) -> R,
) -> Vec<R> {
#[cfg(test)]
PROJECTIONS.with(|count| count.set(count.get() + 1));
let guard = self
.records
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
guard.iter().map(|(id, record)| f(id, record)).collect()
}
pub(crate) fn has_records(&self) -> bool {
let guard = self
.records
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
!guard.is_empty()
}
}
#[cfg(test)]
thread_local! {
static PROJECTIONS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
#[cfg(test)]
pub(crate) fn projections() -> usize {
PROJECTIONS.with(std::cell::Cell::get)
}
#[derive(Debug)]
pub(crate) struct EngineState {
pub(crate) id_counter: IdCounter,
pub(crate) checkpoint_log: Arc<CheckpointLog>,
}
impl EngineState {
pub(crate) fn new_root(checkpoint_log: Arc<CheckpointLog>) -> Self {
Self {
id_counter: IdCounter::new(String::new()),
checkpoint_log,
}
}
pub(crate) fn new_child(
parent_positional_id: &str,
checkpoint_log: Arc<CheckpointLog>,
) -> Self {
Self {
id_counter: IdCounter::child(parent_positional_id),
checkpoint_log,
}
}
pub(crate) fn mint_id(&self) -> OperationId {
self.id_counter.mint()
}
#[cfg(test)]
pub(crate) fn is_replaying_at(&self, positional_id: &str) -> bool {
let wire_id = compute_wire_id(positional_id);
self.checkpoint_log
.status_view(&wire_id)
.is_some_and(|view| view.status.is_terminal())
}
pub(crate) fn is_replaying(&self) -> bool {
if !self.checkpoint_log.has_records() {
return false;
}
let next_positional = self.id_counter.peek_next();
let next_wire = compute_wire_id(&next_positional);
self.checkpoint_log.contains(&next_wire)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn unknown_status_is_terminal_and_keeps_raw_wire_value() {
let unknown = CheckpointStatus::Unknown("PAUSED".to_owned());
assert!(unknown.is_terminal());
assert_eq!(unknown.wire_str(), "PAUSED");
}
#[test]
fn known_status_wire_spellings_are_upper_case() {
assert_eq!(CheckpointStatus::TimedOut.wire_str(), "TIMED_OUT");
assert_eq!(CheckpointStatus::Started.wire_str(), "STARTED");
assert!(!CheckpointStatus::Started.is_terminal());
assert!(CheckpointStatus::TimedOut.is_terminal());
}
#[test]
fn insert_returns_previous_status() {
let log = CheckpointLog::empty();
assert_eq!(
log.insert(
"op".to_owned(),
CheckpointRecord::with_status("op", CheckpointStatus::Started),
),
None
);
assert_eq!(
log.insert(
"op".to_owned(),
CheckpointRecord::with_status("op", CheckpointStatus::Succeeded),
),
Some(CheckpointStatus::Started)
);
}
#[test]
fn counter_starts_at_one() {
let counter = IdCounter::new(String::new());
let id = counter.mint();
assert_eq!(id.positional(), "1");
}
#[test]
fn counter_increments_per_call() {
let counter = IdCounter::new(String::new());
let id1 = counter.mint();
let id2 = counter.mint();
let id3 = counter.mint();
assert_eq!(id1.positional(), "1");
assert_eq!(id2.positional(), "2");
assert_eq!(id3.positional(), "3");
}
#[test]
fn counter_with_prefix() {
let counter = IdCounter::new("2".to_owned());
let id1 = counter.mint();
let id2 = counter.mint();
assert_eq!(id1.positional(), "2-1");
assert_eq!(id2.positional(), "2-2");
}
#[test]
fn child_prefix_chaining() {
let root = IdCounter::new(String::new());
let _id1 = root.mint(); let id2 = root.mint(); assert_eq!(id2.positional(), "2");
let child = IdCounter::child(id2.positional());
let child_id1 = child.mint();
let child_id2 = child.mint();
assert_eq!(child_id1.positional(), "2-1");
assert_eq!(child_id2.positional(), "2-2");
let grandchild = IdCounter::child(child_id1.positional());
let gc_id1 = grandchild.mint();
let gc_id2 = grandchild.mint();
assert_eq!(gc_id1.positional(), "2-1-1");
assert_eq!(gc_id2.positional(), "2-1-2");
}
#[test]
fn wire_id_known_answer_simple() {
let id = compute_wire_id("1");
assert_eq!(
id,
"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b"
);
}
#[test]
fn wire_id_known_answer_prefixed() {
let id = compute_wire_id("2-1");
assert_eq!(
id,
"e2433ac3278279ac90b51abed3d4dd9057d9e1f66541ad9cdfd13762a30e5a43"
);
}
#[test]
fn wire_id_known_answer_deep_path() {
let id = compute_wire_id("2-1-3");
assert_eq!(
id,
"c00876d4d917a061ad5a377270e059994251eea9818733e6abfef69c6080625b"
);
}
#[test]
fn wire_id_length_and_charset() {
let counter = IdCounter::new(String::new());
for _ in 0..20 {
let id = counter.mint();
let wire = id.wire();
assert_eq!(wire.len(), 64);
assert!(
wire.chars().all(|c| c.is_ascii_hexdigit()),
"wire ID contains non-hex character: {wire}"
);
assert!(
wire.chars().all(|c| !c.is_ascii_uppercase()),
"wire ID should be lowercase: {wire}"
);
}
}
#[test]
fn wire_id_at_mint_matches_standalone() {
let counter = IdCounter::new("prefix".to_owned());
let id = counter.mint();
assert_eq!(id.wire(), compute_wire_id(id.positional()));
}
#[tokio::test]
async fn ids_minted_at_creation_not_await() {
let counter = Arc::new(IdCounter::new(String::new()));
let id_first = counter.mint();
let id_second = counter.mint();
assert_eq!(id_first.positional(), "1");
assert_eq!(id_second.positional(), "2");
}
#[tokio::test]
async fn ids_stable_under_interleaved_join() {
let counter = Arc::new(IdCounter::new(String::new()));
let c1 = Arc::clone(&counter);
let c2 = Arc::clone(&counter);
let (id_a, id_b) = tokio::join!(async move { c1.mint() }, async move { c2.mint() },);
assert_eq!(id_a.positional(), "1");
assert_eq!(id_b.positional(), "2");
}
#[test]
fn checkpoint_log_returns_stored_result() {
let log = CheckpointLog::from_records(vec![(
"1".to_owned(),
CheckpointRecord {
id: "wire-1".to_owned(),
status: CheckpointStatus::Succeeded,
result: Some(r#""hello""#.to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)]);
let record = log.get("1");
assert!(record.is_some());
let record = record.unwrap();
assert_eq!(record.status, CheckpointStatus::Succeeded);
assert_eq!(record.result.as_deref(), Some(r#""hello""#));
}
#[test]
fn checkpoint_log_miss_for_unknown_id() {
let log = CheckpointLog::from_records(vec![(
"1".to_owned(),
CheckpointRecord {
id: "wire-1".to_owned(),
status: CheckpointStatus::Succeeded,
result: Some(r"42".to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)]);
assert!(log.get("2").is_none());
assert!(log.get("unknown").is_none());
}
#[test]
fn checkpoint_log_empty_always_miss() {
let log = CheckpointLog::empty();
assert!(log.get("1").is_none());
assert!(log.get("anything").is_none());
}
fn full_record() -> CheckpointRecord {
CheckpointRecord {
id: "wire-1".to_owned(),
status: CheckpointStatus::Failed,
result: Some(r#""payload""#.to_owned()),
error_type: Some("SomeError".to_owned()),
error_message: Some("it broke".to_owned()),
error_data: None,
stack_trace: None,
attempt: 3,
invoke_result: Some(r#""invoke-payload""#.to_owned()),
invoke_error_type: Some("InvokeError".to_owned()),
invoke_error_message: Some("invoke broke".to_owned()),
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: true,
callback_id: Some("cb-123".to_owned()),
op_type: Some("Step".to_owned()),
sub_type: Some("Step".to_owned()),
due_at: None,
op_name: Some("my-step".to_owned()),
parent_id: None,
start_timestamp: None,
end_timestamp: None,
}
}
#[test]
fn status_view_carries_status_attempt_and_replay_children() {
let log = CheckpointLog::from_records(vec![("1".to_owned(), full_record())]);
let view = log.status_view("1");
assert!(view.is_some());
let view = view.unwrap();
assert_eq!(view.status, CheckpointStatus::Failed);
assert_eq!(view.attempt, 3);
assert!(view.replay_children);
let clone = view.clone();
assert_eq!(clone, view);
assert!(log.status_view("missing").is_none());
}
#[test]
fn contains_reports_existence_without_cloning() {
let log = CheckpointLog::from_records(vec![("1".to_owned(), full_record())]);
assert!(log.contains("1"));
assert!(!log.contains("2"));
assert!(!CheckpointLog::empty().contains("1"));
}
#[test]
fn with_record_projects_only_what_the_closure_takes() {
let log = CheckpointLog::from_records(vec![("1".to_owned(), full_record())]);
let callback_id = log.with_record("1", |r| r.callback_id.clone()).flatten();
assert_eq!(callback_id.as_deref(), Some("cb-123"));
let is_failed = log.with_record("1", |r| r.status == CheckpointStatus::Failed);
assert_eq!(is_failed, Some(true));
let ran = log.with_record("missing", |_| true);
assert!(ran.is_none());
}
#[test]
fn get_still_returns_the_full_record() {
let log = CheckpointLog::from_records(vec![("1".to_owned(), full_record())]);
let record = log.get("1");
assert!(record.is_some());
let record = record.unwrap();
assert_eq!(record.result.as_deref(), Some(r#""payload""#));
assert_eq!(record.invoke_error_message.as_deref(), Some("invoke broke"));
assert_eq!(record.op_name.as_deref(), Some("my-step"));
}
#[test]
fn replay_mode_with_checkpointed_operations() {
let wire1 = compute_wire_id("1");
let wire2 = compute_wire_id("2");
let log = Arc::new(CheckpointLog::from_records(vec![
(
wire1.clone(),
CheckpointRecord {
id: wire1,
status: CheckpointStatus::Succeeded,
result: Some("a".to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
),
(
wire2.clone(),
CheckpointRecord {
id: wire2,
status: CheckpointStatus::Succeeded,
result: Some("b".to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
),
]));
let engine = EngineState::new_root(Arc::clone(&log));
assert!(engine.is_replaying());
let id1 = engine.mint_id();
assert_eq!(id1.positional(), "1");
assert!(engine.is_replaying_at(id1.positional()));
assert!(engine.is_replaying());
let id2 = engine.mint_id();
assert_eq!(id2.positional(), "2");
assert!(engine.is_replaying_at(id2.positional()));
assert!(!engine.is_replaying());
}
#[test]
fn replay_mode_empty_log_never_replaying() {
let log = Arc::new(CheckpointLog::empty());
let engine = EngineState::new_root(log);
assert!(!engine.is_replaying());
let id = engine.mint_id();
assert!(!engine.is_replaying_at(id.positional()));
}
#[test]
fn replay_mode_non_terminal_record_still_replaying() {
let wire1 = compute_wire_id("1");
let log = Arc::new(CheckpointLog::from_records(vec![(
wire1.clone(),
CheckpointRecord {
id: wire1,
status: CheckpointStatus::Started,
result: None,
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)]));
let engine = EngineState::new_root(log);
assert!(engine.is_replaying());
assert!(!engine.is_replaying_at("1"));
let _ = engine.mint_id();
assert!(!engine.is_replaying());
}
#[test]
fn replay_mode_child_context() {
let wire_2_1 = compute_wire_id("2-1");
let wire_2_2 = compute_wire_id("2-2");
let log = Arc::new(CheckpointLog::from_records(vec![
(
wire_2_1.clone(),
CheckpointRecord {
id: wire_2_1,
status: CheckpointStatus::Succeeded,
result: Some("x".to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
),
(
wire_2_2.clone(),
CheckpointRecord {
id: wire_2_2,
status: CheckpointStatus::Failed,
result: None,
error_type: Some("StepError".to_owned()),
error_message: Some("oops".to_owned()),
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
),
]));
let engine = EngineState::new_child("2", log);
assert!(engine.is_replaying());
let id1 = engine.mint_id();
assert_eq!(id1.positional(), "2-1");
assert!(engine.is_replaying_at(id1.positional()));
assert!(engine.is_replaying());
let id2 = engine.mint_id();
assert_eq!(id2.positional(), "2-2");
assert!(engine.is_replaying_at(id2.positional()));
assert!(!engine.is_replaying());
}
}