use std::{
any::Any,
collections::VecDeque,
panic::{AssertUnwindSafe, catch_unwind},
sync::{
Arc,
atomic::{AtomicU64, Ordering},
},
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
};
use datum::{
RestartSettings,
actor::{Actor, ActorProcessingErr, ActorRef, Message},
};
use tokio::sync::oneshot;
use super::{
ChangeEventHandler, DurableState, DurableStateBehavior, DurableStateHandle, DurableStateSignal,
effect::{ChainedEffect, MainEffect},
};
use crate::{
AppendOutcome, AtomicWrite, DurableStateStore, EventAdapter, PersistenceError,
PersistenceResult, Revision, SequenceNr, SerializedPayload, SerializerCatalog, SerializerId,
StateAdapter, StoredEvent,
};
const STATE_ENVELOPE_MAGIC: &[u8] = b"datum-durable-state-v1\0";
const CHANGE_EVENT_REVISION_METADATA_MANIFEST: &str = "datum-durable-state-change-revision-v1";
const CHANGE_EVENT_REVISION_METADATA_SERIALIZER_ID: SerializerId = SerializerId::new(u32::MAX);
static NEXT_WRITER_ID: AtomicU64 = AtomicU64::new(1);
static BACKOFF_RANDOM_SEED: AtomicU64 = AtomicU64::new(0xd124_911a_c305_76e7);
pub(crate) trait StateCodec<S>: Send + Sync + 'static {
fn encode(&self, state: &S) -> PersistenceResult<SerializedPayload>;
fn decode(&self, state: &SerializedPayload) -> PersistenceResult<S>;
}
pub(crate) struct AdaptedStateCodec<S, J> {
adapter: Arc<dyn StateAdapter<S, J>>,
catalog: SerializerCatalog<J>,
}
impl<S, J> AdaptedStateCodec<S, J> {
pub(crate) fn new<A>(adapter: A, catalog: SerializerCatalog<J>) -> Self
where
A: StateAdapter<S, J>,
{
Self {
adapter: Arc::new(adapter),
catalog,
}
}
}
impl<S, J> StateCodec<S> for AdaptedStateCodec<S, J>
where
S: 'static,
J: 'static,
{
fn encode(&self, state: &S) -> PersistenceResult<SerializedPayload> {
let (adapter_manifest, journal_state) = self.adapter.to_journal(state)?;
let frame = self.catalog.encode(&journal_state)?;
encode_state_envelope(adapter_manifest, frame)
}
fn decode(&self, state: &SerializedPayload) -> PersistenceResult<S> {
let (adapter_manifest, frame) = decode_state_envelope(state)?;
let journal_state = self.catalog.decode(&frame)?;
self.adapter.from_journal(&adapter_manifest, journal_state)
}
}
pub(crate) struct EncodedChangeEvent {
adapter_manifest: String,
payload: SerializedPayload,
}
pub(crate) trait ChangeEventCodec<C, S>: Send + Sync + 'static {
fn clone_command(&self, command: &C) -> Box<dyn Any + Send>;
fn encode_update(
&self,
previous_state: &S,
new_state: &S,
command: &(dyn Any + Send),
) -> PersistenceResult<EncodedChangeEvent>;
fn encode_delete(
&self,
previous_state: &S,
command: &(dyn Any + Send),
) -> PersistenceResult<EncodedChangeEvent>;
}
pub(crate) struct AdaptedChangeEventCodec<C, S, E, J> {
handler: ChangeEventHandler<C, S, E>,
adapter: Arc<dyn EventAdapter<E, J>>,
catalog: SerializerCatalog<J>,
}
impl<C, S, E, J> AdaptedChangeEventCodec<C, S, E, J>
where
E: 'static,
J: 'static,
{
pub(crate) fn new<A>(
handler: ChangeEventHandler<C, S, E>,
adapter: A,
catalog: SerializerCatalog<J>,
) -> Self
where
A: EventAdapter<E, J>,
{
Self {
handler,
adapter: Arc::new(adapter),
catalog,
}
}
fn encode(&self, event: E) -> PersistenceResult<EncodedChangeEvent> {
let (adapter_manifest, journal_event) = self.adapter.to_journal(&event)?;
let payload = self.catalog.encode(&journal_event)?;
Ok(EncodedChangeEvent {
adapter_manifest,
payload,
})
}
fn command<'a>(&self, command: &'a (dyn Any + Send)) -> PersistenceResult<&'a C>
where
C: 'static,
{
command.downcast_ref::<C>().ok_or_else(|| {
PersistenceError::Failure("durable change-event command type mismatch".into())
})
}
}
impl<C, S, E, J> ChangeEventCodec<C, S> for AdaptedChangeEventCodec<C, S, E, J>
where
C: Clone + Send + 'static,
S: 'static,
E: 'static,
J: 'static,
{
fn clone_command(&self, command: &C) -> Box<dyn Any + Send> {
Box::new(command.clone())
}
fn encode_update(
&self,
previous_state: &S,
new_state: &S,
command: &(dyn Any + Send),
) -> PersistenceResult<EncodedChangeEvent> {
self.encode((self.handler.update)(
previous_state,
new_state,
self.command(command)?,
))
}
fn encode_delete(
&self,
previous_state: &S,
command: &(dyn Any + Send),
) -> PersistenceResult<EncodedChangeEvent> {
self.encode((self.handler.delete)(
previous_state,
self.command(command)?,
))
}
}
pub struct DurableStateMessage<C, S>(RuntimeMessage<C, S>);
impl<C, S> DurableStateMessage<C, S> {
#[must_use]
pub fn command(command: C) -> Self {
Self(RuntimeMessage::Command(command))
}
pub(crate) fn get_state(reply: oneshot::Sender<DurableState<S>>) -> Self {
Self(RuntimeMessage::GetState(reply))
}
}
enum RuntimeMessage<C, S> {
Recover,
Command(C),
GetState(oneshot::Sender<DurableState<S>>),
BackoffElapsed,
}
pub(crate) struct DurableStateActor<C, S, J> {
behavior: DurableStateBehavior<C, S, J>,
store: Arc<dyn DurableStateStore>,
writer_uuid: String,
}
pub(crate) struct RuntimeState<C, S> {
state: S,
revision: Revision,
change_event_watermark: Option<SequenceNr>,
manual_stash: VecDeque<C>,
backoff_stash: VecDeque<C>,
unstash_remaining: usize,
phase: RuntimePhase,
backoff: PersistenceBackoff,
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum RuntimePhase {
Running,
BackingOff,
}
struct PersistenceBackoff {
failures: u32,
restarts_in_window: usize,
window_started: Instant,
}
impl PersistenceBackoff {
fn new() -> Self {
Self {
failures: 0,
restarts_in_window: 0,
window_started: Instant::now(),
}
}
fn reset(&mut self) {
self.failures = 0;
self.restarts_in_window = 0;
self.window_started = Instant::now();
}
fn next_delay(&mut self, settings: &RestartSettings) -> Option<Duration> {
let now = Instant::now();
if now.duration_since(self.window_started) > settings.max_restarts_within() {
self.reset();
}
if self.restarts_in_window >= settings.max_restarts() {
return None;
}
let factor = 1_u32.checked_shl(self.failures.min(31)).unwrap_or(u32::MAX);
let base = settings
.min_backoff()
.checked_mul(factor)
.unwrap_or(settings.max_backoff())
.min(settings.max_backoff());
self.failures = self.failures.saturating_add(1);
self.restarts_in_window += 1;
if settings.random_factor() == 0.0 || base.is_zero() {
Some(base)
} else {
let jitter = base.mul_f64(next_random_fraction() * settings.random_factor());
Some(base.saturating_add(jitter))
}
}
}
pub(crate) async fn spawn<C, S, J>(
behavior: DurableStateBehavior<C, S, J>,
store: Arc<dyn DurableStateStore>,
) -> PersistenceResult<DurableStateHandle<C, S>>
where
C: Message,
S: Clone + Send + Sync + 'static,
J: 'static,
{
if behavior.state_codec.is_none() {
return Err(PersistenceError::Serialization(
"state adapter and serializer catalog must be configured before spawn".into(),
));
}
if behavior.change_event_codec.is_some() && behavior.change_event_journal.is_none() {
return Err(PersistenceError::Failure(
"a change-event handler requires a change-event journal".into(),
));
}
let initial_state = RuntimeState {
state: behavior.empty_state.clone(),
revision: Revision::new(0),
change_event_watermark: None,
manual_stash: VecDeque::new(),
backoff_stash: VecDeque::new(),
unstash_remaining: 0,
phase: RuntimePhase::Running,
backoff: PersistenceBackoff::new(),
};
let actor = DurableStateActor {
behavior,
store,
writer_uuid: next_writer_uuid(),
};
let (control, _control_join) =
Actor::spawn(None, actor, initial_state)
.await
.map_err(|error| {
PersistenceError::Failure(format!("durable-state actor failed to spawn: {error}"))
})?;
Ok(DurableStateHandle { control })
}
impl<C, S, J> Actor for DurableStateActor<C, S, J>
where
C: Message,
S: Clone + Send + Sync + 'static,
J: 'static,
{
type Msg = DurableStateMessage<C, S>;
type State = RuntimeState<C, S>;
type Arguments = RuntimeState<C, S>;
async fn pre_start(
&self,
myself: ActorRef<Self::Msg>,
state: Self::Arguments,
) -> Result<Self::State, ActorProcessingErr> {
myself
.cast(DurableStateMessage(RuntimeMessage::Recover))
.map_err(|error| {
Box::new(PersistenceError::Failure(format!(
"failed to enqueue initial durable-state recovery: {error}"
))) as ActorProcessingErr
})?;
Ok(state)
}
async fn handle(
&self,
myself: ActorRef<Self::Msg>,
message: Self::Msg,
state: &mut Self::State,
) -> Result<(), ActorProcessingErr> {
match message.0 {
RuntimeMessage::Recover => self.recover(state).await.map_err(boxed),
RuntimeMessage::Command(command) => {
if state.phase == RuntimePhase::BackingOff {
state.backoff_stash.push_back(command);
return Ok(());
}
if self.process_commands(command, &myself, state).await? {
myself.stop(None);
}
Ok(())
}
RuntimeMessage::GetState(reply) => {
let _ = reply.send(DurableState {
state: state.state.clone(),
revision: state.revision,
});
Ok(())
}
RuntimeMessage::BackoffElapsed => {
state.phase = RuntimePhase::Running;
self.recover(state).await.map_err(boxed)?;
while state.phase == RuntimePhase::Running {
let Some(command) = state.backoff_stash.pop_front() else {
break;
};
if self.process_commands(command, &myself, state).await? {
myself.stop(None);
break;
}
}
Ok(())
}
}
}
}
fn boxed(error: PersistenceError) -> ActorProcessingErr {
Box::new(error)
}
impl<C, S, J> DurableStateActor<C, S, J>
where
C: Message,
S: Clone + Send + Sync + 'static,
J: 'static,
{
async fn recover(&self, state: &mut RuntimeState<C, S>) -> PersistenceResult<()> {
state.state = self.behavior.empty_state.clone();
state.revision = Revision::new(0);
state.change_event_watermark = None;
let result = self.recover_inner(state).await;
match result {
Ok(()) => {
self.signal(&state.state, &DurableStateSignal::RecoveryCompleted)?;
Ok(())
}
Err(error) => {
let _ = self.signal(
&state.state,
&DurableStateSignal::RecoveryFailed(error.clone()),
);
Err(error)
}
}
}
async fn recover_inner(&self, state: &mut RuntimeState<C, S>) -> PersistenceResult<()> {
let stored = self.store.get(&self.behavior.persistence_id).await?;
state.revision = stored.revision;
if let Some(frame) = stored.state {
state.state = self
.behavior
.state_codec
.as_ref()
.expect("spawn validates state codec")
.decode(&frame)?;
}
Ok(())
}
async fn process_commands(
&self,
mut command: C,
myself: &ActorRef<DurableStateMessage<C, S>>,
state: &mut RuntimeState<C, S>,
) -> Result<bool, ActorProcessingErr> {
loop {
let change_command = self
.behavior
.change_event_codec
.as_ref()
.map(|codec| codec.clone_command(&command));
let effect = catch_unwind(AssertUnwindSafe(|| {
(self.behavior.command_handler)(&state.state, command)
}))
.map_err(|panic| boxed(handler_panic("command handler", panic)))?;
let mut stop = false;
match effect.main {
MainEffect::Persist(new_state) => {
let change_event = self.encode_change_event(
Some(&new_state),
change_command.as_deref(),
&state.state,
)?;
match self.persist(Some(new_state), change_event, state).await {
Ok(()) => {
state.backoff.reset();
stop = self.apply_chained(effect.chained, state)?;
}
Err(error) => {
self.start_backoff_or_fail(error, myself, state)?;
}
}
}
MainEffect::Delete => {
let change_event =
self.encode_change_event(None, change_command.as_deref(), &state.state)?;
match self.persist(None, change_event, state).await {
Ok(()) => {
state.backoff.reset();
stop = self.apply_chained(effect.chained, state)?;
}
Err(error) => {
self.start_backoff_or_fail(error, myself, state)?;
}
}
}
MainEffect::None => {
stop = self.apply_chained(effect.chained, state)?;
}
MainEffect::Unhandled => {
tracing::debug!(
persistence_id = %self.behavior.persistence_id,
"durable-state behavior left a command unhandled"
);
stop = self.apply_chained(effect.chained, state)?;
}
MainEffect::Stash(command) => {
if state.manual_stash.len() < self.behavior.stash_capacity {
state.manual_stash.push_back(command);
stop = self.apply_chained(effect.chained, state)?;
} else {
tracing::warn!(
persistence_id = %self.behavior.persistence_id,
stash_capacity = self.behavior.stash_capacity,
"manual durable-state stash is full; dropping newest command"
);
}
}
}
if stop {
return Ok(true);
}
if state.phase == RuntimePhase::BackingOff {
return Ok(false);
}
if state.unstash_remaining > 0 {
state.unstash_remaining -= 1;
if let Some(stashed) = state.manual_stash.pop_front() {
command = stashed;
continue;
}
state.unstash_remaining = 0;
}
return Ok(false);
}
}
fn start_backoff_or_fail(
&self,
error: PersistenceError,
myself: &ActorRef<DurableStateMessage<C, S>>,
state: &mut RuntimeState<C, S>,
) -> Result<(), ActorProcessingErr> {
let Some(settings) = self.behavior.persist_failure_backoff.as_ref() else {
return Err(boxed(error));
};
let Some(delay) = state.backoff.next_delay(settings) else {
return Err(boxed(error));
};
state.phase = RuntimePhase::BackingOff;
std::mem::drop(myself.send_after(delay, || {
DurableStateMessage(RuntimeMessage::BackoffElapsed)
}));
Ok(())
}
fn apply_chained(
&self,
chained: Vec<ChainedEffect<S>>,
state: &mut RuntimeState<C, S>,
) -> PersistenceResult<bool> {
let mut stop = false;
for effect in chained {
match effect {
ChainedEffect::Run(callback) => {
catch_unwind(AssertUnwindSafe(|| callback(&state.state, state.revision)))
.map_err(|panic| handler_panic("side effect", panic))?;
}
ChainedEffect::Stop => stop = true,
ChainedEffect::UnstashAll => {
state.unstash_remaining = state.manual_stash.len();
}
}
}
Ok(stop)
}
async fn persist(
&self,
new_state: Option<S>,
change_event: Option<EncodedChangeEvent>,
state: &mut RuntimeState<C, S>,
) -> PersistenceResult<()> {
let revision = state
.revision
.as_u64()
.checked_add(1)
.map(Revision::new)
.ok_or_else(|| PersistenceError::RevisionExhausted {
persistence_id: self.behavior.persistence_id.clone(),
})?;
let serialized_state = match &new_state {
Some(next) => Some(
self.behavior
.state_codec
.as_ref()
.expect("spawn validates state codec")
.encode(next)?,
),
None => None,
};
match serialized_state {
Some(frame) => {
self.store
.upsert(
&self.behavior.persistence_id,
revision,
frame,
self.behavior.tag.clone(),
)
.await?;
}
None => {
self.store
.delete(&self.behavior.persistence_id, revision)
.await?;
}
}
if let Some(change_event) = change_event {
self.append_change_event(revision, change_event, state)
.await?;
}
state.state = new_state.unwrap_or_else(|| self.behavior.empty_state.clone());
state.revision = revision;
Ok(())
}
fn encode_change_event(
&self,
new_state: Option<&S>,
change_command: Option<&(dyn Any + Send)>,
previous_state: &S,
) -> PersistenceResult<Option<EncodedChangeEvent>> {
match (&self.behavior.change_event_codec, change_command) {
(Some(codec), Some(command)) => match new_state {
Some(next) => codec.encode_update(previous_state, next, command).map(Some),
None => codec.encode_delete(previous_state, command).map(Some),
},
(None, None) => Ok(None),
_ => Err(PersistenceError::Failure(
"durable change-event configuration became inconsistent".into(),
)),
}
}
async fn append_change_event(
&self,
revision: Revision,
change_event: EncodedChangeEvent,
state: &mut RuntimeState<C, S>,
) -> PersistenceResult<()> {
let journal = self
.behavior
.change_event_journal
.as_ref()
.expect("spawn validates change-event journal");
let highest = match state.change_event_watermark {
Some(highest) => highest,
None => {
let highest = journal
.highest_sequence_nr(&self.behavior.persistence_id, SequenceNr::new(0))
.await?;
state.change_event_watermark = Some(highest);
highest
}
};
let sequence_nr = highest
.as_u64()
.checked_add(1)
.map(SequenceNr::new)
.ok_or_else(|| {
PersistenceError::Failure(format!(
"durable change-event journal sequence exhausted for {}",
self.behavior.persistence_id
))
})?;
let stored = StoredEvent {
persistence_id: self.behavior.persistence_id.clone(),
sequence_nr,
adapter_manifest: change_event.adapter_manifest,
writer_uuid: self.writer_uuid.clone(),
timestamp_ms: now_ms(),
metadata: Some(SerializedPayload::new(
CHANGE_EVENT_REVISION_METADATA_SERIALIZER_ID,
CHANGE_EVENT_REVISION_METADATA_MANIFEST,
revision.as_u64().to_be_bytes().to_vec(),
)),
payload: change_event.payload,
tags: self.behavior.tag.iter().cloned().collect(),
};
let write = AtomicWrite::new(vec![stored])?;
match journal.append_one(write).await {
Ok(AppendOutcome::Accepted) => {
state.change_event_watermark = Some(sequence_nr);
Ok(())
}
Ok(AppendOutcome::Rejected { reason }) => {
Err(PersistenceError::Rejected(reason))
}
Err(error) => {
state.change_event_watermark = None;
Err(error)
}
}
}
fn signal(&self, state: &S, signal: &DurableStateSignal) -> PersistenceResult<()> {
catch_unwind(AssertUnwindSafe(|| {
(self.behavior.signal_handler)(state, signal)
}))
.map_err(|panic| handler_panic("signal handler", panic))
}
}
fn encode_state_envelope(
adapter_manifest: String,
frame: SerializedPayload,
) -> PersistenceResult<SerializedPayload> {
let manifest_len = u32::try_from(adapter_manifest.len()).map_err(|error| {
PersistenceError::Serialization(format!(
"durable state adapter manifest is too large: {error}"
))
})?;
let mut payload = Vec::with_capacity(
STATE_ENVELOPE_MAGIC.len() + 4 + adapter_manifest.len() + frame.payload.len(),
);
payload.extend_from_slice(STATE_ENVELOPE_MAGIC);
payload.extend_from_slice(&manifest_len.to_be_bytes());
payload.extend_from_slice(adapter_manifest.as_bytes());
payload.extend_from_slice(&frame.payload);
Ok(SerializedPayload {
serializer_id: frame.serializer_id,
manifest: frame.manifest,
payload,
})
}
fn decode_state_envelope(
state: &SerializedPayload,
) -> PersistenceResult<(String, SerializedPayload)> {
let payload = state
.payload
.strip_prefix(STATE_ENVELOPE_MAGIC)
.ok_or_else(|| {
PersistenceError::Serialization(
"durable state payload is missing the versioned envelope".into(),
)
})?;
let (manifest_len, payload) = payload.split_at_checked(4).ok_or_else(|| {
PersistenceError::Serialization("durable state adapter envelope is truncated".into())
})?;
let manifest_len = u32::from_be_bytes(
manifest_len
.try_into()
.expect("split_at_checked returned exactly four bytes"),
) as usize;
let (adapter_manifest, serialized_payload) =
payload.split_at_checked(manifest_len).ok_or_else(|| {
PersistenceError::Serialization(
"durable state adapter manifest length exceeds the stored payload".into(),
)
})?;
let adapter_manifest = String::from_utf8(adapter_manifest.to_vec()).map_err(|error| {
PersistenceError::Serialization(format!(
"durable state adapter manifest is not UTF-8: {error}"
))
})?;
Ok((
adapter_manifest,
SerializedPayload {
serializer_id: state.serializer_id,
manifest: state.manifest.clone(),
payload: serialized_payload.to_vec(),
},
))
}
fn next_writer_uuid() -> String {
format!(
"datum-durable-state-{}-{}-{}",
std::process::id(),
now_nanos(),
NEXT_WRITER_ID.fetch_add(1, Ordering::Relaxed)
)
}
fn now_ms() -> i64 {
let millis = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or(Duration::ZERO)
.as_millis();
i64::try_from(millis).unwrap_or(i64::MAX)
}
fn now_nanos() -> u128 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or(Duration::ZERO)
.as_nanos()
}
fn next_random_fraction() -> f64 {
let mut current = BACKOFF_RANDOM_SEED.load(Ordering::Relaxed);
loop {
let mut next = current;
next ^= next << 13;
next ^= next >> 7;
next ^= next << 17;
match BACKOFF_RANDOM_SEED.compare_exchange_weak(
current,
next,
Ordering::Relaxed,
Ordering::Relaxed,
) {
Ok(_) => return ((next >> 11) as f64) / ((1_u64 << 53) as f64),
Err(observed) => current = observed,
}
}
}
fn handler_panic(handler: &str, panic: Box<dyn Any + Send + 'static>) -> PersistenceError {
let message = if let Some(message) = panic.downcast_ref::<&str>() {
(*message).to_owned()
} else if let Some(message) = panic.downcast_ref::<String>() {
message.clone()
} else {
"non-string panic".to_owned()
};
PersistenceError::Failure(format!("{handler} panicked: {message}"))
}
#[cfg(test)]
mod tests {
use super::{STATE_ENVELOPE_MAGIC, decode_state_envelope, encode_state_envelope};
use crate::{SerializedPayload, SerializerId};
#[test]
fn state_envelope_is_always_written_and_preserves_manifests() {
let frame = SerializedPayload::new(SerializerId::new(9), "serializer-v2", vec![1, 2, 3]);
let stored = encode_state_envelope("adapter-v4".into(), frame.clone()).unwrap();
let (adapter_manifest, decoded) = decode_state_envelope(&stored).unwrap();
assert_eq!(adapter_manifest, "adapter-v4");
assert_eq!(decoded, frame);
let identity = encode_state_envelope(String::new(), frame.clone()).unwrap();
assert_ne!(identity, frame);
assert!(identity.payload.starts_with(STATE_ENVELOPE_MAGIC));
let (adapter_manifest, decoded) = decode_state_envelope(&identity).unwrap();
assert_eq!(adapter_manifest, "");
assert_eq!(decoded, frame);
assert!(decode_state_envelope(&frame).is_err());
}
}