Skip to main content

datum_persistence/behavior/
runtime.rs

1use std::{
2    collections::VecDeque,
3    panic::{AssertUnwindSafe, catch_unwind},
4    sync::{
5        Arc, LazyLock,
6        atomic::{AtomicU64, Ordering},
7    },
8    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
9};
10
11use datum::{
12    RestartSettings, Topic, TopicOverflow,
13    actor::{Actor, ActorProcessingErr, ActorRef, Message},
14};
15use futures_util::StreamExt;
16use tokio::sync::oneshot;
17
18use super::{
19    EventSourcedBehavior, EventSourcedHandle, EventSourcedRuntime, EventSourcedSignal,
20    EventSourcedState, PublishedEvent, Recovery, RetentionCriteria, SnapshotSelection,
21    effect::{ChainedEffect, Effect, EventWithMetadata, MainEffect},
22};
23use crate::{
24    AppendOutcome, AtomicWrite, EventAdapter, PersistenceError, PersistenceResult, ReplayRange,
25    SequenceNr, SerializedPayload, SerializerCatalog, SnapshotCriteria, SnapshotMetadata,
26    StateAdapter, StoredEvent,
27};
28
29static NEXT_WRITER_ID: AtomicU64 = AtomicU64::new(1);
30static WRITER_STARTUP_NANOS: LazyLock<u128> = LazyLock::new(|| {
31    SystemTime::now()
32        .duration_since(UNIX_EPOCH)
33        .map(|duration| duration.as_nanos())
34        .unwrap_or_default()
35});
36static BACKOFF_RANDOM_SEED: AtomicU64 = AtomicU64::new(0x517c_c1b7_2722_0a95);
37
38pub(crate) fn next_writer_uuid() -> String {
39    format!(
40        "datum-persistence-{}-{}-{}",
41        std::process::id(),
42        *WRITER_STARTUP_NANOS,
43        NEXT_WRITER_ID.fetch_add(1, Ordering::Relaxed)
44    )
45}
46
47pub(crate) struct EventCodec<E, J> {
48    adapter: Arc<dyn EventAdapter<E, J>>,
49    catalog: Arc<SerializerCatalog<J>>,
50}
51
52impl<E, J> EventCodec<E, J> {
53    pub(crate) fn new<A>(adapter: A, catalog: SerializerCatalog<J>) -> Self
54    where
55        A: EventAdapter<E, J>,
56    {
57        Self {
58            adapter: Arc::new(adapter),
59            catalog: Arc::new(catalog),
60        }
61    }
62}
63
64pub(crate) trait StateCodec<S>: Send + Sync + 'static {
65    fn encode(&self, state: &S) -> PersistenceResult<(String, SerializedPayload)>;
66    fn decode(&self, adapter_manifest: &str, state: &SerializedPayload) -> PersistenceResult<S>;
67}
68
69pub(crate) struct AdaptedStateCodec<S, J> {
70    adapter: Arc<dyn StateAdapter<S, J>>,
71    catalog: SerializerCatalog<J>,
72}
73
74impl<S, J> AdaptedStateCodec<S, J> {
75    pub(crate) fn new<A>(adapter: A, catalog: SerializerCatalog<J>) -> Self
76    where
77        A: StateAdapter<S, J>,
78    {
79        Self {
80            adapter: Arc::new(adapter),
81            catalog,
82        }
83    }
84}
85
86impl<S, J> StateCodec<S> for AdaptedStateCodec<S, J>
87where
88    S: 'static,
89    J: 'static,
90{
91    fn encode(&self, state: &S) -> PersistenceResult<(String, SerializedPayload)> {
92        let (adapter_manifest, journal_state) = self.adapter.to_journal(state)?;
93        let frame = self.catalog.encode(&journal_state)?;
94        Ok((adapter_manifest, frame))
95    }
96
97    fn decode(&self, adapter_manifest: &str, state: &SerializedPayload) -> PersistenceResult<S> {
98        let journal_state = self.catalog.decode(state)?;
99        self.adapter.from_journal(adapter_manifest, journal_state)
100    }
101}
102
103/// A message accepted by an event-sourced behavior's single ordered ingress.
104///
105/// Construct domain-command messages with [`Self::command`]. Runtime control
106/// messages are intentionally private so callers cannot bypass handle APIs.
107pub struct EventSourcedMessage<C, S>(RuntimeMessage<C, S>);
108
109impl<C, S> EventSourcedMessage<C, S> {
110    /// Wraps one domain command for direct actor or `ActorFlow::ask` ingress.
111    #[must_use]
112    pub fn command(command: C) -> Self {
113        Self(RuntimeMessage::Command(command))
114    }
115
116    pub(crate) fn get_state(reply: oneshot::Sender<EventSourcedState<S>>) -> Self {
117        Self(RuntimeMessage::GetState(reply))
118    }
119}
120
121enum RuntimeMessage<C, S> {
122    Recover,
123    Command(C),
124    GetState(oneshot::Sender<EventSourcedState<S>>),
125    BackoffElapsed,
126    RetentionSignal(EventSourcedSignal),
127    RetentionCleanupFinished,
128}
129
130pub(crate) struct EventSourcedActor<C, E: Send + Sync + 'static, S, J> {
131    behavior: EventSourcedBehavior<C, E, S, J>,
132    runtime: EventSourcedRuntime,
133    topic: Option<Topic<PublishedEvent<E>>>,
134}
135
136pub(crate) struct RuntimeState<C, S> {
137    state: S,
138    sequence_nr: SequenceNr,
139    manual_stash: VecDeque<C>,
140    backoff_stash: VecDeque<C>,
141    unstash_remaining: usize,
142    retention_in_flight: bool,
143    phase: RuntimePhase,
144    backoff: PersistenceBackoff,
145}
146
147#[derive(Clone, Copy, PartialEq, Eq)]
148enum RuntimePhase {
149    Running,
150    BackingOff,
151}
152
153struct PersistenceBackoff {
154    failures: u32,
155    restarts_in_window: usize,
156    window_started: Instant,
157}
158
159struct RetentionTaskGuard<C: Message, S: Send + 'static> {
160    actor: ActorRef<EventSourcedMessage<C, S>>,
161}
162
163impl<C: Message, S: Send + 'static> Drop for RetentionTaskGuard<C, S> {
164    fn drop(&mut self) {
165        let _ = self.actor.cast(EventSourcedMessage(
166            RuntimeMessage::RetentionCleanupFinished,
167        ));
168    }
169}
170
171impl PersistenceBackoff {
172    fn new() -> Self {
173        Self {
174            failures: 0,
175            restarts_in_window: 0,
176            window_started: Instant::now(),
177        }
178    }
179
180    fn reset(&mut self) {
181        self.failures = 0;
182        self.restarts_in_window = 0;
183        self.window_started = Instant::now();
184    }
185
186    fn next_delay(&mut self, settings: &RestartSettings) -> Option<Duration> {
187        let now = Instant::now();
188        if now.duration_since(self.window_started) > settings.max_restarts_within() {
189            self.reset();
190        }
191        if self.restarts_in_window >= settings.max_restarts() {
192            return None;
193        }
194
195        let factor = 1_u32.checked_shl(self.failures.min(31)).unwrap_or(u32::MAX);
196        let base = settings
197            .min_backoff()
198            .checked_mul(factor)
199            .unwrap_or(settings.max_backoff())
200            .min(settings.max_backoff());
201        self.failures = self.failures.saturating_add(1);
202        self.restarts_in_window += 1;
203        if settings.random_factor() == 0.0 || base.is_zero() {
204            Some(base)
205        } else {
206            let jitter = base.mul_f64(next_random_fraction() * settings.random_factor());
207            Some(base.saturating_add(jitter))
208        }
209    }
210}
211
212pub(crate) async fn spawn<C, E, S, J>(
213    behavior: EventSourcedBehavior<C, E, S, J>,
214    mut runtime: EventSourcedRuntime,
215) -> PersistenceResult<EventSourcedHandle<C, E, S>>
216where
217    C: Message,
218    E: Send + Sync + 'static,
219    S: Clone + Send + Sync + 'static,
220    J: Send + 'static,
221{
222    if behavior.event_codec.is_none() {
223        return Err(PersistenceError::Serialization(
224            "event adapter and serializer catalog must be configured before spawn".into(),
225        ));
226    }
227    if let RetentionCriteria::SnapshotEvery { every, keep, .. } = behavior.retention {
228        if every == 0 {
229            return Err(PersistenceError::Rejected(
230                "snapshot frequency must be greater than zero".into(),
231            ));
232        }
233        if keep == 0 {
234            return Err(PersistenceError::Rejected(
235                "snapshot retention count must be greater than zero".into(),
236            ));
237        }
238    }
239    if runtime.writer_uuid.is_none() {
240        runtime.writer_uuid = Some(next_writer_uuid());
241    }
242
243    let topic = if behavior.publish_events {
244        Some(
245            Topic::new(runtime.topic_capacity, TopicOverflow::Dropping).map_err(|error| {
246                PersistenceError::Failure(format!("published-event topic failed to start: {error}"))
247            })?,
248        )
249    } else {
250        None
251    };
252
253    let initial_state = RuntimeState {
254        state: behavior.empty_state.clone(),
255        sequence_nr: SequenceNr::new(0),
256        manual_stash: VecDeque::new(),
257        backoff_stash: VecDeque::new(),
258        unstash_remaining: 0,
259        retention_in_flight: false,
260        phase: RuntimePhase::Running,
261        backoff: PersistenceBackoff::new(),
262    };
263    let actor = EventSourcedActor {
264        behavior,
265        runtime,
266        topic: topic.clone(),
267    };
268    let (control, _control_join) =
269        Actor::spawn(None, actor, initial_state)
270            .await
271            .map_err(|error| {
272                PersistenceError::Failure(format!("event-sourced actor failed to spawn: {error}"))
273            })?;
274    Ok(EventSourcedHandle {
275        control,
276        published_events: topic,
277    })
278}
279
280impl<C, E, S, J> Actor for EventSourcedActor<C, E, S, J>
281where
282    C: Message,
283    E: Send + Sync + 'static,
284    S: Clone + Send + Sync + 'static,
285    J: Send + 'static,
286{
287    type Msg = EventSourcedMessage<C, S>;
288    type State = RuntimeState<C, S>;
289    type Arguments = RuntimeState<C, S>;
290
291    async fn pre_start(
292        &self,
293        myself: ActorRef<Self::Msg>,
294        state: Self::Arguments,
295    ) -> Result<Self::State, ActorProcessingErr> {
296        myself
297            .cast(EventSourcedMessage(RuntimeMessage::Recover))
298            .map_err(|error| {
299                Box::new(PersistenceError::Failure(format!(
300                    "failed to enqueue initial recovery: {error}"
301                ))) as ActorProcessingErr
302            })?;
303        Ok(state)
304    }
305
306    async fn handle(
307        &self,
308        myself: ActorRef<Self::Msg>,
309        message: Self::Msg,
310        state: &mut Self::State,
311    ) -> Result<(), ActorProcessingErr> {
312        if let Err(error) = self.handle_runtime_message(&myself, message, state).await {
313            myself.stop(Some(error.to_string()));
314        }
315        Ok(())
316    }
317
318    async fn post_stop(
319        &self,
320        _myself: ActorRef<Self::Msg>,
321        _state: &mut Self::State,
322    ) -> Result<(), ActorProcessingErr> {
323        self.close_topic();
324        Ok(())
325    }
326}
327
328impl<C, E, S, J> EventSourcedActor<C, E, S, J>
329where
330    C: Message,
331    E: Send + Sync + 'static,
332    S: Clone + Send + Sync + 'static,
333    J: Send + 'static,
334{
335    async fn handle_runtime_message(
336        &self,
337        myself: &ActorRef<EventSourcedMessage<C, S>>,
338        message: EventSourcedMessage<C, S>,
339        state: &mut RuntimeState<C, S>,
340    ) -> Result<(), ActorProcessingErr> {
341        match message.0 {
342            RuntimeMessage::Recover => self.recover(state).await.map_err(boxed),
343            RuntimeMessage::Command(command) => {
344                if state.phase == RuntimePhase::BackingOff {
345                    state.backoff_stash.push_back(command);
346                    return Ok(());
347                }
348                let stop = self.process_commands(command, myself, state).await?;
349                if stop {
350                    myself.stop(None);
351                }
352                Ok(())
353            }
354            RuntimeMessage::GetState(reply) => {
355                let _ = reply.send(EventSourcedState {
356                    state: state.state.clone(),
357                    sequence_nr: state.sequence_nr,
358                });
359                Ok(())
360            }
361            RuntimeMessage::BackoffElapsed => {
362                state.phase = RuntimePhase::Running;
363                self.recover(state).await.map_err(boxed)?;
364                while state.phase == RuntimePhase::Running {
365                    let Some(command) = state.backoff_stash.pop_front() else {
366                        break;
367                    };
368                    if self.process_commands(command, myself, state).await? {
369                        myself.stop(None);
370                        break;
371                    }
372                }
373                Ok(())
374            }
375            RuntimeMessage::RetentionSignal(signal) => {
376                self.signal(&state.state, &signal)?;
377                Ok(())
378            }
379            RuntimeMessage::RetentionCleanupFinished => {
380                state.retention_in_flight = false;
381                Ok(())
382            }
383        }
384    }
385}
386
387fn boxed(error: PersistenceError) -> ActorProcessingErr {
388    Box::new(error)
389}
390
391enum PersistError {
392    Rejected {
393        sequence_nr: SequenceNr,
394        reason: String,
395    },
396    Failed {
397        sequence_nr: SequenceNr,
398        error: PersistenceError,
399    },
400    Behavior(PersistenceError),
401}
402
403struct PendingPublication<E> {
404    event: E,
405    sequence_nr: SequenceNr,
406    timestamp_ms: i64,
407    tags: Vec<crate::Tag>,
408}
409
410enum SnapshotReason {
411    Predicate,
412    Retention,
413}
414
415impl<C, E, S, J> EventSourcedActor<C, E, S, J>
416where
417    C: Message,
418    E: Send + Sync + 'static,
419    S: Clone + Send + Sync + 'static,
420    J: Send + 'static,
421{
422    async fn recover(&self, state: &mut RuntimeState<C, S>) -> PersistenceResult<()> {
423        state.state = self.behavior.empty_state.clone();
424        state.sequence_nr = SequenceNr::new(0);
425
426        let result = self.recover_inner(state).await;
427        match result {
428            Ok(()) => {
429                self.signal(&state.state, &EventSourcedSignal::RecoveryCompleted)?;
430                Ok(())
431            }
432            Err(error) => {
433                let signal_error = error.clone();
434                let _ = self.signal(
435                    &state.state,
436                    &EventSourcedSignal::RecoveryFailed(signal_error),
437                );
438                Err(error)
439            }
440        }
441    }
442
443    async fn recover_inner(&self, state: &mut RuntimeState<C, S>) -> PersistenceResult<()> {
444        let Recovery::Enabled(selection) = self.behavior.recovery else {
445            state.sequence_nr = self
446                .runtime
447                .journal
448                .highest_sequence_nr(&self.behavior.persistence_id, SequenceNr::new(0))
449                .await?;
450            return Ok(());
451        };
452
453        if selection != SnapshotSelection::None
454            && let Some(snapshot_store) = &self.runtime.snapshot_store
455        {
456            let criteria = selection.criteria();
457            if let Some(snapshot) = snapshot_store
458                .load(&self.behavior.persistence_id, &criteria)
459                .await?
460            {
461                let codec = self.behavior.state_codec.as_ref().ok_or_else(|| {
462                    PersistenceError::Serialization(
463                        "a state adapter and serializer catalog are required to recover snapshots"
464                            .into(),
465                    )
466                })?;
467                state.state = codec.decode(&snapshot.adapter_manifest, &snapshot.state)?;
468                state.sequence_nr = snapshot.metadata.sequence_nr;
469                self.signal(
470                    &state.state,
471                    &EventSourcedSignal::SnapshotRecovered(snapshot.metadata),
472                )?;
473            }
474        }
475
476        let highest = self
477            .runtime
478            .journal
479            .highest_sequence_nr(&self.behavior.persistence_id, state.sequence_nr)
480            .await?;
481        if state.sequence_nr != SequenceNr::new(u64::MAX) {
482            let from = SequenceNr::new(state.sequence_nr.as_u64() + 1);
483            if from <= highest {
484                let mut replay = self
485                    .runtime
486                    .journal
487                    .replay(ReplayRange {
488                        persistence_id: self.behavior.persistence_id.clone(),
489                        from,
490                        to: highest,
491                        max: u64::MAX,
492                    })
493                    .await?;
494                while let Some(stored) = replay.next().await {
495                    let stored = stored?;
496                    let codec = self
497                        .behavior
498                        .event_codec
499                        .as_ref()
500                        .expect("spawn validates event codec");
501                    let journal_event = codec.catalog.decode(&stored.payload)?;
502                    let events = codec
503                        .adapter
504                        .from_journal_events(&stored.adapter_manifest, journal_event)?;
505                    for event in events {
506                        state.state = self.apply_event(&state.state, &event)?;
507                    }
508                    state.sequence_nr = stored.sequence_nr;
509                }
510            }
511        }
512        state.sequence_nr = state.sequence_nr.max(highest);
513        Ok(())
514    }
515
516    async fn process_commands(
517        &self,
518        mut command: C,
519        myself: &ActorRef<EventSourcedMessage<C, S>>,
520        state: &mut RuntimeState<C, S>,
521    ) -> Result<bool, ActorProcessingErr> {
522        loop {
523            let effect = catch_unwind(AssertUnwindSafe(|| {
524                (self.behavior.command_handler)(&state.state, command)
525            }))
526            .map_err(|panic| boxed(handler_panic("command handler", panic)))?;
527            let effect = Self::resolve_async(effect).await;
528
529            let mut stop = false;
530            match effect.main {
531                MainEffect::Persist(events) if events.is_empty() => {
532                    stop = self.apply_chained(effect.chained, state)?;
533                }
534                MainEffect::Persist(events) => match self.persist(events, myself, state).await {
535                    Ok(()) => {
536                        state.backoff.reset();
537                        stop = self.apply_chained(effect.chained, state)?;
538                    }
539                    Err(PersistError::Rejected {
540                        sequence_nr,
541                        reason,
542                    }) => {
543                        self.signal(
544                            &state.state,
545                            &EventSourcedSignal::PersistRejected {
546                                sequence_nr,
547                                reason: reason.clone(),
548                            },
549                        )?;
550                        return Err(boxed(PersistenceError::Rejected(reason)));
551                    }
552                    Err(PersistError::Failed { sequence_nr, error }) => {
553                        self.signal(
554                            &state.state,
555                            &EventSourcedSignal::PersistFailed {
556                                sequence_nr,
557                                error: error.clone(),
558                            },
559                        )?;
560                        let Some(settings) = self.behavior.persist_failure_backoff.as_ref() else {
561                            return Err(boxed(error));
562                        };
563                        let Some(delay) = state.backoff.next_delay(settings) else {
564                            return Err(boxed(error));
565                        };
566                        state.phase = RuntimePhase::BackingOff;
567                        std::mem::drop(myself.send_after(delay, || {
568                            EventSourcedMessage(RuntimeMessage::BackoffElapsed)
569                        }));
570                    }
571                    Err(PersistError::Behavior(error)) => return Err(boxed(error)),
572                },
573                MainEffect::None => {
574                    stop = self.apply_chained(effect.chained, state)?;
575                }
576                MainEffect::Unhandled => {
577                    tracing::debug!(
578                        persistence_id = %self.behavior.persistence_id,
579                        "persistent behavior left a command unhandled"
580                    );
581                    stop = self.apply_chained(effect.chained, state)?;
582                }
583                MainEffect::Stash(command) => {
584                    if state.manual_stash.len() < self.behavior.stash_capacity {
585                        state.manual_stash.push_back(command);
586                        stop = self.apply_chained(effect.chained, state)?;
587                    } else {
588                        tracing::warn!(
589                            persistence_id = %self.behavior.persistence_id,
590                            stash_capacity = self.behavior.stash_capacity,
591                            "manual persistence stash is full; dropping newest command"
592                        );
593                    }
594                }
595                MainEffect::Async(_) => unreachable!("async effects are resolved before dispatch"),
596            }
597
598            if stop {
599                return Ok(true);
600            }
601            if state.phase == RuntimePhase::BackingOff {
602                return Ok(false);
603            }
604
605            if state.unstash_remaining > 0 {
606                state.unstash_remaining -= 1;
607                if let Some(stashed) = state.manual_stash.pop_front() {
608                    command = stashed;
609                    continue;
610                } else {
611                    state.unstash_remaining = 0;
612                }
613            }
614            return Ok(false);
615        }
616    }
617
618    async fn resolve_async(mut effect: Effect<C, E, S>) -> Effect<C, E, S> {
619        let mut after_async = Vec::new();
620        loop {
621            match effect.main {
622                MainEffect::Async(future) => {
623                    after_async.splice(0..0, effect.chained);
624                    effect = future.await;
625                }
626                _ => {
627                    effect.chained.extend(after_async);
628                    return effect;
629                }
630            }
631        }
632    }
633
634    fn apply_chained(
635        &self,
636        chained: Vec<ChainedEffect<S>>,
637        state: &mut RuntimeState<C, S>,
638    ) -> PersistenceResult<bool> {
639        let mut stop = false;
640        for effect in chained {
641            match effect {
642                ChainedEffect::Run(callback) => {
643                    catch_unwind(AssertUnwindSafe(|| callback(&state.state)))
644                        .map_err(|panic| handler_panic("side effect", panic))?;
645                }
646                ChainedEffect::Stop => stop = true,
647                ChainedEffect::UnstashAll => {
648                    state.unstash_remaining = state.manual_stash.len();
649                }
650            }
651        }
652        Ok(stop)
653    }
654
655    async fn persist(
656        &self,
657        events: Vec<EventWithMetadata<E>>,
658        myself: &ActorRef<EventSourcedMessage<C, S>>,
659        state: &mut RuntimeState<C, S>,
660    ) -> Result<(), PersistError> {
661        let first_sequence_nr = state
662            .sequence_nr
663            .as_u64()
664            .checked_add(1)
665            .map(SequenceNr::new)
666            .ok_or_else(|| PersistError::Rejected {
667                sequence_nr: state.sequence_nr,
668                reason: "event sequence number is exhausted".into(),
669            })?;
670        let codec = self
671            .behavior
672            .event_codec
673            .as_ref()
674            .expect("spawn validates event codec");
675        let mut publications = self
676            .topic
677            .as_ref()
678            .map(|_| Vec::with_capacity(events.len()));
679        let mut staged_state = None;
680        let mut stored_events = Vec::with_capacity(events.len());
681        let mut snapshot_reason = None;
682        let mut last_sequence_nr = first_sequence_nr;
683
684        for (index, event) in events.into_iter().enumerate() {
685            let offset = u64::try_from(index).map_err(|error| PersistError::Rejected {
686                sequence_nr: first_sequence_nr,
687                reason: format!("event batch is too large: {error}"),
688            })?;
689            let sequence_nr = first_sequence_nr
690                .as_u64()
691                .checked_add(offset)
692                .map(SequenceNr::new)
693                .ok_or_else(|| PersistError::Rejected {
694                    sequence_nr: first_sequence_nr,
695                    reason: "event sequence number is exhausted".into(),
696                })?;
697            let current_state = staged_state.as_ref().unwrap_or(&state.state);
698            let next_state = self
699                .apply_event(current_state, &event.event)
700                .map_err(PersistError::Behavior)?;
701
702            let retention_match = matches!(
703                self.behavior.retention,
704                RetentionCriteria::SnapshotEvery { every, .. }
705                    if sequence_nr.as_u64() % every == 0
706            );
707            if retention_match {
708                snapshot_reason = Some(SnapshotReason::Retention);
709            } else if snapshot_reason.is_none()
710                && (self.behavior.snapshot_when)(&next_state, &event.event, sequence_nr)
711            {
712                snapshot_reason = Some(SnapshotReason::Predicate);
713            }
714
715            let (adapter_manifest, journal_event) = codec
716                .adapter
717                .to_journal(&event.event)
718                .map_err(|error| PersistError::Rejected {
719                    sequence_nr,
720                    reason: error.to_string(),
721                })?;
722            let payload =
723                codec
724                    .catalog
725                    .encode(&journal_event)
726                    .map_err(|error| PersistError::Rejected {
727                        sequence_nr,
728                        reason: error.to_string(),
729                    })?;
730            let timestamp_ms = now_ms();
731            let tags = (self.behavior.tagger)(&next_state, &event.event);
732            if let Some(publications) = &mut publications {
733                publications.push(PendingPublication {
734                    event: event.event,
735                    sequence_nr,
736                    timestamp_ms,
737                    tags: tags.clone(),
738                });
739            }
740            stored_events.push(StoredEvent {
741                // `PersistenceId` is `Arc<str>`-backed, so this clone is an
742                // atomic refcount bump, not a fresh allocation of the full
743                // `entity_type|entity_id` string, even though this push runs
744                // once per event in the batch.
745                persistence_id: self.behavior.persistence_id.clone(),
746                sequence_nr,
747                adapter_manifest,
748                // `writer_uuid` stays a plain `String` on `StoredEvent`
749                // (shared by every backend: Postgres binds it via
750                // `event.writer_uuid.as_str()`, RocksDB and Cassandra move it
751                // by value into their own row/envelope encodings). Making it
752                // `Arc<str>` would ripple into those backends for a small,
753                // fixed-length string (a UUID), so this per-event clone
754                // remains a real, but minor, allocation — the accepted
755                // remaining cost per the M15 perf small-batch review.
756                writer_uuid: self
757                    .runtime
758                    .writer_uuid
759                    .clone()
760                    .expect("spawn assigns a writer uuid"),
761                timestamp_ms,
762                metadata: event.metadata,
763                payload,
764                tags,
765            });
766            staged_state = Some(next_state);
767            last_sequence_nr = sequence_nr;
768        }
769
770        let write = AtomicWrite::new(stored_events).map_err(|error| PersistError::Rejected {
771            sequence_nr: first_sequence_nr,
772            reason: error.to_string(),
773        })?;
774        let outcome = self
775            .runtime
776            .journal
777            .append_one(write)
778            .await
779            .map_err(|error| PersistError::Failed {
780                sequence_nr: first_sequence_nr,
781                error,
782            })?;
783        if let AppendOutcome::Rejected { reason } = outcome {
784            return Err(PersistError::Rejected {
785                sequence_nr: first_sequence_nr,
786                reason,
787            });
788        }
789
790        state.state = staged_state.expect("non-empty event batch was validated");
791        state.sequence_nr = last_sequence_nr;
792
793        if let Some(topic) = self
794            .topic
795            .as_ref()
796            .filter(|topic| topic.subscriber_count() > 0)
797        {
798            for item in publications.unwrap_or_default() {
799                let _ = topic
800                    .publish(PublishedEvent {
801                        persistence_id: self.behavior.persistence_id.clone(),
802                        sequence_nr: item.sequence_nr,
803                        event: item.event,
804                        timestamp_ms: item.timestamp_ms,
805                        tags: item.tags,
806                    })
807                    .await;
808            }
809        }
810
811        if let Some(reason) = snapshot_reason {
812            let skip_retention =
813                matches!(reason, SnapshotReason::Retention) && state.retention_in_flight;
814            if skip_retention {
815                tracing::debug!(
816                    persistence_id = %self.behavior.persistence_id,
817                    sequence_nr = state.sequence_nr.as_u64(),
818                    "retention snapshot boundary skipped while cleanup is in flight; the next boundary covers it"
819                );
820            } else {
821                self.save_snapshot(reason, myself, state)
822                    .await
823                    .map_err(PersistError::Behavior)?;
824            }
825        }
826        Ok(())
827    }
828
829    async fn save_snapshot(
830        &self,
831        reason: SnapshotReason,
832        myself: &ActorRef<EventSourcedMessage<C, S>>,
833        state: &mut RuntimeState<C, S>,
834    ) -> PersistenceResult<()> {
835        let Some(snapshot_store) = &self.runtime.snapshot_store else {
836            return Ok(());
837        };
838        let metadata = SnapshotMetadata {
839            persistence_id: self.behavior.persistence_id.clone(),
840            sequence_nr: state.sequence_nr,
841            timestamp_ms: now_ms(),
842        };
843        let save_result = match &self.behavior.state_codec {
844            Some(codec) => match codec.encode(&state.state) {
845                Ok((adapter_manifest, frame)) => {
846                    snapshot_store
847                        .save(metadata.clone(), adapter_manifest, frame)
848                        .await
849                }
850                Err(error) => Err(error),
851            },
852            None => Err(PersistenceError::Serialization(
853                "a state adapter and serializer catalog are required to save snapshots".into(),
854            )),
855        };
856        match save_result {
857            Ok(()) => {
858                self.signal(
859                    &state.state,
860                    &EventSourcedSignal::SnapshotCompleted(metadata.clone()),
861                )?;
862                if matches!(reason, SnapshotReason::Retention) {
863                    self.start_retention_cleanup(
864                        metadata.sequence_nr,
865                        Arc::clone(snapshot_store),
866                        myself,
867                        state,
868                    );
869                }
870            }
871            Err(error) => {
872                self.signal(
873                    &state.state,
874                    &EventSourcedSignal::SnapshotFailed { metadata, error },
875                )?;
876            }
877        }
878        Ok(())
879    }
880
881    fn start_retention_cleanup(
882        &self,
883        snapshot_sequence_nr: SequenceNr,
884        snapshot_store: Arc<dyn crate::SnapshotStore>,
885        myself: &ActorRef<EventSourcedMessage<C, S>>,
886        state: &mut RuntimeState<C, S>,
887    ) {
888        let RetentionCriteria::SnapshotEvery {
889            every,
890            keep,
891            delete_events,
892        } = self.behavior.retention
893        else {
894            return;
895        };
896        state.retention_in_flight = true;
897        let retained_window = every.saturating_mul(keep);
898        let delete_to = SequenceNr::new(
899            snapshot_sequence_nr
900                .as_u64()
901                .saturating_sub(retained_window),
902        );
903        let persistence_id = self.behavior.persistence_id.clone();
904        let journal = Arc::clone(&self.runtime.journal);
905        let actor = myself.clone();
906        tokio::spawn(async move {
907            let _completion = RetentionTaskGuard {
908                actor: actor.clone(),
909            };
910            if delete_events {
911                match journal.delete_to(&persistence_id, delete_to).await {
912                    Ok(()) => {
913                        if actor
914                            .cast(EventSourcedMessage(RuntimeMessage::RetentionSignal(
915                                EventSourcedSignal::DeleteEventsCompleted(delete_to),
916                            )))
917                            .is_err()
918                        {
919                            return;
920                        }
921                    }
922                    Err(error) => {
923                        let _ = actor.cast(EventSourcedMessage(RuntimeMessage::RetentionSignal(
924                            EventSourcedSignal::DeleteEventsFailed {
925                                to: delete_to,
926                                error,
927                            },
928                        )));
929                        return;
930                    }
931                }
932            }
933
934            let max_snapshot = if delete_events {
935                delete_to.as_u64().saturating_sub(1)
936            } else {
937                delete_to.as_u64()
938            };
939            let criteria = SnapshotCriteria {
940                min_sequence_nr: SequenceNr::new(0),
941                max_sequence_nr: SequenceNr::new(max_snapshot),
942                min_timestamp_ms: i64::MIN,
943                max_timestamp_ms: i64::MAX,
944            };
945            let signal = match snapshot_store
946                .delete_matching(&persistence_id, &criteria)
947                .await
948            {
949                Ok(()) => EventSourcedSignal::DeleteSnapshotsCompleted(criteria),
950                Err(error) => EventSourcedSignal::DeleteSnapshotsFailed { criteria, error },
951            };
952            let _ = actor.cast(EventSourcedMessage(RuntimeMessage::RetentionSignal(signal)));
953        });
954    }
955
956    fn close_topic(&self) {
957        if let Some(topic) = &self.topic {
958            let _ = topic.close();
959        }
960    }
961
962    fn apply_event(&self, state: &S, event: &E) -> PersistenceResult<S> {
963        catch_unwind(AssertUnwindSafe(|| {
964            (self.behavior.event_handler)(state, event)
965        }))
966        .map_err(|panic| handler_panic("event handler", panic))?
967    }
968
969    fn signal(&self, state: &S, signal: &EventSourcedSignal) -> PersistenceResult<()> {
970        catch_unwind(AssertUnwindSafe(|| {
971            (self.behavior.signal_handler)(state, signal)
972        }))
973        .map_err(|panic| handler_panic("signal handler", panic))
974    }
975}
976
977fn now_ms() -> i64 {
978    let millis = SystemTime::now()
979        .duration_since(UNIX_EPOCH)
980        .unwrap_or(Duration::ZERO)
981        .as_millis();
982    i64::try_from(millis).unwrap_or(i64::MAX)
983}
984
985fn next_random_fraction() -> f64 {
986    let mut current = BACKOFF_RANDOM_SEED.load(Ordering::Relaxed);
987    loop {
988        let mut next = current;
989        next ^= next << 13;
990        next ^= next >> 7;
991        next ^= next << 17;
992        match BACKOFF_RANDOM_SEED.compare_exchange_weak(
993            current,
994            next,
995            Ordering::Relaxed,
996            Ordering::Relaxed,
997        ) {
998            Ok(_) => return ((next >> 11) as f64) / ((1_u64 << 53) as f64),
999            Err(observed) => current = observed,
1000        }
1001    }
1002}
1003
1004fn handler_panic(
1005    handler: &str,
1006    panic: Box<dyn std::any::Any + Send + 'static>,
1007) -> PersistenceError {
1008    let message = if let Some(message) = panic.downcast_ref::<&str>() {
1009        (*message).to_owned()
1010    } else if let Some(message) = panic.downcast_ref::<String>() {
1011        message.clone()
1012    } else {
1013        "non-string panic".to_owned()
1014    };
1015    PersistenceError::Failure(format!("{handler} panicked: {message}"))
1016}
1017
1018#[cfg(test)]
1019mod tests {
1020    use super::next_writer_uuid;
1021
1022    #[test]
1023    fn generated_writer_ids_include_process_start_entropy_and_a_sequence() {
1024        let first = next_writer_uuid();
1025        let second = next_writer_uuid();
1026        let parse = |value: &str| {
1027            value
1028                .strip_prefix("datum-persistence-")
1029                .expect("writer id prefix")
1030                .split('-')
1031                .map(str::to_owned)
1032                .collect::<Vec<_>>()
1033        };
1034        let first_parts = parse(&first);
1035        let second_parts = parse(&second);
1036        assert_eq!(first_parts.len(), 3);
1037        assert_eq!(second_parts.len(), 3);
1038        assert_eq!(first_parts[0], second_parts[0]);
1039        assert_eq!(first_parts[1], second_parts[1]);
1040        assert_ne!(first_parts[2], second_parts[2]);
1041        assert!(first_parts[1].parse::<u128>().unwrap() > 0);
1042    }
1043}