1use std::{
2 collections::VecDeque,
3 panic::{AssertUnwindSafe, catch_unwind},
4 sync::{
5 Arc, LazyLock,
6 atomic::{AtomicU64, Ordering},
7 },
8 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
9};
10
11use datum::{
12 RestartSettings, Topic, TopicOverflow,
13 actor::{Actor, ActorProcessingErr, ActorRef, Message},
14};
15use futures_util::StreamExt;
16use tokio::sync::oneshot;
17
18use super::{
19 EventSourcedBehavior, EventSourcedHandle, EventSourcedRuntime, EventSourcedSignal,
20 EventSourcedState, PublishedEvent, Recovery, RetentionCriteria, SnapshotSelection,
21 effect::{ChainedEffect, Effect, EventWithMetadata, MainEffect},
22};
23use crate::{
24 AppendOutcome, AtomicWrite, EventAdapter, PersistenceError, PersistenceResult, ReplayRange,
25 SequenceNr, SerializedPayload, SerializerCatalog, SnapshotCriteria, SnapshotMetadata,
26 StateAdapter, StoredEvent,
27};
28
29static NEXT_WRITER_ID: AtomicU64 = AtomicU64::new(1);
30static WRITER_STARTUP_NANOS: LazyLock<u128> = LazyLock::new(|| {
31 SystemTime::now()
32 .duration_since(UNIX_EPOCH)
33 .map(|duration| duration.as_nanos())
34 .unwrap_or_default()
35});
36static BACKOFF_RANDOM_SEED: AtomicU64 = AtomicU64::new(0x517c_c1b7_2722_0a95);
37
38pub(crate) fn next_writer_uuid() -> String {
39 format!(
40 "datum-persistence-{}-{}-{}",
41 std::process::id(),
42 *WRITER_STARTUP_NANOS,
43 NEXT_WRITER_ID.fetch_add(1, Ordering::Relaxed)
44 )
45}
46
47pub(crate) struct EventCodec<E, J> {
48 adapter: Arc<dyn EventAdapter<E, J>>,
49 catalog: Arc<SerializerCatalog<J>>,
50}
51
52impl<E, J> EventCodec<E, J> {
53 pub(crate) fn new<A>(adapter: A, catalog: SerializerCatalog<J>) -> Self
54 where
55 A: EventAdapter<E, J>,
56 {
57 Self {
58 adapter: Arc::new(adapter),
59 catalog: Arc::new(catalog),
60 }
61 }
62}
63
64pub(crate) trait StateCodec<S>: Send + Sync + 'static {
65 fn encode(&self, state: &S) -> PersistenceResult<(String, SerializedPayload)>;
66 fn decode(&self, adapter_manifest: &str, state: &SerializedPayload) -> PersistenceResult<S>;
67}
68
69pub(crate) struct AdaptedStateCodec<S, J> {
70 adapter: Arc<dyn StateAdapter<S, J>>,
71 catalog: SerializerCatalog<J>,
72}
73
74impl<S, J> AdaptedStateCodec<S, J> {
75 pub(crate) fn new<A>(adapter: A, catalog: SerializerCatalog<J>) -> Self
76 where
77 A: StateAdapter<S, J>,
78 {
79 Self {
80 adapter: Arc::new(adapter),
81 catalog,
82 }
83 }
84}
85
86impl<S, J> StateCodec<S> for AdaptedStateCodec<S, J>
87where
88 S: 'static,
89 J: 'static,
90{
91 fn encode(&self, state: &S) -> PersistenceResult<(String, SerializedPayload)> {
92 let (adapter_manifest, journal_state) = self.adapter.to_journal(state)?;
93 let frame = self.catalog.encode(&journal_state)?;
94 Ok((adapter_manifest, frame))
95 }
96
97 fn decode(&self, adapter_manifest: &str, state: &SerializedPayload) -> PersistenceResult<S> {
98 let journal_state = self.catalog.decode(state)?;
99 self.adapter.from_journal(adapter_manifest, journal_state)
100 }
101}
102
103pub struct EventSourcedMessage<C, S>(RuntimeMessage<C, S>);
108
109impl<C, S> EventSourcedMessage<C, S> {
110 #[must_use]
112 pub fn command(command: C) -> Self {
113 Self(RuntimeMessage::Command(command))
114 }
115
116 pub(crate) fn get_state(reply: oneshot::Sender<EventSourcedState<S>>) -> Self {
117 Self(RuntimeMessage::GetState(reply))
118 }
119}
120
121enum RuntimeMessage<C, S> {
122 Recover,
123 Command(C),
124 GetState(oneshot::Sender<EventSourcedState<S>>),
125 BackoffElapsed,
126 RetentionSignal(EventSourcedSignal),
127 RetentionCleanupFinished,
128}
129
130pub(crate) struct EventSourcedActor<C, E: Send + Sync + 'static, S, J> {
131 behavior: EventSourcedBehavior<C, E, S, J>,
132 runtime: EventSourcedRuntime,
133 topic: Option<Topic<PublishedEvent<E>>>,
134}
135
136pub(crate) struct RuntimeState<C, S> {
137 state: S,
138 sequence_nr: SequenceNr,
139 manual_stash: VecDeque<C>,
140 backoff_stash: VecDeque<C>,
141 unstash_remaining: usize,
142 retention_in_flight: bool,
143 phase: RuntimePhase,
144 backoff: PersistenceBackoff,
145}
146
147#[derive(Clone, Copy, PartialEq, Eq)]
148enum RuntimePhase {
149 Running,
150 BackingOff,
151}
152
153struct PersistenceBackoff {
154 failures: u32,
155 restarts_in_window: usize,
156 window_started: Instant,
157}
158
159struct RetentionTaskGuard<C: Message, S: Send + 'static> {
160 actor: ActorRef<EventSourcedMessage<C, S>>,
161}
162
163impl<C: Message, S: Send + 'static> Drop for RetentionTaskGuard<C, S> {
164 fn drop(&mut self) {
165 let _ = self.actor.cast(EventSourcedMessage(
166 RuntimeMessage::RetentionCleanupFinished,
167 ));
168 }
169}
170
171impl PersistenceBackoff {
172 fn new() -> Self {
173 Self {
174 failures: 0,
175 restarts_in_window: 0,
176 window_started: Instant::now(),
177 }
178 }
179
180 fn reset(&mut self) {
181 self.failures = 0;
182 self.restarts_in_window = 0;
183 self.window_started = Instant::now();
184 }
185
186 fn next_delay(&mut self, settings: &RestartSettings) -> Option<Duration> {
187 let now = Instant::now();
188 if now.duration_since(self.window_started) > settings.max_restarts_within() {
189 self.reset();
190 }
191 if self.restarts_in_window >= settings.max_restarts() {
192 return None;
193 }
194
195 let factor = 1_u32.checked_shl(self.failures.min(31)).unwrap_or(u32::MAX);
196 let base = settings
197 .min_backoff()
198 .checked_mul(factor)
199 .unwrap_or(settings.max_backoff())
200 .min(settings.max_backoff());
201 self.failures = self.failures.saturating_add(1);
202 self.restarts_in_window += 1;
203 if settings.random_factor() == 0.0 || base.is_zero() {
204 Some(base)
205 } else {
206 let jitter = base.mul_f64(next_random_fraction() * settings.random_factor());
207 Some(base.saturating_add(jitter))
208 }
209 }
210}
211
212pub(crate) async fn spawn<C, E, S, J>(
213 behavior: EventSourcedBehavior<C, E, S, J>,
214 mut runtime: EventSourcedRuntime,
215) -> PersistenceResult<EventSourcedHandle<C, E, S>>
216where
217 C: Message,
218 E: Send + Sync + 'static,
219 S: Clone + Send + Sync + 'static,
220 J: Send + 'static,
221{
222 if behavior.event_codec.is_none() {
223 return Err(PersistenceError::Serialization(
224 "event adapter and serializer catalog must be configured before spawn".into(),
225 ));
226 }
227 if let RetentionCriteria::SnapshotEvery { every, keep, .. } = behavior.retention {
228 if every == 0 {
229 return Err(PersistenceError::Rejected(
230 "snapshot frequency must be greater than zero".into(),
231 ));
232 }
233 if keep == 0 {
234 return Err(PersistenceError::Rejected(
235 "snapshot retention count must be greater than zero".into(),
236 ));
237 }
238 }
239 if runtime.writer_uuid.is_none() {
240 runtime.writer_uuid = Some(next_writer_uuid());
241 }
242
243 let topic = if behavior.publish_events {
244 Some(
245 Topic::new(runtime.topic_capacity, TopicOverflow::Dropping).map_err(|error| {
246 PersistenceError::Failure(format!("published-event topic failed to start: {error}"))
247 })?,
248 )
249 } else {
250 None
251 };
252
253 let initial_state = RuntimeState {
254 state: behavior.empty_state.clone(),
255 sequence_nr: SequenceNr::new(0),
256 manual_stash: VecDeque::new(),
257 backoff_stash: VecDeque::new(),
258 unstash_remaining: 0,
259 retention_in_flight: false,
260 phase: RuntimePhase::Running,
261 backoff: PersistenceBackoff::new(),
262 };
263 let actor = EventSourcedActor {
264 behavior,
265 runtime,
266 topic: topic.clone(),
267 };
268 let (control, _control_join) =
269 Actor::spawn(None, actor, initial_state)
270 .await
271 .map_err(|error| {
272 PersistenceError::Failure(format!("event-sourced actor failed to spawn: {error}"))
273 })?;
274 Ok(EventSourcedHandle {
275 control,
276 published_events: topic,
277 })
278}
279
280impl<C, E, S, J> Actor for EventSourcedActor<C, E, S, J>
281where
282 C: Message,
283 E: Send + Sync + 'static,
284 S: Clone + Send + Sync + 'static,
285 J: Send + 'static,
286{
287 type Msg = EventSourcedMessage<C, S>;
288 type State = RuntimeState<C, S>;
289 type Arguments = RuntimeState<C, S>;
290
291 async fn pre_start(
292 &self,
293 myself: ActorRef<Self::Msg>,
294 state: Self::Arguments,
295 ) -> Result<Self::State, ActorProcessingErr> {
296 myself
297 .cast(EventSourcedMessage(RuntimeMessage::Recover))
298 .map_err(|error| {
299 Box::new(PersistenceError::Failure(format!(
300 "failed to enqueue initial recovery: {error}"
301 ))) as ActorProcessingErr
302 })?;
303 Ok(state)
304 }
305
306 async fn handle(
307 &self,
308 myself: ActorRef<Self::Msg>,
309 message: Self::Msg,
310 state: &mut Self::State,
311 ) -> Result<(), ActorProcessingErr> {
312 if let Err(error) = self.handle_runtime_message(&myself, message, state).await {
313 myself.stop(Some(error.to_string()));
314 }
315 Ok(())
316 }
317
318 async fn post_stop(
319 &self,
320 _myself: ActorRef<Self::Msg>,
321 _state: &mut Self::State,
322 ) -> Result<(), ActorProcessingErr> {
323 self.close_topic();
324 Ok(())
325 }
326}
327
328impl<C, E, S, J> EventSourcedActor<C, E, S, J>
329where
330 C: Message,
331 E: Send + Sync + 'static,
332 S: Clone + Send + Sync + 'static,
333 J: Send + 'static,
334{
335 async fn handle_runtime_message(
336 &self,
337 myself: &ActorRef<EventSourcedMessage<C, S>>,
338 message: EventSourcedMessage<C, S>,
339 state: &mut RuntimeState<C, S>,
340 ) -> Result<(), ActorProcessingErr> {
341 match message.0 {
342 RuntimeMessage::Recover => self.recover(state).await.map_err(boxed),
343 RuntimeMessage::Command(command) => {
344 if state.phase == RuntimePhase::BackingOff {
345 state.backoff_stash.push_back(command);
346 return Ok(());
347 }
348 let stop = self.process_commands(command, myself, state).await?;
349 if stop {
350 myself.stop(None);
351 }
352 Ok(())
353 }
354 RuntimeMessage::GetState(reply) => {
355 let _ = reply.send(EventSourcedState {
356 state: state.state.clone(),
357 sequence_nr: state.sequence_nr,
358 });
359 Ok(())
360 }
361 RuntimeMessage::BackoffElapsed => {
362 state.phase = RuntimePhase::Running;
363 self.recover(state).await.map_err(boxed)?;
364 while state.phase == RuntimePhase::Running {
365 let Some(command) = state.backoff_stash.pop_front() else {
366 break;
367 };
368 if self.process_commands(command, myself, state).await? {
369 myself.stop(None);
370 break;
371 }
372 }
373 Ok(())
374 }
375 RuntimeMessage::RetentionSignal(signal) => {
376 self.signal(&state.state, &signal)?;
377 Ok(())
378 }
379 RuntimeMessage::RetentionCleanupFinished => {
380 state.retention_in_flight = false;
381 Ok(())
382 }
383 }
384 }
385}
386
387fn boxed(error: PersistenceError) -> ActorProcessingErr {
388 Box::new(error)
389}
390
391enum PersistError {
392 Rejected {
393 sequence_nr: SequenceNr,
394 reason: String,
395 },
396 Failed {
397 sequence_nr: SequenceNr,
398 error: PersistenceError,
399 },
400 Behavior(PersistenceError),
401}
402
403struct PendingPublication<E> {
404 event: E,
405 sequence_nr: SequenceNr,
406 timestamp_ms: i64,
407 tags: Vec<crate::Tag>,
408}
409
410enum SnapshotReason {
411 Predicate,
412 Retention,
413}
414
415impl<C, E, S, J> EventSourcedActor<C, E, S, J>
416where
417 C: Message,
418 E: Send + Sync + 'static,
419 S: Clone + Send + Sync + 'static,
420 J: Send + 'static,
421{
422 async fn recover(&self, state: &mut RuntimeState<C, S>) -> PersistenceResult<()> {
423 state.state = self.behavior.empty_state.clone();
424 state.sequence_nr = SequenceNr::new(0);
425
426 let result = self.recover_inner(state).await;
427 match result {
428 Ok(()) => {
429 self.signal(&state.state, &EventSourcedSignal::RecoveryCompleted)?;
430 Ok(())
431 }
432 Err(error) => {
433 let signal_error = error.clone();
434 let _ = self.signal(
435 &state.state,
436 &EventSourcedSignal::RecoveryFailed(signal_error),
437 );
438 Err(error)
439 }
440 }
441 }
442
443 async fn recover_inner(&self, state: &mut RuntimeState<C, S>) -> PersistenceResult<()> {
444 let Recovery::Enabled(selection) = self.behavior.recovery else {
445 state.sequence_nr = self
446 .runtime
447 .journal
448 .highest_sequence_nr(&self.behavior.persistence_id, SequenceNr::new(0))
449 .await?;
450 return Ok(());
451 };
452
453 if selection != SnapshotSelection::None
454 && let Some(snapshot_store) = &self.runtime.snapshot_store
455 {
456 let criteria = selection.criteria();
457 if let Some(snapshot) = snapshot_store
458 .load(&self.behavior.persistence_id, &criteria)
459 .await?
460 {
461 let codec = self.behavior.state_codec.as_ref().ok_or_else(|| {
462 PersistenceError::Serialization(
463 "a state adapter and serializer catalog are required to recover snapshots"
464 .into(),
465 )
466 })?;
467 state.state = codec.decode(&snapshot.adapter_manifest, &snapshot.state)?;
468 state.sequence_nr = snapshot.metadata.sequence_nr;
469 self.signal(
470 &state.state,
471 &EventSourcedSignal::SnapshotRecovered(snapshot.metadata),
472 )?;
473 }
474 }
475
476 let highest = self
477 .runtime
478 .journal
479 .highest_sequence_nr(&self.behavior.persistence_id, state.sequence_nr)
480 .await?;
481 if state.sequence_nr != SequenceNr::new(u64::MAX) {
482 let from = SequenceNr::new(state.sequence_nr.as_u64() + 1);
483 if from <= highest {
484 let mut replay = self
485 .runtime
486 .journal
487 .replay(ReplayRange {
488 persistence_id: self.behavior.persistence_id.clone(),
489 from,
490 to: highest,
491 max: u64::MAX,
492 })
493 .await?;
494 while let Some(stored) = replay.next().await {
495 let stored = stored?;
496 let codec = self
497 .behavior
498 .event_codec
499 .as_ref()
500 .expect("spawn validates event codec");
501 let journal_event = codec.catalog.decode(&stored.payload)?;
502 let events = codec
503 .adapter
504 .from_journal_events(&stored.adapter_manifest, journal_event)?;
505 for event in events {
506 state.state = self.apply_event(&state.state, &event)?;
507 }
508 state.sequence_nr = stored.sequence_nr;
509 }
510 }
511 }
512 state.sequence_nr = state.sequence_nr.max(highest);
513 Ok(())
514 }
515
516 async fn process_commands(
517 &self,
518 mut command: C,
519 myself: &ActorRef<EventSourcedMessage<C, S>>,
520 state: &mut RuntimeState<C, S>,
521 ) -> Result<bool, ActorProcessingErr> {
522 loop {
523 let effect = catch_unwind(AssertUnwindSafe(|| {
524 (self.behavior.command_handler)(&state.state, command)
525 }))
526 .map_err(|panic| boxed(handler_panic("command handler", panic)))?;
527 let effect = Self::resolve_async(effect).await;
528
529 let mut stop = false;
530 match effect.main {
531 MainEffect::Persist(events) if events.is_empty() => {
532 stop = self.apply_chained(effect.chained, state)?;
533 }
534 MainEffect::Persist(events) => match self.persist(events, myself, state).await {
535 Ok(()) => {
536 state.backoff.reset();
537 stop = self.apply_chained(effect.chained, state)?;
538 }
539 Err(PersistError::Rejected {
540 sequence_nr,
541 reason,
542 }) => {
543 self.signal(
544 &state.state,
545 &EventSourcedSignal::PersistRejected {
546 sequence_nr,
547 reason: reason.clone(),
548 },
549 )?;
550 return Err(boxed(PersistenceError::Rejected(reason)));
551 }
552 Err(PersistError::Failed { sequence_nr, error }) => {
553 self.signal(
554 &state.state,
555 &EventSourcedSignal::PersistFailed {
556 sequence_nr,
557 error: error.clone(),
558 },
559 )?;
560 let Some(settings) = self.behavior.persist_failure_backoff.as_ref() else {
561 return Err(boxed(error));
562 };
563 let Some(delay) = state.backoff.next_delay(settings) else {
564 return Err(boxed(error));
565 };
566 state.phase = RuntimePhase::BackingOff;
567 std::mem::drop(myself.send_after(delay, || {
568 EventSourcedMessage(RuntimeMessage::BackoffElapsed)
569 }));
570 }
571 Err(PersistError::Behavior(error)) => return Err(boxed(error)),
572 },
573 MainEffect::None => {
574 stop = self.apply_chained(effect.chained, state)?;
575 }
576 MainEffect::Unhandled => {
577 tracing::debug!(
578 persistence_id = %self.behavior.persistence_id,
579 "persistent behavior left a command unhandled"
580 );
581 stop = self.apply_chained(effect.chained, state)?;
582 }
583 MainEffect::Stash(command) => {
584 if state.manual_stash.len() < self.behavior.stash_capacity {
585 state.manual_stash.push_back(command);
586 stop = self.apply_chained(effect.chained, state)?;
587 } else {
588 tracing::warn!(
589 persistence_id = %self.behavior.persistence_id,
590 stash_capacity = self.behavior.stash_capacity,
591 "manual persistence stash is full; dropping newest command"
592 );
593 }
594 }
595 MainEffect::Async(_) => unreachable!("async effects are resolved before dispatch"),
596 }
597
598 if stop {
599 return Ok(true);
600 }
601 if state.phase == RuntimePhase::BackingOff {
602 return Ok(false);
603 }
604
605 if state.unstash_remaining > 0 {
606 state.unstash_remaining -= 1;
607 if let Some(stashed) = state.manual_stash.pop_front() {
608 command = stashed;
609 continue;
610 } else {
611 state.unstash_remaining = 0;
612 }
613 }
614 return Ok(false);
615 }
616 }
617
618 async fn resolve_async(mut effect: Effect<C, E, S>) -> Effect<C, E, S> {
619 let mut after_async = Vec::new();
620 loop {
621 match effect.main {
622 MainEffect::Async(future) => {
623 after_async.splice(0..0, effect.chained);
624 effect = future.await;
625 }
626 _ => {
627 effect.chained.extend(after_async);
628 return effect;
629 }
630 }
631 }
632 }
633
634 fn apply_chained(
635 &self,
636 chained: Vec<ChainedEffect<S>>,
637 state: &mut RuntimeState<C, S>,
638 ) -> PersistenceResult<bool> {
639 let mut stop = false;
640 for effect in chained {
641 match effect {
642 ChainedEffect::Run(callback) => {
643 catch_unwind(AssertUnwindSafe(|| callback(&state.state)))
644 .map_err(|panic| handler_panic("side effect", panic))?;
645 }
646 ChainedEffect::Stop => stop = true,
647 ChainedEffect::UnstashAll => {
648 state.unstash_remaining = state.manual_stash.len();
649 }
650 }
651 }
652 Ok(stop)
653 }
654
655 async fn persist(
656 &self,
657 events: Vec<EventWithMetadata<E>>,
658 myself: &ActorRef<EventSourcedMessage<C, S>>,
659 state: &mut RuntimeState<C, S>,
660 ) -> Result<(), PersistError> {
661 let first_sequence_nr = state
662 .sequence_nr
663 .as_u64()
664 .checked_add(1)
665 .map(SequenceNr::new)
666 .ok_or_else(|| PersistError::Rejected {
667 sequence_nr: state.sequence_nr,
668 reason: "event sequence number is exhausted".into(),
669 })?;
670 let codec = self
671 .behavior
672 .event_codec
673 .as_ref()
674 .expect("spawn validates event codec");
675 let mut publications = self
676 .topic
677 .as_ref()
678 .map(|_| Vec::with_capacity(events.len()));
679 let mut staged_state = None;
680 let mut stored_events = Vec::with_capacity(events.len());
681 let mut snapshot_reason = None;
682 let mut last_sequence_nr = first_sequence_nr;
683
684 for (index, event) in events.into_iter().enumerate() {
685 let offset = u64::try_from(index).map_err(|error| PersistError::Rejected {
686 sequence_nr: first_sequence_nr,
687 reason: format!("event batch is too large: {error}"),
688 })?;
689 let sequence_nr = first_sequence_nr
690 .as_u64()
691 .checked_add(offset)
692 .map(SequenceNr::new)
693 .ok_or_else(|| PersistError::Rejected {
694 sequence_nr: first_sequence_nr,
695 reason: "event sequence number is exhausted".into(),
696 })?;
697 let current_state = staged_state.as_ref().unwrap_or(&state.state);
698 let next_state = self
699 .apply_event(current_state, &event.event)
700 .map_err(PersistError::Behavior)?;
701
702 let retention_match = matches!(
703 self.behavior.retention,
704 RetentionCriteria::SnapshotEvery { every, .. }
705 if sequence_nr.as_u64() % every == 0
706 );
707 if retention_match {
708 snapshot_reason = Some(SnapshotReason::Retention);
709 } else if snapshot_reason.is_none()
710 && (self.behavior.snapshot_when)(&next_state, &event.event, sequence_nr)
711 {
712 snapshot_reason = Some(SnapshotReason::Predicate);
713 }
714
715 let (adapter_manifest, journal_event) = codec
716 .adapter
717 .to_journal(&event.event)
718 .map_err(|error| PersistError::Rejected {
719 sequence_nr,
720 reason: error.to_string(),
721 })?;
722 let payload =
723 codec
724 .catalog
725 .encode(&journal_event)
726 .map_err(|error| PersistError::Rejected {
727 sequence_nr,
728 reason: error.to_string(),
729 })?;
730 let timestamp_ms = now_ms();
731 let tags = (self.behavior.tagger)(&next_state, &event.event);
732 if let Some(publications) = &mut publications {
733 publications.push(PendingPublication {
734 event: event.event,
735 sequence_nr,
736 timestamp_ms,
737 tags: tags.clone(),
738 });
739 }
740 stored_events.push(StoredEvent {
741 persistence_id: self.behavior.persistence_id.clone(),
746 sequence_nr,
747 adapter_manifest,
748 writer_uuid: self
757 .runtime
758 .writer_uuid
759 .clone()
760 .expect("spawn assigns a writer uuid"),
761 timestamp_ms,
762 metadata: event.metadata,
763 payload,
764 tags,
765 });
766 staged_state = Some(next_state);
767 last_sequence_nr = sequence_nr;
768 }
769
770 let write = AtomicWrite::new(stored_events).map_err(|error| PersistError::Rejected {
771 sequence_nr: first_sequence_nr,
772 reason: error.to_string(),
773 })?;
774 let outcome = self
775 .runtime
776 .journal
777 .append_one(write)
778 .await
779 .map_err(|error| PersistError::Failed {
780 sequence_nr: first_sequence_nr,
781 error,
782 })?;
783 if let AppendOutcome::Rejected { reason } = outcome {
784 return Err(PersistError::Rejected {
785 sequence_nr: first_sequence_nr,
786 reason,
787 });
788 }
789
790 state.state = staged_state.expect("non-empty event batch was validated");
791 state.sequence_nr = last_sequence_nr;
792
793 if let Some(topic) = self
794 .topic
795 .as_ref()
796 .filter(|topic| topic.subscriber_count() > 0)
797 {
798 for item in publications.unwrap_or_default() {
799 let _ = topic
800 .publish(PublishedEvent {
801 persistence_id: self.behavior.persistence_id.clone(),
802 sequence_nr: item.sequence_nr,
803 event: item.event,
804 timestamp_ms: item.timestamp_ms,
805 tags: item.tags,
806 })
807 .await;
808 }
809 }
810
811 if let Some(reason) = snapshot_reason {
812 let skip_retention =
813 matches!(reason, SnapshotReason::Retention) && state.retention_in_flight;
814 if skip_retention {
815 tracing::debug!(
816 persistence_id = %self.behavior.persistence_id,
817 sequence_nr = state.sequence_nr.as_u64(),
818 "retention snapshot boundary skipped while cleanup is in flight; the next boundary covers it"
819 );
820 } else {
821 self.save_snapshot(reason, myself, state)
822 .await
823 .map_err(PersistError::Behavior)?;
824 }
825 }
826 Ok(())
827 }
828
829 async fn save_snapshot(
830 &self,
831 reason: SnapshotReason,
832 myself: &ActorRef<EventSourcedMessage<C, S>>,
833 state: &mut RuntimeState<C, S>,
834 ) -> PersistenceResult<()> {
835 let Some(snapshot_store) = &self.runtime.snapshot_store else {
836 return Ok(());
837 };
838 let metadata = SnapshotMetadata {
839 persistence_id: self.behavior.persistence_id.clone(),
840 sequence_nr: state.sequence_nr,
841 timestamp_ms: now_ms(),
842 };
843 let save_result = match &self.behavior.state_codec {
844 Some(codec) => match codec.encode(&state.state) {
845 Ok((adapter_manifest, frame)) => {
846 snapshot_store
847 .save(metadata.clone(), adapter_manifest, frame)
848 .await
849 }
850 Err(error) => Err(error),
851 },
852 None => Err(PersistenceError::Serialization(
853 "a state adapter and serializer catalog are required to save snapshots".into(),
854 )),
855 };
856 match save_result {
857 Ok(()) => {
858 self.signal(
859 &state.state,
860 &EventSourcedSignal::SnapshotCompleted(metadata.clone()),
861 )?;
862 if matches!(reason, SnapshotReason::Retention) {
863 self.start_retention_cleanup(
864 metadata.sequence_nr,
865 Arc::clone(snapshot_store),
866 myself,
867 state,
868 );
869 }
870 }
871 Err(error) => {
872 self.signal(
873 &state.state,
874 &EventSourcedSignal::SnapshotFailed { metadata, error },
875 )?;
876 }
877 }
878 Ok(())
879 }
880
881 fn start_retention_cleanup(
882 &self,
883 snapshot_sequence_nr: SequenceNr,
884 snapshot_store: Arc<dyn crate::SnapshotStore>,
885 myself: &ActorRef<EventSourcedMessage<C, S>>,
886 state: &mut RuntimeState<C, S>,
887 ) {
888 let RetentionCriteria::SnapshotEvery {
889 every,
890 keep,
891 delete_events,
892 } = self.behavior.retention
893 else {
894 return;
895 };
896 state.retention_in_flight = true;
897 let retained_window = every.saturating_mul(keep);
898 let delete_to = SequenceNr::new(
899 snapshot_sequence_nr
900 .as_u64()
901 .saturating_sub(retained_window),
902 );
903 let persistence_id = self.behavior.persistence_id.clone();
904 let journal = Arc::clone(&self.runtime.journal);
905 let actor = myself.clone();
906 tokio::spawn(async move {
907 let _completion = RetentionTaskGuard {
908 actor: actor.clone(),
909 };
910 if delete_events {
911 match journal.delete_to(&persistence_id, delete_to).await {
912 Ok(()) => {
913 if actor
914 .cast(EventSourcedMessage(RuntimeMessage::RetentionSignal(
915 EventSourcedSignal::DeleteEventsCompleted(delete_to),
916 )))
917 .is_err()
918 {
919 return;
920 }
921 }
922 Err(error) => {
923 let _ = actor.cast(EventSourcedMessage(RuntimeMessage::RetentionSignal(
924 EventSourcedSignal::DeleteEventsFailed {
925 to: delete_to,
926 error,
927 },
928 )));
929 return;
930 }
931 }
932 }
933
934 let max_snapshot = if delete_events {
935 delete_to.as_u64().saturating_sub(1)
936 } else {
937 delete_to.as_u64()
938 };
939 let criteria = SnapshotCriteria {
940 min_sequence_nr: SequenceNr::new(0),
941 max_sequence_nr: SequenceNr::new(max_snapshot),
942 min_timestamp_ms: i64::MIN,
943 max_timestamp_ms: i64::MAX,
944 };
945 let signal = match snapshot_store
946 .delete_matching(&persistence_id, &criteria)
947 .await
948 {
949 Ok(()) => EventSourcedSignal::DeleteSnapshotsCompleted(criteria),
950 Err(error) => EventSourcedSignal::DeleteSnapshotsFailed { criteria, error },
951 };
952 let _ = actor.cast(EventSourcedMessage(RuntimeMessage::RetentionSignal(signal)));
953 });
954 }
955
956 fn close_topic(&self) {
957 if let Some(topic) = &self.topic {
958 let _ = topic.close();
959 }
960 }
961
962 fn apply_event(&self, state: &S, event: &E) -> PersistenceResult<S> {
963 catch_unwind(AssertUnwindSafe(|| {
964 (self.behavior.event_handler)(state, event)
965 }))
966 .map_err(|panic| handler_panic("event handler", panic))?
967 }
968
969 fn signal(&self, state: &S, signal: &EventSourcedSignal) -> PersistenceResult<()> {
970 catch_unwind(AssertUnwindSafe(|| {
971 (self.behavior.signal_handler)(state, signal)
972 }))
973 .map_err(|panic| handler_panic("signal handler", panic))
974 }
975}
976
977fn now_ms() -> i64 {
978 let millis = SystemTime::now()
979 .duration_since(UNIX_EPOCH)
980 .unwrap_or(Duration::ZERO)
981 .as_millis();
982 i64::try_from(millis).unwrap_or(i64::MAX)
983}
984
985fn next_random_fraction() -> f64 {
986 let mut current = BACKOFF_RANDOM_SEED.load(Ordering::Relaxed);
987 loop {
988 let mut next = current;
989 next ^= next << 13;
990 next ^= next >> 7;
991 next ^= next << 17;
992 match BACKOFF_RANDOM_SEED.compare_exchange_weak(
993 current,
994 next,
995 Ordering::Relaxed,
996 Ordering::Relaxed,
997 ) {
998 Ok(_) => return ((next >> 11) as f64) / ((1_u64 << 53) as f64),
999 Err(observed) => current = observed,
1000 }
1001 }
1002}
1003
1004fn handler_panic(
1005 handler: &str,
1006 panic: Box<dyn std::any::Any + Send + 'static>,
1007) -> PersistenceError {
1008 let message = if let Some(message) = panic.downcast_ref::<&str>() {
1009 (*message).to_owned()
1010 } else if let Some(message) = panic.downcast_ref::<String>() {
1011 message.clone()
1012 } else {
1013 "non-string panic".to_owned()
1014 };
1015 PersistenceError::Failure(format!("{handler} panicked: {message}"))
1016}
1017
1018#[cfg(test)]
1019mod tests {
1020 use super::next_writer_uuid;
1021
1022 #[test]
1023 fn generated_writer_ids_include_process_start_entropy_and_a_sequence() {
1024 let first = next_writer_uuid();
1025 let second = next_writer_uuid();
1026 let parse = |value: &str| {
1027 value
1028 .strip_prefix("datum-persistence-")
1029 .expect("writer id prefix")
1030 .split('-')
1031 .map(str::to_owned)
1032 .collect::<Vec<_>>()
1033 };
1034 let first_parts = parse(&first);
1035 let second_parts = parse(&second);
1036 assert_eq!(first_parts.len(), 3);
1037 assert_eq!(second_parts.len(), 3);
1038 assert_eq!(first_parts[0], second_parts[0]);
1039 assert_eq!(first_parts[1], second_parts[1]);
1040 assert_ne!(first_parts[2], second_parts[2]);
1041 assert!(first_parts[1].parse::<u128>().unwrap() > 0);
1042 }
1043}