datum-persistence 0.11.3

Event sourcing, durable state, and CQRS persistence for Datum
Documentation
//! Persistence Query capability traits and Datum stream adapters.
//!
//! Query backends expose byte-oriented asynchronous streams. [`into_source`]
//! lifts those streams into Datum's stream DSL, and [`decode_envelopes`]
//! performs explicit serializer-catalog and event-adapter decoding at the
//! read-side boundary.

mod slice;

use std::{pin::Pin, sync::Arc};

use datum::{Flow, Source, StreamError};
use futures_core::Stream;
use futures_util::StreamExt;

use crate::{
    EventAdapter, PersistenceId, PersistenceResult, Revision, SequenceNr, SerializedPayload,
    SerializerCatalog, Tag,
};

pub use slice::{SLICE_COUNT, SliceRange, slice_for, slice_ranges};

/// Owned asynchronous stream returned by a persistence-query capability.
///
/// Read failures are emitted as items so a backend can fail after a live query
/// has already emitted elements. [`into_source`] translates such a failure to
/// a failed Datum stream.
pub type QueryStream<T> = Pin<Box<dyn Stream<Item = PersistenceResult<T>> + Send + 'static>>;

/// Backend-defined resume position for ordered query results.
///
/// Offsets are meaningful only to the backend instance that produced them;
/// there is no cross-backend or cross-database global-ordering promise.
/// `Sequence` is the in-memory/RocksDB per-database append counter, and those
/// backends resume strictly after the supplied value. Cassandra per-id
/// envelopes use their journal sequence number; that capability has no
/// offset-resume parameter. `Timestamp` is the
/// PostgreSQL database timestamp plus journal sequence number. PostgreSQL
/// resumes inclusively at an equal composite position: rows sharing the exact
/// timestamp and sequence number may therefore be re-emitted after a restart,
/// but no row at that position is skipped. Consumers must persist and resume
/// offsets with the same backend and database that produced them.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Offset {
    /// Start without a previously observed resume position.
    #[default]
    NoOffset,
    /// Backend-local monotonically increasing sequence position.
    Sequence(u64),
    /// PostgreSQL database timestamp and sequence-number resume position.
    ///
    /// `epoch_micros` retains PostgreSQL's microsecond timestamp precision.
    /// The pair is ordered lexicographically. Since sequence numbers are only
    /// unique within a persistence id, distinct rows can share a pair; resume
    /// is inclusive for that pair and duplicate re-emission is intentional.
    Timestamp {
        /// Microseconds since the Unix epoch.
        epoch_micros: i64,
        /// Journal sequence number within the persistence id.
        seq_nr: u64,
    },
}

/// Whether a query completes at its materialization boundary or stays live.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum QueryMode {
    /// Emit the boundary snapshot, then continue with subsequent changes.
    Live,
    /// Emit only the boundary snapshot and then complete.
    Current,
}

/// One serialized journal event emitted by a persistence query.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct EventEnvelope {
    /// Backend-local resume position for this event.
    pub offset: Offset,
    /// Persistent entity identity.
    pub persistence_id: PersistenceId,
    /// Monotonic sequence number within the persistence id.
    pub sequence_nr: SequenceNr,
    /// Event timestamp as milliseconds since the Unix epoch.
    pub timestamp_ms: i64,
    /// Durable event-adapter manifest, separate from the serializer manifest.
    pub adapter_manifest: String,
    /// Serialized journal payload frame.
    pub payload: SerializedPayload,
    /// Optional serialized event metadata frame.
    pub metadata: Option<SerializedPayload>,
    /// Query tags attached to the stored event.
    pub tags: Vec<Tag>,
}

/// Latest durable value for one persistence id.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UpdatedDurableState {
    /// Backend-local resume position for this change.
    pub offset: Offset,
    /// Durable-state identity.
    pub persistence_id: PersistenceId,
    /// CAS revision of the value.
    pub revision: Revision,
    /// Serialized latest state.
    pub state: SerializedPayload,
    /// Change timestamp as milliseconds since the Unix epoch.
    pub timestamp_ms: i64,
}

/// Latest durable-state tombstone for one persistence id.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DeletedDurableState {
    /// Backend-local resume position for this change.
    pub offset: Offset,
    /// Durable-state identity.
    pub persistence_id: PersistenceId,
    /// CAS revision of the tombstone.
    pub revision: Revision,
    /// Change timestamp as milliseconds since the Unix epoch.
    pub timestamp_ms: i64,
}

/// Coalesced latest durable-state change for one persistence id.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DurableStateChange {
    /// The latest revision contains a serialized state value.
    Updated(UpdatedDurableState),
    /// The latest revision is a deletion tombstone.
    Deleted(DeletedDurableState),
}

impl DurableStateChange {
    /// Returns the backend-local resume position.
    #[must_use]
    pub const fn offset(&self) -> Offset {
        match self {
            Self::Updated(change) => change.offset,
            Self::Deleted(change) => change.offset,
        }
    }

    /// Returns the durable-state identity.
    #[must_use]
    pub fn persistence_id(&self) -> &PersistenceId {
        match self {
            Self::Updated(change) => &change.persistence_id,
            Self::Deleted(change) => &change.persistence_id,
        }
    }

    /// Returns the latest observed revision.
    #[must_use]
    pub const fn revision(&self) -> Revision {
        match self {
            Self::Updated(change) => change.revision,
            Self::Deleted(change) => change.revision,
        }
    }
}

/// Capability for sequence-ordered event queries over one persistence id.
#[async_trait::async_trait]
pub trait EventsByPersistenceIdQuery: Send + Sync {
    /// Queries the inclusive `[from_sequence_nr, to_sequence_nr]` range.
    ///
    /// Results are ordered by sequence number. `Current` completes at the
    /// snapshot boundary; `Live` emits that same snapshot and then later
    /// appends, completing once a finite `to_sequence_nr` is reached.
    async fn events_by_persistence_id(
        &self,
        persistence_id: &PersistenceId,
        from_sequence_nr: SequenceNr,
        to_sequence_nr: SequenceNr,
        mode: QueryMode,
    ) -> PersistenceResult<QueryStream<EventEnvelope>>;
}

/// Capability for globally ordered event queries by tag.
#[async_trait::async_trait]
pub trait EventsByTagQuery: Send + Sync {
    /// Queries events carrying `tag`, resuming gaplessly from `offset`.
    async fn events_by_tag(
        &self,
        tag: &Tag,
        offset: Offset,
        mode: QueryMode,
    ) -> PersistenceResult<QueryStream<EventEnvelope>>;
}

/// Capability for globally ordered event queries over persistence-id slices.
///
/// Slice support is PostgreSQL-first. The fixed slice model is exposed through
/// [`slice_for`] and [`slice_ranges`]; Cassandra and embedded backends do not
/// implement this capability yet.
#[async_trait::async_trait]
pub trait EventsBySlicesQuery: Send + Sync {
    /// Queries one inclusive slice range for `entity_type`.
    ///
    /// Results follow the backend's global tag-query ordering contract.
    /// PostgreSQL resumes inclusively at the supplied timestamp offset, so
    /// exact-position duplicates may be re-emitted and consumers must apply a
    /// provider-complete deduplication boundary.
    ///
    /// For `Live`, PostgreSQL holds recent rows behind its configured
    /// visibility lag to close the ordinary late-commit race. A transaction
    /// open longer than that lag can still commit behind the live cursor and
    /// be skipped permanently; size the lag for the deployment's maximum
    /// transaction-duration envelope.
    async fn events_by_slices(
        &self,
        entity_type: &str,
        min_slice: u16,
        max_slice: u16,
        offset: Offset,
        mode: QueryMode,
    ) -> PersistenceResult<QueryStream<EventEnvelope>>;
}

/// Capability for discovering persistence ids known to a journal.
#[async_trait::async_trait]
pub trait PersistenceIdsQuery: Send + Sync {
    /// Queries each known persistence id once in backend discovery order.
    async fn persistence_ids(
        &self,
        mode: QueryMode,
    ) -> PersistenceResult<QueryStream<PersistenceId>>;
}

/// Capability for querying coalesced latest durable-state changes by tag.
#[async_trait::async_trait]
pub trait DurableStateChangesQuery: Send + Sync {
    /// Queries latest changes carrying `tag`, resuming gaplessly from `offset`.
    ///
    /// Multiple rapid revisions for one persistence id may be coalesced, but
    /// the latest revision for that id must eventually be emitted.
    async fn changes(
        &self,
        tag: &Tag,
        offset: Offset,
        mode: QueryMode,
    ) -> PersistenceResult<QueryStream<DurableStateChange>>;
}

/// Lifts an owned persistence-query stream into the Datum stream DSL.
///
/// The returned source is backed by [`Source::unfold_async`]. A query error
/// fails the Datum stream with [`StreamError::Failed`]. Query streams are
/// single-consumer resources, so materializations of clones of this source
/// share that one underlying query cursor.
#[must_use]
pub fn into_source<T>(stream: QueryStream<T>) -> Source<T>
where
    T: Send + 'static,
{
    let stream = Arc::new(tokio::sync::Mutex::new(stream));
    Source::unfold_async(stream, |stream| async move {
        let next = {
            let mut stream = stream.lock().await;
            stream.next().await
        };
        match next {
            Some(Ok(item)) => Ok(Some((stream, item))),
            Some(Err(error)) => Err(StreamError::Failed(error.to_string())),
            None => Ok(None),
        }
    })
}

/// Builds a Datum flow that decodes byte-frame envelopes into domain events.
///
/// Serializer dispatch happens before the event adapter. The adapter's
/// recovery-side `0..n` cardinality is preserved, allowing old journal events
/// to be filtered or expanded during read-side schema evolution.
#[must_use]
pub fn decode_envelopes<D, J, A>(
    catalog: SerializerCatalog<J>,
    adapter: A,
) -> Flow<EventEnvelope, D>
where
    D: Send + 'static,
    J: Send + 'static,
    A: EventAdapter<D, J>,
{
    let catalog = Arc::new(catalog);
    let adapter = Arc::new(adapter);
    Flow::identity().try_map_concat(move |envelope: EventEnvelope| {
        let journal_event = catalog
            .decode(&envelope.payload)
            .map_err(|error| StreamError::Failed(error.to_string()))?;
        adapter
            .from_journal_events(&envelope.adapter_manifest, journal_event)
            .map_err(|error| StreamError::Failed(error.to_string()))
    })
}