Skip to main content

datum_persistence/query/
mod.rs

1//! Persistence Query capability traits and Datum stream adapters.
2//!
3//! Query backends expose byte-oriented asynchronous streams. [`into_source`]
4//! lifts those streams into Datum's stream DSL, and [`decode_envelopes`]
5//! performs explicit serializer-catalog and event-adapter decoding at the
6//! read-side boundary.
7
8use std::{pin::Pin, sync::Arc};
9
10use datum::{Flow, Source, StreamError};
11use futures_core::Stream;
12use futures_util::StreamExt;
13
14use crate::{
15    EventAdapter, PersistenceId, PersistenceResult, Revision, SequenceNr, SerializedPayload,
16    SerializerCatalog, Tag,
17};
18
19/// Owned asynchronous stream returned by a persistence-query capability.
20///
21/// Read failures are emitted as items so a backend can fail after a live query
22/// has already emitted elements. [`into_source`] translates such a failure to
23/// a failed Datum stream.
24pub type QueryStream<T> = Pin<Box<dyn Stream<Item = PersistenceResult<T>> + Send + 'static>>;
25
26/// Backend-defined resume position for ordered query results.
27///
28/// Offsets are meaningful only to the backend instance that produced them;
29/// there is no cross-backend or cross-database global-ordering promise.
30/// `Sequence` is the in-memory/RocksDB per-database append counter, and those
31/// backends resume strictly after the supplied value. Cassandra per-id
32/// envelopes use their journal sequence number; that capability has no
33/// offset-resume parameter. `Timestamp` is the
34/// PostgreSQL database timestamp plus journal sequence number. PostgreSQL
35/// resumes inclusively at an equal composite position: rows sharing the exact
36/// timestamp and sequence number may therefore be re-emitted after a restart,
37/// but no row at that position is skipped. Consumers must persist and resume
38/// offsets with the same backend and database that produced them.
39#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
40pub enum Offset {
41    /// Start without a previously observed resume position.
42    #[default]
43    NoOffset,
44    /// Backend-local monotonically increasing sequence position.
45    Sequence(u64),
46    /// PostgreSQL database timestamp and sequence-number resume position.
47    ///
48    /// `epoch_micros` retains PostgreSQL's microsecond timestamp precision.
49    /// The pair is ordered lexicographically. Since sequence numbers are only
50    /// unique within a persistence id, distinct rows can share a pair; resume
51    /// is inclusive for that pair and duplicate re-emission is intentional.
52    Timestamp {
53        /// Microseconds since the Unix epoch.
54        epoch_micros: i64,
55        /// Journal sequence number within the persistence id.
56        seq_nr: u64,
57    },
58}
59
60/// Whether a query completes at its materialization boundary or stays live.
61#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
62pub enum QueryMode {
63    /// Emit the boundary snapshot, then continue with subsequent changes.
64    Live,
65    /// Emit only the boundary snapshot and then complete.
66    Current,
67}
68
69/// One serialized journal event emitted by a persistence query.
70#[derive(Debug, Clone, PartialEq, Eq)]
71pub struct EventEnvelope {
72    /// Backend-local resume position for this event.
73    pub offset: Offset,
74    /// Persistent entity identity.
75    pub persistence_id: PersistenceId,
76    /// Monotonic sequence number within the persistence id.
77    pub sequence_nr: SequenceNr,
78    /// Event timestamp as milliseconds since the Unix epoch.
79    pub timestamp_ms: i64,
80    /// Durable event-adapter manifest, separate from the serializer manifest.
81    pub adapter_manifest: String,
82    /// Serialized journal payload frame.
83    pub payload: SerializedPayload,
84    /// Optional serialized event metadata frame.
85    pub metadata: Option<SerializedPayload>,
86    /// Query tags attached to the stored event.
87    pub tags: Vec<Tag>,
88}
89
90/// Latest durable value for one persistence id.
91#[derive(Debug, Clone, PartialEq, Eq)]
92pub struct UpdatedDurableState {
93    /// Backend-local resume position for this change.
94    pub offset: Offset,
95    /// Durable-state identity.
96    pub persistence_id: PersistenceId,
97    /// CAS revision of the value.
98    pub revision: Revision,
99    /// Serialized latest state.
100    pub state: SerializedPayload,
101    /// Change timestamp as milliseconds since the Unix epoch.
102    pub timestamp_ms: i64,
103}
104
105/// Latest durable-state tombstone for one persistence id.
106#[derive(Debug, Clone, PartialEq, Eq)]
107pub struct DeletedDurableState {
108    /// Backend-local resume position for this change.
109    pub offset: Offset,
110    /// Durable-state identity.
111    pub persistence_id: PersistenceId,
112    /// CAS revision of the tombstone.
113    pub revision: Revision,
114    /// Change timestamp as milliseconds since the Unix epoch.
115    pub timestamp_ms: i64,
116}
117
118/// Coalesced latest durable-state change for one persistence id.
119#[derive(Debug, Clone, PartialEq, Eq)]
120pub enum DurableStateChange {
121    /// The latest revision contains a serialized state value.
122    Updated(UpdatedDurableState),
123    /// The latest revision is a deletion tombstone.
124    Deleted(DeletedDurableState),
125}
126
127impl DurableStateChange {
128    /// Returns the backend-local resume position.
129    #[must_use]
130    pub const fn offset(&self) -> Offset {
131        match self {
132            Self::Updated(change) => change.offset,
133            Self::Deleted(change) => change.offset,
134        }
135    }
136
137    /// Returns the durable-state identity.
138    #[must_use]
139    pub fn persistence_id(&self) -> &PersistenceId {
140        match self {
141            Self::Updated(change) => &change.persistence_id,
142            Self::Deleted(change) => &change.persistence_id,
143        }
144    }
145
146    /// Returns the latest observed revision.
147    #[must_use]
148    pub const fn revision(&self) -> Revision {
149        match self {
150            Self::Updated(change) => change.revision,
151            Self::Deleted(change) => change.revision,
152        }
153    }
154}
155
156/// Capability for sequence-ordered event queries over one persistence id.
157#[async_trait::async_trait]
158pub trait EventsByPersistenceIdQuery: Send + Sync {
159    /// Queries the inclusive `[from_sequence_nr, to_sequence_nr]` range.
160    ///
161    /// Results are ordered by sequence number. `Current` completes at the
162    /// snapshot boundary; `Live` emits that same snapshot and then later
163    /// appends, completing once a finite `to_sequence_nr` is reached.
164    async fn events_by_persistence_id(
165        &self,
166        persistence_id: &PersistenceId,
167        from_sequence_nr: SequenceNr,
168        to_sequence_nr: SequenceNr,
169        mode: QueryMode,
170    ) -> PersistenceResult<QueryStream<EventEnvelope>>;
171}
172
173/// Capability for globally ordered event queries by tag.
174#[async_trait::async_trait]
175pub trait EventsByTagQuery: Send + Sync {
176    /// Queries events carrying `tag`, resuming gaplessly from `offset`.
177    async fn events_by_tag(
178        &self,
179        tag: &Tag,
180        offset: Offset,
181        mode: QueryMode,
182    ) -> PersistenceResult<QueryStream<EventEnvelope>>;
183}
184
185/// Capability for discovering persistence ids known to a journal.
186#[async_trait::async_trait]
187pub trait PersistenceIdsQuery: Send + Sync {
188    /// Queries each known persistence id once in backend discovery order.
189    async fn persistence_ids(
190        &self,
191        mode: QueryMode,
192    ) -> PersistenceResult<QueryStream<PersistenceId>>;
193}
194
195/// Capability for querying coalesced latest durable-state changes by tag.
196#[async_trait::async_trait]
197pub trait DurableStateChangesQuery: Send + Sync {
198    /// Queries latest changes carrying `tag`, resuming gaplessly from `offset`.
199    ///
200    /// Multiple rapid revisions for one persistence id may be coalesced, but
201    /// the latest revision for that id must eventually be emitted.
202    async fn changes(
203        &self,
204        tag: &Tag,
205        offset: Offset,
206        mode: QueryMode,
207    ) -> PersistenceResult<QueryStream<DurableStateChange>>;
208}
209
210/// Lifts an owned persistence-query stream into the Datum stream DSL.
211///
212/// The returned source is backed by [`Source::unfold_async`]. A query error
213/// fails the Datum stream with [`StreamError::Failed`]. Query streams are
214/// single-consumer resources, so materializations of clones of this source
215/// share that one underlying query cursor.
216#[must_use]
217pub fn into_source<T>(stream: QueryStream<T>) -> Source<T>
218where
219    T: Send + 'static,
220{
221    let stream = Arc::new(tokio::sync::Mutex::new(stream));
222    Source::unfold_async(stream, |stream| async move {
223        let next = {
224            let mut stream = stream.lock().await;
225            stream.next().await
226        };
227        match next {
228            Some(Ok(item)) => Ok(Some((stream, item))),
229            Some(Err(error)) => Err(StreamError::Failed(error.to_string())),
230            None => Ok(None),
231        }
232    })
233}
234
235/// Builds a Datum flow that decodes byte-frame envelopes into domain events.
236///
237/// Serializer dispatch happens before the event adapter. The adapter's
238/// recovery-side `0..n` cardinality is preserved, allowing old journal events
239/// to be filtered or expanded during read-side schema evolution.
240#[must_use]
241pub fn decode_envelopes<D, J, A>(
242    catalog: SerializerCatalog<J>,
243    adapter: A,
244) -> Flow<EventEnvelope, D>
245where
246    D: Send + 'static,
247    J: Send + 'static,
248    A: EventAdapter<D, J>,
249{
250    let catalog = Arc::new(catalog);
251    let adapter = Arc::new(adapter);
252    Flow::identity().try_map_concat(move |envelope: EventEnvelope| {
253        let journal_event = catalog
254            .decode(&envelope.payload)
255            .map_err(|error| StreamError::Failed(error.to_string()))?;
256        adapter
257            .from_journal_events(&envelope.adapter_manifest, journal_event)
258            .map_err(|error| StreamError::Failed(error.to_string()))
259    })
260}