Skip to main content

datum_persistence/query/
mod.rs

1//! Persistence Query capability traits and Datum stream adapters.
2//!
3//! Query backends expose byte-oriented asynchronous streams. [`into_source`]
4//! lifts those streams into Datum's stream DSL, and [`decode_envelopes`]
5//! performs explicit serializer-catalog and event-adapter decoding at the
6//! read-side boundary.
7
8mod slice;
9
10use std::{pin::Pin, sync::Arc};
11
12use datum::{Flow, Source, StreamError};
13use futures_core::Stream;
14use futures_util::StreamExt;
15
16use crate::{
17    EventAdapter, PersistenceId, PersistenceResult, Revision, SequenceNr, SerializedPayload,
18    SerializerCatalog, Tag,
19};
20
21pub use slice::{SLICE_COUNT, SliceRange, slice_for, slice_ranges};
22
23/// Owned asynchronous stream returned by a persistence-query capability.
24///
25/// Read failures are emitted as items so a backend can fail after a live query
26/// has already emitted elements. [`into_source`] translates such a failure to
27/// a failed Datum stream.
28pub type QueryStream<T> = Pin<Box<dyn Stream<Item = PersistenceResult<T>> + Send + 'static>>;
29
30/// Backend-defined resume position for ordered query results.
31///
32/// Offsets are meaningful only to the backend instance that produced them;
33/// there is no cross-backend or cross-database global-ordering promise.
34/// `Sequence` is the in-memory/RocksDB per-database append counter, and those
35/// backends resume strictly after the supplied value. Cassandra per-id
36/// envelopes use their journal sequence number; that capability has no
37/// offset-resume parameter. `Timestamp` is the
38/// PostgreSQL database timestamp plus journal sequence number. PostgreSQL
39/// resumes inclusively at an equal composite position: rows sharing the exact
40/// timestamp and sequence number may therefore be re-emitted after a restart,
41/// but no row at that position is skipped. Consumers must persist and resume
42/// offsets with the same backend and database that produced them.
43#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
44pub enum Offset {
45    /// Start without a previously observed resume position.
46    #[default]
47    NoOffset,
48    /// Backend-local monotonically increasing sequence position.
49    Sequence(u64),
50    /// PostgreSQL database timestamp and sequence-number resume position.
51    ///
52    /// `epoch_micros` retains PostgreSQL's microsecond timestamp precision.
53    /// The pair is ordered lexicographically. Since sequence numbers are only
54    /// unique within a persistence id, distinct rows can share a pair; resume
55    /// is inclusive for that pair and duplicate re-emission is intentional.
56    Timestamp {
57        /// Microseconds since the Unix epoch.
58        epoch_micros: i64,
59        /// Journal sequence number within the persistence id.
60        seq_nr: u64,
61    },
62}
63
64/// Whether a query completes at its materialization boundary or stays live.
65#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
66pub enum QueryMode {
67    /// Emit the boundary snapshot, then continue with subsequent changes.
68    Live,
69    /// Emit only the boundary snapshot and then complete.
70    Current,
71}
72
73/// One serialized journal event emitted by a persistence query.
74#[derive(Debug, Clone, PartialEq, Eq)]
75pub struct EventEnvelope {
76    /// Backend-local resume position for this event.
77    pub offset: Offset,
78    /// Persistent entity identity.
79    pub persistence_id: PersistenceId,
80    /// Monotonic sequence number within the persistence id.
81    pub sequence_nr: SequenceNr,
82    /// Event timestamp as milliseconds since the Unix epoch.
83    pub timestamp_ms: i64,
84    /// Durable event-adapter manifest, separate from the serializer manifest.
85    pub adapter_manifest: String,
86    /// Serialized journal payload frame.
87    pub payload: SerializedPayload,
88    /// Optional serialized event metadata frame.
89    pub metadata: Option<SerializedPayload>,
90    /// Query tags attached to the stored event.
91    pub tags: Vec<Tag>,
92}
93
94/// Latest durable value for one persistence id.
95#[derive(Debug, Clone, PartialEq, Eq)]
96pub struct UpdatedDurableState {
97    /// Backend-local resume position for this change.
98    pub offset: Offset,
99    /// Durable-state identity.
100    pub persistence_id: PersistenceId,
101    /// CAS revision of the value.
102    pub revision: Revision,
103    /// Serialized latest state.
104    pub state: SerializedPayload,
105    /// Change timestamp as milliseconds since the Unix epoch.
106    pub timestamp_ms: i64,
107}
108
109/// Latest durable-state tombstone for one persistence id.
110#[derive(Debug, Clone, PartialEq, Eq)]
111pub struct DeletedDurableState {
112    /// Backend-local resume position for this change.
113    pub offset: Offset,
114    /// Durable-state identity.
115    pub persistence_id: PersistenceId,
116    /// CAS revision of the tombstone.
117    pub revision: Revision,
118    /// Change timestamp as milliseconds since the Unix epoch.
119    pub timestamp_ms: i64,
120}
121
122/// Coalesced latest durable-state change for one persistence id.
123#[derive(Debug, Clone, PartialEq, Eq)]
124pub enum DurableStateChange {
125    /// The latest revision contains a serialized state value.
126    Updated(UpdatedDurableState),
127    /// The latest revision is a deletion tombstone.
128    Deleted(DeletedDurableState),
129}
130
131impl DurableStateChange {
132    /// Returns the backend-local resume position.
133    #[must_use]
134    pub const fn offset(&self) -> Offset {
135        match self {
136            Self::Updated(change) => change.offset,
137            Self::Deleted(change) => change.offset,
138        }
139    }
140
141    /// Returns the durable-state identity.
142    #[must_use]
143    pub fn persistence_id(&self) -> &PersistenceId {
144        match self {
145            Self::Updated(change) => &change.persistence_id,
146            Self::Deleted(change) => &change.persistence_id,
147        }
148    }
149
150    /// Returns the latest observed revision.
151    #[must_use]
152    pub const fn revision(&self) -> Revision {
153        match self {
154            Self::Updated(change) => change.revision,
155            Self::Deleted(change) => change.revision,
156        }
157    }
158}
159
160/// Capability for sequence-ordered event queries over one persistence id.
161#[async_trait::async_trait]
162pub trait EventsByPersistenceIdQuery: Send + Sync {
163    /// Queries the inclusive `[from_sequence_nr, to_sequence_nr]` range.
164    ///
165    /// Results are ordered by sequence number. `Current` completes at the
166    /// snapshot boundary; `Live` emits that same snapshot and then later
167    /// appends, completing once a finite `to_sequence_nr` is reached.
168    async fn events_by_persistence_id(
169        &self,
170        persistence_id: &PersistenceId,
171        from_sequence_nr: SequenceNr,
172        to_sequence_nr: SequenceNr,
173        mode: QueryMode,
174    ) -> PersistenceResult<QueryStream<EventEnvelope>>;
175}
176
177/// Capability for globally ordered event queries by tag.
178#[async_trait::async_trait]
179pub trait EventsByTagQuery: Send + Sync {
180    /// Queries events carrying `tag`, resuming gaplessly from `offset`.
181    async fn events_by_tag(
182        &self,
183        tag: &Tag,
184        offset: Offset,
185        mode: QueryMode,
186    ) -> PersistenceResult<QueryStream<EventEnvelope>>;
187}
188
189/// Capability for globally ordered event queries over persistence-id slices.
190///
191/// Slice support is PostgreSQL-first. The fixed slice model is exposed through
192/// [`slice_for`] and [`slice_ranges`]; Cassandra and embedded backends do not
193/// implement this capability yet.
194#[async_trait::async_trait]
195pub trait EventsBySlicesQuery: Send + Sync {
196    /// Queries one inclusive slice range for `entity_type`.
197    ///
198    /// Results follow the backend's global tag-query ordering contract.
199    /// PostgreSQL resumes inclusively at the supplied timestamp offset, so
200    /// exact-position duplicates may be re-emitted and consumers must apply a
201    /// provider-complete deduplication boundary.
202    ///
203    /// For `Live`, PostgreSQL holds recent rows behind its configured
204    /// visibility lag to close the ordinary late-commit race. A transaction
205    /// open longer than that lag can still commit behind the live cursor and
206    /// be skipped permanently; size the lag for the deployment's maximum
207    /// transaction-duration envelope.
208    async fn events_by_slices(
209        &self,
210        entity_type: &str,
211        min_slice: u16,
212        max_slice: u16,
213        offset: Offset,
214        mode: QueryMode,
215    ) -> PersistenceResult<QueryStream<EventEnvelope>>;
216}
217
218/// Capability for discovering persistence ids known to a journal.
219#[async_trait::async_trait]
220pub trait PersistenceIdsQuery: Send + Sync {
221    /// Queries each known persistence id once in backend discovery order.
222    async fn persistence_ids(
223        &self,
224        mode: QueryMode,
225    ) -> PersistenceResult<QueryStream<PersistenceId>>;
226}
227
228/// Capability for querying coalesced latest durable-state changes by tag.
229#[async_trait::async_trait]
230pub trait DurableStateChangesQuery: Send + Sync {
231    /// Queries latest changes carrying `tag`, resuming gaplessly from `offset`.
232    ///
233    /// Multiple rapid revisions for one persistence id may be coalesced, but
234    /// the latest revision for that id must eventually be emitted.
235    async fn changes(
236        &self,
237        tag: &Tag,
238        offset: Offset,
239        mode: QueryMode,
240    ) -> PersistenceResult<QueryStream<DurableStateChange>>;
241}
242
243/// Lifts an owned persistence-query stream into the Datum stream DSL.
244///
245/// The returned source is backed by [`Source::unfold_async`]. A query error
246/// fails the Datum stream with [`StreamError::Failed`]. Query streams are
247/// single-consumer resources, so materializations of clones of this source
248/// share that one underlying query cursor.
249#[must_use]
250pub fn into_source<T>(stream: QueryStream<T>) -> Source<T>
251where
252    T: Send + 'static,
253{
254    let stream = Arc::new(tokio::sync::Mutex::new(stream));
255    Source::unfold_async(stream, |stream| async move {
256        let next = {
257            let mut stream = stream.lock().await;
258            stream.next().await
259        };
260        match next {
261            Some(Ok(item)) => Ok(Some((stream, item))),
262            Some(Err(error)) => Err(StreamError::Failed(error.to_string())),
263            None => Ok(None),
264        }
265    })
266}
267
268/// Builds a Datum flow that decodes byte-frame envelopes into domain events.
269///
270/// Serializer dispatch happens before the event adapter. The adapter's
271/// recovery-side `0..n` cardinality is preserved, allowing old journal events
272/// to be filtered or expanded during read-side schema evolution.
273#[must_use]
274pub fn decode_envelopes<D, J, A>(
275    catalog: SerializerCatalog<J>,
276    adapter: A,
277) -> Flow<EventEnvelope, D>
278where
279    D: Send + 'static,
280    J: Send + 'static,
281    A: EventAdapter<D, J>,
282{
283    let catalog = Arc::new(catalog);
284    let adapter = Arc::new(adapter);
285    Flow::identity().try_map_concat(move |envelope: EventEnvelope| {
286        let journal_event = catalog
287            .decode(&envelope.payload)
288            .map_err(|error| StreamError::Failed(error.to_string()))?;
289        adapter
290            .from_journal_events(&envelope.adapter_manifest, journal_event)
291            .map_err(|error| StreamError::Failed(error.to_string()))
292    })
293}