datum_persistence/query/mod.rs
1//! Persistence Query capability traits and Datum stream adapters.
2//!
3//! Query backends expose byte-oriented asynchronous streams. [`into_source`]
4//! lifts those streams into Datum's stream DSL, and [`decode_envelopes`]
5//! performs explicit serializer-catalog and event-adapter decoding at the
6//! read-side boundary.
7
8mod slice;
9
10use std::{pin::Pin, sync::Arc};
11
12use datum::{Flow, Source, StreamError};
13use futures_core::Stream;
14use futures_util::StreamExt;
15
16use crate::{
17 EventAdapter, PersistenceId, PersistenceResult, Revision, SequenceNr, SerializedPayload,
18 SerializerCatalog, Tag,
19};
20
21pub use slice::{SLICE_COUNT, SliceRange, slice_for, slice_ranges};
22
23/// Owned asynchronous stream returned by a persistence-query capability.
24///
25/// Read failures are emitted as items so a backend can fail after a live query
26/// has already emitted elements. [`into_source`] translates such a failure to
27/// a failed Datum stream.
28pub type QueryStream<T> = Pin<Box<dyn Stream<Item = PersistenceResult<T>> + Send + 'static>>;
29
30/// Backend-defined resume position for ordered query results.
31///
32/// Offsets are meaningful only to the backend instance that produced them;
33/// there is no cross-backend or cross-database global-ordering promise.
34/// `Sequence` is the in-memory/RocksDB per-database append counter, and those
35/// backends resume strictly after the supplied value. Cassandra per-id
36/// envelopes use their journal sequence number; that capability has no
37/// offset-resume parameter. `Timestamp` is the
38/// PostgreSQL database timestamp plus journal sequence number. PostgreSQL
39/// resumes inclusively at an equal composite position: rows sharing the exact
40/// timestamp and sequence number may therefore be re-emitted after a restart,
41/// but no row at that position is skipped. Consumers must persist and resume
42/// offsets with the same backend and database that produced them.
43#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
44pub enum Offset {
45 /// Start without a previously observed resume position.
46 #[default]
47 NoOffset,
48 /// Backend-local monotonically increasing sequence position.
49 Sequence(u64),
50 /// PostgreSQL database timestamp and sequence-number resume position.
51 ///
52 /// `epoch_micros` retains PostgreSQL's microsecond timestamp precision.
53 /// The pair is ordered lexicographically. Since sequence numbers are only
54 /// unique within a persistence id, distinct rows can share a pair; resume
55 /// is inclusive for that pair and duplicate re-emission is intentional.
56 Timestamp {
57 /// Microseconds since the Unix epoch.
58 epoch_micros: i64,
59 /// Journal sequence number within the persistence id.
60 seq_nr: u64,
61 },
62}
63
64/// Whether a query completes at its materialization boundary or stays live.
65#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
66pub enum QueryMode {
67 /// Emit the boundary snapshot, then continue with subsequent changes.
68 Live,
69 /// Emit only the boundary snapshot and then complete.
70 Current,
71}
72
73/// One serialized journal event emitted by a persistence query.
74#[derive(Debug, Clone, PartialEq, Eq)]
75pub struct EventEnvelope {
76 /// Backend-local resume position for this event.
77 pub offset: Offset,
78 /// Persistent entity identity.
79 pub persistence_id: PersistenceId,
80 /// Monotonic sequence number within the persistence id.
81 pub sequence_nr: SequenceNr,
82 /// Event timestamp as milliseconds since the Unix epoch.
83 pub timestamp_ms: i64,
84 /// Durable event-adapter manifest, separate from the serializer manifest.
85 pub adapter_manifest: String,
86 /// Serialized journal payload frame.
87 pub payload: SerializedPayload,
88 /// Optional serialized event metadata frame.
89 pub metadata: Option<SerializedPayload>,
90 /// Query tags attached to the stored event.
91 pub tags: Vec<Tag>,
92}
93
94/// Latest durable value for one persistence id.
95#[derive(Debug, Clone, PartialEq, Eq)]
96pub struct UpdatedDurableState {
97 /// Backend-local resume position for this change.
98 pub offset: Offset,
99 /// Durable-state identity.
100 pub persistence_id: PersistenceId,
101 /// CAS revision of the value.
102 pub revision: Revision,
103 /// Serialized latest state.
104 pub state: SerializedPayload,
105 /// Change timestamp as milliseconds since the Unix epoch.
106 pub timestamp_ms: i64,
107}
108
109/// Latest durable-state tombstone for one persistence id.
110#[derive(Debug, Clone, PartialEq, Eq)]
111pub struct DeletedDurableState {
112 /// Backend-local resume position for this change.
113 pub offset: Offset,
114 /// Durable-state identity.
115 pub persistence_id: PersistenceId,
116 /// CAS revision of the tombstone.
117 pub revision: Revision,
118 /// Change timestamp as milliseconds since the Unix epoch.
119 pub timestamp_ms: i64,
120}
121
122/// Coalesced latest durable-state change for one persistence id.
123#[derive(Debug, Clone, PartialEq, Eq)]
124pub enum DurableStateChange {
125 /// The latest revision contains a serialized state value.
126 Updated(UpdatedDurableState),
127 /// The latest revision is a deletion tombstone.
128 Deleted(DeletedDurableState),
129}
130
131impl DurableStateChange {
132 /// Returns the backend-local resume position.
133 #[must_use]
134 pub const fn offset(&self) -> Offset {
135 match self {
136 Self::Updated(change) => change.offset,
137 Self::Deleted(change) => change.offset,
138 }
139 }
140
141 /// Returns the durable-state identity.
142 #[must_use]
143 pub fn persistence_id(&self) -> &PersistenceId {
144 match self {
145 Self::Updated(change) => &change.persistence_id,
146 Self::Deleted(change) => &change.persistence_id,
147 }
148 }
149
150 /// Returns the latest observed revision.
151 #[must_use]
152 pub const fn revision(&self) -> Revision {
153 match self {
154 Self::Updated(change) => change.revision,
155 Self::Deleted(change) => change.revision,
156 }
157 }
158}
159
160/// Capability for sequence-ordered event queries over one persistence id.
161#[async_trait::async_trait]
162pub trait EventsByPersistenceIdQuery: Send + Sync {
163 /// Queries the inclusive `[from_sequence_nr, to_sequence_nr]` range.
164 ///
165 /// Results are ordered by sequence number. `Current` completes at the
166 /// snapshot boundary; `Live` emits that same snapshot and then later
167 /// appends, completing once a finite `to_sequence_nr` is reached.
168 async fn events_by_persistence_id(
169 &self,
170 persistence_id: &PersistenceId,
171 from_sequence_nr: SequenceNr,
172 to_sequence_nr: SequenceNr,
173 mode: QueryMode,
174 ) -> PersistenceResult<QueryStream<EventEnvelope>>;
175}
176
177/// Capability for globally ordered event queries by tag.
178#[async_trait::async_trait]
179pub trait EventsByTagQuery: Send + Sync {
180 /// Queries events carrying `tag`, resuming gaplessly from `offset`.
181 async fn events_by_tag(
182 &self,
183 tag: &Tag,
184 offset: Offset,
185 mode: QueryMode,
186 ) -> PersistenceResult<QueryStream<EventEnvelope>>;
187}
188
189/// Capability for globally ordered event queries over persistence-id slices.
190///
191/// Slice support is PostgreSQL-first. The fixed slice model is exposed through
192/// [`slice_for`] and [`slice_ranges`]; Cassandra and embedded backends do not
193/// implement this capability yet.
194#[async_trait::async_trait]
195pub trait EventsBySlicesQuery: Send + Sync {
196 /// Queries one inclusive slice range for `entity_type`.
197 ///
198 /// Results follow the backend's global tag-query ordering contract.
199 /// PostgreSQL resumes inclusively at the supplied timestamp offset, so
200 /// exact-position duplicates may be re-emitted and consumers must apply a
201 /// provider-complete deduplication boundary.
202 ///
203 /// For `Live`, PostgreSQL holds recent rows behind its configured
204 /// visibility lag to close the ordinary late-commit race. A transaction
205 /// open longer than that lag can still commit behind the live cursor and
206 /// be skipped permanently; size the lag for the deployment's maximum
207 /// transaction-duration envelope.
208 async fn events_by_slices(
209 &self,
210 entity_type: &str,
211 min_slice: u16,
212 max_slice: u16,
213 offset: Offset,
214 mode: QueryMode,
215 ) -> PersistenceResult<QueryStream<EventEnvelope>>;
216}
217
218/// Capability for discovering persistence ids known to a journal.
219#[async_trait::async_trait]
220pub trait PersistenceIdsQuery: Send + Sync {
221 /// Queries each known persistence id once in backend discovery order.
222 async fn persistence_ids(
223 &self,
224 mode: QueryMode,
225 ) -> PersistenceResult<QueryStream<PersistenceId>>;
226}
227
228/// Capability for querying coalesced latest durable-state changes by tag.
229#[async_trait::async_trait]
230pub trait DurableStateChangesQuery: Send + Sync {
231 /// Queries latest changes carrying `tag`, resuming gaplessly from `offset`.
232 ///
233 /// Multiple rapid revisions for one persistence id may be coalesced, but
234 /// the latest revision for that id must eventually be emitted.
235 async fn changes(
236 &self,
237 tag: &Tag,
238 offset: Offset,
239 mode: QueryMode,
240 ) -> PersistenceResult<QueryStream<DurableStateChange>>;
241}
242
243/// Lifts an owned persistence-query stream into the Datum stream DSL.
244///
245/// The returned source is backed by [`Source::unfold_async`]. A query error
246/// fails the Datum stream with [`StreamError::Failed`]. Query streams are
247/// single-consumer resources, so materializations of clones of this source
248/// share that one underlying query cursor.
249#[must_use]
250pub fn into_source<T>(stream: QueryStream<T>) -> Source<T>
251where
252 T: Send + 'static,
253{
254 let stream = Arc::new(tokio::sync::Mutex::new(stream));
255 Source::unfold_async(stream, |stream| async move {
256 let next = {
257 let mut stream = stream.lock().await;
258 stream.next().await
259 };
260 match next {
261 Some(Ok(item)) => Ok(Some((stream, item))),
262 Some(Err(error)) => Err(StreamError::Failed(error.to_string())),
263 None => Ok(None),
264 }
265 })
266}
267
268/// Builds a Datum flow that decodes byte-frame envelopes into domain events.
269///
270/// Serializer dispatch happens before the event adapter. The adapter's
271/// recovery-side `0..n` cardinality is preserved, allowing old journal events
272/// to be filtered or expanded during read-side schema evolution.
273#[must_use]
274pub fn decode_envelopes<D, J, A>(
275 catalog: SerializerCatalog<J>,
276 adapter: A,
277) -> Flow<EventEnvelope, D>
278where
279 D: Send + 'static,
280 J: Send + 'static,
281 A: EventAdapter<D, J>,
282{
283 let catalog = Arc::new(catalog);
284 let adapter = Arc::new(adapter);
285 Flow::identity().try_map_concat(move |envelope: EventEnvelope| {
286 let journal_event = catalog
287 .decode(&envelope.payload)
288 .map_err(|error| StreamError::Failed(error.to_string()))?;
289 adapter
290 .from_journal_events(&envelope.adapter_manifest, journal_event)
291 .map_err(|error| StreamError::Failed(error.to_string()))
292 })
293}