datum_persistence/query/mod.rs
1//! Persistence Query capability traits and Datum stream adapters.
2//!
3//! Query backends expose byte-oriented asynchronous streams. [`into_source`]
4//! lifts those streams into Datum's stream DSL, and [`decode_envelopes`]
5//! performs explicit serializer-catalog and event-adapter decoding at the
6//! read-side boundary.
7
8use std::{pin::Pin, sync::Arc};
9
10use datum::{Flow, Source, StreamError};
11use futures_core::Stream;
12use futures_util::StreamExt;
13
14use crate::{
15 EventAdapter, PersistenceId, PersistenceResult, Revision, SequenceNr, SerializedPayload,
16 SerializerCatalog, Tag,
17};
18
19/// Owned asynchronous stream returned by a persistence-query capability.
20///
21/// Read failures are emitted as items so a backend can fail after a live query
22/// has already emitted elements. [`into_source`] translates such a failure to
23/// a failed Datum stream.
24pub type QueryStream<T> = Pin<Box<dyn Stream<Item = PersistenceResult<T>> + Send + 'static>>;
25
26/// Backend-defined resume position for ordered query results.
27///
28/// Offsets are meaningful only to the backend instance that produced them;
29/// there is no cross-backend or cross-database global-ordering promise.
30/// `Sequence` is the in-memory/RocksDB per-database append counter, and those
31/// backends resume strictly after the supplied value. Cassandra per-id
32/// envelopes use their journal sequence number; that capability has no
33/// offset-resume parameter. `Timestamp` is the
34/// PostgreSQL database timestamp plus journal sequence number. PostgreSQL
35/// resumes inclusively at an equal composite position: rows sharing the exact
36/// timestamp and sequence number may therefore be re-emitted after a restart,
37/// but no row at that position is skipped. Consumers must persist and resume
38/// offsets with the same backend and database that produced them.
39#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
40pub enum Offset {
41 /// Start without a previously observed resume position.
42 #[default]
43 NoOffset,
44 /// Backend-local monotonically increasing sequence position.
45 Sequence(u64),
46 /// PostgreSQL database timestamp and sequence-number resume position.
47 ///
48 /// `epoch_micros` retains PostgreSQL's microsecond timestamp precision.
49 /// The pair is ordered lexicographically. Since sequence numbers are only
50 /// unique within a persistence id, distinct rows can share a pair; resume
51 /// is inclusive for that pair and duplicate re-emission is intentional.
52 Timestamp {
53 /// Microseconds since the Unix epoch.
54 epoch_micros: i64,
55 /// Journal sequence number within the persistence id.
56 seq_nr: u64,
57 },
58}
59
60/// Whether a query completes at its materialization boundary or stays live.
61#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
62pub enum QueryMode {
63 /// Emit the boundary snapshot, then continue with subsequent changes.
64 Live,
65 /// Emit only the boundary snapshot and then complete.
66 Current,
67}
68
69/// One serialized journal event emitted by a persistence query.
70#[derive(Debug, Clone, PartialEq, Eq)]
71pub struct EventEnvelope {
72 /// Backend-local resume position for this event.
73 pub offset: Offset,
74 /// Persistent entity identity.
75 pub persistence_id: PersistenceId,
76 /// Monotonic sequence number within the persistence id.
77 pub sequence_nr: SequenceNr,
78 /// Event timestamp as milliseconds since the Unix epoch.
79 pub timestamp_ms: i64,
80 /// Durable event-adapter manifest, separate from the serializer manifest.
81 pub adapter_manifest: String,
82 /// Serialized journal payload frame.
83 pub payload: SerializedPayload,
84 /// Optional serialized event metadata frame.
85 pub metadata: Option<SerializedPayload>,
86 /// Query tags attached to the stored event.
87 pub tags: Vec<Tag>,
88}
89
90/// Latest durable value for one persistence id.
91#[derive(Debug, Clone, PartialEq, Eq)]
92pub struct UpdatedDurableState {
93 /// Backend-local resume position for this change.
94 pub offset: Offset,
95 /// Durable-state identity.
96 pub persistence_id: PersistenceId,
97 /// CAS revision of the value.
98 pub revision: Revision,
99 /// Serialized latest state.
100 pub state: SerializedPayload,
101 /// Change timestamp as milliseconds since the Unix epoch.
102 pub timestamp_ms: i64,
103}
104
105/// Latest durable-state tombstone for one persistence id.
106#[derive(Debug, Clone, PartialEq, Eq)]
107pub struct DeletedDurableState {
108 /// Backend-local resume position for this change.
109 pub offset: Offset,
110 /// Durable-state identity.
111 pub persistence_id: PersistenceId,
112 /// CAS revision of the tombstone.
113 pub revision: Revision,
114 /// Change timestamp as milliseconds since the Unix epoch.
115 pub timestamp_ms: i64,
116}
117
118/// Coalesced latest durable-state change for one persistence id.
119#[derive(Debug, Clone, PartialEq, Eq)]
120pub enum DurableStateChange {
121 /// The latest revision contains a serialized state value.
122 Updated(UpdatedDurableState),
123 /// The latest revision is a deletion tombstone.
124 Deleted(DeletedDurableState),
125}
126
127impl DurableStateChange {
128 /// Returns the backend-local resume position.
129 #[must_use]
130 pub const fn offset(&self) -> Offset {
131 match self {
132 Self::Updated(change) => change.offset,
133 Self::Deleted(change) => change.offset,
134 }
135 }
136
137 /// Returns the durable-state identity.
138 #[must_use]
139 pub fn persistence_id(&self) -> &PersistenceId {
140 match self {
141 Self::Updated(change) => &change.persistence_id,
142 Self::Deleted(change) => &change.persistence_id,
143 }
144 }
145
146 /// Returns the latest observed revision.
147 #[must_use]
148 pub const fn revision(&self) -> Revision {
149 match self {
150 Self::Updated(change) => change.revision,
151 Self::Deleted(change) => change.revision,
152 }
153 }
154}
155
156/// Capability for sequence-ordered event queries over one persistence id.
157#[async_trait::async_trait]
158pub trait EventsByPersistenceIdQuery: Send + Sync {
159 /// Queries the inclusive `[from_sequence_nr, to_sequence_nr]` range.
160 ///
161 /// Results are ordered by sequence number. `Current` completes at the
162 /// snapshot boundary; `Live` emits that same snapshot and then later
163 /// appends, completing once a finite `to_sequence_nr` is reached.
164 async fn events_by_persistence_id(
165 &self,
166 persistence_id: &PersistenceId,
167 from_sequence_nr: SequenceNr,
168 to_sequence_nr: SequenceNr,
169 mode: QueryMode,
170 ) -> PersistenceResult<QueryStream<EventEnvelope>>;
171}
172
173/// Capability for globally ordered event queries by tag.
174#[async_trait::async_trait]
175pub trait EventsByTagQuery: Send + Sync {
176 /// Queries events carrying `tag`, resuming gaplessly from `offset`.
177 async fn events_by_tag(
178 &self,
179 tag: &Tag,
180 offset: Offset,
181 mode: QueryMode,
182 ) -> PersistenceResult<QueryStream<EventEnvelope>>;
183}
184
185/// Capability for discovering persistence ids known to a journal.
186#[async_trait::async_trait]
187pub trait PersistenceIdsQuery: Send + Sync {
188 /// Queries each known persistence id once in backend discovery order.
189 async fn persistence_ids(
190 &self,
191 mode: QueryMode,
192 ) -> PersistenceResult<QueryStream<PersistenceId>>;
193}
194
195/// Capability for querying coalesced latest durable-state changes by tag.
196#[async_trait::async_trait]
197pub trait DurableStateChangesQuery: Send + Sync {
198 /// Queries latest changes carrying `tag`, resuming gaplessly from `offset`.
199 ///
200 /// Multiple rapid revisions for one persistence id may be coalesced, but
201 /// the latest revision for that id must eventually be emitted.
202 async fn changes(
203 &self,
204 tag: &Tag,
205 offset: Offset,
206 mode: QueryMode,
207 ) -> PersistenceResult<QueryStream<DurableStateChange>>;
208}
209
210/// Lifts an owned persistence-query stream into the Datum stream DSL.
211///
212/// The returned source is backed by [`Source::unfold_async`]. A query error
213/// fails the Datum stream with [`StreamError::Failed`]. Query streams are
214/// single-consumer resources, so materializations of clones of this source
215/// share that one underlying query cursor.
216#[must_use]
217pub fn into_source<T>(stream: QueryStream<T>) -> Source<T>
218where
219 T: Send + 'static,
220{
221 let stream = Arc::new(tokio::sync::Mutex::new(stream));
222 Source::unfold_async(stream, |stream| async move {
223 let next = {
224 let mut stream = stream.lock().await;
225 stream.next().await
226 };
227 match next {
228 Some(Ok(item)) => Ok(Some((stream, item))),
229 Some(Err(error)) => Err(StreamError::Failed(error.to_string())),
230 None => Ok(None),
231 }
232 })
233}
234
235/// Builds a Datum flow that decodes byte-frame envelopes into domain events.
236///
237/// Serializer dispatch happens before the event adapter. The adapter's
238/// recovery-side `0..n` cardinality is preserved, allowing old journal events
239/// to be filtered or expanded during read-side schema evolution.
240#[must_use]
241pub fn decode_envelopes<D, J, A>(
242 catalog: SerializerCatalog<J>,
243 adapter: A,
244) -> Flow<EventEnvelope, D>
245where
246 D: Send + 'static,
247 J: Send + 'static,
248 A: EventAdapter<D, J>,
249{
250 let catalog = Arc::new(catalog);
251 let adapter = Arc::new(adapter);
252 Flow::identity().try_map_concat(move |envelope: EventEnvelope| {
253 let journal_event = catalog
254 .decode(&envelope.payload)
255 .map_err(|error| StreamError::Failed(error.to_string()))?;
256 adapter
257 .from_journal_events(&envelope.adapter_manifest, journal_event)
258 .map_err(|error| StreamError::Failed(error.to_string()))
259 })
260}