datum-persistence 0.11.3

Event sourcing, durable state, and CQRS persistence for Datum
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
//! Provider-complete durable projection cursors.

use std::{collections::BTreeSet, fmt};

use crate::{PersistenceId, SequenceNr};

use super::{ProjectionError, ProjectionResult};

const CURSOR_FORMAT_VERSION: &str = "datum-projection-cursor-v1";

/// Durable identity of a source provider and its cursor semantics.
///
/// A manifest is a persisted protocol identifier, not a Rust type name. Once
/// deployed, changing it makes existing cursors incompatible and must be
/// handled as an explicit migration. Loading a cursor under a different
/// manifest is a fatal [`ProjectionError::ProviderManifestMismatch`].
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ProviderManifest(String);

impl ProviderManifest {
    /// Creates a non-blank provider manifest.
    pub fn new(value: impl Into<String>) -> ProjectionResult<Self> {
        let value = value.into();
        if value.trim().is_empty() {
            return Err(ProjectionError::invalid_config(
                "provider_manifest",
                "provider manifest must not be blank",
            ));
        }
        Ok(Self(value))
    }

    /// Returns the durable manifest text.
    #[must_use]
    pub fn as_str(&self) -> &str {
        &self.0
    }

    /// Consumes the manifest and returns its durable text.
    #[must_use]
    pub fn into_inner(self) -> String {
        self.0
    }
}

impl fmt::Display for ProviderManifest {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter.write_str(self.as_str())
    }
}

/// Provider-defined stable resume position.
///
/// The projection core treats this text as opaque. A source provider owns its
/// meaning and must keep its encoding stable for the lifetime of the provider
/// manifest.
#[derive(Debug, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct CursorPosition(String);

impl CursorPosition {
    /// Creates an opaque provider position.
    #[must_use]
    pub fn new(value: impl Into<String>) -> Self {
        Self(value.into())
    }

    /// Returns the stable provider position text.
    #[must_use]
    pub fn as_str(&self) -> &str {
        &self.0
    }

    /// Consumes the position and returns its stable text.
    #[must_use]
    pub fn into_inner(self) -> String {
        self.0
    }
}

impl fmt::Display for CursorPosition {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter.write_str(self.as_str())
    }
}

impl From<String> for CursorPosition {
    fn from(value: String) -> Self {
        Self::new(value)
    }
}

impl From<&str> for CursorPosition {
    fn from(value: &str) -> Self {
        Self::new(value)
    }
}

/// One event already consumed at a cursor's inclusive resume boundary.
///
/// The pair, rather than sequence number alone, is required because sequence
/// numbers are scoped to a persistence id.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct BoundaryEvent {
    persistence_id: PersistenceId,
    sequence_nr: SequenceNr,
}

impl BoundaryEvent {
    /// Creates a consumed boundary event identity.
    #[must_use]
    pub fn new(persistence_id: PersistenceId, sequence_nr: SequenceNr) -> Self {
        Self {
            persistence_id,
            sequence_nr,
        }
    }

    /// Returns the persistent entity at the boundary.
    #[must_use]
    pub fn persistence_id(&self) -> &PersistenceId {
        &self.persistence_id
    }

    /// Returns the entity-local event sequence number.
    #[must_use]
    pub const fn sequence_nr(&self) -> SequenceNr {
        self.sequence_nr
    }
}

/// Resume boundary plus every event consumed at that exact position.
///
/// Inclusive-resume providers use [`Self::contains`] to drop an event that is
/// re-emitted at the saved position. Events at later positions remain eligible.
/// Providers whose positions are complete still persist an empty set, keeping
/// one cursor shape across backends.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DeduplicationBoundary {
    position: CursorPosition,
    consumed: BTreeSet<BoundaryEvent>,
}

impl DeduplicationBoundary {
    /// Creates a boundary with no consumed event identities.
    #[must_use]
    pub fn new(position: impl Into<CursorPosition>) -> Self {
        Self {
            position: position.into(),
            consumed: BTreeSet::new(),
        }
    }

    /// Creates a boundary with a complete consumed-event set.
    #[must_use]
    pub fn with_consumed(
        position: impl Into<CursorPosition>,
        consumed: impl IntoIterator<Item = BoundaryEvent>,
    ) -> Self {
        Self {
            position: position.into(),
            consumed: consumed.into_iter().collect(),
        }
    }

    /// Returns the provider-defined boundary position.
    #[must_use]
    pub fn position(&self) -> &CursorPosition {
        &self.position
    }

    /// Returns consumed identities in stable persistence-id/sequence order.
    pub fn consumed(&self) -> impl ExactSizeIterator<Item = &BoundaryEvent> {
        self.consumed.iter()
    }

    /// Records an event consumed at this exact position.
    ///
    /// Returns `true` when the identity was not already present.
    pub fn record(&mut self, event: BoundaryEvent) -> bool {
        self.consumed.insert(event)
    }

    /// Returns whether an event was already consumed at this exact position.
    #[must_use]
    pub fn contains(&self, persistence_id: &PersistenceId, sequence_nr: SequenceNr) -> bool {
        self.consumed
            .contains(&BoundaryEvent::new(persistence_id.clone(), sequence_nr))
    }
}

/// Provider-tagged, provider-complete durable projection cursor.
///
/// The cursor owns the opaque source position and the deduplication set needed
/// by inclusive-resume providers. [`Self::encode`] and [`Self::decode`] form the
/// stable store-independent string codec used by durable backends.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProjectionCursor {
    provider_manifest: ProviderManifest,
    boundary: DeduplicationBoundary,
}

impl ProjectionCursor {
    /// Creates a provider-complete cursor.
    #[must_use]
    pub fn new(provider_manifest: ProviderManifest, boundary: DeduplicationBoundary) -> Self {
        Self {
            provider_manifest,
            boundary,
        }
    }

    /// Creates a cursor with an empty deduplication set.
    #[must_use]
    pub fn from_position(
        provider_manifest: ProviderManifest,
        position: impl Into<CursorPosition>,
    ) -> Self {
        Self::new(provider_manifest, DeduplicationBoundary::new(position))
    }

    /// Returns the durable source-provider manifest.
    #[must_use]
    pub fn provider_manifest(&self) -> &ProviderManifest {
        &self.provider_manifest
    }

    /// Returns the provider-defined resume position.
    #[must_use]
    pub fn position(&self) -> &CursorPosition {
        self.boundary.position()
    }

    /// Returns the complete inclusive-resume deduplication boundary.
    #[must_use]
    pub fn boundary(&self) -> &DeduplicationBoundary {
        &self.boundary
    }

    /// Verifies that this cursor belongs to the configured source provider.
    pub fn ensure_manifest(&self, expected: &ProviderManifest) -> ProjectionResult<()> {
        if &self.provider_manifest != expected {
            return Err(ProjectionError::ProviderManifestMismatch {
                expected: expected.as_str().to_owned(),
                actual: self.provider_manifest.as_str().to_owned(),
            });
        }
        Ok(())
    }

    /// Encodes this cursor using Datum's stable versioned string format.
    ///
    /// Strings are UTF-8 encoded as lowercase hexadecimal and boundary events
    /// are ordered by persistence id then sequence number, so equal cursors
    /// always produce byte-for-byte equal output.
    #[must_use]
    pub fn encode(&self) -> String {
        let consumed = self
            .boundary
            .consumed()
            .map(|event| {
                format!(
                    "{}:{}",
                    encode_hex(event.persistence_id().id()),
                    event.sequence_nr().as_u64()
                )
            })
            .collect::<Vec<_>>()
            .join(",");
        format!(
            "{CURSOR_FORMAT_VERSION}|{}|{}|{}|{consumed}",
            encode_hex(self.provider_manifest.as_str()),
            encode_hex(self.position().as_str()),
            self.boundary.consumed().len(),
        )
    }

    /// Decodes and validates Datum's stable versioned string format.
    ///
    /// Malformed, duplicate, out-of-order, or non-canonical data is a fatal
    /// cursor-codec error and must not enter projection restart backoff.
    pub fn decode(encoded: &str) -> ProjectionResult<Self> {
        let mut fields = encoded.splitn(5, '|');
        let version = required_field(&mut fields, "format version")?;
        let manifest = required_field(&mut fields, "provider manifest")?;
        let position = required_field(&mut fields, "cursor position")?;
        let count = required_field(&mut fields, "boundary count")?;
        let consumed = required_field(&mut fields, "boundary events")?;

        if version != CURSOR_FORMAT_VERSION {
            return Err(codec_error(format!(
                "unsupported format version {version:?}"
            )));
        }

        let provider_manifest = ProviderManifest::new(decode_hex(manifest, "provider manifest")?)
            .map_err(|error| codec_error(error.to_string()))?;
        let position = CursorPosition::new(decode_hex(position, "cursor position")?);
        let expected_count = count
            .parse::<usize>()
            .map_err(|_| codec_error(format!("invalid boundary count {count:?}")))?;

        let mut boundary_events = BTreeSet::new();
        if !consumed.is_empty() {
            for item in consumed.split(',') {
                let (persistence_id, sequence_nr) = item
                    .split_once(':')
                    .ok_or_else(|| codec_error(format!("invalid boundary event {item:?}")))?;
                let persistence_id =
                    PersistenceId::from_unique(decode_hex(persistence_id, "persistence id")?)
                        .map_err(|error| codec_error(error.to_string()))?;
                let sequence_nr = sequence_nr.parse::<u64>().map_err(|_| {
                    codec_error(format!("invalid boundary sequence number {sequence_nr:?}"))
                })?;
                if !boundary_events.insert(BoundaryEvent::new(
                    persistence_id,
                    SequenceNr::new(sequence_nr),
                )) {
                    return Err(codec_error("duplicate boundary event"));
                }
            }
        }

        if boundary_events.len() != expected_count {
            return Err(codec_error(format!(
                "boundary count declares {expected_count}, decoded {}",
                boundary_events.len()
            )));
        }

        let cursor = Self::new(
            provider_manifest,
            DeduplicationBoundary::with_consumed(position, boundary_events),
        );
        if cursor.encode() != encoded {
            return Err(codec_error("cursor encoding is not canonical"));
        }
        Ok(cursor)
    }
}

fn required_field<'a>(
    fields: &mut impl Iterator<Item = &'a str>,
    name: &str,
) -> ProjectionResult<&'a str> {
    fields
        .next()
        .ok_or_else(|| codec_error(format!("missing {name}")))
}

fn encode_hex(value: &str) -> String {
    const HEX: &[u8; 16] = b"0123456789abcdef";
    let bytes = value.as_bytes();
    let mut encoded = String::with_capacity(bytes.len().saturating_mul(2));
    for byte in bytes {
        encoded.push(char::from(HEX[usize::from(byte >> 4)]));
        encoded.push(char::from(HEX[usize::from(byte & 0x0f)]));
    }
    encoded
}

fn decode_hex(value: &str, name: &str) -> ProjectionResult<String> {
    if !value.len().is_multiple_of(2) {
        return Err(codec_error(format!("{name} has odd-length hexadecimal")));
    }
    let mut decoded = Vec::with_capacity(value.len() / 2);
    for pair in value.as_bytes().chunks_exact(2) {
        let high = hex_nibble(pair[0])
            .ok_or_else(|| codec_error(format!("{name} contains non-hexadecimal data")))?;
        let low = hex_nibble(pair[1])
            .ok_or_else(|| codec_error(format!("{name} contains non-hexadecimal data")))?;
        decoded.push((high << 4) | low);
    }
    String::from_utf8(decoded).map_err(|_| codec_error(format!("{name} is not valid UTF-8")))
}

const fn hex_nibble(byte: u8) -> Option<u8> {
    match byte {
        b'0'..=b'9' => Some(byte - b'0'),
        b'a'..=b'f' => Some(byte - b'a' + 10),
        _ => None,
    }
}

fn codec_error(reason: impl Into<String>) -> ProjectionError {
    ProjectionError::CursorCodec(reason.into())
}

#[cfg(test)]
mod tests {
    use super::*;

    fn test_manifest() -> ProviderManifest {
        ProviderManifest::new("datum-test-provider/timestamp-v1")
            .expect("test provider manifest is valid")
    }

    #[test]
    fn stable_codec_round_trips_unicode_and_delimiters() {
        let events = [
            BoundaryEvent::new(
                PersistenceId::from_unique("account|a:b,c|δ")
                    .expect("test persistence id is valid"),
                SequenceNr::new(7),
            ),
            BoundaryEvent::new(
                PersistenceId::from_unique("account|z").expect("test persistence id is valid"),
                SequenceNr::new(u64::MAX),
            ),
        ];
        let cursor = ProjectionCursor::new(
            test_manifest(),
            DeduplicationBoundary::with_consumed("ts|1:2,3/λ", events),
        );
        let encoded = cursor.encode();
        assert_eq!(ProjectionCursor::decode(&encoded).unwrap(), cursor);
        assert_eq!(
            ProjectionCursor::decode(&encoded).unwrap().encode(),
            encoded
        );
    }

    #[test]
    fn malformed_and_mismatched_cursors_are_fatal() {
        let malformed =
            ProjectionCursor::decode("not-a-cursor").expect_err("malformed durable data must fail");
        assert!(malformed.is_fatal());

        let cursor = ProjectionCursor::from_position(test_manifest(), "42");
        let other = ProviderManifest::new("datum-test-provider/sequence-v1").unwrap();
        let mismatch = cursor
            .ensure_manifest(&other)
            .expect_err("provider identity must be checked");
        assert!(matches!(
            mismatch,
            ProjectionError::ProviderManifestMismatch { .. }
        ));
        assert!(mismatch.is_fatal());
    }
}