Skip to main content

datum_persistence/projection/
source.rs

1//! Fresh projection source factories and resume-boundary filtering.
2
3use std::{cmp::Ordering, pin::Pin, sync::Arc};
4
5use futures_core::Stream;
6use futures_util::StreamExt;
7
8use crate::query::{EventEnvelope, EventsBySlicesQuery, Offset, QueryMode, SliceRange, slice_for};
9
10use super::{
11    BoundaryEvent, CursorPosition, DeduplicationBoundary, ProjectionCursor, ProjectionError,
12    ProjectionId, ProjectionOffsetStore, ProjectionResult, ProviderManifest,
13};
14
15const SLICE_PROVIDER_MANIFEST_VERSION: &str = "datum-postgres-events-by-slices-v1";
16const TIMESTAMP_POSITION_VERSION: &str = "pg-timestamp-v1";
17
18/// Owned asynchronous stream materialized for one projection incarnation.
19///
20/// A source may fail after emitting envelopes by yielding an error item. The
21/// runtime consumes this stream only from its dedicated source worker.
22pub type ProjectionSource<Envelope> =
23    Pin<Box<dyn Stream<Item = ProjectionResult<Envelope>> + Send + 'static>>;
24
25/// Factory for provider-complete, restartable projection sources.
26///
27/// [`Self::source_from`] must materialize a **new** source every time it is
28/// invoked. Returning a reused stream or a clone sharing one underlying query
29/// cursor violates the projection restart contract.
30///
31/// Delivery guarantees apply only to envelopes emitted by the provider. They
32/// cannot repair source-side omissions: PostgreSQL live tag/slice queries can
33/// permanently skip events committed by transactions open longer than their
34/// configured visibility lag, Cassandra can commit a journal row while losing
35/// its non-atomic tag-index row until external repair, and durable-state change
36/// sources may coalesce rapid revisions to the latest revision per id.
37#[async_trait::async_trait]
38pub trait SourceProvider<Envelope>: Send + Sync + 'static {
39    /// Returns the stable durable identity of this provider and cursor format.
40    fn manifest(&self) -> &ProviderManifest;
41
42    /// Validates that this provider may run under `projection_id`.
43    ///
44    /// The projection runtime invokes this side-effect-free hook during spawn
45    /// validation, before the provider is materialized or an offset-store
46    /// generation is claimed. Most providers accept any valid projection id,
47    /// so the default preserves that behavior. Providers whose durable source
48    /// identity requires a particular instance key can override the hook and
49    /// return a fatal [`ProjectionError::InvalidConfig`].
50    fn validate_projection_id(&self, _projection_id: &ProjectionId) -> ProjectionResult<()> {
51        Ok(())
52    }
53
54    /// Materializes a fresh source beginning at `cursor`.
55    ///
56    /// Inclusive-resume implementations should begin at the cursor position;
57    /// the runtime's demand-side resume filter removes already
58    /// consumed boundary events before they reach the handler.
59    async fn source_from(
60        &self,
61        cursor: Option<&ProjectionCursor>,
62        mode: QueryMode,
63    ) -> ProjectionResult<ProjectionSource<Envelope>>;
64
65    /// Extracts the complete durable cursor after processing `envelope`.
66    ///
67    /// For an inclusive position shared by multiple persistence ids, the
68    /// returned boundary should contain the identity represented by this
69    /// envelope. The runtime merges identities at equal positions.
70    fn extract_cursor(&self, envelope: &Envelope) -> ProjectionResult<ProjectionCursor>;
71
72    /// Compares two opaque provider positions in source order.
73    ///
74    /// Implementations must not use lexical ordering unless their stable
75    /// position codec is explicitly lexicographically sortable.
76    fn compare_positions(
77        &self,
78        left: &super::CursorPosition,
79        right: &super::CursorPosition,
80    ) -> ProjectionResult<Ordering>;
81
82    /// Decides whether `envelope` is after the already-consumed resume cursor.
83    ///
84    /// Providers may override this for backend-specific inclusive-resume
85    /// rules. The default drops earlier positions, accepts later positions,
86    /// and at an equal position checks the emitted envelope's own boundary
87    /// identity. A scalar cursor with no boundary identity treats equality as
88    /// consumed. An equal-position cursor containing multiple identities is
89    /// rejected as ambiguous: [`Self::extract_cursor`] must identify the one
90    /// `(persistence_id, sequence_nr)` represented by this envelope, rather
91    /// than returning a cumulative boundary set.
92    fn resume_allows(
93        &self,
94        cursor: &ProjectionCursor,
95        envelope: &Envelope,
96    ) -> ProjectionResult<bool> {
97        cursor.ensure_manifest(self.manifest())?;
98        let candidate = self.extract_cursor(envelope)?;
99        candidate.ensure_manifest(self.manifest())?;
100        match self.compare_positions(candidate.position(), cursor.position())? {
101            Ordering::Less => Ok(false),
102            Ordering::Greater => Ok(true),
103            Ordering::Equal => {
104                let mut candidate_events = candidate.boundary().consumed();
105                let Some(event) = candidate_events.next() else {
106                    return Ok(false);
107                };
108                if candidate_events.next().is_some() {
109                    return Err(ProjectionError::invalid_config(
110                        "source_provider.resume_boundary",
111                        "extract_cursor must identify exactly one boundary event per envelope",
112                    ));
113                }
114                Ok(!cursor
115                    .boundary()
116                    .contains(event.persistence_id(), event.sequence_nr()))
117            }
118        }
119    }
120}
121
122/// PostgreSQL timestamp-cursor provider over [`EventsBySlicesQuery`].
123///
124/// The provider manifest durably includes the caller-supplied query identity,
125/// entity type, inclusive slice range, and range version. A cursor written for
126/// any different range or version therefore fails with
127/// [`ProjectionError::ProviderManifestMismatch`] instead of being silently
128/// reused. `query_identity` should be a stable, non-secret identifier for the
129/// PostgreSQL journal/database (for example, a deployment database id plus
130/// schema), so cursors are not accidentally moved across databases.
131///
132/// PostgreSQL resumes timestamp offsets inclusively. The provider stores every
133/// consumed `(persistence_id, sequence_nr)` at the exact boundary in
134/// [`DeduplicationBoundary`]. PostgreSQL's projection offset store normalizes
135/// those identities into PJ4's timestamp-boundary table; no second dedup row
136/// shape is introduced here.
137///
138/// Delivery guarantees cover only emitted rows. A live PostgreSQL slice query
139/// can permanently skip an event committed by a transaction open longer than
140/// the configured visibility lag.
141#[derive(Clone)]
142pub struct EventsBySlicesSourceProvider {
143    query: Arc<dyn EventsBySlicesQuery>,
144    query_identity: String,
145    entity_type: String,
146    range: SliceRange,
147    version: u32,
148    projection_key: String,
149    manifest: ProviderManifest,
150}
151
152impl EventsBySlicesSourceProvider {
153    /// Creates an inert, restartable provider for one versioned slice range.
154    pub fn new(
155        query: Arc<dyn EventsBySlicesQuery>,
156        query_identity: impl Into<String>,
157        entity_type: impl Into<String>,
158        range: SliceRange,
159        version: u32,
160    ) -> ProjectionResult<Self> {
161        let query_identity = query_identity.into();
162        if query_identity.trim().is_empty() {
163            return Err(ProjectionError::invalid_config(
164                "events_by_slices.query_identity",
165                "query identity must not be blank",
166            ));
167        }
168        let entity_type = entity_type.into();
169        if entity_type.trim().is_empty() {
170            return Err(ProjectionError::invalid_config(
171                "events_by_slices.entity_type",
172                "entity type must not be blank",
173            ));
174        }
175        if version == 0 {
176            return Err(ProjectionError::invalid_config(
177                "events_by_slices.range_version",
178                "range version must be greater than zero",
179            ));
180        }
181        let projection_key = slice_projection_key(range, version);
182        let manifest = slice_provider_manifest(
183            &query_identity,
184            &entity_type,
185            range,
186            version,
187            &projection_key,
188        )?;
189        Ok(Self {
190            query,
191            query_identity,
192            entity_type,
193            range,
194            version,
195            projection_key,
196            manifest,
197        })
198    }
199
200    /// Returns the stable query/database identity included in the manifest.
201    #[must_use]
202    pub fn query_identity(&self) -> &str {
203        &self.query_identity
204    }
205
206    /// Returns the queried persistence entity type.
207    #[must_use]
208    pub fn entity_type(&self) -> &str {
209        &self.entity_type
210    }
211
212    /// Returns the inclusive slice range owned by this provider.
213    #[must_use]
214    pub const fn range(&self) -> SliceRange {
215        self.range
216    }
217
218    /// Returns the explicit range version.
219    #[must_use]
220    pub const fn version(&self) -> u32 {
221        self.version
222    }
223
224    /// Returns the range-derived projection key, such as `slices-256-511@v1`.
225    #[must_use]
226    pub fn projection_key(&self) -> &str {
227        &self.projection_key
228    }
229
230    fn manifest_for(&self, range: SliceRange, version: u32) -> ProjectionResult<ProviderManifest> {
231        let projection_key = slice_projection_key(range, version);
232        slice_provider_manifest(
233            &self.query_identity,
234            &self.entity_type,
235            range,
236            version,
237            &projection_key,
238        )
239    }
240}
241
242/// One stopped old-range cursor supplied to an explicit reshard seed.
243///
244/// `None` means the old range had not committed a cursor, so any new range it
245/// covers must start from the beginning.
246#[derive(Debug, Clone, PartialEq, Eq)]
247pub struct SliceRangeCursor {
248    range: SliceRange,
249    version: u32,
250    cursor: Option<ProjectionCursor>,
251}
252
253impl SliceRangeCursor {
254    /// Associates a stopped range/version with its final flushed cursor.
255    pub fn new(
256        range: SliceRange,
257        version: u32,
258        cursor: Option<ProjectionCursor>,
259    ) -> ProjectionResult<Self> {
260        if version == 0 {
261            return Err(ProjectionError::invalid_config(
262                "events_by_slices.old_range_version",
263                "range version must be greater than zero",
264            ));
265        }
266        Ok(Self {
267            range,
268            version,
269            cursor,
270        })
271    }
272
273    /// Returns the stopped range.
274    #[must_use]
275    pub const fn range(&self) -> SliceRange {
276        self.range
277    }
278
279    /// Returns the stopped range's version.
280    #[must_use]
281    pub const fn version(&self) -> u32 {
282        self.version
283    }
284
285    /// Returns the final flushed cursor, if that range had one.
286    #[must_use]
287    pub fn cursor(&self) -> Option<&ProjectionCursor> {
288        self.cursor.as_ref()
289    }
290}
291
292/// Explicitly seeds one new slice-range key during resharding.
293///
294/// The required procedure is:
295///
296/// 1. stop every old range and wait for its final cursor flush;
297/// 2. construct the new range/version provider and gather the stopped old
298///    ranges whose intersections cover it exactly;
299/// 3. call this function for the new range-derived [`ProjectionId`];
300/// 4. only after every new key is seeded, start the new projection instances.
301///
302/// This function never remaps or overwrites a live/previously seeded key. It
303/// claims the new key, validates old range/version manifests, verifies exact
304/// contiguous coverage, and persists the minimum cursor position among the
305/// covering old cursors. Equal-minimum PJ4 boundary identities are merged and
306/// filtered to the new range. If any covering old range has no cursor, the new
307/// key intentionally remains at no cursor. Starting from the minimum may
308/// replay work already processed by covering ranges with later positions, but
309/// that replay is bounded by their position spread and cannot lose events.
310///
311/// `projection_id.key()` must equal [`EventsBySlicesSourceProvider::projection_key`],
312/// making an instance-count change an explicit new-key operation.
313pub async fn seed_resharded_slice_range(
314    store: &dyn ProjectionOffsetStore,
315    projection_id: &ProjectionId,
316    provider: &EventsBySlicesSourceProvider,
317    covering_old_ranges: &[SliceRangeCursor],
318) -> ProjectionResult<Option<ProjectionCursor>> {
319    projection_id.validate()?;
320    if projection_id.key() != provider.projection_key() {
321        return Err(ProjectionError::invalid_config(
322            "events_by_slices.reshard_projection_key",
323            format!(
324                "new projection key {:?} must equal range-derived key {:?}",
325                projection_id.key(),
326                provider.projection_key()
327            ),
328        ));
329    }
330
331    let seed = resharding_seed_cursor(provider, covering_old_ranges)?;
332    if store.load(projection_id).await?.is_some() {
333        return Err(ProjectionError::invalid_config(
334            "events_by_slices.reshard_existing_cursor",
335            format!(
336                "new range key {projection_id} already has a cursor; reshard seeding never overwrites"
337            ),
338        ));
339    }
340    let generation = store.claim(projection_id).await?;
341    if store.load(projection_id).await?.is_some() {
342        return Err(ProjectionError::invalid_config(
343            "events_by_slices.reshard_existing_cursor",
344            format!(
345                "new range key {projection_id} already has a cursor; reshard seeding never overwrites"
346            ),
347        ));
348    }
349    if let Some(cursor) = seed.clone() {
350        store.save(projection_id, generation, None, cursor).await?;
351    }
352    Ok(seed)
353}
354
355fn resharding_seed_cursor(
356    provider: &EventsBySlicesSourceProvider,
357    covering_old_ranges: &[SliceRangeCursor],
358) -> ProjectionResult<Option<ProjectionCursor>> {
359    validate_reshard_coverage(provider.range, covering_old_ranges)?;
360    let mut cursors = Vec::with_capacity(covering_old_ranges.len());
361    let mut any_absent = false;
362    for old in covering_old_ranges {
363        let expected_manifest = provider.manifest_for(old.range, old.version)?;
364        if let Some(cursor) = &old.cursor {
365            cursor.ensure_manifest(&expected_manifest)?;
366            provider.compare_positions(cursor.position(), cursor.position())?;
367            cursors.push(cursor);
368        } else {
369            any_absent = true;
370        }
371    }
372    if any_absent {
373        return Ok(None);
374    }
375
376    let mut minimum = cursors
377        .first()
378        .expect("coverage validation requires at least one old range")
379        .position();
380    for cursor in cursors.iter().skip(1) {
381        if provider.compare_positions(cursor.position(), minimum)? == Ordering::Less {
382            minimum = cursor.position();
383        }
384    }
385
386    let mut consumed = Vec::new();
387    for cursor in cursors {
388        if provider.compare_positions(cursor.position(), minimum)? == Ordering::Equal {
389            consumed.extend(
390                cursor
391                    .boundary()
392                    .consumed()
393                    .filter(|event| {
394                        provider
395                            .range
396                            .contains(slice_for(event.persistence_id().id()))
397                    })
398                    .cloned(),
399            );
400        }
401    }
402    Ok(Some(ProjectionCursor::new(
403        provider.manifest.clone(),
404        DeduplicationBoundary::with_consumed(minimum.clone(), consumed),
405    )))
406}
407
408fn validate_reshard_coverage(
409    new_range: SliceRange,
410    covering_old_ranges: &[SliceRangeCursor],
411) -> ProjectionResult<()> {
412    let mut intersections = covering_old_ranges
413        .iter()
414        .map(|old| {
415            let min = old.range.min().max(new_range.min());
416            let max = old.range.max().min(new_range.max());
417            if min > max {
418                return Err(ProjectionError::invalid_config(
419                    "events_by_slices.reshard_coverage",
420                    format!(
421                        "old range {} does not intersect new range {}",
422                        slice_projection_key(old.range, old.version),
423                        slice_projection_key(new_range, 1)
424                    ),
425                ));
426            }
427            Ok((min, max))
428        })
429        .collect::<ProjectionResult<Vec<_>>>()?;
430    intersections.sort_unstable();
431
432    let mut expected = new_range.min();
433    for (min, max) in intersections {
434        if min != expected {
435            let reason = if min < expected {
436                format!("old range intersections overlap at slice {min}")
437            } else {
438                format!("old range intersections leave a gap beginning at slice {expected}")
439            };
440            return Err(ProjectionError::invalid_config(
441                "events_by_slices.reshard_coverage",
442                reason,
443            ));
444        }
445        expected = max + 1;
446    }
447    if expected != new_range.max() + 1 {
448        return Err(ProjectionError::invalid_config(
449            "events_by_slices.reshard_coverage",
450            format!(
451                "old range intersections stop before slice {}",
452                new_range.max()
453            ),
454        ));
455    }
456    Ok(())
457}
458
459#[async_trait::async_trait]
460impl SourceProvider<EventEnvelope> for EventsBySlicesSourceProvider {
461    fn manifest(&self) -> &ProviderManifest {
462        &self.manifest
463    }
464
465    fn validate_projection_id(&self, projection_id: &ProjectionId) -> ProjectionResult<()> {
466        if projection_id.key() != self.projection_key {
467            return Err(ProjectionError::invalid_config(
468                "events_by_slices.projection_key",
469                format!(
470                    "projection key {:?} must equal range-derived key {:?}",
471                    projection_id.key(),
472                    self.projection_key
473                ),
474            ));
475        }
476        Ok(())
477    }
478
479    async fn source_from(
480        &self,
481        cursor: Option<&ProjectionCursor>,
482        mode: QueryMode,
483    ) -> ProjectionResult<ProjectionSource<EventEnvelope>> {
484        let offset = cursor.map_or(Ok(Offset::NoOffset), |cursor| {
485            cursor.ensure_manifest(&self.manifest)?;
486            decode_timestamp_position(cursor.position())
487        })?;
488        let stream = self
489            .query
490            .events_by_slices(
491                &self.entity_type,
492                self.range.min(),
493                self.range.max(),
494                offset,
495                mode,
496            )
497            .await
498            .map_err(|error| ProjectionError::Source(error.to_string()))?;
499        Ok(Box::pin(stream.map(|item| {
500            item.map_err(|error| ProjectionError::Source(error.to_string()))
501        })))
502    }
503
504    fn extract_cursor(&self, envelope: &EventEnvelope) -> ProjectionResult<ProjectionCursor> {
505        let Offset::Timestamp {
506            epoch_micros,
507            seq_nr,
508        } = envelope.offset
509        else {
510            return Err(ProjectionError::Source(format!(
511                "PostgreSQL events-by-slices emitted non-timestamp offset {:?}",
512                envelope.offset
513            )));
514        };
515        if envelope.persistence_id.entity_type() != self.entity_type {
516            return Err(ProjectionError::Source(format!(
517                "events-by-slices provider for entity type {:?} received persistence id {}",
518                self.entity_type, envelope.persistence_id
519            )));
520        }
521        let actual_slice = slice_for(envelope.persistence_id.id());
522        if !self.range.contains(actual_slice) {
523            return Err(ProjectionError::Source(format!(
524                "events-by-slices provider for {} received persistence id {} in slice {actual_slice}",
525                self.projection_key, envelope.persistence_id
526            )));
527        }
528        Ok(ProjectionCursor::new(
529            self.manifest.clone(),
530            DeduplicationBoundary::with_consumed(
531                encode_timestamp_position(epoch_micros, seq_nr),
532                [BoundaryEvent::new(
533                    envelope.persistence_id.clone(),
534                    envelope.sequence_nr,
535                )],
536            ),
537        ))
538    }
539
540    fn compare_positions(
541        &self,
542        left: &CursorPosition,
543        right: &CursorPosition,
544    ) -> ProjectionResult<Ordering> {
545        Ok(decode_timestamp_position_value(left)?.cmp(&decode_timestamp_position_value(right)?))
546    }
547}
548
549/// Returns the mandatory range-derived, versioned projection key.
550#[must_use]
551pub fn slice_projection_key(range: SliceRange, version: u32) -> String {
552    format!("slices-{}-{}@v{version}", range.min(), range.max())
553}
554
555fn slice_provider_manifest(
556    query_identity: &str,
557    entity_type: &str,
558    range: SliceRange,
559    version: u32,
560    projection_key: &str,
561) -> ProjectionResult<ProviderManifest> {
562    ProviderManifest::new(format!(
563        "{SLICE_PROVIDER_MANIFEST_VERSION}|query={}|entity={}|range={}-{}|version={version}|key={projection_key}",
564        encode_utf8_hex(query_identity),
565        encode_utf8_hex(entity_type),
566        range.min(),
567        range.max(),
568    ))
569}
570
571fn encode_timestamp_position(epoch_micros: i64, sequence_nr: u64) -> CursorPosition {
572    CursorPosition::new(format!(
573        "{TIMESTAMP_POSITION_VERSION}|{epoch_micros}|{sequence_nr}"
574    ))
575}
576
577fn decode_timestamp_position(position: &CursorPosition) -> ProjectionResult<Offset> {
578    let (epoch_micros, seq_nr) = decode_timestamp_position_value(position)?;
579    Ok(Offset::Timestamp {
580        epoch_micros,
581        seq_nr,
582    })
583}
584
585fn decode_timestamp_position_value(position: &CursorPosition) -> ProjectionResult<(i64, u64)> {
586    let mut fields = position.as_str().split('|');
587    let version = fields.next();
588    let epoch_micros = fields.next();
589    let sequence_nr = fields.next();
590    if version != Some(TIMESTAMP_POSITION_VERSION)
591        || epoch_micros.is_none()
592        || sequence_nr.is_none()
593        || fields.next().is_some()
594    {
595        return Err(ProjectionError::CursorCodec(format!(
596            "invalid PostgreSQL timestamp cursor position {:?}",
597            position.as_str()
598        )));
599    }
600    let epoch_micros = epoch_micros
601        .expect("presence was checked")
602        .parse::<i64>()
603        .map_err(|_| {
604            ProjectionError::CursorCodec(format!(
605                "invalid PostgreSQL timestamp micros in position {:?}",
606                position.as_str()
607            ))
608        })?;
609    let sequence_nr = sequence_nr
610        .expect("presence was checked")
611        .parse::<u64>()
612        .map_err(|_| {
613            ProjectionError::CursorCodec(format!(
614                "invalid PostgreSQL timestamp sequence in position {:?}",
615                position.as_str()
616            ))
617        })?;
618    Ok((epoch_micros, sequence_nr))
619}
620
621fn encode_utf8_hex(value: &str) -> String {
622    const HEX: &[u8; 16] = b"0123456789abcdef";
623    let mut encoded = String::with_capacity(value.len().saturating_mul(2));
624    for byte in value.bytes() {
625        encoded.push(char::from(HEX[usize::from(byte >> 4)]));
626        encoded.push(char::from(HEX[usize::from(byte & 0x0f)]));
627    }
628    encoded
629}
630
631/// Returns whether an envelope is after the already-consumed resume boundary.
632///
633/// Positions before the cursor are always dropped. At an equal position,
634/// provider-complete boundary identities are checked individually so unseen
635/// persistence ids sharing that exact position remain eligible.
636pub(crate) fn resume_allows<Envelope: 'static>(
637    provider: &dyn SourceProvider<Envelope>,
638    cursor: Option<&ProjectionCursor>,
639    envelope: &Envelope,
640) -> ProjectionResult<bool> {
641    let Some(cursor) = cursor else {
642        return Ok(true);
643    };
644    provider.resume_allows(cursor, envelope)
645}
646
647/// Advances a logical cursor and merges dedup identities at one exact position.
648pub(crate) fn advance_cursor<Envelope: 'static>(
649    provider: &dyn SourceProvider<Envelope>,
650    current: Option<&ProjectionCursor>,
651    envelope: &Envelope,
652) -> ProjectionResult<ProjectionCursor> {
653    let next = provider.extract_cursor(envelope)?;
654    next.ensure_manifest(provider.manifest())?;
655    let Some(current) = current else {
656        return Ok(next);
657    };
658    current.ensure_manifest(provider.manifest())?;
659
660    match provider.compare_positions(next.position(), current.position())? {
661        Ordering::Less => Err(ProjectionError::Source(format!(
662            "provider {} emitted cursor position {} after later position {}",
663            provider.manifest(),
664            next.position(),
665            current.position()
666        ))),
667        Ordering::Greater => Ok(next),
668        Ordering::Equal => {
669            let consumed = current
670                .boundary()
671                .consumed()
672                .chain(next.boundary().consumed())
673                .cloned();
674            Ok(ProjectionCursor::new(
675                provider.manifest().clone(),
676                DeduplicationBoundary::with_consumed(current.position().clone(), consumed),
677            ))
678        }
679    }
680}
681
682#[cfg(test)]
683mod tests {
684    use std::sync::Mutex;
685
686    use futures_util::stream;
687
688    use crate::projection::{
689        GroupedHandler, Handler, MemoryProjectionOffsetStore, Projection, ProjectionRuntime,
690        ProjectionStatus, TestEnvelope, TestSourceProvider,
691    };
692    use crate::query::QueryStream;
693    use crate::{PersistenceId, SequenceNr, SerializedPayload, SerializerId};
694
695    use super::*;
696
697    type RecordedCall = (String, u16, u16, Offset, QueryMode);
698
699    struct RecordingSliceQuery {
700        calls: Mutex<Vec<RecordedCall>>,
701    }
702
703    #[async_trait::async_trait]
704    impl EventsBySlicesQuery for RecordingSliceQuery {
705        async fn events_by_slices(
706            &self,
707            entity_type: &str,
708            min_slice: u16,
709            max_slice: u16,
710            offset: Offset,
711            mode: QueryMode,
712        ) -> crate::PersistenceResult<QueryStream<EventEnvelope>> {
713            self.calls.lock().unwrap().push((
714                entity_type.to_owned(),
715                min_slice,
716                max_slice,
717                offset,
718                mode,
719            ));
720            Ok(Box::pin(stream::empty()))
721        }
722    }
723
724    struct NoopHandler;
725
726    #[async_trait::async_trait]
727    impl Handler<EventEnvelope> for NoopHandler {
728        async fn process(&mut self, _envelope: &EventEnvelope) -> ProjectionResult<()> {
729            Ok(())
730        }
731    }
732
733    struct NoopGroupedHandler;
734
735    #[async_trait::async_trait]
736    impl GroupedHandler<EventEnvelope> for NoopGroupedHandler {
737        async fn process(&mut self, _envelopes: &[EventEnvelope]) -> ProjectionResult<()> {
738            Ok(())
739        }
740    }
741
742    struct NoopTestHandler;
743
744    #[async_trait::async_trait]
745    impl Handler<TestEnvelope<usize>> for NoopTestHandler {
746        async fn process(&mut self, _envelope: &TestEnvelope<usize>) -> ProjectionResult<()> {
747            Ok(())
748        }
749    }
750
751    fn envelope(persistence_id: PersistenceId, epoch_micros: i64) -> EventEnvelope {
752        EventEnvelope {
753            offset: Offset::Timestamp {
754                epoch_micros,
755                seq_nr: 7,
756            },
757            persistence_id,
758            sequence_nr: SequenceNr::new(7),
759            timestamp_ms: 1,
760            adapter_manifest: "adapter-v1".into(),
761            payload: SerializedPayload::new(SerializerId::new(1), "event-v1", vec![]),
762            metadata: None,
763            tags: Vec::new(),
764        }
765    }
766
767    fn id_in(range: SliceRange) -> PersistenceId {
768        (0..10_000)
769            .map(|suffix| PersistenceId::new("account", suffix.to_string()).unwrap())
770            .find(|persistence_id| range.contains(slice_for(persistence_id.id())))
771            .expect("fixture search finds an id in the range")
772    }
773
774    fn id_in_but_not(in_range: SliceRange, excluded: SliceRange) -> PersistenceId {
775        (0..10_000)
776            .map(|suffix| PersistenceId::new("account", format!("outside-{suffix}")).unwrap())
777            .find(|persistence_id| {
778                let slice = slice_for(persistence_id.id());
779                in_range.contains(slice) && !excluded.contains(slice)
780            })
781            .expect("fixture search finds an id in only the old range")
782    }
783
784    #[tokio::test]
785    async fn range_version_is_in_key_manifest_and_query_resume() {
786        let query = Arc::new(RecordingSliceQuery {
787            calls: Mutex::new(Vec::new()),
788        });
789        let range = SliceRange::new(256, 511).unwrap();
790        let provider = EventsBySlicesSourceProvider::new(
791            query.clone(),
792            "postgres:test-db/public",
793            "account",
794            range,
795            1,
796        )
797        .unwrap();
798        assert_eq!(provider.projection_key(), "slices-256-511@v1");
799        assert_eq!(
800            provider.manifest().as_str(),
801            "datum-postgres-events-by-slices-v1|query=706f7374677265733a746573742d64622f7075626c6963|entity=6163636f756e74|range=256-511|version=1|key=slices-256-511@v1"
802        );
803
804        let persistence_id = PersistenceId::new("account", "0").unwrap();
805        assert_eq!(slice_for(persistence_id.id()), 336);
806        let event = envelope(persistence_id, 123);
807        let cursor = provider.extract_cursor(&event).unwrap();
808        assert_eq!(cursor.boundary().consumed().len(), 1);
809        assert_eq!(cursor.position().as_str(), "pg-timestamp-v1|123|7");
810        assert_eq!(
811            cursor.encode(),
812            "datum-projection-cursor-v1|646174756d2d706f7374677265732d6576656e74732d62792d736c696365732d76317c71756572793d373036663733373436373732363537333361373436353733373432643634363232663730373536323663363936337c656e746974793d36313633363336663735366537347c72616e67653d3235362d3531317c76657273696f6e3d317c6b65793d736c696365732d3235362d353131407631|70672d74696d657374616d702d76317c3132337c37|1|6163636f756e747c30:7"
813        );
814        let _source = provider
815            .source_from(Some(&cursor), QueryMode::Current)
816            .await
817            .unwrap();
818        assert_eq!(
819            query.calls.lock().unwrap().as_slice(),
820            [(
821                "account".to_owned(),
822                256,
823                511,
824                Offset::Timestamp {
825                    epoch_micros: 123,
826                    seq_nr: 7,
827                },
828                QueryMode::Current,
829            )]
830        );
831    }
832
833    #[tokio::test]
834    async fn normal_spawn_enforces_slice_key_for_every_builder_before_materialization() {
835        let query = Arc::new(RecordingSliceQuery {
836            calls: Mutex::new(Vec::new()),
837        });
838        let range = SliceRange::new(256, 511).unwrap();
839        let provider =
840            EventsBySlicesSourceProvider::new(query.clone(), "postgres:test", "account", range, 3)
841                .unwrap();
842        let store = Arc::new(MemoryProjectionOffsetStore::new());
843        let runtime = ProjectionRuntime::new(Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>);
844        let wrong_key = "manual";
845        let projections = [
846            Projection::at_least_once(
847                ProjectionId::new("slice-key-alo", wrong_key),
848                provider.clone(),
849                || NoopHandler,
850            ),
851            Projection::at_most_once(
852                ProjectionId::new("slice-key-amo", wrong_key),
853                provider.clone(),
854                || NoopHandler,
855            ),
856            Projection::grouped_at_least_once(
857                ProjectionId::new("slice-key-grouped", wrong_key),
858                provider.clone(),
859                || NoopGroupedHandler,
860            ),
861        ];
862
863        for projection in projections {
864            let error = match projection.spawn(runtime.clone()).await {
865                Ok(_) => panic!("wrong range-derived key must fail normal spawn"),
866                Err(error) => error,
867            };
868            assert!(matches!(
869                error,
870                ProjectionError::InvalidConfig {
871                    setting: "events_by_slices.projection_key",
872                    ..
873                }
874            ));
875            assert!(error.is_fatal());
876        }
877        assert!(
878            query.calls.lock().unwrap().is_empty(),
879            "projection-id validation must run before source materialization"
880        );
881
882        let correct_provider =
883            EventsBySlicesSourceProvider::new(query.clone(), "postgres:test", "account", range, 3)
884                .unwrap();
885        let correct_id = ProjectionId::new("slice-key-correct", correct_provider.projection_key());
886        let correct = Projection::at_least_once(correct_id, correct_provider, || NoopHandler)
887            .with_query_mode(QueryMode::Current)
888            .spawn(runtime.clone())
889            .await
890            .expect("range-derived key is accepted");
891        assert_eq!(correct.wait().await.unwrap(), ProjectionStatus::Completed);
892        assert_eq!(query.calls.lock().unwrap().len(), 1);
893
894        let unrestricted = Projection::at_least_once(
895            ProjectionId::new("non-slice-provider", "manual-key"),
896            TestSourceProvider::new(Vec::<TestEnvelope<usize>>::new()),
897            || NoopTestHandler,
898        )
899        .with_query_mode(QueryMode::Current)
900        .spawn(runtime)
901        .await
902        .expect("the default provider hook preserves arbitrary valid keys");
903        assert_eq!(
904            unrestricted.wait().await.unwrap(),
905            ProjectionStatus::Completed
906        );
907    }
908
909    #[tokio::test]
910    async fn range_or_version_change_is_a_fatal_manifest_mismatch() {
911        let query = Arc::new(RecordingSliceQuery {
912            calls: Mutex::new(Vec::new()),
913        });
914        let old_range = SliceRange::new(0, 511).unwrap();
915        let old = EventsBySlicesSourceProvider::new(
916            query.clone(),
917            "postgres:test-db/public",
918            "account",
919            old_range,
920            1,
921        )
922        .unwrap();
923        let cursor = old
924            .extract_cursor(&envelope(id_in(old_range), 123))
925            .unwrap();
926
927        for (range, version) in [(SliceRange::new(0, 255).unwrap(), 1), (old_range, 2)] {
928            let changed = EventsBySlicesSourceProvider::new(
929                query.clone(),
930                "postgres:test-db/public",
931                "account",
932                range,
933                version,
934            )
935            .unwrap();
936            let error = match changed.source_from(Some(&cursor), QueryMode::Live).await {
937                Ok(_) => panic!("changed provider must reject the old cursor"),
938                Err(error) => error,
939            };
940            assert!(matches!(
941                error,
942                ProjectionError::ProviderManifestMismatch { .. }
943            ));
944            assert!(error.is_fatal());
945        }
946    }
947
948    #[test]
949    fn provider_rejects_invalid_identity_version_and_envelopes() {
950        let query: Arc<dyn EventsBySlicesQuery> = Arc::new(RecordingSliceQuery {
951            calls: Mutex::new(Vec::new()),
952        });
953        let range = SliceRange::new(0, 127).unwrap();
954        assert!(EventsBySlicesSourceProvider::new(query.clone(), "", "account", range, 1).is_err());
955        assert!(
956            EventsBySlicesSourceProvider::new(query.clone(), "postgres:test", "", range, 1)
957                .is_err()
958        );
959        assert!(
960            EventsBySlicesSourceProvider::new(query.clone(), "postgres:test", "account", range, 0)
961                .is_err()
962        );
963
964        let provider =
965            EventsBySlicesSourceProvider::new(query, "postgres:test", "account", range, 1).unwrap();
966        let wrong = PersistenceId::new("other", "id").unwrap();
967        let error = provider
968            .extract_cursor(&envelope(wrong, 1))
969            .expect_err("wrong entity type is a source failure");
970        assert!(matches!(error, ProjectionError::Source(_)));
971
972        let malformed = ProjectionCursor::from_position(
973            provider.manifest().clone(),
974            "not-a-timestamp-position",
975        );
976        let error = provider
977            .compare_positions(malformed.position(), &CursorPosition::new("also-invalid"))
978            .expect_err("malformed position is fatal");
979        assert!(matches!(error, ProjectionError::CursorCodec(_)));
980    }
981
982    #[tokio::test]
983    async fn four_to_eight_seed_persists_new_manifest_and_filters_boundary() {
984        let query: Arc<dyn EventsBySlicesQuery> = Arc::new(RecordingSliceQuery {
985            calls: Mutex::new(Vec::new()),
986        });
987        let old_range = SliceRange::new(0, 255).unwrap();
988        let new_range = SliceRange::new(0, 127).unwrap();
989        let old = EventsBySlicesSourceProvider::new(
990            query.clone(),
991            "postgres:test",
992            "account",
993            old_range,
994            1,
995        )
996        .unwrap();
997        let new =
998            EventsBySlicesSourceProvider::new(query, "postgres:test", "account", new_range, 2)
999                .unwrap();
1000        let included = id_in(new_range);
1001        let excluded = id_in_but_not(old_range, new_range);
1002        let base = old
1003            .extract_cursor(&envelope(included.clone(), 100))
1004            .unwrap();
1005        let old_cursor = ProjectionCursor::new(
1006            old.manifest().clone(),
1007            DeduplicationBoundary::with_consumed(
1008                base.position().clone(),
1009                [
1010                    BoundaryEvent::new(included.clone(), SequenceNr::new(7)),
1011                    BoundaryEvent::new(excluded, SequenceNr::new(7)),
1012                ],
1013            ),
1014        );
1015        let covering = [SliceRangeCursor::new(old_range, 1, Some(old_cursor)).unwrap()];
1016        let projection_id = ProjectionId::new("accounts", new.projection_key());
1017        let store = MemoryProjectionOffsetStore::new();
1018
1019        let seeded = seed_resharded_slice_range(&store, &projection_id, &new, &covering)
1020            .await
1021            .unwrap()
1022            .expect("old range had a cursor");
1023        assert_eq!(seeded.provider_manifest(), new.manifest());
1024        assert_eq!(seeded.boundary().consumed().len(), 1);
1025        assert!(seeded.boundary().contains(&included, SequenceNr::new(7)));
1026        assert_eq!(store.load(&projection_id).await.unwrap(), Some(seeded));
1027
1028        let error = seed_resharded_slice_range(&store, &projection_id, &new, &covering)
1029            .await
1030            .expect_err("seed never overwrites an existing new-range cursor");
1031        assert!(matches!(error, ProjectionError::InvalidConfig { .. }));
1032    }
1033
1034    #[test]
1035    fn seed_takes_minimum_covering_position_and_validates_coverage() {
1036        let query: Arc<dyn EventsBySlicesQuery> = Arc::new(RecordingSliceQuery {
1037            calls: Mutex::new(Vec::new()),
1038        });
1039        let left_range = SliceRange::new(0, 127).unwrap();
1040        let right_range = SliceRange::new(128, 255).unwrap();
1041        let new_range = SliceRange::new(0, 255).unwrap();
1042        let left = EventsBySlicesSourceProvider::new(
1043            query.clone(),
1044            "postgres:test",
1045            "account",
1046            left_range,
1047            1,
1048        )
1049        .unwrap();
1050        let right = EventsBySlicesSourceProvider::new(
1051            query.clone(),
1052            "postgres:test",
1053            "account",
1054            right_range,
1055            1,
1056        )
1057        .unwrap();
1058        let new =
1059            EventsBySlicesSourceProvider::new(query, "postgres:test", "account", new_range, 2)
1060                .unwrap();
1061        let covering = [
1062            SliceRangeCursor::new(
1063                left_range,
1064                1,
1065                Some(
1066                    left.extract_cursor(&envelope(id_in(left_range), 120))
1067                        .unwrap(),
1068                ),
1069            )
1070            .unwrap(),
1071            SliceRangeCursor::new(
1072                right_range,
1073                1,
1074                Some(
1075                    right
1076                        .extract_cursor(&envelope(id_in(right_range), 100))
1077                        .unwrap(),
1078                ),
1079            )
1080            .unwrap(),
1081        ];
1082        let seed = resharding_seed_cursor(&new, &covering)
1083            .unwrap()
1084            .expect("both old ranges had cursors");
1085        assert_eq!(
1086            decode_timestamp_position_value(seed.position()).unwrap(),
1087            (100, 7)
1088        );
1089
1090        let all_absent = [
1091            SliceRangeCursor::new(left_range, 1, None).unwrap(),
1092            SliceRangeCursor::new(right_range, 1, None).unwrap(),
1093        ];
1094        assert_eq!(
1095            resharding_seed_cursor(&new, &all_absent).unwrap(),
1096            None,
1097            "all unstarted covering ranges seed from the beginning"
1098        );
1099        let gap = resharding_seed_cursor(&new, &covering[..1])
1100            .expect_err("partial coverage must be rejected");
1101        assert!(matches!(gap, ProjectionError::InvalidConfig { .. }));
1102    }
1103
1104    #[test]
1105    fn absent_reshard_cursor_never_hides_a_wrong_range_cursor_in_either_order() {
1106        let query: Arc<dyn EventsBySlicesQuery> = Arc::new(RecordingSliceQuery {
1107            calls: Mutex::new(Vec::new()),
1108        });
1109        let left_range = SliceRange::new(0, 127).unwrap();
1110        let right_range = SliceRange::new(128, 255).unwrap();
1111        let new_range = SliceRange::new(0, 255).unwrap();
1112        let left = EventsBySlicesSourceProvider::new(
1113            query.clone(),
1114            "postgres:test",
1115            "account",
1116            left_range,
1117            1,
1118        )
1119        .unwrap();
1120        let new =
1121            EventsBySlicesSourceProvider::new(query, "postgres:test", "account", new_range, 2)
1122                .unwrap();
1123        let cursor_for_left = left
1124            .extract_cursor(&envelope(id_in(left_range), 100))
1125            .unwrap();
1126        let absent_left = SliceRangeCursor::new(left_range, 1, None).unwrap();
1127        let wrong_right = SliceRangeCursor::new(right_range, 1, Some(cursor_for_left)).unwrap();
1128
1129        for covering in [
1130            vec![absent_left.clone(), wrong_right.clone()],
1131            vec![wrong_right.clone(), absent_left.clone()],
1132        ] {
1133            let error = resharding_seed_cursor(&new, &covering)
1134                .expect_err("every present old cursor must be validated before absence wins");
1135            assert!(matches!(
1136                error,
1137                ProjectionError::ProviderManifestMismatch { .. }
1138            ));
1139            assert!(error.is_fatal());
1140        }
1141    }
1142}