use std::{cmp::Ordering, pin::Pin, sync::Arc};
use futures_core::Stream;
use futures_util::StreamExt;
use crate::query::{EventEnvelope, EventsBySlicesQuery, Offset, QueryMode, SliceRange, slice_for};
use super::{
BoundaryEvent, CursorPosition, DeduplicationBoundary, ProjectionCursor, ProjectionError,
ProjectionId, ProjectionOffsetStore, ProjectionResult, ProviderManifest,
};
const SLICE_PROVIDER_MANIFEST_VERSION: &str = "datum-postgres-events-by-slices-v1";
const TIMESTAMP_POSITION_VERSION: &str = "pg-timestamp-v1";
pub type ProjectionSource<Envelope> =
Pin<Box<dyn Stream<Item = ProjectionResult<Envelope>> + Send + 'static>>;
#[async_trait::async_trait]
pub trait SourceProvider<Envelope>: Send + Sync + 'static {
fn manifest(&self) -> &ProviderManifest;
fn validate_projection_id(&self, _projection_id: &ProjectionId) -> ProjectionResult<()> {
Ok(())
}
async fn source_from(
&self,
cursor: Option<&ProjectionCursor>,
mode: QueryMode,
) -> ProjectionResult<ProjectionSource<Envelope>>;
fn extract_cursor(&self, envelope: &Envelope) -> ProjectionResult<ProjectionCursor>;
fn compare_positions(
&self,
left: &super::CursorPosition,
right: &super::CursorPosition,
) -> ProjectionResult<Ordering>;
fn resume_allows(
&self,
cursor: &ProjectionCursor,
envelope: &Envelope,
) -> ProjectionResult<bool> {
cursor.ensure_manifest(self.manifest())?;
let candidate = self.extract_cursor(envelope)?;
candidate.ensure_manifest(self.manifest())?;
match self.compare_positions(candidate.position(), cursor.position())? {
Ordering::Less => Ok(false),
Ordering::Greater => Ok(true),
Ordering::Equal => {
let mut candidate_events = candidate.boundary().consumed();
let Some(event) = candidate_events.next() else {
return Ok(false);
};
if candidate_events.next().is_some() {
return Err(ProjectionError::invalid_config(
"source_provider.resume_boundary",
"extract_cursor must identify exactly one boundary event per envelope",
));
}
Ok(!cursor
.boundary()
.contains(event.persistence_id(), event.sequence_nr()))
}
}
}
}
#[derive(Clone)]
pub struct EventsBySlicesSourceProvider {
query: Arc<dyn EventsBySlicesQuery>,
query_identity: String,
entity_type: String,
range: SliceRange,
version: u32,
projection_key: String,
manifest: ProviderManifest,
}
impl EventsBySlicesSourceProvider {
pub fn new(
query: Arc<dyn EventsBySlicesQuery>,
query_identity: impl Into<String>,
entity_type: impl Into<String>,
range: SliceRange,
version: u32,
) -> ProjectionResult<Self> {
let query_identity = query_identity.into();
if query_identity.trim().is_empty() {
return Err(ProjectionError::invalid_config(
"events_by_slices.query_identity",
"query identity must not be blank",
));
}
let entity_type = entity_type.into();
if entity_type.trim().is_empty() {
return Err(ProjectionError::invalid_config(
"events_by_slices.entity_type",
"entity type must not be blank",
));
}
if version == 0 {
return Err(ProjectionError::invalid_config(
"events_by_slices.range_version",
"range version must be greater than zero",
));
}
let projection_key = slice_projection_key(range, version);
let manifest = slice_provider_manifest(
&query_identity,
&entity_type,
range,
version,
&projection_key,
)?;
Ok(Self {
query,
query_identity,
entity_type,
range,
version,
projection_key,
manifest,
})
}
#[must_use]
pub fn query_identity(&self) -> &str {
&self.query_identity
}
#[must_use]
pub fn entity_type(&self) -> &str {
&self.entity_type
}
#[must_use]
pub const fn range(&self) -> SliceRange {
self.range
}
#[must_use]
pub const fn version(&self) -> u32 {
self.version
}
#[must_use]
pub fn projection_key(&self) -> &str {
&self.projection_key
}
fn manifest_for(&self, range: SliceRange, version: u32) -> ProjectionResult<ProviderManifest> {
let projection_key = slice_projection_key(range, version);
slice_provider_manifest(
&self.query_identity,
&self.entity_type,
range,
version,
&projection_key,
)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SliceRangeCursor {
range: SliceRange,
version: u32,
cursor: Option<ProjectionCursor>,
}
impl SliceRangeCursor {
pub fn new(
range: SliceRange,
version: u32,
cursor: Option<ProjectionCursor>,
) -> ProjectionResult<Self> {
if version == 0 {
return Err(ProjectionError::invalid_config(
"events_by_slices.old_range_version",
"range version must be greater than zero",
));
}
Ok(Self {
range,
version,
cursor,
})
}
#[must_use]
pub const fn range(&self) -> SliceRange {
self.range
}
#[must_use]
pub const fn version(&self) -> u32 {
self.version
}
#[must_use]
pub fn cursor(&self) -> Option<&ProjectionCursor> {
self.cursor.as_ref()
}
}
pub async fn seed_resharded_slice_range(
store: &dyn ProjectionOffsetStore,
projection_id: &ProjectionId,
provider: &EventsBySlicesSourceProvider,
covering_old_ranges: &[SliceRangeCursor],
) -> ProjectionResult<Option<ProjectionCursor>> {
projection_id.validate()?;
if projection_id.key() != provider.projection_key() {
return Err(ProjectionError::invalid_config(
"events_by_slices.reshard_projection_key",
format!(
"new projection key {:?} must equal range-derived key {:?}",
projection_id.key(),
provider.projection_key()
),
));
}
let seed = resharding_seed_cursor(provider, covering_old_ranges)?;
if store.load(projection_id).await?.is_some() {
return Err(ProjectionError::invalid_config(
"events_by_slices.reshard_existing_cursor",
format!(
"new range key {projection_id} already has a cursor; reshard seeding never overwrites"
),
));
}
let generation = store.claim(projection_id).await?;
if store.load(projection_id).await?.is_some() {
return Err(ProjectionError::invalid_config(
"events_by_slices.reshard_existing_cursor",
format!(
"new range key {projection_id} already has a cursor; reshard seeding never overwrites"
),
));
}
if let Some(cursor) = seed.clone() {
store.save(projection_id, generation, None, cursor).await?;
}
Ok(seed)
}
fn resharding_seed_cursor(
provider: &EventsBySlicesSourceProvider,
covering_old_ranges: &[SliceRangeCursor],
) -> ProjectionResult<Option<ProjectionCursor>> {
validate_reshard_coverage(provider.range, covering_old_ranges)?;
let mut cursors = Vec::with_capacity(covering_old_ranges.len());
let mut any_absent = false;
for old in covering_old_ranges {
let expected_manifest = provider.manifest_for(old.range, old.version)?;
if let Some(cursor) = &old.cursor {
cursor.ensure_manifest(&expected_manifest)?;
provider.compare_positions(cursor.position(), cursor.position())?;
cursors.push(cursor);
} else {
any_absent = true;
}
}
if any_absent {
return Ok(None);
}
let mut minimum = cursors
.first()
.expect("coverage validation requires at least one old range")
.position();
for cursor in cursors.iter().skip(1) {
if provider.compare_positions(cursor.position(), minimum)? == Ordering::Less {
minimum = cursor.position();
}
}
let mut consumed = Vec::new();
for cursor in cursors {
if provider.compare_positions(cursor.position(), minimum)? == Ordering::Equal {
consumed.extend(
cursor
.boundary()
.consumed()
.filter(|event| {
provider
.range
.contains(slice_for(event.persistence_id().id()))
})
.cloned(),
);
}
}
Ok(Some(ProjectionCursor::new(
provider.manifest.clone(),
DeduplicationBoundary::with_consumed(minimum.clone(), consumed),
)))
}
fn validate_reshard_coverage(
new_range: SliceRange,
covering_old_ranges: &[SliceRangeCursor],
) -> ProjectionResult<()> {
let mut intersections = covering_old_ranges
.iter()
.map(|old| {
let min = old.range.min().max(new_range.min());
let max = old.range.max().min(new_range.max());
if min > max {
return Err(ProjectionError::invalid_config(
"events_by_slices.reshard_coverage",
format!(
"old range {} does not intersect new range {}",
slice_projection_key(old.range, old.version),
slice_projection_key(new_range, 1)
),
));
}
Ok((min, max))
})
.collect::<ProjectionResult<Vec<_>>>()?;
intersections.sort_unstable();
let mut expected = new_range.min();
for (min, max) in intersections {
if min != expected {
let reason = if min < expected {
format!("old range intersections overlap at slice {min}")
} else {
format!("old range intersections leave a gap beginning at slice {expected}")
};
return Err(ProjectionError::invalid_config(
"events_by_slices.reshard_coverage",
reason,
));
}
expected = max + 1;
}
if expected != new_range.max() + 1 {
return Err(ProjectionError::invalid_config(
"events_by_slices.reshard_coverage",
format!(
"old range intersections stop before slice {}",
new_range.max()
),
));
}
Ok(())
}
#[async_trait::async_trait]
impl SourceProvider<EventEnvelope> for EventsBySlicesSourceProvider {
fn manifest(&self) -> &ProviderManifest {
&self.manifest
}
fn validate_projection_id(&self, projection_id: &ProjectionId) -> ProjectionResult<()> {
if projection_id.key() != self.projection_key {
return Err(ProjectionError::invalid_config(
"events_by_slices.projection_key",
format!(
"projection key {:?} must equal range-derived key {:?}",
projection_id.key(),
self.projection_key
),
));
}
Ok(())
}
async fn source_from(
&self,
cursor: Option<&ProjectionCursor>,
mode: QueryMode,
) -> ProjectionResult<ProjectionSource<EventEnvelope>> {
let offset = cursor.map_or(Ok(Offset::NoOffset), |cursor| {
cursor.ensure_manifest(&self.manifest)?;
decode_timestamp_position(cursor.position())
})?;
let stream = self
.query
.events_by_slices(
&self.entity_type,
self.range.min(),
self.range.max(),
offset,
mode,
)
.await
.map_err(|error| ProjectionError::Source(error.to_string()))?;
Ok(Box::pin(stream.map(|item| {
item.map_err(|error| ProjectionError::Source(error.to_string()))
})))
}
fn extract_cursor(&self, envelope: &EventEnvelope) -> ProjectionResult<ProjectionCursor> {
let Offset::Timestamp {
epoch_micros,
seq_nr,
} = envelope.offset
else {
return Err(ProjectionError::Source(format!(
"PostgreSQL events-by-slices emitted non-timestamp offset {:?}",
envelope.offset
)));
};
if envelope.persistence_id.entity_type() != self.entity_type {
return Err(ProjectionError::Source(format!(
"events-by-slices provider for entity type {:?} received persistence id {}",
self.entity_type, envelope.persistence_id
)));
}
let actual_slice = slice_for(envelope.persistence_id.id());
if !self.range.contains(actual_slice) {
return Err(ProjectionError::Source(format!(
"events-by-slices provider for {} received persistence id {} in slice {actual_slice}",
self.projection_key, envelope.persistence_id
)));
}
Ok(ProjectionCursor::new(
self.manifest.clone(),
DeduplicationBoundary::with_consumed(
encode_timestamp_position(epoch_micros, seq_nr),
[BoundaryEvent::new(
envelope.persistence_id.clone(),
envelope.sequence_nr,
)],
),
))
}
fn compare_positions(
&self,
left: &CursorPosition,
right: &CursorPosition,
) -> ProjectionResult<Ordering> {
Ok(decode_timestamp_position_value(left)?.cmp(&decode_timestamp_position_value(right)?))
}
}
#[must_use]
pub fn slice_projection_key(range: SliceRange, version: u32) -> String {
format!("slices-{}-{}@v{version}", range.min(), range.max())
}
fn slice_provider_manifest(
query_identity: &str,
entity_type: &str,
range: SliceRange,
version: u32,
projection_key: &str,
) -> ProjectionResult<ProviderManifest> {
ProviderManifest::new(format!(
"{SLICE_PROVIDER_MANIFEST_VERSION}|query={}|entity={}|range={}-{}|version={version}|key={projection_key}",
encode_utf8_hex(query_identity),
encode_utf8_hex(entity_type),
range.min(),
range.max(),
))
}
fn encode_timestamp_position(epoch_micros: i64, sequence_nr: u64) -> CursorPosition {
CursorPosition::new(format!(
"{TIMESTAMP_POSITION_VERSION}|{epoch_micros}|{sequence_nr}"
))
}
fn decode_timestamp_position(position: &CursorPosition) -> ProjectionResult<Offset> {
let (epoch_micros, seq_nr) = decode_timestamp_position_value(position)?;
Ok(Offset::Timestamp {
epoch_micros,
seq_nr,
})
}
fn decode_timestamp_position_value(position: &CursorPosition) -> ProjectionResult<(i64, u64)> {
let mut fields = position.as_str().split('|');
let version = fields.next();
let epoch_micros = fields.next();
let sequence_nr = fields.next();
if version != Some(TIMESTAMP_POSITION_VERSION)
|| epoch_micros.is_none()
|| sequence_nr.is_none()
|| fields.next().is_some()
{
return Err(ProjectionError::CursorCodec(format!(
"invalid PostgreSQL timestamp cursor position {:?}",
position.as_str()
)));
}
let epoch_micros = epoch_micros
.expect("presence was checked")
.parse::<i64>()
.map_err(|_| {
ProjectionError::CursorCodec(format!(
"invalid PostgreSQL timestamp micros in position {:?}",
position.as_str()
))
})?;
let sequence_nr = sequence_nr
.expect("presence was checked")
.parse::<u64>()
.map_err(|_| {
ProjectionError::CursorCodec(format!(
"invalid PostgreSQL timestamp sequence in position {:?}",
position.as_str()
))
})?;
Ok((epoch_micros, sequence_nr))
}
fn encode_utf8_hex(value: &str) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut encoded = String::with_capacity(value.len().saturating_mul(2));
for byte in value.bytes() {
encoded.push(char::from(HEX[usize::from(byte >> 4)]));
encoded.push(char::from(HEX[usize::from(byte & 0x0f)]));
}
encoded
}
pub(crate) fn resume_allows<Envelope: 'static>(
provider: &dyn SourceProvider<Envelope>,
cursor: Option<&ProjectionCursor>,
envelope: &Envelope,
) -> ProjectionResult<bool> {
let Some(cursor) = cursor else {
return Ok(true);
};
provider.resume_allows(cursor, envelope)
}
pub(crate) fn advance_cursor<Envelope: 'static>(
provider: &dyn SourceProvider<Envelope>,
current: Option<&ProjectionCursor>,
envelope: &Envelope,
) -> ProjectionResult<ProjectionCursor> {
let next = provider.extract_cursor(envelope)?;
next.ensure_manifest(provider.manifest())?;
let Some(current) = current else {
return Ok(next);
};
current.ensure_manifest(provider.manifest())?;
match provider.compare_positions(next.position(), current.position())? {
Ordering::Less => Err(ProjectionError::Source(format!(
"provider {} emitted cursor position {} after later position {}",
provider.manifest(),
next.position(),
current.position()
))),
Ordering::Greater => Ok(next),
Ordering::Equal => {
let consumed = current
.boundary()
.consumed()
.chain(next.boundary().consumed())
.cloned();
Ok(ProjectionCursor::new(
provider.manifest().clone(),
DeduplicationBoundary::with_consumed(current.position().clone(), consumed),
))
}
}
}
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use futures_util::stream;
use crate::projection::{
GroupedHandler, Handler, MemoryProjectionOffsetStore, Projection, ProjectionRuntime,
ProjectionStatus, TestEnvelope, TestSourceProvider,
};
use crate::query::QueryStream;
use crate::{PersistenceId, SequenceNr, SerializedPayload, SerializerId};
use super::*;
type RecordedCall = (String, u16, u16, Offset, QueryMode);
struct RecordingSliceQuery {
calls: Mutex<Vec<RecordedCall>>,
}
#[async_trait::async_trait]
impl EventsBySlicesQuery for RecordingSliceQuery {
async fn events_by_slices(
&self,
entity_type: &str,
min_slice: u16,
max_slice: u16,
offset: Offset,
mode: QueryMode,
) -> crate::PersistenceResult<QueryStream<EventEnvelope>> {
self.calls.lock().unwrap().push((
entity_type.to_owned(),
min_slice,
max_slice,
offset,
mode,
));
Ok(Box::pin(stream::empty()))
}
}
struct NoopHandler;
#[async_trait::async_trait]
impl Handler<EventEnvelope> for NoopHandler {
async fn process(&mut self, _envelope: &EventEnvelope) -> ProjectionResult<()> {
Ok(())
}
}
struct NoopGroupedHandler;
#[async_trait::async_trait]
impl GroupedHandler<EventEnvelope> for NoopGroupedHandler {
async fn process(&mut self, _envelopes: &[EventEnvelope]) -> ProjectionResult<()> {
Ok(())
}
}
struct NoopTestHandler;
#[async_trait::async_trait]
impl Handler<TestEnvelope<usize>> for NoopTestHandler {
async fn process(&mut self, _envelope: &TestEnvelope<usize>) -> ProjectionResult<()> {
Ok(())
}
}
fn envelope(persistence_id: PersistenceId, epoch_micros: i64) -> EventEnvelope {
EventEnvelope {
offset: Offset::Timestamp {
epoch_micros,
seq_nr: 7,
},
persistence_id,
sequence_nr: SequenceNr::new(7),
timestamp_ms: 1,
adapter_manifest: "adapter-v1".into(),
payload: SerializedPayload::new(SerializerId::new(1), "event-v1", vec![]),
metadata: None,
tags: Vec::new(),
}
}
fn id_in(range: SliceRange) -> PersistenceId {
(0..10_000)
.map(|suffix| PersistenceId::new("account", suffix.to_string()).unwrap())
.find(|persistence_id| range.contains(slice_for(persistence_id.id())))
.expect("fixture search finds an id in the range")
}
fn id_in_but_not(in_range: SliceRange, excluded: SliceRange) -> PersistenceId {
(0..10_000)
.map(|suffix| PersistenceId::new("account", format!("outside-{suffix}")).unwrap())
.find(|persistence_id| {
let slice = slice_for(persistence_id.id());
in_range.contains(slice) && !excluded.contains(slice)
})
.expect("fixture search finds an id in only the old range")
}
#[tokio::test]
async fn range_version_is_in_key_manifest_and_query_resume() {
let query = Arc::new(RecordingSliceQuery {
calls: Mutex::new(Vec::new()),
});
let range = SliceRange::new(256, 511).unwrap();
let provider = EventsBySlicesSourceProvider::new(
query.clone(),
"postgres:test-db/public",
"account",
range,
1,
)
.unwrap();
assert_eq!(provider.projection_key(), "slices-256-511@v1");
assert_eq!(
provider.manifest().as_str(),
"datum-postgres-events-by-slices-v1|query=706f7374677265733a746573742d64622f7075626c6963|entity=6163636f756e74|range=256-511|version=1|key=slices-256-511@v1"
);
let persistence_id = PersistenceId::new("account", "0").unwrap();
assert_eq!(slice_for(persistence_id.id()), 336);
let event = envelope(persistence_id, 123);
let cursor = provider.extract_cursor(&event).unwrap();
assert_eq!(cursor.boundary().consumed().len(), 1);
assert_eq!(cursor.position().as_str(), "pg-timestamp-v1|123|7");
assert_eq!(
cursor.encode(),
"datum-projection-cursor-v1|646174756d2d706f7374677265732d6576656e74732d62792d736c696365732d76317c71756572793d373036663733373436373732363537333361373436353733373432643634363232663730373536323663363936337c656e746974793d36313633363336663735366537347c72616e67653d3235362d3531317c76657273696f6e3d317c6b65793d736c696365732d3235362d353131407631|70672d74696d657374616d702d76317c3132337c37|1|6163636f756e747c30:7"
);
let _source = provider
.source_from(Some(&cursor), QueryMode::Current)
.await
.unwrap();
assert_eq!(
query.calls.lock().unwrap().as_slice(),
[(
"account".to_owned(),
256,
511,
Offset::Timestamp {
epoch_micros: 123,
seq_nr: 7,
},
QueryMode::Current,
)]
);
}
#[tokio::test]
async fn normal_spawn_enforces_slice_key_for_every_builder_before_materialization() {
let query = Arc::new(RecordingSliceQuery {
calls: Mutex::new(Vec::new()),
});
let range = SliceRange::new(256, 511).unwrap();
let provider =
EventsBySlicesSourceProvider::new(query.clone(), "postgres:test", "account", range, 3)
.unwrap();
let store = Arc::new(MemoryProjectionOffsetStore::new());
let runtime = ProjectionRuntime::new(Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>);
let wrong_key = "manual";
let projections = [
Projection::at_least_once(
ProjectionId::new("slice-key-alo", wrong_key),
provider.clone(),
|| NoopHandler,
),
Projection::at_most_once(
ProjectionId::new("slice-key-amo", wrong_key),
provider.clone(),
|| NoopHandler,
),
Projection::grouped_at_least_once(
ProjectionId::new("slice-key-grouped", wrong_key),
provider.clone(),
|| NoopGroupedHandler,
),
];
for projection in projections {
let error = match projection.spawn(runtime.clone()).await {
Ok(_) => panic!("wrong range-derived key must fail normal spawn"),
Err(error) => error,
};
assert!(matches!(
error,
ProjectionError::InvalidConfig {
setting: "events_by_slices.projection_key",
..
}
));
assert!(error.is_fatal());
}
assert!(
query.calls.lock().unwrap().is_empty(),
"projection-id validation must run before source materialization"
);
let correct_provider =
EventsBySlicesSourceProvider::new(query.clone(), "postgres:test", "account", range, 3)
.unwrap();
let correct_id = ProjectionId::new("slice-key-correct", correct_provider.projection_key());
let correct = Projection::at_least_once(correct_id, correct_provider, || NoopHandler)
.with_query_mode(QueryMode::Current)
.spawn(runtime.clone())
.await
.expect("range-derived key is accepted");
assert_eq!(correct.wait().await.unwrap(), ProjectionStatus::Completed);
assert_eq!(query.calls.lock().unwrap().len(), 1);
let unrestricted = Projection::at_least_once(
ProjectionId::new("non-slice-provider", "manual-key"),
TestSourceProvider::new(Vec::<TestEnvelope<usize>>::new()),
|| NoopTestHandler,
)
.with_query_mode(QueryMode::Current)
.spawn(runtime)
.await
.expect("the default provider hook preserves arbitrary valid keys");
assert_eq!(
unrestricted.wait().await.unwrap(),
ProjectionStatus::Completed
);
}
#[tokio::test]
async fn range_or_version_change_is_a_fatal_manifest_mismatch() {
let query = Arc::new(RecordingSliceQuery {
calls: Mutex::new(Vec::new()),
});
let old_range = SliceRange::new(0, 511).unwrap();
let old = EventsBySlicesSourceProvider::new(
query.clone(),
"postgres:test-db/public",
"account",
old_range,
1,
)
.unwrap();
let cursor = old
.extract_cursor(&envelope(id_in(old_range), 123))
.unwrap();
for (range, version) in [(SliceRange::new(0, 255).unwrap(), 1), (old_range, 2)] {
let changed = EventsBySlicesSourceProvider::new(
query.clone(),
"postgres:test-db/public",
"account",
range,
version,
)
.unwrap();
let error = match changed.source_from(Some(&cursor), QueryMode::Live).await {
Ok(_) => panic!("changed provider must reject the old cursor"),
Err(error) => error,
};
assert!(matches!(
error,
ProjectionError::ProviderManifestMismatch { .. }
));
assert!(error.is_fatal());
}
}
#[test]
fn provider_rejects_invalid_identity_version_and_envelopes() {
let query: Arc<dyn EventsBySlicesQuery> = Arc::new(RecordingSliceQuery {
calls: Mutex::new(Vec::new()),
});
let range = SliceRange::new(0, 127).unwrap();
assert!(EventsBySlicesSourceProvider::new(query.clone(), "", "account", range, 1).is_err());
assert!(
EventsBySlicesSourceProvider::new(query.clone(), "postgres:test", "", range, 1)
.is_err()
);
assert!(
EventsBySlicesSourceProvider::new(query.clone(), "postgres:test", "account", range, 0)
.is_err()
);
let provider =
EventsBySlicesSourceProvider::new(query, "postgres:test", "account", range, 1).unwrap();
let wrong = PersistenceId::new("other", "id").unwrap();
let error = provider
.extract_cursor(&envelope(wrong, 1))
.expect_err("wrong entity type is a source failure");
assert!(matches!(error, ProjectionError::Source(_)));
let malformed = ProjectionCursor::from_position(
provider.manifest().clone(),
"not-a-timestamp-position",
);
let error = provider
.compare_positions(malformed.position(), &CursorPosition::new("also-invalid"))
.expect_err("malformed position is fatal");
assert!(matches!(error, ProjectionError::CursorCodec(_)));
}
#[tokio::test]
async fn four_to_eight_seed_persists_new_manifest_and_filters_boundary() {
let query: Arc<dyn EventsBySlicesQuery> = Arc::new(RecordingSliceQuery {
calls: Mutex::new(Vec::new()),
});
let old_range = SliceRange::new(0, 255).unwrap();
let new_range = SliceRange::new(0, 127).unwrap();
let old = EventsBySlicesSourceProvider::new(
query.clone(),
"postgres:test",
"account",
old_range,
1,
)
.unwrap();
let new =
EventsBySlicesSourceProvider::new(query, "postgres:test", "account", new_range, 2)
.unwrap();
let included = id_in(new_range);
let excluded = id_in_but_not(old_range, new_range);
let base = old
.extract_cursor(&envelope(included.clone(), 100))
.unwrap();
let old_cursor = ProjectionCursor::new(
old.manifest().clone(),
DeduplicationBoundary::with_consumed(
base.position().clone(),
[
BoundaryEvent::new(included.clone(), SequenceNr::new(7)),
BoundaryEvent::new(excluded, SequenceNr::new(7)),
],
),
);
let covering = [SliceRangeCursor::new(old_range, 1, Some(old_cursor)).unwrap()];
let projection_id = ProjectionId::new("accounts", new.projection_key());
let store = MemoryProjectionOffsetStore::new();
let seeded = seed_resharded_slice_range(&store, &projection_id, &new, &covering)
.await
.unwrap()
.expect("old range had a cursor");
assert_eq!(seeded.provider_manifest(), new.manifest());
assert_eq!(seeded.boundary().consumed().len(), 1);
assert!(seeded.boundary().contains(&included, SequenceNr::new(7)));
assert_eq!(store.load(&projection_id).await.unwrap(), Some(seeded));
let error = seed_resharded_slice_range(&store, &projection_id, &new, &covering)
.await
.expect_err("seed never overwrites an existing new-range cursor");
assert!(matches!(error, ProjectionError::InvalidConfig { .. }));
}
#[test]
fn seed_takes_minimum_covering_position_and_validates_coverage() {
let query: Arc<dyn EventsBySlicesQuery> = Arc::new(RecordingSliceQuery {
calls: Mutex::new(Vec::new()),
});
let left_range = SliceRange::new(0, 127).unwrap();
let right_range = SliceRange::new(128, 255).unwrap();
let new_range = SliceRange::new(0, 255).unwrap();
let left = EventsBySlicesSourceProvider::new(
query.clone(),
"postgres:test",
"account",
left_range,
1,
)
.unwrap();
let right = EventsBySlicesSourceProvider::new(
query.clone(),
"postgres:test",
"account",
right_range,
1,
)
.unwrap();
let new =
EventsBySlicesSourceProvider::new(query, "postgres:test", "account", new_range, 2)
.unwrap();
let covering = [
SliceRangeCursor::new(
left_range,
1,
Some(
left.extract_cursor(&envelope(id_in(left_range), 120))
.unwrap(),
),
)
.unwrap(),
SliceRangeCursor::new(
right_range,
1,
Some(
right
.extract_cursor(&envelope(id_in(right_range), 100))
.unwrap(),
),
)
.unwrap(),
];
let seed = resharding_seed_cursor(&new, &covering)
.unwrap()
.expect("both old ranges had cursors");
assert_eq!(
decode_timestamp_position_value(seed.position()).unwrap(),
(100, 7)
);
let all_absent = [
SliceRangeCursor::new(left_range, 1, None).unwrap(),
SliceRangeCursor::new(right_range, 1, None).unwrap(),
];
assert_eq!(
resharding_seed_cursor(&new, &all_absent).unwrap(),
None,
"all unstarted covering ranges seed from the beginning"
);
let gap = resharding_seed_cursor(&new, &covering[..1])
.expect_err("partial coverage must be rejected");
assert!(matches!(gap, ProjectionError::InvalidConfig { .. }));
}
#[test]
fn absent_reshard_cursor_never_hides_a_wrong_range_cursor_in_either_order() {
let query: Arc<dyn EventsBySlicesQuery> = Arc::new(RecordingSliceQuery {
calls: Mutex::new(Vec::new()),
});
let left_range = SliceRange::new(0, 127).unwrap();
let right_range = SliceRange::new(128, 255).unwrap();
let new_range = SliceRange::new(0, 255).unwrap();
let left = EventsBySlicesSourceProvider::new(
query.clone(),
"postgres:test",
"account",
left_range,
1,
)
.unwrap();
let new =
EventsBySlicesSourceProvider::new(query, "postgres:test", "account", new_range, 2)
.unwrap();
let cursor_for_left = left
.extract_cursor(&envelope(id_in(left_range), 100))
.unwrap();
let absent_left = SliceRangeCursor::new(left_range, 1, None).unwrap();
let wrong_right = SliceRangeCursor::new(right_range, 1, Some(cursor_for_left)).unwrap();
for covering in [
vec![absent_left.clone(), wrong_right.clone()],
vec![wrong_right.clone(), absent_left.clone()],
] {
let error = resharding_seed_cursor(&new, &covering)
.expect_err("every present old cursor must be validated before absence wins");
assert!(matches!(
error,
ProjectionError::ProviderManifestMismatch { .. }
));
assert!(error.is_fatal());
}
}
}