#![cfg(feature = "postgres")]
use std::{
panic::{AssertUnwindSafe, resume_unwind},
sync::Arc,
sync::atomic::{AtomicU64, Ordering},
time::{Duration, SystemTime, UNIX_EPOCH},
};
use futures_util::{FutureExt, StreamExt};
use sqlx::Row;
pub use datum_persistence::*;
use datum_persistence::{
query::{
EventEnvelope, EventsByPersistenceIdQuery, EventsBySlicesQuery, EventsByTagQuery, Offset,
PersistenceIdsQuery, QueryMode, QueryStream, SliceRange, decode_envelopes, into_source,
slice_for,
},
testkit::{
assert_durable_state_changes_query_conformance,
assert_events_by_persistence_id_query_conformance, assert_events_by_tag_query_conformance,
assert_persistence_ids_query_conformance,
},
};
static TEST_SCHEMA_COUNTER: AtomicU64 = AtomicU64::new(1);
#[tokio::test]
async fn postgres_journal_conformance() {
let Some(database_url) = database_url() else {
return;
};
let config = isolated_config(database_url, PostgresDeletePolicy::Soft);
let schema = config.schema.clone();
let backend = PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated PostgreSQL journal");
let result = AssertUnwindSafe(assert_journal_conformance(&backend))
.catch_unwind()
.await;
drop_schema(&backend, &schema).await;
finish_conformance(result);
}
#[tokio::test]
async fn postgres_hard_delete_journal_conformance() {
let Some(database_url) = database_url() else {
return;
};
let config = isolated_config(database_url, PostgresDeletePolicy::Hard);
let schema = config.schema.clone();
let backend = PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated hard-delete PostgreSQL journal");
let result = AssertUnwindSafe(assert_journal_conformance(&backend))
.catch_unwind()
.await;
drop_schema(&backend, &schema).await;
finish_conformance(result);
}
#[tokio::test]
async fn postgres_snapshot_store_conformance() {
let Some(database_url) = database_url() else {
return;
};
let config = isolated_config(database_url, PostgresDeletePolicy::Soft);
let schema = config.schema.clone();
let backend = PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated PostgreSQL snapshot store");
let result = AssertUnwindSafe(assert_snapshot_store_conformance(&backend))
.catch_unwind()
.await;
drop_schema(&backend, &schema).await;
finish_conformance(result);
}
#[tokio::test]
async fn postgres_durable_state_store_conformance() {
let Some(database_url) = database_url() else {
return;
};
let config = isolated_config(database_url, PostgresDeletePolicy::Soft);
let schema = config.schema.clone();
let backend = PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated PostgreSQL durable-state store");
let result = AssertUnwindSafe(assert_durable_state_store_conformance(&backend))
.catch_unwind()
.await;
drop_schema(&backend, &schema).await;
finish_conformance(result);
}
#[tokio::test]
async fn postgres_events_by_persistence_id_query_conformance() {
let Some(database_url) = database_url() else {
return;
};
let config = isolated_config(database_url, PostgresDeletePolicy::Soft);
let schema = config.schema.clone();
let backend = PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated PostgreSQL per-id query");
let result = AssertUnwindSafe(assert_events_by_persistence_id_query_conformance(
&backend, &backend,
))
.catch_unwind()
.await;
drop_schema(&backend, &schema).await;
finish_conformance(result);
}
#[tokio::test]
async fn postgres_events_by_tag_query_conformance() {
let Some(database_url) = database_url() else {
return;
};
let config = isolated_config(database_url, PostgresDeletePolicy::Soft);
let schema = config.schema.clone();
let backend = PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated PostgreSQL tag query");
let result = AssertUnwindSafe(assert_events_by_tag_query_conformance(&backend, &backend))
.catch_unwind()
.await;
drop_schema(&backend, &schema).await;
finish_conformance(result);
}
#[tokio::test]
async fn postgres_current_events_by_slices_filters_and_orders_like_tag_queries() {
let Some(database_url) = database_url() else {
return;
};
let mut config = isolated_config(database_url, PostgresDeletePolicy::Soft);
config.polling.visibility_lag = Duration::ZERO;
let schema = config.schema.clone();
let backend = PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated PostgreSQL slice query");
let result = AssertUnwindSafe(async {
let entity_type = "slice-current";
let range = SliceRange::new(256, 511).unwrap();
let tag = Tag::new("slice-current-order");
let at_min = id_matching_slice(entity_type, "at-min", |slice| slice == range.min());
let below_min =
id_matching_slice(entity_type, "below-min", |slice| slice == range.min() - 1);
let at_max = id_matching_slice(entity_type, "at-max", |slice| slice == range.max());
let above_max =
id_matching_slice(entity_type, "above-max", |slice| slice == range.max() + 1);
let other_entity = id_matching_slice("slice-current-other", "other", |slice| {
range.contains(slice)
});
assert_eq!(slice_for(at_min.id()), range.min());
assert_eq!(slice_for(below_min.id()), range.min() - 1);
assert_eq!(slice_for(at_max.id()), range.max());
assert_eq!(slice_for(above_max.id()), range.max() + 1);
for persistence_id in [&at_min, &below_min, &other_entity, &at_max, &above_max] {
let mut stored = event(persistence_id, 1);
stored.tags = vec![tag.clone()];
let outcome = backend
.append(vec![atomic(stored)])
.await
.expect("append slice-query fixture");
assert_eq!(outcome, vec![AppendOutcome::Accepted]);
}
let event_table = qualified(&schema, &backend.config().tables.event_journal);
let stored_slices =
sqlx::query(&format!("SELECT persistence_id, slice FROM {event_table}"))
.fetch_all(backend.pool().expect("connected backend pool"))
.await
.expect("inspect persisted slice fixtures");
assert_eq!(stored_slices.len(), 5);
for row in stored_slices {
let persistence_id: String = row.try_get("persistence_id").unwrap();
let stored_slice: i32 = row.try_get("slice").unwrap();
assert_eq!(
stored_slice,
i32::from(slice_for(&persistence_id)),
"journal writes must persist the published frozen hash"
);
}
let slices = collect_query_stream(
backend
.events_by_slices(
entity_type,
range.min(),
range.max(),
Offset::NoOffset,
QueryMode::Current,
)
.await
.expect("materialize current events-by-slices"),
)
.await;
assert_eq!(
slices
.iter()
.map(|envelope| envelope.persistence_id.clone())
.collect::<Vec<_>>(),
vec![at_min.clone(), at_max.clone()]
);
assert!(slices.iter().all(|envelope| {
envelope.persistence_id.entity_type() == entity_type
&& range.contains(slice_for(envelope.persistence_id.id()))
}));
let tagged = collect_query_stream(
backend
.events_by_tag(&tag, Offset::NoOffset, QueryMode::Current)
.await
.expect("materialize parity tag query"),
)
.await;
let tag_filtered = tagged
.into_iter()
.filter(|envelope| {
envelope.persistence_id.entity_type() == entity_type
&& range.contains(slice_for(envelope.persistence_id.id()))
})
.map(|envelope| envelope.persistence_id)
.collect::<Vec<_>>();
assert_eq!(
tag_filtered,
slices
.iter()
.map(|envelope| envelope.persistence_id.clone())
.collect::<Vec<_>>(),
"slice and tag queries must use the same global ordering contract"
);
})
.catch_unwind()
.await;
drop_schema(&backend, &schema).await;
if let Err(panic) = result {
resume_unwind(panic);
}
}
#[tokio::test]
async fn postgres_live_events_by_slices_matches_tag_live_and_resumes_inclusively() {
let Some(database_url) = database_url() else {
return;
};
let mut config = isolated_config(database_url, PostgresDeletePolicy::Soft);
config.polling.min_interval = Duration::from_millis(5);
config.polling.max_interval = Duration::from_millis(20);
config.polling.visibility_lag = Duration::ZERO;
let schema = config.schema.clone();
let backend = PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated PostgreSQL live slice query");
let result = AssertUnwindSafe(async {
let entity_type = "slice-live";
let range = SliceRange::new(512, 767).unwrap();
let tag = Tag::new("slice-live-parity");
let first = id_matching_slice(entity_type, "first", |slice| range.contains(slice));
let outside = id_matching_slice(entity_type, "outside", |slice| !range.contains(slice));
let second = id_matching_slice(entity_type, "second", |slice| range.contains(slice));
let mut live_slices = backend
.events_by_slices(
entity_type,
range.min(),
range.max(),
Offset::NoOffset,
QueryMode::Live,
)
.await
.expect("materialize live events-by-slices before writes");
let mut live_tag = backend
.events_by_tag(&tag, Offset::NoOffset, QueryMode::Live)
.await
.expect("materialize parity live tag query before writes");
for persistence_id in [&first, &outside, &second] {
let mut stored = event(persistence_id, 1);
stored.tags = vec![tag.clone()];
assert_eq!(
backend.append(vec![atomic(stored)]).await.unwrap(),
vec![AppendOutcome::Accepted]
);
}
let observed_slices = tokio::time::timeout(Duration::from_secs(2), async {
let mut events = Vec::with_capacity(2);
while events.len() < 2 {
events.push(
live_slices
.next()
.await
.expect("live slice query completed")
.expect("live slice query failed"),
);
}
events
})
.await
.expect("live slice query timed out");
let observed_tag = tokio::time::timeout(Duration::from_secs(2), async {
let mut matching = Vec::with_capacity(2);
while matching.len() < 2 {
let envelope = live_tag
.next()
.await
.expect("live tag query completed")
.expect("live tag query failed");
if range.contains(slice_for(envelope.persistence_id.id())) {
matching.push(envelope);
}
}
matching
})
.await
.expect("live tag parity query timed out");
assert_eq!(
observed_slices
.iter()
.map(|envelope| &envelope.persistence_id)
.collect::<Vec<_>>(),
vec![&first, &second]
);
assert_eq!(
observed_tag
.iter()
.map(|envelope| &envelope.persistence_id)
.collect::<Vec<_>>(),
vec![&first, &second]
);
let resume_offset = observed_slices[0].offset;
let resumed = collect_query_stream(
backend
.events_by_slices(
entity_type,
range.min(),
range.max(),
resume_offset,
QueryMode::Current,
)
.await
.expect("resume current slice query"),
)
.await;
assert_eq!(
resumed
.iter()
.map(|envelope| &envelope.persistence_id)
.collect::<Vec<_>>(),
vec![&first, &second],
"external timestamp resume must re-emit the exact boundary and lose nothing after it"
);
})
.catch_unwind()
.await;
drop_schema(&backend, &schema).await;
if let Err(panic) = result {
resume_unwind(panic);
}
}
#[tokio::test]
async fn postgres_persistence_ids_query_conformance() {
let Some(database_url) = database_url() else {
return;
};
let config = isolated_config(database_url, PostgresDeletePolicy::Soft);
let schema = config.schema.clone();
let backend = PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated PostgreSQL persistence-ids query");
let result = AssertUnwindSafe(assert_persistence_ids_query_conformance(&backend, &backend))
.catch_unwind()
.await;
drop_schema(&backend, &schema).await;
finish_conformance(result);
}
#[tokio::test]
async fn postgres_durable_state_changes_query_conformance() {
let Some(database_url) = database_url() else {
return;
};
let config = isolated_config(database_url, PostgresDeletePolicy::Soft);
let schema = config.schema.clone();
let backend = PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated PostgreSQL durable changes query");
let result = AssertUnwindSafe(assert_durable_state_changes_query_conformance(
&backend, &backend,
))
.catch_unwind()
.await;
drop_schema(&backend, &schema).await;
finish_conformance(result);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn postgres_behavior_events_flow_through_live_query_and_catalog() {
let Some(database_url) = database_url() else {
return;
};
let config = isolated_config(database_url, PostgresDeletePolicy::Soft);
let schema = config.schema.clone();
let backend = Arc::new(
PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated PostgreSQL query e2e backend"),
);
let persistence_id = PersistenceId::new("postgres-query-e2e", "counter").unwrap();
let behavior = EventSourcedBehavior::new(
persistence_id.clone(),
0_i64,
|_, events: Vec<i64>| Effect::persist_all(events),
|state: &i64, event: &i64| state + event,
)
.with_event_adapter(IdentityAdapter, query_catalog());
let handle = behavior
.spawn(EventSourcedRuntime::new(backend.clone()))
.await
.expect("spawn PostgreSQL-backed behavior");
let query = backend
.events_by_persistence_id(
&persistence_id,
SequenceNr::new(1),
SequenceNr::new(3),
QueryMode::Live,
)
.await
.expect("materialize PostgreSQL live query");
let completion = into_source(query)
.via(decode_envelopes(query_catalog(), IdentityAdapter))
.take(3)
.run_fold(Vec::new(), |mut events, event| {
events.push(event);
events
})
.expect("materialize decoded PostgreSQL query source");
handle.tell(vec![4, 5, 6]).expect("persist behavior events");
let events = tokio::time::timeout(Duration::from_secs(2), completion)
.await
.expect("PostgreSQL live query timed out")
.expect("PostgreSQL live query source failed");
assert_eq!(events, vec![4, 5, 6]);
handle.stop();
drop_schema(&backend, &schema).await;
}
#[tokio::test]
async fn postgres_live_per_id_query_honors_future_requested_lower_bound() {
let Some(database_url) = database_url() else {
return;
};
let mut config = isolated_config(database_url, PostgresDeletePolicy::Soft);
config.polling.min_interval = Duration::from_millis(5);
config.polling.max_interval = Duration::from_millis(20);
let schema = config.schema.clone();
let backend = PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated PostgreSQL future-range query");
let persistence_id = PersistenceId::new("future-range", "one").unwrap();
let mut live = backend
.events_by_persistence_id(
&persistence_id,
SequenceNr::new(10),
SequenceNr::new(12),
QueryMode::Live,
)
.await
.expect("materialize live per-id query before its requested range exists");
let write = AtomicWrite::new(
(1..=12)
.map(|sequence_nr| event(&persistence_id, sequence_nr))
.collect(),
)
.expect("future-range atomic write is valid");
assert_eq!(
backend
.append(vec![write])
.await
.expect("append events after live query materialization"),
vec![AppendOutcome::Accepted]
);
let observed = tokio::time::timeout(Duration::from_secs(2), async {
let mut sequence_nrs = Vec::with_capacity(3);
while sequence_nrs.len() < 3 {
let envelope = live
.next()
.await
.expect("live per-id query completed early")
.expect("live per-id query failed");
sequence_nrs.push(envelope.sequence_nr);
}
sequence_nrs
})
.await
.expect("live per-id query did not emit its future requested range");
drop(live);
drop_schema(&backend, &schema).await;
assert_eq!(
observed,
[
SequenceNr::new(10),
SequenceNr::new(11),
SequenceNr::new(12),
]
);
}
#[tokio::test]
async fn postgres_live_tag_query_does_not_skip_an_earlier_late_commit() {
let Some(database_url) = database_url() else {
return;
};
let mut config = isolated_config(database_url, PostgresDeletePolicy::Soft);
config.polling.min_interval = Duration::from_millis(5);
config.polling.max_interval = Duration::from_millis(20);
config.polling.visibility_lag = Duration::from_millis(50);
let schema = config.schema.clone();
let event_table = qualified(&schema, &config.tables.event_journal);
let backend = PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated PostgreSQL visibility-lag query");
let pool = backend
.pool()
.expect("connected backend has a pool")
.clone();
let tag = Tag::new("postgres-visibility-lag");
let late_id = PersistenceId::new("visibility-lag", "late").unwrap();
let visible_id = PersistenceId::new("visibility-lag", "visible").unwrap();
let insert_late = format!(
"INSERT INTO {event_table} \
(persistence_id, seq_nr, slice, entity_type, db_timestamp, writer, adapter_manifest, \
ser_id, ser_manifest, payload, tags) \
VALUES ($1, 1, 0, 'visibility-lag', \
clock_timestamp() + INTERVAL '250 milliseconds', \
'late-writer', 'adapter-v1', 7, 'payload-v1', $2, ARRAY[$3]::TEXT[])"
);
let insert_visible = format!(
"INSERT INTO {event_table} \
(persistence_id, seq_nr, slice, entity_type, db_timestamp, writer, adapter_manifest, \
ser_id, ser_manifest, payload, tags) \
VALUES ($1, 1, 1, 'visibility-lag', \
clock_timestamp() + INTERVAL '300 milliseconds', \
'visible-writer', 'adapter-v1', 7, 'payload-v1', $2, ARRAY[$3]::TEXT[])"
);
let mut late_transaction = pool.begin().await.expect("begin late transaction");
sqlx::query(&insert_late)
.bind(late_id.id())
.bind(vec![1_u8])
.bind(tag.as_str())
.execute(&mut *late_transaction)
.await
.expect("insert uncommitted earlier-timestamp event");
sqlx::query(&insert_visible)
.bind(visible_id.id())
.bind(vec![2_u8])
.bind(tag.as_str())
.execute(&pool)
.await
.expect("commit later-timestamp event");
let mut live = backend
.events_by_tag(&tag, Offset::NoOffset, QueryMode::Live)
.await
.expect("materialize live tag query before the earlier transaction commits");
late_transaction
.commit()
.await
.expect("commit earlier-timestamp event after live query materialization");
let observed = tokio::time::timeout(Duration::from_secs(2), async {
let mut persistence_ids = Vec::with_capacity(2);
while persistence_ids.len() < 2 {
let envelope = live
.next()
.await
.expect("live tag query completed early")
.expect("live tag query failed");
persistence_ids.push(envelope.persistence_id);
}
persistence_ids
})
.await
.expect("live tag query skipped the earlier late-committing event");
drop(live);
drop_schema(&backend, &schema).await;
assert_eq!(observed, vec![late_id, visible_id]);
}
#[tokio::test]
async fn postgres_live_slice_query_does_not_skip_an_earlier_late_commit() {
let Some(database_url) = database_url() else {
return;
};
let mut config = isolated_config(database_url, PostgresDeletePolicy::Soft);
config.polling.min_interval = Duration::from_millis(5);
config.polling.max_interval = Duration::from_millis(20);
config.polling.visibility_lag = Duration::from_millis(50);
let schema = config.schema.clone();
let event_table = qualified(&schema, &config.tables.event_journal);
let backend = PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated PostgreSQL slice visibility-lag query");
let pool = backend
.pool()
.expect("connected backend has a pool")
.clone();
let entity_type = "slice-visibility-lag";
let late_id = PersistenceId::new(entity_type, "late").unwrap();
let visible_id = PersistenceId::new(entity_type, "visible").unwrap();
let insert_late = format!(
"INSERT INTO {event_table} \
(persistence_id, seq_nr, slice, entity_type, db_timestamp, writer, adapter_manifest, \
ser_id, ser_manifest, payload, tags) \
VALUES ($1, 1, 4, $2, \
clock_timestamp() + INTERVAL '250 milliseconds', \
'late-writer', 'adapter-v1', 7, 'payload-v1', $3, '{{}}'::TEXT[])"
);
let insert_visible = format!(
"INSERT INTO {event_table} \
(persistence_id, seq_nr, slice, entity_type, db_timestamp, writer, adapter_manifest, \
ser_id, ser_manifest, payload, tags) \
VALUES ($1, 1, 5, $2, \
clock_timestamp() + INTERVAL '300 milliseconds', \
'visible-writer', 'adapter-v1', 7, 'payload-v1', $3, '{{}}'::TEXT[])"
);
let mut late_transaction = pool.begin().await.expect("begin late slice transaction");
sqlx::query(&insert_late)
.bind(late_id.id())
.bind(entity_type)
.bind(vec![1_u8])
.execute(&mut *late_transaction)
.await
.expect("insert uncommitted earlier-timestamp slice event");
sqlx::query(&insert_visible)
.bind(visible_id.id())
.bind(entity_type)
.bind(vec![2_u8])
.execute(&pool)
.await
.expect("commit later-timestamp slice event");
let mut live = backend
.events_by_slices(entity_type, 0, 10, Offset::NoOffset, QueryMode::Live)
.await
.expect("materialize live slice query before the earlier transaction commits");
late_transaction
.commit()
.await
.expect("commit earlier-timestamp slice event after live query materialization");
let observed = tokio::time::timeout(Duration::from_secs(2), async {
let mut persistence_ids = Vec::with_capacity(2);
while persistence_ids.len() < 2 {
let envelope = live
.next()
.await
.expect("live slice query completed early")
.expect("live slice query failed");
persistence_ids.push(envelope.persistence_id);
}
persistence_ids
})
.await
.expect("live slice query skipped the earlier late-committing event");
drop(live);
drop_schema(&backend, &schema).await;
assert_eq!(observed, vec![late_id, visible_id]);
}
#[tokio::test]
async fn postgres_soft_and_hard_delete_policies_have_distinct_physical_results() {
let Some(database_url) = database_url() else {
return;
};
for (policy, expected_rows, expected_deleted) in [
(PostgresDeletePolicy::Soft, 1_i64, Some(true)),
(PostgresDeletePolicy::Hard, 0_i64, None),
] {
let config = isolated_config(database_url.clone(), policy);
let schema = config.schema.clone();
let table = qualified(&schema, &config.tables.event_journal);
let backend = PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated delete-policy journal");
let persistence_id = PersistenceId::new("delete-policy", "one").unwrap();
let outcomes = backend
.append(vec![atomic(event(&persistence_id, 1))])
.await
.expect("append delete-policy fixture");
assert_eq!(outcomes, vec![AppendOutcome::Accepted]);
backend
.delete_to(&persistence_id, SequenceNr::new(1))
.await
.expect("delete journal fixture");
let inspect = format!(
"SELECT COUNT(*) AS row_count, BOOL_AND(deleted) AS all_deleted \
FROM {table} WHERE persistence_id = $1"
);
let row = sqlx::query(&inspect)
.bind(persistence_id.id())
.fetch_one(backend.pool().expect("connected backend has a pool"))
.await
.expect("inspect physical delete result");
let row_count: i64 = row.try_get("row_count").unwrap();
let all_deleted: Option<bool> = row.try_get("all_deleted").unwrap();
drop_schema(&backend, &schema).await;
assert_eq!(row_count, expected_rows, "policy {policy:?}");
assert_eq!(all_deleted, expected_deleted, "policy {policy:?}");
}
}
#[tokio::test]
async fn postgres_duplicate_sequence_primary_key_violation_is_failure() {
let Some(database_url) = database_url() else {
return;
};
let config = isolated_config(database_url, PostgresDeletePolicy::Soft);
let schema = config.schema.clone();
let journal_table = qualified(&schema, &config.tables.event_journal);
let head_table = qualified(&schema, &config.tables.journal_head);
let backend = PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated duplicate-sequence journal");
let pool = backend.pool().expect("connected backend has a pool");
let persistence_id = PersistenceId::new("duplicate", "one").unwrap();
let insert_head = format!(
"INSERT INTO {head_table} (persistence_id, highest_seq_nr, deleted_to) VALUES ($1, 0, 0)"
);
sqlx::query(&insert_head)
.bind(persistence_id.id())
.execute(pool)
.await
.expect("insert deliberately inconsistent journal head");
let insert_rogue = format!(
"INSERT INTO {journal_table} \
(persistence_id, seq_nr, slice, entity_type, writer, adapter_manifest, \
ser_id, ser_manifest, payload) \
VALUES ($1, 1, 0, 'duplicate', 'fixture', 'fixture-adapter', 7, 'fixture-v1', $2)"
);
sqlx::query(&insert_rogue)
.bind(persistence_id.id())
.bind(vec![0_u8])
.execute(pool)
.await
.expect("insert rogue row that will force a primary-key violation");
let result = backend
.append(vec![atomic(event(&persistence_id, 1))])
.await;
drop_schema(&backend, &schema).await;
assert!(
matches!(result, Err(PersistenceError::Failure(_))),
"a database-detected primary-key violation is a storage failure, got {result:?}"
);
}
async fn group_commit_backend(database_url: String) -> (Arc<PostgresPersistence>, String) {
let mut config = isolated_config(database_url, PostgresDeletePolicy::Soft);
config.group_commit.bypass_below = 0;
let schema = config.schema.clone();
let backend = Arc::new(
PostgresPersistence::connect(config)
.await
.expect("connect and initialize isolated group-commit backend"),
);
(backend, schema)
}
async fn current_sequence_nrs(
backend: &PostgresPersistence,
persistence_id: &PersistenceId,
to_sequence_nr: u64,
) -> Vec<u64> {
let mut stream = backend
.events_by_persistence_id(
persistence_id,
SequenceNr::new(1),
SequenceNr::new(to_sequence_nr),
QueryMode::Current,
)
.await
.expect("materialize current per-id query");
let mut sequence_nrs = Vec::new();
while let Some(envelope) = stream.next().await {
sequence_nrs.push(envelope.expect("query event").sequence_nr.as_u64());
}
sequence_nrs
}
fn multi_event_write(persistence_id: &PersistenceId, count: u64) -> AtomicWrite {
AtomicWrite::new((1..=count).map(|seq| event(persistence_id, seq)).collect())
.expect("multi-event atomic write is valid")
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn postgres_group_commit_rejects_stale_write_while_sibling_commits() {
let Some(database_url) = database_url() else {
return;
};
let (backend, schema) = group_commit_backend(database_url).await;
backend.force_group_commit_min_group(2);
let good = PersistenceId::new("group-commit-stale", "good").unwrap();
let stale = PersistenceId::new("group-commit-stale", "stale").unwrap();
let good_task = {
let backend = backend.clone();
let good = good.clone();
tokio::spawn(async move { backend.append_one(atomic(event(&good, 1))).await })
};
let stale_task = {
let backend = backend.clone();
let stale = stale.clone();
tokio::spawn(async move { backend.append_one(atomic(event(&stale, 5))).await })
};
let good_outcome = good_task
.await
.unwrap()
.expect("sibling append must not fail");
let stale_outcome = stale_task
.await
.unwrap()
.expect("stale append must not fail");
assert_eq!(good_outcome, AppendOutcome::Accepted);
assert!(
matches!(stale_outcome, AppendOutcome::Rejected { .. }),
"stale write must be rejected, got {stale_outcome:?}"
);
let good_head = backend
.highest_sequence_nr(&good, SequenceNr::new(0))
.await
.unwrap();
let stale_head = backend
.highest_sequence_nr(&stale, SequenceNr::new(0))
.await
.unwrap();
backend.force_group_commit_min_group(0);
drop_schema(&backend, &schema).await;
assert_eq!(good_head, SequenceNr::new(1), "sibling must have committed");
assert_eq!(
stale_head,
SequenceNr::new(0),
"stale write must not persist"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn postgres_group_commit_fresh_rejected_write_leaves_no_head_row() {
let Some(database_url) = database_url() else {
return;
};
let (backend, schema) = group_commit_backend(database_url).await;
let head_table = qualified(&schema, &backend.config().tables.journal_head);
let pool = backend
.pool()
.expect("connected backend has a pool")
.clone();
backend.force_group_commit_min_group(2);
let sibling = PersistenceId::new("group-commit-fresh-seed-leak", "sibling").unwrap();
let fresh = PersistenceId::new("group-commit-fresh-seed-leak", "fresh").unwrap();
let sibling_task = {
let backend = backend.clone();
let sibling = sibling.clone();
tokio::spawn(async move { backend.append_one(atomic(event(&sibling, 1))).await })
};
let fresh_task = {
let backend = backend.clone();
let fresh = fresh.clone();
tokio::spawn(async move { backend.append_one(atomic(event(&fresh, 5))).await })
};
let sibling_outcome = sibling_task
.await
.unwrap()
.expect("sibling append must not fail");
let fresh_outcome = fresh_task
.await
.unwrap()
.expect("fresh append must not fail");
assert_eq!(sibling_outcome, AppendOutcome::Accepted);
assert!(
matches!(fresh_outcome, AppendOutcome::Rejected { .. }),
"fresh id's out-of-sequence write must be rejected, got {fresh_outcome:?}"
);
let count_head_rows =
format!("SELECT COUNT(*) AS count FROM {head_table} WHERE persistence_id = $1");
let row = sqlx::query(&count_head_rows)
.bind(fresh.id())
.fetch_one(&pool)
.await
.expect("query journal_head row count for fresh id");
let head_row_count: i64 = row.try_get("count").expect("decode head row count");
let fresh_head = backend
.highest_sequence_nr(&fresh, SequenceNr::new(0))
.await
.unwrap();
let sibling_head = backend
.highest_sequence_nr(&sibling, SequenceNr::new(0))
.await
.unwrap();
let ids = backend
.persistence_ids(QueryMode::Current)
.await
.expect("materialize persistence_ids query");
let ids: Vec<PersistenceId> = ids
.map(|id| id.expect("query persistence id"))
.collect::<Vec<_>>()
.await;
backend.force_group_commit_min_group(0);
drop_schema(&backend, &schema).await;
assert_eq!(
head_row_count, 0,
"a fresh id with zero accepted writes must not leave a journal_head row behind"
);
assert_eq!(
sibling_head,
SequenceNr::new(1),
"sibling must have committed"
);
assert_eq!(
fresh_head,
SequenceNr::new(0),
"fresh write must not persist"
);
assert!(
ids.contains(&sibling),
"the committed sibling must be advertised by persistence_ids"
);
assert!(
!ids.contains(&fresh),
"an id with zero stored events must not be advertised by persistence_ids"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn postgres_group_commit_preserves_output_order_per_caller() {
let Some(database_url) = database_url() else {
return;
};
let (backend, schema) = group_commit_backend(database_url).await;
backend.force_group_commit_min_group(3);
let id = PersistenceId::new("group-commit-order", "one").unwrap();
let outcomes = backend
.append(vec![
atomic(event(&id, 1)),
atomic(event(&id, 5)),
atomic(event(&id, 2)),
])
.await
.expect("batch append must not fail");
let seqs = current_sequence_nrs(&backend, &id, 5).await;
let head = backend
.highest_sequence_nr(&id, SequenceNr::new(0))
.await
.unwrap();
backend.force_group_commit_min_group(0);
drop_schema(&backend, &schema).await;
assert_eq!(outcomes.len(), 3);
assert_eq!(outcomes[0], AppendOutcome::Accepted);
assert!(matches!(outcomes[1], AppendOutcome::Rejected { .. }));
assert_eq!(outcomes[2], AppendOutcome::Accepted);
assert_eq!(seqs, vec![1, 2]);
assert_eq!(head, SequenceNr::new(2));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn postgres_group_commit_same_id_advances_shadow_head_in_admission_order() {
let Some(database_url) = database_url() else {
return;
};
let (backend, schema) = group_commit_backend(database_url).await;
backend.force_group_commit_min_group(2);
let id = PersistenceId::new("group-commit-shadow", "one").unwrap();
let outcomes = backend
.append(vec![atomic(event(&id, 1)), atomic(event(&id, 2))])
.await
.expect("batch append must not fail");
let seqs = current_sequence_nrs(&backend, &id, 2).await;
let head = backend
.highest_sequence_nr(&id, SequenceNr::new(0))
.await
.unwrap();
backend.force_group_commit_min_group(0);
drop_schema(&backend, &schema).await;
assert_eq!(
outcomes,
vec![AppendOutcome::Accepted, AppendOutcome::Accepted]
);
assert_eq!(seqs, vec![1, 2]);
assert_eq!(head, SequenceNr::new(2));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn postgres_group_commit_rejected_write_does_not_advance_shadow_head() {
let Some(database_url) = database_url() else {
return;
};
let (backend, schema) = group_commit_backend(database_url).await;
backend.force_group_commit_min_group(2);
let id = PersistenceId::new("group-commit-noadvance", "one").unwrap();
let outcomes = backend
.append(vec![atomic(event(&id, 5)), atomic(event(&id, 1))])
.await
.expect("batch append must not fail");
let seqs = current_sequence_nrs(&backend, &id, 5).await;
let head = backend
.highest_sequence_nr(&id, SequenceNr::new(0))
.await
.unwrap();
backend.force_group_commit_min_group(0);
drop_schema(&backend, &schema).await;
assert!(matches!(outcomes[0], AppendOutcome::Rejected { .. }));
assert_eq!(outcomes[1], AppendOutcome::Accepted);
assert_eq!(seqs, vec![1]);
assert_eq!(head, SequenceNr::new(1));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn postgres_group_commit_execution_failure_fails_accepted_but_keeps_rejection() {
let Some(database_url) = database_url() else {
return;
};
let (backend, schema) = group_commit_backend(database_url).await;
let journal_table = qualified(&schema, &backend.config().tables.event_journal);
let head_table = qualified(&schema, &backend.config().tables.journal_head);
let pool = backend
.pool()
.expect("connected backend has a pool")
.clone();
let accepted = PersistenceId::new("group-commit-fail", "accepted").unwrap();
let conflicting = PersistenceId::new("group-commit-fail", "conflicting").unwrap();
let rejected = PersistenceId::new("group-commit-fail", "rejected").unwrap();
let insert_head = format!(
"INSERT INTO {head_table} (persistence_id, highest_seq_nr, deleted_to) VALUES ($1, 0, 0)"
);
sqlx::query(&insert_head)
.bind(conflicting.id())
.execute(&pool)
.await
.expect("insert head at watermark 0 for the conflicting id");
let insert_rogue = format!(
"INSERT INTO {journal_table} \
(persistence_id, seq_nr, slice, entity_type, writer, adapter_manifest, \
ser_id, ser_manifest, payload) \
VALUES ($1, 1, 0, 'group-commit-fail', 'fixture', 'fixture-adapter', 7, 'fixture-v1', $2)"
);
sqlx::query(&insert_rogue)
.bind(conflicting.id())
.bind(vec![0_u8])
.execute(&pool)
.await
.expect("insert rogue row that forces a primary-key violation");
backend.force_group_commit_min_group(3);
let accepted_task = {
let backend = backend.clone();
let accepted = accepted.clone();
tokio::spawn(async move { backend.append_one(atomic(event(&accepted, 1))).await })
};
let conflicting_task = {
let backend = backend.clone();
let conflicting = conflicting.clone();
tokio::spawn(async move { backend.append_one(atomic(event(&conflicting, 1))).await })
};
let rejected_task = {
let backend = backend.clone();
let rejected = rejected.clone();
tokio::spawn(async move { backend.append_one(atomic(event(&rejected, 9))).await })
};
let accepted_result = accepted_task.await.unwrap();
let conflicting_result = conflicting_task.await.unwrap();
let rejected_result = rejected_task.await.unwrap();
backend.force_group_commit_min_group(0);
drop_schema(&backend, &schema).await;
assert!(
matches!(accepted_result, Err(PersistenceError::Failure(_))),
"an accepted sibling must fail on a shared commit failure, got {accepted_result:?}"
);
assert!(
matches!(conflicting_result, Err(PersistenceError::Failure(_))),
"the conflicting write must fail, got {conflicting_result:?}"
);
assert!(
matches!(rejected_result, Ok(AppendOutcome::Rejected { .. })),
"a deterministic rejection must never be relabeled a failure, got {rejected_result:?}"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn postgres_group_commit_dropped_reply_receiver_does_not_cancel_persistence() {
let Some(database_url) = database_url() else {
return;
};
let (backend, schema) = group_commit_backend(database_url).await;
backend.force_group_commit_min_group(2);
let dropped = PersistenceId::new("group-commit-dropped", "dropped").unwrap();
let sibling = PersistenceId::new("group-commit-dropped", "sibling").unwrap();
let _ = tokio::time::timeout(
Duration::from_millis(150),
backend.append_one(atomic(event(&dropped, 1))),
)
.await;
let sibling_outcome = backend
.append_one(atomic(event(&sibling, 1)))
.await
.expect("sibling append must not fail");
let dropped_head = backend
.highest_sequence_nr(&dropped, SequenceNr::new(0))
.await
.unwrap();
backend.force_group_commit_min_group(0);
drop_schema(&backend, &schema).await;
assert_eq!(sibling_outcome, AppendOutcome::Accepted);
assert_eq!(
dropped_head,
SequenceNr::new(1),
"a dropped reply receiver must not cancel the write"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn postgres_group_commit_splits_insert_at_the_bind_limit() {
let Some(database_url) = database_url() else {
return;
};
let (backend, schema) = group_commit_backend(database_url).await;
backend.force_group_commit_min_group(2);
let first = PersistenceId::new("group-commit-bind", "first").unwrap();
let second = PersistenceId::new("group-commit-bind", "second").unwrap();
let first_task = {
let backend = backend.clone();
let first = first.clone();
tokio::spawn(async move { backend.append_one(multi_event_write(&first, 3_000)).await })
};
let second_task = {
let backend = backend.clone();
let second = second.clone();
tokio::spawn(async move { backend.append_one(multi_event_write(&second, 3_000)).await })
};
let first_outcome = first_task
.await
.unwrap()
.expect("first append must not fail");
let second_outcome = second_task
.await
.unwrap()
.expect("second append must not fail");
let first_head = backend
.highest_sequence_nr(&first, SequenceNr::new(0))
.await
.unwrap();
let second_head = backend
.highest_sequence_nr(&second, SequenceNr::new(0))
.await
.unwrap();
backend.force_group_commit_min_group(0);
drop_schema(&backend, &schema).await;
assert_eq!(first_outcome, AppendOutcome::Accepted);
assert_eq!(second_outcome, AppendOutcome::Accepted);
assert_eq!(first_head, SequenceNr::new(3_000));
assert_eq!(second_head, SequenceNr::new(3_000));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn postgres_group_commit_replay_never_observes_later_sequence_first() {
let Some(database_url) = database_url() else {
return;
};
let (backend, schema) = group_commit_backend(database_url).await;
backend.force_group_commit_min_group(3);
let id = PersistenceId::new("group-commit-order2", "one").unwrap();
let outcomes = backend
.append(vec![
atomic(event(&id, 1)),
atomic(event(&id, 2)),
atomic(event(&id, 3)),
])
.await
.expect("batch append must not fail");
let seqs = current_sequence_nrs(&backend, &id, 3).await;
backend.force_group_commit_min_group(0);
drop_schema(&backend, &schema).await;
assert_eq!(
outcomes,
vec![
AppendOutcome::Accepted,
AppendOutcome::Accepted,
AppendOutcome::Accepted,
]
);
assert_eq!(seqs, vec![1, 2, 3]);
assert!(
seqs.windows(2).all(|pair| pair[0] < pair[1]),
"a per-id replay must be strictly ascending, got {seqs:?}"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn postgres_group_commit_bypass_and_flusher_race_same_id_safely() {
let Some(database_url) = database_url() else {
return;
};
let base_config = isolated_config(database_url, PostgresDeletePolicy::Soft);
let schema = base_config.schema.clone();
let mut bypass_config = base_config.clone();
bypass_config.group_commit.bypass_below = usize::MAX;
let bypass_backend = Arc::new(
PostgresPersistence::connect(bypass_config)
.await
.expect("connect always-bypass backend"),
);
let mut flusher_config = base_config;
flusher_config.group_commit.bypass_below = 0;
let flusher_backend = Arc::new(
PostgresPersistence::connect(flusher_config)
.await
.expect("connect always-flusher backend"),
);
let id = PersistenceId::new("group-commit-bypass-race", "one").unwrap();
let barrier = Arc::new(tokio::sync::Barrier::new(2));
let bypass_task = {
let backend = bypass_backend.clone();
let id = id.clone();
let barrier = barrier.clone();
tokio::spawn(async move {
barrier.wait().await;
backend.append_one(atomic(event(&id, 1))).await
})
};
let flusher_task = {
let backend = flusher_backend.clone();
let id = id.clone();
let barrier = barrier.clone();
tokio::spawn(async move {
barrier.wait().await;
backend.append_one(atomic(event(&id, 2))).await
})
};
let bypass_outcome = bypass_task
.await
.unwrap()
.expect("bypass-path append must not fail");
let flusher_outcome = flusher_task
.await
.unwrap()
.expect("flusher-path append must not fail");
let seqs = current_sequence_nrs(&bypass_backend, &id, 2).await;
let head = bypass_backend
.highest_sequence_nr(&id, SequenceNr::new(0))
.await
.unwrap();
drop_schema(&bypass_backend, &schema).await;
match (bypass_outcome, flusher_outcome) {
(AppendOutcome::Accepted, AppendOutcome::Accepted) => {
assert_eq!(seqs, vec![1, 2]);
assert_eq!(head, SequenceNr::new(2));
}
(AppendOutcome::Accepted, AppendOutcome::Rejected { .. }) => {
assert_eq!(seqs, vec![1]);
assert_eq!(head, SequenceNr::new(1));
}
other => panic!(
"bypass vs. flusher same-id race produced an impossible outcome: {other:?} \
(seqs={seqs:?}, head={head:?})"
),
}
}
fn database_url() -> Option<String> {
match std::env::var("DATABASE_URL") {
Ok(database_url) if !database_url.trim().is_empty() => Some(database_url),
_ => {
eprintln!("skipping live PostgreSQL test: DATABASE_URL is unset");
None
}
}
}
fn isolated_config(
database_url: String,
delete_policy: PostgresDeletePolicy,
) -> PostgresPersistenceConfig {
let mut config = PostgresPersistenceConfig::new(database_url);
config.schema = unique_schema();
config.delete_policy = delete_policy;
config.max_connections = 4;
config
}
fn unique_schema() -> String {
let counter = TEST_SCHEMA_COUNTER.fetch_add(1, Ordering::Relaxed);
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("test clock is after the Unix epoch")
.as_nanos();
format!("datum_ps5_{}_{}_{}", std::process::id(), nanos, counter)
}
async fn drop_schema(backend: &PostgresPersistence, schema: &str) {
let statement = format!("DROP SCHEMA IF EXISTS {} CASCADE", quote_identifier(schema));
sqlx::query(&statement)
.execute(backend.pool().expect("initialized backend has a pool"))
.await
.expect("drop isolated PostgreSQL test schema");
}
fn finish_conformance(
result: Result<PersistenceResult<()>, Box<dyn std::any::Any + Send + 'static>>,
) {
match result {
Ok(result) => result.expect("PostgreSQL conformance suite returned a backend error"),
Err(panic) => resume_unwind(panic),
}
}
fn event(persistence_id: &PersistenceId, sequence_nr: u64) -> StoredEvent {
StoredEvent {
persistence_id: persistence_id.clone(),
sequence_nr: SequenceNr::new(sequence_nr),
adapter_manifest: "postgres-conformance-adapter-v1".into(),
writer_uuid: "postgres-conformance-writer".into(),
timestamp_ms: 1_000,
metadata: None,
payload: SerializedPayload::new(
SerializerId::new(7),
"postgres-conformance-v1",
sequence_nr.to_be_bytes().to_vec(),
),
tags: vec![Tag::new("postgres-conformance")],
}
}
fn atomic(event: StoredEvent) -> AtomicWrite {
AtomicWrite::new(vec![event]).expect("test atomic write is valid")
}
fn id_matching_slice(
entity_type: &str,
prefix: &str,
predicate: impl Fn(u16) -> bool,
) -> PersistenceId {
(0_u64..100_000)
.map(|suffix| {
PersistenceId::new(entity_type, format!("{prefix}-{suffix}"))
.expect("generated slice-query persistence id is valid")
})
.find(|persistence_id| predicate(slice_for(persistence_id.id())))
.expect("the deterministic fixture search finds a matching slice")
}
async fn collect_query_stream(mut stream: QueryStream<EventEnvelope>) -> Vec<EventEnvelope> {
tokio::time::timeout(Duration::from_secs(2), async {
let mut events = Vec::new();
while let Some(envelope) = stream.next().await {
events.push(envelope.expect("PostgreSQL query stream succeeds"));
}
events
})
.await
.expect("current PostgreSQL query completes")
}
#[derive(Clone, Copy)]
struct QueryI64Serializer;
impl PersistenceSerializer<i64> for QueryI64Serializer {
fn serializer_id(&self) -> SerializerId {
SerializerId::new(92)
}
fn manifest(&self, _value: &i64) -> String {
"postgres-query-i64-v1".to_owned()
}
fn serialize(&self, value: &i64) -> PersistenceResult<Vec<u8>> {
Ok(value.to_be_bytes().to_vec())
}
fn deserialize(&self, manifest: &str, payload: &[u8]) -> PersistenceResult<i64> {
if manifest != "postgres-query-i64-v1" || payload.len() != 8 {
return Err(PersistenceError::Serialization(
"invalid PostgreSQL query i64 frame".to_owned(),
));
}
let bytes: [u8; 8] = payload.try_into().expect("payload length was checked");
Ok(i64::from_be_bytes(bytes))
}
}
fn query_catalog() -> SerializerCatalog<i64> {
SerializerCatalog::new(QueryI64Serializer)
}
fn quote_identifier(identifier: &str) -> String {
assert!(
identifier
.chars()
.all(|character| character == '_' || character.is_ascii_alphanumeric()),
"test identifier must be conservative ASCII"
);
format!("\"{identifier}\"")
}
fn qualified(schema: &str, table: &str) -> String {
format!("{}.{}", quote_identifier(schema), quote_identifier(table))
}