use std::str::FromStr;
use std::time::Duration;
use happenstance_core::{EventId, SequencePosition};
use happenstance_sync::{Classify, FailureClass, IngestError, IngestGroup, Ingested};
use sqlx::PgPool;
use sqlx::postgres::{PgConnectOptions, PgPoolOptions};
use super::super::FIRST_ROUND_TRIP_SQL;
use super::{append, carrying, event, holds, id, ingest, log, own, peer, rows, server, shipped};
use crate::error::PostgresEventStoreError;
use crate::event_store::PostgresEventStore;
use crate::migration;
async fn named_store() -> (PostgresEventStore, String) {
let schema = server::fresh_schema_name();
let pool = server::pool_onto(&schema).await;
migration::apply(&pool)
.await
.expect("the event log's migrations apply to a fresh schema");
(PostgresEventStore::new(pool), schema)
}
fn bigint(position: SequencePosition) -> i64 {
i64::try_from(position.get()).expect("a Postgres position fits a bigint")
}
pub(super) async fn remove(pool: &PgPool, positions: &[SequencePosition]) {
let positions: Vec<i64> = positions.iter().copied().map(bigint).collect();
let deleted = sqlx::query("DELETE FROM event WHERE position = ANY($1)")
.bind(&positions)
.execute(pool)
.await
.expect("the removal commits")
.rows_affected();
assert_eq!(
usize::try_from(deleted).ok(),
Some(positions.len()),
"every position named was held: {positions:?}"
);
}
async fn position_of(store: &PostgresEventStore, id: EventId) -> SequencePosition {
carrying(&log(store).await, id).position
}
fn counts(ingested: Ingested) -> (usize, usize, usize) {
(ingested.appended, ingested.skipped, ingested.compensated)
}
async fn record(pool: &PgPool) -> Vec<(Vec<u8>, i64)> {
sqlx::query_as(
"SELECT origin_store, origin_position FROM event_removed \
ORDER BY origin_store, origin_position",
)
.fetch_all(pool)
.await
.expect("the record reads")
}
async fn sequence(pool: &PgPool) -> (i64, bool) {
sqlx::query_as("SELECT last_value, is_called FROM event_position_seq")
.fetch_one(pool)
.await
.expect("the sequence reads")
}
async fn impatient_pool(schema: &str) -> PgPool {
let options = PgConnectOptions::from_str(server::server_url().await)
.expect("the server's URL parses")
.options([("search_path", schema), ("lock_timeout", "50ms")]);
PgPoolOptions::new()
.max_connections(2)
.acquire_timeout(Duration::from_secs(5))
.connect_with(options)
.await
.expect("a second pool onto the schema opens")
}
fn lock_not_available(error: &sqlx::Error) -> bool {
error
.as_database_error()
.and_then(sqlx::error::DatabaseError::code)
.as_deref()
== Some("55P03")
}
#[tokio::test]
#[ignore = "needs a live Postgres; run with `-- --ignored` (starts a container, or set HAPPENSTANCE_POSTGRES_URL)"]
async fn a_removed_foreign_event_is_skipped_on_redelivery() {
let (store, _) = named_store().await;
let from = peer(0x61);
let batch = [
shipped(id(from, 1), 1, event("Shipped", "a")),
shipped(id(from, 2), 2, event("Shipped", "b")),
shipped(id(from, 3), 3, event("Shipped", "c")),
];
ingest(&store, &[IngestGroup::new(&batch, &[])])
.await
.expect("three foreign events land");
remove(store.pool(), &[position_of(&store, batch[1].id).await]).await;
let pruned = log(&store).await;
let again = ingest(&store, &[IngestGroup::new(&batch, &[])])
.await
.expect("a re-delivery over a removal is not an error");
assert_eq!(counts(again), (0, 3, 0), "the removed event came back");
assert_eq!(log(&store).await, pruned, "the log is the pruned log");
assert!(
!holds(&store, batch[1].id).await,
"the removed identity reads as held again"
);
}
#[tokio::test]
#[ignore = "needs a live Postgres; run with `-- --ignored` (starts a container, or set HAPPENSTANCE_POSTGRES_URL)"]
async fn a_removed_own_event_is_skipped_not_refused() {
let (store, _) = named_store().await;
let first = append(&store, &[event("Local", "a")]).await;
append(&store, &[event("Local", "b")]).await;
remove(store.pool(), &[first]).await;
let echo = EventId::new(own(&store).await, first);
let arrived = id(peer(0x62), 1);
let batch = [
shipped(echo, 0, event("Local", "a")),
shipped(arrived, 1, event("Shipped", "y")),
];
let landed = ingest(&store, &[IngestGroup::new(&batch, &[])])
.await
.expect("a removed own event is a skip, not SY-38's refusal");
assert_eq!(counts(landed), (1, 1, 0));
assert!(holds(&store, arrived).await);
assert!(
!holds(&store, echo).await,
"the removed own event came back"
);
}
#[tokio::test]
#[ignore = "needs a live Postgres; run with `-- --ignored` (starts a container, or set HAPPENSTANCE_POSTGRES_URL)"]
async fn an_own_claim_never_held_is_refused_after_a_removal() {
let (store, _) = named_store().await;
let first = append(&store, &[event("Local", "a")]).await;
let head = append(&store, &[event("Local", "b")]).await;
remove(store.pool(), &[first]).await;
let never = head
.next()
.and_then(SequencePosition::next)
.and_then(SequencePosition::next)
.expect("three past a fresh head exist");
let claim = EventId::new(own(&store).await, never);
let batch = [
shipped(id(peer(0x63), 1), 1, event("Shipped", "z")),
shipped(claim, 2, event("Local", "forged")),
];
let before = log(&store).await;
let refused = ingest(&store, &[IngestGroup::new(&batch, &[])])
.await
.expect_err("a claim never held is refused after a removal too");
assert!(
matches!(refused, IngestError::UnheldOwnIdentity { id, .. } if id == claim),
"expected UnheldOwnIdentity naming {claim}, got {refused:?}"
);
assert_eq!(refused.class(), FailureClass::Stop);
assert_eq!(
log(&store).await,
before,
"a refused ingest wrote something"
);
}
#[tokio::test]
#[ignore = "needs a live Postgres; run with `-- --ignored` (starts a container, or set HAPPENSTANCE_POSTGRES_URL)"]
async fn a_group_of_removed_and_held_events_writes_no_compensation() {
let (store, _) = named_store().await;
let from = peer(0x64);
let group = [
shipped(id(from, 1), 1, event("Reserved", "seat-1")),
shipped(id(from, 2), 2, event("Reserved", "seat-2")),
];
let answer = [event("ReservationCompensated", "seat")];
ingest(&store, &[IngestGroup::new(&group, &answer)])
.await
.expect("the group lands with its compensation");
remove(store.pool(), &[position_of(&store, group[0].id).await]).await;
let pruned = log(&store).await;
let again = ingest(&store, &[IngestGroup::new(&group, &answer)])
.await
.expect("a re-delivery is not an error");
assert_eq!(
counts(again),
(0, 2, 0),
"compensated a group that landed nothing"
);
assert_eq!(log(&store).await, pruned);
}
#[tokio::test]
#[ignore = "needs a live Postgres; run with `-- --ignored` (starts a container, or set HAPPENSTANCE_POSTGRES_URL)"]
async fn a_group_with_one_new_event_lands_it_and_compensates_once() {
let (store, _) = named_store().await;
let from = peer(0x65);
let group = [
shipped(id(from, 1), 1, event("Reserved", "seat-1")),
shipped(id(from, 2), 2, event("Reserved", "seat-2")),
];
let (removed, fresh) = (&group[0], &group[1]);
ingest(&store, &[IngestGroup::new(&group[..1], &[])])
.await
.expect("the first event lands");
remove(store.pool(), &[position_of(&store, removed.id).await]).await;
let answer = [event("ReservationCompensated", "seat")];
let landed = ingest(&store, &[IngestGroup::new(&group, &answer)])
.await
.expect("a group with one new event lands");
assert_eq!(counts(landed), (1, 1, 1));
assert!(holds(&store, fresh.id).await);
assert!(!holds(&store, removed.id).await);
assert_eq!(rows(&store).await, 2, "the new event and its answer");
}
#[tokio::test]
#[ignore = "needs a live Postgres; run with `-- --ignored` (starts a container, or set HAPPENSTANCE_POSTGRES_URL)"]
async fn a_skip_against_a_removal_draws_no_position() {
let (store, _) = named_store().await;
let from = peer(0x66);
let batch = [
shipped(id(from, 1), 1, event("Shipped", "a")),
shipped(id(from, 2), 2, event("Shipped", "b")),
];
ingest(&store, &[IngestGroup::new(&batch, &[])])
.await
.expect("two foreign events land");
remove(store.pool(), &[position_of(&store, batch[0].id).await]).await;
let before = (sequence(store.pool()).await, log(&store).await);
let again = ingest(&store, &[IngestGroup::new(&batch[..1], &[])])
.await
.expect("a re-delivery is not an error");
assert_eq!(counts(again), (0, 1, 0));
assert_eq!(
(sequence(store.pool()).await, log(&store).await),
before,
"a skip-only call drew a position or wrote a row"
);
}
#[tokio::test]
#[ignore = "needs a live Postgres; run with `-- --ignored` (starts a container, or set HAPPENSTANCE_POSTGRES_URL)"]
async fn the_record_reaches_every_handle_and_survives_reopen() {
let (earlier, schema) = named_store().await;
let batch = [shipped(id(peer(0x67), 1), 1, event("Shipped", "a"))];
ingest(&earlier, &[IngestGroup::new(&batch, &[])])
.await
.expect("a foreign event lands");
remove(earlier.pool(), &[position_of(&earlier, batch[0].id).await]).await;
let later = PostgresEventStore::new(earlier.pool().clone());
for (which, store) in [("earlier", &earlier), ("later", &later)] {
let again = ingest(store, &[IngestGroup::new(&batch, &[])])
.await
.expect("a re-delivery is not an error");
assert_eq!(counts(again), (0, 1, 0), "through the {which} handle");
}
earlier.pool().close().await;
drop((earlier, later));
let reopened = PostgresEventStore::new(server::pool_onto(&schema).await);
let again = ingest(&reopened, &[IngestGroup::new(&batch, &[])])
.await
.expect("a re-delivery is not an error");
assert_eq!(counts(again), (0, 1, 0), "after reopen");
assert!(!holds(&reopened, batch[0].id).await);
}
#[tokio::test]
#[ignore = "needs a live Postgres; run with `-- --ignored` (starts a container, or set HAPPENSTANCE_POSTGRES_URL)"]
async fn a_raw_delete_records_the_identity_whatever_the_search_path() {
let (store, schema) = named_store().await;
let local = append(&store, &[event("Local", "a")]).await;
let foreign = [shipped(id(peer(0xF0), 7), 7, event("Shipped", "b"))];
ingest(&store, &[IngestGroup::new(&foreign, &[])])
.await
.expect("a foreign event lands");
let foreign_at = position_of(&store, foreign[0].id).await;
let decoy = server::fresh_pool().await;
let deleted = sqlx::query(&format!(
r#"DELETE FROM "{schema}".event WHERE position = ANY($1)"#
))
.bind([bigint(local), bigint(foreign_at)])
.execute(&decoy)
.await
.expect("a delete through another schema's path commits")
.rows_affected();
assert_eq!(deleted, 2);
let mut expected = vec![
(own(&store).await.to_bytes().to_vec(), bigint(local)),
(peer(0xF0).to_bytes().to_vec(), 7),
];
expected.sort();
assert_eq!(record(store.pool()).await, expected);
assert!(
record(&decoy).await.is_empty(),
"the trigger wrote to the deleting session's search_path"
);
}
#[tokio::test]
#[ignore = "needs a live Postgres; run with `-- --ignored` (starts a container, or set HAPPENSTANCE_POSTGRES_URL)"]
async fn a_row_with_no_origin_records_nothing() {
let (store, _) = named_store().await;
let pool = store.pool();
sqlx::query(
"INSERT INTO event (position, event_type, data, metadata, tags, origin_store, \
origin_position, recorded_at) VALUES (nextval('event_position_seq'), 'Legacy', \
'\\x00', NULL, '{}', NULL, NULL, 0)",
)
.execute(pool)
.await
.expect("a row from before stamping inserts");
sqlx::query("DELETE FROM event WHERE event_type = 'Legacy'")
.execute(pool)
.await
.expect("deleting a row with no origin is not refused");
assert!(record(pool).await.is_empty());
}
#[tokio::test]
#[ignore = "needs a live Postgres; run with `-- --ignored` (starts a container, or set HAPPENSTANCE_POSTGRES_URL)"]
async fn a_rolled_back_delete_records_nothing() {
let (store, _) = named_store().await;
let local = append(&store, &[event("Local", "a")]).await;
let mut operator = store.pool().begin().await.expect("a transaction opens");
sqlx::query("DELETE FROM event WHERE position = $1")
.bind(bigint(local))
.execute(&mut *operator)
.await
.expect("the delete runs");
operator.rollback().await.expect("the delete rolls back");
assert!(record(store.pool()).await.is_empty());
assert!(holds(&store, EventId::new(own(&store).await, local)).await);
}
#[tokio::test]
#[ignore = "needs a live Postgres; run with `-- --ignored` (starts a container, or set HAPPENSTANCE_POSTGRES_URL)"]
async fn a_version_two_schema_upgrades_in_place_and_reapplies_as_a_no_op() {
let pool = server::empty_pool().await;
sqlx::raw_sql(migration::WRITE_GROUP_LOG)
.execute(&pool)
.await
.expect("a version-2 build's migrations apply");
let store = PostgresEventStore::new(pool.clone());
let before_upgrade = append(&store, &[event("Local", "early")]).await;
let kept = append(&store, &[event("Local", "kept")]).await;
remove(&pool, &[before_upgrade]).await;
for _ in 0..2 {
migration::apply(&pool)
.await
.expect("this build's migrations apply over version 2");
}
let triggers: i64 = sqlx::query_scalar(
"SELECT count(*) FROM pg_trigger \
WHERE tgrelid = 'event'::regclass AND tgname = 'event_record_removal'",
)
.fetch_one(&pool)
.await
.expect("the catalogue reads");
assert_eq!(triggers, 1, "applying twice made a second trigger");
assert!(
record(&pool).await.is_empty(),
"a removal from before the upgrade was recorded"
);
assert!(holds(&store, EventId::new(own(&store).await, kept)).await);
remove(&pool, &[kept]).await;
assert_eq!(
record(&pool).await,
[(own(&store).await.to_bytes().to_vec(), bigint(kept))]
);
}
#[tokio::test]
#[ignore = "needs a live Postgres; run with `-- --ignored` (starts a container, or set HAPPENSTANCE_POSTGRES_URL)"]
async fn a_removal_waits_for_an_ingest_in_flight() {
let (store, schema) = named_store().await;
let local = append(&store, &[event("Local", "a")]).await;
let mut ingesting = store.pool().begin().await.expect("A opens");
sqlx::raw_sql(FIRST_ROUND_TRIP_SQL)
.fetch_one(&mut *ingesting)
.await
.expect("A takes the record's lock and reads the identity");
let impatient = impatient_pool(&schema).await;
let refused = sqlx::query("DELETE FROM event WHERE position = $1")
.bind(bigint(local))
.execute(&impatient)
.await
.expect_err("a removal cannot record while an ingest holds the lock");
assert!(lock_not_available(&refused), "{refused:?}");
assert_eq!(
PostgresEventStoreError::from(refused).class(),
FailureClass::Transient
);
assert!(holds(&store, EventId::new(own(&store).await, local)).await);
ingesting.commit().await.expect("A commits");
remove(&impatient, &[local]).await;
assert_eq!(record(store.pool()).await.len(), 1);
}
#[tokio::test]
#[ignore = "needs a live Postgres; run with `-- --ignored` (starts a container, or set HAPPENSTANCE_POSTGRES_URL)"]
async fn an_ingest_waits_for_a_removal_in_flight() {
let (store, schema) = named_store().await;
let batch = [shipped(id(peer(0x68), 1), 1, event("Shipped", "a"))];
ingest(&store, &[IngestGroup::new(&batch, &[])])
.await
.expect("a foreign event lands");
let at = position_of(&store, batch[0].id).await;
let mut removing = store.pool().begin().await.expect("B opens");
let removing_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")
.fetch_one(&mut *removing)
.await
.expect("B names its backend");
sqlx::query("DELETE FROM event WHERE position = $1")
.bind(bigint(at))
.execute(&mut *removing)
.await
.expect("B deletes, and its trigger takes the record's lock");
let impatient = PostgresEventStore::new(impatient_pool(&schema).await);
let refused = ingest(&impatient, &[IngestGroup::new(&batch, &[])])
.await
.expect_err("an ingest cannot read the record while a removal holds it");
assert!(
matches!(&refused, IngestError::Store(PostgresEventStoreError::Driver(error)) if lock_not_available(error)),
"{refused:?}"
);
assert_eq!(refused.class(), FailureClass::Transient);
let removed = batch[0].id;
let ingesting = tokio::spawn({
let store = store.clone();
async move {
happenstance_sync::SendIngestStore::ingest(&store, &[IngestGroup::new(&batch, &[])])
.await
}
});
wait_until_blocked_behind(store.pool(), removing_pid).await;
removing.commit().await.expect("B commits");
let again = ingesting
.await
.expect("the ingest task finishes")
.expect("after the removal commits, the re-delivery is not an error");
assert_eq!(counts(again), (0, 1, 0), "the removed event came back");
assert!(!holds(&store, removed).await);
}
const BLOCKED_BEHIND_SQL: &str = "SELECT count(*) FROM pg_stat_activity \
WHERE datname = current_database() AND $1 = ANY(pg_blocking_pids(pid))";
const BLOCK_DEADLINE: Duration = Duration::from_secs(30);
const BLOCK_POLL: Duration = Duration::from_millis(10);
async fn wait_until_blocked_behind(pool: &PgPool, pid: i32) {
let blocked = async {
loop {
let waiting: i64 = sqlx::query_scalar(BLOCKED_BEHIND_SQL)
.bind(pid)
.fetch_one(pool)
.await
.expect("pg_stat_activity reads");
if waiting > 0 {
return;
}
tokio::time::sleep(BLOCK_POLL).await;
}
};
tokio::time::timeout(BLOCK_DEADLINE, blocked)
.await
.unwrap_or_else(|_| {
panic!("a broken arrangement: nothing blocked behind backend {pid} within {BLOCK_DEADLINE:?}")
});
}