#![cfg(any(feature = "libpq", feature = "rustls-tls"))]
use pg_walstream::snapshot::SnapshotOutcome;
use pg_walstream::{
CancellationToken, ChangeEvent, EventType, LogicalReplicationStream, PgReplicationConnection,
ReplicationSlotOptions, ReplicationStreamConfig, RetryConfig, StreamingMode,
};
use std::time::Duration;
const TEST_TIMEOUT: Duration = Duration::from_secs(60);
fn replication_conn_string() -> String {
std::env::var("DATABASE_URL").unwrap_or_else(|_| {
"postgresql://postgres:postgres@localhost:5432/test_walstream?replication=database"
.to_string()
})
}
fn regular_conn_string() -> String {
std::env::var("DATABASE_URL_REGULAR").unwrap_or_else(|_| {
let repl = replication_conn_string();
repl.replace("?replication=database", "")
.replace("&replication=database", "")
})
}
fn regular_conn() -> PgReplicationConnection {
PgReplicationConnection::connect(®ular_conn_string()).expect("regular connection")
}
fn server_version() -> i32 {
regular_conn().server_version()
}
fn snapshot_config(slot: &str, publication: &str) -> ReplicationStreamConfig {
ReplicationStreamConfig::new(
slot.to_string(),
publication.to_string(),
2,
StreamingMode::On,
Duration::from_secs(10),
Duration::from_secs(30),
Duration::from_secs(60),
RetryConfig::default(),
)
}
fn drop_slot(slot: &str) {
let mut conn = PgReplicationConnection::connect(&replication_conn_string())
.expect("drop_slot: replication connection");
for attempt in 0..50 {
let _ = conn.exec(&format!(
"SELECT pg_terminate_backend(active_pid) FROM pg_replication_slots \
WHERE slot_name = '{slot}' AND active_pid IS NOT NULL"
));
let _ = conn.exec(&format!(
"SELECT pg_drop_replication_slot('{slot}') WHERE EXISTS \
(SELECT 1 FROM pg_replication_slots WHERE slot_name = '{slot}')"
));
match conn.exec(&format!(
"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '{slot}'"
)) {
Ok(r) if probe_count(&r) == 0 => return,
_ => {}
}
if attempt == 49 {
panic!("could not drop replication slot {slot}; it would pin WAL for the whole run");
}
std::thread::sleep(Duration::from_millis(100));
}
}
struct SlotGuard(&'static str);
impl Drop for SlotGuard {
fn drop(&mut self) {
if std::thread::panicking() {
let _ = std::panic::catch_unwind(|| drop_slot(self.0));
} else {
drop_slot(self.0);
}
}
}
fn slot_guard(slot: &'static str) -> SlotGuard {
drop_slot(slot);
SlotGuard(slot)
}
fn setup(table: &str, publication: &str, rows: &[&str]) {
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {table} CASCADE"));
conn.exec(&format!(
"CREATE TABLE {table} (id SERIAL PRIMARY KEY, name TEXT)"
))
.expect("create table");
for name in rows {
conn.exec(&format!(
"INSERT INTO {table} (name) VALUES ('{}')",
name.replace('\'', "''")
))
.expect("seed");
}
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE {table}"
))
.expect("create publication");
}
async fn collect_snapshot(
stream: LogicalReplicationStream,
) -> (Vec<ChangeEvent>, LogicalReplicationStream) {
match stream.snapshot().await.expect("snapshot") {
SnapshotOutcome::Unavailable(s) => (Vec::new(), s),
SnapshotOutcome::Available(snap) => {
let mut events = snap.events();
let mut collected = Vec::new();
while let Some(event) = events.next_event().await.expect("next_event") {
collected.push(event);
}
(collected, events.finish().await.expect("finish"))
}
}
}
fn probe_count(result: &pg_walstream::PgResult) -> i64 {
result
.get_value(0, 0)
.and_then(|v| v.parse().ok())
.unwrap_or(-1)
}
async fn collect_snapshot_from(
outcome: SnapshotOutcome,
) -> (Vec<ChangeEvent>, LogicalReplicationStream) {
match outcome {
SnapshotOutcome::Unavailable(s) => (Vec::new(), s),
SnapshotOutcome::Available(snap) => {
let mut events = snap.events();
let mut collected = Vec::new();
while let Some(event) = events.next_event().await.expect("next_event") {
collected.push(event);
}
(collected, events.finish().await.expect("finish"))
}
}
}
fn insert_name(event: &ChangeEvent) -> String {
match &event.event_type {
EventType::Insert { data, .. } => data
.get("name")
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string(),
other => panic!("expected an Insert, got {other:?}"),
}
}
async fn drain_inserts(stream: &mut LogicalReplicationStream, want: usize) -> Vec<ChangeEvent> {
let token = CancellationToken::new();
let guard = token.clone();
tokio::spawn(async move {
tokio::time::sleep(Duration::from_secs(10)).await;
guard.cancel();
});
let mut out = Vec::new();
while out.len() < want {
match stream.next_event(&token).await {
Ok(event) if matches!(event.event_type, EventType::Insert { .. }) => out.push(event),
Ok(_) => {}
Err(e) if e.is_cancelled() => break,
Err(e) => panic!("stream failed after {} inserts: {e}", out.len()),
}
}
out
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn snapshot_copies_all_published_rows() {
let (slot, table, publication) = ("it_snap_all", "snap_all", "snap_all_pub");
let _guard = slot_guard(slot);
setup(table, publication, &["alice", "bob", "charlie"]);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let (events, _stream) = tokio::time::timeout(TEST_TIMEOUT, collect_snapshot(stream))
.await
.expect("timed out");
let mut names: Vec<String> = events.iter().map(insert_name).collect();
names.sort();
assert_eq!(names, vec!["alice", "bob", "charlie"]);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn snapshot_events_carry_the_consistent_point_lsn() {
let (slot, table, publication) = ("it_snap_lsn", "snap_lsn", "snap_lsn_pub");
let _guard = slot_guard(slot);
setup(table, publication, &["a", "b"]);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let outcome = stream.snapshot().await.expect("snapshot");
let SnapshotOutcome::Available(snap) = outcome else {
panic!("expected a snapshot for a fresh slot");
};
let point = snap.consistent_point();
assert_ne!(point.value(), 0, "consistent_point must be captured");
let mut events = snap.events();
let mut seen = 0;
while let Some(event) = events.next_event().await.expect("next_event") {
assert_eq!(event.lsn, point, "every snapshot row carries the same LSN");
seen += 1;
}
events.finish().await.expect("finish");
assert_eq!(seen, 2, "premise: both seeded rows must have been copied");
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn snapshot_then_stream_has_no_gap_and_no_duplicate() {
let (slot, table, publication) = ("it_snap_gap", "snap_gap", "snap_gap_pub");
let _guard = slot_guard(slot);
setup(table, publication, &["before_1", "before_2"]);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let outcome = stream.snapshot().await.expect("snapshot");
let SnapshotOutcome::Available(snap) = outcome else {
panic!("expected a snapshot");
};
regular_conn()
.exec(&format!("INSERT INTO {table} (name) VALUES ('after')"))
.expect("insert after export");
let mut events = snap.events();
let mut snapshot_names = Vec::new();
while let Some(event) = events.next_event().await.expect("next_event") {
snapshot_names.push(insert_name(&event));
}
let mut stream = events.finish().await.expect("finish");
snapshot_names.sort();
assert_eq!(
snapshot_names,
vec!["before_1", "before_2"],
"the post-export row must NOT be in the snapshot"
);
stream.start(None).await.expect("start");
let live = tokio::time::timeout(TEST_TIMEOUT, drain_inserts(&mut stream, 1))
.await
.expect("timed out");
let live_names: Vec<String> = live.iter().map(insert_name).collect();
assert_eq!(
live_names.iter().filter(|n| *n == "after").count(),
1,
"the post-export row must arrive on the stream exactly once, got {live_names:?}"
);
assert!(
!live_names.iter().any(|n| n.starts_with("before_")),
"snapshot rows must not be replayed on the stream: {live_names:?}"
);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn copy_text_values_are_byte_identical_to_pgoutput() {
let (slot, table, publication) = ("it_snap_bytes", "snap_bytes", "snap_bytes_pub");
let _guard = slot_guard(slot);
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {table} CASCADE"));
conn.exec(&format!(
"CREATE TABLE {table} (
id SERIAL PRIMARY KEY,
t_tab TEXT, t_nl TEXT, t_cr TEXT, t_bs TEXT, t_lit_null TEXT,
t_empty TEXT, t_null TEXT,
b_bytea BYTEA, j_jsonb JSONB, a_arr TEXT[], n_num NUMERIC, ts TIMESTAMPTZ
)"
))
.expect("create table");
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE {table}"
))
.expect("create publication");
let insert = format!(
"INSERT INTO {table}
(t_tab, t_nl, t_cr, t_bs, t_lit_null, t_empty, t_null,
b_bytea, j_jsonb, a_arr, n_num, ts)
VALUES (E'a\\tb', E'a\\nb', E'a\\rb', E'a\\\\b', E'\\\\N', '', NULL,
'\\x48656c6c6f'::bytea, '{{\"k\": [1, 2]}}'::jsonb,
ARRAY['x','y'], 1234.5678, '2026-01-02 03:04:05+00')"
);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let outcome = stream.snapshot().await.expect("snapshot");
let SnapshotOutcome::Available(snap) = outcome else {
panic!("expected a snapshot");
};
conn.exec(&insert).expect("insert");
let mut stream = snap.abandon().await.expect("abandon");
stream.start(None).await.expect("start");
let live = tokio::time::timeout(TEST_TIMEOUT, drain_inserts(&mut stream, 1))
.await
.expect("timed out");
assert!(!live.is_empty(), "no insert arrived within 10s");
let streamed = match &live[0].event_type {
EventType::Insert { data, .. } => data.clone(),
other => panic!("expected an Insert, got {other:?}"),
};
drop_slot(slot);
let slot2 = "it_snap_bytes_2";
let _guard2 = slot_guard(slot2);
let stream2 = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot2, publication),
)
.await
.expect("connect");
let (events, _s) = tokio::time::timeout(TEST_TIMEOUT, collect_snapshot(stream2))
.await
.expect("timed out");
let snapshotted = match &events[0].event_type {
EventType::Insert { data, .. } => data.clone(),
other => panic!("expected an Insert, got {other:?}"),
};
for column in [
"t_tab",
"t_nl",
"t_cr",
"t_bs",
"t_lit_null",
"t_empty",
"b_bytea",
"j_jsonb",
"a_arr",
"n_num",
"ts",
] {
assert!(
snapshotted.get(column).is_some(),
"column {column} missing from the snapshot decode"
);
assert_eq!(
snapshotted.get(column),
streamed.get(column),
"column {column} differs between COPY TEXT and pgoutput"
);
}
assert_eq!(
snapshotted.get("t_null").map(|v| v.is_null()),
streamed.get("t_null").map(|v| v.is_null()),
"NULL must decode as NULL on both paths"
);
assert_eq!(snapshotted.get("t_null").map(|v| v.is_null()), Some(true));
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn snapshot_rows_api_matches_events_api() {
let (table, publication) = ("snap_parity", "snap_parity_pub");
setup(table, publication, &["one", "two"]);
let mut from_rows = Vec::new();
let slot_rows = "it_snap_parity_rows";
let _guard_rows = slot_guard(slot_rows);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot_rows, publication),
)
.await
.expect("connect");
let SnapshotOutcome::Available(snap) = stream.snapshot().await.expect("snapshot") else {
panic!("a fresh slot must export a snapshot; {slot_rows} leaked from a previous run");
};
let mut rows = snap.rows();
while let Some(row) = rows.next_row().await.expect("next_row") {
from_rows.push(
row.data
.get("name")
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string(),
);
}
rows.finish().await.expect("finish");
let slot_events = "it_snap_parity_events";
let _guard_events = slot_guard(slot_events);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot_events, publication),
)
.await
.expect("connect");
let outcome = stream.snapshot().await.expect("snapshot");
assert!(
outcome.is_available(),
"a fresh slot must export a snapshot; {slot_events} leaked from a previous run"
);
let (events, _s) = collect_snapshot_from(outcome).await;
let from_events: Vec<String> = events.iter().map(insert_name).collect();
let mut seeded = from_rows.clone();
seeded.sort();
assert_eq!(
seeded,
vec!["one", "two"],
"premise: the row API must actually have copied the seeded rows"
);
assert_eq!(from_rows, from_events);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn typed_deserialization_works_on_snapshot_rows() {
#[derive(serde::Deserialize)]
struct Row {
id: i32,
name: String,
}
let (slot, table, publication) = ("it_snap_typed", "snap_typed", "snap_typed_pub");
let _guard = slot_guard(slot);
setup(table, publication, &["typed"]);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let (events, _s) = collect_snapshot(stream).await;
let row: Row = events[0].deserialize_insert().expect("deserialize");
assert!(row.id > 0);
assert_eq!(row.name, "typed");
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn wal_router_runs_across_snapshot_and_stream() {
use pg_walstream::router::WalRouter;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
#[derive(serde::Deserialize)]
#[allow(dead_code)]
struct Row {
name: String,
}
let (slot, table, publication) = ("it_snap_router", "snap_router", "snap_router_pub");
let _guard = slot_guard(slot);
setup(table, publication, &["r1", "r2"]);
let hits = Arc::new(AtomicUsize::new(0));
let counter = Arc::clone(&hits);
let mut router = WalRouter::new();
router.on_insert(table, move |_: Row| {
let counter = Arc::clone(&counter);
async move {
counter.fetch_add(1, Ordering::Relaxed);
Ok(())
}
});
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let stream = match stream.snapshot().await.expect("snapshot") {
SnapshotOutcome::Unavailable(s) => s,
SnapshotOutcome::Available(snap) => {
let mut events = snap.events();
router
.run_snapshot(&mut events)
.await
.expect("run_snapshot");
events.finish().await.expect("finish")
}
};
assert_eq!(
hits.load(Ordering::Relaxed),
2,
"the snapshot phase must reach the same handler as the live phase"
);
drop(stream);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn slot_already_exists_yields_unavailable() {
let (slot, table, publication) = ("it_snap_exists", "snap_exists", "snap_exists_pub");
let _guard = slot_guard(slot);
setup(table, publication, &["x"]);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let outcome = stream.snapshot().await.expect("snapshot");
assert!(
outcome.is_available(),
"a fresh slot must export a snapshot"
);
drop(outcome.skip().await.expect("skip"));
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let outcome = stream.snapshot().await.expect("snapshot");
assert!(
!outcome.is_available(),
"a pre-existing slot exports no snapshot"
);
let mut stream = outcome.skip().await.expect("skip");
stream.start(None).await.expect("start must still work");
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn abandon_returns_the_stream_and_start_succeeds() {
let (slot, table, publication) = ("it_snap_abandon", "snap_abandon", "snap_abandon_pub");
let _guard = slot_guard(slot);
setup(table, publication, &["a"]);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let SnapshotOutcome::Available(snap) = stream.snapshot().await.expect("snapshot") else {
panic!("expected a snapshot");
};
let mut stream = snap.abandon().await.expect("abandon");
stream.start(None).await.expect("start after abandon");
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn abandon_midway_drops_reader_without_hanging() {
let (slot, table, publication) = ("it_snap_mid", "snap_mid", "snap_mid_pub");
let _guard = slot_guard(slot);
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {table} CASCADE"));
conn.exec(&format!("CREATE TABLE {table} (id INT, name TEXT)"))
.expect("create");
conn.exec(&format!(
"INSERT INTO {table} SELECT g, repeat('x', 200) FROM generate_series(1, 20000) g"
))
.expect("seed");
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE {table}"
))
.expect("publication");
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let SnapshotOutcome::Available(snap) = stream.snapshot().await.expect("snapshot") else {
panic!("expected a snapshot");
};
let mut rows = snap.rows();
rows.next_row().await.expect("first row").expect("some");
tokio::time::timeout(Duration::from_secs(20), rows.abandon())
.await
.expect("abandon must not wait for the rest of the table")
.expect("abandon");
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn finish_before_completion_is_rejected() {
let (slot, table, publication) = ("it_snap_partial", "snap_partial", "snap_partial_pub");
let _guard = slot_guard(slot);
setup(table, publication, &["a", "b", "c"]);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let SnapshotOutcome::Available(snap) = stream.snapshot().await.expect("snapshot") else {
panic!("expected a snapshot");
};
let mut rows = snap.rows();
rows.next_row().await.expect("first").expect("some");
assert!(
rows.finish().await.is_err(),
"finish() on a half-consumed snapshot must fail"
);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn a_failed_snapshot_leaves_no_slot_so_a_retry_still_gets_one() {
let (slot, table, publication) = ("it_snap_cleanup", "snap_cleanup", "snap_cleanup_pub");
let _guard = slot_guard(slot);
setup(table, publication, &["a", "b"]);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let SnapshotOutcome::Available(snap) = stream.snapshot().await.expect("snapshot") else {
panic!("a fresh slot must export a snapshot");
};
let failed = snap
.events()
.run(|_| async {
Err(pg_walstream::ReplicationError::protocol(
"simulated sink failure".to_string(),
))
})
.await;
assert!(failed.is_err(), "the handler error must propagate");
let mut probe = regular_conn();
let slots = probe
.exec(&format!(
"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '{slot}'"
))
.expect("probe");
assert_eq!(
probe_count(&slots),
0,
"a failed snapshot must not leave its slot behind"
);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let outcome = stream.snapshot().await.expect("snapshot");
assert!(
outcome.is_available(),
"the retry after a failed snapshot must still export a baseline"
);
let (events, _s) = collect_snapshot_from(outcome).await;
let mut names: Vec<String> = events.iter().map(insert_name).collect();
names.sort();
assert_eq!(names, vec!["a", "b"]);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn a_snapshot_dropped_mid_consumption_leaves_no_slot() {
let (slot, table, publication) = ("it_snap_dropped", "snap_dropped", "snap_dropped_pub");
let _guard = slot_guard(slot);
setup(table, publication, &["a", "b"]);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let SnapshotOutcome::Available(snap) = stream.snapshot().await.expect("snapshot") else {
panic!("a fresh slot must export a snapshot");
};
{
let mut events = snap.events();
let failed = events
.for_each_event(|_| async {
Err(pg_walstream::ReplicationError::protocol(
"simulated sink failure".to_string(),
))
})
.await;
assert!(failed.is_err(), "the handler error must propagate");
}
let mut probe = regular_conn();
let slots = probe
.exec(&format!(
"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '{slot}'"
))
.expect("probe");
assert_eq!(
probe_count(&slots),
0,
"dropping a snapshot handle must discard its slot, or the retry silently \
streams with no baseline"
);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let outcome = stream.snapshot().await.expect("snapshot");
assert!(
outcome.is_available(),
"the retry after a dropped snapshot must still export a baseline"
);
let (events, _s) = collect_snapshot_from(outcome).await;
assert_eq!(events.len(), 2, "the retry must copy every row");
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn a_snapshot_dropped_before_consumption_leaves_no_slot() {
let (slot, table, publication) = ("it_snap_predrop", "snap_predrop", "snap_predrop_pub");
let _guard = slot_guard(slot);
setup(table, publication, &["a"]);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
{
let SnapshotOutcome::Available(snap) = stream.snapshot().await.expect("snapshot") else {
panic!("a fresh slot must export a snapshot");
};
assert_eq!(snap.tables().len(), 1, "premise: the plan resolved a table");
}
let mut probe = regular_conn();
let slots = probe
.exec(&format!(
"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '{slot}'"
))
.expect("probe");
assert_eq!(
probe_count(&slots),
0,
"a Snapshot dropped before consumption must discard its slot"
);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn a_direct_start_clears_the_exported_snapshot_name() {
let (slot, table, publication) = ("it_snap_startclear", "snap_startclear", "snap_sc_pub");
let _guard = slot_guard(slot);
setup(table, publication, &["a"]);
let mut stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
stream.start(None).await.expect("start");
assert!(
stream.exported_snapshot_name().is_none(),
"START_REPLICATION destroyed the exported snapshot; the name must not outlive it"
);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn killing_the_reader_mid_copy_poisons_the_handle_and_drops_the_slot() {
let (slot, table, publication) = ("it_snap_killrdr", "snap_killrdr", "snap_killrdr_pub");
let _guard = slot_guard(slot);
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {table} CASCADE"));
conn.exec(&format!("CREATE TABLE {table} (id INT, payload TEXT)"))
.expect("create");
conn.exec(&format!(
"INSERT INTO {table} SELECT g, repeat('x', 200) FROM generate_series(1, 200000) g"
))
.expect("seed");
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE {table}"
))
.expect("publication");
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let SnapshotOutcome::Available(snap) = stream.snapshot().await.expect("snapshot") else {
panic!("expected a snapshot");
};
let mut rows = snap.rows();
rows.next_row().await.expect("first row").expect("some");
let mut probe = regular_conn();
let killed = probe
.exec(
"SELECT count(*) FROM (
SELECT pg_terminate_backend(pid) FROM pg_stat_activity
WHERE datname = current_database()
AND pid <> pg_backend_pid()
AND state = 'active'
AND query LIKE 'COPY %'
AND query LIKE '%snap_killrdr%'
) t",
)
.expect("terminate");
assert!(
probe_count(&killed) >= 1,
"premise: the reader's COPY backend must have been found and killed"
);
let err = loop {
match rows.next_row().await {
Ok(Some(_)) => continue,
Ok(None) => panic!("a killed reader must not look like a completed snapshot"),
Err(e) => break e,
}
};
assert!(
!err.is_cancelled(),
"a killed backend is a failure, not a cancellation: {err}"
);
let salvage = rows.finish().await;
assert!(
salvage.is_err(),
"a failed snapshot must not yield the stream"
);
let mut probe = regular_conn();
let slots = probe
.exec(&format!(
"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '{slot}'"
))
.expect("probe");
assert_eq!(
probe_count(&slots),
0,
"a reader failure must discard the slot, exactly like a handler failure"
);
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {table} CASCADE"));
}
#[test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
fn dropping_a_half_consumed_snapshot_terminates_and_discards_the_slot() {
let (slot, table, publication) = ("it_snap_drophang", "snap_drophang", "snap_drophang_pub");
let _guard = slot_guard(slot);
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {table} CASCADE"));
conn.exec(&format!("CREATE TABLE {table} (id INT, payload TEXT)"))
.expect("create");
conn.exec(&format!(
"INSERT INTO {table} SELECT g, repeat('y', 500) FROM generate_series(1, 200000) g"
))
.expect("seed");
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE {table}"
))
.expect("publication");
let (done_tx, done_rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime");
rt.block_on(async {
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let SnapshotOutcome::Available(snap) = stream.snapshot().await.expect("snapshot")
else {
panic!("expected a snapshot");
};
let mut rows = snap.rows();
rows.next_row().await.expect("first row").expect("some");
tokio::time::sleep(Duration::from_millis(500)).await;
drop(rows);
});
let _ = done_tx.send(());
});
done_rx
.recv_timeout(Duration::from_secs(60))
.expect("dropping a half-consumed snapshot must not hang");
let mut probe = regular_conn();
let slots = probe
.exec(&format!(
"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '{slot}'"
))
.expect("probe");
assert_eq!(
probe_count(&slots),
0,
"the drop must discard the slot even when it happens mid-COPY"
);
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {table} CASCADE"));
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn abandon_keeps_the_slot() {
let (slot, table, publication) = ("it_snap_keep", "snap_keep", "snap_keep_pub");
let _guard = slot_guard(slot);
setup(table, publication, &["a"]);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let SnapshotOutcome::Available(snap) = stream.snapshot().await.expect("snapshot") else {
panic!("expected a snapshot");
};
drop(snap.abandon().await.expect("abandon"));
let mut probe = regular_conn();
let slots = probe
.exec(&format!(
"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '{slot}'"
))
.expect("probe");
assert_eq!(
probe_count(&slots),
1,
"abandon is a deliberate choice; the slot must survive it"
);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn handler_error_consumes_the_handle() {
let (slot, table, publication) = ("it_snap_handler", "snap_handler", "snap_handler_pub");
let _guard = slot_guard(slot);
setup(table, publication, &["a", "b"]);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let SnapshotOutcome::Available(snap) = stream.snapshot().await.expect("snapshot") else {
panic!("expected a snapshot");
};
let failed = snap
.events()
.run(|_| async {
Err(pg_walstream::ReplicationError::protocol(
"handler exploded".to_string(),
))
})
.await;
assert!(failed.is_err(), "a handler error must propagate");
}
fn seed_large(table: &str, publication: &str, rows: u32) {
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {table} CASCADE"));
conn.exec(&format!("CREATE TABLE {table} (id INT, payload TEXT)"))
.expect("create");
conn.exec(&format!(
"INSERT INTO {table} SELECT g, repeat('x', 400) FROM generate_series(1, {rows}) g"
))
.expect("seed");
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE {table}"
))
.expect("publication");
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn cancelling_mid_snapshot_stops_promptly() {
let (slot, table, publication) = ("it_snap_cancel", "snap_cancel", "snap_cancel_pub");
let _guard = slot_guard(slot);
seed_large(table, publication, 200_000);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let token = CancellationToken::new();
let SnapshotOutcome::Available(snap) = stream.snapshot().await.expect("snapshot") else {
panic!("expected a snapshot");
};
let mut rows = snap.with_cancellation(token.clone()).rows();
rows.next_row().await.expect("first row").expect("some");
token.cancel();
let outcome = tokio::time::timeout(Duration::from_secs(20), async {
loop {
match rows.next_row().await {
Ok(Some(_)) => continue,
other => return other,
}
}
})
.await
.expect("cancellation must unblock the copy loop, not wait for the whole table");
let err = outcome.expect_err("a cancelled snapshot must not report completion");
assert!(
err.is_cancelled() || format!("{err}").contains("poisoned"),
"expected a cancellation, got {err:?}"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn cancelling_mid_snapshot_stops_promptly_on_the_inline_driver() {
let (slot, table, publication) = (
"it_snap_cancel_inline",
"snap_cancel_inline",
"snap_cancel_inline_pub",
);
let _guard = slot_guard(slot);
seed_large(table, publication, 200_000);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let token = CancellationToken::new();
let SnapshotOutcome::Available(snap) = stream.snapshot().await.expect("snapshot") else {
panic!("expected a snapshot");
};
let mut rows = snap.with_cancellation(token.clone()).rows();
rows.next_row().await.expect("first row").expect("some");
token.cancel();
let outcome = tokio::time::timeout(Duration::from_secs(20), async {
loop {
match rows.next_row().await {
Ok(Some(_)) => continue,
other => return other,
}
}
})
.await
.expect("cancellation must unblock the inline copy loop too");
let err = outcome.expect_err("a cancelled snapshot must not report completion");
assert!(
err.is_cancelled() || format!("{err}").contains("poisoned"),
"expected a cancellation, got {err:?}"
);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn a_cancelled_snapshot_is_not_reported_as_success() {
let (slot, table, publication) = (
"it_snap_cancel_run",
"snap_cancel_run",
"snap_cancel_run_pub",
);
let _guard = slot_guard(slot);
seed_large(table, publication, 200_000);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let token = CancellationToken::new();
let SnapshotOutcome::Available(snap) = stream.snapshot().await.expect("snapshot") else {
panic!("expected a snapshot");
};
let events = snap.with_cancellation(token.clone()).events();
let seen = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let counter = std::sync::Arc::clone(&seen);
let guard = token.clone();
let result = tokio::time::timeout(
Duration::from_secs(30),
events.run(move |_| {
let counter = std::sync::Arc::clone(&counter);
let guard = guard.clone();
async move {
if counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed) == 50 {
guard.cancel();
}
Ok(())
}
}),
)
.await
.expect("run must return once cancelled");
assert!(
result.is_err(),
"a cancelled, half-copied snapshot must NOT return the stream as success"
);
assert!(
seen.load(std::sync::atomic::Ordering::Relaxed) < 200_000,
"the test did not actually cancel mid-copy"
);
let mut probe = regular_conn();
let slots = probe
.exec(&format!(
"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '{slot}'"
))
.expect("probe");
assert_eq!(
probe_count(&slots),
0,
"a cancelled snapshot must not leave its slot behind"
);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn empty_table_produces_no_events() {
let (slot, table, publication) = ("it_snap_empty", "snap_empty", "snap_empty_pub");
let _guard = slot_guard(slot);
setup(table, publication, &[]);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let outcome = stream.snapshot().await.expect("snapshot");
assert!(
outcome.is_available(),
"a fresh slot must export a snapshot; {slot} leaked from a previous run"
);
let (events, _s) = collect_snapshot_from(outcome).await;
assert!(events.is_empty());
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn multi_table_publication_snapshots_every_table() {
let (slot, publication) = ("it_snap_multi", "snap_multi_pub");
let _guard = slot_guard(slot);
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
for t in ["snap_multi_a", "snap_multi_b"] {
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {t} CASCADE"));
conn.exec(&format!(
"CREATE TABLE {t} (id SERIAL PRIMARY KEY, name TEXT)"
))
.expect("create");
conn.exec(&format!("INSERT INTO {t} (name) VALUES ('{t}')"))
.expect("seed");
}
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE snap_multi_a, snap_multi_b"
))
.expect("publication");
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let (events, _s) = collect_snapshot(stream).await;
let mut names: Vec<String> = events.iter().map(insert_name).collect();
names.sort();
assert_eq!(names, vec!["snap_multi_a", "snap_multi_b"]);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn large_table_spans_many_copydata_frames() {
let (slot, table, publication) = ("it_snap_large", "snap_large", "snap_large_pub");
let _guard = slot_guard(slot);
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {table} CASCADE"));
conn.exec(&format!("CREATE TABLE {table} (id INT, payload TEXT)"))
.expect("create");
conn.exec(&format!(
"INSERT INTO {table} SELECT g, repeat('abc', 400) FROM generate_series(1, 50000) g"
))
.expect("seed");
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE {table}"
))
.expect("publication");
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let (events, _s) = tokio::time::timeout(Duration::from_secs(180), collect_snapshot(stream))
.await
.expect("timed out");
assert_eq!(events.len(), 50_000);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn snapshot_is_isolated_from_concurrent_writes() {
let (slot, table, publication) = ("it_snap_iso", "snap_iso", "snap_iso_pub");
let _guard = slot_guard(slot);
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {table} CASCADE"));
conn.exec(&format!("CREATE TABLE {table} (id INT, name TEXT)"))
.expect("create");
conn.exec(&format!(
"INSERT INTO {table} SELECT g, 'seed' FROM generate_series(1, 5000) g"
))
.expect("seed");
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE {table}"
))
.expect("publication");
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let SnapshotOutcome::Available(snap) = stream.snapshot().await.expect("snapshot") else {
panic!("expected a snapshot");
};
let mut rows = snap.rows();
rows.next_row().await.expect("first").expect("some");
regular_conn()
.exec(&format!("INSERT INTO {table} VALUES (99999, 'intruder')"))
.expect("concurrent insert");
let mut count = 1;
let mut saw_intruder = false;
while let Some(row) = rows.next_row().await.expect("next_row") {
count += 1;
if row.data.get("name").and_then(|v| v.as_str()) == Some("intruder") {
saw_intruder = true;
}
}
let mut stream = rows.finish().await.expect("finish");
assert_eq!(count, 5000, "the snapshot must see exactly the seeded rows");
assert!(!saw_intruder, "a concurrent write leaked into the snapshot");
stream.start(None).await.expect("start");
let live = drain_inserts(&mut stream, 2).await;
let intruders = live
.iter()
.filter(|e| match &e.event_type {
EventType::Insert { data, .. } => {
data.get("name").and_then(|v| v.as_str()) == Some("intruder")
}
_ => false,
})
.count();
assert_eq!(
intruders, 1,
"a write committed mid-copy must arrive on the stream exactly once"
);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn multiple_publications_are_all_snapshotted() {
let slot = "it_snap_multipub";
let _guard = slot_guard(slot);
let mut conn = regular_conn();
for (t, p) in [
("snap_mp_a", "snap_mp_pub_a"),
("snap_mp_b", "snap_mp_pub_b"),
] {
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {p}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {t} CASCADE"));
conn.exec(&format!(
"CREATE TABLE {t} (id SERIAL PRIMARY KEY, name TEXT)"
))
.expect("create");
conn.exec(&format!("INSERT INTO {t} (name) VALUES ('{t}')"))
.expect("seed");
conn.exec(&format!("CREATE PUBLICATION {p} FOR TABLE {t}"))
.expect("publication");
}
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, "\"snap_mp_pub_a\",\"snap_mp_pub_b\""),
)
.await
.expect("connect");
let (events, _s) = collect_snapshot(stream).await;
let mut names: Vec<String> = events.iter().map(insert_name).collect();
names.sort();
assert_eq!(names, vec!["snap_mp_a", "snap_mp_b"]);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn non_public_schema_is_qualified_correctly() {
let (slot, publication) = ("it_snap_schema", "snap_schema_pub");
let _guard = slot_guard(slot);
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec("DROP SCHEMA IF EXISTS snap_sch CASCADE");
conn.exec("CREATE SCHEMA snap_sch").expect("schema");
let _ = conn.exec("DROP TABLE IF EXISTS public.snap_dup CASCADE");
conn.exec("CREATE TABLE public.snap_dup (id INT, name TEXT)")
.expect("decoy");
conn.exec("INSERT INTO public.snap_dup VALUES (1, 'WRONG_public')")
.expect("decoy seed");
conn.exec("CREATE TABLE snap_sch.snap_dup (id INT, name TEXT)")
.expect("create");
conn.exec("INSERT INTO snap_sch.snap_dup VALUES (1, 'right_schema')")
.expect("seed");
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE snap_sch.snap_dup"
))
.expect("publication");
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let (events, _s) = collect_snapshot(stream).await;
let names: Vec<String> = events.iter().map(insert_name).collect();
assert_eq!(
names,
vec!["right_schema"],
"an unqualified COPY would have read public.snap_dup instead"
);
let EventType::Insert { schema, .. } = &events[0].event_type else {
panic!("expected an Insert");
};
assert_eq!(&**schema, "snap_sch");
let _ = conn.exec("DROP SCHEMA IF EXISTS snap_sch CASCADE");
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn wide_table_beyond_smallvec_inline_capacity() {
let (slot, table, publication) = ("it_snap_wide", "snap_wide", "snap_wide_pub");
let _guard = slot_guard(slot);
let columns: Vec<String> = (0..40).map(|i| format!("c{i} TEXT")).collect();
let values: Vec<String> = (0..40).map(|i| format!("'v{i}'")).collect();
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {table} CASCADE"));
conn.exec(&format!("CREATE TABLE {table} ({})", columns.join(", ")))
.expect("create");
conn.exec(&format!(
"INSERT INTO {table} VALUES ({})",
values.join(", ")
))
.expect("seed");
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE {table}"
))
.expect("publication");
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let (events, _s) = collect_snapshot(stream).await;
assert_eq!(events.len(), 1);
let EventType::Insert { data, .. } = &events[0].event_type else {
panic!("expected an Insert");
};
for i in 0..40 {
assert_eq!(
data.get(&format!("c{i}")).and_then(|v| v.as_str()),
Some(format!("v{i}").as_str()),
"column c{i} must survive the SmallVec spill"
);
}
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn inheritance_children_are_not_copied_twice() {
let ver = server_version();
if ver < 150000 {
println!(
"SKIP: inheritance_children_are_not_copied_twice requires PG >= 15 (server {ver})"
);
return;
}
let (slot, publication) = ("it_snap_inherit", "snap_inherit_pub");
let _guard = slot_guard(slot);
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec("DROP TABLE IF EXISTS snap_inherit_child, snap_inherit_parent CASCADE");
conn.exec("CREATE TABLE snap_inherit_parent (id INT, name TEXT)")
.expect("parent");
conn.exec("CREATE TABLE snap_inherit_child () INHERITS (snap_inherit_parent)")
.expect("child");
conn.exec("INSERT INTO snap_inherit_parent VALUES (1, 'parent-row')")
.expect("parent row");
conn.exec("INSERT INTO snap_inherit_child VALUES (2, 'child-row')")
.expect("child row");
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE snap_inherit_parent WHERE (id > 0)"
))
.expect("publication");
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let (events, _s) = collect_snapshot(stream).await;
let mut names: Vec<String> = events.iter().map(insert_name).collect();
names.sort();
assert_eq!(
names,
vec!["child-row", "parent-row"],
"each row must be copied exactly once; a duplicated child row means the \
parent's COPY read through to the child"
);
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec("DROP TABLE IF EXISTS snap_inherit_child, snap_inherit_parent CASCADE");
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn publication_column_list_is_respected() {
let ver = server_version();
if ver < 150000 {
println!("SKIP: publication_column_list_is_respected requires PG >= 15 (server {ver})");
return;
}
let (slot, table, publication) = ("it_snap_collist", "snap_collist", "snap_collist_pub");
let _guard = slot_guard(slot);
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {table} CASCADE"));
conn.exec(&format!(
"CREATE TABLE {table} (a INT PRIMARY KEY, b TEXT, c TEXT)"
))
.expect("create");
conn.exec(&format!("INSERT INTO {table} VALUES (1, 'bee', 'cee')"))
.expect("seed");
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE {table} (a, c)"
))
.expect("publication");
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let (events, _s) = collect_snapshot(stream).await;
let EventType::Insert { data, .. } = &events[0].event_type else {
panic!("expected an Insert");
};
assert_eq!(data.get("a").and_then(|v| v.as_str()), Some("1"));
assert_eq!(
data.get("c").and_then(|v| v.as_str()),
Some("cee"),
"values must align with the published column list"
);
assert!(data.get("b").is_none(), "unpublished column must be absent");
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn publication_row_filter_is_respected() {
let ver = server_version();
if ver < 150000 {
println!("SKIP: publication_row_filter_is_respected requires PG >= 15 (server {ver})");
return;
}
let (slot, table, publication) = ("it_snap_filter", "snap_filter", "snap_filter_pub");
let _guard = slot_guard(slot);
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {table} CASCADE"));
conn.exec(&format!(
"CREATE TABLE {table} (id INT PRIMARY KEY, name TEXT)"
))
.expect("create");
conn.exec(&format!(
"INSERT INTO {table} SELECT g, 'row' || g FROM generate_series(1, 10) g"
))
.expect("seed");
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE {table} WHERE (id > 5)"
))
.expect("publication");
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let (events, _s) = collect_snapshot(stream).await;
assert_eq!(
events.len(),
5,
"the row filter must be applied to the copy"
);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn partitioned_publication_via_root_uses_the_root_name() {
let (slot, publication) = ("it_snap_part", "snap_part_pub");
let root = "snap_part_root";
let _guard = slot_guard(slot);
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {root} CASCADE"));
conn.exec(&format!(
"CREATE TABLE {root} (id INT, name TEXT) PARTITION BY RANGE (id)"
))
.expect("create root");
conn.exec(&format!(
"CREATE TABLE {root}_p1 PARTITION OF {root} FOR VALUES FROM (1) TO (100)"
))
.expect("create p1");
conn.exec(&format!(
"CREATE TABLE {root}_p2 PARTITION OF {root} FOR VALUES FROM (100) TO (200)"
))
.expect("create p2");
conn.exec(&format!(
"INSERT INTO {root} VALUES (1, 'in_p1'), (150, 'in_p2')"
))
.expect("seed");
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE {root} \
WITH (publish_via_partition_root = true)"
))
.expect("publication");
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let SnapshotOutcome::Available(snap) = stream.snapshot().await.expect("snapshot") else {
panic!("expected a snapshot");
};
let table_names: Vec<String> = snap.tables().iter().map(|t| t.qualified_name()).collect();
assert_eq!(
table_names,
vec![format!("public.{root}")],
"with publish_via_partition_root the snapshot must target the ROOT, not the leaves"
);
let mut events = snap.events();
let mut seen = Vec::new();
while let Some(event) = events.next_event().await.expect("next_event") {
match &event.event_type {
EventType::Insert { table, data, .. } => {
assert_eq!(&**table, root, "events must carry the root relation name");
seen.push(
data.get("name")
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string(),
);
}
other => panic!("expected an Insert, got {other:?}"),
}
}
events.finish().await.expect("finish");
seen.sort();
assert_eq!(
seen,
vec!["in_p1", "in_p2"],
"copying the root must read every partition"
);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn partitioned_root_with_a_row_filter_still_reads_every_partition() {
let ver = server_version();
if ver < 150000 {
println!(
"SKIP: partitioned_root_with_a_row_filter_still_reads_every_partition \
requires PG >= 15 (server {ver})"
);
return;
}
let (slot, publication) = ("it_snap_partfilter", "snap_partfilter_pub");
let root = "snap_partfilter_root";
let _guard = slot_guard(slot);
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {root} CASCADE"));
conn.exec(&format!(
"CREATE TABLE {root} (id INT, name TEXT) PARTITION BY RANGE (id)"
))
.expect("create root");
conn.exec(&format!(
"CREATE TABLE {root}_p1 PARTITION OF {root} FOR VALUES FROM (1) TO (100)"
))
.expect("create p1");
conn.exec(&format!(
"CREATE TABLE {root}_p2 PARTITION OF {root} FOR VALUES FROM (100) TO (200)"
))
.expect("create p2");
conn.exec(&format!(
"INSERT INTO {root} VALUES (1, 'filtered_out'), (50, 'in_p1'), (150, 'in_p2')"
))
.expect("seed");
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE {root} WHERE (id > 10) \
WITH (publish_via_partition_root = true)"
))
.expect("publication");
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let (events, _s) = collect_snapshot(stream).await;
let mut names: Vec<String> = events.iter().map(insert_name).collect();
names.sort();
assert_eq!(
names,
vec!["in_p1", "in_p2"],
"a filtered partitioned root must still read every partition; an empty result \
means `ONLY` was applied to a root that holds no rows itself"
);
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {root} CASCADE"));
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn a_pre_existing_slot_reports_unavailable_so_the_caller_can_refuse() {
let (slot, table, publication) = ("it_snap_required", "snap_required", "snap_required_pub");
let _guard = slot_guard(slot);
setup(table, publication, &["a"]);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let outcome = stream.snapshot().await.expect("first snapshot");
assert!(
outcome.is_available(),
"premise: a fresh slot must export a snapshot"
);
drop(outcome.skip().await.expect("skip keeps the slot"));
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let refused: Result<(), String> = match stream.snapshot().await.expect("snapshot") {
SnapshotOutcome::Unavailable(_) => {
Err("bootstrap requires a fresh slot; drop the existing one".to_string())
}
SnapshotOutcome::Available(snap) => {
snap.abandon().await.expect("abandon");
Ok(())
}
};
let err = refused.expect_err("a pre-existing slot must reach the Unavailable arm");
assert!(err.contains("fresh slot"), "{err}");
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn temporary_slot_is_rejected() {
use pg_walstream::ReplicationSlotOptions;
let (slot, table, publication) = ("it_snap_temp", "snap_temp", "snap_temp_pub");
let _guard = slot_guard(slot);
setup(table, publication, &["a"]);
let config = snapshot_config(slot, publication).with_slot_options(ReplicationSlotOptions {
temporary: true,
snapshot: Some("export".to_string()),
..Default::default()
});
let stream = LogicalReplicationStream::new(&replication_conn_string(), config)
.await
.expect("connect");
match stream.snapshot().await {
Ok(_) => panic!("a temporary slot must be rejected: a reconnect loses data silently"),
Err(e) => assert!(format!("{e}").contains("temporary slot"), "{e}"),
}
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn snapshot_forces_the_export_regardless_of_slot_options() {
let (slot, table, publication) = (
"it_snap_forces_export",
"snap_forces_export",
"snap_forces_export_pub",
);
let _guard = slot_guard(slot);
setup(table, publication, &["a"]);
let config = snapshot_config(slot, publication).with_slot_options(ReplicationSlotOptions {
snapshot: Some("nothing".to_string()),
..Default::default()
});
let stream = LogicalReplicationStream::new(&replication_conn_string(), config)
.await
.expect("connect");
let (events, _stream) = tokio::time::timeout(TEST_TIMEOUT, collect_snapshot(stream))
.await
.expect("timed out");
let names: Vec<String> = events.iter().map(insert_name).collect();
assert_eq!(
names,
vec!["a"],
"snapshot() must export regardless of what the caller set"
);
}
fn setup_generated(table: &str, publication: &str, publication_opts: &str) {
let mut conn = regular_conn();
let _ = conn.exec(&format!("DROP PUBLICATION IF EXISTS {publication}"));
let _ = conn.exec(&format!("DROP TABLE IF EXISTS {table} CASCADE"));
conn.exec(&format!(
"CREATE TABLE {table} (
id SERIAL PRIMARY KEY,
a INT NOT NULL,
g INT GENERATED ALWAYS AS (a * 2) STORED
)"
))
.expect("create table");
conn.exec(&format!("INSERT INTO {table} (a) VALUES (5)"))
.expect("seed");
conn.exec(&format!(
"CREATE PUBLICATION {publication} FOR TABLE {table}{publication_opts}"
))
.expect("create publication");
}
fn insert_columns(event: &ChangeEvent) -> Vec<String> {
match &event.event_type {
EventType::Insert { data, .. } => data.iter().map(|(name, _)| name.to_string()).collect(),
other => panic!("expected an Insert, got {other:?}"),
}
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn snapshot_handles_a_generated_column() {
let (slot, table, publication) = ("it_snap_gen", "snap_gen", "snap_gen_pub");
let _guard = slot_guard(slot);
setup_generated(table, publication, "");
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let (events, _stream) = tokio::time::timeout(TEST_TIMEOUT, collect_snapshot(stream))
.await
.expect("timed out");
assert_eq!(events.len(), 1, "the seeded row must be copied");
let columns = insert_columns(&events[0]);
assert!(
columns.contains(&"id".to_string()) && columns.contains(&"a".to_string()),
"the ordinary columns must be present: {columns:?}"
);
assert!(
!columns.contains(&"g".to_string()),
"without publish_generated_columns, pgoutput never sends the generated \
column, so the snapshot must not invent it: {columns:?}"
);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn a_generated_column_has_the_same_shape_in_both_phases() {
let (slot, table, publication) = ("it_snap_gen_shape", "snap_gen_shape", "snap_gen_shape_pub");
let _guard = slot_guard(slot);
setup_generated(table, publication, "");
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let outcome = stream.snapshot().await.expect("snapshot");
let SnapshotOutcome::Available(snap) = outcome else {
panic!("expected a snapshot");
};
let mut events = snap.events();
let snapshot_event = events
.next_event()
.await
.expect("next_event")
.expect("the seeded row");
let snapshot_columns = insert_columns(&snapshot_event);
while events.next_event().await.expect("next_event").is_some() {}
let mut stream = events.finish().await.expect("finish");
regular_conn()
.exec(&format!("INSERT INTO {table} (a) VALUES (7)"))
.expect("insert");
stream.start(None).await.expect("start");
let live = tokio::time::timeout(TEST_TIMEOUT, drain_inserts(&mut stream, 1))
.await
.expect("timed out");
assert!(!live.is_empty(), "no insert arrived within 10s");
let live_columns = insert_columns(&live[0]);
assert_eq!(
snapshot_columns, live_columns,
"COPY TEXT and pgoutput must agree on the column set, or the two phases \
deserialize into different shapes"
);
}
#[tokio::test]
#[ignore = "requires live PostgreSQL with wal_level=logical"]
async fn snapshot_includes_a_published_generated_column_on_pg18() {
let ver = server_version();
if ver < 180000 {
println!(
"SKIP: snapshot_includes_a_published_generated_column_on_pg18 \
requires PG >= 18 (server {ver})"
);
return;
}
let (slot, table, publication) = ("it_snap_gen_pub", "snap_gen_pub_t", "snap_gen_pub_p");
let _guard = slot_guard(slot);
setup_generated(
table,
publication,
" WITH (publish_generated_columns = stored)",
);
let stream = LogicalReplicationStream::new(
&replication_conn_string(),
snapshot_config(slot, publication),
)
.await
.expect("connect");
let outcome = stream.snapshot().await.expect("snapshot");
let SnapshotOutcome::Available(snap) = outcome else {
panic!("expected a snapshot");
};
let mut events = snap.events();
let snapshot_event = events
.next_event()
.await
.expect("next_event")
.expect("the seeded row");
let snapshot_columns = insert_columns(&snapshot_event);
let snapshot_data = match &snapshot_event.event_type {
EventType::Insert { data, .. } => data.clone(),
other => panic!("expected an Insert, got {other:?}"),
};
while events.next_event().await.expect("next_event").is_some() {}
let mut stream = events.finish().await.expect("finish");
assert!(
snapshot_columns.contains(&"g".to_string()),
"publish_generated_columns=stored replicates g, so the snapshot must \
carry it too: {snapshot_columns:?}"
);
assert_eq!(
snapshot_data.get("g").map(|v| v.as_str()),
Some(Some("10")),
"the generated value must be the server's, not a placeholder"
);
regular_conn()
.exec(&format!("INSERT INTO {table} (a) VALUES (7)"))
.expect("insert");
stream.start(None).await.expect("start");
let live = tokio::time::timeout(TEST_TIMEOUT, drain_inserts(&mut stream, 1))
.await
.expect("timed out");
assert!(!live.is_empty(), "no insert arrived within 10s");
assert_eq!(
snapshot_columns,
insert_columns(&live[0]),
"the whole point: including g in one phase and not the other is the bug"
);
}