Skip to main content

faucet_source_postgres_cdc/
replication.rs

1//! Low-level replication-connection wrapper.
2//!
3//! This module wraps [`pgwire_replication`] to provide the slot lifecycle
4//! (`ensure_slot`) and streaming helpers (`start_replication`, `recv`,
5//! `send_status_update`) used by the rest of the CDC source.
6//!
7//! # Design
8//!
9//! `pgwire_replication` handles everything from TCP connect through auth,
10//! `START_REPLICATION`, keepalive replies, and `StandbyStatusUpdate` — all
11//! internally.  The library delivers events as a typed enum; [`recv`] surfaces
12//! the full [`ReplicationEvent`] to callers (absorbing only [`KeepAlive`] and
13//! [`StoppedAt`] internally) so Tasks 9+ can observe transaction boundaries.
14//!
15//! Slot creation (`CREATE_REPLICATION_SLOT`) is a control-plane operation that
16//! requires an ordinary (non-replication) SQL connection, so `ensure_slot`
17//! uses [`sqlx`] for that single query.
18//!
19//! # Type aliases
20//!
21//! The plan requires stable names `Client` and `Duplex` so that Tasks 9+ can
22//! refer to concrete types.  We define:
23//!
24//! - [`Client`] — a lightweight holder of `ReplicationParams` used to verify
25//!   connectivity and create the replication slot, before the stream is
26//!   opened.
27//! - [`Duplex`] — the live replication stream; a thin wrapper around
28//!   [`pgwire_replication::ReplicationClient`].
29//!
30//! [`KeepAlive`]: ReplicationEvent::KeepAlive
31//! [`StoppedAt`]: ReplicationEvent::StoppedAt
32
33use std::time::{Duration, SystemTime, UNIX_EPOCH};
34
35use faucet_core::FaucetError;
36use pgwire_replication::{Lsn, ReplicationClient, ReplicationConfig, TlsConfig};
37use sqlx::postgres::PgConnectOptions;
38
39/// Re-export so downstream modules (`stream.rs`, Task 9) can import the event
40/// type without depending on `pgwire_replication` directly.
41pub use pgwire_replication::ReplicationEvent;
42use sqlx::{Executor, PgConnection};
43use tracing::debug;
44
45/// Microseconds between the Unix epoch (1970-01-01) and the Postgres epoch
46/// (2000-01-01).  Used for converting between Postgres timestamps and Unix time.
47pub const POSTGRES_EPOCH_MICROS: i64 = 946_684_800_000_000;
48
49// ── Public type aliases ────────────────────────────────────────────────────
50
51/// Pre-stream handle returned by [`connect`] once the connection URL has been
52/// validated. The actual connection is opened by [`ensure_slot`] /
53/// [`start_replication`] from the borrowed [`ReplicationParams`], so this is a
54/// lightweight marker — it deliberately holds no owned copy of the connection
55/// string, which previously sat in leaked (`Box::leak`) heap for the process
56/// lifetime, including the password (#78/#13).
57pub struct Client {
58    _private: (),
59}
60
61/// Live replication stream.  Wraps [`pgwire_replication::ReplicationClient`].
62/// Obtained from [`start_replication`].
63pub struct Duplex {
64    inner: ReplicationClient,
65}
66
67// ── Parameters ────────────────────────────────────────────────────────────
68
69/// All parameters required to establish a logical replication connection.
70///
71/// This struct is accepted by every function in this module.
72#[derive(Clone, Debug)]
73pub struct ReplicationParams<'a> {
74    /// `postgres://user:pass@host:port/db` style URL.
75    pub connection_url: &'a str,
76    /// Name of the replication slot (must already exist, or `create_slot_if_missing = true`).
77    pub slot_name: &'a str,
78    /// Publication name — must already exist on the server.
79    pub publication_name: &'a str,
80    /// pgoutput protocol version. Only `1` is currently supported.
81    pub proto_version: u32,
82    /// Create the slot if it does not already exist.
83    pub create_slot_if_missing: bool,
84    /// Optional LSN to resume from.  `None` means "start from the slot's
85    /// `confirmed_flush_lsn`".
86    pub start_lsn: Option<u64>,
87    /// Protocol-level Standby Status Update cadence — must be shorter than
88    /// the server's `wal_sender_timeout`.
89    pub status_update_interval: Duration,
90    /// TCP-level keepalive interval. Larger than `status_update_interval`
91    /// in normal operation.
92    pub tcp_keepalive: Duration,
93    /// Whether a newly-created slot is permanent or temporary.
94    pub slot_type: crate::config::SlotType,
95    /// TLS settings for the replication connection.
96    pub tls: &'a crate::config::CdcTls,
97}
98
99// ── Helper: parse a postgres URL into (host, port, user, password, dbname) ─
100
101#[cfg_attr(test, derive(Debug))]
102struct PgCoords {
103    host: String,
104    port: u16,
105    user: String,
106    password: String,
107    dbname: String,
108}
109
110fn parse_url(url: &str) -> Result<PgCoords, FaucetError> {
111    // Parse via the standard `url` crate so we can extract all components
112    // without relying on sqlx accessor methods that may not be public.
113    let parsed = url::Url::parse(url)
114        .map_err(|e| FaucetError::Config(format!("postgres-cdc: invalid connection URL: {e}")))?;
115
116    // Fail fast on a missing host or user rather than silently defaulting to
117    // `localhost` / empty — a malformed URL otherwise connects to an
118    // unintended local instance or produces a confusing late auth failure
119    // (#78/#47). Port (5432) and dbname (the user's DB) keep their standard
120    // libpq-style defaults since omitting them is conventional.
121    let host = parsed
122        .host_str()
123        .filter(|h| !h.is_empty())
124        .ok_or_else(|| {
125            FaucetError::Config(
126                "postgres-cdc: connection URL is missing a host (expected \
127                 postgres://user@host[:port]/dbname)"
128                    .to_owned(),
129            )
130        })?
131        .to_owned();
132    let port = parsed.port().unwrap_or(5432);
133    let user = parsed.username().to_owned();
134    if user.is_empty() {
135        return Err(FaucetError::Config(
136            "postgres-cdc: connection URL is missing a user (expected \
137             postgres://user@host[:port]/dbname)"
138                .to_owned(),
139        ));
140    }
141    let password = parsed.password().unwrap_or("").to_owned();
142    let dbname = parsed.path().trim_start_matches('/').to_owned();
143    let dbname = if dbname.is_empty() {
144        "postgres".to_owned()
145    } else {
146        dbname
147    };
148
149    Ok(PgCoords {
150        host,
151        port,
152        user,
153        password,
154        dbname,
155    })
156}
157
158// ── Public API ─────────────────────────────────────────────────────────────
159
160/// Validate connectivity and return a [`Client`] handle.
161///
162/// This function parses the connection URL and records the parameters.
163/// It does **not** open a TCP connection; actual connectivity is verified
164/// lazily when [`ensure_slot`] or [`start_replication`] is called.
165pub async fn connect(params: &ReplicationParams<'_>) -> Result<Client, FaucetError> {
166    // Eagerly validate the URL so bad configs fail fast. No part of the
167    // connection string is retained past this call.
168    let _ = parse_url(params.connection_url)?;
169    Ok(Client { _private: () })
170}
171
172/// Ensure the replication slot exists.
173///
174/// If the slot already exists this is a no-op.  If it does not exist and
175/// `create_if_missing` is `true`, the slot is created via
176/// `pg_create_logical_replication_slot`.  If `create_if_missing` is `false`
177/// and the slot is absent, an error is returned.
178pub async fn ensure_slot(
179    _client: &Client,
180    connection_url: &str,
181    slot_name: &str,
182    create_if_missing: bool,
183    slot_type: crate::config::SlotType,
184) -> Result<(), FaucetError> {
185    use crate::config::SlotType;
186    // Use sqlx for the control-plane query (not a replication connection).
187    let opts: PgConnectOptions = connection_url
188        .parse()
189        .map_err(|e| FaucetError::Config(format!("postgres-cdc: invalid connection URL: {e}")))?;
190
191    use sqlx::ConnectOptions as _;
192    let mut conn: PgConnection = opts
193        .connect()
194        .await
195        .map_err(|e| FaucetError::Source(format!("postgres-cdc ensure_slot connect: {e}")))?;
196
197    // Check whether the slot already exists.
198    let row: Option<(String,)> =
199        sqlx::query_as("SELECT slot_name::text FROM pg_replication_slots WHERE slot_name = $1")
200            .bind(slot_name)
201            .fetch_optional(&mut conn)
202            .await
203            .map_err(|e| FaucetError::Source(format!("postgres-cdc slot lookup: {e}")))?;
204
205    if row.is_some() {
206        debug!("postgres-cdc: replication slot '{slot_name}' already exists");
207        return Ok(());
208    }
209
210    if !create_if_missing {
211        return Err(FaucetError::Source(format!(
212            "postgres-cdc: replication slot '{slot_name}' does not exist \
213             and create_slot_if_missing = false"
214        )));
215    }
216
217    // Create the slot using the pgoutput plugin. The third arg to
218    // pg_create_logical_replication_slot is `temporary`: a temporary slot is
219    // dropped when this session disconnects; a permanent slot persists (and
220    // pins WAL) until explicitly dropped.
221    // `escape_simple` prevents injection via the slot name (already validated
222    // to [a-z0-9_] by config, but defence-in-depth doesn't hurt).
223    let temporary = matches!(slot_type, SlotType::Temporary);
224    let sql = format!(
225        "SELECT pg_create_logical_replication_slot({}, 'pgoutput', {})",
226        quote_literal(slot_name),
227        temporary
228    );
229    conn.execute(sql.as_str())
230        .await
231        .map_err(|e| FaucetError::Source(format!("postgres-cdc create slot: {e}")))?;
232
233    if temporary {
234        debug!("postgres-cdc: created temporary replication slot '{slot_name}'");
235    } else {
236        // A permanent slot retains WAL until consumed or dropped — surface it
237        // loudly so an abandoned slot doesn't silently fill pg_wal (#78/#12).
238        tracing::warn!(
239            "postgres-cdc: created PERMANENT replication slot '{slot_name}' — it will retain \
240             WAL on the server until consumed or explicitly dropped (drop_slot). Use \
241             slot_type=temporary for ephemeral runs."
242        );
243    }
244    Ok(())
245}
246
247/// Drop a logical replication slot via a control-plane SQL call
248/// (`pg_drop_replication_slot`). A missing slot is treated as success (no-op);
249/// an active slot (currently in use by another connection) surfaces an error.
250pub async fn drop_slot(connection_url: &str, slot_name: &str) -> Result<(), FaucetError> {
251    let opts: PgConnectOptions = connection_url
252        .parse()
253        .map_err(|e| FaucetError::Config(format!("postgres-cdc: invalid connection URL: {e}")))?;
254    use sqlx::ConnectOptions as _;
255    let mut conn: PgConnection = opts
256        .connect()
257        .await
258        .map_err(|e| FaucetError::Source(format!("postgres-cdc drop_slot connect: {e}")))?;
259
260    let exists: Option<(String,)> =
261        sqlx::query_as("SELECT slot_name::text FROM pg_replication_slots WHERE slot_name = $1")
262            .bind(slot_name)
263            .fetch_optional(&mut conn)
264            .await
265            .map_err(|e| FaucetError::Source(format!("postgres-cdc slot lookup: {e}")))?;
266    if exists.is_none() {
267        debug!("postgres-cdc: replication slot '{slot_name}' already absent; drop is a no-op");
268        return Ok(());
269    }
270
271    sqlx::query("SELECT pg_drop_replication_slot($1)")
272        .bind(slot_name)
273        .execute(&mut conn)
274        .await
275        .map_err(|e| FaucetError::Source(format!("postgres-cdc drop slot: {e}")))?;
276    debug!("postgres-cdc: dropped replication slot '{slot_name}'");
277    Ok(())
278}
279
280/// Map the user-facing [`CdcTls`](crate::config::CdcTls) config onto
281/// pgwire-replication's [`TlsConfig`].
282fn tls_config(tls: &crate::config::CdcTls) -> TlsConfig {
283    use crate::config::CdcTls;
284    use std::path::PathBuf;
285    match tls {
286        CdcTls::Disable => TlsConfig::disabled(),
287        CdcTls::Require => TlsConfig::require(),
288        CdcTls::VerifyCa { ca_path } => TlsConfig::verify_ca(ca_path.clone().map(PathBuf::from)),
289        CdcTls::VerifyFull { ca_path } => {
290            TlsConfig::verify_full(ca_path.clone().map(PathBuf::from))
291        }
292    }
293}
294
295/// Advance the slot's `confirmed_flush_lsn` to `lsn` via a control-plane SQL
296/// call (`pg_replication_slot_advance`), **before** the replication stream is
297/// opened.
298///
299/// This is how the connector resumes past already-consumed, durably-persisted
300/// changes without ever advancing the slot ahead of durability (#78/#1). For
301/// a logical slot, `START_REPLICATION` resumes decoding from the slot's
302/// `confirmed_flush_lsn` — the client-supplied start LSN does not filter
303/// transactions that committed below it — so the only way to skip consumed
304/// changes is to move `confirmed_flush_lsn` forward here, while the slot is
305/// inactive. `pg_replication_slot_advance` never moves a slot backwards or
306/// past the server's insert pointer, so a stale or zero `lsn` is a safe no-op.
307///
308/// The slot must be inactive, which it is between [`ensure_slot`] and
309/// [`start_replication`].
310pub async fn advance_slot(
311    connection_url: &str,
312    slot_name: &str,
313    lsn: u64,
314) -> Result<(), FaucetError> {
315    if lsn == 0 {
316        return Ok(());
317    }
318    let opts: PgConnectOptions = connection_url
319        .parse()
320        .map_err(|e| FaucetError::Config(format!("postgres-cdc: invalid connection URL: {e}")))?;
321
322    use sqlx::ConnectOptions as _;
323    let mut conn: PgConnection = opts
324        .connect()
325        .await
326        .map_err(|e| FaucetError::Source(format!("postgres-cdc advance_slot connect: {e}")))?;
327
328    // Bind the slot name and the LSN (as pg_lsn text) as parameters — no string
329    // interpolation into the SQL. `format_lsn` emits Postgres' canonical
330    // `X/X` text form.
331    sqlx::query("SELECT pg_replication_slot_advance($1, $2::pg_lsn)")
332        .bind(slot_name)
333        .bind(crate::state::format_lsn(lsn))
334        .execute(&mut conn)
335        .await
336        .map_err(|e| FaucetError::Source(format!("postgres-cdc advance_slot: {e}")))?;
337
338    debug!("postgres-cdc: advanced slot '{slot_name}' confirmed_flush_lsn to {lsn:#x}");
339    Ok(())
340}
341
342/// Ensure the slot exists, then return the slot's **own consistent point** —
343/// its `confirmed_flush_lsn` (falling back to `restart_lsn`, then the server's
344/// `pg_current_wal_lsn()` only if both are null).
345///
346/// Anchoring on the slot's consistent point — rather than the server's *current*
347/// WAL position — is what makes capture-before-snapshot gapless even when the
348/// slot **already existed** with unconsumed WAL behind it. Using
349/// `pg_current_wal_lsn()` would skip every change the slot had retained between
350/// its confirmed position and "now", silently losing those CDC events (F8).
351/// For a freshly-created logical slot, `confirmed_flush_lsn` is the slot's
352/// creation consistent point, so the fresh-slot behaviour is preserved.
353///
354/// Ensuring the slot **first** guarantees the server retains WAL from the
355/// returned LSN onward, so a later `START_REPLICATION` from this position has
356/// no gap. Used by the replication snapshot→CDC handoff
357/// ([`Source::capture_resume_position`](faucet_core::Source::capture_resume_position)).
358pub async fn ensure_slot_and_current_lsn(
359    connection_url: &str,
360    slot_name: &str,
361    create_if_missing: bool,
362    slot_type: crate::config::SlotType,
363) -> Result<u64, FaucetError> {
364    // `ensure_slot`'s `_client` arg is unused; `Client` is constructible here
365    // (same module). This reuses the existing slot existence/create logic.
366    let client = Client { _private: () };
367    ensure_slot(
368        &client,
369        connection_url,
370        slot_name,
371        create_if_missing,
372        slot_type,
373    )
374    .await?;
375
376    let opts: PgConnectOptions = connection_url
377        .parse()
378        .map_err(|e| FaucetError::Config(format!("postgres-cdc: invalid connection URL: {e}")))?;
379    use sqlx::ConnectOptions as _;
380    let mut conn: PgConnection = opts
381        .connect()
382        .await
383        .map_err(|e| FaucetError::Source(format!("postgres-cdc capture_position connect: {e}")))?;
384    // Prefer the slot's retained consistent point over the server's current LSN.
385    let slot_lsn: Option<(Option<String>, Option<String>)> = sqlx::query_as(
386        "SELECT confirmed_flush_lsn::text, restart_lsn::text \
387         FROM pg_replication_slots WHERE slot_name = $1",
388    )
389    .bind(slot_name)
390    .fetch_optional(&mut conn)
391    .await
392    .map_err(|e| FaucetError::Source(format!("postgres-cdc slot lsn lookup: {e}")))?;
393    let anchor = match slot_lsn {
394        Some((confirmed, restart)) => confirmed.or(restart),
395        None => None,
396    };
397    let lsn_text = match anchor {
398        Some(t) => t,
399        None => {
400            // Slot reports no LSN yet (e.g. just-created temporary slot on some
401            // versions) — fall back to the server's current WAL position.
402            let (cur,): (String,) = sqlx::query_as("SELECT pg_current_wal_lsn()::text")
403                .fetch_one(&mut conn)
404                .await
405                .map_err(|e| {
406                    FaucetError::Source(format!("postgres-cdc pg_current_wal_lsn: {e}"))
407                })?;
408            cur
409        }
410    };
411    crate::state::parse_lsn(&lsn_text)
412}
413
414/// Open a logical replication stream and return a [`Duplex`] handle.
415///
416/// Internally this calls `pgwire_replication::ReplicationClient::connect`
417/// which handles TCP, TLS negotiation, auth, and `START_REPLICATION` in one
418/// shot.
419pub async fn start_replication(
420    _client: &Client,
421    params: &ReplicationParams<'_>,
422) -> Result<Duplex, FaucetError> {
423    if params.proto_version != 1 {
424        return Err(FaucetError::Config(format!(
425            "postgres-cdc: pgwire-replication 0.3.2 supports proto_version = 1 only; \
426             got {}",
427            params.proto_version
428        )));
429    }
430
431    let coords = parse_url(params.connection_url)?;
432
433    let start_lsn = Lsn::from_u64(params.start_lsn.unwrap_or(0));
434
435    let cfg = ReplicationConfig {
436        host: coords.host,
437        port: coords.port,
438        user: coords.user,
439        password: coords.password,
440        database: coords.dbname,
441        tls: tls_config(params.tls),
442        slot: params.slot_name.to_owned(),
443        publication: params.publication_name.to_owned(),
444        start_lsn,
445        stop_at_lsn: None,
446        // Use the dedicated status-update interval (not tcp_keepalive) so that
447        // Standby Status Updates fire on their own cadence.
448        status_interval: params.status_update_interval,
449        // Wake up the worker at least as often as we send status updates.
450        idle_wakeup_interval: params.status_update_interval,
451        buffer_events: 8192,
452    };
453
454    let inner = ReplicationClient::connect(cfg)
455        .await
456        .map_err(|e| FaucetError::Source(format!("postgres-cdc start_replication: {e}")))?;
457
458    Ok(Duplex { inner })
459}
460
461/// Report progress to the server (Standby Status Update).
462///
463/// `confirmed_lsn` is the highest LSN whose changes have been durably
464/// written to the sink.  The underlying library sends this feedback on its
465/// own keepalive schedule; calling this function additionally marks the
466/// progress so the next automatic feedback includes the latest position.
467///
468/// `reply_requested` mirrors the flag from the server's KeepAlive message
469/// (no-op here since the library handles immediate replies internally).
470pub async fn send_status_update(
471    duplex: &mut Duplex,
472    confirmed_lsn: u64,
473    _reply_requested: bool,
474) -> Result<(), FaucetError> {
475    duplex
476        .inner
477        .update_applied_lsn(Lsn::from_u64(confirmed_lsn));
478    Ok(())
479}
480
481/// Receive the next meaningful replication event from the server.
482///
483/// Returns:
484/// - `Ok(Some(event))` — the next [`ReplicationEvent`] that the caller should
485///   handle.  This includes [`ReplicationEvent::XLogData`],
486///   [`ReplicationEvent::Begin`], [`ReplicationEvent::Commit`], and
487///   [`ReplicationEvent::Message`].  Callers (Task 9+) can match on the full
488///   event type to observe transaction boundaries.
489/// - `Ok(None)` — stream ended cleanly (slot stopped, stop LSN reached, or
490///   `Duplex` was shut down).
491/// - `Err(_)` — network / protocol error.
492///
493/// [`ReplicationEvent::KeepAlive`] events are absorbed here.  We deliberately
494/// do **not** advance the applied-LSN to the server's `wal_end` on a keepalive
495/// (the previous behaviour): that position is not yet durable downstream, and
496/// advertising it as `confirmed_flush_lsn` would authorise Postgres to recycle
497/// WAL for changes the consumer never persisted — a crash in that window loses
498/// data (#78/#1).  The applied-LSN is advanced only from the durable bookmark,
499/// via [`send_status_update`] at the start of each run; the library keeps
500/// sending its periodic Standby Status Updates (carrying that durable
501/// position) to hold the connection open.  [`ReplicationEvent::StoppedAt`] is
502/// converted to `Ok(None)`.
503pub async fn recv(duplex: &mut Duplex) -> Result<Option<ReplicationEvent>, FaucetError> {
504    loop {
505        match duplex
506            .inner
507            .recv()
508            .await
509            .map_err(|e| FaucetError::Source(format!("postgres-cdc recv: {e}")))?
510        {
511            None => return Ok(None),
512
513            Some(ReplicationEvent::StoppedAt { .. }) => {
514                return Ok(None);
515            }
516
517            Some(ReplicationEvent::KeepAlive { .. }) => {
518                // Absorb keepalives without touching the applied-LSN — see the
519                // function doc. Continue the loop; do not surface to the caller.
520            }
521
522            Some(ev) => {
523                // Surface Begin, Commit, XLogData, Message (and any future
524                // variants) to the caller. The commit_lsn carried by Commit is
525                // what becomes the durable bookmark once the pipeline persists
526                // it — that, not wal_end, is the only position fed back to PG.
527                return Ok(Some(ev));
528            }
529        }
530    }
531}
532
533// ── Clock helpers ──────────────────────────────────────────────────────────
534
535/// Current time as a Postgres-epoch timestamp (µs since 2000-01-01 UTC).
536///
537/// Used in Standby Status Update messages.
538pub fn postgres_clock_now() -> i64 {
539    let now = SystemTime::now()
540        .duration_since(UNIX_EPOCH)
541        .unwrap_or_default();
542    let unix_micros = (now.as_secs() as i64) * 1_000_000 + (now.subsec_micros() as i64);
543    unix_micros - POSTGRES_EPOCH_MICROS
544}
545
546/// Convert a Postgres-epoch timestamp (µs since 2000-01-01) to Unix
547/// milliseconds (ms since 1970-01-01).
548pub fn postgres_clock_to_unix_ms(ts: i64) -> i64 {
549    (POSTGRES_EPOCH_MICROS.saturating_add(ts)) / 1_000
550}
551
552// ── Private SQL helpers ────────────────────────────────────────────────────
553
554/// Wrap `s` in double-quotes for use as a Postgres identifier.
555/// Any embedded double-quote is doubled (`"` → `""`).
556/// Reserved for DDL statements (e.g. `DROP REPLICATION SLOT`); used in tests.
557#[allow(dead_code)]
558fn quote_slot(s: &str) -> String {
559    format!("\"{}\"", s.replace('"', "\"\""))
560}
561
562/// Escape a string for use in a Postgres literal (single-quote context).
563/// Any embedded single-quote is doubled (`'` → `''`).
564fn escape_simple(s: &str) -> String {
565    s.replace('\'', "''")
566}
567
568/// Produce a single-quoted Postgres string literal.
569fn quote_literal(s: &str) -> String {
570    format!("'{}'", escape_simple(s))
571}
572
573// ── Tests ──────────────────────────────────────────────────────────────────
574
575/// Returns `true` if `err` is Postgres reporting that the replication slot is
576/// still **active** — held by a backend that has not yet released it. Postgres
577/// raises *"replication slot \"…\" is active for PID …"* (SQLSTATE `55006`).
578///
579/// This is transient on a rapid restart: a scheduler or `serve` re-running the
580/// pipeline before the previous connection's backend has fully exited finds the
581/// slot momentarily still in use. It clears within a short window, so it is
582/// safe to retry after a backoff (#146 M12).
583pub fn is_slot_active_error(err: &FaucetError) -> bool {
584    let msg = err.to_string().to_ascii_lowercase();
585    msg.contains("is active") || msg.contains("55006")
586}
587
588/// Exponential backoff for slot-acquisition retries: `250ms · 2^attempt`,
589/// capped at 4 s.
590fn slot_acquire_backoff(attempt: u32) -> Duration {
591    let factor = 1u64.checked_shl(attempt).unwrap_or(u64::MAX);
592    let ms = 250u64.saturating_mul(factor).min(4000);
593    Duration::from_millis(ms)
594}
595
596/// Run `op`, retrying up to `max_retries` times with exponential backoff while
597/// it fails because the replication slot is still active (#146 M12). Any other
598/// error — and the final attempt's error after exhausting retries — is returned
599/// immediately. `max_retries = 0` preserves the previous fail-fast behaviour.
600pub async fn retry_on_slot_active<F, Fut, T>(max_retries: u32, op: F) -> Result<T, FaucetError>
601where
602    F: Fn() -> Fut,
603    Fut: std::future::Future<Output = Result<T, FaucetError>>,
604{
605    let mut attempt = 0u32;
606    loop {
607        match op().await {
608            Ok(value) => return Ok(value),
609            Err(e) if attempt < max_retries && is_slot_active_error(&e) => {
610                let backoff = slot_acquire_backoff(attempt);
611                tracing::warn!(
612                    attempt = attempt + 1,
613                    max_retries,
614                    backoff_ms = backoff.as_millis() as u64,
615                    error = %e,
616                    "postgres-cdc: replication slot still active; retrying after backoff"
617                );
618                tokio::time::sleep(backoff).await;
619                attempt += 1;
620            }
621            Err(e) => return Err(e),
622        }
623    }
624}
625
626#[cfg(test)]
627mod tests {
628    use super::*;
629    use crate::config::CdcTls;
630    use chrono::{TimeZone, Utc};
631    use pgwire_replication::SslMode;
632
633    #[test]
634    fn tls_config_maps_each_mode() {
635        assert_eq!(tls_config(&CdcTls::Disable).mode, SslMode::Disable);
636        assert_eq!(tls_config(&CdcTls::Require).mode, SslMode::Require);
637        assert_eq!(
638            tls_config(&CdcTls::VerifyCa { ca_path: None }).mode,
639            SslMode::VerifyCa
640        );
641        assert_eq!(
642            tls_config(&CdcTls::VerifyFull {
643                ca_path: Some("/ca.pem".into())
644            })
645            .mode,
646            SslMode::VerifyFull
647        );
648    }
649
650    /// Convenience: turn `postgres_clock_to_unix_ms`-compatible math into a
651    /// `DateTime<Utc>`.  Used by tests only.
652    fn postgres_clock_to_datetime(ts: i64) -> chrono::DateTime<Utc> {
653        Utc.timestamp_micros(POSTGRES_EPOCH_MICROS.saturating_add(ts))
654            .single()
655            .unwrap_or_else(Utc::now)
656    }
657
658    #[test]
659    fn postgres_clock_round_trip() {
660        let dt = Utc.with_ymd_and_hms(2026, 5, 17, 12, 0, 0).unwrap();
661        let pg_ts = dt.timestamp_micros() - POSTGRES_EPOCH_MICROS;
662        let back = postgres_clock_to_datetime(pg_ts);
663        assert_eq!(back, dt);
664    }
665
666    #[test]
667    fn unix_ms_conversion() {
668        let dt = Utc.with_ymd_and_hms(2026, 5, 17, 12, 0, 0).unwrap();
669        let pg_ts = dt.timestamp_micros() - POSTGRES_EPOCH_MICROS;
670        assert_eq!(postgres_clock_to_unix_ms(pg_ts), 1_779_019_200_000);
671    }
672
673    #[test]
674    fn quote_slot_simple() {
675        assert_eq!(quote_slot("faucet_slot"), "\"faucet_slot\"");
676    }
677
678    #[test]
679    fn escape_simple_doubles_quotes() {
680        assert_eq!(escape_simple("foo'bar"), "foo''bar");
681    }
682
683    #[test]
684    fn parse_url_extracts_all_components() {
685        let c = parse_url("postgres://alice:secret@db.example.com:5544/analytics").unwrap();
686        assert_eq!(c.host, "db.example.com");
687        assert_eq!(c.port, 5544);
688        assert_eq!(c.user, "alice");
689        assert_eq!(c.password, "secret");
690        assert_eq!(c.dbname, "analytics");
691    }
692
693    #[test]
694    fn parse_url_defaults_port_and_dbname() {
695        let c = parse_url("postgres://alice@db.example.com").unwrap();
696        assert_eq!(c.port, 5432);
697        assert_eq!(c.dbname, "postgres");
698        assert_eq!(c.password, "");
699    }
700
701    #[test]
702    fn parse_url_rejects_missing_host() {
703        // `postgres:///db` parses with an empty host — must fail fast (#78/#47).
704        let err = parse_url("postgres:///analytics").unwrap_err();
705        assert!(format!("{err}").contains("missing a host"), "{err}");
706    }
707
708    #[test]
709    fn parse_url_rejects_missing_user() {
710        let err = parse_url("postgres://db.example.com/analytics").unwrap_err();
711        assert!(format!("{err}").contains("missing a user"), "{err}");
712    }
713
714    #[test]
715    fn is_slot_active_error_classifies_the_postgres_message() {
716        // The canonical Postgres message (SQLSTATE 55006).
717        assert!(is_slot_active_error(&FaucetError::Source(
718            "postgres-cdc start_replication: db error: ERROR: replication slot \"s\" \
719             is active for PID 4242"
720                .into()
721        )));
722        // SQLSTATE code present.
723        assert!(is_slot_active_error(&FaucetError::Source(
724            "55006: replication slot is in use".into()
725        )));
726        // Unrelated errors are NOT slot-active.
727        assert!(!is_slot_active_error(&FaucetError::Source(
728            "connection refused".into()
729        )));
730        assert!(!is_slot_active_error(&FaucetError::Config(
731            "bad url".into()
732        )));
733    }
734
735    #[test]
736    fn slot_acquire_backoff_grows_and_is_capped() {
737        assert_eq!(slot_acquire_backoff(0), Duration::from_millis(250));
738        assert_eq!(slot_acquire_backoff(1), Duration::from_millis(500));
739        assert_eq!(slot_acquire_backoff(2), Duration::from_millis(1000));
740        // Capped at 4s no matter how large the attempt.
741        assert_eq!(slot_acquire_backoff(20), Duration::from_millis(4000));
742        assert_eq!(slot_acquire_backoff(64), Duration::from_millis(4000));
743    }
744
745    #[tokio::test]
746    async fn retry_on_slot_active_retries_then_succeeds() {
747        use std::sync::atomic::{AtomicU32, Ordering};
748        let calls = AtomicU32::new(0);
749        let result = retry_on_slot_active(5, || {
750            let n = calls.fetch_add(1, Ordering::SeqCst);
751            async move {
752                if n < 2 {
753                    Err(FaucetError::Source(
754                        "replication slot \"s\" is active for PID 1".into(),
755                    ))
756                } else {
757                    Ok::<u32, FaucetError>(42)
758                }
759            }
760        })
761        .await;
762        assert_eq!(result.unwrap(), 42);
763        assert_eq!(calls.load(Ordering::SeqCst), 3, "2 failures + 1 success");
764    }
765
766    #[tokio::test]
767    async fn retry_on_slot_active_gives_up_after_max_retries() {
768        use std::sync::atomic::{AtomicU32, Ordering};
769        let calls = AtomicU32::new(0);
770        let result: Result<(), _> = retry_on_slot_active(2, || {
771            calls.fetch_add(1, Ordering::SeqCst);
772            async { Err(FaucetError::Source("slot is active".into())) }
773        })
774        .await;
775        assert!(result.is_err());
776        assert_eq!(
777            calls.load(Ordering::SeqCst),
778            3,
779            "initial attempt + 2 retries"
780        );
781    }
782
783    #[tokio::test]
784    async fn retry_on_slot_active_does_not_retry_unrelated_errors() {
785        use std::sync::atomic::{AtomicU32, Ordering};
786        let calls = AtomicU32::new(0);
787        let result: Result<(), _> = retry_on_slot_active(5, || {
788            calls.fetch_add(1, Ordering::SeqCst);
789            async { Err(FaucetError::Source("connection refused".into())) }
790        })
791        .await;
792        assert!(result.is_err());
793        assert_eq!(
794            calls.load(Ordering::SeqCst),
795            1,
796            "a non-slot-active error must not be retried"
797        );
798    }
799}