Skip to main content

pgwire_replication/client/
tokio_client.rs

1use crate::config::ReplicationConfig;
2use crate::error::{PgWireError, Result};
3use crate::lsn::Lsn;
4
5use tokio::net::TcpStream;
6#[cfg(unix)]
7use tokio::net::UnixStream;
8
9use tokio::sync::{mpsc, oneshot, watch, Semaphore};
10use tokio::task::JoinHandle;
11
12use std::sync::Arc;
13use std::time::Duration;
14
15#[cfg(not(feature = "tls-rustls"))]
16use crate::config::SslMode;
17
18use super::worker::{ReplicationEvent, ReplicationEventReceiver, SharedProgress, WorkerState};
19
20const DROP_SHUTDOWN_GRACE: Duration = Duration::from_millis(100);
21
22/// PostgreSQL logical replication client.
23///
24/// This client spawns a background worker task that maintains the replication
25/// connection and streams events to the consumer via a bounded channel.
26///
27/// # Example
28///
29/// ```no_run
30/// use pgwire_replication::client::{ReplicationClient, ReplicationEvent};
31/// use pgwire_replication::config::ReplicationConfig;
32///
33/// #[tokio::main]
34/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
35///     let config = ReplicationConfig::new(
36///         "localhost",
37///         "postgres",
38///         "password",
39///         "mydb",
40///         "my_slot",
41///         "my_pub",
42///     );
43///
44///     let mut client = ReplicationClient::connect(config).await?;
45///
46///     while let Some(ev) = client.recv().await? {
47///         match ev {
48///             ReplicationEvent::XLogData { data, wal_end, .. } => {
49///                 process_change(&data);
50///                 client.update_applied_lsn(wal_end);
51///             }
52///             ReplicationEvent::KeepAlive { .. } => {}
53///             ReplicationEvent::StoppedAt { reached } => {
54///                 println!("Reached stop LSN: {reached}");
55///                 break;
56///             }
57///             _ => {}
58///         }
59///     }
60///
61///     Ok(())
62/// }
63///
64/// fn process_change(_data: &bytes::Bytes) {
65///     // user-defined
66/// }
67/// ```
68pub struct ReplicationClient {
69    rx: ReplicationEventReceiver,
70    progress: Arc<SharedProgress>,
71    stop_tx: watch::Sender<bool>,
72    join: Option<JoinHandle<std::result::Result<(), PgWireError>>>,
73    runtime: tokio::runtime::Handle,
74}
75
76/// Cloneable writer for one client's applied/durable LSN.
77///
78/// The worker reads this value for every standby status update, including while the
79/// event channel is full, so durable progress reaches the server without a consumer poll.
80#[derive(Clone)]
81pub struct AppliedLsnHandle(Arc<SharedProgress>);
82
83impl AppliedLsnHandle {
84    /// A handle not attached to any worker, starting at `start`.
85    pub fn new(start: Lsn) -> Self {
86        Self(Arc::new(SharedProgress::new(start)))
87    }
88
89    /// The applied LSN the worker will report next.
90    #[inline]
91    pub fn get(&self) -> Lsn {
92        self.0.load_applied()
93    }
94
95    /// Monotonically raise the applied LSN; call only once every event up to `lsn` is durable.
96    #[inline]
97    pub fn update(&self, lsn: Lsn) {
98        self.0.update_applied(lsn);
99    }
100
101    /// Make the worker send a status update with `reply_requested` set at once, so the server
102    /// answers with a keepalive carrying its current sent position.
103    pub fn request_keepalive(&self) {
104        self.0.request_keepalive();
105    }
106}
107
108impl std::fmt::Debug for AppliedLsnHandle {
109    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
110        formatter
111            .debug_tuple("AppliedLsnHandle")
112            .field(&self.0.load_applied())
113            .finish()
114    }
115}
116
117/// Aborts a not-yet-published replication worker when `connect()` is
118/// cancelled or returns an error. Once startup succeeds, ownership moves into
119/// `ReplicationClient` and its normal shutdown path applies.
120struct StartupWorker {
121    join: Option<JoinHandle<std::result::Result<(), PgWireError>>>,
122}
123
124impl StartupWorker {
125    fn new(join: JoinHandle<std::result::Result<(), PgWireError>>) -> Self {
126        Self { join: Some(join) }
127    }
128
129    fn take(&mut self) -> JoinHandle<std::result::Result<(), PgWireError>> {
130        self.join
131            .take()
132            .expect("startup worker ownership is transferred once")
133    }
134}
135
136impl Drop for StartupWorker {
137    fn drop(&mut self) {
138        if let Some(join) = self.join.take() {
139            join.abort();
140        }
141    }
142}
143
144impl ReplicationClient {
145    /// Connect to PostgreSQL and start streaming replication events.
146    ///
147    /// This establishes a TCP connection (optionally upgrading to TLS),
148    /// authenticates, and starts the replication stream. Events are buffered
149    /// in a channel of size `config.buffer_events`, with raw payload ownership
150    /// additionally capped by `config.max_in_flight_bytes`.
151    ///
152    /// # Errors
153    ///
154    /// Returns an error if:
155    /// - TCP connection fails
156    /// - TLS handshake fails (when enabled)
157    /// - Authentication fails
158    /// - Replication slot doesn't exist
159    /// - Publication doesn't exist
160    /// - Unix socket does not exist (when host starts with `/`)
161    /// - TLS requested with Unix socket connection
162    pub async fn connect(cfg: ReplicationConfig) -> Result<Self> {
163        Self::connect_with_worker_lifetime(cfg, ()).await
164    }
165
166    /// Connect while retaining `worker_lifetime` for the exact lifetime of the
167    /// spawned replication worker.
168    ///
169    /// This lets callers fence replacement work until a cancelled startup
170    /// worker has actually been destroyed, rather than merely requested to
171    /// abort.
172    pub async fn connect_with_worker_lifetime<G>(
173        cfg: ReplicationConfig,
174        worker_lifetime: G,
175    ) -> Result<Self>
176    where
177        G: Send + 'static,
178    {
179        validate_config(&cfg)?;
180        let (tx, rx) = mpsc::channel(cfg.buffer_events);
181        let wire_byte_budget = Arc::new(Semaphore::new(cfg.max_in_flight_bytes));
182
183        // Progress is shared via atomics: cheap, monotonic, no async backpressure.
184        let progress = Arc::new(SharedProgress::new(cfg.start_lsn));
185
186        let (stop_tx, stop_rx) = watch::channel(false);
187        let (startup_tx, startup_rx) = oneshot::channel();
188
189        let progress_for_worker = Arc::clone(&progress);
190        let cfg_for_worker = cfg.clone();
191
192        let join = tokio::spawn(async move {
193            let _worker_lifetime = worker_lifetime;
194            let mut worker = WorkerState::new(
195                cfg_for_worker,
196                progress_for_worker,
197                stop_rx,
198                tx,
199                wire_byte_budget,
200            );
201            worker.install_startup_notifier(startup_tx);
202            let res = run_worker(&mut worker, &cfg).await;
203            if let Err(ref e) = res {
204                // Covers TCP, Unix-socket, and TLS failures that happen before
205                // `WorkerState::run_on_stream` can publish its own result.
206                worker.report_startup_failure(e);
207                tracing::error!("replication worker terminated with error: {e}");
208            }
209            res
210        });
211
212        let mut startup_worker = StartupWorker::new(join);
213        match startup_rx.await {
214            Ok(Ok(())) => {}
215            Ok(Err(error)) => return Err(error),
216            Err(_) => {
217                return Err(PgWireError::Task(
218                    "replication worker terminated before publishing startup readiness".into(),
219                ));
220            }
221        }
222
223        Ok(Self {
224            rx,
225            progress,
226            stop_tx,
227            join: Some(startup_worker.take()),
228            runtime: tokio::runtime::Handle::current(),
229        })
230    }
231
232    /// Receive the next replication event.
233    ///
234    /// - `Ok(Some(event))` => received an event
235    /// - `Ok(None)`        => replication ended normally (stop requested or stop_at_lsn reached)
236    /// - `Err(e)`          => replication ended abnormally
237    pub async fn recv(&mut self) -> Result<Option<ReplicationEvent>> {
238        match self.rx.recv().await {
239            Some(Ok(ev)) => Ok(Some(ev)),
240            Some(Err(e)) => Err(e),
241            None => self.handle_worker_shutdown().await,
242        }
243    }
244
245    async fn handle_worker_shutdown(&mut self) -> Result<Option<ReplicationEvent>> {
246        let join = self
247            .join
248            .take()
249            .ok_or_else(|| PgWireError::Internal("replication worker already joined".into()))?;
250
251        match join.await {
252            Ok(Ok(())) => Ok(None),
253            Ok(Err(e)) => Err(e),
254            Err(join_err) => Err(PgWireError::Task(format!(
255                "replication worker panicked: {join_err}"
256            ))),
257        }
258    }
259
260    /// Update the applied/durable LSN reported to the server.
261    ///
262    /// Semantics: call this only once you have durably persisted all events up to `lsn`.
263    /// This update is monotonic and cheap; wire feedback is still governed by the worker’s
264    /// `status_interval` and keepalive reply requests.
265    #[inline]
266    pub fn update_applied_lsn(&self, lsn: Lsn) {
267        self.progress.update_applied(lsn);
268    }
269
270    /// A cloneable writer for the same applied/durable LSN as
271    /// [`update_applied_lsn`](Self::update_applied_lsn), usable without borrowing the client.
272    pub fn applied_lsn_handle(&self) -> AppliedLsnHandle {
273        AppliedLsnHandle(Arc::clone(&self.progress))
274    }
275
276    /// Request the worker to stop gracefully.
277    ///
278    /// After calling this, [`recv()`](Self::recv) will return remaining buffered
279    /// events, then `Ok(None)` once the worker exits cleanly.
280    ///
281    /// This sends a CopyDone message to the server to cleanly terminate
282    /// the replication stream.
283    #[inline]
284    pub fn stop(&self) {
285        let _ = self.stop_tx.send(true);
286    }
287
288    pub fn is_running(&self) -> bool {
289        self.join
290            .as_ref()
291            .map(|j| !j.is_finished())
292            .unwrap_or(false)
293    }
294
295    /// Wait for the worker task to complete and return its result.
296    ///
297    /// This consumes the client. Use this for diagnostics or to ensure
298    /// clean shutdown after calling [`stop()`](Self::stop).
299    pub async fn join(mut self) -> Result<()> {
300        let join = self
301            .join
302            .take()
303            .ok_or_else(|| PgWireError::Task("worker already joined".into()))?;
304
305        match join.await {
306            Ok(inner) => inner,
307            Err(e) => Err(PgWireError::Task(format!("join error: {e}"))),
308        }
309    }
310
311    /// Abort the worker task immediately.
312    ///
313    /// This is a hard cancel and does not send CopyDone.
314    /// Prefer `stop()`/`shutdown()` for graceful termination.
315    pub fn abort(&mut self) {
316        if let Some(join) = self.join.take() {
317            join.abort();
318        }
319    }
320
321    /// Request a graceful stop and wait for the worker to exit.
322    pub async fn shutdown(&mut self) -> Result<()> {
323        self.stop();
324
325        // Drain events until the worker closes the channel.
326        while let Some(msg) = self.rx.recv().await {
327            match msg {
328                Ok(_ev) => {} //discard; caller can drain themselves if they need events
329                Err(e) => return Err(e),
330            }
331        }
332
333        self.join_mut().await
334    }
335
336    /// Wait for the worker task to complete and return its result.
337    async fn join_mut(&mut self) -> Result<()> {
338        let join = self
339            .join
340            .take()
341            .ok_or_else(|| PgWireError::Task("worker already joined".into()))?;
342
343        match join.await {
344            Ok(inner) => inner,
345            Err(e) => Err(PgWireError::Task(format!("join error: {e}"))),
346        }
347    }
348}
349
350pub(super) fn validate_config(cfg: &ReplicationConfig) -> Result<()> {
351    if cfg.slot.is_empty()
352        || cfg.slot.len() > 63
353        || !cfg
354            .slot
355            .bytes()
356            .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_')
357    {
358        return Err(PgWireError::Configuration(
359            "slot must be at most 63 bytes and contain only lower-case ASCII letters, digits, or underscore"
360                .into(),
361        ));
362    }
363    if cfg.publication.is_empty()
364        || cfg.publication.len() > 63
365        || !cfg
366            .publication
367            .bytes()
368            .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_')
369        || cfg.publication.as_bytes()[0].is_ascii_digit()
370    {
371        return Err(PgWireError::Configuration(
372            "publication must start with a lower-case ASCII letter or underscore and contain only lower-case ASCII letters, digits, or underscore (63 bytes maximum)"
373                .into(),
374        ));
375    }
376    if cfg.buffer_events == 0 {
377        return Err(PgWireError::Configuration(
378            "buffer_events must be greater than zero".into(),
379        ));
380    }
381    if cfg.max_message_bytes == 0 {
382        return Err(PgWireError::Configuration(
383            "max_message_bytes must be greater than zero".into(),
384        ));
385    }
386    if cfg.max_in_flight_bytes == 0 {
387        return Err(PgWireError::Configuration(
388            "max_in_flight_bytes must be greater than zero".into(),
389        ));
390    }
391    if cfg.max_message_bytes > cfg.max_in_flight_bytes {
392        return Err(PgWireError::Configuration(format!(
393            "max_message_bytes ({}) must not exceed max_in_flight_bytes ({})",
394            cfg.max_message_bytes, cfg.max_in_flight_bytes
395        )));
396    }
397    if cfg.max_message_bytes > u32::MAX as usize {
398        return Err(PgWireError::Configuration(format!(
399            "max_message_bytes must not exceed {}",
400            u32::MAX
401        )));
402    }
403    if cfg.max_in_flight_bytes > Semaphore::MAX_PERMITS {
404        return Err(PgWireError::Configuration(format!(
405            "max_in_flight_bytes must not exceed {}",
406            Semaphore::MAX_PERMITS
407        )));
408    }
409    if cfg.status_interval.is_zero() {
410        return Err(PgWireError::Configuration(
411            "status_interval must be greater than zero".into(),
412        ));
413    }
414    if cfg.idle_wakeup_interval.is_zero() {
415        return Err(PgWireError::Configuration(
416            "idle_wakeup_interval must be greater than zero".into(),
417        ));
418    }
419    Ok(())
420}
421
422impl Drop for ReplicationClient {
423    fn drop(&mut self) {
424        let _ = self.stop_tx.send(true);
425
426        if let Some(join) = self.join.take() {
427            drop(self.runtime.spawn(reap_dropped_worker(join)));
428        }
429    }
430}
431
432async fn reap_dropped_worker(mut join: JoinHandle<std::result::Result<(), PgWireError>>) {
433    if tokio::time::timeout(DROP_SHUTDOWN_GRACE, &mut join)
434        .await
435        .is_err()
436    {
437        join.abort();
438        let _ = join.await;
439    }
440}
441
442async fn run_worker(worker: &mut WorkerState, cfg: &ReplicationConfig) -> Result<()> {
443    #[cfg(unix)]
444    if cfg.is_unix_socket() {
445        if cfg.tls.mode.requires_tls() {
446            return Err(PgWireError::Tls(
447                "TLS is not supported over Unix domain sockets".into(),
448            ));
449        }
450
451        let path = cfg.unix_socket_path();
452        let mut stream = UnixStream::connect(&path).await.map_err(|e| {
453            PgWireError::Io(std::sync::Arc::new(std::io::Error::new(
454                e.kind(),
455                format!("failed to connect to Unix socket {}: {e}", path.display()),
456            )))
457        })?;
458
459        return worker.run_on_stream(&mut stream).await;
460    }
461
462    let tcp = TcpStream::connect((cfg.host.as_str(), cfg.port)).await?;
463    tcp.set_nodelay(true)?;
464
465    #[cfg(feature = "tls-rustls")]
466    {
467        use crate::tls::rustls::{maybe_upgrade_to_tls, MaybeTlsStream};
468        let upgraded = maybe_upgrade_to_tls(tcp, &cfg.tls, &cfg.host).await?;
469        match upgraded {
470            MaybeTlsStream::Plain(mut s) => worker.run_on_stream(&mut s).await,
471            MaybeTlsStream::Tls(mut s) => worker.run_on_stream(s.as_mut()).await,
472        }
473    }
474
475    #[cfg(not(feature = "tls-rustls"))]
476    {
477        if !matches!(cfg.tls.mode, SslMode::Disable) {
478            return Err(PgWireError::Tls("tls-rustls feature not enabled".into()));
479        }
480        let mut s = tcp;
481        worker.run_on_stream(&mut s).await
482    }
483}
484
485#[cfg(test)]
486mod tests {
487    use std::sync::atomic::{AtomicBool, Ordering};
488    use std::sync::Arc;
489    use std::time::Duration;
490
491    use tokio::net::TcpListener;
492    use tokio::sync::{mpsc, oneshot, watch};
493
494    use super::{validate_config, ReplicationClient, DROP_SHUTDOWN_GRACE};
495    use crate::client::worker::{ReplicationEvent, SharedProgress};
496    use crate::config::ReplicationConfig;
497    use crate::error::PgWireError;
498    use crate::lsn::Lsn;
499
500    #[test]
501    fn buffering_config_requires_a_fittable_frame_and_nonzero_channel() {
502        let mut config = ReplicationConfig::default();
503        config.buffer_events = 0;
504        assert!(validate_config(&config).is_err());
505
506        config.buffer_events = 1;
507        config.max_message_bytes = 9;
508        config.max_in_flight_bytes = 8;
509        let error = validate_config(&config).unwrap_err();
510        assert!(error.to_string().contains("must not exceed"), "{error}");
511
512        config.max_message_bytes = 8;
513        assert!(validate_config(&config).is_ok());
514    }
515
516    #[test]
517    fn replication_identifiers_use_the_server_wire_grammar() {
518        for (field, value) in [
519            ("slot", "Mixed-Case"),
520            ("publication", "quoted publication"),
521            ("publication", "pub'option"),
522            ("publication", "9publication"),
523        ] {
524            let mut config = ReplicationConfig::default();
525            if field == "slot" {
526                config.slot = value.into();
527            } else {
528                config.publication = value.into();
529            }
530            let error = validate_config(&config).unwrap_err();
531            assert!(error.to_string().contains(field), "{error}");
532        }
533    }
534
535    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
536    async fn drop_reaps_a_worker_stalled_after_stop() {
537        struct WorkerLifetime {
538            active: Arc<AtomicBool>,
539            terminated: Option<oneshot::Sender<()>>,
540        }
541
542        impl Drop for WorkerLifetime {
543            fn drop(&mut self) {
544                self.active.store(false, Ordering::Release);
545                if let Some(terminated) = self.terminated.take() {
546                    let _ = terminated.send(());
547                }
548            }
549        }
550
551        let (events_tx, rx) =
552            mpsc::channel::<std::result::Result<ReplicationEvent, PgWireError>>(1);
553        let (stop_tx, mut stop_rx) = watch::channel(false);
554        let (started_tx, started_rx) = oneshot::channel();
555        let (saw_stop_tx, saw_stop_rx) = oneshot::channel();
556        let (terminated_tx, terminated_rx) = oneshot::channel();
557        let active = Arc::new(AtomicBool::new(false));
558        let worker_active = Arc::clone(&active);
559        let join = tokio::spawn(async move {
560            let _events_tx = events_tx;
561            let _lifetime = WorkerLifetime {
562                active: Arc::clone(&worker_active),
563                terminated: Some(terminated_tx),
564            };
565            worker_active.store(true, Ordering::Release);
566            let _ = started_tx.send(());
567            stop_rx.changed().await.unwrap();
568            assert!(*stop_rx.borrow());
569            let _ = saw_stop_tx.send(());
570            std::future::pending::<()>().await;
571            Ok::<(), PgWireError>(())
572        });
573        let client = ReplicationClient {
574            rx,
575            progress: Arc::new(SharedProgress::new(Lsn::ZERO)),
576            stop_tx,
577            join: Some(join),
578            runtime: tokio::runtime::Handle::current(),
579        };
580        tokio::time::timeout(Duration::from_secs(1), started_rx)
581            .await
582            .expect("stalled worker must start")
583            .unwrap();
584        assert!(active.load(Ordering::Acquire));
585
586        drop(client);
587
588        tokio::time::timeout(Duration::from_secs(1), saw_stop_rx)
589            .await
590            .expect("Drop must request graceful stop")
591            .unwrap();
592        tokio::time::timeout(DROP_SHUTDOWN_GRACE + Duration::from_secs(1), terminated_rx)
593            .await
594            .expect("stalled worker must be aborted and reaped within the cleanup bound")
595            .unwrap();
596        assert!(!active.load(Ordering::Acquire));
597    }
598
599    #[tokio::test]
600    async fn cancelled_connect_retains_lifetime_until_startup_worker_is_destroyed() {
601        struct WorkerLifetime(Option<oneshot::Sender<()>>);
602
603        impl Drop for WorkerLifetime {
604            fn drop(&mut self) {
605                if let Some(terminated) = self.0.take() {
606                    let _ = terminated.send(());
607                }
608            }
609        }
610
611        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
612        let mut config = ReplicationConfig::default();
613        config.port = listener.local_addr().unwrap().port();
614        let (terminated_tx, terminated_rx) = oneshot::channel();
615
616        let connect = tokio::spawn(ReplicationClient::connect_with_worker_lifetime(
617            config,
618            WorkerLifetime(Some(terminated_tx)),
619        ));
620        let (_socket, _) = tokio::time::timeout(Duration::from_secs(1), listener.accept())
621            .await
622            .expect("startup worker must connect")
623            .unwrap();
624
625        connect.abort();
626        let error = match connect.await {
627            Err(error) => error,
628            Ok(_) => panic!("connect task must be cancelled"),
629        };
630        assert!(error.is_cancelled());
631        tokio::time::timeout(Duration::from_secs(1), terminated_rx)
632            .await
633            .expect("worker lifetime must end after startup cancellation")
634            .unwrap();
635    }
636}