Skip to main content

pg_bus/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2//! pg-bus: a message bus on PostgreSQL.
3//!
4//! Messages are published inside the caller's transaction, kept in a
5//! backlog table, and delivered to subscribers that resume from a cursor.
6//!
7//! # Example
8//!
9//! ```no_run
10//! use pg_bus::{Bus, Config, Filter, Item};
11//! use serde_json::json;
12//!
13//! # async fn example(pool: sqlx::PgPool) -> Result<(), Box<dyn std::error::Error>> {
14//! let bus = Bus::start(pool.clone(), Config::default()).await?;
15//!
16//! // Publish inside the transaction that makes the change: delivered once
17//! // it commits, never if it rolls back.
18//! let mut tx = pool.begin().await?;
19//! bus.publish(&mut tx, "/topic/35", &json!({ "post_id": 52 }), None).await?;
20//! tx.commit().await?;
21//!
22//! // Subscribe from a position (here, from now on) to some channels, with
23//! // the tags this subscriber holds.
24//! let filter = Filter {
25//!     channels: vec!["/topic/35".into()],
26//!     tags: vec!["user:3".into()],
27//! };
28//! let mut subscription = bus.subscribe(bus.now().await?, filter);
29//! while let Ok(item) = subscription.next().await {
30//!     match item {
31//!         Item::Message(m) => println!("{} {} {}", m.position, m.channel, m.data),
32//!         Item::Gap => println!("missed some: reload"),
33//!     }
34//! }
35//! # Ok(())
36//! # }
37//! ```
38//!
39//! # Ordering
40//!
41//! Ids come from a sequence at insert, but transactions commit in their own
42//! order, so a cursor over ids alone would skip a message whose transaction
43//! commits after a later one was delivered. Each message therefore records
44//! the id of the transaction that wrote it (`xid8`), and a message is only
45//! delivered once every older transaction has finished: below the current
46//! snapshot's `xmin`. Delivery runs in `(xid, id)` order, and since nothing
47//! can commit below the horizon afterwards, a [`Position`] in that order
48//! never skips a committed message. The cost is that a long-running write
49//! transaction anywhere in the database holds delivery back until it ends.
50//!
51//! # Limits
52//!
53//! - **PostgreSQL 13 or newer.** Positions use `xid8` and
54//!   `pg_current_xact_id`; `Bus::start` refuses older servers.
55//! - **One listening connection per process,** held open for LISTEN. It must
56//!   reach PostgreSQL directly or through a pooler in session mode: LISTEN
57//!   does not work through pgbouncer in transaction mode. When the pool goes
58//!   through one, set `Config::listen_url` to a direct URL for the listener.
59//! - **Publish throughput is bounded by NOTIFY.** A transaction that sends a
60//!   notification takes a global lock at commit, so publishing transactions
61//!   commit one at a time. BENCH.md has about 4,000 single-message
62//!   transactions a second on a laptop; batching several messages into one
63//!   transaction sends one notification.
64//! - **No two-phase commit.** Publishing queues a NOTIFY, and PostgreSQL
65//!   refuses to PREPARE a transaction that has one.
66//! - **Long write transactions delay delivery.** A message is delivered once
67//!   every older write transaction in the database has ended, so one left
68//!   open (idle in transaction, a forgotten prepared transaction) holds back
69//!   everything after it. Nothing is lost, and `Bus::health` and the
70//!   listener's warning name the transaction.
71//! - **Gaps are not per channel.** A subscriber resuming from before a trim
72//!   is told it may have missed messages even when the trim was on another
73//!   channel. It reloads when it need not, never the other way round.
74//! - **Memory:** each process keeps its most recent messages
75//!   (`Config::recent`, default 10,000) for catch-ups, and each subscription
76//!   queues at most a batch.
77//! - **Channels match exactly;** there are no wildcards or prefixes.
78//! - **One order across channels:** delivery follows transaction ids (roughly
79//!   when each transaction first wrote), the same for every subscriber; a
80//!   transaction that commits late is held back, not delivered out of turn.
81//!   There are no per-channel sequence numbers.
82
83mod health;
84mod listener;
85mod recent;
86mod schema;
87#[cfg(feature = "axum")]
88#[cfg_attr(docsrs, doc(cfg(feature = "axum")))]
89pub mod sse;
90mod stats;
91mod subscription;
92mod trim;
93
94use std::fmt;
95use std::str::FromStr;
96use std::sync::{Arc, Mutex};
97use std::time::Duration;
98
99use serde_json::Value;
100use sqlx::{PgConnection, PgPool};
101use tokio::sync::broadcast;
102
103pub use health::{Blocker, Health, stall_report};
104pub use stats::Stats;
105pub use subscription::{Item, Subscription};
106pub use trim::Trim;
107
108/// Where a subscriber is in the bus: the last message it has seen, as
109/// `(transaction id, message id)`. Opaque to clients, which hand back its
110/// string form (an SSE `Last-Event-ID`).
111#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
112pub struct Position {
113    xid: u64,
114    id: i64,
115}
116
117impl Position {
118    /// Before every message.
119    pub const START: Position = Position { xid: 0, id: 0 };
120}
121
122impl fmt::Display for Position {
123    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
124        write!(f, "{}-{}", self.xid, self.id)
125    }
126}
127
128impl FromStr for Position {
129    type Err = Error;
130
131    fn from_str(s: &str) -> Result<Self, Self::Err> {
132        let invalid = || Error::InvalidPosition(s.to_string());
133        let (xid, id) = s.split_once('-').ok_or_else(invalid)?;
134        Ok(Position {
135            xid: xid.parse().map_err(|_| invalid())?,
136            id: id.parse().map_err(|_| invalid())?,
137        })
138    }
139}
140
141/// A delivered message.
142#[derive(Debug, Clone, PartialEq)]
143pub struct Message {
144    pub position: Position,
145    pub channel: String,
146    pub data: Value,
147    /// `None`: every subscriber of the channel; otherwise the tags a
148    /// subscriber needs at least one of.
149    pub audience: Option<Vec<String>>,
150}
151
152/// Which messages a subscriber receives: those on its channels whose
153/// audience is everyone or shares a tag with the subscriber's.
154#[derive(Debug, Clone, Default)]
155pub struct Filter {
156    pub channels: Vec<String>,
157    pub tags: Vec<String>,
158}
159
160impl Filter {
161    pub fn matches(&self, message: &Message) -> bool {
162        self.channels.contains(&message.channel)
163            && match &message.audience {
164                None => true,
165                Some(audience) => audience.iter().any(|a| self.tags.contains(a)),
166            }
167    }
168}
169
170#[derive(Debug)]
171pub enum Error {
172    Db(sqlx::Error),
173    InvalidPosition(String),
174    InvalidSchema(String),
175    /// The server is older than PostgreSQL 13 (no `xid8`); its
176    /// `server_version_num`.
177    UnsupportedServer(i64),
178    /// The bus was dropped while a subscriber waited.
179    Closed,
180}
181
182impl fmt::Display for Error {
183    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
184        match self {
185            Error::Db(e) => write!(f, "pg-bus: database: {e}"),
186            Error::InvalidPosition(s) => write!(f, "pg-bus: not a position: {s:?}"),
187            Error::InvalidSchema(s) => write!(f, "pg-bus: not a schema name: {s:?}"),
188            Error::UnsupportedServer(v) => write!(
189                f,
190                "pg-bus: needs PostgreSQL 13 or newer (xid8), the server is {v}"
191            ),
192            Error::Closed => write!(f, "pg-bus: the bus was dropped"),
193        }
194    }
195}
196
197impl std::error::Error for Error {
198    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
199        match self {
200            Error::Db(e) => Some(e),
201            _ => None,
202        }
203    }
204}
205
206impl From<sqlx::Error> for Error {
207    fn from(e: sqlx::Error) -> Self {
208        Error::Db(e)
209    }
210}
211
212/// How the bus runs.
213#[derive(Debug, Clone)]
214pub struct Config {
215    /// The schema holding the backlog; also the NOTIFY channel's name.
216    pub schema: String,
217    /// Messages a subscriber may fall behind the live feed before it
218    /// catches up from the backlog instead.
219    pub capacity: usize,
220    /// How often the listener looks again while committed messages wait
221    /// behind an older transaction (the delay doubles up to `max_poll`).
222    pub min_poll: Duration,
223    pub max_poll: Duration,
224    /// Without a notification, the listener still looks this often.
225    pub idle_poll: Duration,
226    /// The listener logs a warning (at most once a minute) when a committed
227    /// message has waited longer than this behind an open transaction.
228    pub stall_warning: Duration,
229    /// Recent messages each process keeps in memory, so subscribers
230    /// resuming from a recent position catch up without a query.
231    pub recent: usize,
232    /// How old a message must be before [`Bus::trim`] may delete it: longer
233    /// than any listener could still be behind. A subscriber whose position
234    /// is within a process's recent ring is caught up from memory without a
235    /// gap check, which holds only if trim never deletes a message that
236    /// listener has not read yet.
237    pub trim_grace: Duration,
238    /// The background trim: on by default with [`Trim`]'s defaults; `None`
239    /// leaves trimming to calls of [`Bus::trim`].
240    pub trim: Option<Trim>,
241    /// Where the listener connects, when the pool goes through a pooler
242    /// in transaction mode (LISTEN needs a session of its own): a direct
243    /// or session-mode URL. `None`: the pool's settings.
244    pub listen_url: Option<String>,
245}
246
247impl Default for Config {
248    fn default() -> Self {
249        Config {
250            schema: "pg_bus".to_string(),
251            capacity: 1024,
252            min_poll: Duration::from_millis(20),
253            max_poll: Duration::from_secs(1),
254            idle_poll: Duration::from_secs(30),
255            stall_warning: Duration::from_secs(30),
256            recent: 10_000,
257            trim_grace: Duration::from_secs(60),
258            trim: Some(Trim::default()),
259            listen_url: None,
260        }
261    }
262}
263
264/// The bus: publishing, the backlog, and one listener per process feeding
265/// its subscribers.
266#[derive(Clone)]
267pub struct Bus {
268    inner: Arc<Inner>,
269}
270
271struct Inner {
272    pool: PgPool,
273    config: Config,
274    shared: Arc<Shared>,
275    /// Where the listener started, for stats before it reads anything.
276    start: Position,
277    listener: tokio::task::JoinHandle<()>,
278    trimmer: tokio::task::JoinHandle<()>,
279}
280
281/// What the listener and the subscriptions in one process share: the live
282/// feed, the recent ring and the counters.
283pub(crate) struct Shared {
284    pub(crate) feed: broadcast::Sender<Arc<Message>>,
285    pub(crate) recent: Mutex<recent::Recent>,
286    pub(crate) counters: stats::Counters,
287}
288
289impl Drop for Inner {
290    fn drop(&mut self) {
291        self.listener.abort();
292        self.trimmer.abort();
293    }
294}
295
296/// The columns a message is read with, in [`Message`]'s order plus the
297/// position. The transaction id comes out as text under its own name: an
298/// output column called `xid` would be what an unqualified `ORDER BY xid`
299/// sorts by, and as text "1000000" sorts before "999997".
300const COLUMNS: &str = "messages.xid::text AS xid_text, messages.id, messages.channel, messages.data, messages.audience";
301
302type Row = (String, i64, String, Value, Option<Vec<String>>);
303
304fn message(row: Row) -> Result<Message, Error> {
305    let (xid, id, channel, data, audience) = row;
306    let xid = xid.parse().map_err(|_| Error::InvalidPosition(xid))?;
307    Ok(Message {
308        position: Position { xid, id },
309        channel,
310        data,
311        audience,
312    })
313}
314
315impl Bus {
316    /// Creates the schema if needed and starts the listener.
317    pub async fn start(pool: PgPool, config: Config) -> Result<Bus, Error> {
318        schema::validate(&config.schema)?;
319        schema::check_server(&pool).await?;
320        schema::migrate(&pool, &config.schema).await?;
321        let (feed, _) = broadcast::channel(config.capacity);
322        let start = horizon_on(&pool).await?;
323        let shared = Arc::new(Shared {
324            feed,
325            recent: Mutex::new(recent::Recent::new(start, config.recent)),
326            counters: stats::Counters::default(),
327        });
328        let listener = tokio::spawn(listener::run(
329            pool.clone(),
330            config.clone(),
331            shared.clone(),
332            start,
333        ));
334        let trimmer = tokio::spawn(trim::run(pool.clone(), config.clone()));
335        Ok(Bus {
336            inner: Arc::new(Inner {
337                pool,
338                config,
339                shared,
340                start,
341                listener,
342                trimmer,
343            }),
344        })
345    }
346
347    fn schema(&self) -> &str {
348        &self.inner.config.schema
349    }
350
351    /// Writes a message in the caller's transaction. It is delivered once
352    /// that transaction commits, and never if it rolls back. Returns its
353    /// position, which a client that made the change can resume from.
354    ///
355    /// The transaction cannot then use two-phase commit: publishing queues a
356    /// NOTIFY, and Postgres refuses to PREPARE a transaction that has one.
357    pub async fn publish(
358        &self,
359        conn: &mut PgConnection,
360        channel: &str,
361        data: &Value,
362        audience: Option<&[String]>,
363    ) -> Result<Position, Error> {
364        let schema = self.schema();
365        let (xid, id): (String, i64) = sqlx::query_as(&format!(
366            "INSERT INTO {schema}.messages (channel, data, audience) VALUES ($1, $2, $3) \
367             RETURNING xid::text, id"
368        ))
369        .bind(channel)
370        .bind(data)
371        .bind(audience)
372        .fetch_one(&mut *conn)
373        .await?;
374        let xid = xid.parse().map_err(|_| Error::InvalidPosition(xid))?;
375        // Postgres sends it at commit, once per transaction however many
376        // messages it wrote.
377        sqlx::query("SELECT pg_notify($1, '')")
378            .bind(schema)
379            .execute(&mut *conn)
380            .await?;
381        Ok(Position { xid, id })
382    }
383
384    /// What waits behind the delivery horizon and the transactions holding
385    /// it, for monitoring (see [`Health`]).
386    pub async fn health(&self) -> Result<Health, Error> {
387        health::health_on(&self.inner.pool, self.schema()).await
388    }
389
390    /// The position a new subscriber starts from: after every message
391    /// already deliverable, so it receives what commits from now on.
392    pub async fn now(&self) -> Result<Position, Error> {
393        horizon_on(&self.inner.pool).await
394    }
395
396    /// Deliverable messages after `from`, oldest first, at most `limit`.
397    /// A message committed a moment ago may not be here yet: it waits until
398    /// every older write transaction has ended (the crate docs, Limits).
399    pub async fn backlog(
400        &self,
401        from: Position,
402        filter: &Filter,
403        limit: i64,
404    ) -> Result<Vec<Message>, Error> {
405        backlog_on(&self.inner.pool, self.schema(), from, Some(filter), limit).await
406    }
407
408    /// True when messages after `from` were trimmed away: the subscriber
409    /// has missed some and should start over from fresh state. Not per
410    /// channel: a trim on any channel counts, so a subscriber may reload
411    /// when it did not need to, but never misses one it did.
412    pub async fn trimmed_after(&self, from: Position) -> Result<bool, Error> {
413        let schema = self.schema();
414        let trimmed: Option<(String, i64)> = sqlx::query_as(&format!(
415            "SELECT trimmed_xid::text, trimmed_id FROM {schema}.state"
416        ))
417        .fetch_optional(&self.inner.pool)
418        .await?;
419        let Some((xid, id)) = trimmed else {
420            return Ok(false);
421        };
422        let xid: u64 = xid.parse().map_err(|_| Error::InvalidPosition(xid))?;
423        Ok(from < Position { xid, id })
424    }
425
426    /// Deletes delivered messages older than `max_age`, and on each channel
427    /// all but the newest `keep_per_channel`. Returns how many went. Never
428    /// deletes a message younger than [`Config::trim_grace`]: a listener may
429    /// not have read it yet, and its subscribers would miss it unawares.
430    /// Waits while another process trims (the background task, see
431    /// [`Config::trim`], does not: it skips the round).
432    pub async fn trim(&self, max_age: Duration, keep_per_channel: i64) -> Result<u64, Error> {
433        let config = &self.inner.config;
434        let deleted = trim::trim_on(
435            &self.inner.pool,
436            &config.schema,
437            config.trim_grace,
438            max_age,
439            keep_per_channel,
440            true,
441        )
442        .await?;
443        Ok(deleted.unwrap_or(0))
444    }
445
446    /// Subscribes from `from` (see [`Bus::now`] for a new subscriber):
447    /// the backlog first, then the live feed.
448    pub fn subscribe(&self, from: Position, filter: Filter) -> Subscription {
449        Subscription::new(self.clone(), from, filter)
450    }
451
452    /// This process's counters (see [`Stats`]).
453    pub fn stats(&self) -> Stats {
454        let shared = &self.inner.shared;
455        shared
456            .counters
457            .snapshot(shared.feed.receiver_count(), self.inner.start)
458    }
459
460    pub(crate) fn feed(&self) -> broadcast::Receiver<Arc<Message>> {
461        self.inner.shared.feed.subscribe()
462    }
463
464    pub(crate) fn counters(&self) -> &stats::Counters {
465        &self.inner.shared.counters
466    }
467
468    /// The recent messages after `after`, if this process still holds all
469    /// of them in memory.
470    pub(crate) fn recent_after(
471        &self,
472        after: Position,
473        filter: &Filter,
474        limit: usize,
475    ) -> Option<Vec<Message>> {
476        self.inner
477            .shared
478            .recent
479            .lock()
480            .unwrap_or_else(|e| e.into_inner())
481            .after(after, filter, limit)
482    }
483}
484
485/// The delivery horizon as a position: every transaction below the
486/// snapshot's xmin has finished, and ids start at 1, so `(xmin, 0)` is
487/// after all of their messages and before any later transaction's.
488pub(crate) async fn horizon_on(pool: &PgPool) -> Result<Position, Error> {
489    let xmin: String = sqlx::query_scalar("SELECT pg_snapshot_xmin(pg_current_snapshot())::text")
490        .fetch_one(pool)
491        .await?;
492    let xid = xmin.parse().map_err(|_| Error::InvalidPosition(xmin))?;
493    Ok(Position { xid, id: 0 })
494}
495
496/// Deliverable messages after `from` in `(xid, id)` order: written by
497/// transactions below the snapshot's xmin, so none can commit behind them.
498pub(crate) async fn backlog_on(
499    pool: &PgPool,
500    schema: &str,
501    from: Position,
502    filter: Option<&Filter>,
503    limit: i64,
504) -> Result<Vec<Message>, Error> {
505    let filter_sql = if filter.is_some() {
506        "AND channel = ANY($4) AND (audience IS NULL OR audience && $5)"
507    } else {
508        ""
509    };
510    let sql = format!(
511        "SELECT {COLUMNS} FROM {schema}.messages \
512         WHERE (xid, id) > ($1::text::xid8, $2) \
513           AND xid < pg_snapshot_xmin(pg_current_snapshot()) {filter_sql} \
514         ORDER BY messages.xid, messages.id LIMIT $3"
515    );
516    let mut q = sqlx::query_as::<_, Row>(&sql)
517        .bind(from.xid.to_string())
518        .bind(from.id)
519        .bind(limit);
520    if let Some(f) = filter {
521        q = q.bind(&f.channels).bind(&f.tags);
522    }
523    q.fetch_all(pool).await?.into_iter().map(message).collect()
524}
525
526/// Whether any message after `from` is still waiting on an older
527/// transaction.
528pub(crate) async fn pending_after(
529    pool: &PgPool,
530    schema: &str,
531    from: Position,
532) -> Result<bool, Error> {
533    Ok(sqlx::query_scalar(&format!(
534        "SELECT EXISTS (SELECT 1 FROM {schema}.messages WHERE (xid, id) > ($1::text::xid8, $2))"
535    ))
536    .bind(from.xid.to_string())
537    .bind(from.id)
538    .fetch_one(pool)
539    .await?)
540}