pg_bus/lib.rs
1#![cfg_attr(docsrs, feature(doc_cfg))]
2//! pg-bus: a message bus on PostgreSQL.
3//!
4//! Messages are published inside the caller's transaction, kept in a
5//! backlog table, and delivered to subscribers that resume from a cursor.
6//!
7//! # Example
8//!
9//! ```no_run
10//! use pg_bus::{Bus, Config, Filter, Item};
11//! use serde_json::json;
12//!
13//! # async fn example(pool: sqlx::PgPool) -> Result<(), Box<dyn std::error::Error>> {
14//! let bus = Bus::start(pool.clone(), Config::default()).await?;
15//!
16//! // Publish inside the transaction that makes the change: delivered once
17//! // it commits, never if it rolls back.
18//! let mut tx = pool.begin().await?;
19//! bus.publish(&mut tx, "/topic/35", &json!({ "post_id": 52 }), None).await?;
20//! tx.commit().await?;
21//!
22//! // Subscribe from a position (here, from now on) to some channels, with
23//! // the tags this subscriber holds.
24//! let filter = Filter {
25//! channels: vec!["/topic/35".into()],
26//! tags: vec!["user:3".into()],
27//! };
28//! let mut subscription = bus.subscribe(bus.now().await?, filter);
29//! while let Ok(item) = subscription.next().await {
30//! match item {
31//! Item::Message(m) => println!("{} {} {}", m.position, m.channel, m.data),
32//! Item::Gap => println!("missed some: reload"),
33//! }
34//! }
35//! # Ok(())
36//! # }
37//! ```
38//!
39//! # Ordering
40//!
41//! Ids come from a sequence at insert, but transactions commit in their own
42//! order, so a cursor over ids alone would skip a message whose transaction
43//! commits after a later one was delivered. Each message therefore records
44//! the id of the transaction that wrote it (`xid8`), and a message is only
45//! delivered once every older transaction has finished: below the current
46//! snapshot's `xmin`. Delivery runs in `(xid, id)` order, and since nothing
47//! can commit below the horizon afterwards, a [`Position`] in that order
48//! never skips a committed message. The cost is that a long-running write
49//! transaction anywhere in the database holds delivery back until it ends.
50//!
51//! # Limits
52//!
53//! - **PostgreSQL 13 or newer.** Positions use `xid8` and
54//! `pg_current_xact_id`; `Bus::start` refuses older servers.
55//! - **One listening connection per process,** held open for LISTEN. It must
56//! reach PostgreSQL directly or through a pooler in session mode: LISTEN
57//! does not work through pgbouncer in transaction mode. When the pool goes
58//! through one, set `Config::listen_url` to a direct URL for the listener.
59//! - **Publish throughput is bounded by NOTIFY.** A transaction that sends a
60//! notification takes a global lock at commit, so publishing transactions
61//! commit one at a time. BENCH.md has about 4,000 single-message
62//! transactions a second on a laptop; batching several messages into one
63//! transaction sends one notification.
64//! - **No two-phase commit.** Publishing queues a NOTIFY, and PostgreSQL
65//! refuses to PREPARE a transaction that has one.
66//! - **Long write transactions delay delivery.** A message is delivered once
67//! every older write transaction in the database has ended, so one left
68//! open (idle in transaction, a forgotten prepared transaction) holds back
69//! everything after it. Nothing is lost, and `Bus::health` and the
70//! listener's warning name the transaction.
71//! - **Gaps are not per channel.** A subscriber resuming from before a trim
72//! is told it may have missed messages even when the trim was on another
73//! channel. It reloads when it need not, never the other way round.
74//! - **Memory:** each process keeps its most recent messages
75//! (`Config::recent`, default 10,000) for catch-ups, and each subscription
76//! queues at most a batch.
77//! - **Channels match exactly;** there are no wildcards or prefixes.
78//! - **One order across channels:** delivery follows transaction ids (roughly
79//! when each transaction first wrote), the same for every subscriber; a
80//! transaction that commits late is held back, not delivered out of turn.
81//! There are no per-channel sequence numbers.
82
83mod health;
84mod listener;
85mod recent;
86mod schema;
87#[cfg(feature = "axum")]
88#[cfg_attr(docsrs, doc(cfg(feature = "axum")))]
89pub mod sse;
90mod stats;
91mod subscription;
92mod trim;
93
94use std::fmt;
95use std::str::FromStr;
96use std::sync::{Arc, Mutex};
97use std::time::Duration;
98
99use serde_json::Value;
100use sqlx::{PgConnection, PgPool};
101use tokio::sync::broadcast;
102
103pub use health::{Blocker, Health, stall_report};
104pub use stats::Stats;
105pub use subscription::{Item, Subscription};
106pub use trim::Trim;
107
108/// Where a subscriber is in the bus: the last message it has seen, as
109/// `(transaction id, message id)`. Opaque to clients, which hand back its
110/// string form (an SSE `Last-Event-ID`).
111#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
112pub struct Position {
113 xid: u64,
114 id: i64,
115}
116
117impl Position {
118 /// Before every message.
119 pub const START: Position = Position { xid: 0, id: 0 };
120}
121
122impl fmt::Display for Position {
123 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
124 write!(f, "{}-{}", self.xid, self.id)
125 }
126}
127
128impl FromStr for Position {
129 type Err = Error;
130
131 fn from_str(s: &str) -> Result<Self, Self::Err> {
132 let invalid = || Error::InvalidPosition(s.to_string());
133 let (xid, id) = s.split_once('-').ok_or_else(invalid)?;
134 Ok(Position {
135 xid: xid.parse().map_err(|_| invalid())?,
136 id: id.parse().map_err(|_| invalid())?,
137 })
138 }
139}
140
141/// A delivered message.
142#[derive(Debug, Clone, PartialEq)]
143pub struct Message {
144 pub position: Position,
145 pub channel: String,
146 pub data: Value,
147 /// `None`: every subscriber of the channel; otherwise the tags a
148 /// subscriber needs at least one of.
149 pub audience: Option<Vec<String>>,
150}
151
152/// Which messages a subscriber receives: those on its channels whose
153/// audience is everyone or shares a tag with the subscriber's.
154#[derive(Debug, Clone, Default)]
155pub struct Filter {
156 pub channels: Vec<String>,
157 pub tags: Vec<String>,
158}
159
160impl Filter {
161 pub fn matches(&self, message: &Message) -> bool {
162 self.channels.contains(&message.channel)
163 && match &message.audience {
164 None => true,
165 Some(audience) => audience.iter().any(|a| self.tags.contains(a)),
166 }
167 }
168}
169
170#[derive(Debug)]
171pub enum Error {
172 Db(sqlx::Error),
173 InvalidPosition(String),
174 InvalidSchema(String),
175 /// The server is older than PostgreSQL 13 (no `xid8`); its
176 /// `server_version_num`.
177 UnsupportedServer(i64),
178 /// The bus was dropped while a subscriber waited.
179 Closed,
180}
181
182impl fmt::Display for Error {
183 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
184 match self {
185 Error::Db(e) => write!(f, "pg-bus: database: {e}"),
186 Error::InvalidPosition(s) => write!(f, "pg-bus: not a position: {s:?}"),
187 Error::InvalidSchema(s) => write!(f, "pg-bus: not a schema name: {s:?}"),
188 Error::UnsupportedServer(v) => write!(
189 f,
190 "pg-bus: needs PostgreSQL 13 or newer (xid8), the server is {v}"
191 ),
192 Error::Closed => write!(f, "pg-bus: the bus was dropped"),
193 }
194 }
195}
196
197impl std::error::Error for Error {
198 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
199 match self {
200 Error::Db(e) => Some(e),
201 _ => None,
202 }
203 }
204}
205
206impl From<sqlx::Error> for Error {
207 fn from(e: sqlx::Error) -> Self {
208 Error::Db(e)
209 }
210}
211
212/// How the bus runs.
213#[derive(Debug, Clone)]
214pub struct Config {
215 /// The schema holding the backlog; also the NOTIFY channel's name.
216 pub schema: String,
217 /// Messages a subscriber may fall behind the live feed before it
218 /// catches up from the backlog instead.
219 pub capacity: usize,
220 /// How often the listener looks again while committed messages wait
221 /// behind an older transaction (the delay doubles up to `max_poll`).
222 pub min_poll: Duration,
223 pub max_poll: Duration,
224 /// Without a notification, the listener still looks this often.
225 pub idle_poll: Duration,
226 /// The listener logs a warning (at most once a minute) when a committed
227 /// message has waited longer than this behind an open transaction.
228 pub stall_warning: Duration,
229 /// Recent messages each process keeps in memory, so subscribers
230 /// resuming from a recent position catch up without a query.
231 pub recent: usize,
232 /// How old a message must be before [`Bus::trim`] may delete it: longer
233 /// than any listener could still be behind. A subscriber whose position
234 /// is within a process's recent ring is caught up from memory without a
235 /// gap check, which holds only if trim never deletes a message that
236 /// listener has not read yet.
237 pub trim_grace: Duration,
238 /// The background trim: on by default with [`Trim`]'s defaults; `None`
239 /// leaves trimming to calls of [`Bus::trim`].
240 pub trim: Option<Trim>,
241 /// Where the listener connects, when the pool goes through a pooler
242 /// in transaction mode (LISTEN needs a session of its own): a direct
243 /// or session-mode URL. `None`: the pool's settings.
244 pub listen_url: Option<String>,
245}
246
247impl Default for Config {
248 fn default() -> Self {
249 Config {
250 schema: "pg_bus".to_string(),
251 capacity: 1024,
252 min_poll: Duration::from_millis(20),
253 max_poll: Duration::from_secs(1),
254 idle_poll: Duration::from_secs(30),
255 stall_warning: Duration::from_secs(30),
256 recent: 10_000,
257 trim_grace: Duration::from_secs(60),
258 trim: Some(Trim::default()),
259 listen_url: None,
260 }
261 }
262}
263
264/// The bus: publishing, the backlog, and one listener per process feeding
265/// its subscribers.
266#[derive(Clone)]
267pub struct Bus {
268 inner: Arc<Inner>,
269}
270
271struct Inner {
272 pool: PgPool,
273 config: Config,
274 shared: Arc<Shared>,
275 /// Where the listener started, for stats before it reads anything.
276 start: Position,
277 listener: tokio::task::JoinHandle<()>,
278 trimmer: tokio::task::JoinHandle<()>,
279}
280
281/// What the listener and the subscriptions in one process share: the live
282/// feed, the recent ring and the counters.
283pub(crate) struct Shared {
284 pub(crate) feed: broadcast::Sender<Arc<Message>>,
285 pub(crate) recent: Mutex<recent::Recent>,
286 pub(crate) counters: stats::Counters,
287}
288
289impl Drop for Inner {
290 fn drop(&mut self) {
291 self.listener.abort();
292 self.trimmer.abort();
293 }
294}
295
296/// The columns a message is read with, in [`Message`]'s order plus the
297/// position. The transaction id comes out as text under its own name: an
298/// output column called `xid` would be what an unqualified `ORDER BY xid`
299/// sorts by, and as text "1000000" sorts before "999997".
300const COLUMNS: &str = "messages.xid::text AS xid_text, messages.id, messages.channel, messages.data, messages.audience";
301
302type Row = (String, i64, String, Value, Option<Vec<String>>);
303
304fn message(row: Row) -> Result<Message, Error> {
305 let (xid, id, channel, data, audience) = row;
306 let xid = xid.parse().map_err(|_| Error::InvalidPosition(xid))?;
307 Ok(Message {
308 position: Position { xid, id },
309 channel,
310 data,
311 audience,
312 })
313}
314
315impl Bus {
316 /// Creates the schema if needed and starts the listener.
317 pub async fn start(pool: PgPool, config: Config) -> Result<Bus, Error> {
318 schema::validate(&config.schema)?;
319 schema::check_server(&pool).await?;
320 schema::migrate(&pool, &config.schema).await?;
321 let (feed, _) = broadcast::channel(config.capacity);
322 let start = horizon_on(&pool).await?;
323 let shared = Arc::new(Shared {
324 feed,
325 recent: Mutex::new(recent::Recent::new(start, config.recent)),
326 counters: stats::Counters::default(),
327 });
328 let listener = tokio::spawn(listener::run(
329 pool.clone(),
330 config.clone(),
331 shared.clone(),
332 start,
333 ));
334 let trimmer = tokio::spawn(trim::run(pool.clone(), config.clone()));
335 Ok(Bus {
336 inner: Arc::new(Inner {
337 pool,
338 config,
339 shared,
340 start,
341 listener,
342 trimmer,
343 }),
344 })
345 }
346
347 fn schema(&self) -> &str {
348 &self.inner.config.schema
349 }
350
351 /// Writes a message in the caller's transaction. It is delivered once
352 /// that transaction commits, and never if it rolls back. Returns its
353 /// position, which a client that made the change can resume from.
354 ///
355 /// The transaction cannot then use two-phase commit: publishing queues a
356 /// NOTIFY, and Postgres refuses to PREPARE a transaction that has one.
357 pub async fn publish(
358 &self,
359 conn: &mut PgConnection,
360 channel: &str,
361 data: &Value,
362 audience: Option<&[String]>,
363 ) -> Result<Position, Error> {
364 let schema = self.schema();
365 let (xid, id): (String, i64) = sqlx::query_as(&format!(
366 "INSERT INTO {schema}.messages (channel, data, audience) VALUES ($1, $2, $3) \
367 RETURNING xid::text, id"
368 ))
369 .bind(channel)
370 .bind(data)
371 .bind(audience)
372 .fetch_one(&mut *conn)
373 .await?;
374 let xid = xid.parse().map_err(|_| Error::InvalidPosition(xid))?;
375 // Postgres sends it at commit, once per transaction however many
376 // messages it wrote.
377 sqlx::query("SELECT pg_notify($1, '')")
378 .bind(schema)
379 .execute(&mut *conn)
380 .await?;
381 Ok(Position { xid, id })
382 }
383
384 /// What waits behind the delivery horizon and the transactions holding
385 /// it, for monitoring (see [`Health`]).
386 pub async fn health(&self) -> Result<Health, Error> {
387 health::health_on(&self.inner.pool, self.schema()).await
388 }
389
390 /// The position a new subscriber starts from: after every message
391 /// already deliverable, so it receives what commits from now on.
392 pub async fn now(&self) -> Result<Position, Error> {
393 horizon_on(&self.inner.pool).await
394 }
395
396 /// Deliverable messages after `from`, oldest first, at most `limit`.
397 /// A message committed a moment ago may not be here yet: it waits until
398 /// every older write transaction has ended (the crate docs, Limits).
399 pub async fn backlog(
400 &self,
401 from: Position,
402 filter: &Filter,
403 limit: i64,
404 ) -> Result<Vec<Message>, Error> {
405 backlog_on(&self.inner.pool, self.schema(), from, Some(filter), limit).await
406 }
407
408 /// True when messages after `from` were trimmed away: the subscriber
409 /// has missed some and should start over from fresh state. Not per
410 /// channel: a trim on any channel counts, so a subscriber may reload
411 /// when it did not need to, but never misses one it did.
412 pub async fn trimmed_after(&self, from: Position) -> Result<bool, Error> {
413 let schema = self.schema();
414 let trimmed: Option<(String, i64)> = sqlx::query_as(&format!(
415 "SELECT trimmed_xid::text, trimmed_id FROM {schema}.state"
416 ))
417 .fetch_optional(&self.inner.pool)
418 .await?;
419 let Some((xid, id)) = trimmed else {
420 return Ok(false);
421 };
422 let xid: u64 = xid.parse().map_err(|_| Error::InvalidPosition(xid))?;
423 Ok(from < Position { xid, id })
424 }
425
426 /// Deletes delivered messages older than `max_age`, and on each channel
427 /// all but the newest `keep_per_channel`. Returns how many went. Never
428 /// deletes a message younger than [`Config::trim_grace`]: a listener may
429 /// not have read it yet, and its subscribers would miss it unawares.
430 /// Waits while another process trims (the background task, see
431 /// [`Config::trim`], does not: it skips the round).
432 pub async fn trim(&self, max_age: Duration, keep_per_channel: i64) -> Result<u64, Error> {
433 let config = &self.inner.config;
434 let deleted = trim::trim_on(
435 &self.inner.pool,
436 &config.schema,
437 config.trim_grace,
438 max_age,
439 keep_per_channel,
440 true,
441 )
442 .await?;
443 Ok(deleted.unwrap_or(0))
444 }
445
446 /// Subscribes from `from` (see [`Bus::now`] for a new subscriber):
447 /// the backlog first, then the live feed.
448 pub fn subscribe(&self, from: Position, filter: Filter) -> Subscription {
449 Subscription::new(self.clone(), from, filter)
450 }
451
452 /// This process's counters (see [`Stats`]).
453 pub fn stats(&self) -> Stats {
454 let shared = &self.inner.shared;
455 shared
456 .counters
457 .snapshot(shared.feed.receiver_count(), self.inner.start)
458 }
459
460 pub(crate) fn feed(&self) -> broadcast::Receiver<Arc<Message>> {
461 self.inner.shared.feed.subscribe()
462 }
463
464 pub(crate) fn counters(&self) -> &stats::Counters {
465 &self.inner.shared.counters
466 }
467
468 /// The recent messages after `after`, if this process still holds all
469 /// of them in memory.
470 pub(crate) fn recent_after(
471 &self,
472 after: Position,
473 filter: &Filter,
474 limit: usize,
475 ) -> Option<Vec<Message>> {
476 self.inner
477 .shared
478 .recent
479 .lock()
480 .unwrap_or_else(|e| e.into_inner())
481 .after(after, filter, limit)
482 }
483}
484
485/// The delivery horizon as a position: every transaction below the
486/// snapshot's xmin has finished, and ids start at 1, so `(xmin, 0)` is
487/// after all of their messages and before any later transaction's.
488pub(crate) async fn horizon_on(pool: &PgPool) -> Result<Position, Error> {
489 let xmin: String = sqlx::query_scalar("SELECT pg_snapshot_xmin(pg_current_snapshot())::text")
490 .fetch_one(pool)
491 .await?;
492 let xid = xmin.parse().map_err(|_| Error::InvalidPosition(xmin))?;
493 Ok(Position { xid, id: 0 })
494}
495
496/// Deliverable messages after `from` in `(xid, id)` order: written by
497/// transactions below the snapshot's xmin, so none can commit behind them.
498pub(crate) async fn backlog_on(
499 pool: &PgPool,
500 schema: &str,
501 from: Position,
502 filter: Option<&Filter>,
503 limit: i64,
504) -> Result<Vec<Message>, Error> {
505 let filter_sql = if filter.is_some() {
506 "AND channel = ANY($4) AND (audience IS NULL OR audience && $5)"
507 } else {
508 ""
509 };
510 let sql = format!(
511 "SELECT {COLUMNS} FROM {schema}.messages \
512 WHERE (xid, id) > ($1::text::xid8, $2) \
513 AND xid < pg_snapshot_xmin(pg_current_snapshot()) {filter_sql} \
514 ORDER BY messages.xid, messages.id LIMIT $3"
515 );
516 let mut q = sqlx::query_as::<_, Row>(&sql)
517 .bind(from.xid.to_string())
518 .bind(from.id)
519 .bind(limit);
520 if let Some(f) = filter {
521 q = q.bind(&f.channels).bind(&f.tags);
522 }
523 q.fetch_all(pool).await?.into_iter().map(message).collect()
524}
525
526/// Whether any message after `from` is still waiting on an older
527/// transaction.
528pub(crate) async fn pending_after(
529 pool: &PgPool,
530 schema: &str,
531 from: Position,
532) -> Result<bool, Error> {
533 Ok(sqlx::query_scalar(&format!(
534 "SELECT EXISTS (SELECT 1 FROM {schema}.messages WHERE (xid, id) > ($1::text::xid8, $2))"
535 ))
536 .bind(from.xid.to_string())
537 .bind(from.id)
538 .fetch_one(pool)
539 .await?)
540}