#![cfg_attr(docsrs, feature(doc_cfg))]
mod health;
mod listener;
mod recent;
mod schema;
#[cfg(feature = "axum")]
#[cfg_attr(docsrs, doc(cfg(feature = "axum")))]
pub mod sse;
mod stats;
mod subscription;
mod trim;
use std::fmt;
use std::str::FromStr;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use serde_json::Value;
use sqlx::{PgConnection, PgPool};
use tokio::sync::broadcast;
pub use health::{Blocker, Health, stall_report};
pub use stats::Stats;
pub use subscription::{Item, Subscription};
pub use trim::Trim;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Position {
xid: u64,
id: i64,
}
impl Position {
pub const START: Position = Position { xid: 0, id: 0 };
}
impl fmt::Display for Position {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}-{}", self.xid, self.id)
}
}
impl FromStr for Position {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let invalid = || Error::InvalidPosition(s.to_string());
let (xid, id) = s.split_once('-').ok_or_else(invalid)?;
Ok(Position {
xid: xid.parse().map_err(|_| invalid())?,
id: id.parse().map_err(|_| invalid())?,
})
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct Message {
pub position: Position,
pub channel: String,
pub data: Value,
pub audience: Option<Vec<String>>,
}
#[derive(Debug, Clone, Default)]
pub struct Filter {
pub channels: Vec<String>,
pub tags: Vec<String>,
}
impl Filter {
pub fn matches(&self, message: &Message) -> bool {
self.channels.contains(&message.channel)
&& match &message.audience {
None => true,
Some(audience) => audience.iter().any(|a| self.tags.contains(a)),
}
}
}
#[derive(Debug)]
pub enum Error {
Db(sqlx::Error),
InvalidPosition(String),
InvalidSchema(String),
UnsupportedServer(i64),
Closed,
}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Error::Db(e) => write!(f, "pg-bus: database: {e}"),
Error::InvalidPosition(s) => write!(f, "pg-bus: not a position: {s:?}"),
Error::InvalidSchema(s) => write!(f, "pg-bus: not a schema name: {s:?}"),
Error::UnsupportedServer(v) => write!(
f,
"pg-bus: needs PostgreSQL 13 or newer (xid8), the server is {v}"
),
Error::Closed => write!(f, "pg-bus: the bus was dropped"),
}
}
}
impl std::error::Error for Error {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Error::Db(e) => Some(e),
_ => None,
}
}
}
impl From<sqlx::Error> for Error {
fn from(e: sqlx::Error) -> Self {
Error::Db(e)
}
}
#[derive(Debug, Clone)]
pub struct Config {
pub schema: String,
pub capacity: usize,
pub min_poll: Duration,
pub max_poll: Duration,
pub idle_poll: Duration,
pub stall_warning: Duration,
pub recent: usize,
pub trim_grace: Duration,
pub trim: Option<Trim>,
pub listen_url: Option<String>,
}
impl Default for Config {
fn default() -> Self {
Config {
schema: "pg_bus".to_string(),
capacity: 1024,
min_poll: Duration::from_millis(20),
max_poll: Duration::from_secs(1),
idle_poll: Duration::from_secs(30),
stall_warning: Duration::from_secs(30),
recent: 10_000,
trim_grace: Duration::from_secs(60),
trim: Some(Trim::default()),
listen_url: None,
}
}
}
#[derive(Clone)]
pub struct Bus {
inner: Arc<Inner>,
}
struct Inner {
pool: PgPool,
config: Config,
shared: Arc<Shared>,
start: Position,
listener: tokio::task::JoinHandle<()>,
trimmer: tokio::task::JoinHandle<()>,
}
pub(crate) struct Shared {
pub(crate) feed: broadcast::Sender<Arc<Message>>,
pub(crate) recent: Mutex<recent::Recent>,
pub(crate) counters: stats::Counters,
}
impl Drop for Inner {
fn drop(&mut self) {
self.listener.abort();
self.trimmer.abort();
}
}
const COLUMNS: &str = "messages.xid::text AS xid_text, messages.id, messages.channel, messages.data, messages.audience";
type Row = (String, i64, String, Value, Option<Vec<String>>);
fn message(row: Row) -> Result<Message, Error> {
let (xid, id, channel, data, audience) = row;
let xid = xid.parse().map_err(|_| Error::InvalidPosition(xid))?;
Ok(Message {
position: Position { xid, id },
channel,
data,
audience,
})
}
impl Bus {
pub async fn start(pool: PgPool, config: Config) -> Result<Bus, Error> {
schema::validate(&config.schema)?;
schema::check_server(&pool).await?;
schema::migrate(&pool, &config.schema).await?;
let (feed, _) = broadcast::channel(config.capacity);
let start = horizon_on(&pool).await?;
let shared = Arc::new(Shared {
feed,
recent: Mutex::new(recent::Recent::new(start, config.recent)),
counters: stats::Counters::default(),
});
let listener = tokio::spawn(listener::run(
pool.clone(),
config.clone(),
shared.clone(),
start,
));
let trimmer = tokio::spawn(trim::run(pool.clone(), config.clone()));
Ok(Bus {
inner: Arc::new(Inner {
pool,
config,
shared,
start,
listener,
trimmer,
}),
})
}
fn schema(&self) -> &str {
&self.inner.config.schema
}
pub async fn publish(
&self,
conn: &mut PgConnection,
channel: &str,
data: &Value,
audience: Option<&[String]>,
) -> Result<Position, Error> {
let schema = self.schema();
let (xid, id): (String, i64) = sqlx::query_as(&format!(
"INSERT INTO {schema}.messages (channel, data, audience) VALUES ($1, $2, $3) \
RETURNING xid::text, id"
))
.bind(channel)
.bind(data)
.bind(audience)
.fetch_one(&mut *conn)
.await?;
let xid = xid.parse().map_err(|_| Error::InvalidPosition(xid))?;
sqlx::query("SELECT pg_notify($1, '')")
.bind(schema)
.execute(&mut *conn)
.await?;
Ok(Position { xid, id })
}
pub async fn health(&self) -> Result<Health, Error> {
health::health_on(&self.inner.pool, self.schema()).await
}
pub async fn now(&self) -> Result<Position, Error> {
horizon_on(&self.inner.pool).await
}
pub async fn backlog(
&self,
from: Position,
filter: &Filter,
limit: i64,
) -> Result<Vec<Message>, Error> {
backlog_on(&self.inner.pool, self.schema(), from, Some(filter), limit).await
}
pub async fn trimmed_after(&self, from: Position) -> Result<bool, Error> {
let schema = self.schema();
let trimmed: Option<(String, i64)> = sqlx::query_as(&format!(
"SELECT trimmed_xid::text, trimmed_id FROM {schema}.state"
))
.fetch_optional(&self.inner.pool)
.await?;
let Some((xid, id)) = trimmed else {
return Ok(false);
};
let xid: u64 = xid.parse().map_err(|_| Error::InvalidPosition(xid))?;
Ok(from < Position { xid, id })
}
pub async fn trim(&self, max_age: Duration, keep_per_channel: i64) -> Result<u64, Error> {
let config = &self.inner.config;
let deleted = trim::trim_on(
&self.inner.pool,
&config.schema,
config.trim_grace,
max_age,
keep_per_channel,
true,
)
.await?;
Ok(deleted.unwrap_or(0))
}
pub fn subscribe(&self, from: Position, filter: Filter) -> Subscription {
Subscription::new(self.clone(), from, filter)
}
pub fn stats(&self) -> Stats {
let shared = &self.inner.shared;
shared
.counters
.snapshot(shared.feed.receiver_count(), self.inner.start)
}
pub(crate) fn feed(&self) -> broadcast::Receiver<Arc<Message>> {
self.inner.shared.feed.subscribe()
}
pub(crate) fn counters(&self) -> &stats::Counters {
&self.inner.shared.counters
}
pub(crate) fn recent_after(
&self,
after: Position,
filter: &Filter,
limit: usize,
) -> Option<Vec<Message>> {
self.inner
.shared
.recent
.lock()
.unwrap_or_else(|e| e.into_inner())
.after(after, filter, limit)
}
}
pub(crate) async fn horizon_on(pool: &PgPool) -> Result<Position, Error> {
let xmin: String = sqlx::query_scalar("SELECT pg_snapshot_xmin(pg_current_snapshot())::text")
.fetch_one(pool)
.await?;
let xid = xmin.parse().map_err(|_| Error::InvalidPosition(xmin))?;
Ok(Position { xid, id: 0 })
}
pub(crate) async fn backlog_on(
pool: &PgPool,
schema: &str,
from: Position,
filter: Option<&Filter>,
limit: i64,
) -> Result<Vec<Message>, Error> {
let filter_sql = if filter.is_some() {
"AND channel = ANY($4) AND (audience IS NULL OR audience && $5)"
} else {
""
};
let sql = format!(
"SELECT {COLUMNS} FROM {schema}.messages \
WHERE (xid, id) > ($1::text::xid8, $2) \
AND xid < pg_snapshot_xmin(pg_current_snapshot()) {filter_sql} \
ORDER BY messages.xid, messages.id LIMIT $3"
);
let mut q = sqlx::query_as::<_, Row>(&sql)
.bind(from.xid.to_string())
.bind(from.id)
.bind(limit);
if let Some(f) = filter {
q = q.bind(&f.channels).bind(&f.tags);
}
q.fetch_all(pool).await?.into_iter().map(message).collect()
}
pub(crate) async fn pending_after(
pool: &PgPool,
schema: &str,
from: Position,
) -> Result<bool, Error> {
Ok(sqlx::query_scalar(&format!(
"SELECT EXISTS (SELECT 1 FROM {schema}.messages WHERE (xid, id) > ($1::text::xid8, $2))"
))
.bind(from.xid.to_string())
.bind(from.id)
.fetch_one(pool)
.await?)
}