use super::{MAX_PAYLOAD_BYTES, MAX_TOPIC_BYTES, Pubsub, Subscription};
use crate::error::{ForgeError, Result};
use crate::obs;
use crate::util::key_hash;
use async_trait::async_trait;
use bytes::Bytes;
use futures_util::StreamExt;
use sqlx::PgPool;
use sqlx::postgres::PgListener;
use std::collections::HashMap;
use std::time::Duration;
use tokio::sync::{OnceCell, broadcast, mpsc, oneshot};
use tracing::field::Empty;
const BROADCAST_CAPACITY: usize = 1024;
enum Cmd {
Register {
channel: String,
ack: oneshot::Sender<Result<broadcast::Receiver<Bytes>>>,
},
Unregister {
channel: String,
},
Shutdown {
ack: oneshot::Sender<()>,
},
}
struct Broker {
cmd_tx: mpsc::UnboundedSender<Cmd>,
}
struct SubGuard {
channel: String,
cmd_tx: mpsc::UnboundedSender<Cmd>,
}
impl Drop for SubGuard {
fn drop(&mut self) {
let _ = self.cmd_tx.send(Cmd::Unregister {
channel: std::mem::take(&mut self.channel),
});
}
}
pub(crate) struct PgPubsub {
pool: PgPool,
url: String,
namespace: String,
broker: OnceCell<Broker>,
}
impl PgPubsub {
pub(crate) fn new(pool: PgPool, url: String, namespace: String) -> Self {
Self {
pool,
url,
namespace,
broker: OnceCell::new(),
}
}
fn channel(&self, topic: &str) -> String {
super::hashed_channel_for(&crate::util::namespaced(&self.namespace, topic))
}
fn check_topic(topic: &str) -> Result<()> {
if topic.is_empty() {
return Err(ForgeError::invalid("pubsub topic must not be empty"));
}
if topic.len() > MAX_TOPIC_BYTES {
return Err(ForgeError::invalid(format!(
"pubsub topic exceeds {MAX_TOPIC_BYTES} bytes"
)));
}
Ok(())
}
async fn broker(&self) -> Result<&Broker> {
self.broker
.get_or_try_init(|| async {
let listener = PgListener::connect(&self.url)
.await
.map_err(ForgeError::from_sqlx)?;
let (cmd_tx, cmd_rx) = mpsc::unbounded_channel();
tokio::spawn(broker_task(listener, cmd_rx, self.url.clone()));
Ok::<Broker, ForgeError>(Broker { cmd_tx })
})
.await
}
pub(crate) async fn shutdown(&self) -> Result<()> {
let Some(broker) = self.broker.get() else {
return Ok(());
};
let (ack_tx, ack_rx) = oneshot::channel();
if broker.cmd_tx.send(Cmd::Shutdown { ack: ack_tx }).is_err() {
return Ok(());
}
let _ = ack_rx.await;
Ok(())
}
}
async fn broker_task(
mut listener: PgListener,
mut cmd_rx: mpsc::UnboundedReceiver<Cmd>,
url: String,
) {
let mut channels: HashMap<String, broadcast::Sender<Bytes>> = HashMap::new();
loop {
tokio::select! {
biased;
cmd = cmd_rx.recv() => {
match cmd {
None => break,
Some(Cmd::Register { channel, ack }) => {
let tx = match channels.get(&channel) {
Some(tx) => tx.clone(),
None => {
if let Err(e) = listener.listen(&channel).await {
let _ = ack.send(Err(ForgeError::from_sqlx(e)));
continue;
}
let (tx, _) = broadcast::channel(BROADCAST_CAPACITY);
channels.insert(channel, tx.clone());
tx
}
};
let _ = ack.send(Ok(tx.subscribe()));
}
Some(Cmd::Unregister { channel }) => {
if let Some(tx) = channels.get(&channel)
&& tx.receiver_count() == 0
{
let _ = listener.unlisten(&channel).await;
channels.remove(&channel);
}
}
Some(Cmd::Shutdown { ack }) => {
let _ = ack.send(());
break;
}
}
}
res = listener.recv() => {
match res {
Ok(note) => {
if let Some(tx) = channels.get(note.channel()) {
let _ = tx.send(Bytes::copy_from_slice(note.payload().as_bytes()));
if tx.receiver_count() == 0 {
let ch = note.channel().to_string();
let _ = listener.unlisten(&ch).await;
channels.remove(&ch);
}
}
}
Err(e) => {
tracing::warn!(error = %e, "pubsub listener dropped; reconnecting");
listener = reconnect(&url, &channels).await;
}
}
}
}
}
}
async fn reconnect(url: &str, channels: &HashMap<String, broadcast::Sender<Bytes>>) -> PgListener {
loop {
match PgListener::connect(url).await {
Ok(mut listener) => {
let mut all_ok = true;
for ch in channels.keys() {
if let Err(e) = listener.listen(ch).await {
tracing::warn!(error = %e, channel = %ch, "pubsub re-listen failed");
all_ok = false;
break;
}
}
if all_ok {
return listener;
}
}
Err(e) => tracing::warn!(error = %e, "pubsub reconnect failed; retrying"),
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
}
#[async_trait]
impl Pubsub for PgPubsub {
fn channel_for(&self, topic: &str) -> Result<String> {
Self::check_topic(topic)?;
Ok(self.channel(topic))
}
async fn publish(&self, topic: &str, payload: Bytes) -> Result<()> {
let span = tracing::info_span!(
"forge.pubsub.publish",
pubsub.topic_hash = %key_hash(topic),
pubsub.payload_bytes = payload.len(),
outcome = Empty,
error.variant = Empty,
);
obs::instrument("pubsub", "publish", span, async move {
Self::check_topic(topic)?;
if payload.len() > MAX_PAYLOAD_BYTES {
return Err(ForgeError::limit(format!(
"pubsub payload is {} bytes; max is {MAX_PAYLOAD_BYTES}",
payload.len()
)));
}
let text = std::str::from_utf8(&payload).map_err(|_| {
ForgeError::invalid("pubsub payload must be valid UTF-8 (NOTIFY payloads are text)")
})?;
let channel = self.channel(topic);
sqlx::query!("SELECT pg_notify($1, $2)", channel, text)
.execute(&self.pool)
.await?;
Ok(())
})
.await
}
async fn subscribe(&self, topic: &str) -> Result<Subscription> {
Self::check_topic(topic)?;
let channel = self.channel(topic);
let broker = self.broker().await?;
let (ack_tx, ack_rx) = oneshot::channel();
broker
.cmd_tx
.send(Cmd::Register {
channel: channel.clone(),
ack: ack_tx,
})
.map_err(|_| ForgeError::unavailable("pubsub broker has stopped"))?;
let rx = ack_rx
.await
.map_err(|_| ForgeError::unavailable("pubsub broker dropped before acknowledging"))??;
let guard = SubGuard {
channel,
cmd_tx: broker.cmd_tx.clone(),
};
let stream = futures_util::stream::unfold((rx, guard), |(mut rx, guard)| async move {
loop {
match rx.recv().await {
Ok(payload) => return Some((Ok(payload), (rx, guard))),
Err(broadcast::error::RecvError::Lagged(n)) => {
tracing::warn!(skipped = n, "pubsub subscriber lagged; skipped messages");
continue;
}
Err(broadcast::error::RecvError::Closed) => return None,
}
}
});
Ok(stream.boxed())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn channel_is_valid_fixed_length_identifier() {
let c = super::super::hashed_channel_for("chat:550e8400-e29b-41d4-a716-446655440000");
assert_eq!(c.len(), 38); assert!(c.starts_with("forge_"));
assert!(c.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_'));
assert!(c.len() <= 63, "must fit a Postgres channel identifier");
}
#[test]
fn distinct_topics_get_distinct_channels() {
assert_ne!(
super::super::hashed_channel_for("chat:1"),
super::super::hashed_channel_for("chat:2")
);
assert_eq!(
super::super::hashed_channel_for("presence"),
super::super::hashed_channel_for("presence")
);
}
#[test]
fn empty_topic_is_invalid() {
assert!(matches!(
PgPubsub::check_topic(""),
Err(ForgeError::Invalid(_))
));
}
}