use std::time::Instant;
use futures_util::Stream;
use sqlx::PgPool;
use tokio::sync::broadcast;
use crate::observability::{ConnectionGuard, Metrics};
use crate::persistence::{
files::events::{
EventCursor, EventEntity, EventRepository, EventType, EventVisibility, PathFilter,
},
sql::{SqlDb, UnifiedExecutor},
};
use crate::shared::webdav::EntryPath;
pub const MAX_EVENT_STREAM_USERS: usize = 50;
pub(crate) const PG_NOTIFY_CHANNEL: &str = "events";
#[derive(Clone, Debug)]
pub struct EventsService {
event_tx: broadcast::Sender<EventEntity>,
channel_capacity: usize,
}
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum Mode {
Forward,
ForwardLive,
Reverse,
}
impl Mode {
fn reverse(self) -> bool {
matches!(self, Self::Reverse)
}
fn live(self) -> bool {
matches!(self, Self::ForwardLive)
}
}
pub(crate) struct AllEventsFilter {
pub start_cursor: Option<EventCursor>,
pub user_ids: Option<Vec<i32>>,
pub paths: Vec<PathFilter>,
pub mode: Mode,
pub limit: Option<u16>,
}
impl EventsService {
pub fn new(channel_capacity: usize) -> Self {
let (event_tx, _rx) = broadcast::channel(channel_capacity);
Self {
event_tx,
channel_capacity,
}
}
pub fn subscribe(&self) -> broadcast::Receiver<EventEntity> {
self.event_tx.subscribe()
}
pub fn channel_capacity(&self) -> usize {
self.channel_capacity
}
pub async fn create_event<'a>(
&self,
user_id: i32,
event_type: EventType,
path: &EntryPath,
executor: &mut UnifiedExecutor<'a>,
) -> Result<EventEntity, sqlx::Error> {
EventRepository::create(user_id, event_type, path, executor).await
}
pub(crate) fn broadcast_event(&self, event: EventEntity) {
match self.event_tx.send(event) {
Ok(_) => {} Err(broadcast::error::SendError(_)) => {
}
}
}
pub async fn notify_event(pool: &PgPool) {
if let Err(e) = sqlx::query("SELECT pg_notify($1, '')")
.bind(PG_NOTIFY_CHANNEL)
.execute(pool)
.await
{
tracing::error!("Failed to send NOTIFY: {}", e);
}
}
pub async fn parse_cursor<'a>(
&self,
cursor: &str,
executor: &mut UnifiedExecutor<'a>,
) -> Result<EventCursor, sqlx::Error> {
EventRepository::parse_cursor(cursor, executor).await
}
pub async fn get_public_by_cursor<'a>(
&self,
cursor: Option<EventCursor>,
limit: Option<u16>,
executor: &mut UnifiedExecutor<'a>,
) -> Result<Vec<EventEntity>, sqlx::Error> {
EventRepository::get_by_cursor(cursor, limit, EventVisibility::Public, executor).await
}
pub async fn get_all_events<'a>(
&self,
cursor: Option<EventCursor>,
limit: Option<u16>,
reverse: bool,
path_filters: &[PathFilter],
user_ids: Option<&[i32]>,
executor: &mut UnifiedExecutor<'a>,
) -> Result<Vec<EventEntity>, sqlx::Error> {
EventRepository::get_all_filtered_by_cursor(
cursor,
limit,
reverse,
path_filters,
user_ids,
executor,
)
.await
}
pub async fn get_by_user_cursors<'a>(
&self,
user_cursors: Vec<(i32, Option<EventCursor>)>,
reverse: bool,
allowed_paths: &[PathFilter],
executor: &mut UnifiedExecutor<'a>,
) -> Result<Vec<EventEntity>, sqlx::Error> {
EventRepository::get_by_user_cursors(user_cursors, reverse, allowed_paths, executor).await
}
pub(crate) fn all_events_stream(
&self,
sql_db: SqlDb,
metrics: Metrics,
filter: AllEventsFilter,
) -> impl Stream<Item = EventEntity> {
let service = self.clone();
let mut rx = self.subscribe();
let half_capacity = self.channel_capacity() / 2;
async_stream::stream! {
let _guard = ConnectionGuard::new(metrics.clone());
let mut last_cursor = filter.start_cursor;
let mut total_sent: usize = 0;
let reverse = filter.mode.reverse();
let live = filter.mode.live();
loop {
while rx.try_recv().is_ok() {}
let batch_limit = match filter.limit {
Some(max) => match (max as usize).saturating_sub(total_sent) {
0 => return,
remaining => Some(remaining as u16),
},
None => None,
};
let query_start = Instant::now();
let events = match service
.get_all_events(
last_cursor,
batch_limit,
reverse,
&filter.paths,
filter.user_ids.as_deref(),
&mut sql_db.pool().into(),
)
.await
{
Ok(events) => events,
Err(e) => {
tracing::error!("Database error while streaming admin events: {}", e);
break;
}
};
metrics.record_event_stream_db_query(query_start.elapsed().as_millis());
let caught_up = events.is_empty();
for event in events {
last_cursor = Some(event.cursor());
yield event;
total_sent += 1;
}
if caught_up {
if !live {
return;
}
break;
}
}
if live {
loop {
match rx.recv().await {
Ok(event) => {
if rx.len() >= half_capacity {
metrics.record_broadcast_half_full();
}
if !accept_live_event(&event, last_cursor, &filter) {
continue;
}
if filter.limit.is_some_and(|max| total_sent >= max as usize) {
return;
}
last_cursor = Some(event.cursor());
yield event;
total_sent += 1;
}
Err(broadcast::error::RecvError::Lagged(skipped)) => {
metrics.record_broadcast_lagged();
tracing::warn!(
"Slow admin client detected: broadcast channel lagged by {} events. Closing connection.",
skipped
);
return;
}
Err(_) => break, }
}
}
}
}
}
fn accept_live_event(
event: &EventEntity,
last_cursor: Option<EventCursor>,
filter: &AllEventsFilter,
) -> bool {
if let Some(cursor) = last_cursor {
if event.cursor() <= cursor {
return false;
}
}
if let Some(ids) = filter.user_ids.as_deref() {
if !ids.contains(&event.user_id) {
return false;
}
}
if !filter.paths.is_empty() {
let path = event.path.path();
if !filter.paths.iter().any(|f| f.matches(path.as_str())) {
return false;
}
}
true
}
#[cfg(test)]
mod tests {
use super::*;
use crate::persistence::sql::{user::UserRepository, SqlDb};
use crate::shared::webdav::StoragePath;
use pubky_common::crypto::Keypair;
#[tokio::test]
#[pubky_test_utils::test]
async fn test_events_service_create_and_broadcast() {
let db = SqlDb::test().await;
let events_service = EventsService::new(100);
let user_pubkey = Keypair::random().public_key();
let user = UserRepository::create(&user_pubkey, &mut db.pool().into())
.await
.unwrap();
let path = EntryPath::new(user_pubkey.clone(), StoragePath::new("/test.txt").unwrap());
let mut rx = events_service.subscribe();
let mut tx = db.pool().begin().await.unwrap();
let event = events_service
.create_event(
user.id,
EventType::Put {
content_hash: pubky_common::crypto::Hash::from_bytes([0; 32]),
},
&path,
&mut (&mut tx).into(),
)
.await
.unwrap();
tx.commit().await.unwrap();
events_service.broadcast_event(event.clone());
let received = rx.recv().await.unwrap();
assert_eq!(received.id, event.id);
assert_eq!(received.user_id, user.id);
assert!(matches!(received.event_type, EventType::Put { .. }));
}
#[tokio::test]
#[pubky_test_utils::test]
async fn test_events_service_get_public_by_cursor() {
let db = SqlDb::test().await;
let events_service = EventsService::new(100);
let user_pubkey = Keypair::random().public_key();
let user = UserRepository::create(&user_pubkey, &mut db.pool().into())
.await
.unwrap();
let paths = ["/pub/a", "/priv/x", "/pub/b", "/priv/y", "/pub/c"];
for p in paths {
let path = EntryPath::new(user_pubkey.clone(), StoragePath::new(p).unwrap());
events_service
.create_event(
user.id,
EventType::Put {
content_hash: pubky_common::crypto::Hash::from_bytes([0; 32]),
},
&path,
&mut db.pool().into(),
)
.await
.unwrap();
}
let events = events_service
.get_public_by_cursor(None, None, &mut db.pool().into())
.await
.unwrap();
let returned: Vec<&str> = events.iter().map(|e| e.path.path().as_str()).collect();
assert_eq!(returned, vec!["/pub/a", "/pub/b", "/pub/c"]);
let page = events_service
.get_public_by_cursor(None, Some(2), &mut db.pool().into())
.await
.unwrap();
assert_eq!(page.len(), 2);
assert_eq!(page[0].id, 1); assert_eq!(page[1].id, 3);
let next_cursor = page.last().unwrap().cursor();
let page = events_service
.get_public_by_cursor(Some(next_cursor), Some(2), &mut db.pool().into())
.await
.unwrap();
assert_eq!(page.len(), 1);
assert_eq!(page[0].id, 5); }
#[tokio::test]
#[pubky_test_utils::test]
async fn test_events_service_get_all_events_includes_private() {
let db = SqlDb::test().await;
let events_service = EventsService::new(100);
let user_pubkey = Keypair::random().public_key();
let user = UserRepository::create(&user_pubkey, &mut db.pool().into())
.await
.unwrap();
let paths = ["/pub/a", "/priv/x", "/pub/b", "/priv/y", "/pub/c"];
for p in paths {
let path = EntryPath::new(user_pubkey.clone(), StoragePath::new(p).unwrap());
events_service
.create_event(
user.id,
EventType::Put {
content_hash: pubky_common::crypto::Hash::from_bytes([0; 32]),
},
&path,
&mut db.pool().into(),
)
.await
.unwrap();
}
let events = events_service
.get_all_events(None, None, false, &[], None, &mut db.pool().into())
.await
.unwrap();
let returned: Vec<&str> = events.iter().map(|e| e.path.path().as_str()).collect();
assert_eq!(
returned,
vec!["/pub/a", "/priv/x", "/pub/b", "/priv/y", "/pub/c"]
);
let events = events_service
.get_all_events(None, None, true, &[], None, &mut db.pool().into())
.await
.unwrap();
let returned: Vec<&str> = events.iter().map(|e| e.path.path().as_str()).collect();
assert_eq!(
returned,
vec!["/pub/c", "/priv/y", "/pub/b", "/priv/x", "/pub/a"]
);
let path_filters = [PathFilter::from(StoragePath::new("/priv/").unwrap())];
let events = events_service
.get_all_events(
None,
None,
false,
&path_filters,
None,
&mut db.pool().into(),
)
.await
.unwrap();
let returned: Vec<&str> = events.iter().map(|e| e.path.path().as_str()).collect();
assert_eq!(returned, vec!["/priv/x", "/priv/y"]);
let path_filters = [
PathFilter::from(StoragePath::new("/pub/a").unwrap()),
PathFilter::from(StoragePath::new("/priv/").unwrap()),
];
let events = events_service
.get_all_events(
None,
None,
false,
&path_filters,
None,
&mut db.pool().into(),
)
.await
.unwrap();
let returned: Vec<&str> = events.iter().map(|e| e.path.path().as_str()).collect();
assert_eq!(returned, vec!["/pub/a", "/priv/x", "/priv/y"]);
let events = events_service
.get_all_events(
None,
None,
false,
&[],
Some(&[user.id]),
&mut db.pool().into(),
)
.await
.unwrap();
assert_eq!(events.len(), 5);
}
}