use std::time::Duration;
use futures_util::{SinkExt, StreamExt};
use tokio::sync::mpsc;
use tokio::time::Instant;
use tokio_tungstenite::connect_async;
use tokio_tungstenite::tungstenite::Message;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
use url::form_urlencoded;
use crate::error::Result;
use crate::ingest::backoff::BackoffManager;
use crate::ingest::cursor::CursorTracker;
use crate::ingest::events::{parse_jetstream_frame, JetstreamCommit, JetstreamEvent};
pub const DEFAULT_JETSTREAM_ENDPOINT: &str = "wss://jetstream2.us-east.bsky.network/subscribe";
pub const INITIAL_BACKOFF: Duration = Duration::from_millis(500);
pub const MAX_BACKOFF: Duration = Duration::from_secs(30);
pub const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(30);
pub const DEFAULT_PING_INTERVAL: Duration = Duration::from_secs(15);
pub const MAX_JETSTREAM_REPLAY_WINDOW_US: u64 = 72 * 3600 * 1_000_000;
pub const CURSOR_REWIND_OVERLAP_US: u64 = 5_000_000;
#[must_use]
pub fn resolve_resume_cursor(persisted_cursor_us: u64, now_us: u64) -> Option<u64> {
if persisted_cursor_us == 0 {
return None;
}
let rewound = persisted_cursor_us.saturating_sub(CURSOR_REWIND_OVERLAP_US);
let min_allowed = now_us.saturating_sub(MAX_JETSTREAM_REPLAY_WINDOW_US);
Some(rewound.max(min_allowed)).filter(|c| *c > 0)
}
#[must_use]
pub fn build_subscription_url(
base_url: &str,
wanted_collections: &[String],
cursor: Option<u64>,
) -> String {
build_subscription_url_full(base_url, wanted_collections, &[], cursor)
}
#[must_use]
pub fn build_subscription_url_full(
base_url: &str,
wanted_collections: &[String],
wanted_dids: &[String],
cursor: Option<u64>,
) -> String {
let mut url = base_url.trim().to_string();
let lower = url.to_ascii_lowercase();
if !lower.starts_with("ws://") && !lower.starts_with("wss://") {
url = format!("wss://{url}");
}
if let Some(scheme_idx) = url.find("://") {
let after_scheme = &url[scheme_idx + 3..];
if !after_scheme.contains('/') && !after_scheme.contains('?') {
url.push('/');
}
} else if !url.contains('/') && !url.contains('?') {
url.push('/');
}
let has_query = url.contains('?');
let mut query_params: Vec<String> = Vec::new();
if !url.contains("wantedCollections=") {
for col in wanted_collections {
let trimmed = col.trim();
if !trimmed.is_empty() {
let encoded: String = form_urlencoded::byte_serialize(trimmed.as_bytes()).collect();
query_params.push(format!("wantedCollections={encoded}"));
}
}
}
if !url.contains("wantedDids=") {
for did in wanted_dids {
let trimmed = did.trim();
if !trimmed.is_empty() {
let encoded: String = form_urlencoded::byte_serialize(trimmed.as_bytes()).collect();
let safe_did = encoded.replace("%3A", ":");
query_params.push(format!("wantedDids={safe_did}"));
}
}
}
if let Some(c) = cursor {
if c > 0 && !url.contains("cursor=") {
query_params.push(format!("cursor={c}"));
}
}
if !query_params.is_empty() {
let separator = if has_query { '&' } else { '?' };
if !url.ends_with('?') && !url.ends_with('&') {
url.push(separator);
}
url.push_str(&query_params.join("&"));
}
url
}
#[derive(Debug, Clone)]
pub struct StreamConfig {
pub endpoint: String,
pub collections: Vec<String>,
pub wanted_dids: Vec<String>,
pub cursor: Option<u64>,
pub read_timeout: Duration,
pub ping_interval: Option<Duration>,
}
impl Default for StreamConfig {
fn default() -> Self {
Self {
endpoint: DEFAULT_JETSTREAM_ENDPOINT.to_string(),
collections: Vec::new(),
wanted_dids: Vec::new(),
cursor: None,
read_timeout: DEFAULT_READ_TIMEOUT,
ping_interval: Some(DEFAULT_PING_INTERVAL),
}
}
}
impl StreamConfig {
#[must_use]
pub fn new(endpoint: impl Into<String>) -> Self {
Self {
endpoint: endpoint.into(),
..Self::default()
}
}
#[must_use]
pub fn with_collections<I, S>(mut self, collections: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.collections = collections.into_iter().map(Into::into).collect();
self
}
#[must_use]
pub fn with_collection(mut self, collection: impl Into<String>) -> Self {
self.collections.push(collection.into());
self
}
#[must_use]
pub fn with_did(mut self, did: impl Into<String>) -> Self {
self.wanted_dids.push(did.into());
self
}
#[must_use]
pub fn with_cursor(mut self, cursor: u64) -> Self {
self.cursor = if cursor > 0 { Some(cursor) } else { None };
self
}
#[must_use]
pub fn with_read_timeout(mut self, timeout: Duration) -> Self {
self.read_timeout = timeout.max(Duration::from_millis(50));
self
}
#[must_use]
pub const fn with_ping_interval(mut self, interval: Option<Duration>) -> Self {
self.ping_interval = interval;
self
}
#[must_use]
pub fn build_url(&self) -> String {
build_subscription_url_full(
&self.endpoint,
&self.collections,
&self.wanted_dids,
self.cursor.filter(|c| *c > 0),
)
}
}
pub async fn run_jetstream_streamer(
config: StreamConfig,
tx: mpsc::Sender<JetstreamCommit>,
cancel: CancellationToken,
) -> Result<()> {
let mut backoff = BackoffManager::new(INITIAL_BACKOFF, MAX_BACKOFF);
let cursor_tracker = CursorTracker::from_option(config.cursor);
let mut has_connected_once = false;
while !cancel.is_cancelled() {
let mut active_config = config.clone();
if let Some(cursor) = cursor_tracker.get_opt() {
active_config.cursor = if has_connected_once {
Some(cursor.saturating_sub(CURSOR_REWIND_OVERLAP_US))
} else {
Some(cursor)
};
}
has_connected_once = true;
let url = active_config.build_url();
info!(endpoint = %url, cursor = ?active_config.cursor, "Connecting to Jetstream firehose");
match connect_async(&url).await {
Ok((ws_stream, response)) => {
debug!(status = %response.status(), "WebSocket connection established");
let (mut write_half, mut read_half) = ws_stream.split();
let ping_dur = active_config
.ping_interval
.unwrap_or(Duration::from_secs(3600));
let mut ping_timer = tokio::time::interval_at(Instant::now() + ping_dur, ping_dur);
loop {
tokio::select! {
biased;
() = cancel.cancelled() => {
info!("Streamer received cancellation signal; closing connection");
let _ = write_half.send(Message::Close(None)).await;
return Ok(());
}
_ = ping_timer.tick(), if active_config.ping_interval.is_some() => {
if let Err(e) = write_half.send(Message::Ping(Vec::new().into())).await {
warn!(error = %e, "Failed to send WebSocket keepalive ping; reconnecting");
break;
}
}
msg_res = tokio::time::timeout(active_config.read_timeout, read_half.next()) => {
match msg_res {
Ok(Some(Ok(Message::Text(text)))) => {
if let Some(evt) = parse_jetstream_frame(&text) {
cursor_tracker.update(evt.time_us());
backoff.reset();
if let JetstreamEvent::Commit(commit) = evt {
if tx.send(commit).await.is_err() {
debug!("Commit receiver dropped; stopping streamer");
return Ok(());
}
}
}
}
Ok(Some(Ok(Message::Binary(bytes)))) => {
if let Ok(text) = std::str::from_utf8(&bytes) {
if let Some(evt) = parse_jetstream_frame(text) {
cursor_tracker.update(evt.time_us());
backoff.reset();
if let JetstreamEvent::Commit(commit) = evt {
if tx.send(commit).await.is_err() {
debug!("Commit receiver dropped; stopping streamer");
return Ok(());
}
}
}
}
}
Ok(Some(Ok(Message::Ping(payload)))) => {
let _ = write_half.send(Message::Pong(payload)).await;
}
Ok(Some(Ok(Message::Pong(_)))) => {}
Ok(Some(Ok(Message::Close(frame)))) => {
warn!(frame = ?frame, "Jetstream server sent close frame; reconnecting");
break;
}
Ok(Some(Ok(Message::Frame(_)))) => {}
Ok(Some(Err(e))) => {
warn!(error = %e, "WebSocket read error; reconnecting");
break;
}
Ok(None) => {
warn!("Jetstream stream ended unexpectedly; reconnecting");
break;
}
Err(_elapsed) => {
warn!(
timeout = ?active_config.read_timeout,
cursor = ?cursor_tracker.get_opt(),
"Jetstream WebSocket read timed out (silent stall); closing and reconnecting"
);
let _ = tokio::time::timeout(
Duration::from_secs(1),
write_half.send(Message::Close(None)),
)
.await;
break;
}
}
}
}
}
}
Err(e) => {
error!(error = %e, "Failed to connect to Jetstream");
}
}
let delay = backoff.next_backoff();
tokio::select! {
biased;
() = cancel.cancelled() => {
return Ok(());
}
() = tokio::time::sleep(delay) => {}
}
}
Ok(())
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic, missing_docs)]
mod tests {
use super::*;
#[test]
fn test_resolve_resume_cursor() {
let now_us = 1_800_000_000_000_000_u64;
assert_eq!(resolve_resume_cursor(0, now_us), None);
let recent = now_us - 600_000_000;
assert_eq!(
resolve_resume_cursor(recent, now_us),
Some(recent - CURSOR_REWIND_OVERLAP_US)
);
let stale = now_us - (10 * 24 * 3600 * 1_000_000);
assert_eq!(
resolve_resume_cursor(stale, now_us),
Some(now_us - MAX_JETSTREAM_REPLAY_WINDOW_US)
);
}
#[test]
fn test_stream_config_url_building() {
let cfg = StreamConfig::new("wss://example.test/subscribe")
.with_collection("app.bsky.feed.post")
.with_cursor(1_700_000_000_000_000);
let url = cfg.build_url();
assert!(url.contains("wantedCollections=app.bsky.feed.post"));
assert!(url.contains("cursor=1700000000000000"));
}
}