use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Duration;
use futures_util::{SinkExt, StreamExt};
use tokio::time::Instant;
use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
use tokio_tungstenite::tungstenite::protocol::{CloseFrame, Message};
use tokio_util::sync::CancellationToken;
use url::form_urlencoded;
use crate::error::{Result, SkybaseError};
use crate::index::{RecordStore, StoreOperation};
use crate::ingest::backoff::BackoffManager;
use crate::ingest::cursor::CursorTracker;
use crate::ingest::events::{
parse_jetstream_frame, CommitOperation, JetstreamCommit, JetstreamEvent,
};
#[derive(Debug, Clone)]
pub struct IngesterConfig {
pub endpoint: String,
pub wanted_collections: Vec<String>,
pub wanted_dids: Vec<String>,
pub initial_cursor: Option<u64>,
pub channel_capacity: usize,
pub inactivity_timeout: Duration,
pub ping_interval: Option<Duration>,
pub initial_backoff: Duration,
pub max_backoff: Duration,
pub batch_size: usize,
}
impl Default for IngesterConfig {
fn default() -> Self {
Self {
endpoint: "wss://jetstream2.us-east.bsky.network/subscribe".to_string(),
wanted_collections: Vec::new(),
wanted_dids: Vec::new(),
initial_cursor: None,
channel_capacity: 1024,
inactivity_timeout: Duration::from_secs(30),
ping_interval: Some(Duration::from_secs(15)),
initial_backoff: Duration::from_millis(500),
max_backoff: Duration::from_secs(30),
batch_size: 1,
}
}
}
impl IngesterConfig {
pub fn new(endpoint: impl Into<String>) -> Self {
Self {
endpoint: endpoint.into(),
..Self::default()
}
}
#[must_use]
pub fn with_collection(mut self, collection: impl Into<String>) -> Self {
self.wanted_collections.push(collection.into());
self
}
#[must_use]
pub fn with_collections<I, S>(mut self, collections: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.wanted_collections = collections.into_iter().map(Into::into).collect();
self
}
#[must_use]
pub fn with_did(mut self, did: impl Into<String>) -> Self {
self.wanted_dids.push(did.into());
self
}
#[must_use]
pub fn with_dids<I, S>(mut self, dids: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.wanted_dids = dids.into_iter().map(Into::into).collect();
self
}
#[must_use]
pub fn with_cursor(mut self, cursor: u64) -> Self {
self.initial_cursor = if cursor > 0 { Some(cursor) } else { None };
self
}
#[must_use]
pub fn with_channel_capacity(mut self, capacity: usize) -> Self {
self.channel_capacity = capacity.max(16);
self
}
#[must_use]
pub fn with_inactivity_timeout(mut self, timeout: Duration) -> Self {
self.inactivity_timeout = timeout;
self
}
#[must_use]
pub fn with_ping_interval(mut self, interval: Option<Duration>) -> Self {
self.ping_interval = interval;
self
}
#[must_use]
pub fn with_backoff(mut self, initial: Duration, max: Duration) -> Self {
self.initial_backoff = initial;
self.max_backoff = max;
self
}
#[must_use]
pub fn with_batch_size(mut self, batch_size: usize) -> Self {
self.batch_size = batch_size.max(1);
self
}
}
#[derive(Debug, Default)]
pub struct ConsumerStats {
pub frames_received: AtomicU64,
pub bytes_received: AtomicU64,
pub records_upserted: AtomicU64,
pub records_deleted: AtomicU64,
pub sync_errors: AtomicU64,
pub reconnect_count: AtomicU64,
pub last_activity_timestamp: AtomicU64,
}
impl ConsumerStats {
#[must_use]
pub fn frames_received(&self) -> u64 {
self.frames_received.load(Ordering::Relaxed)
}
#[must_use]
pub fn bytes_received(&self) -> u64 {
self.bytes_received.load(Ordering::Relaxed)
}
#[must_use]
pub fn records_upserted(&self) -> u64 {
self.records_upserted.load(Ordering::Relaxed)
}
#[must_use]
pub fn records_deleted(&self) -> u64 {
self.records_deleted.load(Ordering::Relaxed)
}
#[must_use]
pub fn sync_errors(&self) -> u64 {
self.sync_errors.load(Ordering::Relaxed)
}
#[must_use]
pub fn reconnect_count(&self) -> u64 {
self.reconnect_count.load(Ordering::Relaxed)
}
#[must_use]
pub fn last_activity_timestamp(&self) -> u64 {
self.last_activity_timestamp.load(Ordering::Relaxed)
}
}
#[must_use]
pub fn build_subscription_url(
base_url: &str,
wanted_collections: &[String],
cursor: Option<u64>,
) -> String {
build_subscription_url_full(base_url, wanted_collections, &[], cursor)
}
#[must_use]
pub fn build_subscription_url_full(
base_url: &str,
wanted_collections: &[String],
wanted_dids: &[String],
cursor: Option<u64>,
) -> String {
let mut url = base_url.trim().to_string();
let lower = url.to_ascii_lowercase();
if !lower.starts_with("ws://") && !lower.starts_with("wss://") {
url = format!("wss://{url}");
}
if let Some(scheme_idx) = url.find("://") {
let after_scheme = &url[scheme_idx + 3..];
if !after_scheme.contains('/') && !after_scheme.contains('?') {
url.push('/');
}
} else if !url.contains('/') && !url.contains('?') {
url.push('/');
}
let has_query = url.contains('?');
let mut query_params: Vec<String> = Vec::new();
if !url.contains("wantedCollections=") {
for col in wanted_collections {
let trimmed = col.trim();
if !trimmed.is_empty() {
let encoded: String = form_urlencoded::byte_serialize(trimmed.as_bytes()).collect();
query_params.push(format!("wantedCollections={encoded}"));
}
}
}
if !url.contains("wantedDids=") {
for did in wanted_dids {
let trimmed = did.trim();
if !trimmed.is_empty() {
let encoded: String = form_urlencoded::byte_serialize(trimmed.as_bytes()).collect();
let safe_did = encoded.replace("%3A", ":");
query_params.push(format!("wantedDids={safe_did}"));
}
}
}
if let Some(c) = cursor {
if c > 0 && !url.contains("cursor=") {
query_params.push(format!("cursor={c}"));
}
}
if !query_params.is_empty() {
let separator = if has_query { '&' } else { '?' };
if !url.ends_with('?') && !url.ends_with('&') {
url.push(separator);
}
url.push_str(&query_params.join("&"));
}
url
}
#[derive(Clone)]
pub struct JetstreamConsumer {
config: IngesterConfig,
store: RecordStore,
cursor: Arc<CursorTracker>,
stats: Arc<ConsumerStats>,
}
impl JetstreamConsumer {
pub fn new(config: IngesterConfig, store: RecordStore) -> Self {
let initial_cursor = config.initial_cursor.unwrap_or(0);
Self {
config,
store,
cursor: Arc::new(CursorTracker::new(initial_cursor)),
stats: Arc::new(ConsumerStats::default()),
}
}
#[must_use]
pub fn with_cursor_tracker(mut self, cursor: Arc<CursorTracker>) -> Self {
self.cursor = cursor;
self
}
#[must_use]
pub fn config(&self) -> &IngesterConfig {
&self.config
}
#[must_use]
pub fn cursor(&self) -> u64 {
self.cursor.get()
}
#[must_use]
pub fn cursor_tracker(&self) -> Arc<CursorTracker> {
Arc::clone(&self.cursor)
}
#[must_use]
pub fn stats(&self) -> Arc<ConsumerStats> {
Arc::clone(&self.stats)
}
pub async fn run(&self, cancel: CancellationToken) -> Result<()> {
if cancel.is_cancelled() {
return Ok(());
}
if self.config.initial_cursor.is_none() && self.cursor.get() == 0 {
if let Ok(Some(persisted_cursor)) = self.store.get_cursor() {
self.cursor.force_rewind(persisted_cursor);
}
}
let capacity = self.config.channel_capacity.max(1);
let (tx, rx) = tokio::sync::mpsc::channel::<JetstreamCommit>(capacity);
let mut join_set = tokio::task::JoinSet::new();
let worker_store = self.store.clone();
let worker_stats = Arc::clone(&self.stats);
let worker_cursor = Arc::clone(&self.cursor);
let worker_token = cancel.child_token();
let batch_size = self.config.batch_size;
join_set.spawn(async move {
run_storage_sync_worker(
rx,
worker_store,
worker_stats,
batch_size,
worker_cursor,
worker_token,
)
.await;
Ok(())
});
let reader_config = self.config.clone();
let reader_cursor = Arc::clone(&self.cursor);
let reader_stats = Arc::clone(&self.stats);
let reader_token = cancel.child_token();
join_set.spawn(async move {
run_reader_reconnect_loop(reader_config, tx, reader_cursor, reader_stats, reader_token)
.await
});
while let Some(join_res) = join_set.join_next().await {
match join_res {
Ok(task_res) => task_res?,
Err(join_err) => {
if !join_err.is_cancelled() {
return Err(SkybaseError::Event(format!(
"Ingestion task failed: {join_err}"
)));
}
}
}
}
Ok(())
}
pub fn start(&self, cancel: CancellationToken) -> JetstreamConsumerHandle {
let this = self.clone();
let run_token = cancel.clone();
let join_handle = tokio::spawn(async move { this.run(run_token).await });
JetstreamConsumerHandle {
join_handle: Some(join_handle),
cancel,
}
}
}
pub struct JetstreamConsumerHandle {
join_handle: Option<tokio::task::JoinHandle<Result<()>>>,
cancel: CancellationToken,
}
impl Drop for JetstreamConsumerHandle {
fn drop(&mut self) {
self.cancel.cancel();
}
}
impl JetstreamConsumerHandle {
pub fn stop(&self) {
self.cancel.cancel();
}
pub fn abort(&self) {
self.cancel.cancel();
if let Some(ref handle) = self.join_handle {
handle.abort();
}
}
pub async fn join(mut self) -> Result<()> {
if let Some(handle) = self.join_handle.take() {
match handle.await {
Ok(res) => res,
Err(join_err) => {
if join_err.is_cancelled() {
Ok(())
} else {
Err(SkybaseError::Event(format!(
"Consumer task join error: {join_err}"
)))
}
}
}
} else {
Ok(())
}
}
}
async fn run_reader_reconnect_loop(
config: IngesterConfig,
tx: tokio::sync::mpsc::Sender<JetstreamCommit>,
cursor: Arc<CursorTracker>,
stats: Arc<ConsumerStats>,
cancel: CancellationToken,
) -> Result<()> {
let mut backoff = BackoffManager::new(config.initial_backoff, config.max_backoff);
while !cancel.is_cancelled() {
let current_cursor = cursor.get_opt();
let url = build_subscription_url_full(
&config.endpoint,
&config.wanted_collections,
&config.wanted_dids,
current_cursor,
);
tracing::debug!(endpoint = %url, "Attempting connection to Jetstream firehose");
let connect_res = tokio::select! {
() = cancel.cancelled() => break,
res = tokio_tungstenite::connect_async(&url) => res,
};
let (mut ws_stream, _) = match connect_res {
Ok(pair) => pair,
Err(err) => {
stats.reconnect_count.fetch_add(1, Ordering::Relaxed);
let delay = backoff.next_backoff();
tracing::warn!(
error = %err,
retry_in = ?delay,
"Failed to connect to Jetstream. Retrying."
);
tokio::select! {
() = cancel.cancelled() => break,
() = tokio::time::sleep(delay) => continue,
}
}
};
tracing::info!(endpoint = %url, "Connected to Jetstream firehose");
let mut last_activity = Instant::now();
let mut ping_interval = config
.ping_interval
.filter(|d| *d > Duration::ZERO)
.map(tokio::time::interval);
loop {
let timeout_duration = config
.inactivity_timeout
.min(Duration::from_secs(86400 * 365));
let deadline = last_activity
.checked_add(timeout_duration)
.unwrap_or_else(|| last_activity + Duration::from_secs(86400 * 365));
let sleep_watchdog = tokio::time::sleep_until(deadline);
tokio::select! {
() = cancel.cancelled() => {
let _ = ws_stream.close(Some(CloseFrame {
code: CloseCode::Normal,
reason: "Graceful shutdown".into(),
})).await;
return Ok(());
}
() = sleep_watchdog => {
stats.reconnect_count.fetch_add(1, Ordering::Relaxed);
tracing::warn!(
"Inactivity watchdog triggered ({timeout_duration:?} elapsed without frames). Reconnecting."
);
let _ = ws_stream.close(None).await;
break;
}
_ = async {
if let Some(ref mut interval) = ping_interval {
interval.tick().await
} else {
futures_util::future::pending().await
}
} => {
if ws_stream.send(Message::Ping(Vec::new().into())).await.is_err() {
stats.reconnect_count.fetch_add(1, Ordering::Relaxed);
tracing::warn!("Failed to send keepalive ping; reconnecting.");
break;
}
}
msg_opt = ws_stream.next() => {
match msg_opt {
Some(Ok(msg)) => {
last_activity = Instant::now();
let now_secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_secs());
stats.last_activity_timestamp.store(now_secs, Ordering::Relaxed);
match msg {
Message::Text(text) => {
stats.frames_received.fetch_add(1, Ordering::Relaxed);
stats.bytes_received.fetch_add(text.len() as u64, Ordering::Relaxed);
if let Some(event) = parse_jetstream_frame(&text) {
backoff.reset();
match event {
JetstreamEvent::Commit(commit) => {
let matches_collection = config.wanted_collections.is_empty()
|| config.wanted_collections.iter().any(|c| c == &commit.collection);
let matches_did = config.wanted_dids.is_empty()
|| config.wanted_dids.iter().any(|d| d == &commit.did);
if matches_collection && matches_did {
if tx.send(commit).await.is_err() {
return Ok(());
}
} else {
cursor.update(commit.time_us);
}
}
JetstreamEvent::Heartbeat { time_us } => {
cursor.update(time_us);
}
JetstreamEvent::Other { time_us, .. } => {
cursor.update(time_us);
}
}
}
}
Message::Binary(bin) => {
stats.frames_received.fetch_add(1, Ordering::Relaxed);
stats.bytes_received.fetch_add(bin.len() as u64, Ordering::Relaxed);
if let Ok(text) = std::str::from_utf8(&bin) {
if let Some(event) = parse_jetstream_frame(text) {
backoff.reset();
match event {
JetstreamEvent::Commit(commit) => {
let matches_collection = config.wanted_collections.is_empty()
|| config.wanted_collections.iter().any(|c| c == &commit.collection);
let matches_did = config.wanted_dids.is_empty()
|| config.wanted_dids.iter().any(|d| d == &commit.did);
if matches_collection && matches_did {
if tx.send(commit).await.is_err() {
return Ok(());
}
} else {
cursor.update(commit.time_us);
}
}
JetstreamEvent::Heartbeat { time_us }
| JetstreamEvent::Other { time_us, .. } => {
cursor.update(time_us);
}
}
}
}
}
Message::Ping(data) => {
let _ = ws_stream.send(Message::Pong(data)).await;
}
Message::Pong(_) => {}
Message::Close(_) => {
stats.reconnect_count.fetch_add(1, Ordering::Relaxed);
tracing::debug!("WebSocket close frame received; reconnecting");
break;
}
Message::Frame(_) => {}
}
}
Some(Err(err)) => {
stats.reconnect_count.fetch_add(1, Ordering::Relaxed);
tracing::warn!("WebSocket error: {err}. Reconnecting.");
break;
}
None => {
stats.reconnect_count.fetch_add(1, Ordering::Relaxed);
tracing::warn!("WebSocket connection closed by remote peer. Reconnecting.");
break;
}
}
}
}
}
if !cancel.is_cancelled() {
let delay = backoff.next_backoff();
tracing::info!(retry_in = ?delay, "Reconnecting to Jetstream firehose after backoff");
tokio::select! {
() = cancel.cancelled() => break,
() = tokio::time::sleep(delay) => continue,
}
}
}
Ok(())
}
async fn run_storage_sync_worker(
mut rx: tokio::sync::mpsc::Receiver<JetstreamCommit>,
store: RecordStore,
stats: Arc<ConsumerStats>,
batch_size: usize,
cursor: Arc<CursorTracker>,
cancel: CancellationToken,
) {
if batch_size <= 1 {
loop {
tokio::select! {
() = cancel.cancelled() => {
while let Ok(commit) = rx.try_recv() {
apply_single_commit(&commit, &store, &stats, &cursor);
}
break;
}
commit_opt = rx.recv() => {
match commit_opt {
Some(commit) => apply_single_commit(&commit, &store, &stats, &cursor),
None => break,
}
}
}
}
} else {
let mut buffer = Vec::with_capacity(batch_size);
loop {
tokio::select! {
() = cancel.cancelled() => {
while let Ok(commit) = rx.try_recv() {
buffer.push(commit);
}
flush_commit_batch(&mut buffer, &store, &stats, &cursor);
break;
}
count = rx.recv_many(&mut buffer, batch_size) => {
if count == 0 {
flush_commit_batch(&mut buffer, &store, &stats, &cursor);
break;
}
flush_commit_batch(&mut buffer, &store, &stats, &cursor);
}
}
}
}
}
fn apply_single_commit(
commit: &JetstreamCommit,
store: &RecordStore,
stats: &ConsumerStats,
cursor: &CursorTracker,
) {
let uri = commit.uri();
match commit.operation {
CommitOperation::Create | CommitOperation::Update => {
let Some(input) = commit.to_record_input(None) else {
stats.sync_errors.fetch_add(1, Ordering::Relaxed);
tracing::warn!(uri = %uri, "Ignoring commit with missing record payload");
return;
};
match store.upsert_record(&input) {
Ok(()) => {
stats.records_upserted.fetch_add(1, Ordering::Relaxed);
cursor.update(commit.time_us);
let _ = store.set_cursor(commit.time_us);
}
Err(err) => {
stats.sync_errors.fetch_add(1, Ordering::Relaxed);
tracing::error!(uri = %input.uri, "Failed to upsert record: {err}");
}
}
}
CommitOperation::Delete => match store.soft_delete_record(&uri, commit.time_us) {
Ok(true) => {
stats.records_deleted.fetch_add(1, Ordering::Relaxed);
cursor.update(commit.time_us);
let _ = store.set_cursor(commit.time_us);
}
Ok(false) => {
cursor.update(commit.time_us);
let _ = store.set_cursor(commit.time_us);
}
Err(err) => {
stats.sync_errors.fetch_add(1, Ordering::Relaxed);
tracing::error!(uri = %uri, "Failed to soft delete record: {err}");
}
},
}
}
fn flush_commit_batch(
buffer: &mut Vec<JetstreamCommit>,
store: &RecordStore,
stats: &ConsumerStats,
cursor: &CursorTracker,
) {
if buffer.is_empty() {
return;
}
let mut operations = Vec::with_capacity(buffer.len());
let mut latest_cursor: Option<u64> = None;
for commit in buffer.drain(..) {
latest_cursor = Some(latest_cursor.map_or(commit.time_us, |c| c.max(commit.time_us)));
match commit.operation {
CommitOperation::Create | CommitOperation::Update => {
if let Some(input) = commit.to_record_input(None) {
operations.push(StoreOperation::Upsert(input));
} else {
stats.sync_errors.fetch_add(1, Ordering::Relaxed);
tracing::warn!(uri = %commit.uri(), "Ignoring commit with missing record payload");
}
}
CommitOperation::Delete => {
operations.push(StoreOperation::Delete {
uri: commit.uri(),
indexed_at: commit.time_us,
});
}
}
}
match store.apply_operations_batch(&operations, latest_cursor) {
Ok(batch_stats) => {
stats
.records_upserted
.fetch_add(batch_stats.upserted as u64, Ordering::Relaxed);
stats
.records_deleted
.fetch_add(batch_stats.deleted as u64, Ordering::Relaxed);
stats
.sync_errors
.fetch_add(batch_stats.skipped as u64, Ordering::Relaxed);
if let Some(cur) = latest_cursor {
cursor.update(cur);
}
}
Err(err) => {
stats
.sync_errors
.fetch_add(operations.len() as u64, Ordering::Relaxed);
tracing::error!("Failed to batch apply records: {err}");
}
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic, missing_docs)]
mod tests {
use super::*;
#[test]
fn test_build_subscription_url_plain() {
let url = build_subscription_url("ws://127.0.0.1:8080/subscribe", &[], None);
assert_eq!(url, "ws://127.0.0.1:8080/subscribe");
}
#[test]
fn test_build_subscription_url_with_collections_and_cursor() {
let cols = vec![
"app.bsky.feed.post".to_string(),
"com.example.review".to_string(),
];
let url = build_subscription_url(
"ws://127.0.0.1:8080/subscribe",
&cols,
Some(1700000000000000),
);
assert_eq!(
url,
"ws://127.0.0.1:8080/subscribe?wantedCollections=app.bsky.feed.post&wantedCollections=com.example.review&cursor=1700000000000000"
);
}
#[test]
fn test_build_subscription_url_preserves_existing_query() {
let cols = vec!["app.bsky.feed.post".to_string()];
let url = build_subscription_url(
"wss://jetstream.example.com/subscribe?compress=true",
&cols,
Some(12345),
);
assert_eq!(
url,
"wss://jetstream.example.com/subscribe?compress=true&wantedCollections=app.bsky.feed.post&cursor=12345"
);
}
#[test]
fn test_build_subscription_url_zero_cursor_omitted() {
let cols = vec!["app.bsky.feed.post".to_string()];
let url = build_subscription_url("ws://127.0.0.1:8080/subscribe", &cols, Some(0));
assert_eq!(
url,
"ws://127.0.0.1:8080/subscribe?wantedCollections=app.bsky.feed.post"
);
}
#[test]
fn test_ingester_config_builder() {
let config = IngesterConfig::new("ws://127.0.0.1:9000/subscribe")
.with_collection("app.bsky.feed.post")
.with_did("did:plc:alice")
.with_cursor(1000)
.with_channel_capacity(512)
.with_inactivity_timeout(Duration::from_secs(45))
.with_batch_size(10);
assert_eq!(config.endpoint, "ws://127.0.0.1:9000/subscribe");
assert_eq!(config.wanted_collections, vec!["app.bsky.feed.post"]);
assert_eq!(config.wanted_dids, vec!["did:plc:alice"]);
assert_eq!(config.initial_cursor, Some(1000));
assert_eq!(config.channel_capacity, 512);
assert_eq!(config.inactivity_timeout, Duration::from_secs(45));
assert_eq!(config.batch_size, 10);
}
#[tokio::test]
async fn test_consumer_stats_telemetry() {
let stats = ConsumerStats::default();
assert_eq!(stats.frames_received(), 0);
assert_eq!(stats.records_upserted(), 0);
stats.frames_received.fetch_add(5, Ordering::Relaxed);
stats.records_upserted.fetch_add(3, Ordering::Relaxed);
assert_eq!(stats.frames_received(), 5);
assert_eq!(stats.records_upserted(), 3);
}
#[tokio::test]
async fn test_jetstream_consumer_end_to_end_flow() {
use crate::index::ChangeNotification;
use crate::ingest::mock::MockJetstreamServer;
use serde_json::json;
let server = MockJetstreamServer::start()
.await
.expect("start mock server");
let store = RecordStore::open_in_memory().expect("open store");
let mut bus = store.subscribe();
let config = IngesterConfig::new(server.ws_url())
.with_collection("app.bsky.feed.post")
.with_inactivity_timeout(Duration::from_secs(5));
let consumer = JetstreamConsumer::new(config, store.clone());
let cancel = CancellationToken::new();
let handle = consumer.start(cancel.clone());
tokio::time::sleep(Duration::from_millis(60)).await;
let commit = JetstreamCommit {
did: "did:plc:consumer_test".to_string(),
time_us: 1_715_000_000_000_000,
collection: "app.bsky.feed.post".to_string(),
rkey: "post_1".to_string(),
operation: CommitOperation::Create,
cid: Some("bafyrei_test_cid".to_string()),
record: Some(json!({ "text": "Consumer e2e test post" })),
};
server.emit_commit(&commit).expect("emit commit");
let notification = tokio::time::timeout(Duration::from_secs(2), bus.recv())
.await
.expect("timeout waiting for bus notification")
.expect("notification error");
match notification {
ChangeNotification::Upsert(row) => {
assert_eq!(
row.uri,
"at://did:plc:consumer_test/app.bsky.feed.post/post_1"
);
assert_eq!(row.cid, "bafyrei_test_cid");
assert_eq!(row.record_json["text"], "Consumer e2e test post");
}
_ => panic!("Expected ChangeNotification::Upsert"),
}
let stored = store
.get_record("at://did:plc:consumer_test/app.bsky.feed.post/post_1")
.expect("get_record")
.expect("record found");
assert_eq!(stored.cid, "bafyrei_test_cid");
assert_eq!(consumer.cursor(), 1_715_000_000_000_000);
assert_eq!(consumer.stats().records_upserted(), 1);
handle.stop();
handle.join().await.expect("join failed");
}
#[tokio::test]
async fn test_jetstream_consumer_filtering_ignores_unwanted_collections() {
use crate::ingest::mock::MockJetstreamServer;
use serde_json::json;
let server = MockJetstreamServer::start()
.await
.expect("start mock server");
let store = RecordStore::open_in_memory().expect("open store");
let config = IngesterConfig::new(server.ws_url())
.with_collection("app.bsky.feed.post")
.with_inactivity_timeout(Duration::from_secs(5));
let consumer = JetstreamConsumer::new(config, store.clone());
let cancel = CancellationToken::new();
let handle = consumer.start(cancel.clone());
tokio::time::sleep(Duration::from_millis(60)).await;
let ignored_commit = JetstreamCommit {
did: "did:plc:other_user".to_string(),
time_us: 1_715_000_000_100_000,
collection: "unwanted.collection".to_string(),
rkey: "rkey_ignored".to_string(),
operation: CommitOperation::Create,
cid: Some("cid_ignored".to_string()),
record: Some(json!({ "text": "ignored" })),
};
server
.emit_commit(&ignored_commit)
.expect("emit ignored commit");
tokio::time::sleep(Duration::from_millis(100)).await;
let stored = store
.get_record("at://did:plc:other_user/unwanted.collection/rkey_ignored")
.expect("get_record");
assert_eq!(stored, None);
handle.stop();
handle.join().await.expect("clean join");
}
#[tokio::test]
async fn test_jetstream_consumer_graceful_shutdown() {
use crate::ingest::mock::MockJetstreamServer;
let server = MockJetstreamServer::start()
.await
.expect("start mock server");
let store = RecordStore::open_in_memory().expect("open store");
let config = IngesterConfig::new(server.ws_url());
let consumer = JetstreamConsumer::new(config, store);
let cancel = CancellationToken::new();
let handle = consumer.start(cancel.clone());
tokio::time::sleep(Duration::from_millis(40)).await;
handle.stop();
let join_res = handle.join().await;
assert!(join_res.is_ok());
}
#[tokio::test]
async fn test_jetstream_consumer_inactivity_watchdog() {
use crate::ingest::mock::MockJetstreamServer;
let server = MockJetstreamServer::start()
.await
.expect("start mock server");
let store = RecordStore::open_in_memory().expect("open store");
let config = IngesterConfig::new(server.ws_url())
.with_inactivity_timeout(Duration::from_millis(80))
.with_ping_interval(None)
.with_backoff(Duration::from_millis(50), Duration::from_millis(100));
let consumer = JetstreamConsumer::new(config, store);
let cancel = CancellationToken::new();
let handle = consumer.start(cancel.clone());
tokio::time::sleep(Duration::from_millis(250)).await;
assert!(
consumer.stats().reconnect_count() >= 1,
"Watchdog should have triggered at least one reconnect"
);
handle.stop();
handle.join().await.expect("clean join");
}
}