use serde::{Deserialize, Serialize};
#[cfg(feature = "storage")]
use crate::error::Result;
#[cfg(feature = "storage")]
use crate::index::{RecordInput, RecordStore};
#[must_use]
pub const fn normalize_indexed_at(time_us: u64) -> u64 {
time_us
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CommitOperation {
Create,
Update,
Delete,
}
impl CommitOperation {
#[must_use]
pub fn from_str_loose(s: &str) -> Option<Self> {
if s.eq_ignore_ascii_case("create") {
Some(Self::Create)
} else if s.eq_ignore_ascii_case("update") {
Some(Self::Update)
} else if s.eq_ignore_ascii_case("delete") {
Some(Self::Delete)
} else {
None
}
}
#[must_use]
pub const fn as_str(&self) -> &'static str {
match self {
Self::Create => "create",
Self::Update => "update",
Self::Delete => "delete",
}
}
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct RawJetstreamMessage {
#[serde(default)]
pub did: Option<String>,
#[serde(default)]
pub time_us: Option<u64>,
#[serde(default)]
pub kind: Option<String>,
#[serde(default)]
pub commit: Option<RawJetstreamCommit>,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct RawJetstreamCommit {
#[serde(default)]
pub operation: String,
#[serde(default)]
pub collection: String,
#[serde(default)]
pub rkey: String,
#[serde(default)]
pub cid: Option<String>,
#[serde(default)]
pub record: Option<serde_json::Value>,
#[serde(default)]
pub rev: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct JetstreamCommit {
pub did: String,
pub time_us: u64,
pub collection: String,
pub rkey: String,
pub operation: CommitOperation,
pub cid: Option<String>,
pub record: Option<serde_json::Value>,
}
impl JetstreamCommit {
#[must_use]
pub fn uri(&self) -> String {
format!("at://{}/{}/{}", self.did, self.collection, self.rkey)
}
#[cfg(feature = "storage")]
#[must_use]
pub fn to_record_input(&self, indexed_at_timestamp: Option<u64>) -> Option<RecordInput> {
if self.operation == CommitOperation::Delete {
return None;
}
let record_json = self.record.clone()?;
let cid = self.cid.clone().unwrap_or_default();
let indexed_at = indexed_at_timestamp.unwrap_or_else(|| normalize_indexed_at(self.time_us));
Some(RecordInput {
uri: self.uri(),
cid,
did: self.did.clone(),
collection: self.collection.clone(),
rkey: self.rkey.clone(),
record_json,
indexed_at,
})
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum JetstreamEvent {
Commit(JetstreamCommit),
Heartbeat {
time_us: u64,
},
Other {
kind: String,
did: Option<String>,
time_us: u64,
},
}
impl JetstreamEvent {
#[must_use]
pub fn time_us(&self) -> u64 {
match self {
Self::Commit(c) => c.time_us,
Self::Heartbeat { time_us } | Self::Other { time_us, .. } => *time_us,
}
}
#[must_use]
pub fn as_commit(&self) -> Option<&JetstreamCommit> {
match self {
Self::Commit(c) => Some(c),
_ => None,
}
}
}
#[must_use]
pub fn parse_jetstream_frame(text: &str) -> Option<JetstreamEvent> {
let trimmed = text.trim();
if trimmed.is_empty() {
return None;
}
let msg: RawJetstreamMessage = match serde_json::from_str(trimmed) {
Ok(m) => m,
Err(err) => {
tracing::debug!(error = %err, "Discarding unparseable Jetstream JSON frame");
return None;
}
};
let time_us = msg.time_us.unwrap_or(0);
if time_us > 0 && !is_valid_time_us(time_us) {
tracing::debug!(
time_us,
"Discarding frame with far-future or poisoned timestamp"
);
return None;
}
if msg.kind.as_deref() == Some("commit") {
let did = match msg.did {
Some(d) if !d.trim().is_empty() => d,
_ => {
tracing::debug!("Discarding commit frame with missing DID");
return if time_us > 0 {
Some(JetstreamEvent::Heartbeat { time_us })
} else {
None
};
}
};
let commit = match msg.commit {
Some(c) => c,
None => {
tracing::debug!("Commit event missing 'commit' payload");
return if time_us > 0 {
Some(JetstreamEvent::Heartbeat { time_us })
} else {
None
};
}
};
if !is_valid_did(&did)
|| !is_valid_collection(&commit.collection)
|| crate::repo::validate_rkey(&commit.rkey).is_err()
{
tracing::debug!("Discarding commit with malformed did, collection, or rkey");
return if time_us > 0 {
Some(JetstreamEvent::Heartbeat { time_us })
} else {
None
};
}
let operation = match CommitOperation::from_str_loose(&commit.operation) {
Some(op) => op,
None => {
tracing::debug!(
operation = %commit.operation,
"Discarding commit with unknown operation"
);
return if time_us > 0 {
Some(JetstreamEvent::Heartbeat { time_us })
} else {
None
};
}
};
return Some(JetstreamEvent::Commit(JetstreamCommit {
did,
time_us,
collection: commit.collection,
rkey: commit.rkey,
operation,
cid: commit.cid,
record: commit.record,
}));
}
if msg.kind.is_none() && time_us > 0 {
return Some(JetstreamEvent::Heartbeat { time_us });
}
if let Some(kind) = msg.kind {
if time_us > 0 {
return Some(JetstreamEvent::Other {
kind,
did: msg.did,
time_us,
});
}
}
None
}
pub(crate) const MAX_FUTURE_SKEW_US: u64 = 5 * 365 * 86_400 * 1_000_000;
pub(crate) fn is_valid_time_us(time_us: u64) -> bool {
let now_us = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_micros() as u64);
if now_us > 0 && time_us > now_us.saturating_add(MAX_FUTURE_SKEW_US) {
return false;
}
true
}
fn is_valid_did(did: &str) -> bool {
let trimmed = did.trim();
if !trimmed.starts_with("did:") || trimmed.len() < 7 {
return false;
}
if trimmed.split(':').count() < 3 {
return false;
}
for ch in trimmed.chars() {
if !ch.is_ascii_alphanumeric()
&& ch != ':'
&& ch != '.'
&& ch != '_'
&& ch != '-'
&& ch != '%'
{
return false;
}
}
true
}
fn is_valid_collection(coll: &str) -> bool {
let trimmed = coll.trim();
if trimmed.is_empty() || trimmed.len() > 317 || !trimmed.contains('.') {
return false;
}
for part in trimmed.split('.') {
if part.is_empty() {
return false;
}
for ch in part.chars() {
if !ch.is_ascii_alphanumeric() && ch != '-' {
return false;
}
}
}
true
}
#[must_use]
pub fn parse_frame_timestamp(text: &str) -> Option<u64> {
parse_jetstream_frame(text).map(|evt| evt.time_us())
}
#[must_use]
pub fn parse_jetstream_commit(text: &str) -> Option<JetstreamCommit> {
match parse_jetstream_frame(text) {
Some(JetstreamEvent::Commit(commit)) => Some(commit),
_ => None,
}
}
#[cfg(feature = "storage")]
pub fn sync_commit_to_store(store: &RecordStore, commit: &JetstreamCommit) -> Result<()> {
let uri = commit.uri();
match commit.operation {
CommitOperation::Create | CommitOperation::Update => {
let Some(input) = commit.to_record_input(None) else {
return Err(crate::error::SkybaseError::Index(format!(
"Missing record payload for commit '{uri}'"
)));
};
store.upsert_record(&input)?;
}
CommitOperation::Delete => {
store.soft_delete_record(&uri, commit.time_us)?;
}
}
Ok(())
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic, missing_docs)]
mod tests {
use super::*;
#[cfg(feature = "storage")]
use serde_json::json;
#[test]
fn test_commit_operation_loose_parsing() {
assert_eq!(
CommitOperation::from_str_loose("create"),
Some(CommitOperation::Create)
);
assert_eq!(
CommitOperation::from_str_loose("CREATE"),
Some(CommitOperation::Create)
);
assert_eq!(
CommitOperation::from_str_loose("Update"),
Some(CommitOperation::Update)
);
assert_eq!(
CommitOperation::from_str_loose("DELETE"),
Some(CommitOperation::Delete)
);
assert_eq!(CommitOperation::from_str_loose("unknown"), None);
}
#[test]
fn test_parse_valid_commit_frame() {
let json_str = r#"{
"did": "did:plc:alice",
"time_us": 1710000000123456,
"kind": "commit",
"commit": {
"operation": "create",
"collection": "app.bsky.feed.post",
"rkey": "post1",
"cid": "bafyrei1",
"record": { "text": "Hello world" }
}
}"#;
let event = parse_jetstream_frame(json_str).expect("should parse");
match event {
JetstreamEvent::Commit(c) => {
assert_eq!(c.did, "did:plc:alice");
assert_eq!(c.time_us, 1710000000123456);
assert_eq!(c.collection, "app.bsky.feed.post");
assert_eq!(c.rkey, "post1");
assert_eq!(c.operation, CommitOperation::Create);
assert_eq!(c.cid.as_deref(), Some("bafyrei1"));
assert_eq!(c.uri(), "at://did:plc:alice/app.bsky.feed.post/post1");
#[cfg(feature = "storage")]
{
let input = c.to_record_input(None).expect("to record input");
assert_eq!(input.indexed_at, 1710000000123456);
}
}
_ => panic!("Expected commit event"),
}
}
#[test]
fn test_parse_heartbeat_frame() {
let json_str = r#"{"time_us": 1720000000999999}"#;
let event = parse_jetstream_frame(json_str).expect("should parse");
match event {
JetstreamEvent::Heartbeat { time_us } => {
assert_eq!(time_us, 1720000000999999);
}
_ => panic!("Expected heartbeat"),
}
}
#[test]
fn test_parse_malformed_json_graceful() {
assert_eq!(parse_jetstream_frame(""), None);
assert_eq!(parse_jetstream_frame(" "), None);
assert_eq!(parse_jetstream_frame("{ corrupt json:"), None);
assert_eq!(parse_jetstream_frame("42"), None);
}
#[cfg(feature = "storage")]
#[test]
fn test_sync_commit_to_store() {
let store = RecordStore::open_in_memory().expect("open store");
let commit = JetstreamCommit {
did: "did:plc:bob".to_string(),
time_us: 1715000000000000,
collection: "app.bsky.feed.post".to_string(),
rkey: "r1".to_string(),
operation: CommitOperation::Create,
cid: Some("cid1".to_string()),
record: Some(json!({ "text": "Testing sync" })),
};
sync_commit_to_store(&store, &commit).expect("sync commit");
let rec = store
.get_record("at://did:plc:bob/app.bsky.feed.post/r1")
.expect("get record")
.expect("record found");
assert_eq!(rec.cid, "cid1");
assert_eq!(rec.record_json["text"], "Testing sync");
let delete_commit = JetstreamCommit {
did: "did:plc:bob".to_string(),
time_us: 1715000001000000,
collection: "app.bsky.feed.post".to_string(),
rkey: "r1".to_string(),
operation: CommitOperation::Delete,
cid: None,
record: None,
};
sync_commit_to_store(&store, &delete_commit).expect("sync delete");
assert_eq!(
store
.get_record("at://did:plc:bob/app.bsky.feed.post/r1")
.expect("get record"),
None
);
let rec_after = store
.get_record_including_deleted("at://did:plc:bob/app.bsky.feed.post/r1")
.expect("get record including deleted")
.expect("record found");
assert!(rec_after.is_deleted);
}
}