use std::{
collections::{HashMap, HashSet, VecDeque},
path::{Path, PathBuf},
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
mpsc as std_mpsc,
},
thread::{self, JoinHandle},
time::{SystemTime, UNIX_EPOCH},
};
use futures_util::stream;
use rocksdb::{
BlockBasedOptions, ColumnFamilyDescriptor, ColumnFamilyRef, DB, DBCompressionType, Direction,
IteratorMode, Options as DbOptions, ReadOptions, SliceTransform, WriteBatch, WriteOptions,
};
use tokio::sync::{mpsc, oneshot, watch};
use crate::{
AppendOutcome, AtomicWrite, DurableStateStore, EventStream, Journal, PersistenceError,
PersistenceId, PersistenceResult, ReplayRange, Revision, SelectedSnapshot, SequenceNr,
SerializedPayload, SerializerId, SnapshotCriteria, SnapshotMetadata, SnapshotStore,
StoredEvent, Tag, VersionedState,
projection::{
CursorPosition, ProjectionCursor, ProjectionError, ProjectionGeneration, ProjectionId,
ProjectionOffsetStore, ProjectionResult,
},
query::{
DeletedDurableState, DurableStateChange, DurableStateChangesQuery, EventEnvelope,
EventsByPersistenceIdQuery, EventsByTagQuery, Offset, PersistenceIdsQuery, QueryMode,
QueryStream, UpdatedDurableState,
},
};
const JOURNAL_CF: &str = "journal";
const SNAPSHOTS_CF: &str = "snapshots";
const DURABLE_STATE_CF: &str = "durable_state";
const TAGS_CF: &str = "tags";
const META_CF: &str = "meta";
const PROJECTION_CF: &str = "projection";
const DEFAULT_COMMAND_CAPACITY: usize = 256;
const ENVELOPE_VERSION: u8 = 1;
const TAG_INDEX_ENVELOPE_VERSION: u8 = 2;
const PROJECTION_ENTRY_VERSION: u8 = 1;
const HEAD_KEY_MARKER: u8 = 1;
const READ_SIDE_OFFSET_KEY: &[u8] = b"\x02read-side-offset";
const ID_HASH_BYTES: usize = 8;
const ENVELOPE_OFFSET_START: usize = 1;
const ENVELOPE_OFFSET_END: usize = ENVELOPE_OFFSET_START + 8;
const GROUP_COMMIT_MAX_WRITES: usize = 256;
const GROUP_COMMIT_MAX_EVENTS: usize = 4_096;
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum RocksDbDurability {
#[default]
Sync,
Relaxed,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RocksDbOptions {
pub command_capacity: usize,
pub durability: RocksDbDurability,
}
impl Default for RocksDbOptions {
fn default() -> Self {
Self {
command_capacity: DEFAULT_COMMAND_CAPACITY,
durability: RocksDbDurability::Sync,
}
}
}
pub struct RocksDbBackend {
sender: Option<mpsc::Sender<Command>>,
notifications: watch::Sender<u64>,
owner: Option<JoinHandle<()>>,
active_owner_threads: Arc<AtomicUsize>,
}
impl std::fmt::Debug for RocksDbBackend {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("RocksDbBackend")
.field("channel_closed", &self.sender.is_none())
.field(
"active_owner_threads",
&self.active_owner_threads.load(Ordering::Acquire),
)
.finish_non_exhaustive()
}
}
impl RocksDbBackend {
pub fn open(path: impl AsRef<Path>) -> PersistenceResult<Self> {
Self::open_with_options(path, RocksDbOptions::default())
}
pub fn open_with_options(
path: impl AsRef<Path>,
options: RocksDbOptions,
) -> PersistenceResult<Self> {
if options.command_capacity == 0 {
return Err(PersistenceError::Rejected(
"RocksDB command capacity must be greater than zero".to_owned(),
));
}
let path = path.as_ref().to_path_buf();
let (sender, receiver) = mpsc::channel(options.command_capacity);
let (notifications, _notification_receiver) = watch::channel(0);
let (startup_sender, startup_receiver) = std_mpsc::sync_channel(1);
let active_owner_threads = Arc::new(AtomicUsize::new(0));
let owner_active = Arc::clone(&active_owner_threads);
let durability = options.durability;
let owner = thread::Builder::new()
.name("datum-rocksdb-owner".to_owned())
.spawn({
let notifications = notifications.clone();
move || {
owner_main(
path,
durability,
receiver,
startup_sender,
owner_active,
notifications,
);
}
})
.map_err(|error| {
PersistenceError::Connection(format!(
"failed to start RocksDB storage-owner thread: {error}"
))
})?;
match startup_receiver.recv() {
Ok(Ok(())) => Ok(Self {
sender: Some(sender),
notifications,
owner: Some(owner),
active_owner_threads,
}),
Ok(Err(error)) => {
let _ = owner.join();
Err(error)
}
Err(_) => {
let _ = owner.join();
Err(PersistenceError::Connection(
"RocksDB storage-owner thread ended during startup".to_owned(),
))
}
}
}
#[doc(hidden)]
#[must_use]
pub fn storage_owner_thread_count(&self) -> usize {
self.active_owner_threads.load(Ordering::Acquire)
}
fn sender(&self) -> PersistenceResult<&mpsc::Sender<Command>> {
self.sender.as_ref().ok_or_else(|| {
PersistenceError::Connection("RocksDB storage owner is closed".to_owned())
})
}
}
impl Drop for RocksDbBackend {
fn drop(&mut self) {
self.sender.take();
if let Some(owner) = self.owner.take() {
let _ = owner.join();
}
}
}
#[cfg(test)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ProjectionSaveCrashPoint {
BeforeCommit,
AfterCommitBeforeAcknowledgement,
}
enum Command {
Append {
writes: Vec<AtomicWrite>,
reply: oneshot::Sender<PersistenceResult<Vec<AppendOutcome>>>,
},
HighestSequenceNr {
persistence_id: PersistenceId,
reply: oneshot::Sender<PersistenceResult<SequenceNr>>,
},
Replay {
range: ReplayRange,
reply: oneshot::Sender<PersistenceResult<Vec<StoredEvent>>>,
},
DeleteJournalTo {
persistence_id: PersistenceId,
inclusive: SequenceNr,
reply: oneshot::Sender<PersistenceResult<()>>,
},
LoadSnapshot {
persistence_id: PersistenceId,
criteria: SnapshotCriteria,
reply: oneshot::Sender<PersistenceResult<Option<SelectedSnapshot>>>,
},
SaveSnapshot {
metadata: SnapshotMetadata,
adapter_manifest: String,
state: SerializedPayload,
reply: oneshot::Sender<PersistenceResult<()>>,
},
DeleteSnapshot {
metadata: SnapshotMetadata,
reply: oneshot::Sender<PersistenceResult<()>>,
},
DeleteSnapshotsMatching {
persistence_id: PersistenceId,
criteria: SnapshotCriteria,
reply: oneshot::Sender<PersistenceResult<()>>,
},
GetDurableState {
persistence_id: PersistenceId,
reply: oneshot::Sender<PersistenceResult<VersionedState>>,
},
UpsertDurableState {
persistence_id: PersistenceId,
new_revision: Revision,
state: SerializedPayload,
tag: Option<Tag>,
reply: oneshot::Sender<PersistenceResult<()>>,
},
DeleteDurableState {
persistence_id: PersistenceId,
new_revision: Revision,
reply: oneshot::Sender<PersistenceResult<()>>,
},
QueryEventsByPersistenceId {
persistence_id: PersistenceId,
from_sequence_nr: SequenceNr,
to_sequence_nr: SequenceNr,
reply: oneshot::Sender<PersistenceResult<PersistenceIdQueryPage>>,
},
QueryEventsByTag {
tag: Tag,
after_offset: u64,
reply: oneshot::Sender<PersistenceResult<OffsetQueryPage<EventEnvelope>>>,
},
QueryPersistenceIds {
reply: oneshot::Sender<PersistenceResult<Vec<PersistenceId>>>,
},
QueryDurableStateChanges {
tag: Tag,
after_offset: u64,
reply: oneshot::Sender<PersistenceResult<OffsetQueryPage<DurableStateChange>>>,
},
ClaimProjection {
projection_id: ProjectionId,
reply: oneshot::Sender<ProjectionResult<ProjectionGeneration>>,
},
LoadProjection {
projection_id: ProjectionId,
reply: oneshot::Sender<ProjectionResult<Option<ProjectionCursor>>>,
},
SaveProjection {
projection_id: ProjectionId,
generation: ProjectionGeneration,
expected_position: Option<CursorPosition>,
cursor: ProjectionCursor,
reply: oneshot::Sender<ProjectionResult<()>>,
},
ClearProjection {
projection_id: ProjectionId,
generation: ProjectionGeneration,
reply: oneshot::Sender<ProjectionResult<()>>,
},
LoadProjectionPaused {
projection_id: ProjectionId,
reply: oneshot::Sender<ProjectionResult<bool>>,
},
SetProjectionPaused {
projection_id: ProjectionId,
paused: bool,
reply: oneshot::Sender<ProjectionResult<()>>,
},
#[cfg(test)]
CrashProjectionSave {
projection_id: ProjectionId,
generation: ProjectionGeneration,
expected_position: Option<CursorPosition>,
cursor: ProjectionCursor,
point: ProjectionSaveCrashPoint,
},
}
struct PersistenceIdQueryPage {
events: Vec<EventEnvelope>,
highest_sequence_nr: SequenceNr,
}
struct OffsetQueryPage<T> {
items: Vec<T>,
boundary: u64,
}
#[async_trait::async_trait]
impl Journal for RocksDbBackend {
async fn append(&self, writes: Vec<AtomicWrite>) -> PersistenceResult<Vec<AppendOutcome>> {
if writes.is_empty() {
return Ok(Vec::new());
}
let (reply, receiver) = oneshot::channel();
self.enqueue(Command::Append { writes, reply }).await?;
receive_reply(receiver, "journal append").await
}
async fn append_one(&self, write: AtomicWrite) -> PersistenceResult<AppendOutcome> {
let outcomes = self.append(vec![write]).await?;
outcomes.into_iter().next().ok_or_else(|| {
PersistenceError::Failure("RocksDB returned no outcome for one atomic write".to_owned())
})
}
async fn highest_sequence_nr(
&self,
persistence_id: &PersistenceId,
_from_hint: SequenceNr,
) -> PersistenceResult<SequenceNr> {
let (reply, receiver) = oneshot::channel();
self.enqueue(Command::HighestSequenceNr {
persistence_id: persistence_id.clone(),
reply,
})
.await?;
receive_reply(receiver, "highest sequence number").await
}
async fn replay(&self, range: ReplayRange) -> PersistenceResult<EventStream> {
if range.from > range.to || range.max == 0 {
return Ok(Box::pin(stream::empty()));
}
let (reply, receiver) = oneshot::channel();
self.enqueue(Command::Replay { range, reply }).await?;
let events = receive_reply(receiver, "journal replay").await?;
Ok(Box::pin(stream::iter(events.into_iter().map(Ok))))
}
async fn delete_to(
&self,
persistence_id: &PersistenceId,
inclusive: SequenceNr,
) -> PersistenceResult<()> {
let (reply, receiver) = oneshot::channel();
self.enqueue(Command::DeleteJournalTo {
persistence_id: persistence_id.clone(),
inclusive,
reply,
})
.await?;
receive_reply(receiver, "journal delete").await
}
}
#[async_trait::async_trait]
impl SnapshotStore for RocksDbBackend {
async fn load(
&self,
persistence_id: &PersistenceId,
criteria: &SnapshotCriteria,
) -> PersistenceResult<Option<SelectedSnapshot>> {
if criteria.min_sequence_nr > criteria.max_sequence_nr
|| criteria.min_timestamp_ms > criteria.max_timestamp_ms
{
return Ok(None);
}
let (reply, receiver) = oneshot::channel();
self.enqueue(Command::LoadSnapshot {
persistence_id: persistence_id.clone(),
criteria: *criteria,
reply,
})
.await?;
receive_reply(receiver, "snapshot load").await
}
async fn save(
&self,
metadata: SnapshotMetadata,
adapter_manifest: String,
state: SerializedPayload,
) -> PersistenceResult<()> {
let (reply, receiver) = oneshot::channel();
self.enqueue(Command::SaveSnapshot {
metadata,
adapter_manifest,
state,
reply,
})
.await?;
receive_reply(receiver, "snapshot save").await
}
async fn delete(&self, metadata: SnapshotMetadata) -> PersistenceResult<()> {
let (reply, receiver) = oneshot::channel();
self.enqueue(Command::DeleteSnapshot { metadata, reply })
.await?;
receive_reply(receiver, "snapshot delete").await
}
async fn delete_matching(
&self,
persistence_id: &PersistenceId,
criteria: &SnapshotCriteria,
) -> PersistenceResult<()> {
if criteria.min_sequence_nr > criteria.max_sequence_nr
|| criteria.min_timestamp_ms > criteria.max_timestamp_ms
{
return Ok(());
}
let (reply, receiver) = oneshot::channel();
self.enqueue(Command::DeleteSnapshotsMatching {
persistence_id: persistence_id.clone(),
criteria: *criteria,
reply,
})
.await?;
receive_reply(receiver, "snapshot range delete").await
}
}
#[async_trait::async_trait]
impl DurableStateStore for RocksDbBackend {
async fn get(&self, persistence_id: &PersistenceId) -> PersistenceResult<VersionedState> {
let (reply, receiver) = oneshot::channel();
self.enqueue(Command::GetDurableState {
persistence_id: persistence_id.clone(),
reply,
})
.await?;
receive_reply(receiver, "durable-state get").await
}
async fn upsert(
&self,
persistence_id: &PersistenceId,
new_revision: Revision,
state: SerializedPayload,
tag: Option<Tag>,
) -> PersistenceResult<()> {
let (reply, receiver) = oneshot::channel();
self.enqueue(Command::UpsertDurableState {
persistence_id: persistence_id.clone(),
new_revision,
state,
tag,
reply,
})
.await?;
receive_reply(receiver, "durable-state upsert").await
}
async fn delete(
&self,
persistence_id: &PersistenceId,
new_revision: Revision,
) -> PersistenceResult<()> {
let (reply, receiver) = oneshot::channel();
self.enqueue(Command::DeleteDurableState {
persistence_id: persistence_id.clone(),
new_revision,
reply,
})
.await?;
receive_reply(receiver, "durable-state delete").await
}
}
#[async_trait::async_trait]
impl EventsByPersistenceIdQuery for RocksDbBackend {
async fn events_by_persistence_id(
&self,
persistence_id: &PersistenceId,
from_sequence_nr: SequenceNr,
to_sequence_nr: SequenceNr,
mode: QueryMode,
) -> PersistenceResult<QueryStream<EventEnvelope>> {
if from_sequence_nr > to_sequence_nr {
return Ok(Box::pin(stream::empty()));
}
let notifications = self.notifications.subscribe();
let (reply, receiver) = oneshot::channel();
self.enqueue(Command::QueryEventsByPersistenceId {
persistence_id: persistence_id.clone(),
from_sequence_nr,
to_sequence_nr,
reply,
})
.await?;
let page = receive_reply(receiver, "events-by-persistence-id query").await?;
if mode == QueryMode::Current {
return Ok(Box::pin(stream::iter(page.events.into_iter().map(Ok))));
}
let next_sequence_nr = page
.highest_sequence_nr
.as_u64()
.checked_add(1)
.map(SequenceNr::new);
let cursor = RocksPersistenceIdCursor {
sender: self.sender()?.downgrade(),
notifications,
persistence_id: persistence_id.clone(),
next_sequence_nr,
to_sequence_nr,
buffered: VecDeque::from(page.events),
reached_end: page.highest_sequence_nr >= to_sequence_nr,
};
Ok(Box::pin(stream::unfold(
cursor,
next_rocks_persistence_id_event,
)))
}
}
#[async_trait::async_trait]
impl EventsByTagQuery for RocksDbBackend {
async fn events_by_tag(
&self,
tag: &Tag,
offset: Offset,
mode: QueryMode,
) -> PersistenceResult<QueryStream<EventEnvelope>> {
let after_offset = rocks_sequence_offset(offset)?;
let notifications = self.notifications.subscribe();
let page = self.query_events_by_tag(tag.clone(), after_offset).await?;
if mode == QueryMode::Current {
return Ok(Box::pin(stream::iter(page.items.into_iter().map(Ok))));
}
let cursor = RocksOffsetCursor {
sender: self.sender()?.downgrade(),
notifications,
query: RocksOffsetQuery::EventsByTag(tag.clone()),
after_offset: page.boundary,
buffered: page.items.into_iter().map(RocksOffsetItem::Event).collect(),
};
Ok(Box::pin(stream::unfold(cursor, next_rocks_tag_event)))
}
}
#[async_trait::async_trait]
impl PersistenceIdsQuery for RocksDbBackend {
async fn persistence_ids(
&self,
mode: QueryMode,
) -> PersistenceResult<QueryStream<PersistenceId>> {
let notifications = self.notifications.subscribe();
let ids = self.query_persistence_ids().await?;
if mode == QueryMode::Current {
return Ok(Box::pin(stream::iter(ids.into_iter().map(Ok))));
}
let known = ids.iter().cloned().collect();
let cursor = RocksPersistenceIdsCursor {
sender: self.sender()?.downgrade(),
notifications,
known,
buffered: VecDeque::from(ids),
};
Ok(Box::pin(stream::unfold(
cursor,
next_rocks_persistence_id_value,
)))
}
}
#[async_trait::async_trait]
impl DurableStateChangesQuery for RocksDbBackend {
async fn changes(
&self,
tag: &Tag,
offset: Offset,
mode: QueryMode,
) -> PersistenceResult<QueryStream<DurableStateChange>> {
let after_offset = rocks_sequence_offset(offset)?;
let notifications = self.notifications.subscribe();
let page = self
.query_durable_state_changes(tag.clone(), after_offset)
.await?;
if mode == QueryMode::Current {
return Ok(Box::pin(stream::iter(page.items.into_iter().map(Ok))));
}
let cursor = RocksOffsetCursor {
sender: self.sender()?.downgrade(),
notifications,
query: RocksOffsetQuery::DurableStateChanges(tag.clone()),
after_offset: page.boundary,
buffered: page
.items
.into_iter()
.map(RocksOffsetItem::DurableStateChange)
.collect(),
};
Ok(Box::pin(stream::unfold(cursor, next_rocks_durable_change)))
}
}
#[async_trait::async_trait]
impl ProjectionOffsetStore for RocksDbBackend {
async fn claim(&self, projection_id: &ProjectionId) -> ProjectionResult<ProjectionGeneration> {
projection_id.validate()?;
let (reply, receiver) = oneshot::channel();
self.enqueue_projection(Command::ClaimProjection {
projection_id: projection_id.clone(),
reply,
})
.await?;
receive_projection_reply(receiver, "projection generation claim").await
}
async fn load(
&self,
projection_id: &ProjectionId,
) -> ProjectionResult<Option<ProjectionCursor>> {
projection_id.validate()?;
let (reply, receiver) = oneshot::channel();
self.enqueue_projection(Command::LoadProjection {
projection_id: projection_id.clone(),
reply,
})
.await?;
receive_projection_reply(receiver, "projection cursor load").await
}
async fn save(
&self,
projection_id: &ProjectionId,
generation: ProjectionGeneration,
expected_position: Option<&CursorPosition>,
cursor: ProjectionCursor,
) -> ProjectionResult<()> {
projection_id.validate()?;
let (reply, receiver) = oneshot::channel();
self.enqueue_projection(Command::SaveProjection {
projection_id: projection_id.clone(),
generation,
expected_position: expected_position.cloned(),
cursor,
reply,
})
.await?;
receive_projection_reply(receiver, "projection cursor save").await
}
async fn clear(
&self,
projection_id: &ProjectionId,
generation: ProjectionGeneration,
) -> ProjectionResult<()> {
projection_id.validate()?;
let (reply, receiver) = oneshot::channel();
self.enqueue_projection(Command::ClearProjection {
projection_id: projection_id.clone(),
generation,
reply,
})
.await?;
receive_projection_reply(receiver, "projection cursor clear").await
}
async fn load_paused(&self, projection_id: &ProjectionId) -> ProjectionResult<bool> {
projection_id.validate()?;
let (reply, receiver) = oneshot::channel();
self.enqueue_projection(Command::LoadProjectionPaused {
projection_id: projection_id.clone(),
reply,
})
.await?;
receive_projection_reply(receiver, "projection management load").await
}
async fn set_paused(&self, projection_id: &ProjectionId, paused: bool) -> ProjectionResult<()> {
projection_id.validate()?;
let (reply, receiver) = oneshot::channel();
self.enqueue_projection(Command::SetProjectionPaused {
projection_id: projection_id.clone(),
paused,
reply,
})
.await?;
receive_projection_reply(receiver, "projection management save").await
}
}
impl RocksDbBackend {
async fn enqueue(&self, command: Command) -> PersistenceResult<()> {
self.sender()?
.send(command)
.await
.map_err(|_| PersistenceError::Connection("RocksDB command channel closed".to_owned()))
}
async fn enqueue_projection(&self, command: Command) -> ProjectionResult<()> {
let sender = self.sender.as_ref().ok_or_else(|| {
ProjectionError::OffsetStore("RocksDB storage owner is closed".to_owned())
})?;
sender
.send(command)
.await
.map_err(|_| ProjectionError::OffsetStore("RocksDB command channel closed".to_owned()))
}
#[cfg(test)]
async fn crash_projection_save(
&self,
projection_id: ProjectionId,
generation: ProjectionGeneration,
expected_position: Option<CursorPosition>,
cursor: ProjectionCursor,
point: ProjectionSaveCrashPoint,
) -> ProjectionResult<()> {
self.enqueue_projection(Command::CrashProjectionSave {
projection_id,
generation,
expected_position,
cursor,
point,
})
.await?;
std::future::pending().await
}
async fn query_events_by_tag(
&self,
tag: Tag,
after_offset: u64,
) -> PersistenceResult<OffsetQueryPage<EventEnvelope>> {
let (reply, receiver) = oneshot::channel();
self.enqueue(Command::QueryEventsByTag {
tag,
after_offset,
reply,
})
.await?;
receive_reply(receiver, "events-by-tag query").await
}
async fn query_persistence_ids(&self) -> PersistenceResult<Vec<PersistenceId>> {
let (reply, receiver) = oneshot::channel();
self.enqueue(Command::QueryPersistenceIds { reply }).await?;
receive_reply(receiver, "persistence-ids query").await
}
async fn query_durable_state_changes(
&self,
tag: Tag,
after_offset: u64,
) -> PersistenceResult<OffsetQueryPage<DurableStateChange>> {
let (reply, receiver) = oneshot::channel();
self.enqueue(Command::QueryDurableStateChanges {
tag,
after_offset,
reply,
})
.await?;
receive_reply(receiver, "durable-state changes query").await
}
}
struct RocksPersistenceIdCursor {
sender: mpsc::WeakSender<Command>,
notifications: watch::Receiver<u64>,
persistence_id: PersistenceId,
next_sequence_nr: Option<SequenceNr>,
to_sequence_nr: SequenceNr,
buffered: VecDeque<EventEnvelope>,
reached_end: bool,
}
async fn next_rocks_persistence_id_event(
mut cursor: RocksPersistenceIdCursor,
) -> Option<(PersistenceResult<EventEnvelope>, RocksPersistenceIdCursor)> {
loop {
if let Some(envelope) = cursor.buffered.pop_front() {
return Some((Ok(envelope), cursor));
}
if cursor.reached_end || cursor.next_sequence_nr.is_none() {
return None;
}
if cursor.notifications.changed().await.is_err() {
return None;
}
let sender = cursor.sender.upgrade()?;
let (reply, receiver) = oneshot::channel();
let command = Command::QueryEventsByPersistenceId {
persistence_id: cursor.persistence_id.clone(),
from_sequence_nr: cursor.next_sequence_nr.expect("checked above"),
to_sequence_nr: cursor.to_sequence_nr,
reply,
};
if sender.send(command).await.is_err() {
return Some((Err(rocks_query_channel_closed()), cursor));
}
let page = match receive_reply(receiver, "live events-by-persistence-id query").await {
Ok(page) => page,
Err(error) => return Some((Err(error), cursor)),
};
cursor.next_sequence_nr = page
.highest_sequence_nr
.as_u64()
.checked_add(1)
.map(SequenceNr::new);
cursor.reached_end = page.highest_sequence_nr >= cursor.to_sequence_nr;
cursor.buffered = VecDeque::from(page.events);
}
}
enum RocksOffsetQuery {
EventsByTag(Tag),
DurableStateChanges(Tag),
}
enum RocksOffsetItem {
Event(EventEnvelope),
DurableStateChange(DurableStateChange),
}
struct RocksOffsetCursor {
sender: mpsc::WeakSender<Command>,
notifications: watch::Receiver<u64>,
query: RocksOffsetQuery,
after_offset: u64,
buffered: VecDeque<RocksOffsetItem>,
}
async fn refill_rocks_offset_cursor(
cursor: &mut RocksOffsetCursor,
) -> Option<PersistenceResult<()>> {
if cursor.notifications.changed().await.is_err() {
return None;
}
let sender = cursor.sender.upgrade()?;
let (reply, receiver) = oneshot::channel();
match &cursor.query {
RocksOffsetQuery::EventsByTag(tag) => {
if sender
.send(Command::QueryEventsByTag {
tag: tag.clone(),
after_offset: cursor.after_offset,
reply,
})
.await
.is_err()
{
return Some(Err(rocks_query_channel_closed()));
}
match receive_reply(receiver, "live events-by-tag query").await {
Ok(page) => {
cursor.after_offset = page.boundary;
cursor
.buffered
.extend(page.items.into_iter().map(RocksOffsetItem::Event));
Some(Ok(()))
}
Err(error) => Some(Err(error)),
}
}
RocksOffsetQuery::DurableStateChanges(_) => unreachable!("reply type differs by query"),
}
}
async fn next_rocks_tag_event(
mut cursor: RocksOffsetCursor,
) -> Option<(PersistenceResult<EventEnvelope>, RocksOffsetCursor)> {
loop {
if let Some(RocksOffsetItem::Event(envelope)) = cursor.buffered.pop_front() {
return Some((Ok(envelope), cursor));
}
match refill_rocks_offset_cursor(&mut cursor).await? {
Ok(()) => {}
Err(error) => return Some((Err(error), cursor)),
}
}
}
async fn next_rocks_durable_change(
mut cursor: RocksOffsetCursor,
) -> Option<(PersistenceResult<DurableStateChange>, RocksOffsetCursor)> {
loop {
if let Some(RocksOffsetItem::DurableStateChange(change)) = cursor.buffered.pop_front() {
return Some((Ok(change), cursor));
}
if cursor.notifications.changed().await.is_err() {
return None;
}
let sender = cursor.sender.upgrade()?;
let (reply, receiver) = oneshot::channel();
let RocksOffsetQuery::DurableStateChanges(tag) = &cursor.query else {
unreachable!("durable cursor has durable query")
};
if sender
.send(Command::QueryDurableStateChanges {
tag: tag.clone(),
after_offset: cursor.after_offset,
reply,
})
.await
.is_err()
{
return Some((Err(rocks_query_channel_closed()), cursor));
}
let page = match receive_reply(receiver, "live durable-state changes query").await {
Ok(page) => page,
Err(error) => return Some((Err(error), cursor)),
};
cursor.after_offset = page.boundary;
cursor.buffered.extend(
page.items
.into_iter()
.map(RocksOffsetItem::DurableStateChange),
);
}
}
struct RocksPersistenceIdsCursor {
sender: mpsc::WeakSender<Command>,
notifications: watch::Receiver<u64>,
known: HashSet<PersistenceId>,
buffered: VecDeque<PersistenceId>,
}
async fn next_rocks_persistence_id_value(
mut cursor: RocksPersistenceIdsCursor,
) -> Option<(PersistenceResult<PersistenceId>, RocksPersistenceIdsCursor)> {
loop {
if let Some(persistence_id) = cursor.buffered.pop_front() {
return Some((Ok(persistence_id), cursor));
}
if cursor.notifications.changed().await.is_err() {
return None;
}
let sender = cursor.sender.upgrade()?;
let (reply, receiver) = oneshot::channel();
if sender
.send(Command::QueryPersistenceIds { reply })
.await
.is_err()
{
return Some((Err(rocks_query_channel_closed()), cursor));
}
let ids = match receive_reply(receiver, "live persistence-ids query").await {
Ok(ids) => ids,
Err(error) => return Some((Err(error), cursor)),
};
for persistence_id in ids {
if cursor.known.insert(persistence_id.clone()) {
cursor.buffered.push_back(persistence_id);
}
}
}
}
fn rocks_query_channel_closed() -> PersistenceError {
PersistenceError::Connection("RocksDB query command channel closed".to_owned())
}
fn rocks_sequence_offset(offset: Offset) -> PersistenceResult<u64> {
match offset {
Offset::NoOffset => Ok(0),
Offset::Sequence(value) => Ok(value),
Offset::Timestamp { .. } => Err(PersistenceError::Rejected(
"RocksDB queries require a Sequence offset produced by the same database".to_owned(),
)),
}
}
async fn receive_reply<T>(
receiver: oneshot::Receiver<PersistenceResult<T>>,
operation: &str,
) -> PersistenceResult<T> {
receiver.await.map_err(|_| {
PersistenceError::Connection(format!(
"RocksDB storage owner ended before replying to {operation}"
))
})?
}
async fn receive_projection_reply<T>(
receiver: oneshot::Receiver<ProjectionResult<T>>,
operation: &str,
) -> ProjectionResult<T> {
receiver.await.map_err(|_| {
ProjectionError::OffsetStore(format!(
"RocksDB storage owner ended before replying to {operation}"
))
})?
}
#[derive(Default)]
struct OwnerState {
heads: HashMap<PersistenceId, JournalHead>,
read_side_offset: Option<u64>,
}
impl OwnerState {
fn journal_head(
&mut self,
db: &DB,
persistence_id: &PersistenceId,
) -> PersistenceResult<JournalHead> {
if let Some(head) = self.heads.get(persistence_id) {
return Ok(*head);
}
let head = read_head(db, persistence_id)?;
self.heads.insert(persistence_id.clone(), head);
Ok(head)
}
fn read_side_offset(&mut self, db: &DB) -> PersistenceResult<u64> {
if let Some(offset) = self.read_side_offset {
return Ok(offset);
}
let offset = read_offset(db)?;
self.read_side_offset = Some(offset);
Ok(offset)
}
fn invalidate_all(&mut self) {
self.heads.clear();
self.read_side_offset = None;
}
}
struct AppendCommand {
writes: Vec<AtomicWrite>,
reply: oneshot::Sender<PersistenceResult<Vec<AppendOutcome>>>,
}
fn owner_main(
path: PathBuf,
durability: RocksDbDurability,
mut receiver: mpsc::Receiver<Command>,
startup: std_mpsc::SyncSender<PersistenceResult<()>>,
active_owner_threads: Arc<AtomicUsize>,
notifications: watch::Sender<u64>,
) {
let db = match open_database(&path) {
Ok(db) => db,
Err(error) => {
let _ = startup.send(Err(error));
return;
}
};
active_owner_threads.store(1, Ordering::Release);
let _active_guard = ActiveOwnerThread(active_owner_threads);
if startup.send(Ok(())).is_err() {
return;
}
let mut write_options = WriteOptions::default();
write_options.set_sync(durability == RocksDbDurability::Sync);
write_options.disable_wal(false);
let mut state = OwnerState::default();
let mut deferred = None;
loop {
let command = match deferred.take() {
Some(command) => command,
None => match receiver.blocking_recv() {
Some(command) => command,
None => break,
},
};
if let Command::Append { writes, reply } = command {
let (mut write_count, mut event_count) = append_command_budget(&writes);
let mut appends = vec![AppendCommand { writes, reply }];
while write_count < GROUP_COMMIT_MAX_WRITES && event_count < GROUP_COMMIT_MAX_EVENTS {
match receiver.try_recv() {
Ok(Command::Append { writes, reply }) => {
let (next_writes, next_events) = append_command_budget(&writes);
let fits = write_count
.checked_add(next_writes)
.is_some_and(|count| count <= GROUP_COMMIT_MAX_WRITES)
&& event_count
.checked_add(next_events)
.is_some_and(|count| count <= GROUP_COMMIT_MAX_EVENTS);
if !fits {
deferred = Some(Command::Append { writes, reply });
break;
}
write_count += next_writes;
event_count += next_events;
appends.push(AppendCommand { writes, reply });
}
Ok(barrier) => {
deferred = Some(barrier);
break;
}
Err(
mpsc::error::TryRecvError::Empty | mpsc::error::TryRecvError::Disconnected,
) => {
break;
}
}
}
process_append_group(&db, &write_options, &mut state, appends);
notifications.send_modify(|generation| *generation = generation.wrapping_add(1));
} else if process_command(&db, &write_options, &mut state, command) {
notifications.send_modify(|generation| *generation = generation.wrapping_add(1));
}
}
}
fn append_command_budget(writes: &[AtomicWrite]) -> (usize, usize) {
(
writes.len(),
writes.iter().fold(0_usize, |events, write| {
events.saturating_add(write.events().len())
}),
)
}
struct ActiveOwnerThread(Arc<AtomicUsize>);
impl Drop for ActiveOwnerThread {
fn drop(&mut self) {
self.0.store(0, Ordering::Release);
}
}
fn open_database(path: &Path) -> PersistenceResult<DB> {
let mut database_options = DbOptions::default();
database_options.create_if_missing(true);
database_options.create_missing_column_families(true);
database_options.set_max_background_jobs(2);
let descriptors = vec![
ColumnFamilyDescriptor::new("default", column_family_options(false)),
ColumnFamilyDescriptor::new(JOURNAL_CF, column_family_options(true)),
ColumnFamilyDescriptor::new(SNAPSHOTS_CF, column_family_options(true)),
ColumnFamilyDescriptor::new(DURABLE_STATE_CF, column_family_options(true)),
ColumnFamilyDescriptor::new(TAGS_CF, column_family_options(false)),
ColumnFamilyDescriptor::new(META_CF, column_family_options(false)),
ColumnFamilyDescriptor::new(PROJECTION_CF, column_family_options(false)),
];
DB::open_cf_descriptors(&database_options, path, descriptors).map_err(|error| {
PersistenceError::Connection(format!(
"failed to open RocksDB at {}: {error}",
path.display()
))
})
}
fn column_family_options(id_hash_prefix: bool) -> DbOptions {
let mut options = DbOptions::default();
options.set_compression_type(DBCompressionType::Lz4);
let mut table = BlockBasedOptions::default();
table.set_bloom_filter(10.0, false);
if id_hash_prefix {
options.set_prefix_extractor(SliceTransform::create_fixed_prefix(ID_HASH_BYTES));
table.set_whole_key_filtering(false);
}
options.set_block_based_table_factory(&table);
options
}
fn process_command(
db: &DB,
write_options: &WriteOptions,
state: &mut OwnerState,
command: Command,
) -> bool {
match command {
Command::Append { .. } => unreachable!("append commands are grouped by the owner loop"),
Command::HighestSequenceNr {
persistence_id,
reply,
} => {
let result = state
.journal_head(db, &persistence_id)
.map(|head| head.highest_sequence_nr);
let _ = reply.send(result);
false
}
Command::Replay { range, reply } => {
let _ = reply.send(replay_events(db, &range));
false
}
Command::DeleteJournalTo {
persistence_id,
inclusive,
reply,
} => {
let result = match state.journal_head(db, &persistence_id) {
Ok(head) => delete_journal_to(db, write_options, &persistence_id, inclusive, head),
Err(error) => Err(error),
};
match &result {
Ok(head) => {
state.heads.insert(persistence_id, *head);
}
Err(_) => state.invalidate_all(),
}
let _ = reply.send(result.map(|_| ()));
false
}
Command::LoadSnapshot {
persistence_id,
criteria,
reply,
} => {
let _ = reply.send(load_snapshot(db, &persistence_id, &criteria));
false
}
Command::SaveSnapshot {
metadata,
adapter_manifest,
state: snapshot_state,
reply,
} => {
let result = save_snapshot(
db,
write_options,
metadata,
adapter_manifest,
snapshot_state,
);
if result.is_err() {
state.invalidate_all();
}
let _ = reply.send(result);
false
}
Command::DeleteSnapshot { metadata, reply } => {
let result = delete_snapshot(db, write_options, &metadata);
if result.is_err() {
state.invalidate_all();
}
let _ = reply.send(result);
false
}
Command::DeleteSnapshotsMatching {
persistence_id,
criteria,
reply,
} => {
let result = delete_snapshots_matching(db, write_options, &persistence_id, &criteria);
if result.is_err() {
state.invalidate_all();
}
let _ = reply.send(result);
false
}
Command::GetDurableState {
persistence_id,
reply,
} => {
let _ = reply.send(get_durable_state(db, &persistence_id));
false
}
Command::UpsertDurableState {
persistence_id,
new_revision,
state: durable_state,
tag,
reply,
} => {
let result = write_durable_state(
db,
write_options,
state,
&persistence_id,
new_revision,
Some(durable_state),
tag,
);
if result.is_err() {
state.invalidate_all();
}
let _ = reply.send(result);
true
}
Command::DeleteDurableState {
persistence_id,
new_revision,
reply,
} => {
let result = write_durable_state(
db,
write_options,
state,
&persistence_id,
new_revision,
None,
None,
);
if result.is_err() {
state.invalidate_all();
}
let _ = reply.send(result);
true
}
Command::QueryEventsByPersistenceId {
persistence_id,
from_sequence_nr,
to_sequence_nr,
reply,
} => {
let result = match state.journal_head(db, &persistence_id) {
Ok(head) => query_events_by_persistence_id(
db,
&persistence_id,
from_sequence_nr,
to_sequence_nr,
head.highest_sequence_nr,
),
Err(error) => Err(error),
};
let _ = reply.send(result);
false
}
Command::QueryEventsByTag {
tag,
after_offset,
reply,
} => {
let result = match state.read_side_offset(db) {
Ok(boundary) => query_events_by_tag(db, state, &tag, after_offset, boundary),
Err(error) => Err(error),
};
let _ = reply.send(result);
false
}
Command::QueryPersistenceIds { reply } => {
let _ = reply.send(query_persistence_ids(db));
false
}
Command::QueryDurableStateChanges {
tag,
after_offset,
reply,
} => {
let result = match state.read_side_offset(db) {
Ok(boundary) => query_durable_state_changes(db, &tag, after_offset, boundary),
Err(error) => Err(error),
};
let _ = reply.send(result);
false
}
Command::ClaimProjection {
projection_id,
reply,
} => {
let _ = reply.send(claim_projection(db, write_options, &projection_id));
false
}
Command::LoadProjection {
projection_id,
reply,
} => {
let _ = reply.send(load_projection_cursor(db, &projection_id));
false
}
Command::SaveProjection {
projection_id,
generation,
expected_position,
cursor,
reply,
} => {
let _ = reply.send(save_projection_cursor(
db,
write_options,
&projection_id,
generation,
expected_position.as_ref(),
cursor,
));
false
}
Command::ClearProjection {
projection_id,
generation,
reply,
} => {
let _ = reply.send(clear_projection_cursor(
db,
write_options,
&projection_id,
generation,
));
false
}
Command::LoadProjectionPaused {
projection_id,
reply,
} => {
let _ = reply.send(load_projection_paused(db, &projection_id));
false
}
Command::SetProjectionPaused {
projection_id,
paused,
reply,
} => {
let _ = reply.send(set_projection_paused(
db,
write_options,
&projection_id,
paused,
));
false
}
#[cfg(test)]
Command::CrashProjectionSave {
projection_id,
generation,
expected_position,
cursor,
point,
} => crash_projection_save(
db,
write_options,
&projection_id,
generation,
expected_position.as_ref(),
cursor,
point,
),
}
}
struct AppendCaller {
reply: Option<oneshot::Sender<PersistenceResult<Vec<AppendOutcome>>>>,
outcomes: Vec<Option<AppendOutcome>>,
has_accepted: bool,
}
struct PreparedAtomicWrite {
persistence_id: PersistenceId,
head_key: Vec<u8>,
first_sequence_nr: SequenceNr,
last_sequence_nr: SequenceNr,
event_count: u64,
events: Vec<PreparedEvent>,
}
struct PreparedEvent {
journal_key: Vec<u8>,
journal_value: Vec<u8>,
tag_value: Vec<u8>,
tag_prefixes: Vec<Vec<u8>>,
}
fn process_append_group(
db: &DB,
write_options: &WriteOptions,
state: &mut OwnerState,
appends: Vec<AppendCommand>,
) {
let mut callers = Vec::with_capacity(appends.len());
let mut prepared = Vec::new();
for append in appends {
let caller_index = callers.len();
let mut caller = AppendCaller {
reply: Some(append.reply),
outcomes: vec![None; append.writes.len()],
has_accepted: false,
};
for (write_index, write) in append.writes.into_iter().enumerate() {
match prepare_atomic_write(write) {
Ok(write) => prepared.push((caller_index, write_index, write)),
Err(reason) => {
caller.outcomes[write_index] = Some(AppendOutcome::Rejected { reason });
}
}
}
callers.push(caller);
}
let journal = match cf(db, JOURNAL_CF) {
Ok(column_family) => column_family,
Err(error) => {
reply_append_group_error(&mut callers, error);
return;
}
};
let tags = match cf(db, TAGS_CF) {
Ok(column_family) => column_family,
Err(error) => {
reply_append_group_error(&mut callers, error);
return;
}
};
let meta = match cf(db, META_CF) {
Ok(column_family) => column_family,
Err(error) => {
reply_append_group_error(&mut callers, error);
return;
}
};
let mut batch = WriteBatch::default();
let mut shadow_heads = HashMap::new();
let mut committed_heads = HashMap::new();
let mut shadow_offset = None;
let mut accepted_writes = 0_usize;
for (caller_index, write_index, mut write) in prepared {
let head = match shadow_heads.get(&write.persistence_id) {
Some(head) => *head,
None => match state.journal_head(db, &write.persistence_id) {
Ok(head) => {
shadow_heads.insert(write.persistence_id.clone(), head);
head
}
Err(error) => {
reply_append_group_error(&mut callers, error);
return;
}
},
};
let Some(expected) = head.highest_sequence_nr.as_u64().checked_add(1) else {
callers[caller_index].outcomes[write_index] = Some(AppendOutcome::Rejected {
reason: format!(
"persistence id {} already reached the maximum sequence number",
write.persistence_id
),
});
continue;
};
if write.first_sequence_nr != SequenceNr::new(expected) {
callers[caller_index].outcomes[write_index] = Some(AppendOutcome::Rejected {
reason: format!(
"expected first sequence number {expected} for {}, got {}",
write.persistence_id, write.first_sequence_nr
),
});
continue;
}
let current_offset = match shadow_offset {
Some(offset) => offset,
None => match state.read_side_offset(db) {
Ok(offset) => {
shadow_offset = Some(offset);
offset
}
Err(error) => {
reply_append_group_error(&mut callers, error);
return;
}
},
};
let Some(final_offset) = current_offset.checked_add(write.event_count) else {
callers[caller_index].outcomes[write_index] = Some(AppendOutcome::Rejected {
reason: "RocksDB read-side offset counter is exhausted".to_owned(),
});
continue;
};
let mut event_offset = current_offset;
for event in &mut write.events {
event_offset += 1;
patch_event_offset(&mut event.journal_value, event_offset);
batch.put_cf(&journal, &event.journal_key, &event.journal_value);
if !event.tag_prefixes.is_empty() {
patch_tag_index_offset(&mut event.tag_value, event_offset);
for tag_prefix in &event.tag_prefixes {
batch.put_cf(
&tags,
tag_index_key_from_prefix(tag_prefix, event_offset),
&event.tag_value,
);
}
}
}
debug_assert_eq!(event_offset, final_offset);
let new_head = JournalHead {
highest_sequence_nr: write.last_sequence_nr,
deleted_to: head.deleted_to,
};
shadow_heads.insert(write.persistence_id.clone(), new_head);
committed_heads.insert(write.persistence_id, (write.head_key, new_head));
shadow_offset = Some(final_offset);
accepted_writes += 1;
callers[caller_index].outcomes[write_index] = Some(AppendOutcome::Accepted);
callers[caller_index].has_accepted = true;
}
for caller in callers.iter_mut().filter(|caller| !caller.has_accepted) {
reply_append_caller(caller, Ok(append_outcomes(caller)));
}
if accepted_writes == 0 {
return;
}
for (head_key, head) in committed_heads.values() {
batch.put_cf(&meta, head_key, encode_head(*head));
}
let final_offset = shadow_offset.expect("an accepted write advances the offset");
batch.put_cf(&meta, READ_SIDE_OFFSET_KEY, encode_offset(final_offset));
if let Err(error) = db.write_opt(batch, write_options) {
state.invalidate_all();
let error = PersistenceError::Failure(format!(
"RocksDB merged journal write failed; commit outcome is uncertain for every merged caller: {error}"
));
for caller in callers.iter_mut().filter(|caller| caller.has_accepted) {
reply_append_caller(caller, Err(error.clone()));
}
return;
}
for (persistence_id, (_, head)) in committed_heads {
state.heads.insert(persistence_id, head);
}
state.read_side_offset = Some(final_offset);
for caller in callers.iter_mut().filter(|caller| caller.has_accepted) {
reply_append_caller(caller, Ok(append_outcomes(caller)));
}
}
fn prepare_atomic_write(write: AtomicWrite) -> Result<PreparedAtomicWrite, String> {
let persistence_id = write.persistence_id().clone();
let head_key = head_key(&persistence_id)?;
let event_count = u64::try_from(write.events().len())
.map_err(|_| "atomic write contains too many events".to_owned())?;
let first_sequence_nr = write.events()[0].sequence_nr;
let last_sequence_nr = write.events()[write.events().len() - 1].sequence_nr;
let mut events = Vec::with_capacity(write.events().len());
for event in write.into_events() {
let journal_key = journal_key(&event.persistence_id, event.sequence_nr)?;
let journal_value = encode_event(&event, 0)?;
let tag_prefixes = event
.tags
.iter()
.map(tag_index_prefix)
.collect::<Result<Vec<_>, _>>()?;
let tag_value = if tag_prefixes.is_empty() {
Vec::new()
} else {
encode_tag_index_value(&journal_key, &event, 0)?
};
events.push(PreparedEvent {
journal_key,
journal_value,
tag_value,
tag_prefixes,
});
}
Ok(PreparedAtomicWrite {
persistence_id,
head_key,
first_sequence_nr,
last_sequence_nr,
event_count,
events,
})
}
fn reply_append_group_error(callers: &mut [AppendCaller], error: PersistenceError) {
for caller in callers {
if !caller.has_accepted && caller.outcomes.iter().all(Option::is_some) {
reply_append_caller(caller, Ok(append_outcomes(caller)));
} else {
reply_append_caller(caller, Err(error.clone()));
}
}
}
fn append_outcomes(caller: &AppendCaller) -> Vec<AppendOutcome> {
caller
.outcomes
.iter()
.cloned()
.map(|outcome| outcome.expect("every prepared write was validated"))
.collect()
}
fn reply_append_caller(caller: &mut AppendCaller, result: PersistenceResult<Vec<AppendOutcome>>) {
if let Some(reply) = caller.reply.take() {
let _ = reply.send(result);
}
}
fn replay_events(db: &DB, range: &ReplayRange) -> PersistenceResult<Vec<StoredEvent>> {
let prefix = id_prefix(&range.persistence_id).map_err(PersistenceError::Rejected)?;
let lower = sequence_key(&prefix, range.from);
let upper = if range.to.as_u64() == u64::MAX {
prefix_successor(&prefix)
} else {
sequence_key(&prefix, SequenceNr::new(range.to.as_u64() + 1))
};
let journal = cf(db, JOURNAL_CF)?;
let snapshot = db.snapshot();
let mut read_options = ReadOptions::default();
read_options.set_iterate_lower_bound(lower.clone());
read_options.set_iterate_upper_bound(upper);
let iterator = snapshot.iterator_cf_opt(
&journal,
read_options,
IteratorMode::From(&lower, Direction::Forward),
);
let limit = usize::try_from(range.max).unwrap_or(usize::MAX);
let mut events = Vec::with_capacity(limit.min(1024));
for item in iterator.take(limit) {
let (key, value) = item.map_err(rocksdb_read_error("journal replay iterator"))?;
let event = decode_indexed_event(&value)?.event;
if !journal_key_matches_prefix(&key, &prefix, event.sequence_nr)
|| event.persistence_id != range.persistence_id
{
return Err(PersistenceError::Serialization(
"RocksDB journal key does not match its stored event envelope".to_owned(),
));
}
events.push(event);
}
Ok(events)
}
fn query_events_by_persistence_id(
db: &DB,
persistence_id: &PersistenceId,
from_sequence_nr: SequenceNr,
to_sequence_nr: SequenceNr,
highest_sequence_nr: SequenceNr,
) -> PersistenceResult<PersistenceIdQueryPage> {
if from_sequence_nr > to_sequence_nr || from_sequence_nr > highest_sequence_nr {
return Ok(PersistenceIdQueryPage {
events: Vec::new(),
highest_sequence_nr,
});
}
let range = ReplayRange {
persistence_id: persistence_id.clone(),
from: from_sequence_nr,
to: to_sequence_nr.min(highest_sequence_nr),
max: u64::MAX,
};
let prefix = id_prefix(persistence_id).map_err(PersistenceError::Rejected)?;
let lower = sequence_key(&prefix, range.from);
let upper = if range.to.as_u64() == u64::MAX {
prefix_successor(&prefix)
} else {
sequence_key(&prefix, SequenceNr::new(range.to.as_u64() + 1))
};
let journal = cf(db, JOURNAL_CF)?;
let snapshot = db.snapshot();
let mut read_options = ReadOptions::default();
read_options.set_iterate_lower_bound(lower.clone());
read_options.set_iterate_upper_bound(upper);
let iterator = snapshot.iterator_cf_opt(
&journal,
read_options,
IteratorMode::From(&lower, Direction::Forward),
);
let mut events = Vec::new();
for item in iterator {
let (key, value) = item.map_err(rocksdb_read_error("per-id query iterator"))?;
let indexed = decode_indexed_event(&value)?;
if !journal_key_matches_prefix(&key, &prefix, indexed.event.sequence_nr)
|| indexed.event.persistence_id != *persistence_id
{
return Err(PersistenceError::Serialization(
"RocksDB per-id query crossed persistence ids".to_owned(),
));
}
events.push(rocks_event_envelope(indexed));
}
Ok(PersistenceIdQueryPage {
events,
highest_sequence_nr,
})
}
fn query_events_by_tag(
db: &DB,
state: &mut OwnerState,
tag: &Tag,
after_offset: u64,
boundary: u64,
) -> PersistenceResult<OffsetQueryPage<EventEnvelope>> {
let Some(first_offset) = after_offset.checked_add(1) else {
return Ok(OffsetQueryPage {
items: Vec::new(),
boundary,
});
};
if first_offset > boundary {
return Ok(OffsetQueryPage {
items: Vec::new(),
boundary,
});
}
let lower = tag_index_key(tag, first_offset).map_err(PersistenceError::Rejected)?;
let upper = if boundary == u64::MAX {
prefix_successor(&tag_index_prefix(tag).map_err(PersistenceError::Rejected)?)
} else {
tag_index_key(tag, boundary + 1).map_err(PersistenceError::Rejected)?
};
let tags = cf(db, TAGS_CF)?;
let journal = cf(db, JOURNAL_CF)?;
let snapshot = db.snapshot();
let mut read_options = ReadOptions::default();
read_options.set_iterate_lower_bound(lower.clone());
read_options.set_iterate_upper_bound(upper);
let iterator = snapshot.iterator_cf_opt(
&tags,
read_options,
IteratorMode::From(&lower, Direction::Forward),
);
let mut candidates = Vec::new();
for item in iterator {
let (key, value) = item.map_err(rocksdb_read_error("tag query iterator"))?;
let offset = tag_index_offset(tag, &key)?;
let tag_value = decode_tag_index_value(&value, offset)?;
let expected_key = journal_key(
&tag_value.envelope.persistence_id,
tag_value.envelope.sequence_nr,
)
.map_err(PersistenceError::Serialization)?;
if tag_value.journal_key != expected_key || !tag_value.envelope.tags.contains(tag) {
return Err(PersistenceError::Serialization(
"RocksDB tag index does not match its embedded journal event".to_owned(),
));
}
let head = state.journal_head(db, &tag_value.envelope.persistence_id)?;
if tag_value.envelope.sequence_nr <= head.deleted_to {
continue;
}
if tag_value.envelope.sequence_nr > head.highest_sequence_nr {
return Err(PersistenceError::Serialization(
"RocksDB tag index points beyond its journal head".to_owned(),
));
}
candidates.push(TagQueryCandidate {
journal_key: tag_value.journal_key,
envelope: tag_value.envelope,
requires_existence_check: head.deleted_to > SequenceNr::new(0),
});
}
let canonical_values = snapshot.multi_get_cf(
candidates
.iter()
.filter(|candidate| candidate.requires_existence_check)
.map(|candidate| (&journal, candidate.journal_key.as_slice())),
);
let mut items = Vec::with_capacity(candidates.len());
let mut canonical_values = canonical_values.into_iter();
for candidate in candidates {
if candidate.requires_existence_check {
let canonical = canonical_values
.next()
.expect("one canonical result exists per deletion-filtered candidate")
.map_err(rocksdb_read_error("tag query journal existence check"))?;
if canonical.is_none() {
continue;
}
}
items.push(candidate.envelope);
}
debug_assert!(canonical_values.next().is_none());
Ok(OffsetQueryPage { items, boundary })
}
struct TagQueryCandidate {
journal_key: Vec<u8>,
envelope: EventEnvelope,
requires_existence_check: bool,
}
fn query_persistence_ids(db: &DB) -> PersistenceResult<Vec<PersistenceId>> {
let meta = cf(db, META_CF)?;
let lower = [HEAD_KEY_MARKER];
let upper = [HEAD_KEY_MARKER + 1];
let mut read_options = ReadOptions::default();
read_options.set_iterate_lower_bound(lower);
read_options.set_iterate_upper_bound(upper);
let iterator = db.iterator_cf_opt(
&meta,
read_options,
IteratorMode::From(&lower, Direction::Forward),
);
let mut ids = Vec::new();
for item in iterator {
let (key, _) = item.map_err(rocksdb_read_error("persistence-ids iterator"))?;
let Some(prefix) = key.get(1..) else {
return Err(PersistenceError::Serialization(
"RocksDB journal-head key is truncated".to_owned(),
));
};
ids.push(persistence_id_from_prefix(prefix)?);
}
Ok(ids)
}
fn query_durable_state_changes(
db: &DB,
tag: &Tag,
after_offset: u64,
boundary: u64,
) -> PersistenceResult<OffsetQueryPage<DurableStateChange>> {
let durable = cf(db, DURABLE_STATE_CF)?;
let snapshot = db.snapshot();
let iterator = snapshot.iterator_cf(&durable, IteratorMode::Start);
let mut items = Vec::new();
for item in iterator {
let (key, value) = item.map_err(rocksdb_read_error("durable changes iterator"))?;
let persistence_id = persistence_id_from_prefix(&key)?;
let record = decode_durable_record(&value, &persistence_id)?;
if record.change_offset <= after_offset
|| record.change_offset > boundary
|| record.change_tag.as_ref() != Some(tag)
{
continue;
}
let offset = Offset::Sequence(record.change_offset);
let revision = record.versioned_state.revision;
let change = match record.versioned_state.state {
Some(state) => DurableStateChange::Updated(UpdatedDurableState {
offset,
persistence_id,
revision,
state,
timestamp_ms: record.change_timestamp_ms,
}),
None => DurableStateChange::Deleted(DeletedDurableState {
offset,
persistence_id,
revision,
timestamp_ms: record.change_timestamp_ms,
}),
};
items.push(change);
}
items.sort_by_key(DurableStateChange::offset);
Ok(OffsetQueryPage { items, boundary })
}
fn rocks_event_envelope(indexed: IndexedEvent) -> EventEnvelope {
EventEnvelope {
offset: Offset::Sequence(indexed.offset),
persistence_id: indexed.event.persistence_id,
sequence_nr: indexed.event.sequence_nr,
timestamp_ms: indexed.event.timestamp_ms,
adapter_manifest: indexed.event.adapter_manifest,
payload: indexed.event.payload,
metadata: indexed.event.metadata,
tags: indexed.event.tags,
}
}
fn delete_journal_to(
db: &DB,
write_options: &WriteOptions,
persistence_id: &PersistenceId,
inclusive: SequenceNr,
head: JournalHead,
) -> PersistenceResult<JournalHead> {
if head.highest_sequence_nr == SequenceNr::new(0) {
return Ok(head);
}
let prefix = id_prefix(persistence_id).map_err(PersistenceError::Rejected)?;
let from = sequence_key(&prefix, SequenceNr::new(0));
let to = if inclusive.as_u64() == u64::MAX {
prefix_successor(&prefix)
} else {
sequence_key(&prefix, SequenceNr::new(inclusive.as_u64() + 1))
};
let journal = cf(db, JOURNAL_CF)?;
let meta = cf(db, META_CF)?;
let mut batch = WriteBatch::default();
batch.delete_range_cf(&journal, from, to);
let new_head = JournalHead {
highest_sequence_nr: head.highest_sequence_nr,
deleted_to: SequenceNr::new(
head.deleted_to
.as_u64()
.max(inclusive.as_u64().min(head.highest_sequence_nr.as_u64())),
),
};
batch.put_cf(
&meta,
head_key(persistence_id).map_err(PersistenceError::Rejected)?,
encode_head(new_head),
);
db.write_opt(batch, write_options).map_err(|error| {
PersistenceError::Failure(format!(
"RocksDB hard journal delete failed; commit outcome is uncertain: {error}"
))
})?;
Ok(new_head)
}
fn read_head(db: &DB, persistence_id: &PersistenceId) -> PersistenceResult<JournalHead> {
let meta = cf(db, META_CF)?;
let key = head_key(persistence_id).map_err(PersistenceError::Rejected)?;
match db
.get_cf(&meta, key)
.map_err(rocksdb_read_error("journal head read"))?
{
Some(value) => decode_head(&value),
None => Ok(JournalHead::default()),
}
}
fn read_offset(db: &DB) -> PersistenceResult<u64> {
let meta = cf(db, META_CF)?;
match db
.get_cf(&meta, READ_SIDE_OFFSET_KEY)
.map_err(rocksdb_read_error("read-side offset read"))?
{
Some(value) => decode_offset(&value),
None => Ok(0),
}
}
fn load_snapshot(
db: &DB,
persistence_id: &PersistenceId,
criteria: &SnapshotCriteria,
) -> PersistenceResult<Option<SelectedSnapshot>> {
let prefix = id_prefix(persistence_id).map_err(PersistenceError::Rejected)?;
let lower = sequence_key(&prefix, criteria.min_sequence_nr);
let upper = if criteria.max_sequence_nr.as_u64() == u64::MAX {
prefix_successor(&prefix)
} else {
sequence_key(
&prefix,
SequenceNr::new(criteria.max_sequence_nr.as_u64() + 1),
)
};
let snapshots = cf(db, SNAPSHOTS_CF)?;
let snapshot = db.snapshot();
let mut read_options = ReadOptions::default();
read_options.set_iterate_lower_bound(lower.clone());
read_options.set_iterate_upper_bound(upper);
let iterator = snapshot.iterator_cf_opt(
&snapshots,
read_options,
IteratorMode::From(&lower, Direction::Forward),
);
let mut selected: Option<SelectedSnapshot> = None;
for item in iterator {
let (key, value) = item.map_err(rocksdb_read_error("snapshot iterator"))?;
let candidate = decode_snapshot(&value)?;
let expected_key = snapshot_key(
&candidate.metadata.persistence_id,
candidate.metadata.sequence_nr,
)
.map_err(PersistenceError::Serialization)?;
if key.as_ref() != expected_key || candidate.metadata.persistence_id != *persistence_id {
return Err(PersistenceError::Serialization(
"RocksDB snapshot key does not match its stored envelope".to_owned(),
));
}
if criteria.matches(&candidate.metadata)
&& selected.as_ref().is_none_or(|current| {
(
candidate.metadata.sequence_nr,
candidate.metadata.timestamp_ms,
) > (current.metadata.sequence_nr, current.metadata.timestamp_ms)
})
{
selected = Some(candidate);
}
}
Ok(selected)
}
fn save_snapshot(
db: &DB,
write_options: &WriteOptions,
metadata: SnapshotMetadata,
adapter_manifest: String,
state: SerializedPayload,
) -> PersistenceResult<()> {
let key = snapshot_key(&metadata.persistence_id, metadata.sequence_nr)
.map_err(PersistenceError::Rejected)?;
let value = encode_snapshot(&SelectedSnapshot {
metadata,
adapter_manifest,
state,
})
.map_err(PersistenceError::Rejected)?;
let snapshots = cf(db, SNAPSHOTS_CF)?;
let mut batch = WriteBatch::default();
batch.put_cf(&snapshots, key, value);
write_batch(db, write_options, batch, "snapshot save")
}
fn delete_snapshot(
db: &DB,
write_options: &WriteOptions,
metadata: &SnapshotMetadata,
) -> PersistenceResult<()> {
let key = snapshot_key(&metadata.persistence_id, metadata.sequence_nr)
.map_err(PersistenceError::Rejected)?;
let snapshots = cf(db, SNAPSHOTS_CF)?;
let Some(value) = db
.get_cf(&snapshots, &key)
.map_err(rocksdb_read_error("snapshot delete read"))?
else {
return Ok(());
};
if decode_snapshot(&value)?.metadata != *metadata {
return Ok(());
}
let mut batch = WriteBatch::default();
batch.delete_cf(&snapshots, key);
write_batch(db, write_options, batch, "snapshot delete")
}
fn delete_snapshots_matching(
db: &DB,
write_options: &WriteOptions,
persistence_id: &PersistenceId,
criteria: &SnapshotCriteria,
) -> PersistenceResult<()> {
let prefix = id_prefix(persistence_id).map_err(PersistenceError::Rejected)?;
let lower = sequence_key(&prefix, criteria.min_sequence_nr);
let upper = if criteria.max_sequence_nr.as_u64() == u64::MAX {
prefix_successor(&prefix)
} else {
sequence_key(
&prefix,
SequenceNr::new(criteria.max_sequence_nr.as_u64() + 1),
)
};
let snapshots = cf(db, SNAPSHOTS_CF)?;
let snapshot = db.snapshot();
let mut read_options = ReadOptions::default();
read_options.set_iterate_lower_bound(lower.clone());
read_options.set_iterate_upper_bound(upper);
let iterator = snapshot.iterator_cf_opt(
&snapshots,
read_options,
IteratorMode::From(&lower, Direction::Forward),
);
let mut keys = Vec::new();
for item in iterator {
let (key, value) = item.map_err(rocksdb_read_error("snapshot delete iterator"))?;
let candidate = decode_snapshot(&value)?;
if candidate.metadata.persistence_id != *persistence_id {
return Err(PersistenceError::Serialization(
"RocksDB snapshot range crossed persistence ids".to_owned(),
));
}
if criteria.matches(&candidate.metadata) {
keys.push(key.into_vec());
}
}
drop(snapshot);
if keys.is_empty() {
return Ok(());
}
let mut batch = WriteBatch::default();
for key in keys {
batch.delete_cf(&snapshots, key);
}
write_batch(db, write_options, batch, "snapshot range delete")
}
fn get_durable_state(db: &DB, persistence_id: &PersistenceId) -> PersistenceResult<VersionedState> {
Ok(read_durable_record(db, persistence_id)?.versioned_state)
}
fn read_durable_record(
db: &DB,
persistence_id: &PersistenceId,
) -> PersistenceResult<DurableRecord> {
let durable = cf(db, DURABLE_STATE_CF)?;
let key = durable_state_key(persistence_id).map_err(PersistenceError::Rejected)?;
match db
.get_cf(&durable, key)
.map_err(rocksdb_read_error("durable-state read"))?
{
Some(value) => decode_durable_record(&value, persistence_id),
None => Ok(DurableRecord::default()),
}
}
fn write_durable_state(
db: &DB,
write_options: &WriteOptions,
owner_state: &mut OwnerState,
persistence_id: &PersistenceId,
new_revision: Revision,
state: Option<SerializedPayload>,
tag: Option<Tag>,
) -> PersistenceResult<()> {
let stored = read_durable_record(db, persistence_id)?;
let expected = stored
.versioned_state
.revision
.as_u64()
.checked_add(1)
.map(Revision::new)
.ok_or_else(|| PersistenceError::RevisionExhausted {
persistence_id: persistence_id.clone(),
})?;
if new_revision != expected {
return Err(PersistenceError::RevisionConflict {
persistence_id: persistence_id.clone(),
expected,
actual: new_revision,
});
}
let current_offset = owner_state.read_side_offset(db)?;
let change_offset = current_offset.checked_add(1).ok_or_else(|| {
PersistenceError::Rejected("RocksDB read-side offset counter is exhausted".to_owned())
})?;
let change_tag = if state.is_some() {
tag.clone()
} else {
stored.change_tag
};
let key = durable_state_key(persistence_id).map_err(PersistenceError::Rejected)?;
let value = encode_durable_state(
persistence_id,
new_revision,
state.as_ref(),
change_tag.as_ref(),
change_offset,
unix_timestamp_ms(),
)
.map_err(PersistenceError::Rejected)?;
let durable = cf(db, DURABLE_STATE_CF)?;
let meta = cf(db, META_CF)?;
let mut batch = WriteBatch::default();
batch.put_cf(&durable, key, value);
batch.put_cf(&meta, READ_SIDE_OFFSET_KEY, encode_offset(change_offset));
write_batch(db, write_options, batch, "durable-state mutation")?;
owner_state.read_side_offset = Some(change_offset);
Ok(())
}
#[derive(Debug, Default)]
struct ProjectionEntry {
generation: ProjectionGeneration,
paused: bool,
cursor: Option<ProjectionCursor>,
}
fn claim_projection(
db: &DB,
write_options: &WriteOptions,
projection_id: &ProjectionId,
) -> ProjectionResult<ProjectionGeneration> {
let mut entry = read_projection_entry(db, projection_id)?;
let next = entry.generation.as_u64().checked_add(1).ok_or_else(|| {
ProjectionError::GenerationExhausted {
projection_id: projection_id.clone(),
}
})?;
entry.generation = ProjectionGeneration::new(next);
write_projection_entry(db, write_options, projection_id, &entry)?;
Ok(entry.generation)
}
fn load_projection_cursor(
db: &DB,
projection_id: &ProjectionId,
) -> ProjectionResult<Option<ProjectionCursor>> {
Ok(read_projection_entry(db, projection_id)?.cursor)
}
fn save_projection_cursor(
db: &DB,
write_options: &WriteOptions,
projection_id: &ProjectionId,
generation: ProjectionGeneration,
expected_position: Option<&CursorPosition>,
cursor: ProjectionCursor,
) -> ProjectionResult<()> {
let mut entry = read_projection_entry(db, projection_id)?;
validate_projection_generation(projection_id, generation, entry.generation)?;
let actual_position = entry.cursor.as_ref().map(ProjectionCursor::position);
if actual_position != expected_position {
return Err(ProjectionError::UnexpectedPosition {
projection_id: projection_id.clone(),
expected: expected_position.map(ToString::to_string),
actual: actual_position.map(ToString::to_string),
});
}
entry.cursor = Some(cursor);
write_projection_entry(db, write_options, projection_id, &entry)
}
fn clear_projection_cursor(
db: &DB,
write_options: &WriteOptions,
projection_id: &ProjectionId,
generation: ProjectionGeneration,
) -> ProjectionResult<()> {
let mut entry = read_projection_entry(db, projection_id)?;
validate_projection_generation(projection_id, generation, entry.generation)?;
entry.cursor = None;
write_projection_entry(db, write_options, projection_id, &entry)
}
fn load_projection_paused(db: &DB, projection_id: &ProjectionId) -> ProjectionResult<bool> {
Ok(read_projection_entry(db, projection_id)?.paused)
}
fn set_projection_paused(
db: &DB,
write_options: &WriteOptions,
projection_id: &ProjectionId,
paused: bool,
) -> ProjectionResult<()> {
let mut entry = read_projection_entry(db, projection_id)?;
entry.paused = paused;
write_projection_entry(db, write_options, projection_id, &entry)
}
fn validate_projection_generation(
projection_id: &ProjectionId,
attempted: ProjectionGeneration,
current: ProjectionGeneration,
) -> ProjectionResult<()> {
if attempted == ProjectionGeneration::default() || attempted != current {
return Err(ProjectionError::Fenced {
projection_id: projection_id.clone(),
attempted_generation: attempted.as_u64(),
current_generation: current.as_u64(),
});
}
Ok(())
}
fn read_projection_entry(
db: &DB,
projection_id: &ProjectionId,
) -> ProjectionResult<ProjectionEntry> {
let projection = projection_cf(db)?;
let key = projection_key(projection_id);
match db
.get_cf(&projection, key)
.map_err(|error| projection_read_error("projection row read", error))?
{
Some(value) => decode_projection_entry(&value),
None => Ok(ProjectionEntry::default()),
}
}
fn write_projection_entry(
db: &DB,
write_options: &WriteOptions,
projection_id: &ProjectionId,
entry: &ProjectionEntry,
) -> ProjectionResult<()> {
let projection = projection_cf(db)?;
let key = projection_key(projection_id);
let value = encode_projection_entry(entry)?;
db.put_cf_opt(&projection, key, value, write_options)
.map_err(|error| {
ProjectionError::OffsetStore(format!(
"RocksDB projection row write failed; commit outcome is uncertain: {error}"
))
})
}
fn projection_cf(db: &DB) -> ProjectionResult<ColumnFamilyRef<'_>> {
db.cf_handle(PROJECTION_CF).ok_or_else(|| {
ProjectionError::OffsetStore(format!(
"RocksDB column family {PROJECTION_CF:?} is missing"
))
})
}
fn projection_key(projection_id: &ProjectionId) -> Vec<u8> {
let name = projection_id.name().as_bytes();
let key = projection_id.key().as_bytes();
let mut encoded = Vec::with_capacity(
16_usize
.saturating_add(name.len())
.saturating_add(key.len()),
);
encoded.extend_from_slice(&(name.len() as u64).to_be_bytes());
encoded.extend_from_slice(name);
encoded.extend_from_slice(&(key.len() as u64).to_be_bytes());
encoded.extend_from_slice(key);
encoded
}
fn encode_projection_entry(entry: &ProjectionEntry) -> ProjectionResult<Vec<u8>> {
let mut writer = EnvelopeWriter::new();
writer.u8(PROJECTION_ENTRY_VERSION);
writer.u64(entry.generation.as_u64());
writer.u8(u8::from(entry.paused));
let cursor = entry.cursor.as_ref().map(ProjectionCursor::encode);
writer
.option(cursor.as_ref(), |writer, cursor| writer.string(cursor))
.map_err(|error| {
ProjectionError::CursorCodec(format!(
"RocksDB projection cursor cannot be encoded: {error}"
))
})?;
Ok(writer.finish())
}
fn decode_projection_entry(bytes: &[u8]) -> ProjectionResult<ProjectionEntry> {
let mut reader = EnvelopeReader::new(bytes, "projection offset");
let version = reader.u8().map_err(projection_codec_error)?;
if version != PROJECTION_ENTRY_VERSION {
return Err(ProjectionError::CursorCodec(format!(
"invalid RocksDB projection offset envelope: unsupported version {version}"
)));
}
let generation = reader.u64().map_err(projection_codec_error)?;
let paused = match reader.u8().map_err(projection_codec_error)? {
0 => false,
1 => true,
value => {
return Err(ProjectionError::CursorCodec(format!(
"invalid RocksDB projection offset envelope: invalid paused flag {value}"
)));
}
};
let cursor = reader
.option(EnvelopeReader::string)
.map_err(projection_codec_error)?;
reader.finish().map_err(projection_codec_error)?;
Ok(ProjectionEntry {
generation: ProjectionGeneration::new(generation),
paused,
cursor: cursor
.as_deref()
.map(ProjectionCursor::decode)
.transpose()?,
})
}
fn projection_codec_error(error: PersistenceError) -> ProjectionError {
ProjectionError::CursorCodec(format!("invalid RocksDB projection row: {error}"))
}
fn projection_read_error(operation: &str, error: rocksdb::Error) -> ProjectionError {
ProjectionError::OffsetStore(format!("RocksDB {operation} failed: {error}"))
}
#[cfg(test)]
fn crash_projection_save(
db: &DB,
write_options: &WriteOptions,
projection_id: &ProjectionId,
generation: ProjectionGeneration,
expected_position: Option<&CursorPosition>,
cursor: ProjectionCursor,
point: ProjectionSaveCrashPoint,
) -> ! {
const BEFORE_COMMIT_EXIT: i32 = 86;
const AFTER_COMMIT_EXIT: i32 = 87;
const INJECTION_FAILED_EXIT: i32 = 88;
if point == ProjectionSaveCrashPoint::BeforeCommit {
std::process::exit(BEFORE_COMMIT_EXIT);
}
if let Err(error) = save_projection_cursor(
db,
write_options,
projection_id,
generation,
expected_position,
cursor,
) {
eprintln!("RocksDB projection crash injection could not commit: {error}");
std::process::exit(INJECTION_FAILED_EXIT);
}
std::process::exit(AFTER_COMMIT_EXIT);
}
fn write_batch(
db: &DB,
write_options: &WriteOptions,
batch: WriteBatch,
operation: &str,
) -> PersistenceResult<()> {
db.write_opt(batch, write_options).map_err(|error| {
PersistenceError::Failure(format!(
"RocksDB {operation} failed; commit outcome is uncertain: {error}"
))
})
}
fn cf<'a>(db: &'a DB, name: &str) -> PersistenceResult<ColumnFamilyRef<'a>> {
db.cf_handle(name).ok_or_else(|| {
PersistenceError::Failure(format!("RocksDB column family {name:?} is missing"))
})
}
fn rocksdb_read_error(operation: &'static str) -> impl FnOnce(rocksdb::Error) -> PersistenceError {
move |error| PersistenceError::Failure(format!("RocksDB {operation} failed: {error}"))
}
#[derive(Debug, Clone, Copy, Default)]
struct JournalHead {
highest_sequence_nr: SequenceNr,
deleted_to: SequenceNr,
}
fn id_prefix(persistence_id: &PersistenceId) -> Result<Vec<u8>, String> {
let id = persistence_id.id().as_bytes();
let id_len = u32::try_from(id.len())
.map_err(|_| format!("persistence id {persistence_id} exceeds the RocksDB key limit"))?;
let mut key = Vec::with_capacity(ID_HASH_BYTES + 4 + id.len());
key.extend_from_slice(&fnv1a_64(id).to_be_bytes());
key.extend_from_slice(&id_len.to_be_bytes());
key.extend_from_slice(id);
Ok(key)
}
fn persistence_id_from_prefix(prefix: &[u8]) -> PersistenceResult<PersistenceId> {
let length_bytes: [u8; 4] = prefix
.get(ID_HASH_BYTES..ID_HASH_BYTES + 4)
.ok_or_else(|| {
PersistenceError::Serialization("RocksDB persistence-id key is truncated".to_owned())
})?
.try_into()
.expect("slice length was checked");
let id_len = u32::from_be_bytes(length_bytes) as usize;
let id_start = ID_HASH_BYTES + 4;
let id_end = id_start.checked_add(id_len).ok_or_else(|| {
PersistenceError::Serialization("RocksDB persistence-id key length overflow".to_owned())
})?;
if prefix.len() != id_end {
return Err(PersistenceError::Serialization(
"RocksDB persistence-id key has an invalid framed length".to_owned(),
));
}
let id = std::str::from_utf8(&prefix[id_start..id_end]).map_err(|error| {
PersistenceError::Serialization(format!("RocksDB persistence-id key is not UTF-8: {error}"))
})?;
let persistence_id = PersistenceId::from_unique(id.to_owned())?;
let expected = id_prefix(&persistence_id).map_err(PersistenceError::Serialization)?;
if expected != prefix {
return Err(PersistenceError::Serialization(
"RocksDB persistence-id key hash does not match its raw id".to_owned(),
));
}
Ok(persistence_id)
}
fn sequence_key(prefix: &[u8], sequence_nr: SequenceNr) -> Vec<u8> {
let mut key = Vec::with_capacity(prefix.len() + 8);
key.extend_from_slice(prefix);
key.extend_from_slice(&sequence_nr.as_u64().to_be_bytes());
key
}
fn journal_key_matches_prefix(key: &[u8], prefix: &[u8], sequence_nr: SequenceNr) -> bool {
let sequence = sequence_nr.as_u64().to_be_bytes();
key.strip_prefix(prefix) == Some(sequence.as_slice())
}
fn journal_key(persistence_id: &PersistenceId, sequence_nr: SequenceNr) -> Result<Vec<u8>, String> {
Ok(sequence_key(&id_prefix(persistence_id)?, sequence_nr))
}
fn snapshot_key(
persistence_id: &PersistenceId,
sequence_nr: SequenceNr,
) -> Result<Vec<u8>, String> {
journal_key(persistence_id, sequence_nr)
}
fn durable_state_key(persistence_id: &PersistenceId) -> Result<Vec<u8>, String> {
id_prefix(persistence_id)
}
fn head_key(persistence_id: &PersistenceId) -> Result<Vec<u8>, String> {
let prefix = id_prefix(persistence_id)?;
let mut key = Vec::with_capacity(prefix.len() + 1);
key.push(HEAD_KEY_MARKER);
key.extend_from_slice(&prefix);
Ok(key)
}
fn prefix_successor(prefix: &[u8]) -> Vec<u8> {
let mut upper = prefix.to_vec();
for index in (0..upper.len()).rev() {
if upper[index] != u8::MAX {
upper[index] += 1;
upper.truncate(index + 1);
return upper;
}
}
unreachable!("framed UTF-8 id prefix always has a lexicographic successor")
}
fn tag_index_key(tag: &Tag, offset: u64) -> Result<Vec<u8>, String> {
Ok(tag_index_key_from_prefix(&tag_index_prefix(tag)?, offset))
}
fn tag_index_key_from_prefix(prefix: &[u8], offset: u64) -> Vec<u8> {
let mut key = Vec::with_capacity(prefix.len() + 8);
key.extend_from_slice(prefix);
key.reserve(8);
key.extend_from_slice(&offset.to_be_bytes());
key
}
fn tag_index_prefix(tag: &Tag) -> Result<Vec<u8>, String> {
let bytes = tag.as_str().as_bytes();
let len = u32::try_from(bytes.len())
.map_err(|_| "journal tag exceeds the RocksDB key limit".to_owned())?;
let mut prefix = Vec::with_capacity(4 + bytes.len());
prefix.extend_from_slice(&len.to_be_bytes());
prefix.extend_from_slice(bytes);
Ok(prefix)
}
fn tag_index_offset(tag: &Tag, key: &[u8]) -> PersistenceResult<u64> {
let prefix = tag_index_prefix(tag).map_err(PersistenceError::Serialization)?;
let offset_bytes: [u8; 8] = key
.strip_prefix(prefix.as_slice())
.filter(|suffix| suffix.len() == 8)
.ok_or_else(|| {
PersistenceError::Serialization(
"RocksDB tag-index key does not match the requested tag".to_owned(),
)
})?
.try_into()
.expect("suffix length was checked");
Ok(u64::from_be_bytes(offset_bytes))
}
const fn fnv1a_64(bytes: &[u8]) -> u64 {
let mut hash = 0xcbf2_9ce4_8422_2325_u64;
let mut index = 0;
while index < bytes.len() {
hash ^= bytes[index] as u64;
hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
index += 1;
}
hash
}
fn encode_event(event: &StoredEvent, offset: u64) -> Result<Vec<u8>, String> {
let mut writer = EnvelopeWriter::new();
writer.u8(ENVELOPE_VERSION);
writer.u64(offset);
writer.string(event.persistence_id.id())?;
writer.u64(event.sequence_nr.as_u64());
writer.string(&event.adapter_manifest)?;
writer.string(&event.writer_uuid)?;
writer.i64(event.timestamp_ms);
writer.u32_len(event.tags.len(), "journal tag count")?;
for tag in &event.tags {
writer.string(tag.as_str())?;
}
writer.option(event.metadata.as_ref(), |writer, metadata| {
writer.payload_frame(metadata)
})?;
writer.payload_frame(&event.payload)?;
Ok(writer.finish())
}
fn patch_event_offset(event_value: &mut [u8], offset: u64) {
debug_assert_eq!(event_value.first(), Some(&ENVELOPE_VERSION));
event_value
.get_mut(ENVELOPE_OFFSET_START..ENVELOPE_OFFSET_END)
.expect("a freshly encoded event contains its fixed-width offset")
.copy_from_slice(&offset.to_be_bytes());
}
fn patch_tag_index_offset(tag_value: &mut [u8], offset: u64) {
debug_assert_eq!(tag_value.first(), Some(&TAG_INDEX_ENVELOPE_VERSION));
tag_value
.get_mut(ENVELOPE_OFFSET_START..ENVELOPE_OFFSET_END)
.expect("a freshly encoded tag index contains its fixed-width offset")
.copy_from_slice(&offset.to_be_bytes());
}
struct IndexedEvent {
offset: u64,
event: StoredEvent,
}
fn decode_indexed_event(bytes: &[u8]) -> PersistenceResult<IndexedEvent> {
let mut reader = EnvelopeReader::new(bytes, "journal event");
reader.version()?;
let offset = reader.u64()?;
let persistence_id = reader.persistence_id()?;
let sequence_nr = SequenceNr::new(reader.u64()?);
let adapter_manifest = reader.string()?;
let writer_uuid = reader.string()?;
let timestamp_ms = reader.i64()?;
let tag_count = reader.u32()? as usize;
let mut tags = Vec::with_capacity(tag_count.min(1024));
for _ in 0..tag_count {
tags.push(Tag::new(reader.string()?));
}
let metadata = reader.option(EnvelopeReader::payload_frame)?;
let payload = reader.payload_frame()?;
reader.finish()?;
Ok(IndexedEvent {
offset,
event: StoredEvent {
persistence_id,
sequence_nr,
adapter_manifest,
writer_uuid,
timestamp_ms,
metadata,
payload,
tags,
},
})
}
fn encode_snapshot(snapshot: &SelectedSnapshot) -> Result<Vec<u8>, String> {
let mut writer = EnvelopeWriter::new();
writer.u8(ENVELOPE_VERSION);
writer.string(snapshot.metadata.persistence_id.id())?;
writer.u64(snapshot.metadata.sequence_nr.as_u64());
writer.i64(snapshot.metadata.timestamp_ms);
writer.string(&snapshot.adapter_manifest)?;
writer.payload_frame(&snapshot.state)?;
Ok(writer.finish())
}
fn decode_snapshot(bytes: &[u8]) -> PersistenceResult<SelectedSnapshot> {
let mut reader = EnvelopeReader::new(bytes, "snapshot");
reader.version()?;
let persistence_id = reader.persistence_id()?;
let sequence_nr = SequenceNr::new(reader.u64()?);
let timestamp_ms = reader.i64()?;
let adapter_manifest = reader.string()?;
let state = reader.payload_frame()?;
reader.finish()?;
Ok(SelectedSnapshot {
metadata: SnapshotMetadata {
persistence_id,
sequence_nr,
timestamp_ms,
},
adapter_manifest,
state,
})
}
fn encode_durable_state(
persistence_id: &PersistenceId,
revision: Revision,
state: Option<&SerializedPayload>,
change_tag: Option<&Tag>,
change_offset: u64,
change_timestamp_ms: i64,
) -> Result<Vec<u8>, String> {
let mut writer = EnvelopeWriter::new();
writer.u8(ENVELOPE_VERSION);
writer.string(persistence_id.id())?;
writer.u64(revision.as_u64());
writer.option(state, |writer, state| writer.payload_frame(state))?;
writer.option(change_tag, |writer, tag| writer.string(tag.as_str()))?;
writer.u64(change_offset);
writer.i64(change_timestamp_ms);
Ok(writer.finish())
}
#[derive(Default)]
struct DurableRecord {
versioned_state: VersionedState,
change_tag: Option<Tag>,
change_offset: u64,
change_timestamp_ms: i64,
}
fn decode_durable_record(
bytes: &[u8],
expected_id: &PersistenceId,
) -> PersistenceResult<DurableRecord> {
let mut reader = EnvelopeReader::new(bytes, "durable state");
reader.version()?;
let persistence_id = reader.persistence_id()?;
if persistence_id != *expected_id {
return Err(PersistenceError::Serialization(format!(
"RocksDB durable-state key for {expected_id} contains envelope for {persistence_id}"
)));
}
let revision = Revision::new(reader.u64()?);
let state = reader.option(EnvelopeReader::payload_frame)?;
let change_tag = reader.option(|reader| reader.string().map(Tag::new))?;
let change_offset = reader.u64()?;
let change_timestamp_ms = reader.i64()?;
reader.finish()?;
Ok(DurableRecord {
versioned_state: VersionedState {
tag: state.as_ref().and(change_tag.clone()),
state,
revision,
},
change_tag,
change_offset,
change_timestamp_ms,
})
}
fn encode_head(head: JournalHead) -> Vec<u8> {
let mut writer = EnvelopeWriter::new();
writer.u8(ENVELOPE_VERSION);
writer.u64(head.highest_sequence_nr.as_u64());
writer.u64(head.deleted_to.as_u64());
writer.finish()
}
fn decode_head(bytes: &[u8]) -> PersistenceResult<JournalHead> {
let mut reader = EnvelopeReader::new(bytes, "journal head");
reader.version()?;
let head = JournalHead {
highest_sequence_nr: SequenceNr::new(reader.u64()?),
deleted_to: SequenceNr::new(reader.u64()?),
};
reader.finish()?;
Ok(head)
}
fn encode_offset(offset: u64) -> Vec<u8> {
let mut writer = EnvelopeWriter::new();
writer.u8(ENVELOPE_VERSION);
writer.u64(offset);
writer.finish()
}
fn decode_offset(bytes: &[u8]) -> PersistenceResult<u64> {
let mut reader = EnvelopeReader::new(bytes, "read-side offset");
reader.version()?;
let offset = reader.u64()?;
reader.finish()?;
Ok(offset)
}
fn encode_tag_index_value(
journal_key: &[u8],
event: &StoredEvent,
offset: u64,
) -> Result<Vec<u8>, String> {
let mut writer = EnvelopeWriter::new();
writer.u8(TAG_INDEX_ENVELOPE_VERSION);
writer.u64(offset);
writer.bytes(journal_key)?;
writer.string(event.persistence_id.id())?;
writer.u64(event.sequence_nr.as_u64());
writer.i64(event.timestamp_ms);
writer.string(&event.adapter_manifest)?;
writer.payload_frame(&event.payload)?;
writer.option(event.metadata.as_ref(), |writer, metadata| {
writer.payload_frame(metadata)
})?;
writer.u32_len(event.tags.len(), "tag-index tag count")?;
for tag in &event.tags {
writer.string(tag.as_str())?;
}
Ok(writer.finish())
}
struct TagIndexValue {
journal_key: Vec<u8>,
envelope: EventEnvelope,
}
fn decode_tag_index_value(bytes: &[u8], offset: u64) -> PersistenceResult<TagIndexValue> {
let mut reader = EnvelopeReader::new(bytes, "tag index");
let version = reader.u8()?;
if version != TAG_INDEX_ENVELOPE_VERSION {
return Err(reader.error(format!("unsupported version {version}")));
}
let embedded_offset = reader.u64()?;
if embedded_offset != offset {
return Err(reader.error(format!(
"embedded offset {embedded_offset} does not match tag key offset {offset}"
)));
}
let journal_key = reader.bytes()?;
let persistence_id = reader.persistence_id()?;
let sequence_nr = SequenceNr::new(reader.u64()?);
let timestamp_ms = reader.i64()?;
let adapter_manifest = reader.string()?;
let payload = reader.payload_frame()?;
let metadata = reader.option(EnvelopeReader::payload_frame)?;
let tag_count = reader.u32()? as usize;
let mut tags = Vec::with_capacity(tag_count.min(1024));
for _ in 0..tag_count {
tags.push(Tag::new(reader.string()?));
}
reader.finish()?;
Ok(TagIndexValue {
journal_key,
envelope: EventEnvelope {
offset: Offset::Sequence(offset),
persistence_id,
sequence_nr,
timestamp_ms,
adapter_manifest,
payload,
metadata,
tags,
},
})
}
fn unix_timestamp_ms() -> i64 {
let millis = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis();
i64::try_from(millis).unwrap_or(i64::MAX)
}
struct EnvelopeWriter {
bytes: Vec<u8>,
}
impl EnvelopeWriter {
fn new() -> Self {
Self { bytes: Vec::new() }
}
fn finish(self) -> Vec<u8> {
self.bytes
}
fn u8(&mut self, value: u8) {
self.bytes.push(value);
}
fn u32(&mut self, value: u32) {
self.bytes.extend_from_slice(&value.to_be_bytes());
}
fn u64(&mut self, value: u64) {
self.bytes.extend_from_slice(&value.to_be_bytes());
}
fn i64(&mut self, value: i64) {
self.bytes.extend_from_slice(&value.to_be_bytes());
}
fn u32_len(&mut self, len: usize, field: &str) -> Result<(), String> {
let len = u32::try_from(len).map_err(|_| format!("{field} exceeds the u32 frame limit"))?;
self.u32(len);
Ok(())
}
fn bytes(&mut self, bytes: &[u8]) -> Result<(), String> {
self.u32_len(bytes.len(), "byte field")?;
self.bytes.extend_from_slice(bytes);
Ok(())
}
fn string(&mut self, value: &str) -> Result<(), String> {
self.u32_len(value.len(), "string field")?;
self.bytes.extend_from_slice(value.as_bytes());
Ok(())
}
fn option<T>(
&mut self,
value: Option<&T>,
write: impl FnOnce(&mut Self, &T) -> Result<(), String>,
) -> Result<(), String> {
match value {
Some(value) => {
self.u8(1);
write(self, value)
}
None => {
self.u8(0);
Ok(())
}
}
}
fn payload_frame(&mut self, payload: &SerializedPayload) -> Result<(), String> {
self.u32(payload.serializer_id.as_u32());
self.string(&payload.manifest)?;
self.bytes(&payload.payload)
}
}
struct EnvelopeReader<'a> {
bytes: &'a [u8],
cursor: usize,
kind: &'static str,
}
impl<'a> EnvelopeReader<'a> {
fn new(bytes: &'a [u8], kind: &'static str) -> Self {
Self {
bytes,
cursor: 0,
kind,
}
}
fn error(&self, message: impl std::fmt::Display) -> PersistenceError {
PersistenceError::Serialization(format!(
"invalid RocksDB {} envelope: {message}",
self.kind
))
}
fn take(&mut self, len: usize) -> PersistenceResult<&'a [u8]> {
let end = self
.cursor
.checked_add(len)
.ok_or_else(|| self.error("field length overflow"))?;
let value = self
.bytes
.get(self.cursor..end)
.ok_or_else(|| self.error("truncated field"))?;
self.cursor = end;
Ok(value)
}
fn u8(&mut self) -> PersistenceResult<u8> {
Ok(self.take(1)?[0])
}
fn u32(&mut self) -> PersistenceResult<u32> {
let bytes: [u8; 4] = self
.take(4)?
.try_into()
.map_err(|_| self.error("invalid u32"))?;
Ok(u32::from_be_bytes(bytes))
}
fn u64(&mut self) -> PersistenceResult<u64> {
let bytes: [u8; 8] = self
.take(8)?
.try_into()
.map_err(|_| self.error("invalid u64"))?;
Ok(u64::from_be_bytes(bytes))
}
fn i64(&mut self) -> PersistenceResult<i64> {
let bytes: [u8; 8] = self
.take(8)?
.try_into()
.map_err(|_| self.error("invalid i64"))?;
Ok(i64::from_be_bytes(bytes))
}
fn bytes(&mut self) -> PersistenceResult<Vec<u8>> {
let len = self.u32()? as usize;
Ok(self.take(len)?.to_vec())
}
fn string(&mut self) -> PersistenceResult<String> {
String::from_utf8(self.bytes()?).map_err(|error| self.error(error))
}
fn persistence_id(&mut self) -> PersistenceResult<PersistenceId> {
let id = self.string()?;
PersistenceId::from_unique(id).map_err(|error| self.error(error))
}
fn option<T>(
&mut self,
read: impl FnOnce(&mut Self) -> PersistenceResult<T>,
) -> PersistenceResult<Option<T>> {
match self.u8()? {
0 => Ok(None),
1 => read(self).map(Some),
value => Err(self.error(format!("invalid option discriminant {value}"))),
}
}
fn payload_frame(&mut self) -> PersistenceResult<SerializedPayload> {
let serializer_id = SerializerId::new(self.u32()?);
let manifest = self.string()?;
let payload = self.bytes()?;
Ok(SerializedPayload::new(serializer_id, manifest, payload))
}
fn version(&mut self) -> PersistenceResult<()> {
let version = self.u8()?;
if version != ENVELOPE_VERSION {
return Err(self.error(format!("unsupported version {version}")));
}
Ok(())
}
fn finish(&self) -> PersistenceResult<()> {
if self.cursor != self.bytes.len() {
return Err(self.error(format!("{} trailing bytes", self.bytes.len() - self.cursor)));
}
Ok(())
}
}
#[cfg(test)]
#[path = "rocksdb_projection_tests.rs"]
mod projection_tests;