#![forbid(unsafe_code)]
use std::{
collections::{HashMap, HashSet, VecDeque},
fmt,
sync::Arc,
time::{Duration, SystemTime, UNIX_EPOCH},
};
use futures_util::{StreamExt, stream};
use scylla::{
client::{session::Session, session_builder::SessionBuilder},
response::{PagingState, PagingStateResponse, query_result::QueryResult},
serialize::{
SerializationError,
row::{RowSerializationContext, SerializeRow},
writers::RowWriter,
},
statement::{
Consistency, SerialConsistency,
batch::{Batch, BatchType},
prepared::PreparedStatement,
},
value::{CqlValue, Row},
};
use tokio::sync::OnceCell;
use crate::{
AppendOutcome, AtomicWrite, DurableStateStore, EventStream, Journal, PersistenceError,
PersistenceId, PersistenceResult, ReplayRange, Revision, SelectedSnapshot, SequenceNr,
SerializedPayload, SerializerId, SnapshotCriteria, SnapshotMetadata, SnapshotStore,
StoredEvent, Tag, VersionedState,
query::{
DeletedDurableState, DurableStateChange, DurableStateChangesQuery, EventEnvelope,
EventsByPersistenceIdQuery, EventsByTagQuery, Offset, PersistenceIdsQuery, QueryMode,
QueryStream, UpdatedDurableState,
},
};
const DEFAULT_REPLICATION_FACTOR: u32 = 1;
const DEFAULT_PARTITION_SIZE: u64 = 100_000;
const DEFAULT_REPLAY_PAGE_SIZE: i32 = 256;
const DEFAULT_POLL_MIN_INTERVAL: Duration = Duration::from_millis(50);
const DEFAULT_POLL_MAX_INTERVAL: Duration = Duration::from_secs(2);
const DEFAULT_POLL_BATCH_SIZE: u32 = 256;
const MAX_CQL_IDENTIFIER_BYTES: usize = 48;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CassandraPollingConfig {
pub min_interval: Duration,
pub max_interval: Duration,
pub batch_size: u32,
}
impl Default for CassandraPollingConfig {
fn default() -> Self {
Self {
min_interval: DEFAULT_POLL_MIN_INTERVAL,
max_interval: DEFAULT_POLL_MAX_INTERVAL,
batch_size: DEFAULT_POLL_BATCH_SIZE,
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
pub enum CassandraWriteMode {
#[default]
Lwt,
SingleWriter,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CassandraTableNames {
pub event_journal: String,
pub journal_head: String,
pub snapshot: String,
pub durable_state: String,
pub tag_events: String,
pub durable_state_changes: String,
pub durable_state_change_log: String,
}
impl Default for CassandraTableNames {
fn default() -> Self {
Self {
event_journal: "event_journal".into(),
journal_head: "journal_head".into(),
snapshot: "snapshot".into(),
durable_state: "durable_state".into(),
tag_events: "tag_events".into(),
durable_state_changes: "durable_state_changes".into(),
durable_state_change_log: "durable_state_change_log".into(),
}
}
}
#[derive(Clone, PartialEq, Eq)]
pub struct CassandraPersistenceConfig {
pub contact_points: Vec<String>,
pub keyspace: String,
pub replication_factor: u32,
pub tables: CassandraTableNames,
pub partition_size: u64,
pub write_consistency: Consistency,
pub read_consistency: Consistency,
pub serial_consistency: SerialConsistency,
pub replay_page_size: i32,
pub polling: CassandraPollingConfig,
pub write_mode: CassandraWriteMode,
}
impl fmt::Debug for CassandraPersistenceConfig {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("CassandraPersistenceConfig")
.field("contact_points", &self.contact_points)
.field("keyspace", &self.keyspace)
.field("replication_factor", &self.replication_factor)
.field("tables", &self.tables)
.field("partition_size", &self.partition_size)
.field("write_consistency", &self.write_consistency)
.field("read_consistency", &self.read_consistency)
.field("serial_consistency", &self.serial_consistency)
.field("replay_page_size", &self.replay_page_size)
.field("polling", &self.polling)
.field("write_mode", &self.write_mode)
.finish()
}
}
impl CassandraPersistenceConfig {
#[must_use]
pub fn new(contact_point: impl Into<String>, keyspace: impl Into<String>) -> Self {
Self {
contact_points: vec![contact_point.into()],
keyspace: keyspace.into(),
replication_factor: DEFAULT_REPLICATION_FACTOR,
tables: CassandraTableNames::default(),
partition_size: DEFAULT_PARTITION_SIZE,
write_consistency: Consistency::LocalQuorum,
read_consistency: Consistency::LocalQuorum,
serial_consistency: SerialConsistency::LocalSerial,
replay_page_size: DEFAULT_REPLAY_PAGE_SIZE,
polling: CassandraPollingConfig::default(),
write_mode: CassandraWriteMode::default(),
}
}
}
pub struct CassandraPersistence {
config: CassandraPersistenceConfig,
schema: CassandraSchema,
connected: OnceCell<Arc<ConnectedCassandra>>,
}
impl fmt::Debug for CassandraPersistence {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("CassandraPersistence")
.field("config", &self.config)
.field("connected", &self.connected.get().is_some())
.finish_non_exhaustive()
}
}
impl CassandraPersistence {
pub fn new(config: CassandraPersistenceConfig) -> PersistenceResult<Self> {
validate_config(&config)?;
let schema = CassandraSchema::new(&config);
Ok(Self {
config,
schema,
connected: OnceCell::new(),
})
}
pub async fn connect(config: CassandraPersistenceConfig) -> PersistenceResult<Self> {
let backend = Self::new(config)?;
backend.initialize().await?;
Ok(backend)
}
pub async fn initialize(&self) -> PersistenceResult<()> {
self.connected
.get_or_try_init(|| connect_cassandra(&self.config, &self.schema))
.await
.map(|_| ())
}
#[must_use]
pub fn config(&self) -> &CassandraPersistenceConfig {
&self.config
}
#[must_use]
pub fn schema_ddl(&self) -> String {
let mut ddl = self.schema.ddl.join(";\n");
ddl.push(';');
ddl
}
async fn ready(&self) -> PersistenceResult<Arc<ConnectedCassandra>> {
self.initialize().await?;
self.connected.get().cloned().ok_or_else(|| {
PersistenceError::Connection(
"Cassandra session unavailable after successful initialization".into(),
)
})
}
async fn append_write(&self, write: AtomicWrite) -> PersistenceResult<AppendOutcome> {
let prepared = match PreparedAtomicWrite::new(write, self.config.partition_size) {
Ok(prepared) => prepared,
Err(reason) => return Ok(AppendOutcome::Rejected { reason }),
};
let connected = self.ready().await?;
let reconciled = read_reconciled_head(
&connected,
&prepared.persistence_id,
self.config.partition_size,
)
.await?;
let head = repair_reconciled_head(
&connected,
&prepared.persistence_id,
reconciled,
self.config.write_mode,
"repair Cassandra journal head before append",
)
.await?;
let expected = head
.highest_seq_nr
.checked_add(1)
.ok_or_else(|| PersistenceError::Failure("Cassandra journal head exhausted".into()))?;
if prepared.first_sequence_nr != expected {
return Ok(AppendOutcome::Rejected {
reason: format!(
"expected first sequence number {expected} after journal head {}, got {}",
head.highest_seq_nr, prepared.first_sequence_nr
),
});
}
let mut batch = Batch::new(BatchType::Unlogged);
batch.set_consistency(self.config.write_consistency);
for _ in &prepared.events {
batch.append_statement(connected.statements.insert_event.clone());
}
for _ in &prepared.tag_events {
batch.append_statement(connected.statements.insert_tag_event.clone());
}
let mut values = Vec::with_capacity(prepared.events.len() + prepared.tag_events.len());
values.extend(prepared.events.into_iter().map(JournalBatchValues::Event));
values.extend(prepared.tag_events.into_iter().map(JournalBatchValues::Tag));
connected
.session
.batch(&batch, values)
.await
.map_err(|error| storage_error("execute atomic Cassandra journal batch", error))?;
transition_head(
&connected,
&prepared.persistence_id,
head,
prepared.last_sequence_nr,
head.logical_deleted_to(),
self.config.write_mode,
"advance Cassandra journal head after append",
)
.await?;
Ok(AppendOutcome::Accepted)
}
async fn revision_conflict(
&self,
persistence_id: &PersistenceId,
actual: Revision,
) -> PersistenceError {
match self.current_revision(persistence_id).await {
Ok(stored) if stored.as_u64() == i64::MAX as u64 => {
PersistenceError::RevisionExhausted {
persistence_id: persistence_id.clone(),
}
}
Ok(stored) => match stored.as_u64().checked_add(1) {
Some(expected) => PersistenceError::RevisionConflict {
persistence_id: persistence_id.clone(),
expected: Revision::new(expected),
actual,
},
None => PersistenceError::RevisionExhausted {
persistence_id: persistence_id.clone(),
},
},
Err(error) => error,
}
}
async fn current_revision(
&self,
persistence_id: &PersistenceId,
) -> PersistenceResult<Revision> {
Ok(self.get(persistence_id).await?.revision)
}
}
#[async_trait::async_trait]
impl Journal for CassandraPersistence {
async fn append(&self, writes: Vec<AtomicWrite>) -> PersistenceResult<Vec<AppendOutcome>> {
let mut outcomes = Vec::with_capacity(writes.len());
for write in writes {
outcomes.push(self.append_write(write).await?);
}
Ok(outcomes)
}
async fn append_one(&self, write: AtomicWrite) -> PersistenceResult<AppendOutcome> {
self.append_write(write).await
}
async fn highest_sequence_nr(
&self,
persistence_id: &PersistenceId,
_from_hint: SequenceNr,
) -> PersistenceResult<SequenceNr> {
let connected = self.ready().await?;
let head =
read_reconciled_head(&connected, persistence_id, self.config.partition_size).await?;
Ok(SequenceNr::new(non_negative_u64(
head.highest_seq_nr,
"Cassandra journal head",
)?))
}
async fn replay(&self, range: ReplayRange) -> PersistenceResult<EventStream> {
if range.from > range.to || range.max == 0 || range.from.as_u64() > i64::MAX as u64 {
return Ok(Box::pin(stream::empty()));
}
let connected = self.ready().await?;
let head = read_reconciled_head(
&connected,
&range.persistence_id,
self.config.partition_size,
)
.await?;
let highest = SequenceNr::new(non_negative_u64(
head.highest_seq_nr,
"Cassandra replay journal head",
)?);
let to = range.to.min(highest);
if range.from > to {
return Ok(Box::pin(stream::empty()));
}
Ok(cassandra_event_stream(
connected,
range.persistence_id,
range.from,
to,
range.max,
head.logical_deleted_to(),
self.config.partition_size,
))
}
async fn delete_to(
&self,
persistence_id: &PersistenceId,
inclusive: SequenceNr,
) -> PersistenceResult<()> {
let connected = self.ready().await?;
let reconciled =
read_reconciled_head(&connected, persistence_id, self.config.partition_size).await?;
let head = repair_reconciled_head(
&connected,
persistence_id,
reconciled,
self.config.write_mode,
"repair Cassandra journal head before delete",
)
.await?;
if head.highest_seq_nr == 0 {
return Ok(());
}
let inclusive = i64::try_from(inclusive.as_u64()).unwrap_or(i64::MAX);
let effective = inclusive.min(head.highest_seq_nr);
if effective <= head.logical_deleted_to() {
return Ok(());
}
let first_partition = partition_nr(
u64::try_from(head.logical_deleted_to().saturating_add(1)).unwrap_or_default(),
self.config.partition_size,
);
let last_partition = partition_nr(
u64::try_from(effective).expect("positive Cassandra sequence fits u64"),
self.config.partition_size,
);
for partition in first_partition..=last_partition {
connected
.session
.execute_unpaged(
&connected.statements.delete_event_range,
(persistence_id.id(), partition, effective),
)
.await
.map_err(|error| {
storage_error("delete Cassandra journal partition range", error)
})?;
}
transition_head(
&connected,
persistence_id,
head,
head.highest_seq_nr,
effective,
self.config.write_mode,
"advance Cassandra journal deletion marker",
)
.await?;
Ok(())
}
}
#[async_trait::async_trait]
impl SnapshotStore for CassandraPersistence {
async fn load(
&self,
persistence_id: &PersistenceId,
criteria: &SnapshotCriteria,
) -> PersistenceResult<Option<SelectedSnapshot>> {
let Some((min_sequence_nr, max_sequence_nr)) = snapshot_sequence_bounds(criteria) else {
return Ok(None);
};
if criteria.min_timestamp_ms > criteria.max_timestamp_ms {
return Ok(None);
}
let connected = self.ready().await?;
let result = connected
.session
.execute_unpaged(
&connected.statements.load_snapshot,
(
persistence_id.id(),
min_sequence_nr,
max_sequence_nr,
criteria.min_timestamp_ms,
criteria.max_timestamp_ms,
),
)
.await
.map_err(|error| storage_error("load Cassandra snapshot", error))?;
let rows = result
.into_rows_result()
.map_err(|error| storage_error("decode Cassandra snapshot result", error))?;
let row = rows
.maybe_first_row::<SnapshotRow>()
.map_err(|error| storage_error("decode Cassandra snapshot row", error))?;
row.map(|row| decode_snapshot(persistence_id, row))
.transpose()
}
async fn save(
&self,
metadata: SnapshotMetadata,
adapter_manifest: String,
state: SerializedPayload,
) -> PersistenceResult<()> {
let sequence_nr = sql_i64(metadata.sequence_nr.as_u64(), "snapshot sequence number")?;
let connected = self.ready().await?;
connected
.session
.execute_unpaged(
&connected.statements.save_snapshot,
(
metadata.persistence_id.id(),
sequence_nr,
metadata.timestamp_ms,
adapter_manifest,
i64::from(state.serializer_id.as_u32()),
state.manifest,
state.payload,
),
)
.await
.map_err(|error| storage_error("save Cassandra snapshot", error))?;
Ok(())
}
async fn delete(&self, metadata: SnapshotMetadata) -> PersistenceResult<()> {
let Ok(sequence_nr) = i64::try_from(metadata.sequence_nr.as_u64()) else {
return Ok(());
};
let connected = self.ready().await?;
connected
.session
.execute_unpaged(
&connected.statements.delete_snapshot_exact,
(
metadata.persistence_id.id(),
sequence_nr,
metadata.timestamp_ms,
),
)
.await
.map_err(|error| storage_error("delete exact Cassandra snapshot", error))?;
Ok(())
}
async fn delete_matching(
&self,
persistence_id: &PersistenceId,
criteria: &SnapshotCriteria,
) -> PersistenceResult<()> {
let Some((min_sequence_nr, max_sequence_nr)) = snapshot_sequence_bounds(criteria) else {
return Ok(());
};
if criteria.min_timestamp_ms > criteria.max_timestamp_ms {
return Ok(());
}
let connected = self.ready().await?;
let pager = connected
.session
.execute_iter(
connected.statements.select_matching_snapshots.clone(),
(
persistence_id.id(),
min_sequence_nr,
max_sequence_nr,
criteria.min_timestamp_ms,
criteria.max_timestamp_ms,
),
)
.await
.map_err(|error| storage_error("select matching Cassandra snapshots", error))?;
let mut rows = pager
.rows_stream::<SnapshotMetadataRow>()
.map_err(|error| storage_error("decode matching Cassandra snapshots", error))?;
while let Some(row) = rows.next().await {
let row = row
.map_err(|error| storage_error("decode matching Cassandra snapshot row", error))?;
connected
.session
.execute_unpaged(
&connected.statements.delete_snapshot_exact,
(persistence_id.id(), row.sequence_nr, row.timestamp_ms),
)
.await
.map_err(|error| storage_error("delete matching Cassandra snapshot", error))?;
}
Ok(())
}
}
#[async_trait::async_trait]
impl DurableStateStore for CassandraPersistence {
async fn get(&self, persistence_id: &PersistenceId) -> PersistenceResult<VersionedState> {
let connected = self.ready().await?;
let result = connected
.session
.execute_unpaged(
&connected.statements.get_durable_state,
(persistence_id.id(),),
)
.await
.map_err(|error| storage_error("get Cassandra durable state", error))?;
let rows = result
.into_rows_result()
.map_err(|error| storage_error("decode Cassandra durable-state result", error))?;
rows.maybe_first_row::<DurableStateRow>()
.map_err(|error| storage_error("decode Cassandra durable-state row", error))?
.map_or(Ok(VersionedState::default()), decode_durable_state)
}
async fn upsert(
&self,
persistence_id: &PersistenceId,
new_revision: Revision,
state: SerializedPayload,
tag: Option<Tag>,
) -> PersistenceResult<()> {
let Ok(revision) = i64::try_from(new_revision.as_u64()) else {
return Err(self.revision_conflict(persistence_id, new_revision).await);
};
if revision == 0 {
return Err(self.revision_conflict(persistence_id, new_revision).await);
}
let connected = self.ready().await?;
let previous_tag = if revision > 1 {
read_raw_durable_tag(&connected, persistence_id).await?
} else {
None
};
let change_state = tag.as_ref().map(|_| state.clone());
let result = if revision == 1 {
connected
.session
.execute_unpaged(
&connected.statements.insert_durable_state,
(
persistence_id.id(),
revision,
i64::from(state.serializer_id.as_u32()),
state.manifest,
state.payload,
tag.as_ref().map(Tag::as_str),
),
)
.await
} else {
connected
.session
.execute_unpaged(
&connected.statements.update_durable_state,
(
revision,
i64::from(state.serializer_id.as_u32()),
state.manifest,
state.payload,
tag.as_ref().map(Tag::as_str),
persistence_id.id(),
revision - 1,
),
)
.await
}
.map_err(|error| storage_error("upsert Cassandra durable state with LWT", error))?;
if !lwt_applied(result)? {
return Err(self.revision_conflict(persistence_id, new_revision).await);
}
if let Some(old_tag) = &previous_tag
&& Some(old_tag) != tag.as_ref()
{
remove_durable_state_change(&connected, old_tag, persistence_id).await?;
}
if let (Some(tag), Some(change_state)) = (tag, change_state) {
write_durable_state_upsert_change(
&connected,
&tag,
persistence_id,
new_revision,
change_state,
)
.await?;
}
Ok(())
}
async fn delete(
&self,
persistence_id: &PersistenceId,
new_revision: Revision,
) -> PersistenceResult<()> {
let Ok(revision) = i64::try_from(new_revision.as_u64()) else {
return Err(self.revision_conflict(persistence_id, new_revision).await);
};
if revision == 0 {
return Err(self.revision_conflict(persistence_id, new_revision).await);
}
let connected = self.ready().await?;
let result = if revision == 1 {
connected
.session
.execute_unpaged(
&connected.statements.insert_durable_tombstone,
(persistence_id.id(), revision),
)
.await
} else {
connected
.session
.execute_unpaged(
&connected.statements.update_durable_tombstone,
(revision, persistence_id.id(), revision - 1),
)
.await
}
.map_err(|error| storage_error("delete Cassandra durable state with LWT", error))?;
if !lwt_applied(result)? {
return Err(self.revision_conflict(persistence_id, new_revision).await);
}
if let Some(tag) = read_raw_durable_tag(&connected, persistence_id).await? {
write_durable_state_delete_change(&connected, &tag, persistence_id, new_revision)
.await?;
}
Ok(())
}
}
#[async_trait::async_trait]
impl EventsByPersistenceIdQuery for CassandraPersistence {
async fn events_by_persistence_id(
&self,
persistence_id: &PersistenceId,
from_sequence_nr: SequenceNr,
to_sequence_nr: SequenceNr,
mode: QueryMode,
) -> PersistenceResult<QueryStream<EventEnvelope>> {
if from_sequence_nr > to_sequence_nr || from_sequence_nr.as_u64() > i64::MAX as u64 {
return Ok(Box::pin(stream::empty()));
}
let connected = self.ready().await?;
let head =
read_reconciled_head(&connected, persistence_id, self.config.partition_size).await?;
let highest = SequenceNr::new(non_negative_u64(
head.highest_seq_nr,
"Cassandra query journal head",
)?);
let boundary = highest.min(to_sequence_nr);
let initial = (from_sequence_nr <= boundary).then(|| {
cassandra_event_stream(
connected.clone(),
persistence_id.clone(),
from_sequence_nr,
boundary,
u64::MAX,
head.logical_deleted_to(),
self.config.partition_size,
)
});
if mode == QueryMode::Current {
return Ok(initial.map_or_else(
|| Box::pin(stream::empty()) as QueryStream<EventEnvelope>,
|events| Box::pin(events.map(|event| event.map(event_envelope))),
));
}
let after_highest = highest.as_u64().checked_add(1).map(SequenceNr::new);
let next_sequence_nr = after_highest.map(|next| next.max(from_sequence_nr));
let cursor = CassandraLiveCursor {
connected,
persistence_id: persistence_id.clone(),
initial,
next_sequence_nr,
to_sequence_nr,
reached_end: highest >= to_sequence_nr,
partition_size: self.config.partition_size,
polling: self.config.polling.clone(),
delay: self.config.polling.min_interval,
buffered: VecDeque::new(),
};
Ok(Box::pin(stream::unfold(cursor, next_cassandra_live_event)))
}
}
#[async_trait::async_trait]
impl PersistenceIdsQuery for CassandraPersistence {
async fn persistence_ids(
&self,
mode: QueryMode,
) -> PersistenceResult<QueryStream<PersistenceId>> {
if mode == QueryMode::Live {
return Err(PersistenceError::Rejected(
"Cassandra live persistence-id discovery is unsupported; use QueryMode::Current"
.into(),
));
}
let connected = self.ready().await?;
let result = connected
.session
.execute_unpaged(&connected.statements.select_persistence_ids, ())
.await
.map_err(|error| storage_error("query Cassandra persistence ids", error))?;
let rows = result
.into_rows_result()
.map_err(|error| storage_error("decode Cassandra persistence ids", error))?;
let persistence_ids = rows
.rows::<PersistenceIdRow>()
.map_err(|error| storage_error("decode Cassandra persistence-id rows", error))?
.map(|row| {
row.map_err(|error| storage_error("decode Cassandra persistence-id row", error))
.and_then(|row| PersistenceId::from_unique(row.persistence_id))
})
.collect::<Vec<_>>();
Ok(Box::pin(stream::iter(persistence_ids)))
}
}
#[async_trait::async_trait]
impl EventsByTagQuery for CassandraPersistence {
async fn events_by_tag(
&self,
tag: &Tag,
offset: Offset,
mode: QueryMode,
) -> PersistenceResult<QueryStream<EventEnvelope>> {
let resume = cassandra_tag_resume(offset)?;
let connected = self.ready().await?;
let events = cassandra_events_by_tag(&connected, tag, &resume, None).await?;
if mode == QueryMode::Current {
return Ok(Box::pin(stream::iter(events.into_iter().map(Ok))));
}
let resume = events
.last()
.map(cassandra_tag_resume_after)
.unwrap_or(resume);
let cursor = CassandraTagCursor {
connected,
tag: tag.clone(),
resume,
buffered: VecDeque::from(events),
polling: self.config.polling.clone(),
delay: self.config.polling.min_interval,
};
Ok(Box::pin(stream::unfold(cursor, next_cassandra_tag_event)))
}
}
#[async_trait::async_trait]
impl DurableStateChangesQuery for CassandraPersistence {
async fn changes(
&self,
tag: &Tag,
offset: Offset,
mode: QueryMode,
) -> PersistenceResult<QueryStream<DurableStateChange>> {
let since_timestamp_ms = cassandra_change_since_timestamp_ms(offset)?;
let connected = self.ready().await?;
let bootstrap_timestamp_ms = current_timestamp_ms();
let changes =
cassandra_current_durable_state_changes(&connected, tag, since_timestamp_ms).await?;
if mode == QueryMode::Current {
return Ok(Box::pin(stream::iter(changes.into_iter().map(Ok))));
}
let cursor = CassandraDurableChangeLogCursor {
connected,
tag: tag.clone(),
resume: CassandraDurableChangeLogResume::Inclusive {
timestamp_ms: bootstrap_timestamp_ms,
revision: i64::MIN,
},
buffered: VecDeque::from(changes),
polling: self.config.polling.clone(),
delay: self.config.polling.min_interval,
};
Ok(Box::pin(stream::unfold(
cursor,
next_cassandra_durable_change_log,
)))
}
}
#[derive(Debug)]
struct ConnectedCassandra {
session: Arc<Session>,
statements: CassandraStatements,
}
#[derive(Debug)]
struct CassandraStatements {
insert_event: PreparedStatement,
select_head: PreparedStatement,
select_partition_highest: PreparedStatement,
insert_head: PreparedStatement,
update_head: PreparedStatement,
insert_head_plain: PreparedStatement,
update_head_plain: PreparedStatement,
replay_partition: PreparedStatement,
delete_event_range: PreparedStatement,
load_snapshot: PreparedStatement,
save_snapshot: PreparedStatement,
delete_snapshot_exact: PreparedStatement,
select_matching_snapshots: PreparedStatement,
get_durable_state: PreparedStatement,
insert_durable_state: PreparedStatement,
update_durable_state: PreparedStatement,
insert_durable_tombstone: PreparedStatement,
update_durable_tombstone: PreparedStatement,
select_persistence_ids: PreparedStatement,
insert_tag_event: PreparedStatement,
select_tag_events_all: PreparedStatement,
select_tag_events_inclusive: PreparedStatement,
select_tag_events_after: PreparedStatement,
upsert_durable_state_change: PreparedStatement,
delete_durable_state_change: PreparedStatement,
remove_durable_state_change: PreparedStatement,
select_durable_state_changes: PreparedStatement,
insert_durable_state_change_log: PreparedStatement,
select_durable_state_change_log_inclusive: PreparedStatement,
select_durable_state_change_log_after: PreparedStatement,
}
#[derive(Debug)]
struct CassandraSchema {
event_journal: String,
journal_head: String,
snapshot: String,
durable_state: String,
tag_events: String,
durable_state_changes: String,
durable_state_change_log: String,
ddl: Vec<String>,
}
impl CassandraSchema {
fn new(config: &CassandraPersistenceConfig) -> Self {
let event_journal = qualified(&config.keyspace, &config.tables.event_journal);
let journal_head = qualified(&config.keyspace, &config.tables.journal_head);
let snapshot = qualified(&config.keyspace, &config.tables.snapshot);
let durable_state = qualified(&config.keyspace, &config.tables.durable_state);
let tag_events = qualified(&config.keyspace, &config.tables.tag_events);
let durable_state_changes =
qualified(&config.keyspace, &config.tables.durable_state_changes);
let durable_state_change_log =
qualified(&config.keyspace, &config.tables.durable_state_change_log);
let ddl = vec![
format!(
"CREATE KEYSPACE IF NOT EXISTS {} WITH replication = {{'class': \
'SimpleStrategy', 'replication_factor': {}}} AND durable_writes = true",
config.keyspace, config.replication_factor
),
format!(
"CREATE TABLE IF NOT EXISTS {event_journal} (\
persistence_id text, partition_nr bigint, sequence_nr bigint, writer text, \
adapter_manifest text, ser_id bigint, ser_manifest text, payload blob, \
meta_ser_id bigint, meta_manifest text, meta_payload blob, tags set<text>, \
timestamp_ms bigint, PRIMARY KEY ((persistence_id, partition_nr), sequence_nr)) \
WITH CLUSTERING ORDER BY (sequence_nr ASC)"
),
format!(
"CREATE TABLE IF NOT EXISTS {journal_head} (persistence_id text PRIMARY KEY, \
highest_seq_nr bigint, deleted_to bigint)"
),
format!(
"CREATE TABLE IF NOT EXISTS {snapshot} (persistence_id text, sequence_nr bigint, \
timestamp_ms bigint, adapter_manifest text, ser_id bigint, ser_manifest text, \
payload blob, PRIMARY KEY (persistence_id, sequence_nr)) \
WITH CLUSTERING ORDER BY (sequence_nr DESC)"
),
format!(
"CREATE TABLE IF NOT EXISTS {durable_state} (persistence_id text PRIMARY KEY, \
revision bigint, ser_id bigint, ser_manifest text, payload blob, tag text)"
),
format!(
"CREATE TABLE IF NOT EXISTS {tag_events} (\
tag text, timestamp_ms bigint, sequence_nr bigint, persistence_id text, \
writer text, adapter_manifest text, ser_id bigint, ser_manifest text, \
payload blob, meta_ser_id bigint, meta_manifest text, meta_payload blob, \
tags set<text>, \
PRIMARY KEY (tag, timestamp_ms, sequence_nr, persistence_id)) \
WITH CLUSTERING ORDER BY (timestamp_ms ASC, sequence_nr ASC, persistence_id ASC)"
),
format!(
"CREATE TABLE IF NOT EXISTS {durable_state_changes} (\
tag text, persistence_id text, revision bigint, timestamp_ms bigint, \
ser_id bigint, ser_manifest text, payload blob, deleted boolean, \
PRIMARY KEY (tag, persistence_id))"
),
format!(
"CREATE TABLE IF NOT EXISTS {durable_state_change_log} (\
tag text, timestamp_ms bigint, revision bigint, persistence_id text, \
ser_id bigint, ser_manifest text, payload blob, deleted boolean, \
PRIMARY KEY (tag, timestamp_ms, revision, persistence_id)) \
WITH CLUSTERING ORDER BY (timestamp_ms ASC, revision ASC, persistence_id ASC)"
),
];
Self {
event_journal,
journal_head,
snapshot,
durable_state,
tag_events,
durable_state_changes,
durable_state_change_log,
ddl,
}
}
}
async fn connect_cassandra(
config: &CassandraPersistenceConfig,
schema: &CassandraSchema,
) -> PersistenceResult<Arc<ConnectedCassandra>> {
let session = SessionBuilder::new()
.known_nodes(config.contact_points.iter().map(String::as_str))
.build()
.await
.map_err(|error| connection_error("connect Cassandra session", error))?;
let session = Arc::new(session);
for ddl in &schema.ddl {
session
.query_unpaged(ddl.as_str(), ())
.await
.map_err(|error| storage_error("execute Cassandra schema bootstrap", error))?;
}
let statements = prepare_statements(&session, config, schema).await?;
Ok(Arc::new(ConnectedCassandra {
session,
statements,
}))
}
async fn prepare_statements(
session: &Session,
config: &CassandraPersistenceConfig,
schema: &CassandraSchema,
) -> PersistenceResult<CassandraStatements> {
let insert_event = prepare(
session,
format!(
"INSERT INTO {} (persistence_id, partition_nr, sequence_nr, writer, \
adapter_manifest, ser_id, ser_manifest, payload, meta_ser_id, meta_manifest, \
meta_payload, tags, timestamp_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
schema.event_journal
),
config.write_consistency,
None,
None,
)
.await?;
let select_head = prepare(
session,
format!(
"SELECT highest_seq_nr, deleted_to FROM {} WHERE persistence_id = ?",
schema.journal_head
),
config.read_consistency,
None,
None,
)
.await?;
let select_partition_highest = prepare(
session,
format!(
"SELECT sequence_nr FROM {} WHERE persistence_id = ? AND partition_nr = ? \
AND sequence_nr > ? ORDER BY sequence_nr DESC LIMIT 1",
schema.event_journal
),
config.read_consistency,
None,
None,
)
.await?;
let insert_head = prepare(
session,
format!(
"INSERT INTO {} (persistence_id, highest_seq_nr, deleted_to) VALUES (?, ?, ?) \
IF NOT EXISTS",
schema.journal_head
),
config.write_consistency,
None,
Some(config.serial_consistency),
)
.await?;
let update_head = prepare(
session,
format!(
"UPDATE {} SET highest_seq_nr = ?, deleted_to = ? WHERE persistence_id = ? \
IF highest_seq_nr = ? AND deleted_to = ?",
schema.journal_head
),
config.write_consistency,
None,
Some(config.serial_consistency),
)
.await?;
let replay_partition = prepare(
session,
format!(
"SELECT sequence_nr, writer, adapter_manifest, ser_id, ser_manifest, payload, \
meta_ser_id, meta_manifest, meta_payload, tags, timestamp_ms FROM {} \
WHERE persistence_id = ? AND partition_nr = ? \
AND sequence_nr >= ? AND sequence_nr <= ?",
schema.event_journal
),
config.read_consistency,
Some(config.replay_page_size),
None,
)
.await?;
let delete_event_range = prepare(
session,
format!(
"DELETE FROM {} WHERE persistence_id = ? AND partition_nr = ? \
AND sequence_nr <= ?",
schema.event_journal
),
config.write_consistency,
None,
None,
)
.await?;
let load_snapshot = prepare(
session,
format!(
"SELECT sequence_nr, timestamp_ms, adapter_manifest, ser_id, ser_manifest, payload \
FROM {} WHERE persistence_id = ? AND sequence_nr >= ? AND sequence_nr <= ? \
AND timestamp_ms >= ? AND timestamp_ms <= ? LIMIT 1 ALLOW FILTERING",
schema.snapshot
),
config.read_consistency,
None,
None,
)
.await?;
let save_snapshot = prepare(
session,
format!(
"INSERT INTO {} (persistence_id, sequence_nr, timestamp_ms, adapter_manifest, \
ser_id, ser_manifest, payload) VALUES (?, ?, ?, ?, ?, ?, ?)",
schema.snapshot
),
config.write_consistency,
None,
None,
)
.await?;
let delete_snapshot_exact = prepare(
session,
format!(
"DELETE FROM {} WHERE persistence_id = ? AND sequence_nr = ? IF timestamp_ms = ?",
schema.snapshot
),
config.write_consistency,
None,
Some(config.serial_consistency),
)
.await?;
let select_matching_snapshots = prepare(
session,
format!(
"SELECT sequence_nr, timestamp_ms FROM {} WHERE persistence_id = ? \
AND sequence_nr >= ? AND sequence_nr <= ? AND timestamp_ms >= ? \
AND timestamp_ms <= ? ALLOW FILTERING",
schema.snapshot
),
config.read_consistency,
Some(config.replay_page_size),
None,
)
.await?;
let get_durable_state = prepare(
session,
format!(
"SELECT revision, ser_id, ser_manifest, payload, tag FROM {} WHERE persistence_id = ?",
schema.durable_state
),
config.read_consistency,
None,
None,
)
.await?;
let insert_durable_state = prepare(
session,
format!(
"INSERT INTO {} (persistence_id, revision, ser_id, ser_manifest, payload, tag) \
VALUES (?, ?, ?, ?, ?, ?) IF NOT EXISTS",
schema.durable_state
),
config.write_consistency,
None,
Some(config.serial_consistency),
)
.await?;
let update_durable_state = prepare(
session,
format!(
"UPDATE {} SET revision = ?, ser_id = ?, ser_manifest = ?, payload = ?, tag = ? \
WHERE persistence_id = ? IF revision = ?",
schema.durable_state
),
config.write_consistency,
None,
Some(config.serial_consistency),
)
.await?;
let insert_durable_tombstone = prepare(
session,
format!(
"INSERT INTO {} (persistence_id, revision) VALUES (?, ?) IF NOT EXISTS",
schema.durable_state
),
config.write_consistency,
None,
Some(config.serial_consistency),
)
.await?;
let update_durable_tombstone = prepare(
session,
format!(
"UPDATE {} SET revision = ?, ser_id = null, ser_manifest = null, payload = null \
WHERE persistence_id = ? IF revision = ?",
schema.durable_state
),
config.write_consistency,
None,
Some(config.serial_consistency),
)
.await?;
let select_persistence_ids = prepare(
session,
format!(
"SELECT DISTINCT persistence_id FROM {}",
schema.journal_head
),
config.read_consistency,
Some(config.replay_page_size),
None,
)
.await?;
let insert_head_plain = prepare(
session,
format!(
"INSERT INTO {} (persistence_id, highest_seq_nr, deleted_to) VALUES (?, ?, ?)",
schema.journal_head
),
config.write_consistency,
None,
None,
)
.await?;
let update_head_plain = prepare(
session,
format!(
"UPDATE {} SET highest_seq_nr = ?, deleted_to = ? WHERE persistence_id = ?",
schema.journal_head
),
config.write_consistency,
None,
None,
)
.await?;
let insert_tag_event = prepare(
session,
format!(
"INSERT INTO {} (tag, timestamp_ms, sequence_nr, persistence_id, writer, \
adapter_manifest, ser_id, ser_manifest, payload, meta_ser_id, meta_manifest, \
meta_payload, tags) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
schema.tag_events
),
config.write_consistency,
None,
None,
)
.await?;
let tag_event_columns = "timestamp_ms, sequence_nr, persistence_id, writer, \
adapter_manifest, ser_id, ser_manifest, payload, meta_ser_id, meta_manifest, \
meta_payload, tags";
let select_tag_events_all = prepare(
session,
format!(
"SELECT {tag_event_columns} FROM {} WHERE tag = ?",
schema.tag_events
),
config.read_consistency,
Some(config.replay_page_size),
None,
)
.await?;
let select_tag_events_inclusive = prepare(
session,
format!(
"SELECT {tag_event_columns} FROM {} WHERE tag = ? \
AND (timestamp_ms, sequence_nr) >= (?, ?)",
schema.tag_events
),
config.read_consistency,
Some(config.replay_page_size),
None,
)
.await?;
let select_tag_events_after = prepare(
session,
format!(
"SELECT {tag_event_columns} FROM {} WHERE tag = ? \
AND (timestamp_ms, sequence_nr, persistence_id) > (?, ?, ?)",
schema.tag_events
),
config.read_consistency,
Some(config.replay_page_size),
None,
)
.await?;
let upsert_durable_state_change = prepare(
session,
format!(
"UPDATE {} SET revision = ?, timestamp_ms = ?, ser_id = ?, ser_manifest = ?, \
payload = ?, deleted = false WHERE tag = ? AND persistence_id = ?",
schema.durable_state_changes
),
config.write_consistency,
None,
None,
)
.await?;
let delete_durable_state_change = prepare(
session,
format!(
"UPDATE {} SET revision = ?, timestamp_ms = ?, ser_id = null, ser_manifest = null, \
payload = null, deleted = true WHERE tag = ? AND persistence_id = ?",
schema.durable_state_changes
),
config.write_consistency,
None,
None,
)
.await?;
let remove_durable_state_change = prepare(
session,
format!(
"DELETE FROM {} WHERE tag = ? AND persistence_id = ?",
schema.durable_state_changes
),
config.write_consistency,
None,
None,
)
.await?;
let select_durable_state_changes = prepare(
session,
format!(
"SELECT persistence_id, revision, timestamp_ms, ser_id, ser_manifest, payload, \
deleted FROM {} WHERE tag = ? AND timestamp_ms >= ? ALLOW FILTERING",
schema.durable_state_changes
),
config.read_consistency,
Some(config.replay_page_size),
None,
)
.await?;
let insert_durable_state_change_log = prepare(
session,
format!(
"INSERT INTO {} (tag, timestamp_ms, revision, persistence_id, ser_id, \
ser_manifest, payload, deleted) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
schema.durable_state_change_log
),
config.write_consistency,
None,
None,
)
.await?;
let durable_state_change_log_columns =
"persistence_id, revision, timestamp_ms, ser_id, ser_manifest, payload, deleted";
let select_durable_state_change_log_inclusive = prepare(
session,
format!(
"SELECT {durable_state_change_log_columns} FROM {} WHERE tag = ? \
AND (timestamp_ms, revision) >= (?, ?)",
schema.durable_state_change_log
),
config.read_consistency,
Some(config.replay_page_size),
None,
)
.await?;
let select_durable_state_change_log_after = prepare(
session,
format!(
"SELECT {durable_state_change_log_columns} FROM {} WHERE tag = ? \
AND (timestamp_ms, revision, persistence_id) > (?, ?, ?)",
schema.durable_state_change_log
),
config.read_consistency,
Some(config.replay_page_size),
None,
)
.await?;
Ok(CassandraStatements {
insert_event,
select_head,
select_partition_highest,
insert_head,
update_head,
insert_head_plain,
update_head_plain,
replay_partition,
delete_event_range,
load_snapshot,
save_snapshot,
delete_snapshot_exact,
select_matching_snapshots,
get_durable_state,
insert_durable_state,
update_durable_state,
insert_durable_tombstone,
update_durable_tombstone,
select_persistence_ids,
insert_tag_event,
select_tag_events_all,
select_tag_events_inclusive,
select_tag_events_after,
upsert_durable_state_change,
delete_durable_state_change,
remove_durable_state_change,
select_durable_state_changes,
insert_durable_state_change_log,
select_durable_state_change_log_inclusive,
select_durable_state_change_log_after,
})
}
async fn prepare(
session: &Session,
cql: String,
consistency: Consistency,
page_size: Option<i32>,
serial_consistency: Option<SerialConsistency>,
) -> PersistenceResult<PreparedStatement> {
let mut statement = session
.prepare(cql)
.await
.map_err(|error| storage_error("prepare Cassandra statement", error))?;
statement.set_consistency(consistency);
if let Some(page_size) = page_size {
statement.set_page_size(page_size);
}
if let Some(serial_consistency) = serial_consistency {
statement.set_serial_consistency(Some(serial_consistency));
}
Ok(statement)
}
#[derive(Debug, Clone, Copy)]
struct JournalHead {
exists: bool,
highest_seq_nr: i64,
deleted_to: i64,
}
impl JournalHead {
fn logical_deleted_to(self) -> i64 {
self.deleted_to.min(self.highest_seq_nr)
}
}
#[derive(Debug, Clone, Copy)]
struct ReconciledJournalHead {
stored: JournalHead,
highest_seq_nr: i64,
}
impl ReconciledJournalHead {
fn logical_deleted_to(self) -> i64 {
self.stored.logical_deleted_to().min(self.highest_seq_nr)
}
}
async fn read_head(
connected: &ConnectedCassandra,
persistence_id: &PersistenceId,
) -> PersistenceResult<JournalHead> {
let result = connected
.session
.execute_unpaged(&connected.statements.select_head, (persistence_id.id(),))
.await
.map_err(|error| storage_error("read Cassandra journal head", error))?;
let rows = result
.into_rows_result()
.map_err(|error| storage_error("decode Cassandra journal-head result", error))?;
let row = rows
.maybe_first_row::<HeadRow>()
.map_err(|error| storage_error("decode Cassandra journal-head row", error))?;
match row {
Some(row) if row.highest_seq_nr >= 0 && row.deleted_to >= 0 => Ok(JournalHead {
exists: true,
highest_seq_nr: row.highest_seq_nr,
deleted_to: row.deleted_to,
}),
Some(row) => Err(PersistenceError::Failure(format!(
"Cassandra journal head contains negative values ({}, {})",
row.highest_seq_nr, row.deleted_to
))),
None => Ok(JournalHead {
exists: false,
highest_seq_nr: 0,
deleted_to: 0,
}),
}
}
async fn read_reconciled_head(
connected: &ConnectedCassandra,
persistence_id: &PersistenceId,
partition_size: u64,
) -> PersistenceResult<ReconciledJournalHead> {
let stored = read_head(connected, persistence_id).await?;
let head_partition = partition_nr(
u64::try_from(stored.highest_seq_nr)
.expect("validated Cassandra journal head is non-negative"),
partition_size,
);
let mut highest_seq_nr = stored.highest_seq_nr;
if let Some(highest) = read_partition_highest(
connected,
persistence_id,
head_partition,
stored.highest_seq_nr,
)
.await?
{
highest_seq_nr = highest_seq_nr.max(highest);
}
if let Some(next_partition) = head_partition.checked_add(1)
&& let Some(highest) = read_partition_highest(
connected,
persistence_id,
next_partition,
stored.highest_seq_nr,
)
.await?
{
highest_seq_nr = highest_seq_nr.max(highest);
}
Ok(ReconciledJournalHead {
stored,
highest_seq_nr,
})
}
async fn read_partition_highest(
connected: &ConnectedCassandra,
persistence_id: &PersistenceId,
partition_nr: i64,
above_sequence_nr: i64,
) -> PersistenceResult<Option<i64>> {
let result = connected
.session
.execute_unpaged(
&connected.statements.select_partition_highest,
(persistence_id.id(), partition_nr, above_sequence_nr),
)
.await
.map_err(|error| storage_error("reconcile Cassandra journal head", error))?;
let rows = result
.into_rows_result()
.map_err(|error| storage_error("decode Cassandra journal reconciliation result", error))?;
let row = rows
.maybe_first_row::<SequenceNrRow>()
.map_err(|error| storage_error("decode Cassandra journal reconciliation row", error))?;
row.map(|row| {
if row.sequence_nr <= above_sequence_nr {
return Err(PersistenceError::Failure(format!(
"Cassandra journal reconciliation returned sequence {} at or below bound {}",
row.sequence_nr, above_sequence_nr
)));
}
Ok(row.sequence_nr)
})
.transpose()
}
async fn repair_reconciled_head(
connected: &ConnectedCassandra,
persistence_id: &PersistenceId,
reconciled: ReconciledJournalHead,
write_mode: CassandraWriteMode,
context: &str,
) -> PersistenceResult<JournalHead> {
if reconciled.highest_seq_nr == reconciled.stored.highest_seq_nr {
return Ok(reconciled.stored);
}
transition_head(
connected,
persistence_id,
reconciled.stored,
reconciled.highest_seq_nr,
reconciled.logical_deleted_to(),
write_mode,
context,
)
.await?;
Ok(JournalHead {
exists: true,
highest_seq_nr: reconciled.highest_seq_nr,
deleted_to: reconciled.logical_deleted_to(),
})
}
async fn transition_head(
connected: &ConnectedCassandra,
persistence_id: &PersistenceId,
expected: JournalHead,
highest_seq_nr: i64,
deleted_to: i64,
write_mode: CassandraWriteMode,
context: &str,
) -> PersistenceResult<()> {
match write_mode {
CassandraWriteMode::Lwt => {
let result = if expected.exists {
connected
.session
.execute_unpaged(
&connected.statements.update_head,
(
highest_seq_nr,
deleted_to,
persistence_id.id(),
expected.highest_seq_nr,
expected.deleted_to,
),
)
.await
} else {
connected
.session
.execute_unpaged(
&connected.statements.insert_head,
(persistence_id.id(), highest_seq_nr, deleted_to),
)
.await
}
.map_err(|error| storage_error(context, error))?;
if lwt_applied(result)? {
Ok(())
} else {
Err(PersistenceError::Failure(format!(
"{context}: journal-head LWT contention for {persistence_id}; write outcome \
is uncertain"
)))
}
}
CassandraWriteMode::SingleWriter => {
if expected.exists {
connected
.session
.execute_unpaged(
&connected.statements.update_head_plain,
(highest_seq_nr, deleted_to, persistence_id.id()),
)
.await
} else {
connected
.session
.execute_unpaged(
&connected.statements.insert_head_plain,
(persistence_id.id(), highest_seq_nr, deleted_to),
)
.await
}
.map_err(|error| storage_error(context, error))?;
Ok(())
}
}
}
#[derive(Debug, scylla::SerializeRow)]
#[scylla(flavor = "enforce_order", skip_name_checks)]
struct JournalInsertValues {
persistence_id: String,
partition_nr: i64,
sequence_nr: i64,
writer: String,
adapter_manifest: String,
serializer_id: i64,
serializer_manifest: String,
payload: Vec<u8>,
metadata_serializer_id: Option<i64>,
metadata_manifest: Option<String>,
metadata_payload: Option<Vec<u8>>,
tags: HashSet<String>,
timestamp_ms: i64,
}
#[derive(Debug, scylla::SerializeRow)]
#[scylla(flavor = "enforce_order", skip_name_checks)]
struct TagEventInsertValues {
tag: String,
timestamp_ms: i64,
sequence_nr: i64,
persistence_id: String,
writer: String,
adapter_manifest: String,
serializer_id: i64,
serializer_manifest: String,
payload: Vec<u8>,
metadata_serializer_id: Option<i64>,
metadata_manifest: Option<String>,
metadata_payload: Option<Vec<u8>>,
tags: HashSet<String>,
}
enum JournalBatchValues {
Event(JournalInsertValues),
Tag(TagEventInsertValues),
}
impl SerializeRow for JournalBatchValues {
fn serialize(
&self,
ctx: &RowSerializationContext<'_>,
writer: &mut RowWriter,
) -> Result<(), SerializationError> {
match self {
Self::Event(values) => values.serialize(ctx, writer),
Self::Tag(values) => values.serialize(ctx, writer),
}
}
fn is_empty(&self) -> bool {
match self {
Self::Event(values) => values.is_empty(),
Self::Tag(values) => values.is_empty(),
}
}
}
#[derive(Debug)]
struct PreparedAtomicWrite {
persistence_id: PersistenceId,
first_sequence_nr: i64,
last_sequence_nr: i64,
events: Vec<JournalInsertValues>,
tag_events: Vec<TagEventInsertValues>,
}
impl PreparedAtomicWrite {
fn new(write: AtomicWrite, partition_size: u64) -> Result<Self, String> {
let source = write.into_events();
let first = source
.first()
.expect("AtomicWrite guarantees at least one event");
let last = source.last().expect("AtomicWrite guarantees events");
let first_sequence_nr = i64::try_from(first.sequence_nr.as_u64()).map_err(|_| {
format!(
"Cassandra sequence number {} exceeds bigint range",
first.sequence_nr.as_u64()
)
})?;
let last_sequence_nr = i64::try_from(last.sequence_nr.as_u64()).map_err(|_| {
format!(
"Cassandra sequence number {} exceeds bigint range",
last.sequence_nr.as_u64()
)
})?;
let first_partition = partition_nr(first.sequence_nr.as_u64(), partition_size);
let last_partition = partition_nr(last.sequence_nr.as_u64(), partition_size);
if first_partition != last_partition {
return Err(format!(
"atomic write from sequence {} through {} crosses Cassandra journal partitions \
{first_partition} and {last_partition}",
first.sequence_nr.as_u64(),
last.sequence_nr.as_u64()
));
}
let persistence_id = first.persistence_id.clone();
let mut events = Vec::with_capacity(source.len());
let mut tag_events = Vec::new();
for event in source {
let (journal_values, mut event_tag_rows) =
journal_insert_values(event, first_partition);
tag_events.append(&mut event_tag_rows);
events.push(journal_values);
}
Ok(Self {
persistence_id,
first_sequence_nr,
last_sequence_nr,
events,
tag_events,
})
}
}
fn journal_insert_values(
event: StoredEvent,
partition_nr: i64,
) -> (JournalInsertValues, Vec<TagEventInsertValues>) {
let persistence_id = event.persistence_id.id().to_owned();
let sequence_nr = i64::try_from(event.sequence_nr.as_u64())
.expect("atomic write sequence range was validated");
let tag_names: Vec<String> = event
.tags
.iter()
.map(|tag| tag.as_str().to_owned())
.collect();
let tags_set: HashSet<String> = tag_names.iter().cloned().collect();
let tag_rows = tag_names
.iter()
.map(|tag| TagEventInsertValues {
tag: tag.clone(),
timestamp_ms: event.timestamp_ms,
sequence_nr,
persistence_id: persistence_id.clone(),
writer: event.writer_uuid.clone(),
adapter_manifest: event.adapter_manifest.clone(),
serializer_id: i64::from(event.payload.serializer_id.as_u32()),
serializer_manifest: event.payload.manifest.clone(),
payload: event.payload.payload.clone(),
metadata_serializer_id: event
.metadata
.as_ref()
.map(|metadata| i64::from(metadata.serializer_id.as_u32())),
metadata_manifest: event
.metadata
.as_ref()
.map(|metadata| metadata.manifest.clone()),
metadata_payload: event
.metadata
.as_ref()
.map(|metadata| metadata.payload.clone()),
tags: tags_set.clone(),
})
.collect::<Vec<_>>();
let (metadata_serializer_id, metadata_manifest, metadata_payload) =
event.metadata.map_or((None, None, None), |metadata| {
(
Some(i64::from(metadata.serializer_id.as_u32())),
Some(metadata.manifest),
Some(metadata.payload),
)
});
let journal_values = JournalInsertValues {
persistence_id,
partition_nr,
sequence_nr,
writer: event.writer_uuid,
adapter_manifest: event.adapter_manifest,
serializer_id: i64::from(event.payload.serializer_id.as_u32()),
serializer_manifest: event.payload.manifest,
payload: event.payload.payload,
metadata_serializer_id,
metadata_manifest,
metadata_payload,
tags: tags_set,
timestamp_ms: event.timestamp_ms,
};
(journal_values, tag_rows)
}
#[derive(Debug, scylla::DeserializeRow)]
#[scylla(flavor = "enforce_order", skip_name_checks)]
struct HeadRow {
highest_seq_nr: i64,
deleted_to: i64,
}
#[derive(Debug, scylla::DeserializeRow)]
#[scylla(flavor = "enforce_order", skip_name_checks)]
struct SequenceNrRow {
sequence_nr: i64,
}
#[derive(Debug, scylla::DeserializeRow)]
#[scylla(flavor = "enforce_order", skip_name_checks)]
struct JournalRow {
sequence_nr: i64,
writer: String,
adapter_manifest: String,
serializer_id: i64,
serializer_manifest: String,
payload: Vec<u8>,
metadata_serializer_id: Option<i64>,
metadata_manifest: Option<String>,
metadata_payload: Option<Vec<u8>>,
tags: HashSet<String>,
timestamp_ms: i64,
}
struct CassandraReplayState {
connected: Arc<ConnectedCassandra>,
persistence_id: PersistenceId,
current_partition: i64,
last_partition: i64,
from_sequence_nr: i64,
to_sequence_nr: i64,
remaining: u64,
partition_size: u64,
paging_state: PagingState,
buffered: VecDeque<StoredEvent>,
finished: bool,
}
fn cassandra_event_stream(
connected: Arc<ConnectedCassandra>,
persistence_id: PersistenceId,
from: SequenceNr,
to: SequenceNr,
max: u64,
deleted_to: i64,
partition_size: u64,
) -> EventStream {
let deleted_from = deleted_to
.checked_add(1)
.map_or(u64::MAX, |value| u64::try_from(value).unwrap_or_default());
let from = from.as_u64().max(deleted_from);
let to = to.as_u64().min(i64::MAX as u64);
if from > to || max == 0 || from > i64::MAX as u64 {
return Box::pin(stream::empty());
}
let state = CassandraReplayState {
connected,
persistence_id,
current_partition: partition_nr(from, partition_size),
last_partition: partition_nr(to, partition_size),
from_sequence_nr: i64::try_from(from).expect("replay bound was capped"),
to_sequence_nr: i64::try_from(to).expect("replay bound was capped"),
remaining: max,
partition_size,
paging_state: PagingState::start(),
buffered: VecDeque::new(),
finished: false,
};
Box::pin(stream::unfold(state, next_replay_event))
}
async fn next_replay_event(
mut state: CassandraReplayState,
) -> Option<(PersistenceResult<StoredEvent>, CassandraReplayState)> {
loop {
if state.finished || state.remaining == 0 {
return None;
}
if let Some(event) = state.buffered.pop_front() {
state.remaining -= 1;
return Some((Ok(event), state));
}
if state.current_partition > state.last_partition {
return None;
}
let partition_start = partition_start(state.current_partition, state.partition_size);
let partition_end = partition_end(state.current_partition, state.partition_size);
let from = state.from_sequence_nr.max(partition_start);
let to = state.to_sequence_nr.min(partition_end);
let paging_state = std::mem::replace(&mut state.paging_state, PagingState::start());
let page = state
.connected
.session
.execute_single_page(
&state.connected.statements.replay_partition,
(state.persistence_id.id(), state.current_partition, from, to),
paging_state,
)
.await;
let (result, next_page) = match page {
Ok(page) => page,
Err(error) => {
state.finished = true;
return Some((
Err(storage_error("page Cassandra journal replay", error)),
state,
));
}
};
let rows = match decode_journal_page(&state.persistence_id, result) {
Ok(rows) => rows,
Err(error) => {
state.finished = true;
return Some((Err(error), state));
}
};
state.buffered = VecDeque::from(rows);
match next_page {
PagingStateResponse::HasMorePages {
state: paging_state,
} => {
state.paging_state = paging_state;
}
PagingStateResponse::NoMorePages => {
state.current_partition += 1;
state.paging_state = PagingState::start();
}
}
}
}
fn decode_journal_page(
persistence_id: &PersistenceId,
result: QueryResult,
) -> PersistenceResult<Vec<StoredEvent>> {
let rows = result
.into_rows_result()
.map_err(|error| storage_error("decode Cassandra journal page", error))?;
rows.rows::<JournalRow>()
.map_err(|error| storage_error("decode Cassandra journal page rows", error))?
.map(|row| {
row.map_err(|error| storage_error("decode Cassandra journal row", error))
.and_then(|row| decode_stored_event(persistence_id, row))
})
.collect()
}
fn decode_stored_event(
persistence_id: &PersistenceId,
row: JournalRow,
) -> PersistenceResult<StoredEvent> {
let sequence_nr = SequenceNr::new(non_negative_u64(
row.sequence_nr,
"Cassandra journal sequence number",
)?);
let metadata = decode_optional_frame(
row.metadata_serializer_id,
row.metadata_manifest,
row.metadata_payload,
"Cassandra journal metadata",
)?;
let mut tags = row.tags.into_iter().map(Tag::new).collect::<Vec<_>>();
tags.sort_by(|left, right| left.as_str().cmp(right.as_str()));
Ok(StoredEvent {
persistence_id: persistence_id.clone(),
sequence_nr,
adapter_manifest: row.adapter_manifest,
writer_uuid: row.writer,
timestamp_ms: row.timestamp_ms,
metadata,
payload: decode_frame(
row.serializer_id,
row.serializer_manifest,
row.payload,
"Cassandra journal payload",
)?,
tags,
})
}
struct CassandraLiveCursor {
connected: Arc<ConnectedCassandra>,
persistence_id: PersistenceId,
initial: Option<EventStream>,
next_sequence_nr: Option<SequenceNr>,
to_sequence_nr: SequenceNr,
reached_end: bool,
partition_size: u64,
polling: CassandraPollingConfig,
delay: Duration,
buffered: VecDeque<EventEnvelope>,
}
async fn next_cassandra_live_event(
mut cursor: CassandraLiveCursor,
) -> Option<(PersistenceResult<EventEnvelope>, CassandraLiveCursor)> {
loop {
if let Some(initial) = &mut cursor.initial {
match initial.next().await {
Some(event) => return Some((event.map(event_envelope), cursor)),
None => cursor.initial = None,
}
}
if let Some(event) = cursor.buffered.pop_front() {
return Some((Ok(event), cursor));
}
let next_sequence_nr = cursor.next_sequence_nr?;
if cursor.reached_end {
return None;
}
let head = match read_reconciled_head(
&cursor.connected,
&cursor.persistence_id,
cursor.partition_size,
)
.await
{
Ok(head) => head,
Err(error) => return Some((Err(error), cursor)),
};
let highest = SequenceNr::new(
match non_negative_u64(head.highest_seq_nr, "Cassandra live-query journal head") {
Ok(highest) => highest,
Err(error) => return Some((Err(error), cursor)),
},
);
let boundary = highest.min(cursor.to_sequence_nr);
if next_sequence_nr <= boundary {
let events = cassandra_event_stream(
cursor.connected.clone(),
cursor.persistence_id.clone(),
next_sequence_nr,
boundary,
u64::from(cursor.polling.batch_size),
head.logical_deleted_to(),
cursor.partition_size,
);
let mut events = events;
let mut buffered = VecDeque::new();
while let Some(event) = events.next().await {
match event {
Ok(event) => buffered.push_back(event_envelope(event)),
Err(error) => return Some((Err(error), cursor)),
}
}
if let Some(last) = buffered.back() {
cursor.next_sequence_nr = last
.sequence_nr
.as_u64()
.checked_add(1)
.map(SequenceNr::new);
cursor.reached_end = last.sequence_nr >= cursor.to_sequence_nr;
cursor.buffered = buffered;
cursor.delay = cursor.polling.min_interval;
continue;
}
cursor.next_sequence_nr = highest.as_u64().checked_add(1).map(SequenceNr::new);
}
cursor.reached_end = highest >= cursor.to_sequence_nr;
if cursor.reached_end {
return None;
}
tokio::time::sleep(cursor.delay).await;
cursor.delay = cursor
.delay
.saturating_mul(2)
.min(cursor.polling.max_interval);
}
}
fn event_envelope(event: StoredEvent) -> EventEnvelope {
EventEnvelope {
offset: Offset::Sequence(event.sequence_nr.as_u64()),
persistence_id: event.persistence_id,
sequence_nr: event.sequence_nr,
timestamp_ms: event.timestamp_ms,
adapter_manifest: event.adapter_manifest,
payload: event.payload,
metadata: event.metadata,
tags: event.tags,
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum CassandraTagResume {
None,
Inclusive {
timestamp_ms: i64,
sequence_nr: i64,
},
After {
timestamp_ms: i64,
sequence_nr: i64,
persistence_id: String,
},
}
fn cassandra_tag_resume(offset: Offset) -> PersistenceResult<CassandraTagResume> {
match offset {
Offset::NoOffset => Ok(CassandraTagResume::None),
Offset::Timestamp {
epoch_micros,
seq_nr,
} => Ok(CassandraTagResume::Inclusive {
timestamp_ms: epoch_micros.div_euclid(1000),
sequence_nr: sql_i64(seq_nr, "Cassandra tag-query offset sequence number")?,
}),
Offset::Sequence(_) => Err(PersistenceError::Rejected(
"Cassandra tag queries require a Timestamp offset produced by the same backend".into(),
)),
}
}
fn cassandra_tag_resume_after(last: &EventEnvelope) -> CassandraTagResume {
let Offset::Timestamp {
epoch_micros,
seq_nr,
} = last.offset
else {
unreachable!("Cassandra tag envelopes always carry timestamp offsets")
};
CassandraTagResume::After {
timestamp_ms: epoch_micros.div_euclid(1000),
sequence_nr: i64::try_from(seq_nr).expect("Cassandra sequence numbers fit i64"),
persistence_id: last.persistence_id.id().to_owned(),
}
}
async fn cassandra_events_by_tag(
connected: &ConnectedCassandra,
tag: &Tag,
resume: &CassandraTagResume,
limit: Option<u32>,
) -> PersistenceResult<Vec<EventEnvelope>> {
let pager = match resume {
CassandraTagResume::None => {
connected
.session
.execute_iter(
connected.statements.select_tag_events_all.clone(),
(tag.as_str(),),
)
.await
}
CassandraTagResume::Inclusive {
timestamp_ms,
sequence_nr,
} => {
connected
.session
.execute_iter(
connected.statements.select_tag_events_inclusive.clone(),
(tag.as_str(), *timestamp_ms, *sequence_nr),
)
.await
}
CassandraTagResume::After {
timestamp_ms,
sequence_nr,
persistence_id,
} => {
connected
.session
.execute_iter(
connected.statements.select_tag_events_after.clone(),
(
tag.as_str(),
*timestamp_ms,
*sequence_nr,
persistence_id.as_str(),
),
)
.await
}
}
.map_err(|error| storage_error("query Cassandra tag events", error))?;
let mut rows = pager
.rows_stream::<TagEventRow>()
.map_err(|error| storage_error("decode Cassandra tag-query rows", error))?;
let cap = limit.map_or(usize::MAX, |limit| limit as usize);
let mut candidates = Vec::new();
while candidates.len() < cap {
match rows.next().await {
Some(Ok(row)) => candidates.push(decode_tag_event(row)?),
Some(Err(error)) => {
return Err(storage_error("decode Cassandra tag-query row", error));
}
None => break,
}
}
filter_deleted_tag_events(connected, candidates).await
}
async fn filter_deleted_tag_events(
connected: &ConnectedCassandra,
candidates: Vec<StoredEvent>,
) -> PersistenceResult<Vec<EventEnvelope>> {
let mut seen: HashMap<String, i64> = HashMap::new();
let mut events = Vec::with_capacity(candidates.len());
for candidate in candidates {
let deleted_to = match seen.get(candidate.persistence_id.id()) {
Some(deleted_to) => *deleted_to,
None => {
let head = read_head(connected, &candidate.persistence_id).await?;
let deleted_to = head.logical_deleted_to();
seen.insert(candidate.persistence_id.id().to_owned(), deleted_to);
deleted_to
}
};
if candidate.sequence_nr.as_u64() <= u64::try_from(deleted_to).unwrap_or(0) {
continue;
}
events.push(tag_event_envelope(candidate));
}
Ok(events)
}
struct CassandraTagCursor {
connected: Arc<ConnectedCassandra>,
tag: Tag,
resume: CassandraTagResume,
buffered: VecDeque<EventEnvelope>,
polling: CassandraPollingConfig,
delay: Duration,
}
async fn next_cassandra_tag_event(
mut cursor: CassandraTagCursor,
) -> Option<(PersistenceResult<EventEnvelope>, CassandraTagCursor)> {
loop {
if let Some(event) = cursor.buffered.pop_front() {
return Some((Ok(event), cursor));
}
let events = match cassandra_events_by_tag(
&cursor.connected,
&cursor.tag,
&cursor.resume,
Some(cursor.polling.batch_size),
)
.await
{
Ok(events) => events,
Err(error) => return Some((Err(error), cursor)),
};
if let Some(last) = events.last() {
cursor.resume = cassandra_tag_resume_after(last);
cursor.buffered = VecDeque::from(events);
cursor.delay = cursor.polling.min_interval;
continue;
}
tokio::time::sleep(cursor.delay).await;
cursor.delay = cursor
.delay
.saturating_mul(2)
.min(cursor.polling.max_interval);
}
}
#[derive(Debug, scylla::DeserializeRow)]
#[scylla(flavor = "enforce_order", skip_name_checks)]
struct TagEventRow {
timestamp_ms: i64,
sequence_nr: i64,
persistence_id: String,
writer: String,
adapter_manifest: String,
serializer_id: i64,
serializer_manifest: String,
payload: Vec<u8>,
metadata_serializer_id: Option<i64>,
metadata_manifest: Option<String>,
metadata_payload: Option<Vec<u8>>,
tags: HashSet<String>,
}
fn decode_tag_event(row: TagEventRow) -> PersistenceResult<StoredEvent> {
let persistence_id = PersistenceId::from_unique(row.persistence_id)?;
let sequence_nr = SequenceNr::new(non_negative_u64(
row.sequence_nr,
"Cassandra tag-query sequence number",
)?);
let metadata = decode_optional_frame(
row.metadata_serializer_id,
row.metadata_manifest,
row.metadata_payload,
"Cassandra tag-query metadata",
)?;
let mut tags = row.tags.into_iter().map(Tag::new).collect::<Vec<_>>();
tags.sort_by(|left, right| left.as_str().cmp(right.as_str()));
Ok(StoredEvent {
persistence_id,
sequence_nr,
adapter_manifest: row.adapter_manifest,
writer_uuid: row.writer,
timestamp_ms: row.timestamp_ms,
metadata,
payload: decode_frame(
row.serializer_id,
row.serializer_manifest,
row.payload,
"Cassandra tag-query payload",
)?,
tags,
})
}
fn tag_event_envelope(event: StoredEvent) -> EventEnvelope {
EventEnvelope {
offset: Offset::Timestamp {
epoch_micros: event.timestamp_ms.saturating_mul(1000),
seq_nr: event.sequence_nr.as_u64(),
},
persistence_id: event.persistence_id,
sequence_nr: event.sequence_nr,
timestamp_ms: event.timestamp_ms,
adapter_manifest: event.adapter_manifest,
payload: event.payload,
metadata: event.metadata,
tags: event.tags,
}
}
fn cassandra_change_since_timestamp_ms(offset: Offset) -> PersistenceResult<i64> {
match offset {
Offset::NoOffset => Ok(i64::MIN),
Offset::Timestamp { epoch_micros, .. } => Ok(epoch_micros.div_euclid(1000)),
Offset::Sequence(_) => Err(PersistenceError::Rejected(
"Cassandra durable-state changes require a Timestamp offset produced by the same \
backend"
.into(),
)),
}
}
async fn cassandra_current_durable_state_changes(
connected: &ConnectedCassandra,
tag: &Tag,
since_timestamp_ms: i64,
) -> PersistenceResult<Vec<DurableStateChange>> {
let pager = connected
.session
.execute_iter(
connected.statements.select_durable_state_changes.clone(),
(tag.as_str(), since_timestamp_ms),
)
.await
.map_err(|error| storage_error("query Cassandra durable-state changes", error))?;
let mut rows = pager
.rows_stream::<DurableStateChangeRow>()
.map_err(|error| storage_error("decode Cassandra durable-state change rows", error))?;
let mut changes = Vec::new();
while let Some(row) = rows.next().await {
let row =
row.map_err(|error| storage_error("decode Cassandra durable-state change row", error))?;
changes.push(decode_durable_state_change(row)?);
}
Ok(changes)
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum CassandraDurableChangeLogResume {
Inclusive { timestamp_ms: i64, revision: i64 },
After {
timestamp_ms: i64,
revision: i64,
persistence_id: String,
},
}
fn cassandra_change_log_resume_after(last: &DurableStateChange) -> CassandraDurableChangeLogResume {
let Offset::Timestamp {
epoch_micros,
seq_nr,
} = last.offset()
else {
unreachable!("Cassandra durable changes always carry timestamp offsets")
};
CassandraDurableChangeLogResume::After {
timestamp_ms: epoch_micros.div_euclid(1000),
revision: i64::try_from(seq_nr).expect("Cassandra revisions fit i64"),
persistence_id: last.persistence_id().id().to_owned(),
}
}
async fn cassandra_durable_state_change_log(
connected: &ConnectedCassandra,
tag: &Tag,
resume: &CassandraDurableChangeLogResume,
limit: Option<u32>,
) -> PersistenceResult<Vec<DurableStateChange>> {
let pager = match resume {
CassandraDurableChangeLogResume::Inclusive {
timestamp_ms,
revision,
} => {
connected
.session
.execute_iter(
connected
.statements
.select_durable_state_change_log_inclusive
.clone(),
(tag.as_str(), *timestamp_ms, *revision),
)
.await
}
CassandraDurableChangeLogResume::After {
timestamp_ms,
revision,
persistence_id,
} => {
connected
.session
.execute_iter(
connected
.statements
.select_durable_state_change_log_after
.clone(),
(
tag.as_str(),
*timestamp_ms,
*revision,
persistence_id.as_str(),
),
)
.await
}
}
.map_err(|error| storage_error("query Cassandra durable-state change log", error))?;
let mut rows = pager
.rows_stream::<DurableStateChangeRow>()
.map_err(|error| storage_error("decode Cassandra durable-state change log rows", error))?;
let cap = limit.map_or(usize::MAX, |limit| limit as usize);
let mut changes = Vec::new();
while changes.len() < cap {
match rows.next().await {
Some(Ok(row)) => changes.push(decode_durable_state_change(row)?),
Some(Err(error)) => {
return Err(storage_error(
"decode Cassandra durable-state change log row",
error,
));
}
None => break,
}
}
Ok(changes)
}
struct CassandraDurableChangeLogCursor {
connected: Arc<ConnectedCassandra>,
tag: Tag,
resume: CassandraDurableChangeLogResume,
buffered: VecDeque<DurableStateChange>,
polling: CassandraPollingConfig,
delay: Duration,
}
async fn next_cassandra_durable_change_log(
mut cursor: CassandraDurableChangeLogCursor,
) -> Option<(
PersistenceResult<DurableStateChange>,
CassandraDurableChangeLogCursor,
)> {
loop {
if let Some(change) = cursor.buffered.pop_front() {
return Some((Ok(change), cursor));
}
let changes = match cassandra_durable_state_change_log(
&cursor.connected,
&cursor.tag,
&cursor.resume,
Some(cursor.polling.batch_size),
)
.await
{
Ok(changes) => changes,
Err(error) => return Some((Err(error), cursor)),
};
if let Some(last) = changes.last() {
cursor.resume = cassandra_change_log_resume_after(last);
cursor.buffered = VecDeque::from(changes);
cursor.delay = cursor.polling.min_interval;
continue;
}
tokio::time::sleep(cursor.delay).await;
cursor.delay = cursor
.delay
.saturating_mul(2)
.min(cursor.polling.max_interval);
}
}
#[derive(Debug, scylla::DeserializeRow)]
#[scylla(flavor = "enforce_order", skip_name_checks)]
struct DurableStateChangeRow {
persistence_id: String,
revision: i64,
timestamp_ms: i64,
serializer_id: Option<i64>,
serializer_manifest: Option<String>,
payload: Option<Vec<u8>>,
deleted: bool,
}
fn decode_durable_state_change(
row: DurableStateChangeRow,
) -> PersistenceResult<DurableStateChange> {
let persistence_id = PersistenceId::from_unique(row.persistence_id)?;
let revision = Revision::new(non_negative_u64(
row.revision,
"Cassandra durable-state change revision",
)?);
let offset = Offset::Timestamp {
epoch_micros: row.timestamp_ms.saturating_mul(1000),
seq_nr: revision.as_u64(),
};
if row.deleted {
return Ok(DurableStateChange::Deleted(DeletedDurableState {
offset,
persistence_id,
revision,
timestamp_ms: row.timestamp_ms,
}));
}
let state = decode_optional_frame(
row.serializer_id,
row.serializer_manifest,
row.payload,
"Cassandra durable-state change",
)?
.ok_or_else(|| {
PersistenceError::Failure(
"Cassandra durable-state change is live but has no serialization frame".into(),
)
})?;
Ok(DurableStateChange::Updated(UpdatedDurableState {
offset,
persistence_id,
revision,
state,
timestamp_ms: row.timestamp_ms,
}))
}
#[derive(Debug, scylla::DeserializeRow)]
#[scylla(flavor = "enforce_order", skip_name_checks)]
struct SnapshotRow {
sequence_nr: i64,
timestamp_ms: i64,
adapter_manifest: String,
serializer_id: i64,
serializer_manifest: String,
payload: Vec<u8>,
}
#[derive(Debug, scylla::DeserializeRow)]
#[scylla(flavor = "enforce_order", skip_name_checks)]
struct SnapshotMetadataRow {
sequence_nr: i64,
timestamp_ms: i64,
}
fn decode_snapshot(
persistence_id: &PersistenceId,
row: SnapshotRow,
) -> PersistenceResult<SelectedSnapshot> {
Ok(SelectedSnapshot {
metadata: SnapshotMetadata {
persistence_id: persistence_id.clone(),
sequence_nr: SequenceNr::new(non_negative_u64(
row.sequence_nr,
"Cassandra snapshot sequence number",
)?),
timestamp_ms: row.timestamp_ms,
},
adapter_manifest: row.adapter_manifest,
state: decode_frame(
row.serializer_id,
row.serializer_manifest,
row.payload,
"Cassandra snapshot state",
)?,
})
}
#[derive(Debug, scylla::DeserializeRow)]
#[scylla(flavor = "enforce_order", skip_name_checks)]
struct DurableStateRow {
revision: i64,
serializer_id: Option<i64>,
serializer_manifest: Option<String>,
payload: Option<Vec<u8>>,
tag: Option<String>,
}
fn decode_durable_state(row: DurableStateRow) -> PersistenceResult<VersionedState> {
let state = decode_optional_frame(
row.serializer_id,
row.serializer_manifest,
row.payload,
"Cassandra durable state",
)?;
let tag = state.is_some().then(|| row.tag.map(Tag::new)).flatten();
Ok(VersionedState {
state,
revision: Revision::new(non_negative_u64(
row.revision,
"Cassandra durable-state revision",
)?),
tag,
})
}
async fn read_raw_durable_tag(
connected: &ConnectedCassandra,
persistence_id: &PersistenceId,
) -> PersistenceResult<Option<Tag>> {
let result = connected
.session
.execute_unpaged(
&connected.statements.get_durable_state,
(persistence_id.id(),),
)
.await
.map_err(|error| {
storage_error("read Cassandra durable-state tag for change index", error)
})?;
let rows = result
.into_rows_result()
.map_err(|error| storage_error("decode Cassandra durable-state tag result", error))?;
let row = rows
.maybe_first_row::<DurableStateRow>()
.map_err(|error| storage_error("decode Cassandra durable-state tag row", error))?;
Ok(row.and_then(|row| row.tag).map(Tag::new))
}
async fn remove_durable_state_change(
connected: &ConnectedCassandra,
old_tag: &Tag,
persistence_id: &PersistenceId,
) -> PersistenceResult<()> {
connected
.session
.execute_unpaged(
&connected.statements.remove_durable_state_change,
(old_tag.as_str(), persistence_id.id()),
)
.await
.map_err(|error| storage_error("remove stale Cassandra durable-state change", error))?;
Ok(())
}
async fn write_durable_state_upsert_change(
connected: &ConnectedCassandra,
tag: &Tag,
persistence_id: &PersistenceId,
revision: Revision,
state: SerializedPayload,
) -> PersistenceResult<()> {
let revision = sql_i64(revision.as_u64(), "durable-state change revision")?;
let timestamp_ms = current_timestamp_ms();
let serializer_id = i64::from(state.serializer_id.as_u32());
connected
.session
.execute_unpaged(
&connected.statements.upsert_durable_state_change,
(
revision,
timestamp_ms,
serializer_id,
state.manifest.clone(),
state.payload.clone(),
tag.as_str(),
persistence_id.id(),
),
)
.await
.map_err(|error| storage_error("index Cassandra durable-state change", error))?;
connected
.session
.execute_unpaged(
&connected.statements.insert_durable_state_change_log,
(
tag.as_str(),
timestamp_ms,
revision,
persistence_id.id(),
serializer_id,
state.manifest,
state.payload,
false,
),
)
.await
.map_err(|error| storage_error("append Cassandra durable-state change log", error))?;
Ok(())
}
async fn write_durable_state_delete_change(
connected: &ConnectedCassandra,
tag: &Tag,
persistence_id: &PersistenceId,
revision: Revision,
) -> PersistenceResult<()> {
let revision = sql_i64(revision.as_u64(), "durable-state change tombstone revision")?;
let timestamp_ms = current_timestamp_ms();
connected
.session
.execute_unpaged(
&connected.statements.delete_durable_state_change,
(revision, timestamp_ms, tag.as_str(), persistence_id.id()),
)
.await
.map_err(|error| storage_error("index Cassandra durable-state change tombstone", error))?;
connected
.session
.execute_unpaged(
&connected.statements.insert_durable_state_change_log,
(
tag.as_str(),
timestamp_ms,
revision,
persistence_id.id(),
None::<i64>,
None::<String>,
None::<Vec<u8>>,
true,
),
)
.await
.map_err(|error| {
storage_error("append Cassandra durable-state change log tombstone", error)
})?;
Ok(())
}
fn current_timestamp_ms() -> i64 {
i64::try_from(
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_millis())
.unwrap_or_default(),
)
.unwrap_or(i64::MAX)
}
#[derive(Debug, scylla::DeserializeRow)]
#[scylla(flavor = "enforce_order", skip_name_checks)]
struct PersistenceIdRow {
persistence_id: String,
}
fn lwt_applied(result: QueryResult) -> PersistenceResult<bool> {
let rows = result
.into_rows_result()
.map_err(|error| storage_error("decode Cassandra LWT result", error))?;
let row = rows
.first_row::<Row>()
.map_err(|error| storage_error("decode Cassandra LWT row", error))?;
match row.columns.first().and_then(Option::as_ref) {
Some(CqlValue::Boolean(applied)) => Ok(*applied),
other => Err(PersistenceError::Failure(format!(
"Cassandra LWT result did not contain boolean [applied], got {other:?}"
))),
}
}
fn decode_frame(
serializer_id: i64,
manifest: String,
payload: Vec<u8>,
context: &str,
) -> PersistenceResult<SerializedPayload> {
let serializer_id = u32::try_from(serializer_id).map_err(|_| {
PersistenceError::Failure(format!(
"{context} serializer id {serializer_id} is outside the u32 range"
))
})?;
Ok(SerializedPayload::new(
SerializerId::new(serializer_id),
manifest,
payload,
))
}
fn decode_optional_frame(
serializer_id: Option<i64>,
manifest: Option<String>,
payload: Option<Vec<u8>>,
context: &str,
) -> PersistenceResult<Option<SerializedPayload>> {
match (serializer_id, manifest, payload) {
(None, None, None) => Ok(None),
(Some(serializer_id), Some(manifest), Some(payload)) => {
decode_frame(serializer_id, manifest, payload, context).map(Some)
}
_ => Err(PersistenceError::Failure(format!(
"{context} contains an incomplete serialization frame"
))),
}
}
fn snapshot_sequence_bounds(criteria: &SnapshotCriteria) -> Option<(i64, i64)> {
if criteria.min_sequence_nr > criteria.max_sequence_nr
|| criteria.min_sequence_nr.as_u64() > i64::MAX as u64
{
return None;
}
Some((
i64::try_from(criteria.min_sequence_nr.as_u64()).expect("bound was checked"),
i64::try_from(criteria.max_sequence_nr.as_u64()).unwrap_or(i64::MAX),
))
}
fn partition_nr(sequence_nr: u64, partition_size: u64) -> i64 {
let zero_based = sequence_nr.saturating_sub(1) / partition_size;
i64::try_from(zero_based).unwrap_or(i64::MAX)
}
fn partition_start(partition_nr: i64, partition_size: u64) -> i64 {
let partition = u64::try_from(partition_nr).expect("partition numbers are non-negative");
let start = partition
.saturating_mul(partition_size)
.saturating_add(1)
.min(i64::MAX as u64);
i64::try_from(start).expect("partition start was capped")
}
fn partition_end(partition_nr: i64, partition_size: u64) -> i64 {
let partition = u64::try_from(partition_nr).expect("partition numbers are non-negative");
let end = partition
.saturating_add(1)
.saturating_mul(partition_size)
.min(i64::MAX as u64);
i64::try_from(end).expect("partition end was capped")
}
fn validate_config(config: &CassandraPersistenceConfig) -> PersistenceResult<()> {
if config.contact_points.is_empty()
|| config
.contact_points
.iter()
.any(|contact_point| contact_point.trim().is_empty())
{
return Err(PersistenceError::Rejected(
"Cassandra contact_points must contain at least one non-empty seed".into(),
));
}
validate_identifier("keyspace", &config.keyspace)?;
validate_identifier("event journal table", &config.tables.event_journal)?;
validate_identifier("journal-head table", &config.tables.journal_head)?;
validate_identifier("snapshot table", &config.tables.snapshot)?;
validate_identifier("durable-state table", &config.tables.durable_state)?;
validate_identifier("tag-events table", &config.tables.tag_events)?;
validate_identifier(
"durable-state-changes table",
&config.tables.durable_state_changes,
)?;
if config.replication_factor == 0 {
return Err(PersistenceError::Rejected(
"Cassandra replication_factor must be greater than zero".into(),
));
}
if config.partition_size == 0 {
return Err(PersistenceError::Rejected(
"Cassandra partition_size must be greater than zero".into(),
));
}
if config.replay_page_size <= 0 {
return Err(PersistenceError::Rejected(
"Cassandra replay_page_size must be greater than zero".into(),
));
}
if config.polling.batch_size == 0 || config.polling.batch_size > i32::MAX as u32 {
return Err(PersistenceError::Rejected(
"Cassandra polling batch_size must be in 1..=i32::MAX".into(),
));
}
if config.polling.min_interval > config.polling.max_interval {
return Err(PersistenceError::Rejected(
"Cassandra polling min_interval must not exceed max_interval".into(),
));
}
if matches!(
config.write_consistency,
Consistency::Any | Consistency::Serial | Consistency::LocalSerial
) {
return Err(PersistenceError::Rejected(
"Cassandra write_consistency cannot be ANY, SERIAL, or LOCAL_SERIAL; ANY may \
acknowledge only a hinted handoff and silently lose an acknowledged write"
.into(),
));
}
if matches!(
config.read_consistency,
Consistency::Any | Consistency::Serial | Consistency::LocalSerial
) {
return Err(PersistenceError::Rejected(
"Cassandra read_consistency must be a non-serial read consistency".into(),
));
}
Ok(())
}
fn validate_identifier(name: &str, value: &str) -> PersistenceResult<()> {
let mut characters = value.chars();
let valid_first = characters
.next()
.is_some_and(|character| character == '_' || character.is_ascii_lowercase());
if !valid_first
|| !characters.all(|character| {
character == '_' || character.is_ascii_lowercase() || character.is_ascii_digit()
})
|| value.len() > MAX_CQL_IDENTIFIER_BYTES
{
return Err(PersistenceError::Rejected(format!(
"invalid Cassandra {name} identifier {value:?}; expected a lowercase ASCII identifier \
of at most {MAX_CQL_IDENTIFIER_BYTES} bytes"
)));
}
Ok(())
}
fn qualified(keyspace: &str, table: &str) -> String {
format!("{keyspace}.{table}")
}
fn sql_i64(value: u64, context: &str) -> PersistenceResult<i64> {
i64::try_from(value).map_err(|_| {
PersistenceError::Rejected(format!("{context} {value} exceeds Cassandra bigint range"))
})
}
fn non_negative_u64(value: i64, context: &str) -> PersistenceResult<u64> {
u64::try_from(value).map_err(|_| {
PersistenceError::Failure(format!("{context} contains negative value {value}"))
})
}
fn connection_error(context: &str, error: impl fmt::Display) -> PersistenceError {
PersistenceError::Connection(format!("{context}: {error}"))
}
fn storage_error(context: &str, error: impl fmt::Display) -> PersistenceError {
PersistenceError::Failure(format!("{context}: {error}"))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn defaults_and_partition_math_match_the_contract() {
let config = CassandraPersistenceConfig::new("127.0.0.1:9042", "datum_persistence");
assert_eq!(config.replication_factor, 1);
assert_eq!(config.partition_size, 100_000);
assert_eq!(config.write_consistency, Consistency::LocalQuorum);
assert_eq!(config.read_consistency, Consistency::LocalQuorum);
assert_eq!(partition_nr(0, 2), 0);
assert_eq!(partition_nr(1, 2), 0);
assert_eq!(partition_nr(2, 2), 0);
assert_eq!(partition_nr(3, 2), 1);
assert_eq!(partition_nr(6, 2), 2);
assert_eq!(partition_start(2, 2), 5);
assert_eq!(partition_end(2, 2), 6);
}
#[test]
fn configuration_validation_is_side_effect_free_and_strict() {
let valid = CassandraPersistenceConfig::new("127.0.0.1:9042", "datum_persistence");
let backend = CassandraPersistence::new(valid.clone()).expect("default config is valid");
assert!(backend.connected.get().is_none());
let mut invalid = valid.clone();
invalid.contact_points.clear();
assert!(matches!(
CassandraPersistence::new(invalid),
Err(PersistenceError::Rejected(_))
));
let mut invalid = valid.clone();
invalid.partition_size = 0;
assert!(matches!(
CassandraPersistence::new(invalid),
Err(PersistenceError::Rejected(_))
));
let mut invalid = valid;
invalid.keyspace = "quoted-Keyspace".into();
assert!(matches!(
CassandraPersistence::new(invalid),
Err(PersistenceError::Rejected(_))
));
}
#[test]
fn schema_ddl_contains_the_settled_tables_and_keys() {
let backend = CassandraPersistence::new(CassandraPersistenceConfig::new(
"127.0.0.1:9042",
"datum_persistence",
))
.expect("config is valid");
let ddl = backend.schema_ddl();
assert!(ddl.contains("'class': 'SimpleStrategy'"));
assert!(ddl.contains("'replication_factor': 1"));
assert!(ddl.contains("PRIMARY KEY ((persistence_id, partition_nr), sequence_nr)"));
assert!(ddl.contains("CLUSTERING ORDER BY (sequence_nr ASC)"));
assert!(ddl.contains("datum_persistence.journal_head"));
assert!(ddl.contains("CLUSTERING ORDER BY (sequence_nr DESC)"));
assert!(ddl.contains("datum_persistence.durable_state"));
assert!(ddl.contains("datum_persistence.durable_state_change_log"));
assert!(ddl.contains("PRIMARY KEY (tag, timestamp_ms, revision, persistence_id)"));
}
#[test]
fn atomic_write_crossing_a_partition_is_pre_storage_rejection() {
let persistence_id = PersistenceId::new("test", "partition").expect("id is valid");
let event = |sequence_nr| StoredEvent {
persistence_id: persistence_id.clone(),
sequence_nr: SequenceNr::new(sequence_nr),
adapter_manifest: String::new(),
writer_uuid: "writer".into(),
timestamp_ms: 0,
metadata: None,
payload: SerializedPayload::new(SerializerId::new(1), "", Vec::new()),
tags: Vec::new(),
};
let write = AtomicWrite::new(vec![event(2), event(3)]).expect("write is gapless");
let rejection = PreparedAtomicWrite::new(write, 2).expect_err("write crosses partitions");
assert!(rejection.contains("crosses Cassandra journal partitions"));
}
}