use std::{
env,
path::{Path, PathBuf},
process::Command as ProcessCommand,
};
use tempfile::TempDir;
use super::*;
use crate::{
PersistenceId, SequenceNr,
projection::{
BoundaryEvent, CrashSaveRequest, DeduplicationBoundary,
DurableProjectionOffsetStoreHarness, ProviderManifest, SaveCrashOutcome, SaveCrashPoint,
assert_projection_offset_store_durable_conformance,
assert_projection_offset_store_semantic_conformance,
},
};
const CRASH_CHILD_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_CHILD";
const CRASH_PATH_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_PATH";
const CRASH_POINT_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_POINT";
const CRASH_NAME_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_NAME";
const CRASH_KEY_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_KEY";
const CRASH_GENERATION_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_GENERATION";
const CRASH_EXPECTED_PRESENT_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_EXPECTED_PRESENT";
const CRASH_EXPECTED_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_EXPECTED";
const CRASH_CURSOR_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_CURSOR";
const CRASH_CHILD_TEST: &str = "backend::rocksdb::projection_tests::rocksdb_projection_crash_child";
const BEFORE_COMMIT_EXIT: i32 = 86;
const AFTER_COMMIT_EXIT: i32 = 87;
struct RocksDbDurableHarness {
_directory: TempDir,
path: PathBuf,
migration_path: PathBuf,
}
impl RocksDbDurableHarness {
fn new() -> Self {
let directory = tempfile::tempdir().expect("temporary RocksDB projection directory");
let path = directory.path().join("current");
let migration_path = directory.path().join("m15-migration");
Self {
_directory: directory,
path,
migration_path,
}
}
}
#[async_trait::async_trait]
impl DurableProjectionOffsetStoreHarness for RocksDbDurableHarness {
async fn create_new(&self) -> ProjectionResult<Box<dyn ProjectionOffsetStore>> {
open_projection_store(&self.path)
}
async fn reopen_existing(&self) -> ProjectionResult<Box<dyn ProjectionOffsetStore>> {
open_projection_store(&self.path)
}
async fn exercise_reopen_migration(&self) -> ProjectionResult<()> {
create_m15_database(&self.migration_path)?;
let store = RocksDbBackend::open(&self.migration_path).map_err(projection_open_error)?;
let projection_id = ProjectionId::new("rocksdb-migration", "projection-cf");
let generation = store.claim(&projection_id).await?;
let cursor = migration_cursor();
ProjectionOffsetStore::save(&store, &projection_id, generation, None, cursor.clone())
.await?;
store.set_paused(&projection_id, true).await?;
drop(store);
let column_families =
DB::list_cf(&DbOptions::default(), &self.migration_path).map_err(|error| {
ProjectionError::OffsetStore(format!(
"failed to list migrated RocksDB column families: {error}"
))
})?;
assert!(
column_families
.iter()
.any(|column_family| column_family == PROJECTION_CF),
"normal reopen must add the projection column family"
);
let db = open_database(&self.migration_path).map_err(projection_open_error)?;
let meta = cf(&db, META_CF).map_err(projection_open_error)?;
let sentinel = db
.get_cf(&meta, b"m15-migration-sentinel")
.map_err(|error| projection_read_error("M15 migration sentinel read", error))?;
assert_eq!(
sentinel.as_deref(),
Some(b"legacy-data".as_slice()),
"adding the projection column family must preserve M15 data"
);
assert_eq!(
load_projection_cursor(&db, &projection_id)?,
Some(cursor),
"the newly added projection column family must be usable after reopen"
);
assert!(load_projection_paused(&db, &projection_id)?);
Ok(())
}
async fn save_with_crash(
&self,
request: CrashSaveRequest,
crash_point: SaveCrashPoint,
) -> ProjectionResult<SaveCrashOutcome> {
let point = match crash_point {
SaveCrashPoint::BeforeCommit => "before",
SaveCrashPoint::AfterCommitBeforeAcknowledgement => "after",
};
let expected_exit = match crash_point {
SaveCrashPoint::BeforeCommit => BEFORE_COMMIT_EXIT,
SaveCrashPoint::AfterCommitBeforeAcknowledgement => AFTER_COMMIT_EXIT,
};
let expected_position = request
.expected_position
.as_ref()
.map(CursorPosition::as_str);
let status = ProcessCommand::new(env::current_exe().map_err(|error| {
ProjectionError::OffsetStore(format!(
"failed to locate the RocksDB crash-test executable: {error}"
))
})?)
.arg("--exact")
.arg(CRASH_CHILD_TEST)
.arg("--nocapture")
.env(CRASH_CHILD_ENV, "1")
.env(CRASH_PATH_ENV, &self.path)
.env(CRASH_POINT_ENV, point)
.env(CRASH_NAME_ENV, request.projection_id.name())
.env(CRASH_KEY_ENV, request.projection_id.key())
.env(
CRASH_GENERATION_ENV,
request.generation.as_u64().to_string(),
)
.env(
CRASH_EXPECTED_PRESENT_ENV,
if expected_position.is_some() {
"1"
} else {
"0"
},
)
.env(CRASH_EXPECTED_ENV, expected_position.unwrap_or_default())
.env(CRASH_CURSOR_ENV, request.cursor.encode())
.status()
.map_err(|error| {
ProjectionError::OffsetStore(format!(
"failed to start the RocksDB crash-test process: {error}"
))
})?;
if status.code() != Some(expected_exit) {
return Err(ProjectionError::OffsetStore(format!(
"RocksDB crash-test process exited with {status}; expected code {expected_exit}"
)));
}
Ok(match crash_point {
SaveCrashPoint::BeforeCommit => SaveCrashOutcome::NotCommitted,
SaveCrashPoint::AfterCommitBeforeAcknowledgement => {
SaveCrashOutcome::CommitOutcomeUncertain
}
})
}
}
fn open_projection_store(path: &Path) -> ProjectionResult<Box<dyn ProjectionOffsetStore>> {
RocksDbBackend::open(path)
.map(|store| Box::new(store) as Box<dyn ProjectionOffsetStore>)
.map_err(projection_open_error)
}
fn projection_open_error(error: PersistenceError) -> ProjectionError {
ProjectionError::OffsetStore(error.to_string())
}
fn create_m15_database(path: &Path) -> ProjectionResult<()> {
let mut options = DbOptions::default();
options.create_if_missing(true);
options.create_missing_column_families(true);
let descriptors = vec![
ColumnFamilyDescriptor::new("default", column_family_options(false)),
ColumnFamilyDescriptor::new(JOURNAL_CF, column_family_options(true)),
ColumnFamilyDescriptor::new(SNAPSHOTS_CF, column_family_options(true)),
ColumnFamilyDescriptor::new(DURABLE_STATE_CF, column_family_options(true)),
ColumnFamilyDescriptor::new(TAGS_CF, column_family_options(false)),
ColumnFamilyDescriptor::new(META_CF, column_family_options(false)),
];
let db = DB::open_cf_descriptors(&options, path, descriptors).map_err(|error| {
ProjectionError::OffsetStore(format!(
"failed to create the M15 RocksDB migration fixture: {error}"
))
})?;
assert!(
db.cf_handle(PROJECTION_CF).is_none(),
"the M15 fixture must predate the projection column family"
);
let meta = db
.cf_handle(META_CF)
.expect("the exact M15 descriptor set contains meta");
db.put_cf(&meta, b"m15-migration-sentinel", b"legacy-data")
.map_err(|error| {
ProjectionError::OffsetStore(format!(
"failed to write the M15 RocksDB migration sentinel: {error}"
))
})
}
fn migration_cursor() -> ProjectionCursor {
ProjectionCursor::new(
ProviderManifest::new("datum-rocksdb-migration/provider-v1")
.expect("fixed provider manifest is valid"),
DeduplicationBoundary::with_consumed(
"shared-position-λ",
[
BoundaryEvent::new(
PersistenceId::new("rocksdb-migration", "first")
.expect("fixed persistence id is valid"),
SequenceNr::new(7),
),
BoundaryEvent::new(
PersistenceId::new("rocksdb-migration", "second")
.expect("fixed persistence id is valid"),
SequenceNr::new(11),
),
],
),
)
}
fn required_env(name: &str) -> ProjectionResult<String> {
env::var(name).map_err(|error| {
ProjectionError::OffsetStore(format!(
"RocksDB projection crash child is missing {name}: {error}"
))
})
}
#[tokio::test(flavor = "current_thread")]
async fn rocksdb_projection_offset_store_passes_semantic_conformance() -> ProjectionResult<()> {
let directory = tempfile::tempdir().expect("temporary RocksDB projection directory");
let store = RocksDbBackend::open(directory.path()).map_err(projection_open_error)?;
assert_projection_offset_store_semantic_conformance(&store).await
}
#[tokio::test(flavor = "current_thread")]
async fn rocksdb_projection_offset_store_passes_durable_conformance() -> ProjectionResult<()> {
assert_projection_offset_store_durable_conformance(&RocksDbDurableHarness::new()).await
}
#[tokio::test(flavor = "current_thread")]
async fn rocksdb_projection_reopens_m15_database_and_adds_projection_cf() -> ProjectionResult<()> {
RocksDbDurableHarness::new()
.exercise_reopen_migration()
.await
}
#[tokio::test(flavor = "current_thread")]
async fn rocksdb_projection_crash_child() -> ProjectionResult<()> {
if env::var_os(CRASH_CHILD_ENV).is_none() {
return Ok(());
}
let path = required_env(CRASH_PATH_ENV)?;
let point = match required_env(CRASH_POINT_ENV)?.as_str() {
"before" => ProjectionSaveCrashPoint::BeforeCommit,
"after" => ProjectionSaveCrashPoint::AfterCommitBeforeAcknowledgement,
value => {
return Err(ProjectionError::OffsetStore(format!(
"unknown RocksDB projection crash point {value:?}"
)));
}
};
let projection_id =
ProjectionId::new(required_env(CRASH_NAME_ENV)?, required_env(CRASH_KEY_ENV)?);
let generation = required_env(CRASH_GENERATION_ENV)?
.parse::<u64>()
.map(ProjectionGeneration::new)
.map_err(|error| {
ProjectionError::OffsetStore(format!(
"invalid RocksDB projection crash generation: {error}"
))
})?;
let expected_position = match required_env(CRASH_EXPECTED_PRESENT_ENV)?.as_str() {
"0" => None,
"1" => Some(CursorPosition::new(required_env(CRASH_EXPECTED_ENV)?)),
value => {
return Err(ProjectionError::OffsetStore(format!(
"invalid RocksDB projection expected-position marker {value:?}"
)));
}
};
let cursor = ProjectionCursor::decode(&required_env(CRASH_CURSOR_ENV)?)?;
let store = RocksDbBackend::open(path).map_err(projection_open_error)?;
store
.crash_projection_save(projection_id, generation, expected_position, cursor, point)
.await
}