datum-persistence 0.11.3

Event sourcing, durable state, and CQRS persistence for Datum
Documentation
use std::{
    env,
    path::{Path, PathBuf},
    process::Command as ProcessCommand,
};

use tempfile::TempDir;

use super::*;
use crate::{
    PersistenceId, SequenceNr,
    projection::{
        BoundaryEvent, CrashSaveRequest, DeduplicationBoundary,
        DurableProjectionOffsetStoreHarness, ProviderManifest, SaveCrashOutcome, SaveCrashPoint,
        assert_projection_offset_store_durable_conformance,
        assert_projection_offset_store_semantic_conformance,
    },
};

const CRASH_CHILD_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_CHILD";
const CRASH_PATH_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_PATH";
const CRASH_POINT_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_POINT";
const CRASH_NAME_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_NAME";
const CRASH_KEY_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_KEY";
const CRASH_GENERATION_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_GENERATION";
const CRASH_EXPECTED_PRESENT_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_EXPECTED_PRESENT";
const CRASH_EXPECTED_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_EXPECTED";
const CRASH_CURSOR_ENV: &str = "DATUM_ROCKSDB_PROJECTION_CRASH_CURSOR";
const CRASH_CHILD_TEST: &str = "backend::rocksdb::projection_tests::rocksdb_projection_crash_child";
const BEFORE_COMMIT_EXIT: i32 = 86;
const AFTER_COMMIT_EXIT: i32 = 87;

struct RocksDbDurableHarness {
    _directory: TempDir,
    path: PathBuf,
    migration_path: PathBuf,
}

impl RocksDbDurableHarness {
    fn new() -> Self {
        let directory = tempfile::tempdir().expect("temporary RocksDB projection directory");
        let path = directory.path().join("current");
        let migration_path = directory.path().join("m15-migration");
        Self {
            _directory: directory,
            path,
            migration_path,
        }
    }
}

#[async_trait::async_trait]
impl DurableProjectionOffsetStoreHarness for RocksDbDurableHarness {
    async fn create_new(&self) -> ProjectionResult<Box<dyn ProjectionOffsetStore>> {
        open_projection_store(&self.path)
    }

    async fn reopen_existing(&self) -> ProjectionResult<Box<dyn ProjectionOffsetStore>> {
        open_projection_store(&self.path)
    }

    async fn exercise_reopen_migration(&self) -> ProjectionResult<()> {
        create_m15_database(&self.migration_path)?;

        let store = RocksDbBackend::open(&self.migration_path).map_err(projection_open_error)?;
        let projection_id = ProjectionId::new("rocksdb-migration", "projection-cf");
        let generation = store.claim(&projection_id).await?;
        let cursor = migration_cursor();
        ProjectionOffsetStore::save(&store, &projection_id, generation, None, cursor.clone())
            .await?;
        store.set_paused(&projection_id, true).await?;
        drop(store);

        let column_families =
            DB::list_cf(&DbOptions::default(), &self.migration_path).map_err(|error| {
                ProjectionError::OffsetStore(format!(
                    "failed to list migrated RocksDB column families: {error}"
                ))
            })?;
        assert!(
            column_families
                .iter()
                .any(|column_family| column_family == PROJECTION_CF),
            "normal reopen must add the projection column family"
        );

        let db = open_database(&self.migration_path).map_err(projection_open_error)?;
        let meta = cf(&db, META_CF).map_err(projection_open_error)?;
        let sentinel = db
            .get_cf(&meta, b"m15-migration-sentinel")
            .map_err(|error| projection_read_error("M15 migration sentinel read", error))?;
        assert_eq!(
            sentinel.as_deref(),
            Some(b"legacy-data".as_slice()),
            "adding the projection column family must preserve M15 data"
        );
        assert_eq!(
            load_projection_cursor(&db, &projection_id)?,
            Some(cursor),
            "the newly added projection column family must be usable after reopen"
        );
        assert!(load_projection_paused(&db, &projection_id)?);
        Ok(())
    }

    async fn save_with_crash(
        &self,
        request: CrashSaveRequest,
        crash_point: SaveCrashPoint,
    ) -> ProjectionResult<SaveCrashOutcome> {
        let point = match crash_point {
            SaveCrashPoint::BeforeCommit => "before",
            SaveCrashPoint::AfterCommitBeforeAcknowledgement => "after",
        };
        let expected_exit = match crash_point {
            SaveCrashPoint::BeforeCommit => BEFORE_COMMIT_EXIT,
            SaveCrashPoint::AfterCommitBeforeAcknowledgement => AFTER_COMMIT_EXIT,
        };
        let expected_position = request
            .expected_position
            .as_ref()
            .map(CursorPosition::as_str);
        let status = ProcessCommand::new(env::current_exe().map_err(|error| {
            ProjectionError::OffsetStore(format!(
                "failed to locate the RocksDB crash-test executable: {error}"
            ))
        })?)
        .arg("--exact")
        .arg(CRASH_CHILD_TEST)
        .arg("--nocapture")
        .env(CRASH_CHILD_ENV, "1")
        .env(CRASH_PATH_ENV, &self.path)
        .env(CRASH_POINT_ENV, point)
        .env(CRASH_NAME_ENV, request.projection_id.name())
        .env(CRASH_KEY_ENV, request.projection_id.key())
        .env(
            CRASH_GENERATION_ENV,
            request.generation.as_u64().to_string(),
        )
        .env(
            CRASH_EXPECTED_PRESENT_ENV,
            if expected_position.is_some() {
                "1"
            } else {
                "0"
            },
        )
        .env(CRASH_EXPECTED_ENV, expected_position.unwrap_or_default())
        .env(CRASH_CURSOR_ENV, request.cursor.encode())
        .status()
        .map_err(|error| {
            ProjectionError::OffsetStore(format!(
                "failed to start the RocksDB crash-test process: {error}"
            ))
        })?;
        if status.code() != Some(expected_exit) {
            return Err(ProjectionError::OffsetStore(format!(
                "RocksDB crash-test process exited with {status}; expected code {expected_exit}"
            )));
        }
        Ok(match crash_point {
            SaveCrashPoint::BeforeCommit => SaveCrashOutcome::NotCommitted,
            SaveCrashPoint::AfterCommitBeforeAcknowledgement => {
                SaveCrashOutcome::CommitOutcomeUncertain
            }
        })
    }
}

fn open_projection_store(path: &Path) -> ProjectionResult<Box<dyn ProjectionOffsetStore>> {
    RocksDbBackend::open(path)
        .map(|store| Box::new(store) as Box<dyn ProjectionOffsetStore>)
        .map_err(projection_open_error)
}

fn projection_open_error(error: PersistenceError) -> ProjectionError {
    ProjectionError::OffsetStore(error.to_string())
}

fn create_m15_database(path: &Path) -> ProjectionResult<()> {
    let mut options = DbOptions::default();
    options.create_if_missing(true);
    options.create_missing_column_families(true);
    let descriptors = vec![
        ColumnFamilyDescriptor::new("default", column_family_options(false)),
        ColumnFamilyDescriptor::new(JOURNAL_CF, column_family_options(true)),
        ColumnFamilyDescriptor::new(SNAPSHOTS_CF, column_family_options(true)),
        ColumnFamilyDescriptor::new(DURABLE_STATE_CF, column_family_options(true)),
        ColumnFamilyDescriptor::new(TAGS_CF, column_family_options(false)),
        ColumnFamilyDescriptor::new(META_CF, column_family_options(false)),
    ];
    let db = DB::open_cf_descriptors(&options, path, descriptors).map_err(|error| {
        ProjectionError::OffsetStore(format!(
            "failed to create the M15 RocksDB migration fixture: {error}"
        ))
    })?;
    assert!(
        db.cf_handle(PROJECTION_CF).is_none(),
        "the M15 fixture must predate the projection column family"
    );
    let meta = db
        .cf_handle(META_CF)
        .expect("the exact M15 descriptor set contains meta");
    db.put_cf(&meta, b"m15-migration-sentinel", b"legacy-data")
        .map_err(|error| {
            ProjectionError::OffsetStore(format!(
                "failed to write the M15 RocksDB migration sentinel: {error}"
            ))
        })
}

fn migration_cursor() -> ProjectionCursor {
    ProjectionCursor::new(
        ProviderManifest::new("datum-rocksdb-migration/provider-v1")
            .expect("fixed provider manifest is valid"),
        DeduplicationBoundary::with_consumed(
            "shared-position-λ",
            [
                BoundaryEvent::new(
                    PersistenceId::new("rocksdb-migration", "first")
                        .expect("fixed persistence id is valid"),
                    SequenceNr::new(7),
                ),
                BoundaryEvent::new(
                    PersistenceId::new("rocksdb-migration", "second")
                        .expect("fixed persistence id is valid"),
                    SequenceNr::new(11),
                ),
            ],
        ),
    )
}

fn required_env(name: &str) -> ProjectionResult<String> {
    env::var(name).map_err(|error| {
        ProjectionError::OffsetStore(format!(
            "RocksDB projection crash child is missing {name}: {error}"
        ))
    })
}

#[tokio::test(flavor = "current_thread")]
async fn rocksdb_projection_offset_store_passes_semantic_conformance() -> ProjectionResult<()> {
    let directory = tempfile::tempdir().expect("temporary RocksDB projection directory");
    let store = RocksDbBackend::open(directory.path()).map_err(projection_open_error)?;
    assert_projection_offset_store_semantic_conformance(&store).await
}

#[tokio::test(flavor = "current_thread")]
async fn rocksdb_projection_offset_store_passes_durable_conformance() -> ProjectionResult<()> {
    assert_projection_offset_store_durable_conformance(&RocksDbDurableHarness::new()).await
}

#[tokio::test(flavor = "current_thread")]
async fn rocksdb_projection_reopens_m15_database_and_adds_projection_cf() -> ProjectionResult<()> {
    RocksDbDurableHarness::new()
        .exercise_reopen_migration()
        .await
}

#[tokio::test(flavor = "current_thread")]
async fn rocksdb_projection_crash_child() -> ProjectionResult<()> {
    if env::var_os(CRASH_CHILD_ENV).is_none() {
        return Ok(());
    }

    let path = required_env(CRASH_PATH_ENV)?;
    let point = match required_env(CRASH_POINT_ENV)?.as_str() {
        "before" => ProjectionSaveCrashPoint::BeforeCommit,
        "after" => ProjectionSaveCrashPoint::AfterCommitBeforeAcknowledgement,
        value => {
            return Err(ProjectionError::OffsetStore(format!(
                "unknown RocksDB projection crash point {value:?}"
            )));
        }
    };
    let projection_id =
        ProjectionId::new(required_env(CRASH_NAME_ENV)?, required_env(CRASH_KEY_ENV)?);
    let generation = required_env(CRASH_GENERATION_ENV)?
        .parse::<u64>()
        .map(ProjectionGeneration::new)
        .map_err(|error| {
            ProjectionError::OffsetStore(format!(
                "invalid RocksDB projection crash generation: {error}"
            ))
        })?;
    let expected_position = match required_env(CRASH_EXPECTED_PRESENT_ENV)?.as_str() {
        "0" => None,
        "1" => Some(CursorPosition::new(required_env(CRASH_EXPECTED_ENV)?)),
        value => {
            return Err(ProjectionError::OffsetStore(format!(
                "invalid RocksDB projection expected-position marker {value:?}"
            )));
        }
    };
    let cursor = ProjectionCursor::decode(&required_env(CRASH_CURSOR_ENV)?)?;
    let store = RocksDbBackend::open(path).map_err(projection_open_error)?;
    store
        .crash_projection_save(projection_id, generation, expected_position, cursor, point)
        .await
}