armdb 0.15.3

sharded bitcask key-value storage optimized for NVMe
Documentation
//! Configuration of a segmented collection.

use std::time::{Duration, Instant};

use crate::config::{Config, IoBackend};
use crate::error::{DbError, DbResult};

const MIN_BLOCK_SIZE: u32 = 256;
const MAX_BLOCK_SIZE: u32 = 16 << 20;

/// Configuration of a segmented collection. Every field may change between
/// opens; `shard_count` applies to generations created from then on.
#[derive(Debug, Clone)]
pub struct SegConfig {
    /// Freeze the active generation when its index holds this many keys,
    /// tombstones included. Bounds the index RAM.
    pub seal_max_keys: u64,
    /// Freeze the active generation when its journal holds this many bytes.
    pub seal_max_bytes: u64,
    /// Freeze the active generation when it is this old and not empty.
    pub seal_max_age: Duration,
    /// Drop a segment this long after it was frozen. `None` keeps everything.
    pub retention: Option<Duration>,
    /// Target size of a data block in a sealed segment.
    pub block_size: u32,
    /// Budget of the cache of sealed blocks. `0` disables it.
    pub block_cache_bytes: u64,
    /// Hot sealed metadata budget, newest first. `None` keeps all metadata hot;
    /// `Some(0)` pages every segment. Changes apply at the next open.
    pub resident_metadata_bytes: Option<u64>,
    /// Generations that may wait for a seal before writers are held back.
    pub max_frozen_generations: usize,
    /// How long a write waits for room before it fails with
    /// `SealBackpressure`.
    pub seal_wait_timeout: Duration,
    /// How often age and retention are evaluated.
    pub maintenance_interval: Duration,
    /// Shards of a new generation.
    pub shard_count: usize,
    /// Size of each generation journal write buffer.
    pub write_buffer_size: usize,
    /// Maximum size of a generation journal file.
    pub max_file_size: u64,
    /// Writes per generation journal durability sync.
    pub sync_batch_size: u32,
    /// Optional interval for generation journal durability syncs.
    pub sync_interval: Option<Duration>,
    /// Whether generation journals write recovery hints.
    pub hints: bool,
    /// I/O backend used by generation journals.
    pub io_backend: IoBackend,
}

/// Size thresholds of a generation.
#[derive(Debug, Clone, Copy)]
pub(crate) struct Limits {
    pub max_keys: u64,
    pub max_bytes: u64,
}

impl SegConfig {
    /// Balanced thresholds and journal settings for general workloads.
    pub fn balanced() -> Self {
        let journal = Config::balanced().build();
        Self {
            // Measured index and recovery scratch exceed the 8 GB envelope at 24M.
            // See docs/benches/26-09-29-segmented.md for the conservative projection.
            seal_max_keys: 6_000_000,
            seal_max_bytes: 16 << 30,
            seal_max_age: Duration::from_secs(24 * 3600),
            retention: None,
            block_size: 16 * 1024,
            block_cache_bytes: 256 << 20,
            resident_metadata_bytes: None,
            max_frozen_generations: 1,
            seal_wait_timeout: Duration::from_secs(30),
            maintenance_interval: Duration::from_secs(60),
            shard_count: journal.shard_count,
            write_buffer_size: journal.write_buffer_size,
            max_file_size: journal.max_file_size,
            sync_batch_size: journal.sync_batch_size,
            sync_interval: journal.sync_interval,
            hints: true,
            io_backend: journal.io_backend,
        }
    }

    /// Balanced byte-ordered generation thresholds using the measured companion index budget.
    pub fn tree_balanced() -> Self {
        Self {
            seal_max_keys: 3_600_000,
            ..Self::balanced()
        }
    }

    /// Small thresholds so that freeze and seal happen inside a unit test.
    pub fn test() -> Self {
        let config = Self {
            seal_max_keys: 1_000,
            seal_max_bytes: 4 << 20,
            block_size: 1024,
            block_cache_bytes: 1 << 20,
            seal_wait_timeout: Duration::from_millis(500),
            shard_count: 2,
            write_buffer_size: 64 * 1024,
            max_file_size: 1 << 20,
            ..Self::balanced()
        };
        #[cfg(any(test, feature = "test-faults"))]
        if let Ok(value) = std::env::var("ARMDB_SEG_TEST_RESIDENT_BYTES") {
            return Self {
                resident_metadata_bytes: (value != "none")
                    .then(|| value.parse().expect("ARMDB_SEG_TEST_RESIDENT_BYTES")),
                ..config
            };
        }
        config
    }

    /// Validate generation thresholds, block size and journal settings.
    pub fn validate(&self) -> DbResult<()> {
        if self.shard_count == 0 || self.shard_count > 255 {
            return Err(DbError::Config("shard_count must be between 1 and 255"));
        }
        if self.seal_max_keys == 0 {
            return Err(DbError::Config("seal_max_keys must be at least 1"));
        }
        if self.seal_max_bytes == 0 {
            return Err(DbError::Config("seal_max_bytes must be at least 1"));
        }
        // Ages are compared in whole milliseconds: a shorter age would read
        // as zero and freeze the active generation on every tick.
        if self.seal_max_age < Duration::from_millis(1) {
            return Err(DbError::Config("seal_max_age must be at least 1 ms"));
        }
        if self.retention.is_some_and(|r| r.is_zero()) {
            return Err(DbError::Config(
                "retention must not be zero; use None to keep everything",
            ));
        }
        if self.max_frozen_generations == 0 {
            return Err(DbError::Config("max_frozen_generations must be at least 1"));
        }
        // The manifest holds at most `MAX_GENERATIONS` generations, one of
        // them active; a deeper queue would fail at freeze time instead.
        if self.max_frozen_generations >= crate::seg::manifest::MAX_GENERATIONS {
            return Err(DbError::Config(
                "max_frozen_generations must be at most 1023",
            ));
        }
        if !(MIN_BLOCK_SIZE..=MAX_BLOCK_SIZE).contains(&self.block_size) {
            return Err(DbError::Config(
                "block_size must be between 256 bytes and 16 MiB",
            ));
        }
        self.seal_deadline()?;
        self.journal(self.shard_count).validate()
    }

    pub(crate) fn seal_deadline(&self) -> DbResult<Instant> {
        Instant::now()
            .checked_add(self.seal_wait_timeout)
            .ok_or(DbError::Config("seal_wait_timeout exceeds the clock range"))
    }

    pub(crate) fn limits(&self) -> Limits {
        Limits {
            max_keys: self.seal_max_keys,
            max_bytes: self.seal_max_bytes,
        }
    }

    /// The Bitcask configuration of one generation. `shard_count` is the
    /// generation's own: the current setting for a new one, the recorded
    /// value for an existing one.
    pub(crate) fn journal(&self, shard_count: usize) -> Config {
        Config {
            shard_count,
            shard_prefix_bits: 0,
            write_buffer_size: self.write_buffer_size,
            max_file_size: self.max_file_size,
            sync_batch_size: self.sync_batch_size,
            sync_interval: self.sync_interval,
            hints: Some(self.hints),
            io_backend: self.io_backend,
            iterable: false,
            direct_io: false,
            ..Config::balanced().build()
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::time::Duration;

    #[test]
    fn the_balanced_preset_has_the_documented_defaults() {
        let c = SegConfig::balanced();
        assert_eq!(c.seal_max_keys, 6_000_000);
        assert_eq!(c.seal_max_bytes, 16 << 30);
        assert_eq!(c.seal_max_age, Duration::from_secs(24 * 3600));
        assert_eq!(c.retention, None);
        assert_eq!(c.block_size, 16 * 1024);
        assert_eq!(c.block_cache_bytes, 256 << 20);
        assert_eq!(c.max_frozen_generations, 1);
        assert_eq!(c.seal_wait_timeout, Duration::from_secs(30));
        assert_eq!(c.maintenance_interval, Duration::from_secs(60));
        assert!(c.hints);
        c.validate().unwrap();
    }

    #[test]
    fn the_test_preset_is_small_and_valid() {
        let c = SegConfig::test();
        assert_eq!(c.shard_count, 2);
        assert!(c.seal_max_keys <= 10_000);
        assert!(c.block_size <= 4 * 1024);
        c.validate().unwrap();
    }

    #[test]
    fn the_journal_config_never_routes_by_key_prefix() {
        let c = SegConfig::test();
        let journal = c.journal(7);
        assert_eq!(journal.shard_count, 7, "the generation's own count wins");
        assert_eq!(journal.shard_prefix_bits, 0);
        assert_eq!(journal.hints, Some(true));
        assert_eq!(journal.write_buffer_size, c.write_buffer_size);
        assert_eq!(journal.max_file_size, c.max_file_size);
        assert!(!journal.iterable);
        journal.validate().unwrap();
    }

    #[test]
    fn nonsense_is_refused() {
        let bad = [
            SegConfig {
                shard_count: 0,
                ..SegConfig::test()
            },
            SegConfig {
                shard_count: 256,
                ..SegConfig::test()
            },
            SegConfig {
                seal_max_keys: 0,
                ..SegConfig::test()
            },
            SegConfig {
                seal_max_bytes: 0,
                ..SegConfig::test()
            },
            SegConfig {
                max_frozen_generations: 0,
                ..SegConfig::test()
            },
            SegConfig {
                max_frozen_generations: crate::seg::manifest::MAX_GENERATIONS,
                ..SegConfig::test()
            },
            SegConfig {
                block_size: 16,
                ..SegConfig::test()
            },
            SegConfig {
                block_size: 1 << 30,
                ..SegConfig::test()
            },
            SegConfig {
                seal_max_age: Duration::ZERO,
                ..SegConfig::test()
            },
            SegConfig {
                seal_max_age: Duration::from_micros(500),
                ..SegConfig::test()
            },
            SegConfig {
                retention: Some(Duration::ZERO),
                ..SegConfig::test()
            },
        ];
        for config in bad {
            assert!(config.validate().is_err(), "{config:?}");
        }
    }
    #[test]
    fn tree_balanced_changes_only_key_limit() {
        let map = SegConfig::balanced();
        let mut tree = SegConfig::tree_balanced();
        assert_eq!(tree.seal_max_keys, 3_600_000);
        tree.seal_max_keys = map.seal_max_keys;
        assert_eq!(format!("{tree:?}"), format!("{map:?}"));
    }
}