use std::path::{Path, PathBuf};
use crate::{codec::CodecId, limits, prefix::PrefixExtractor, runtime::RuntimeOptions};
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum StorageMode {
InMemory,
Persistent {
path: PathBuf,
},
HostPersistent {
backend: HostStorageBackend,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum HostStorageBackend {
Wasi {
path: PathBuf,
},
Browser {
path: PathBuf,
},
ObjectStore,
}
impl HostStorageBackend {
pub(crate) const fn as_str(&self) -> &'static str {
match self {
Self::Wasi { .. } => "WASI persistent storage backend",
Self::Browser { .. } => "browser persistent storage backend",
Self::ObjectStore => "object-store persistent storage backend",
}
}
}
impl StorageMode {
pub(crate) fn persistent_path(&self) -> Option<&Path> {
match self {
Self::Persistent { path }
| Self::HostPersistent {
backend: HostStorageBackend::Wasi { path },
} => Some(path.as_path()),
Self::InMemory
| Self::HostPersistent {
backend: HostStorageBackend::Browser { .. } | HostStorageBackend::ObjectStore,
} => None,
}
}
#[cfg_attr(
not(all(target_arch = "wasm32", target_os = "unknown")),
allow(dead_code)
)]
pub(crate) fn browser_path(&self) -> Option<&Path> {
match self {
Self::HostPersistent {
backend: HostStorageBackend::Browser { path },
} => Some(path.as_path()),
Self::InMemory
| Self::Persistent { .. }
| Self::HostPersistent {
backend: HostStorageBackend::Wasi { .. } | HostStorageBackend::ObjectStore,
} => None,
}
}
pub(crate) const fn is_wasi_persistent(&self) -> bool {
matches!(
self,
Self::HostPersistent {
backend: HostStorageBackend::Wasi { .. }
}
)
}
pub(crate) const fn is_browser_persistent(&self) -> bool {
matches!(
self,
Self::HostPersistent {
backend: HostStorageBackend::Browser { .. }
}
)
}
pub(crate) const fn is_object_store_persistent(&self) -> bool {
matches!(
self,
Self::HostPersistent {
backend: HostStorageBackend::ObjectStore
}
)
}
}
impl Default for StorageMode {
fn default() -> Self {
Self::InMemory
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum DurabilityMode {
#[default]
Buffered,
Flush,
SyncData,
SyncAll,
SyncAllStrict,
}
impl DurabilityMode {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Buffered => "buffered",
Self::Flush => "flush",
Self::SyncData => "sync-data",
Self::SyncAll => "sync-all",
Self::SyncAllStrict => "sync-all-strict",
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum CompressionProfile {
None,
#[default]
Fast,
}
impl CompressionProfile {
#[must_use]
pub(crate) const fn codec_id(self) -> CodecId {
match self {
Self::None => CodecId::None,
Self::Fast => CodecId::FastLz4Block,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FilterPolicy {
Disabled,
Bloom {
bits_per_key: u8,
},
}
impl Default for FilterPolicy {
fn default() -> Self {
Self::Bloom { bits_per_key: 10 }
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PrefixFilterPolicy {
Disabled,
Bloom {
bits_per_prefix: u8,
},
}
impl Default for PrefixFilterPolicy {
fn default() -> Self {
Self::Bloom {
bits_per_prefix: 10,
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum IndexSearchPolicy {
Linear,
Binary,
#[default]
Auto,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum FailOnCorruptionPolicy {
#[default]
FailClosed,
RepairSafeTemporaryFiles,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DbOptions {
pub storage_mode: StorageMode,
pub create_if_missing: bool,
pub read_only: bool,
pub default_bucket_options: BucketOptions,
pub durability: DurabilityMode,
pub write_buffer_bytes: usize,
pub max_immutable_memtables: usize,
pub target_table_bytes: usize,
pub level_size_multiplier: usize,
pub max_l0_files: usize,
pub object_client_trust: ObjectClientTrustMode,
pub max_key_bytes: usize,
pub max_value_bytes: usize,
pub block_cache_bytes: usize,
pub background_worker_count: usize,
pub wal_shards: WalShardPolicy,
pub runtime: RuntimeOptions,
pub fail_on_corruption: FailOnCorruptionPolicy,
pub keep_last_read_versions: u64,
pub blob_gc_enabled: bool,
pub blob_gc_discardable_ratio: BlobGcRatio,
pub blob_gc_min_file_bytes: u64,
}
impl DbOptions {
pub const DEFAULT_WRITE_BUFFER_BYTES: usize = 64 * 1024 * 1024;
pub const DEFAULT_TARGET_TABLE_BYTES: usize = 64 * 1024 * 1024;
pub const DEFAULT_MAX_KEY_BYTES: usize = limits::MAX_DECODED_BLOCK_BYTES;
pub const DEFAULT_MAX_VALUE_BYTES: usize = limits::MAX_DECODED_BLOCK_BYTES;
pub const MAX_WRITE_FIELD_BYTES: usize = limits::MAX_DECODED_BLOCK_BYTES;
pub const DEFAULT_BLOCK_CACHE_BYTES: usize = 256 * 1024 * 1024;
pub const DEFAULT_BLOB_GC_MIN_FILE_BYTES: u64 = 64 * 1024 * 1024;
pub const DEFAULT_KEEP_LAST_READ_VERSIONS: u64 = 1;
#[must_use]
pub fn new(path: impl Into<PathBuf>) -> Self {
Self::persistent(path)
}
#[must_use]
pub fn memory() -> Self {
Self::default()
}
#[must_use]
pub fn persistent(path: impl Into<PathBuf>) -> Self {
Self {
storage_mode: StorageMode::Persistent { path: path.into() },
durability: DurabilityMode::SyncAll,
..Self::default()
}
}
#[must_use]
pub fn wasi_persistent(path: impl Into<PathBuf>) -> Self {
Self {
storage_mode: StorageMode::HostPersistent {
backend: HostStorageBackend::Wasi { path: path.into() },
},
background_worker_count: 0,
durability: DurabilityMode::Flush,
runtime: RuntimeOptions::inline(),
..Self::default()
}
}
#[must_use]
pub fn wasi_persistent_read_only(path: impl Into<PathBuf>) -> Self {
Self::wasi_persistent(path).read_only()
}
#[must_use]
pub fn browser_persistent() -> Self {
Self::browser_persistent_at("")
}
#[must_use]
pub fn browser_persistent_at(path: impl Into<PathBuf>) -> Self {
Self {
storage_mode: StorageMode::HostPersistent {
backend: HostStorageBackend::Browser { path: path.into() },
},
background_worker_count: 0,
durability: DurabilityMode::Flush,
runtime: RuntimeOptions::inline(),
..Self::default()
}
}
#[doc(hidden)]
#[must_use]
pub fn object_store() -> Self {
Self {
storage_mode: StorageMode::HostPersistent {
backend: HostStorageBackend::ObjectStore,
},
background_worker_count: 0,
durability: DurabilityMode::Flush,
runtime: RuntimeOptions::inline(),
..Self::default()
}
}
#[must_use]
pub fn browser_persistent_read_only() -> Self {
Self::browser_persistent().read_only()
}
#[must_use]
pub fn browser_persistent_read_only_at(path: impl Into<PathBuf>) -> Self {
Self::browser_persistent_at(path).read_only()
}
#[must_use]
pub fn persistent_read_only(path: impl Into<PathBuf>) -> Self {
Self::persistent(path).read_only()
}
#[must_use]
pub const fn with_durability(mut self, durability: DurabilityMode) -> Self {
self.durability = durability;
self
}
#[must_use]
pub fn with_default_bucket_options(mut self, options: BucketOptions) -> Self {
self.default_bucket_options = options;
self
}
#[must_use]
pub const fn read_only(mut self) -> Self {
self.read_only = true;
self.create_if_missing = false;
self
}
#[must_use]
pub const fn with_keep_last_read_versions(mut self, count: u64) -> Self {
self.keep_last_read_versions = count;
self
}
#[must_use]
pub const fn with_wal_shards(mut self, policy: WalShardPolicy) -> Self {
self.wal_shards = policy;
self
}
#[must_use]
pub const fn with_wal_shard_count(mut self, count: usize) -> Self {
self.wal_shards = WalShardPolicy::Fixed(count);
self
}
#[must_use]
pub const fn with_object_client_trust(mut self, trust: ObjectClientTrustMode) -> Self {
self.object_client_trust = trust;
self
}
#[must_use]
pub const fn with_max_key_bytes(mut self, max_key_bytes: usize) -> Self {
self.max_key_bytes = max_key_bytes;
self
}
#[must_use]
pub const fn with_max_value_bytes(mut self, max_value_bytes: usize) -> Self {
self.max_value_bytes = max_value_bytes;
self
}
}
impl Default for DbOptions {
fn default() -> Self {
Self {
storage_mode: StorageMode::InMemory,
create_if_missing: true,
read_only: false,
default_bucket_options: BucketOptions::default(),
durability: DurabilityMode::Buffered,
write_buffer_bytes: Self::DEFAULT_WRITE_BUFFER_BYTES,
max_immutable_memtables: 4,
target_table_bytes: Self::DEFAULT_TARGET_TABLE_BYTES,
level_size_multiplier: 10,
max_l0_files: 8,
object_client_trust: ObjectClientTrustMode::default(),
max_key_bytes: Self::DEFAULT_MAX_KEY_BYTES,
max_value_bytes: Self::DEFAULT_MAX_VALUE_BYTES,
block_cache_bytes: Self::DEFAULT_BLOCK_CACHE_BYTES,
background_worker_count: 1,
wal_shards: WalShardPolicy::Auto,
runtime: RuntimeOptions::default(),
fail_on_corruption: FailOnCorruptionPolicy::FailClosed,
keep_last_read_versions: Self::DEFAULT_KEEP_LAST_READ_VERSIONS,
blob_gc_enabled: true,
blob_gc_discardable_ratio: BlobGcRatio::HALF,
blob_gc_min_file_bytes: Self::DEFAULT_BLOB_GC_MIN_FILE_BYTES,
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum ObjectClientTrustMode {
#[default]
Trusted,
VerifyOnOpen,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WalShardPolicy {
Auto,
Fixed(usize),
}
impl WalShardPolicy {
const AUTO_PARALLEL_LANES: usize = crate::wal::DEFAULT_WAL_SHARD_COUNT;
pub(crate) fn resolve(self, storage_mode: &StorageMode, durability: DurabilityMode) -> usize {
match self {
Self::Fixed(count) => count.max(1),
Self::Auto => {
if storage_mode.is_wasi_persistent() || storage_mode.is_browser_persistent() {
return 1;
}
let per_commit_fsync = matches!(storage_mode, StorageMode::Persistent { .. })
&& matches!(
durability,
DurabilityMode::SyncData
| DurabilityMode::SyncAll
| DurabilityMode::SyncAllStrict
);
if per_commit_fsync {
1
} else {
Self::AUTO_PARALLEL_LANES
}
}
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct BlobGcRatio {
millionths: u32,
}
impl BlobGcRatio {
pub const HALF: Self = Self {
millionths: 500_000,
};
pub const FULL: Self = Self {
millionths: 1_000_000,
};
#[must_use]
pub const fn from_millionths(millionths: u32) -> Self {
Self { millionths }
}
#[must_use]
pub const fn millionths(self) -> u32 {
self.millionths
}
pub(crate) fn should_collect(self, discardable_bytes: u64, total_bytes: u64) -> bool {
if total_bytes == 0 {
return false;
}
u128::from(discardable_bytes).saturating_mul(1_000_000)
>= u128::from(total_bytes).saturating_mul(u128::from(self.millionths))
}
}
impl Default for BlobGcRatio {
fn default() -> Self {
Self::HALF
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum BlobLevelMergePolicy {
Disabled,
#[default]
Auto,
Always,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum FilterDepthCurve {
#[default]
Auto,
Uniform,
Custom {
step: u8,
floor: u8,
},
CostWeighted {
step: u8,
ceil: u8,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BucketOptions {
pub allow_empty_keys: bool,
pub compression: CompressionProfile,
pub block_bytes: usize,
pub filter_policy: FilterPolicy,
pub prefix_extractor: PrefixExtractor,
pub prefix_filter_policy: PrefixFilterPolicy,
pub index_search_policy: IndexSearchPolicy,
pub blob_threshold_bytes: usize,
pub blob_level_merge_policy: BlobLevelMergePolicy,
pub filter_depth_curve: FilterDepthCurve,
}
impl BucketOptions {
pub const DEFAULT_BLOCK_BYTES: usize = 16 * 1024;
pub const DEFAULT_BLOB_THRESHOLD_BYTES: usize = 1024 * 1024;
#[must_use]
pub fn with_prefix_extractor(mut self, prefix_extractor: PrefixExtractor) -> Self {
self.prefix_extractor = prefix_extractor;
self
}
#[must_use]
pub const fn with_blob_threshold_bytes(mut self, blob_threshold_bytes: usize) -> Self {
self.blob_threshold_bytes = blob_threshold_bytes;
self
}
#[must_use]
pub const fn with_blob_level_merge_policy(mut self, policy: BlobLevelMergePolicy) -> Self {
self.blob_level_merge_policy = policy;
self
}
#[must_use]
pub const fn with_blob_level_merge_enabled(mut self, enabled: bool) -> Self {
self.blob_level_merge_policy = if enabled {
BlobLevelMergePolicy::Always
} else {
BlobLevelMergePolicy::Disabled
};
self
}
#[must_use]
pub const fn with_filter_depth_curve(mut self, curve: FilterDepthCurve) -> Self {
self.filter_depth_curve = curve;
self
}
}
impl Default for BucketOptions {
fn default() -> Self {
Self {
allow_empty_keys: true,
compression: CompressionProfile::Fast,
block_bytes: Self::DEFAULT_BLOCK_BYTES,
filter_policy: FilterPolicy::default(),
prefix_extractor: PrefixExtractor::default(),
prefix_filter_policy: PrefixFilterPolicy::default(),
index_search_policy: IndexSearchPolicy::Auto,
blob_threshold_bytes: Self::DEFAULT_BLOB_THRESHOLD_BYTES,
blob_level_merge_policy: BlobLevelMergePolicy::Auto,
filter_depth_curve: FilterDepthCurve::Auto,
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct WriteOptions {
pub durability: DurabilityMode,
}
impl WriteOptions {
#[must_use]
pub const fn new(durability: DurabilityMode) -> Self {
Self { durability }
}
#[must_use]
pub const fn buffered() -> Self {
Self::new(DurabilityMode::Buffered)
}
#[must_use]
pub const fn flush() -> Self {
Self::new(DurabilityMode::Flush)
}
#[must_use]
pub const fn sync_data() -> Self {
Self::new(DurabilityMode::SyncData)
}
#[must_use]
pub const fn sync_all() -> Self {
Self::new(DurabilityMode::SyncAll)
}
#[must_use]
pub const fn sync_all_strict() -> Self {
Self::new(DurabilityMode::SyncAllStrict)
}
}
#[cfg(test)]
mod tests {
use super::{DurabilityMode, HostStorageBackend, StorageMode, WalShardPolicy};
use std::path::PathBuf;
fn persistent() -> StorageMode {
StorageMode::Persistent {
path: PathBuf::from("/tmp/trine-wal-policy-test"),
}
}
fn wasi_persistent() -> StorageMode {
StorageMode::HostPersistent {
backend: HostStorageBackend::Wasi {
path: PathBuf::from("/tmp/trine-wasi-wal-policy-test"),
},
}
}
fn browser_persistent() -> StorageMode {
StorageMode::HostPersistent {
backend: HostStorageBackend::Browser {
path: PathBuf::from("/trine-browser-wal-policy-test"),
},
}
}
#[test]
fn auto_uses_one_lane_for_per_commit_fsync_durability() {
assert_eq!(
WalShardPolicy::Auto.resolve(&persistent(), DurabilityMode::SyncAll),
1
);
assert_eq!(
WalShardPolicy::Auto.resolve(&persistent(), DurabilityMode::SyncData),
1
);
}
#[test]
fn auto_uses_parallel_lanes_without_per_commit_fsync() {
assert_eq!(
WalShardPolicy::Auto.resolve(&persistent(), DurabilityMode::Buffered),
WalShardPolicy::AUTO_PARALLEL_LANES
);
assert_eq!(
WalShardPolicy::Auto.resolve(&StorageMode::InMemory, DurabilityMode::SyncAll),
WalShardPolicy::AUTO_PARALLEL_LANES
);
}
#[test]
fn fixed_clamps_to_at_least_one_lane() {
assert_eq!(
WalShardPolicy::Fixed(0).resolve(&persistent(), DurabilityMode::SyncAll),
1
);
assert_eq!(
WalShardPolicy::Fixed(8).resolve(&persistent(), DurabilityMode::SyncAll),
8
);
}
#[test]
fn auto_uses_one_lane_for_wasm_host_backends() {
assert_eq!(
WalShardPolicy::Auto.resolve(&wasi_persistent(), DurabilityMode::Flush),
1
);
assert_eq!(
WalShardPolicy::Auto.resolve(&browser_persistent(), DurabilityMode::Flush),
1
);
}
#[test]
fn fixed_overrides_wasm_host_backend_lane_count() {
assert_eq!(
WalShardPolicy::Fixed(3).resolve(&browser_persistent(), DurabilityMode::Flush),
3
);
}
}