use fjall::config::{
BlockSizePolicy, BloomConstructionPolicy, CompressionPolicy, FilterPolicy, FilterPolicyEntry,
};
use fjall::{
Config, CompressionType, KeyspaceCreateOptions, KvSeparationOptions, SingleWriterTxDatabase,
};
use nitrite::common::{atomic, Atomic, ReadExecutor, WriteExecutor};
use nitrite::store::{StoreConfigProvider, StoreEventListener};
use std::any::Any;
use std::path::Path;
use std::sync::atomic::{
AtomicBool, AtomicI8, AtomicU16, AtomicU32, AtomicU64, AtomicUsize, Ordering,
};
use std::sync::{Arc, OnceLock};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum Strategy {
#[default]
Leveled,
Fifo,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Durability {
OnCommit,
Periodic,
}
impl Durability {
#[inline]
pub(crate) fn is_on_commit(self) -> bool {
matches!(self, Durability::OnCommit)
}
#[inline]
pub(crate) fn from_on_commit(on_commit: bool) -> Self {
if on_commit {
Durability::OnCommit
} else {
Durability::Periodic
}
}
}
#[derive(Clone)]
pub struct FjallConfig {
inner: Arc<FjallConfigInner>,
}
impl Default for FjallConfig {
fn default() -> Self {
Self::new()
}
}
impl FjallConfig {
#[inline]
pub fn new() -> FjallConfig {
FjallConfig {
inner: Arc::new(FjallConfigInner::new()),
}
}
#[inline]
pub(crate) fn keyspace_config(&self) -> Config {
let worker_threads = self
.inner
.flush_workers()
.max(self.inner.compaction_workers())
.max(1);
#[allow(deprecated)] SingleWriterTxDatabase::builder(Path::new(self.inner.db_path()))
.manual_journal_persist(self.inner.manual_journal_persist())
.worker_threads(worker_threads)
.cache_size(self.inner.block_cache_capacity() + self.inner.blob_cache_capacity())
.max_journaling_size(self.inner.max_journaling_size())
.max_write_buffer_size(Some(self.inner.max_write_buffer_size()))
.into_config()
}
#[inline]
pub(crate) fn partition_config(&self) -> KeyspaceCreateOptions {
let bits = self.inner.bloom_filter_bits();
let filter_policy = if bits <= 0 {
FilterPolicy::disabled()
} else {
FilterPolicy::all(FilterPolicyEntry::Bloom(
BloomConstructionPolicy::BitsPerKey(f32::from(bits)),
))
};
let mut config = KeyspaceCreateOptions::default()
.filter_policy(filter_policy)
.data_block_compression_policy(CompressionPolicy::all(self.inner.compression_type()))
.max_memtable_size(u64::from(self.inner.max_memtable_size()))
.data_block_size_policy(BlockSizePolicy::all(self.inner.block_size()));
config = match self.inner.compaction_strategy() {
Strategy::Leveled => {
config.compaction_strategy(Arc::new(fjall::compaction::Leveled::default()))
}
Strategy::Fifo => config.compaction_strategy(Arc::new(fjall::compaction::Fifo::new(
self.inner.max_journaling_size(),
None,
))),
};
if self.inner.kv_separated() {
config = config.with_kv_separation(Some(
KvSeparationOptions::default()
.staleness_threshold(self.inner.staleness_threshold()),
));
}
config
}
#[inline]
pub fn db_path(&self) -> &str {
self.inner.db_path()
}
#[inline]
pub(crate) fn set_db_path(&self, db_path: &str) {
self.inner.set_db_path(db_path)
}
#[inline]
pub fn manual_journal_persist(&self) -> bool {
self.inner.manual_journal_persist()
}
#[inline]
pub(crate) fn set_manual_journal_persist(&self, v: bool) {
self.inner.set_manual_journal_persist(v)
}
#[inline]
pub fn flush_workers(&self) -> usize {
self.inner.flush_workers()
}
#[inline]
pub(crate) fn set_flush_workers(&self, c: usize) {
self.inner.set_flush_workers(c)
}
#[inline]
pub fn compaction_workers(&self) -> usize {
self.inner.compaction_workers()
}
#[inline]
pub(crate) fn set_compaction_workers(&self, c: usize) {
self.inner.set_compaction_workers(c)
}
#[inline]
pub fn block_cache_capacity(&self) -> u64 {
self.inner.block_cache_capacity()
}
#[inline]
pub(crate) fn set_block_cache_capacity(&self, c: u64) {
self.inner.set_block_cache_capacity(c)
}
#[inline]
pub fn blob_cache_capacity(&self) -> u64 {
self.inner.blob_cache_capacity()
}
#[inline]
pub(crate) fn set_blob_cache_capacity(&self, c: u64) {
self.inner.set_blob_cache_capacity(c)
}
#[inline]
pub fn max_journaling_size(&self) -> u64 {
self.inner.max_journaling_size()
}
#[inline]
pub(crate) fn set_max_journaling_size(&self, s: u64) {
self.inner.set_max_journaling_size(s)
}
#[inline]
pub fn max_write_buffer_size(&self) -> u64 {
self.inner.max_write_buffer_size()
}
#[inline]
pub(crate) fn set_max_write_buffer_size(&self, s: u64) {
self.inner.set_max_write_buffer_size(s)
}
#[inline]
pub fn fsync_frequency(&self) -> u16 {
self.inner.fsync_frequency()
}
#[inline]
pub(crate) fn set_fsync_frequency(&self, f: u16) {
self.inner.set_fsync_frequency(f)
}
#[inline]
pub fn event_listeners(&self) -> Vec<StoreEventListener> {
self.inner.event_listeners()
}
#[inline]
pub(crate) fn add_event_listener(&self, listener: StoreEventListener) {
self.inner.add_event_listener(listener)
}
#[inline]
pub fn commit_before_close(&self) -> bool {
self.inner.commit_before_close()
}
#[inline]
#[allow(dead_code)]
pub(crate) fn set_commit_before_close(&self, v: bool) {
self.inner.set_commit_before_close(v)
}
#[inline]
pub fn durability_on_commit(&self) -> bool {
self.inner.durability_on_commit()
}
#[inline]
pub fn durability(&self) -> Durability {
Durability::from_on_commit(self.inner.durability_on_commit())
}
#[inline]
pub(crate) fn set_durability_on_commit(&self, v: bool) {
self.inner.set_durability_on_commit(v)
}
#[inline]
pub(crate) fn set_durability(&self, durability: Durability) {
self.set_durability_on_commit(durability.is_on_commit())
}
#[inline]
pub fn bloom_filter_bits(&self) -> i8 {
self.inner.bloom_filter_bits()
}
#[inline]
pub(crate) fn set_bloom_filter_bits(&self, b: i8) {
self.inner.set_bloom_filter_bits(b)
}
#[inline]
pub fn compression_type(&self) -> CompressionType {
self.inner.compression_type()
}
#[inline]
pub(crate) fn set_compression_type(&self, ct: CompressionType) {
self.inner.set_compression_type(ct)
}
#[inline]
pub fn compaction_strategy(&self) -> Strategy {
self.inner.compaction_strategy()
}
#[inline]
pub(crate) fn set_compaction_strategy(&self, s: Strategy) {
self.inner.set_compaction_strategy(s)
}
#[inline]
pub fn max_memtable_size(&self) -> u32 {
self.inner.max_memtable_size()
}
#[inline]
pub(crate) fn set_max_memtable_size(&self, s: u32) {
self.inner.set_max_memtable_size(s)
}
#[inline]
pub fn block_size(&self) -> u32 {
self.inner.block_size()
}
#[inline]
pub(crate) fn set_block_size(&self, s: u32) {
self.inner.set_block_size(s)
}
#[inline]
pub fn kv_separated(&self) -> bool {
self.inner.kv_separated()
}
#[inline]
pub(crate) fn set_kv_separated(&self, v: bool) {
self.inner.set_kv_separated(v)
}
#[inline]
pub fn staleness_threshold(&self) -> f32 {
self.inner.staleness_threshold()
}
#[inline]
pub(crate) fn set_staleness_threshold(&self, t: f32) {
self.inner.set_staleness_threshold(t)
}
}
impl StoreConfigProvider for FjallConfig {
fn file_path(&self) -> String {
self.db_path().to_string()
}
fn is_read_only(&self) -> bool {
false
}
fn add_store_listener(&self, listener: StoreEventListener) {
self.add_event_listener(listener)
}
fn as_any(&self) -> &dyn Any {
self
}
}
struct FjallConfigInner {
db_path: OnceLock<String>,
manual_journal_persist: AtomicBool,
flush_workers_count: AtomicUsize,
compaction_workers_count: AtomicUsize,
block_cache_capacity: AtomicU64,
blob_cache_capacity: AtomicU64,
max_journaling_size: AtomicU64,
max_write_buffer_size: AtomicU64,
fsync_frequency: AtomicU16,
event_listeners: Atomic<Vec<StoreEventListener>>,
commit_before_close: AtomicBool,
durability_on_commit: AtomicBool,
bloom_filter_bits: AtomicI8,
compression_type: Atomic<CompressionType>,
compaction_strategy: Atomic<Strategy>,
max_memtable_size: AtomicU32,
block_size: AtomicU32,
kv_separated: AtomicBool,
staleness_threshold: Atomic<f32>,
}
impl FjallConfigInner {
pub const DEFAULT_BLOCK_CACHE_MB: u64 = 64;
pub const DEFAULT_BLOB_CACHE_MB: u64 = 32;
pub const DEFAULT_WRITE_BUFFER_MB: u64 = 128;
pub const DEFAULT_MAX_JOURNALING_MB: u64 = 512;
pub const DEFAULT_MEMTABLE_MB: u32 = 32;
fn new() -> FjallConfigInner {
let queried_cores = std::thread::available_parallelism().map(usize::from);
let cpus = queried_cores.unwrap_or(4);
let flush_workers = cpus.max(1);
let compaction_workers = (cpus / 2).max(1);
FjallConfigInner {
db_path: OnceLock::new(),
manual_journal_persist: AtomicBool::new(false),
flush_workers_count: AtomicUsize::new(flush_workers),
compaction_workers_count: AtomicUsize::new(compaction_workers),
block_cache_capacity: AtomicU64::new(Self::DEFAULT_BLOCK_CACHE_MB * 1_024 * 1_024),
blob_cache_capacity: AtomicU64::new(Self::DEFAULT_BLOB_CACHE_MB * 1_024 * 1_024),
max_journaling_size: AtomicU64::new(Self::DEFAULT_MAX_JOURNALING_MB * 1_024 * 1_024),
max_write_buffer_size: AtomicU64::new(Self::DEFAULT_WRITE_BUFFER_MB * 1_024 * 1_024),
fsync_frequency: AtomicU16::new(1000),
event_listeners: atomic(Vec::new()),
commit_before_close: AtomicBool::new(true),
durability_on_commit: AtomicBool::new(false),
bloom_filter_bits: AtomicI8::new(10),
compression_type: atomic(CompressionType::Lz4),
compaction_strategy: atomic(Strategy::default()),
max_memtable_size: AtomicU32::new(Self::DEFAULT_MEMTABLE_MB * 1_024 * 1_024),
block_size: AtomicU32::new(4 * 1_024),
kv_separated: AtomicBool::new(false),
staleness_threshold: atomic(0.8),
}
}
#[inline]
pub fn db_path(&self) -> &str {
self.db_path.get_or_init(|| "".to_string()).as_str()
}
#[inline]
pub(crate) fn set_db_path(&self, db_path: &str) {
self.db_path.get_or_init(|| db_path.to_string());
}
#[inline]
pub fn manual_journal_persist(&self) -> bool {
self.manual_journal_persist.load(Ordering::Relaxed)
}
#[inline]
pub(crate) fn set_manual_journal_persist(&self, manual_journal_persist: bool) {
self.manual_journal_persist
.store(manual_journal_persist, Ordering::Relaxed)
}
#[inline]
pub fn flush_workers(&self) -> usize {
self.flush_workers_count.load(Ordering::Relaxed)
}
#[inline]
pub(crate) fn set_flush_workers(&self, flush_workers_count: usize) {
self.flush_workers_count
.store(flush_workers_count, Ordering::Relaxed)
}
#[inline]
pub fn compaction_workers(&self) -> usize {
self.compaction_workers_count.load(Ordering::Relaxed)
}
#[inline]
pub(crate) fn set_compaction_workers(&self, compaction_workers_count: usize) {
self.compaction_workers_count
.store(compaction_workers_count, Ordering::Relaxed)
}
#[inline]
pub fn block_cache_capacity(&self) -> u64 {
self.block_cache_capacity.load(Ordering::Relaxed)
}
#[inline]
pub(crate) fn set_block_cache_capacity(&self, block_cache_capacity: u64) {
self.block_cache_capacity
.store(block_cache_capacity, Ordering::Relaxed)
}
#[inline]
pub fn blob_cache_capacity(&self) -> u64 {
self.blob_cache_capacity.load(Ordering::Relaxed)
}
#[inline]
pub(crate) fn set_blob_cache_capacity(&self, blob_cache_capacity: u64) {
self.blob_cache_capacity
.store(blob_cache_capacity, Ordering::Relaxed)
}
#[inline]
pub fn max_journaling_size(&self) -> u64 {
self.max_journaling_size.load(Ordering::Relaxed)
}
#[inline]
pub(crate) fn set_max_journaling_size(&self, max_journaling_size: u64) {
self.max_journaling_size
.store(max_journaling_size, Ordering::Relaxed)
}
#[inline]
pub fn max_write_buffer_size(&self) -> u64 {
self.max_write_buffer_size.load(Ordering::Relaxed)
}
#[inline]
pub(crate) fn set_max_write_buffer_size(&self, max_write_buffer_size: u64) {
self.max_write_buffer_size
.store(max_write_buffer_size, Ordering::Relaxed)
}
#[inline]
pub fn fsync_frequency(&self) -> u16 {
self.fsync_frequency.load(Ordering::Relaxed)
}
#[inline]
pub(crate) fn set_fsync_frequency(&self, fsync_frequency: u16) {
self.fsync_frequency
.store(fsync_frequency, Ordering::Relaxed)
}
#[inline]
pub fn event_listeners(&self) -> Vec<StoreEventListener> {
self.event_listeners.read_with(|it| it.clone())
}
#[inline]
pub(crate) fn add_event_listener(&self, listener: StoreEventListener) {
self.event_listeners.write_with(|it| it.push(listener))
}
#[inline]
pub fn commit_before_close(&self) -> bool {
self.commit_before_close.load(Ordering::Relaxed)
}
#[inline]
#[allow(dead_code)]
pub(crate) fn set_commit_before_close(&self, commit_before_close: bool) {
self.commit_before_close
.store(commit_before_close, Ordering::Relaxed)
}
#[inline]
pub fn durability_on_commit(&self) -> bool {
self.durability_on_commit.load(Ordering::Relaxed)
}
#[inline]
#[allow(dead_code)]
pub(crate) fn set_durability_on_commit(&self, durability_on_commit: bool) {
self.durability_on_commit
.store(durability_on_commit, Ordering::Relaxed)
}
#[inline]
pub fn bloom_filter_bits(&self) -> i8 {
self.bloom_filter_bits.load(Ordering::Relaxed)
}
#[inline]
pub(crate) fn set_bloom_filter_bits(&self, bloom_filter_bits: i8) {
self.bloom_filter_bits
.store(bloom_filter_bits, Ordering::Relaxed)
}
#[inline]
pub fn compression_type(&self) -> CompressionType {
self.compression_type.read_with(|it| *it)
}
#[inline]
pub(crate) fn set_compression_type(&self, compression_type: CompressionType) {
self.compression_type
.write_with(|it| *it = compression_type)
}
#[inline]
pub fn compaction_strategy(&self) -> Strategy {
self.compaction_strategy.read_with(|it| *it)
}
#[inline]
pub(crate) fn set_compaction_strategy(&self, compaction_strategy: Strategy) {
self.compaction_strategy
.write_with(|it| *it = compaction_strategy)
}
#[inline]
pub fn max_memtable_size(&self) -> u32 {
self.max_memtable_size.load(Ordering::Relaxed)
}
#[inline]
pub(crate) fn set_max_memtable_size(&self, max_memtable_size: u32) {
self.max_memtable_size
.store(max_memtable_size, Ordering::Relaxed)
}
#[inline]
pub fn block_size(&self) -> u32 {
self.block_size.load(Ordering::Relaxed)
}
#[inline]
pub(crate) fn set_block_size(&self, block_size: u32) {
self.block_size.store(block_size, Ordering::Relaxed)
}
#[inline]
pub fn kv_separated(&self) -> bool {
self.kv_separated.load(Ordering::Relaxed)
}
#[inline]
pub(crate) fn set_kv_separated(&self, kv_separated: bool) {
self.kv_separated.store(kv_separated, Ordering::Relaxed)
}
#[inline]
pub fn staleness_threshold(&self) -> f32 {
self.staleness_threshold.read_with(|it| *it)
}
#[inline]
pub(crate) fn set_staleness_threshold(&self, staleness_threshold: f32) {
self.staleness_threshold
.write_with(|it| *it = staleness_threshold)
}
}
#[cfg(test)]
mod tests {
use super::*;
use fjall::CompressionType;
use nitrite::store::StoreEventListener;
#[ctor::ctor(unsafe)]
fn init() {
colog::init();
}
#[inline(never)]
fn black_box<T>(x: T) -> T {
x
}
#[test]
fn test_fjall_config_new() {
let config = FjallConfig::new();
let cpus = std::thread::available_parallelism()
.map(usize::from)
.unwrap_or(4);
assert_eq!(config.db_path(), "");
assert!(!config.manual_journal_persist());
assert_eq!(config.flush_workers(), cpus.max(1));
assert_eq!(config.compaction_workers(), (cpus / 2).max(1));
assert_eq!(config.block_cache_capacity(), 64 * 1_024 * 1_024);
assert_eq!(config.blob_cache_capacity(), 32 * 1_024 * 1_024);
assert_eq!(config.max_journaling_size(), 512 * 1_024 * 1_024);
assert_eq!(config.max_write_buffer_size(), 128 * 1_024 * 1_024);
assert_eq!(config.fsync_frequency(), 1000);
assert!(!config.durability_on_commit());
assert_eq!(config.durability(), Durability::Periodic);
assert!(config.commit_before_close());
assert_eq!(config.bloom_filter_bits(), 10);
assert_eq!(config.compression_type(), CompressionType::Lz4);
assert_eq!(config.max_memtable_size(), 32 * 1_024 * 1_024);
assert_eq!(config.block_size(), 4 * 1_024);
assert!(!config.kv_separated());
assert_eq!(config.staleness_threshold(), 0.8);
}
#[test]
fn test_setters_and_getters() {
let config = FjallConfig::new();
config.set_db_path("test_path");
assert_eq!(config.db_path(), "test_path");
config.set_manual_journal_persist(true);
assert!(config.manual_journal_persist());
config.set_flush_workers(8);
assert_eq!(config.flush_workers(), 8);
config.set_compaction_workers(8);
assert_eq!(config.compaction_workers(), 8);
config.set_block_cache_capacity(32 * 1_024 * 1_024);
assert_eq!(config.block_cache_capacity(), 32 * 1_024 * 1_024);
config.set_blob_cache_capacity(32 * 1_024 * 1_024);
assert_eq!(config.blob_cache_capacity(), 32 * 1_024 * 1_024);
config.set_max_journaling_size(1_024 * 1_024);
assert_eq!(config.max_journaling_size(), 1_024 * 1_024);
config.set_max_write_buffer_size(128 * 1_024 * 1_024);
assert_eq!(config.max_write_buffer_size(), 128 * 1_024 * 1_024);
config.set_fsync_frequency(100);
assert_eq!(config.fsync_frequency(), 100);
let listener = StoreEventListener::new(|_| Ok(()));
config.add_event_listener(listener.clone());
assert_eq!(config.event_listeners().len(), 1);
config.set_commit_before_close(false);
assert!(!config.commit_before_close());
config.set_bloom_filter_bits(10);
assert_eq!(config.bloom_filter_bits(), 10);
config.set_compression_type(CompressionType::Lz4);
assert_eq!(config.compression_type(), CompressionType::Lz4);
config.set_max_memtable_size(32 * 1_024 * 1_024);
assert_eq!(config.max_memtable_size(), 32 * 1_024 * 1_024);
config.set_block_size(8 * 1_024);
assert_eq!(config.block_size(), 8 * 1_024);
config.set_kv_separated(false);
assert!(!config.kv_separated());
config.set_staleness_threshold(0.9);
assert_eq!(config.staleness_threshold(), 0.9);
}
#[test]
fn test_config_creation_perf() {
for _ in 0..1000 {
let config = black_box(FjallConfig::new());
black_box(config);
}
}
#[test]
fn test_config_atomic_getters_perf() {
let config = FjallConfig::new();
for _ in 0..10000 {
black_box(config.flush_workers());
black_box(config.compaction_workers());
black_box(config.block_cache_capacity());
black_box(config.blob_cache_capacity());
}
}
#[test]
fn test_config_building_perf() {
for _ in 0..1000 {
let cfg = FjallConfig::new();
cfg.set_flush_workers(4);
cfg.set_compaction_workers(2);
cfg.set_block_cache_capacity(256 * 1_024 * 1_024);
cfg.set_blob_cache_capacity(64 * 1_024 * 1_024);
black_box(cfg);
}
}
}