pub struct Config {Show 26 fields
pub level_count: u8,
pub data_block_restart_interval_policy: RestartIntervalPolicy,
pub index_block_restart_interval_policy: RestartIntervalPolicy,
pub data_block_size_policy: BlockSizePolicy,
pub columnar_row_group_size_policy: BlockSizePolicy,
pub columnar_page_size_policy: BlockSizePolicy,
pub column_encoding_policy: ColumnEncodingPolicy,
pub columnar_read_budget: ReadBudget,
pub columnar_scan_budget: u64,
pub multi_get_metadata_budget: u64,
pub index_block_pinning_policy: PinningPolicy,
pub filter_block_pinning_policy: PinningPolicy,
pub top_level_index_block_pinning_policy: PinningPolicy,
pub top_level_filter_block_pinning_policy: PinningPolicy,
pub data_block_hash_ratio_policy: HashRatioPolicy,
pub index_block_partitioning_policy: PartitioningPolicy,
pub filter_block_partitioning_policy: PartitioningPolicy,
pub index_block_partition_size_policy: BlockSizePolicy,
pub filter_block_partition_size_policy: BlockSizePolicy,
pub filter_policy: FilterPolicy,
pub filter_advisor: Option<FilterAdvisor>,
pub locator_policy: LocatorPolicy,
pub compaction_filter_factory: Option<Arc<dyn Factory>>,
pub prefix_extractor: Option<Arc<dyn PrefixExtractor>>,
pub key_domains: Option<Arc<dyn KeyDomains>>,
pub merge_operator: Option<Arc<dyn MergeOperator>>,
/* private fields */
}Expand description
Tree configuration builder
Fields§
§level_count: u8Number of levels of the LSM tree (depth of tree)
Once set, the level count is fixed (in the manifest snapshot v<id>)
data_block_restart_interval_policy: RestartIntervalPolicyRestart interval inside data blocks
index_block_restart_interval_policy: RestartIntervalPolicyRestart interval inside index blocks
data_block_size_policy: BlockSizePolicyBlock size of data blocks
columnar_row_group_size_policy: BlockSizePolicyUncompressed size a columnar table’s row groups are cut at, per level:
the columnar counterpart of Self::data_block_size_policy, which a
columnar table does not use for its data. A row group is split into
independently readable column pages, so it is sized for what a read of
one page is worth rather than for what a read of the whole group costs.
columnar_page_size_policy: BlockSizePolicyUncompressed bytes of row data a columnar row page holds, per level. A row group’s rows are cut into row pages shared by every column, so a read of a few rows decodes only the pages that hold them.
column_encoding_policy: ColumnEncodingPolicyHow a columnar table’s column pages store their values, per level;
see ColumnEncoding.
columnar_read_budget: ReadBudgetHow columnar reads of this tree fetch their pages; see ReadBudget.
columnar_scan_budget: u64The page bytes one projected columnar scan holds at once, shared by
the segments it merges; see Self::columnar_scan_budget.
multi_get_metadata_budget: u64The filter and index bytes a multi-get holds at once across the tables
of a level it reads stage by stage; see
Self::multi_get_metadata_budget.
index_block_pinning_policy: PinningPolicyWhether to pin index blocks
filter_block_pinning_policy: PinningPolicyWhether to pin filter blocks
top_level_index_block_pinning_policy: PinningPolicyWhether to pin top level index of partitioned index
top_level_filter_block_pinning_policy: PinningPolicyWhether to pin top level index of partitioned filter
data_block_hash_ratio_policy: HashRatioPolicyData block hash ratio
index_block_partitioning_policy: PartitioningPolicyWhether to partition index blocks
filter_block_partitioning_policy: PartitioningPolicyWhether to partition filter blocks
index_block_partition_size_policy: BlockSizePolicyPartition size when using partitioned indexes, per level; see
Self::index_block_partition_size_policy.
filter_block_partition_size_policy: BlockSizePolicyPartition size when using partitioned filters, per level; see
Self::filter_block_partition_size_policy.
filter_policy: FilterPolicyFilter construction policy
filter_advisor: Option<FilterAdvisor>Allocates filter memory by measured negative-probe load; None (the
default) leaves every filter to Self::filter_policy.
locator_policy: LocatorPolicyRetrieval-ribbon locator policy (per level). Defaults to
LocatorPolicy::block_level: written SSTs carry an optional locator
section mapping each key to its data block for O(1) point reads (skipping
the index-block binary search). Set LocatorPolicy::disabled to opt
out — disabled levels produce byte-identical SSTs (no section).
compaction_filter_factory: Option<Arc<dyn Factory>>Compaction filter factory
prefix_extractor: Option<Arc<dyn PrefixExtractor>>Prefix extractor for prefix bloom filters.
When set, the bloom filter indexes extracted prefixes in addition to full keys, allowing prefix scans to skip segments that contain no matching prefixes.
key_domains: Option<Arc<dyn KeyDomains>>Key domains no table spans (see Config::key_domains).
merge_operator: Option<Arc<dyn MergeOperator>>Merge operator for commutative operations
When set, enables merge() operations that store partial updates
which are lazily combined during reads and compaction.
Implementations§
Source§impl Config
impl Config
Sourcepub fn new<P: AsRef<Path>>(
path: P,
seqno: SequenceNumberCounter,
visible_seqno: SequenceNumberCounter,
) -> Self
pub fn new<P: AsRef<Path>>( path: P, seqno: SequenceNumberCounter, visible_seqno: SequenceNumberCounter, ) -> Self
Initializes a new config
Sourcepub fn with_fs<F: Fs>(self, fs: F) -> Self
pub fn with_fs<F: Fs>(self, fs: F) -> Self
Sets the default filesystem backend used for levels without an explicit route.
Defaults to StdFs. Use MemFs for
in-memory trees (testing, ephemeral indexes).
§Example
use lsm_tree::{Config, SequenceNumberCounter};
use lsm_tree::fs::MemFs;
let tree = Config::new(
"/virtual/tree",
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(MemFs::new())
.open()?;Sets the default filesystem backend from an existing shared handle.
Useful when multiple configs should reuse the same backend
instance, including trait objects and backends that are not Clone.
Sourcepub fn open(self) -> Result<AnyTree>
pub fn open(self) -> Result<AnyTree>
Opens a tree using the config.
§Errors
Will return Err if an IO error occurs.
Returns Error::ZstdDictMismatch if
the compression policy references a dict_id that doesn’t match the
configured dictionary.
Sourcepub fn open_many(
configs: impl IntoIterator<Item = Self>,
) -> Result<Vec<AnyTree>>
pub fn open_many( configs: impl IntoIterator<Item = Self>, ) -> Result<Vec<AnyTree>>
Opens a tree for each config, making each one that does not exist yet, and makes the new trees’ folder names durable with one sync of each folder that holds them, after all of them are made, rather than one sync of it per tree. A consumer that makes many trees in one data directory at once pays one device flush for that directory.
The trees come back in the order of configs. They are durable once
this returns: before that a power loss may take back any new tree of
the batch, with everything in it. A batch that fails takes back every
tree it made and closes the ones it opened, so a retry finds what was
there before it.
§Errors
As Self::open, for the first config whose tree fails to open, or a
failed sync of a folder holding a new tree’s name.
§Examples
use lsm_tree::fs::MemFs;
use lsm_tree::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let fs = Arc::new(MemFs::new());
let config = |name: &str| {
Config::new(
format!("/data/{name}"),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(fs.clone())
};
let trees = Config::open_many([config("nodes"), config("edges")])?;
trees[0].insert("a", "node", 0);
assert!(trees[1].is_empty(lsm_tree::SeqNo::MAX, None)?);Sourcepub fn restore_pinned(
self,
layout: &[u8],
entries: impl IntoIterator<Item = PinnedEntry>,
range_tombstones: impl IntoIterator<Item = PinnedRangeTombstone>,
) -> Result<AnyTree>
pub fn restore_pinned( self, layout: &[u8], entries: impl IntoIterator<Item = PinnedEntry>, range_tombstones: impl IntoIterator<Item = PinnedRangeTombstone>, ) -> Result<AnyTree>
Builds a tree at Self::path from a PinnedVersion
taken elsewhere, and opens it.
Before the call, the pin’s files lie in the tree’s folders under their
ids: each of tables as tables/{id}
(with its heal_attestation,
when it has one, as tables/{id}.heal-attest), each of
blob_files as blobs/{id}, each of
dictionaries as dicts/{id}.
layout is the pin’s layout, and
entries and range_tombstones are its
memtable_entries and
memtable_range_tombstones.
The directory lock is taken first and held into the open, so no other
opener sees the folder half restored. Until the restored memtable
contents are durable a restoring marker lies in the folder: an
ordinary Self::open refuses it, and calling this again with the
same pin finishes an interrupted restore. The files are synced and the
layout published as the tree’s manifest, then the memtable contents are
written with their sequence numbers and flushed. The returned tree reads
as the source did at the pin, for every key and every snapshot the
source served then. The config’s sequence number counters are raised to
the pin’s seqno (and the seqno counter past every restored version), so
later writes land above the restored data. The config must match the
source’s where it shapes the files: the tree type, comparator,
encryption provider and dictionaries.
§Errors
Error::InvalidHeader for a layout too
short to hold its header; Error::Locked when
another instance holds the folder; an AlreadyExists I/O error, before
anything is written, when the folder already holds a tree (a manifest
snapshot or edit log) that no interrupted restore left; otherwise
the failures of the syncs, the
manifest write, Self::open and the flush.
Sourcepub fn new_with_generators<P: AsRef<Path>>(
path: P,
seqno: SharedSequenceNumberGenerator,
visible_seqno: SharedSequenceNumberGenerator,
) -> Self
pub fn new_with_generators<P: AsRef<Path>>( path: P, seqno: SharedSequenceNumberGenerator, visible_seqno: SharedSequenceNumberGenerator, ) -> Self
Like Config::new, but accepts pre-built shared generators.
This is useful when the caller already has
SharedSequenceNumberGenerator instances (e.g., from a higher-level
database that shares generators across multiple trees).
Source§impl Config
impl Config
Sourcepub fn tables_folder_for_level(&self, level: u8) -> (PathBuf, Arc<dyn Fs>)
pub fn tables_folder_for_level(&self, level: u8) -> (PathBuf, Arc<dyn Fs>)
Returns the tables folder path and Fs backend for the given level.
If level_routes has an entry covering this
level, uses that entry’s path and Fs. Otherwise falls back to the
primary path and fs.
Sourcepub fn all_tables_folders(&self) -> Vec<(PathBuf, Arc<dyn Fs>)>
pub fn all_tables_folders(&self) -> Vec<(PathBuf, Arc<dyn Fs>)>
Returns all unique tables folders that need to be scanned during
recovery: the primary folder plus every LevelRoute folder.
Sourcepub fn level_routes(self, routes: Vec<LevelRoute>) -> Self
pub fn level_routes(self, routes: Vec<LevelRoute>) -> Self
Configures per-level filesystem routing for tiered storage.
Each LevelRoute maps a range of LSM levels to a base directory
and filesystem backend. Levels not covered by any route fall back to
the primary path and fs.
§Reopen contract
The route configuration is not persisted in the manifest.
On reopen, the Config must specify level_routes such that
all_tables_folders includes every
directory and filesystem pair that may contain existing SST files
for this tree.
Changing the mapping from levels to paths is allowed as long as
the previously used folders remain covered. If old folders are
omitted, recovery may fail with
RouteMismatch (when all missing
tables are on uncovered levels) or
Unrecoverable (when some missing
tables are on levels that are still covered).
§Panics
Panics if any route has an empty range or if any two routes have overlapping level ranges.
Sourcepub fn seqno_generator(self, generator: SharedSequenceNumberGenerator) -> Self
pub fn seqno_generator(self, generator: SharedSequenceNumberGenerator) -> Self
Overrides the sequence number generator.
By default, SequenceNumberCounter is used. This allows plugging in
a custom generator (e.g., HLC for distributed databases).
Sourcepub fn visible_seqno_generator(
self,
generator: SharedSequenceNumberGenerator,
) -> Self
pub fn visible_seqno_generator( self, generator: SharedSequenceNumberGenerator, ) -> Self
Overrides the visible sequence number generator.
Sourcepub fn use_cache(self, cache: Arc<Cache>) -> Self
pub fn use_cache(self, cache: Arc<Cache>) -> Self
Sets the global cache.
You can create a global Cache and share it between multiple
trees to cap global cache memory usage.
Defaults to a cache with 16 MiB of capacity per tree.
Sourcepub fn use_descriptor_table(
self,
descriptor_table: Option<Arc<DescriptorTable>>,
) -> Self
pub fn use_descriptor_table( self, descriptor_table: Option<Arc<DescriptorTable>>, ) -> Self
Sets the file descriptor cache.
Can be shared across trees.
Sourcepub fn expect_point_read_hits(self, b: bool) -> Self
pub fn expect_point_read_hits(self, b: bool) -> Self
If true, the last level will not build filters, reducing the filter size of a database
by ~90% typically.
Enable this only if you know that point reads generally are expected to find a key-value pair.
Sourcepub fn compaction_output_alignment(self, enabled: bool) -> Self
pub fn compaction_output_alignment(self, enabled: bool) -> Self
Whether a compaction ends its outputs on the boundaries between the tables of the level they are merged into next.
An output straddling such a boundary drags both tables into that next merge, which reads and rewrites the extra one. Aligned, an output ends at the first boundary past half the target size; each boundary it passes without ending raises that share by five percent, up to ninety; it ends at twice the target whatever the boundaries, and at the target when no boundary is left ahead. A user key’s versions stay in one output above every size bound. Off, outputs end at the target size alone.
Default = true.
§Examples
use lsm_tree::{Config, SequenceNumberCounter};
let config = Config::new(
folder,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.compaction_output_alignment(false);Sourcepub fn page_ecc(self, enabled: bool) -> Self
pub fn page_ecc(self, enabled: bool) -> Self
Enables per-block Page ECC.
When enabled, every block written by this tree carries a parity
trailer; on read, if the block’s XXH3 disagrees with the on-disk
bytes, the reader attempts recovery from the trailer before surfacing
the corruption. The correction scheme defaults to per-word SEC-DED and
is selectable at runtime (update_runtime_config): per-word SEC-DED,
single XOR parity, or Reed-Solomon.
Opening a tree with page_ecc = true on a build that does not
have the page_ecc cargo feature enabled returns
crate::Error::PageEccUnsupported at Tree::open — the
reader has no way to honour the parity trailer without the
codec, so silently downgrading integrity is not an option.
Wired into the on-disk write path via MultiWriter::use_page_ecc
at every Tree::open / Tree::ingestion / compaction-worker
MultiWriter construction site. With this flag set, every
Block::write_into call those writers make upgrades its
BlockTransform to the matching *Ecc variant — emitting the
configured scheme’s parity trailer and setting the ECC_PARITY flag
in each block header (the trailer length is derived from
data_length, not stored).
Sourcepub fn with_directory_lock(self, enabled: bool) -> Self
pub fn with_directory_lock(self, enabled: bool) -> Self
Enables or disables the cross-process directory lock (default: enabled).
When enabled, Config::open and Config::repair acquire an
exclusive advisory lock on a LOCK file in the tree directory, so a
second process opening / repairing the same directory fails fast with
Error::Locked rather than corrupting the shared
manifest. Disable ONLY when exclusive directory ownership is already
guaranteed at a higher layer (e.g. an embedding keyspace / journal
manager that opens each directory at most once per host).
Sourcepub fn with_recovery_progress(self, progress: Arc<RecoveryProgress>) -> Self
pub fn with_recovery_progress(self, progress: Arc<RecoveryProgress>) -> Self
Wires shared live-progress counters into the repair / salvage paths.
A repair over a large store streams every SST and blob file and can run
for a long time; the handle set here is ticked as files are discovered
and blocks / rows are recovered, so another thread can poll
RecoveryProgress::snapshot while
Config::repair (or a salvage it triggers) runs. Without it, repair
publishes no progress (zero overhead).
§Examples
use lsm_tree::{Config, RecoveryProgress, SequenceNumberCounter};
use std::sync::Arc;
let progress = Arc::new(RecoveryProgress::default());
let config = Config::new(
"my-tree",
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_recovery_progress(progress.clone());
// spawn the repair, then poll `progress.snapshot()` elsewhere
let report = config.repair()?;Sourcepub fn ecc_scheme(self, scheme: EccScheme) -> Self
pub fn ecc_scheme(self, scheme: EccScheme) -> Self
Sets the Page ECC scheme used when Self::page_ecc is enabled.
ECC is off until page_ecc(true). When on, this picks the
algorithm:
EccScheme::Secded
(per-word single-bit correct / double-bit detect, the default, supported
at Block granularity),
EccScheme::Xor (RAID-5
single-parity), or
EccScheme::ReedSolomon.
There is no implicit RS(4,2) default.
Sourcepub fn disable_cow_on_sst_files(self, enabled: bool) -> Self
pub fn disable_cow_on_sst_files(self, enabled: bool) -> Self
Sets whether the writer clears per-file copy-on-write on newly created SST / blob files when the backing filesystem is copy-on-write (Btrfs).
Default true: write-once SSTs gain no benefit from CoW but suffer a
fragmentation penalty (~20% write throughput on Btrfs), so clearing it
recovers the ext4-equivalent baseline. A no-op on non-CoW filesystems.
Set false to preserve CoW (e.g. Btrfs subvolume snapshots that depend
on it). See crate::runtime_config::RuntimeConfig::disable_cow_on_sst_files.
Sourcepub fn use_reflink_for_checkpoint(self, enabled: bool) -> Self
pub fn use_reflink_for_checkpoint(self, enabled: bool) -> Self
Sets whether crate::AbstractTree::create_checkpoint clones files via
reflink (FICLONE / clonefile) when the filesystem supports it,
falling back to a hard link otherwise.
Default true: a reflinked checkpoint has an independent inode (no
max-links constraint, modifications never touch the original) at O(1)
cost via copy-on-write block sharing. A no-op (hard-link path) on
filesystems without reflink. See
crate::runtime_config::RuntimeConfig::use_reflink_for_checkpoint.
Sourcepub fn with_runtime_config(self, runtime: RuntimeConfig) -> Self
pub fn with_runtime_config(self, runtime: RuntimeConfig) -> Self
Sets the initial crate::runtime_config::RuntimeConfig
snapshot the tree will start with.
Seeds both the first manifest write and the live
RuntimeConfigHandle exposed via
crate::Tree::runtime_config.
Replaces the snapshot, including what the builders that seed it set
earlier (Self::ecc_scheme and the other RuntimeConfig-backed
builders), EXCEPT the three compression policies: those keep what
Self::data_block_compression_policy,
Self::index_block_compression_policy and Self::blob_compression
set, in whichever order they are called, and the policies in runtime
are ignored. A caller that set its compression through the builders and
its runtime toggles through this does not lose the former to the
latter’s defaults. On a live tree, all three change through
crate::Tree::update_runtime_config.
Manifest-hardening toggles in the supplied snapshot
that are currently wired through the writer
(manifest_footer_mirror, page_ecc as consumed by
manifest_blocks::writer when picking the BlockTransform
variant) take effect from byte zero of the on-disk
manifest rather than waiting for a post-open
crate::Tree::update_runtime_config call. Subsequent
updates still flow through the live handle and apply to
the next manifest write.
manifest_kv_checksums is plumbed in the snapshot but the
writer does NOT yet consult or persist it (per-entry
framing + footer-flag slot land in a follow-up). Setting
it here today has no on-disk effect; it is exposed for
forward-compat with no behaviour break.
Note on data-block ECC: RuntimeConfig::page_ecc
currently affects manifest Blocks only — data-block ECC is
still gated by Config::page_ecc at tree-open time. The
SST writer path consumes the tree-static config, not the
runtime handle. Wiring through SST emission is a follow-up.
Sourcepub fn filter_block_partitioning_policy(self, policy: PinningPolicy) -> Self
pub fn filter_block_partitioning_policy(self, policy: PinningPolicy) -> Self
Sets the partitioning policy for filter blocks.
Sourcepub fn index_block_partitioning_policy(self, policy: PinningPolicy) -> Self
pub fn index_block_partitioning_policy(self, policy: PinningPolicy) -> Self
Sets the partitioning policy for index blocks.
Sourcepub fn filter_block_pinning_policy(self, policy: PinningPolicy) -> Self
pub fn filter_block_pinning_policy(self, policy: PinningPolicy) -> Self
Sets the pinning policy for filter blocks.
Sourcepub fn index_block_pinning_policy(self, policy: PinningPolicy) -> Self
pub fn index_block_pinning_policy(self, policy: PinningPolicy) -> Self
Sets the pinning policy for index blocks.
Sourcepub fn data_block_restart_interval_policy(
self,
policy: RestartIntervalPolicy,
) -> Self
pub fn data_block_restart_interval_policy( self, policy: RestartIntervalPolicy, ) -> Self
Sets the restart interval inside data blocks.
A higher restart interval saves space while increasing lookup times inside data blocks.
Default = 16
§Panics
Panics if any restart interval in policy is zero.
Sourcepub fn index_block_restart_interval_policy(
self,
policy: RestartIntervalPolicy,
) -> Self
pub fn index_block_restart_interval_policy( self, policy: RestartIntervalPolicy, ) -> Self
Sets the restart interval inside index blocks.
A higher restart interval saves space while increasing lookup times inside index blocks.
Default = 1
§Panics
Panics if any restart interval in policy is zero.
Sourcepub fn filter_policy(self, policy: FilterPolicy) -> Self
pub fn filter_policy(self, policy: FilterPolicy) -> Self
Sets the filter construction policy.
Sourcepub fn filter_advisor(self, advisor: Option<FilterAdvisor>) -> Self
pub fn filter_advisor(self, advisor: Option<FilterAdvisor>) -> Self
Allocates filter memory by measured negative-probe load; see
FilterAdvisor. None (the default) leaves every filter to the
FilterPolicy and counts nothing on the read path, so the tables it
writes are the ones a tree without the advisor writes.
With an advisor, each flush and compaction chooses the width of the
filters it writes; AbstractTree::filter_memory
reports the serialised and resident filter bytes against the budget.
§Examples
use lsm_tree::{Config, config::FilterAdvisor};
// 64 MiB of filters, widths from 8 to 16 bits per key.
let advisor = FilterAdvisor::new(64 << 20).with_bits_per_key([8, 10, 12, 14, 16]);
let tree = Config::new(&folder, Default::default(), Default::default())
.filter_advisor(Some(advisor))
.open()?;Sourcepub fn locator_policy(self, policy: LocatorPolicy) -> Self
pub fn locator_policy(self, policy: LocatorPolicy) -> Self
Sets the retrieval-ribbon locator policy.
On by default at LocatorPrecision::Block (see
LocatorPolicy::block_level). When enabled for a level, written SSTs on
that level carry an optional locator section mapping each key to its
data block (and, at finer precisions, its slot), letting point reads skip
the index-block binary search. Set LocatorPolicy::disabled to opt out;
disabled levels emit byte-identical SSTs.
Sourcepub fn data_block_compression_policy(self, policy: CompressionPolicy) -> Self
pub fn data_block_compression_policy(self, policy: CompressionPolicy) -> Self
Sets the compression the tree starts with for data blocks.
The initial value of
RuntimeConfig::data_block_compression_policy,
which a live tree changes with crate::Tree::update_runtime_config.
Sourcepub fn index_block_compression_policy(self, policy: CompressionPolicy) -> Self
pub fn index_block_compression_policy(self, policy: CompressionPolicy) -> Self
Sets the compression the tree starts with for index blocks.
The initial value of
RuntimeConfig::index_block_compression_policy,
which a live tree changes with crate::Tree::update_runtime_config.
Sourcepub fn blob_compression(self, compression: CompressionType) -> Self
pub fn blob_compression(self, compression: CompressionType) -> Self
Sets the compression a KV-separated tree starts with for blob files.
The initial value of
RuntimeConfig::blob_compression,
which a live tree changes with crate::BlobTree::update_runtime_config.
A standard tree ignores it.
Sourcepub fn blob_cache_fill(self, fill: BlobCacheFill) -> Self
pub fn blob_cache_fill(self, fill: BlobCacheFill) -> Self
Sets whether a KV-separated tree starts caching the values it reads from blob files. It does not by default.
The initial value of
RuntimeConfig::blob_cache_fill,
which a live tree changes with crate::BlobTree::update_runtime_config.
A standard tree ignores it.
§Examples
use lsm_tree::{Config, KvSeparationOptions, SequenceNumberCounter, config::BlobCacheFill};
let config = Config::new(&folder, SequenceNumberCounter::default(), SequenceNumberCounter::default())
.with_kv_separation(Some(KvSeparationOptions::default()))
.blob_cache_fill(BlobCacheFill::Always);Sourcepub fn data_block_size_policy(self, policy: BlockSizePolicy) -> Self
pub fn data_block_size_policy(self, policy: BlockSizePolicy) -> Self
Sets the data block size policy.
Sourcepub fn index_block_partition_size_policy(self, policy: BlockSizePolicy) -> Self
pub fn index_block_partition_size_policy(self, policy: BlockSizePolicy) -> Self
Sets the size a partitioned block index cuts its partitions at, per
level. Applies where Self::index_block_partitioning_policy
partitions the index. The default is 4 KiB at every level.
§Examples
use lsm_tree::{Config, SequenceNumberCounter, config::BlockSizePolicy};
let config = Config::new(
folder.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.index_block_partition_size_policy(BlockSizePolicy::all(16 * 1_024));Sourcepub fn filter_block_partition_size_policy(self, policy: BlockSizePolicy) -> Self
pub fn filter_block_partition_size_policy(self, policy: BlockSizePolicy) -> Self
Sets the size a partitioned filter cuts its partitions at, per level,
in the bytes a partition takes on disk. Applies where
Self::filter_block_partitioning_policy partitions the filter. A
point read that misses the block cache loads one partition, so a
smaller partition costs less per read and more top-level index
entries. The default is 4 KiB at every level.
§Examples
use lsm_tree::{Config, SequenceNumberCounter, config::BlockSizePolicy};
let config = Config::new(
folder.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.filter_block_partition_size_policy(BlockSizePolicy::all(16 * 1_024));Sourcepub fn columnar_row_group_size_policy(self, policy: BlockSizePolicy) -> Self
pub fn columnar_row_group_size_policy(self, policy: BlockSizePolicy) -> Self
Sets the size a columnar table’s row groups are cut at, per level.
Applies to the tables written while the runtime columnar switch is
on; a row-major table cuts its blocks at
Self::data_block_size_policy. The default is
DEFAULT_COLUMNAR_ROW_GROUP_SIZE at every level.
§Examples
use lsm_tree::{Config, SequenceNumberCounter, config::BlockSizePolicy};
let config = Config::new(
folder.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.columnar_row_group_size_policy(BlockSizePolicy::all(256 * 1_024));Sourcepub fn columnar_page_size_policy(self, policy: BlockSizePolicy) -> Self
pub fn columnar_page_size_policy(self, policy: BlockSizePolicy) -> Self
Sets the size a columnar row group’s rows are cut into row pages at, per level.
A row page closes once its rows reach this many uncompressed bytes, so
a page holds at least one row and a group at least one page. A size at
or above the row group size writes one row page per group the writer
closes at its size; a group an ingested batch took past it is still cut
into pages of this size. The default is DEFAULT_COLUMNAR_PAGE_SIZE
at every level, four row pages to a group of the default size.
§Examples
use lsm_tree::{Config, SequenceNumberCounter, config::BlockSizePolicy};
let config = Config::new(
folder.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.columnar_row_group_size_policy(BlockSizePolicy::all(256 * 1_024))
.columnar_page_size_policy(BlockSizePolicy::all(8 * 1_024));Sourcepub fn column_encoding_policy(self, policy: ColumnEncodingPolicy) -> Self
pub fn column_encoding_policy(self, policy: ColumnEncodingPolicy) -> Self
Sets how a columnar table’s column pages store their values, per level.
The default is ColumnEncoding::Plain at every level: a read serves
a plain page as a view of itself, while an encoded one is built back
into its layout by each read that wants the column whole. Choose
ColumnEncoding::Auto for the levels whose tables are mostly stored
rather than read, as a stronger compression is chosen for them. A table
reads the same under either; only what its pages hold differs.
§Examples
use lsm_tree::{
Config, SequenceNumberCounter,
config::{ColumnEncoding, ColumnEncodingPolicy},
};
// Plain in the first three levels, encoded below them.
let config = Config::new(
folder.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.column_encoding_policy(ColumnEncodingPolicy::new([
ColumnEncoding::Plain,
ColumnEncoding::Plain,
ColumnEncoding::Plain,
ColumnEncoding::Auto,
]));Sourcepub fn columnar_read_budget(self, budget: ReadBudget) -> Self
pub fn columnar_read_budget(self, budget: ReadBudget) -> Self
Sets how columnar reads of this tree fetch their pages: the bytes one
request may ask for and the requests kept in flight at once. A reader
setting, not a format one: a tree reopened under another budget reads
the same tables and returns the same rows. The default is
ReadBudget::default.
§Examples
use lsm_tree::{Config, SequenceNumberCounter, config::ReadBudget};
let config = Config::new(
folder.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.columnar_read_budget(ReadBudget::new(64 * 1_024, 4));Sourcepub fn columnar_scan_budget(self, bytes: u64) -> Self
pub fn columnar_scan_budget(self, bytes: u64) -> Self
Sets the page bytes a projected columnar scan of this tree holds at
once. A scan over overlapping segments shares it between them, so a
scan that merges more of them reads each in smaller runs of row pages
rather than holding proportionally more. A row page is the least a
read takes, so one larger than a segment’s share is read on its own
and counted, not refused; so is a row group a pushed-down predicate
reads whole to select its row pages. A reader setting: the rows returned are the
same under any budget. The default is DEFAULT_COLUMNAR_SCAN_BUDGET.
§Examples
use lsm_tree::{Config, SequenceNumberCounter};
let config = Config::new(
folder.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.columnar_scan_budget(256 * 1_024);Sourcepub fn multi_get_metadata_budget(self, bytes: u64) -> Self
pub fn multi_get_metadata_budget(self, bytes: u64) -> Self
Sets the filter and index bytes a multi-get, blocking or resumable,
holds at once, in flight or read, across the tables of a level it
reads stage by stage. A block
in flight counts at the size it is read as, a block read at its decoded
size. A table whose blocks would pass it waits until a table before it
is planned and lets its blocks go; one that alone passes it is read
when no other is in flight. A stage is let in at its on-disk size, so
with compressed index blocks the read can go past the budget by what
the stages let in together grow when decoded; no further stage is let
in until it is back under. Independent of the block cache, so a small
cache does not make the tables of a level wait for each other. A reader
setting: the values returned are the same under any budget. The
default is DEFAULT_MULTI_GET_METADATA_BUDGET.
§Examples
use lsm_tree::{Config, SequenceNumberCounter};
let config = Config::new(
folder.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.multi_get_metadata_budget(8 * 1_024 * 1_024);Sourcepub fn data_block_hash_ratio_policy(self, policy: HashRatioPolicy) -> Self
pub fn data_block_hash_ratio_policy(self, policy: HashRatioPolicy) -> Self
Sets the hash ratio policy for data blocks.
If greater than 0.0, a hash index is embedded into data blocks that can speed up reads inside the data block.
Sourcepub fn with_kv_separation(self, opts: Option<KvSeparationOptions>) -> Self
pub fn with_kv_separation(self, opts: Option<KvSeparationOptions>) -> Self
Toggles key-value separation.
Sourcepub fn with_compaction_filter_factory(
self,
factory: Option<Arc<dyn Factory>>,
) -> Self
pub fn with_compaction_filter_factory( self, factory: Option<Arc<dyn Factory>>, ) -> Self
Installs a custom compaction filter.
Sourcepub fn prefix_extractor(self, extractor: Arc<dyn PrefixExtractor>) -> Self
pub fn prefix_extractor(self, extractor: Arc<dyn PrefixExtractor>) -> Self
Sets the prefix extractor for prefix bloom filters.
When configured, bloom filters will index key prefixes returned by the extractor. Prefix scans can then skip segments whose bloom filter reports no match for the scan prefix.
The extractor may be added or replaced on reopen. A table records the
name of the extractor its filter indexes,
and a prefix scan skips by a table’s filter only when that is the
configured extractor: tables written without it, or under another,
are read instead, until compaction rewrites them.
Sourcepub fn key_domains(self, domains: Arc<dyn KeyDomains>) -> Self
pub fn key_domains(self, domains: Arc<dyn KeyDomains>) -> Self
Ends every table that flush, ingest and compaction write before the
first key of the next of domains, even below the target size, so no
new table holds keys of two domains.
The policy may be added or replaced on reopen: tables written before it keep their keys until compaction rewrites them.
Sourcepub fn with_merge_operator(self, op: Option<Arc<dyn MergeOperator>>) -> Self
pub fn with_merge_operator(self, op: Option<Arc<dyn MergeOperator>>) -> Self
Installs a merge operator for commutative operations.
When set, enables crate::AbstractTree::merge which stores partial updates
(operands) that are lazily combined during reads and compaction.
Sourcepub fn comparator(self, comparator: SharedComparator) -> Self
pub fn comparator(self, comparator: SharedComparator) -> Self
Sets a custom user key comparator.
When configured, all key ordering (memtable, block index, merge, range scans) uses this comparator instead of the default lexicographic byte ordering.
§Important
The comparator’s crate::UserComparator::name is persisted when a tree is
first created. On subsequent opens the stored name is compared against
the supplied comparator’s name — a mismatch causes the open to fail
with Error::ComparatorMismatch.
Sourcepub fn with_encryption(
self,
encryption: Option<Arc<dyn EncryptionProvider>>,
) -> Self
pub fn with_encryption( self, encryption: Option<Arc<dyn EncryptionProvider>>, ) -> Self
Sets the block-level encryption provider for encryption at rest.
When set, all blocks written to SST files are encrypted after
compression and before checksumming, using the provided
EncryptionProvider. The compression dictionaries the tree stores
are sealed with it too, since a dictionary keeps literal stretches of
the records it was trained on.
The caller is responsible for key management and rotation.
See crate::Aes256GcmProvider (behind the encryption feature)
for a ready-to-use AES-256-GCM implementation.
Important constraints:
- Encryption state is NOT recorded in SST metadata. Opening an encrypted tree without the correct provider (or vice versa) will cause block validation errors, not silent corruption.
- Blob files (KV-separated large values) are NOT covered by block-level encryption. Large values stored via KV separation remain in plaintext on disk.
Sourcepub fn with_clock(self, clock: Arc<dyn Clock>) -> Self
pub fn with_clock(self, clock: Arc<dyn Clock>) -> Self
Sets the wall clock this tree reads: the creation time stamped on its tables and blob files, those a repair salvages included, and the time FIFO compaction measures TTL against.
Without one the tree reads the process clock: SystemClock
under std, or the clock registered with set_clock under no_std.
A fixed clock makes the files a tree writes reproducible: two trees given the same fixed clock, the same writes in the same order and the same configuration write byte-identical tables and blob files, wherever their folders are. Encryption breaks that, since every sealed block takes a fresh nonce.
§Examples
struct Epoch;
impl Clock for Epoch {
fn unix_time(&self) -> Duration {
Duration::from_secs(1_700_000_000)
}
}
let config = Config::new(
folder.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_clock(Arc::new(Epoch));Sourcepub fn manifest_recovery_mode(self, mode: ManifestRecoveryMode) -> Self
pub fn manifest_recovery_mode(self, mode: ManifestRecoveryMode) -> Self
Sets what Tree::open does with a torn tail of the manifest edit log.
The only difference between the modes is an edit-log record the writer
never finished appending (power loss mid-append): the default
(ManifestRecoveryMode::AbsoluteConsistency) fails the open,
ManifestRecoveryMode::TolerateCorruptedTailRecords drops it and
opens at the last complete edit. That record was never acknowledged, so
dropping it loses no committed state. Damage to anything committed (a
snapshot section, a fully written edit) fails the open in either mode;
Config::repair rebuilds the manifest from the tables on disk.
See the ManifestRecoveryMode doc for per-variant semantics.
Sourcepub fn sync_mode(self, mode: SyncMode) -> Self
pub fn sync_mode(self, mode: SyncMode) -> Self
Sets the durability level for every fsync the tree issues.
Defaults to SyncMode::Normal (plain fsync, matching RocksDB /
SQLite defaults). Pass SyncMode::Full to force F_FULLFSYNC on
macOS for power-loss durability without an external journal — at a
large per-flush cost. Pass SyncMode::Barrier to keep writes in
order at a barrier’s cost and make them durable with
AbstractTree::sync_devices, once
per batch of flushes. On non-macOS platforms all three are plain
fsync.
Sourcepub fn flush_durability(self, durability: FlushDurability) -> Self
pub fn flush_durability(self, durability: FlushDurability) -> Self
Sets when the tables a flush writes become durable.
Defaults to FlushDurability::Synced: a flush that returns Ok is
on disk. FlushDurability::Deferred leaves a flush unsynced until
AbstractTree::persist_flushes,
for a caller whose own write-ahead log already makes its writes
durable.
§Examples
use lsm_tree::{AbstractTree, Config, SequenceNumberCounter, config::FlushDurability};
let tree = Config::new(
folder.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.flush_durability(FlushDurability::Deferred)
.open()?;
tree.insert("a", "1", 0);
// Written and readable, not yet durable: the caller's log still holds it.
tree.flush_active_memtable(0)?;
// Durable from here: the caller may drop its log up to this point.
tree.persist_flushes()?;Sourcepub fn writeback_bytes(self, bytes: u64) -> Self
pub fn writeback_bytes(self, bytes: u64) -> Self
Sets how many bytes a table or blob file gathers before its writer
starts writing them back, without waiting (default 1 MiB; 0 starts
none).
The final sync of a large file otherwise writes all its dirty pages at
once, and on a device shared with the caller’s journal that burst
delays the journal’s own syncs. Writeback makes nothing durable; the
final sync still does. Only Linux starts one
(sync_file_range(SYNC_FILE_RANGE_WRITE)); elsewhere this is unused.
§Examples
use lsm_tree::{Config, SequenceNumberCounter};
let folder = tempfile::tempdir()?;
let tree = Config::new(
&folder,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.writeback_bytes(4 * 1_024 * 1_024)
.open()?;Sourcepub fn table_target_size(self, bytes: u64) -> Self
pub fn table_target_size(self, bytes: u64) -> Self
Sets the size a flush or an ingestion cuts its output tables at
(default DEFAULT_TABLE_TARGET_SIZE, 64 MiB).
A memtable larger than this flushes into several tables of one run; an ingestion cuts its stream the same way, at whatever level it writes to. Compaction output is sized by its strategy, not by this value.
The leveled L0 trigger counts runs, so the cut does not change how often L0 merges into L1; it bounds how much per-key state a flush’s writer holds. A flush of fewer, larger tables writes less in the merges that follow, so with a memtable much larger than the default a matching target is the cheaper shape, as far as the writer’s memory allows.
§Examples
use lsm_tree::{Config, SequenceNumberCounter};
let folder = tempfile::tempdir()?;
let tree = Config::new(
&folder,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.table_target_size(8 * 1_024 * 1_024)
.open()?;Sourcepub fn manifest_log_rotate_bytes(self, bytes: u64) -> Self
pub fn manifest_log_rotate_bytes(self, bytes: u64) -> Self
Sets the edit-log rotation threshold in bytes (default 1 MiB).
Once the manifest edit log exceeds this size, the next version upgrade
writes a fresh full snapshot and starts an empty log instead of appending
another edit. Lower it to shorten recovery replay and cap log size at the
cost of more frequent full-snapshot writes; 0 rotates on every upgrade.
Sourcepub fn table_file_pool(self, files: usize) -> Self
pub fn table_file_pool(self, files: usize) -> Self
Sets how many empty table files each tables folder creates at a time, ahead of the tables written into them (default 16).
A new table’s name has to be durable before the manifest edit that
names it, which costs a sync of its folder. A pooled file’s name was
made durable with its batch, one folder sync for the whole batch, so a
flush or a compaction into one syncs its tables and the manifest edit
only. The next batch is made off the writing thread once a claim
leaves half a batch or less. An unused pooled file is empty and takes
no space; the next open
uses it again. 0 creates each table file as its table is written and
syncs the folder at every install that names one.
§Examples
use lsm_tree::{Config, SequenceNumberCounter};
let folder = tempfile::tempdir()?;
let tree = Config::new(
&folder,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.table_file_pool(64)
.open()?;Sourcepub fn repair_retention_floor(self, floor: SeqNo) -> Self
pub fn repair_retention_floor(self, floor: SeqNo) -> Self
Sets the retention floor a manifest repair seeds the rebuilt tree with
(default 0): after repair /
open_or_repair every snapshot at or below
floor is refused with
Error::SnapshotBelowRetention,
exactly as the tree refused it before the manifest was lost.
A normal open needs no help: the manifest carries the floor every
retention-advancing install established (a GC compaction, clear, a
table drop, a filtering compaction; see
AbstractTree::retention_floor).
A repair rebuilds the manifest from the tables, which do not record it
(a GC compaction zeroes the seqnos of the rows it settles), so only the
deployment knows it: record
retention_floor() in your own
durable state (it already folds in every operation that raised the
boundary, so no per-operation bookkeeping is needed) and pass the last
recorded value here. Left at 0, a repaired tree serves every
snapshot, which is correct only if history was never collected. Has no
effect on an open that finds a manifest.
Sourcepub fn compaction_rate_limit(self, bytes_per_sec: u64) -> Self
pub fn compaction_rate_limit(self, bytes_per_sec: u64) -> Self
Sets the compaction I/O rate limit in bytes per second.
Caps how fast the compaction worker may issue I/O so background
compaction does not saturate the device and spike user read P99.
0 (the default) disables throttling. Only compaction is limited;
flush and user reads always pass through.
The tree builds its own limiter from this figure at open; change it
later through AbstractTree::compaction_rate_limiter.
Ignored when a shared limiter is supplied through
Self::compaction_rate_limiter.
The figure is the share of the device the deployment gives compaction,
and only the deployment knows it. Below what the write rate times the
strategy’s write amplification needs, compaction falls behind: runs
pile up and every read visits more of them. The
Planned strategy reads the limit and
plans a tree it can keep (see its section on the compaction budget);
Leveled keeps its shape and lags.
Sourcepub fn compaction_rate_limiter(self, limiter: Arc<RateLimiter>) -> Self
pub fn compaction_rate_limiter(self, limiter: Arc<RateLimiter>) -> Self
Supplies a compaction rate limiter shared with other trees, used in
place of the per-tree one built from Self::compaction_rate_limit.
Hand the same Arc to every tree on one device: their compactions are
then bounded by it together, at the rate it holds, whatever each
tree’s configured Self::compaction_rate_limit says. A retune through
RateLimiter::set_rate
on any holder applies to all of them on their next request.
§Examples
use lsm_tree::rate_limiter::RateLimiter;
use lsm_tree::Config;
use std::sync::Arc;
let device = Arc::new(RateLimiter::new(64 * 1_024 * 1_024));
let first = Config::new(a.path(), Default::default(), Default::default())
.compaction_rate_limiter(Arc::clone(&device))
.open()?;
let second = Config::new(b.path(), Default::default(), Default::default())
.compaction_rate_limiter(Arc::clone(&device))
.open()?;
// Both trees now share the device's 64 MiB/s.Sourcepub fn compaction_capacity(self, pool: Arc<CapacityPool>) -> Self
pub fn compaction_capacity(self, pool: Arc<CapacityPool>) -> Self
Supplies the pool this tree’s compactions reserve the room for their outputs in, shared with other trees, in place of a pool of its own.
Hand the same Arc to every tree whose files share volumes: each
compaction then reserves against what the others have still to write,
rather than two of them counting on the same free bytes. See
CapacityPool.
Sourcepub fn write_rate_limit(self, bytes_per_sec: u64) -> Self
pub fn write_rate_limit(self, bytes_per_sec: u64) -> Self
Sets the byte rate writes are paced to while the tree’s backpressure
verdict is in its slowdown tier (default 0: no byte pacing).
With a rate set,
AbstractTree::write_backpressure_for
answers a slowdown with what the write’s bytes owe at this rate, so a
caller honouring it writes no faster than the rate however large or
small its writes are; below the slowdown thresholds nothing is debited
and writes run at full speed. The tree builds its own limiter from
this figure at open; change it later through
AbstractTree::write_rate_limiter.
Ignored when a shared limiter is supplied through
Self::write_rate_limiter.
§Examples
use lsm_tree::{Config, SequenceNumberCounter};
let folder = tempfile::tempdir()?;
let tree = Config::new(
&folder,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.write_rate_limit(16 * 1_024 * 1_024)
.open()?;Sourcepub fn write_rate_limiter(self, limiter: Arc<RateLimiter>) -> Self
pub fn write_rate_limiter(self, limiter: Arc<RateLimiter>) -> Self
Supplies a write rate limiter shared with other trees, used in place of
the per-tree one built from Self::write_rate_limit: writes paced
under slowdown by any of them draw on one budget, at the rate it holds.
Sourcepub fn write_buffer_budget(self, budget: Arc<WriteBufferBudget>) -> Self
pub fn write_buffer_budget(self, budget: Arc<WriteBufferBudget>) -> Self
Joins the tree to a WriteBufferBudget
shared with other trees: the memtable axis of its backpressure verdict
then compares the unflushed memtable bytes of every member together.
Sourcepub fn compaction_threads(self, threads: usize) -> Self
pub fn compaction_threads(self, threads: usize) -> Self
Sets the compaction worker-thread count.
Under std this both sizes the per-tree pool built at open when no
shared pool is supplied (see Self::compaction_pool), which prepares
a table’s data blocks and builds its partitioned filter’s partitions,
and caps how many range-parallel sub-compactions a compaction splits
into.
1 keeps compaction serial. Default is max(1, available_parallelism / 2). Without the parallel feature there is no built-in pool, so the
work runs serially even for a value > 1.
Sourcepub fn parallel_compression_inline_below(self, bytes: Option<u32>) -> Self
pub fn parallel_compression_inline_below(self, bytes: Option<u32>) -> Self
Sets the payload size (bytes) below which a block of a table compressed
in parallel is prepared on the writer thread instead of a worker. A
block that small costs less to compress than to hand to a worker and
back. None (default) derives the threshold from the table’s block
codec: 1 KiB for lz4; 8 KiB with no codec, where only encryption or
page ECC brings a table to the workers and costs less per block than
lz4; and 0 for zstd, whose per-frame setup makes any block worth a
worker. Some(0) sends every block to the workers.
§Examples
use lsm_tree::{Config, SequenceNumberCounter};
let config = Config::new(
folder.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.parallel_compression_inline_below(Some(1_024));Sourcepub fn subcompaction_min_bytes(self, bytes: u64) -> Self
pub fn subcompaction_min_bytes(self, bytes: u64) -> Self
Sets the minimum total input size (bytes) for a compaction to be split
into parallel sub-compactions. Default 8 MiB. 0 splits every eligible
compaction; a large value effectively disables sub-compaction (block
compression still parallelizes via Self::compaction_threads).
Sourcepub fn compaction_pool(self, pool: Option<Arc<dyn CompactionSpawner>>) -> Self
pub fn compaction_pool(self, pool: Option<Arc<dyn CompactionSpawner>>) -> Self
Supplies a shared compaction thread pool, used in place of the per-tree
default. Pass one [crate::table::writer::CompactionSpawner] (e.g. a
RayonSpawner wrapping a shared rayon thread pool) to several trees so
the total worker-thread count stays bounded by the pool size rather than
the number of open trees.
Sourcepub fn zstd_dictionary(self, dictionary: Option<Arc<ZstdDictionary>>) -> Self
pub fn zstd_dictionary(self, dictionary: Option<Arc<ZstdDictionary>>) -> Self
Supplies a pre-trained zstd dictionary for the data block policy to name.
The open registers it with the tree, which stores it, so a later open
needs nothing supplied. Which blocks are written against it is decided
by the policy: a CompressionType::ZstdDict entry names a dictionary
by id, and every block reads back through the id it recorded. The
dictionary should be trained on representative data samples for best
results.
Create a dictionary with ZstdDictionary::new,
then use CompressionType::zstd_dict to create a matching
compression type:
use lsm_tree::{CompressionType, ZstdDictionary};
let dict = ZstdDictionary::new(&training_data);
let compression = CompressionType::zstd_dict(3, dict.id()).unwrap();
config
.zstd_dictionary(Some(Arc::new(dict)))
.data_block_compression_policy(CompressionPolicy::all(compression));Source§impl Config
impl Config
Sourcepub fn repair(&self) -> Result<RepairReport>
pub fn repair(&self) -> Result<RepairReport>
Rebuilds the MANIFEST for the tree at this config’s path from the SST
files present on disk, then returns a RepairReport.
Use this only when a tree fails to open because its manifest is missing or corrupt but the SST files are intact. After a successful repair the tree opens normally; all recovered data is at L0 and a background compaction restructures it into proper levels (expect elevated I/O for a period proportional to the data size).
Version edits the lost manifest carried are gone with it (the report’s standing warning says so): in particular, a committed compaction whose filter removed EVERY record left no output to carry lineage, so its still-lingering inputs are republished as one consistent pre-compaction history — the filter’s removals transiently reappear until the next compaction re-applies the standing policy. No operand is doubled by that window.
§The report
The rebuilt tree keeps the returned report until
AbstractTree::acknowledge_repair_report
takes its id; until then Config::open_or_repair hands it out
again, so a process that ends before the report was acted on loses
nothing.
§Exclusive access
Repair writes and installs a fresh snapshot and removes the stale
edits-* logs in place, so it requires exclusive access to the tree
directory. It acquires the same cross-process directory lock as
Config::open for the duration of the call: if another live instance
holds the directory (open or repairing), this fails fast with
crate::Error::Locked instead of corrupting that instance’s manifest
state. The lock can be disabled via
Config::with_directory_lock for
embedders enforcing exclusivity at a higher layer.
§Errors
Returns crate::Error::FeatureUnsupported for KV-separated (blob)
trees, and propagates any I/O error from scanning the directory or
writing the new manifest. Individual unreadable SSTs do not fail the
repair; they are reported in RepairReport::unreadable_files.
§Examples
use lsm_tree::{AbstractTree, Config, SequenceNumberCounter};
let folder = tempfile::tempdir()?;
let config = Config::new(
&folder,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
);
{
let tree = config.clone().open()?;
tree.insert("a", "1", 0);
tree.flush_active_memtable(0)?;
}
let report = config.repair()?;
println!("recovered {} tables, {} unreadable", report.recovered, report.unreadable);
// `repair` borrows, so the same config opens the rebuilt tree. Once the
// report's replay obligation is met, the tree stops owing it.
let tree = config.open()?;
assert!(tree.get("a", 1)?.is_some());
assert!(tree.acknowledge_repair_report(report.id)?);Sourcepub fn repair_with_salvage(&self, salvage: bool) -> Result<RepairReport>
pub fn repair_with_salvage(&self, salvage: bool) -> Result<RepairReport>
Like repair, but when an SST fails whole-file recovery
(salvage = true) it is block-salvaged instead of being left out: a fresh
SST holding its recoverable blocks is built beside it, referenced by the
rebuilt manifest, and swapped onto its name once that manifest is durable.
A salvaged table may be missing the key ranges of its corrupt blocks
(reported per file via RepairReport::salvaged); use this only as a
last resort when losing the whole SST is worse than losing some keys.
SSTs whose metadata, index, or SFA trailer is itself unreadable still
cannot be salvaged and are reported unreadable.
§Errors
Returns an error if the directory lock cannot be taken or the rebuilt
manifest cannot be persisted; per-file recovery / salvage failures are
reported in the RepairReport, not returned.
§Examples
use lsm_tree::{Config, SequenceNumberCounter};
let config = Config::new(
"/var/lib/mydb",
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
);
let report = config.repair_with_salvage(true)?;
println!(
"recovered {} table(s), {} of them by salvage",
report.recovered, report.salvaged,
);Sourcepub fn repair_with_resurrection(
&self,
salvage: bool,
allow_resurrection: bool,
) -> Result<RepairReport>
pub fn repair_with_resurrection( &self, salvage: bool, allow_resurrection: bool, ) -> Result<RepairReport>
Like repair_with_salvage, but with an
explicit allow_resurrection policy. When false (the default the other
entry points use), recovery drops data whose visibility became ambiguous
after a lost restriction bound or a lost / forged delete mask; when
true, it keeps that data, accepting that superseded or deleted rows may
reappear. Either setting yields a valid, openable tree; the flag is the
ONLY recovery decision an operator makes.
The flag governs AMBIGUOUS VISIBILITY, not LOST BYTES. When a table (or
one of its blocks) cannot be read at all, what it said about its keys is
gone with it, and older versions of those keys survive in other tables:
a value it had overwritten, or a key its tombstone had deleted, becomes
visible again. Neither setting changes that, because nothing left on
disk distinguishes “this key was deleted” from “this key was simply
never rewritten”. Deleting the lost range instead would destroy intact
data — a flushed table’s key range routinely spans most of the keyspace
while its sequence numbers sit above every older level, so one corrupt
block would erase most of the tree, irreversibly, where the stale read
leaves every byte in place. The lost ranges are therefore REPORTED
rather than acted on; see
RepairReport::lost_coverage.
§Errors
Returns an error if the tables directory cannot be scanned or a transient
I/O fault interrupts recovery (retryable), or if the rebuilt manifest
cannot be durably installed. See
repair_with_salvage.
Sourcepub fn open_or_repair(
self,
policy: RepairPolicy,
) -> Result<(AnyTree, Option<RepairReport>)>
pub fn open_or_repair( self, policy: RepairPolicy, ) -> Result<(AnyTree, Option<RepairReport>)>
Opens the tree, and when the open fails STRUCTURALLY, repairs per
policy and opens again — the one-call recovery entry point for a
caller that pairs the engine with an external write-ahead log.
Ok((tree, None)) is a healthy open. Ok((tree, Some(report))) is an
open that succeeded only after a repair; the caller MUST then consult
RepairReport::wal_replay_scope before its WAL replay, since the
repair may have regressed persisted state below the log’s trim
watermark (see docs/external-wal.md § Replay after repair).
Only failures that positively identify repairable on-disk structural
damage engage the repair (see is_repairable_structural). Everything
else is returned as-is: a TRANSIENT I/O failure (a repair would
rebuild the manifest around files a healthy retry could still read), a
held directory lock (Error::Locked — the
repair would contend on the same lock), an UNSUPPORTED format version
(Error::InvalidVersion — the store
needs offline conversion or a matching binary, not a rebuild in the
current format that would reject every table), a ROUTED tree’s
Error::Unrecoverable (route
provenance is not persisted, so a missing routed table is
indistinguishable from a route path change or an unmounted tier — a
rebuild would omit every SST still on the old route), and every
CONFIGURATION mismatch (wrong comparator, level route, dictionary, or
encryption key) — repairing under a wrong configuration would rebuild,
and could drop, perfectly healthy data.
§Errors
Propagates a transient-I/O or lock failure from the open and any
repair failure. A failure of the POST-repair open surfaces as
Error::RepairedButUnopened,
carrying the completed RepairReport: consume it from the error.
The rebuilt snapshot holds its report until the caller acknowledges it
with AbstractTree::acknowledge_repair_report,
and every call before that answers Some(report) again, a healthy
open included. A report is handed out at least once whatever ends the
process between the hand-out and its use; one handed out again carries
the same RepairReport::id, so a caller that records the ids it
applied skips it. Acknowledge only once the report’s replay obligation
is met.
§Examples
use lsm_tree::{Config, RepairPolicy, SequenceNumberCounter};
let config = Config::new(
"/var/lib/mydb",
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
);
let (tree, repaired) = config.open_or_repair(RepairPolicy::default().salvage(true))?;
if let Some(report) = repaired {
println!("repaired; WAL must replay {:?}", report.wal_replay_scope());
}Trait Implementations§
Auto Trait Implementations§
impl !RefUnwindSafe for Config
impl !UnwindSafe for Config
impl Freeze for Config
impl Send for Config
impl Sync for Config
impl Unpin for Config
impl UnsafeUnpin for Config
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
impl<ST, DT> CastableFrom<ST, Initialized, Initialized> for DT
impl<ST, DT> CastableFrom<ST, Uninit, Uninit> for DT
Source§impl<T> CloneToUninit for Twhere
T: Clone,
impl<T> CloneToUninit for Twhere
T: Clone,
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more