dig-blockstore 0.1.0

DIG L2 block persistence — RocksDB-backed block store with canonical chain indexing
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163

//! [`BlockStoreConfig`] — paths, cache sizes, RocksDB tuning, pipeline, and pruning hooks.
//!
//! **Requirements trace**
//! - [`TYP-008`](../../docs/requirements/domains/storage_types/specs/TYP-008.md) — field set, production defaults, manual [`Default`] (non-empty [`PathBuf`](std::path::PathBuf))
//! - [`NORMATIVE` TYP-008](../../docs/requirements/domains/storage_types/NORMATIVE.md#typ-008-blockstoreconfig-struct)
//! - [`STR-002`](../../docs/requirements/domains/crate_structure/specs/STR-002.md), [`STR-004`](../../docs/requirements/domains/crate_structure/specs/STR-004.md), [`STR-005`](../../docs/requirements/domains/crate_structure/specs/STR-005.md) (`test_config` overrides)
//! - Shared numeric tunables: [`TYP-002`](../../docs/requirements/domains/storage_types/specs/TYP-002.md) via [`crate::constants`] (`DEFAULT_*`, [`ZSTD_COMPRESSION_LEVEL`](crate::constants::ZSTD_COMPRESSION_LEVEL))
//!
//! ## Defaults and `Default` impl
//!
//! [`Default::default`] follows TYP-008 for the **core** knobs (path, caches, RocksDB, zstd, pipeline flags,
//! pruning flags). Numeric write-buffer / block-cache / cache capacities reuse [`crate::constants`] so
//! TYP-002 and TYP-008 stay numerically identical ([`tests/typ_002_tests.rs`](../../tests/typ_002_tests.rs)).
//!
//! **Why manual `Default`:** `#[derive(Default)]` would set `path` to an empty [`PathBuf`]; the spec requires a
//! conventional relative layout (`data/blockstore`) suitable for local dev and examples.
//!
//! ## Extension fields (beyond TYP-008’s core table)
//!
//! The crate also carries forward-looking fields wired by [`crate::store::BlockStore::open`] and future BLK/CAC work:
//!
//! - **`warm_cache_depth`** — how many trailing canonical heights to touch when [`warm_cache_on_open`] is true ([`CAC-006`](../../docs/requirements/domains/caching/specs/CAC-006_cache_warming_on_startup.md)).
//! - **`write_pipeline_channel_capacity`** — bounded queue depth for [`BLK-008`](../../docs/requirements/domains/block_storage/specs/BLK-008.md).
//! - **`readahead_size`** — sequential read hint ([`BLK-006`](../../docs/requirements/domains/block_storage/specs/BLK-006.md)).
//!
//! These are **not** duplicated in the short TYP-008 markdown table but are part of the public Rust API and are
//! covered by [`tests/typ_008_tests.rs`](../../tests/typ_008_tests.rs).

use std::path::PathBuf;

use crate::constants::{
    DEFAULT_BLOCK_CACHE_CAPACITY, DEFAULT_BLOCK_CACHE_SIZE, DEFAULT_HEADER_CACHE_CAPACITY,
    DEFAULT_MAX_DECOMPRESSED_BLOCK_BYTES, DEFAULT_MAX_OPEN_FILES, DEFAULT_WRITE_BUFFER_SIZE,
    ZSTD_COMPRESSION_LEVEL,
};

/// Configuration for opening or creating a [`crate::store::BlockStore`].
///
/// **Construction:** Use [`BlockStoreConfig::default`] and override fields, [`std::default::Default::default`]
/// with struct update syntax (`BlockStoreConfig { path: my_dir, ..Default::default() }`), or [`STR-005`](../../docs/requirements/domains/crate_structure/specs/STR-005.md) `test_config` for tiny test tunables.
///
/// **Validation:** [`crate::store::BlockStore::open`] should eventually enforce `cache_shards` is a power of two ([`TYP-008`](../../docs/requirements/domains/storage_types/specs/TYP-008.md) implementation notes); today callers should follow that invariant.
#[derive(Debug, Clone)]
pub struct BlockStoreConfig {
    // --- Storage path (TYP-008) ---
    /// Root directory for the RocksDB database files (TYP-008 `path`).
    pub path: PathBuf,

    // --- In-memory blockstore caches (CAC-001 / CAC-002 precursors) ---
    /// Max blocks retained in the sharded block cache.
    pub block_cache_capacity: usize,

    /// Max headers retained in the sharded header cache.
    pub header_cache_capacity: usize,

    /// Shard count for block/header caches; must be a power of two when enforced ([`TYP-008`](../../docs/requirements/domains/storage_types/specs/TYP-008.md)).
    pub cache_shards: usize,

    /// When true, [`crate::store::BlockStore::open`] preloads recent canonical blocks ([`STR-004`](../../docs/requirements/domains/crate_structure/specs/STR-004.md), [`CAC-006`](../../docs/requirements/domains/caching/specs/CAC-006_cache_warming_on_startup.md)).
    pub warm_cache_on_open: bool,

    /// Trailing heights (inclusive) to touch when warming, starting from tip ([`CAC-006`](../../docs/requirements/domains/caching/specs/CAC-006_cache_warming_on_startup.md)).
    pub warm_cache_depth: u64,

    // --- RocksDB tuning (TYP-002 / TYP-003 precursors) ---
    /// RocksDB memtable / write buffer budget per column family (bytes).
    pub write_buffer_size: usize,

    /// Shared block cache for RocksDB (bytes).
    pub block_cache_size: usize,

    /// `max_open_files` passed to RocksDB options.
    pub max_open_files: i32,

    /// When true, enable BlobDB-style large-value handling for block bodies ([`TYP-003`](../../docs/requirements/domains/storage_types/specs/TYP-003.md)).
    pub enable_blob_db: bool,

    /// When true, store compressed block payloads ([`SER-001`](../../docs/requirements/domains/serialization/specs/SER-001.md)); tests often disable for simplicity ([`STR-005`](../../docs/requirements/domains/crate_structure/specs/STR-005.md)).
    pub compress_blocks: bool,

    /// Zstd level for block compression when `compress_blocks` is true.
    pub compression_level: i32,

    /// Whether to use a trained zstd dictionary ([`SER-005`](../../docs/requirements/domains/serialization/specs/SER-005.md)).
    pub use_compression_dict: bool,

    /// Max accepted **decompressed** byte length for a stored block body ([`SER-001`](../../docs/requirements/domains/serialization/specs/SER-001.md)).
    pub max_decompressed_block_bytes: usize,

    /// Optional zstd dictionary bytes wired at open time ([`SER-001`](../../docs/requirements/domains/serialization/specs/SER-001.md)).
    ///
    /// **Production:** leave `None` — [`crate::store::BlockStore::open`] loads [`crate::constants::META_ZSTD_DICT`]
    /// from `CF_METADATA` when [`Self::use_compression_dict`] is true ([`SER-005`](../../docs/requirements/domains/serialization/specs/SER-005.md)).
    /// **Tests:** set to a trained dictionary to exercise dictionary compress/decompress without persisting metadata first.
    pub zstd_dictionary_override: Option<Vec<u8>>,

    // --- Write pipeline (BLK-008 precursor) ---
    /// Max blocks batched before a pipeline flush.
    pub write_pipeline_batch_size: usize,

    /// Max wait before flushing a partial pipeline batch (milliseconds).
    pub write_pipeline_flush_ms: u64,

    /// Bounded async channel capacity feeding the write pipeline ([`BLK-008`](../../docs/requirements/domains/block_storage/specs/BLK-008.md)).
    pub write_pipeline_channel_capacity: usize,

    /// When true, sync the WAL after each write (durability vs throughput).
    pub sync_writes: bool,

    /// Hint for sequential readahead ([`BLK-006`](../../docs/requirements/domains/block_storage/specs/BLK-006.md)).
    pub readahead_size: usize,

    // --- Caching (CAC-004 / CAC-005) ---
    /// Max entries in the canonical height→hash `BTreeMap` cache ([`CAC-004`](../../docs/requirements/domains/caching/specs/CAC-004_canonical_height_index_cache.md)).
    ///
    /// When the cache exceeds this size, the **lowest** height entry is evicted.
    /// Set to `0` to disable the height cache entirely.
    pub canonical_height_cache_capacity: usize,

    /// Max entries in the hash→height reverse lookup cache ([`CAC-005`](../../docs/requirements/domains/caching/specs/CAC-005_hash_to_height_reverse_cache.md)).
    ///
    /// Uses the same sharded LRU infrastructure as block/header caches.
    /// Higher capacity is practical because each entry is only 40 bytes (32 hash + 8 height).
    pub hash_to_height_cache_capacity: usize,

    // --- Pruning (PRN-003 / PRN-004 precursors) ---
    /// Register compaction-time pruning when true ([`PRN-003`](../../docs/requirements/domains/pruning/specs/PRN-003_compaction_filter.md)).
    pub enable_compaction_pruning: bool,

    /// Optional floor height below which compaction may drop data; `None` disables.
    pub min_retained_height: Option<u64>,
}

impl Default for BlockStoreConfig {
    fn default() -> Self {
        Self {
            path: PathBuf::from("data/blockstore"),
            block_cache_capacity: DEFAULT_BLOCK_CACHE_CAPACITY,
            header_cache_capacity: DEFAULT_HEADER_CACHE_CAPACITY,
            cache_shards: 16,
            warm_cache_on_open: true,
            warm_cache_depth: 64,
            write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
            block_cache_size: DEFAULT_BLOCK_CACHE_SIZE,
            max_open_files: DEFAULT_MAX_OPEN_FILES,
            enable_blob_db: true,
            compress_blocks: true,
            compression_level: ZSTD_COMPRESSION_LEVEL,
            use_compression_dict: true,
            max_decompressed_block_bytes: DEFAULT_MAX_DECOMPRESSED_BLOCK_BYTES,
            zstd_dictionary_override: None,
            write_pipeline_batch_size: 64,
            write_pipeline_flush_ms: 100,
            write_pipeline_channel_capacity: 256,
            sync_writes: false,
            readahead_size: 2_097_152,
            canonical_height_cache_capacity: 10_000,
            hash_to_height_cache_capacity: 10_000,
            enable_compaction_pruning: false,
            min_retained_height: None,
        }
    }
}