kimberlite_storage/lib.rs
1//! kmb-storage: Append-only segment storage for `Kimberlite`
2//!
3//! This crate implements the durable event log storage layer. Events are
4//! stored in segment files with a simple binary format that includes
5//! cryptographic hash chains for tamper detection and CRC32 checksums
6//! for corruption detection.
7//!
8//! # Record Format
9//!
10//! Each record is stored as:
11//! ```text
12//! [offset:i64][prev_hash:32B][length:u32][payload:bytes][crc32:u32]
13//! 8B 32B 4B variable 4B
14//! ```
15//!
16//! - **offset**: The logical position of this event in the stream
17//! - **`prev_hash`**: SHA-256 hash of the previous record (all zeros for genesis)
18//! - **length**: Size of the payload in bytes
19//! - **payload**: The event data
20//! - **crc32**: Checksum of all preceding fields for corruption detection
21//!
22//! # Hash Chain
23//!
24//! Records form a tamper-evident chain where each record includes the hash
25//! of the previous record. This allows verification that the log has not
26//! been modified:
27//!
28//! ```text
29//! Record 0: prev_hash = [0; 32] → hash_0 = SHA-256(payload_0)
30//! Record 1: prev_hash = hash_0 → hash_1 = SHA-256(hash_0 || payload_1)
31//! Record 2: prev_hash = hash_1 → hash_2 = SHA-256(hash_1 || payload_2)
32//! ```
33//!
34//! # File Layout
35//!
36//! ```text
37//! data_dir/
38//! {stream_id}/
39//! segment_000000.log # First segment (future: rotation)
40//! segment_000001.log # Second segment, etc.
41//! ```
42//!
43//! # Example
44//!
45//! ```ignore
46//! use kimberlite_storage::Storage;
47//! use kimberlite_types::{Offset, StreamId};
48//! use bytes::Bytes;
49//!
50//! let storage = Storage::new("/data/kimberlite");
51//!
52//! // Append events
53//! let events = vec![Bytes::from("event1"), Bytes::from("event2")];
54//! let new_offset = storage.append_batch(
55//! StreamId::new(1),
56//! events,
57//! Offset::new(0),
58//! true, // fsync for durability
59//! )?;
60//!
61//! // Read events back
62//! let events = storage.read_from(StreamId::new(1), Offset::new(0), 1024)?;
63//! ```
64
65// Modules
66mod backend;
67mod checkpoint;
68pub mod codec;
69mod compaction;
70mod error;
71mod index;
72mod memory;
73mod pipeline;
74mod record;
75mod storage;
76
77// Re-exports
78pub use backend::StorageBackend;
79pub use checkpoint::{
80 CheckpointIndex, create_checkpoint, deserialize_checkpoint_payload,
81 serialize_checkpoint_payload, should_create_checkpoint,
82};
83pub use codec::{Codec, CodecRegistry, Lz4Codec, NoneCodec, ZstdCodec};
84pub use compaction::{CompactionConfig, CompactionResult};
85pub use error::StorageError;
86pub use index::OffsetIndex;
87pub use memory::MemoryStorage;
88pub use pipeline::{AppendPipeline, PreparedBatch};
89pub use record::Record;
90pub use storage::Storage;
91
92#[cfg(test)]
93mod tests;
94
95// Kani verification harnesses for bounded model checking
96#[cfg(kani)]
97mod kani_proofs;