kimberlite_storage/
compaction.rs

1//! Log compaction for merging and cleaning old segments.
2//!
3//! Compaction merges consecutive completed segments, removes tombstoned records
4//! whose data records exist in later segments, and optionally compresses the
5//! output. The hash chain is rebuilt for the compacted segment.
6//!
7//! # Algorithm
8//!
9//! 1. Select consecutive completed segments below `merge_threshold_bytes`
10//! 2. Read all records, filter tombstones whose data exists in later segments
11//! 3. Rewrite surviving records to a new segment, rebuilding the hash chain
12//! 4. Rebuild offset index for the compacted segment
13//! 5. Atomically update manifest (remove old entries, insert compacted)
14//! 6. Delete old segment files and indexes
15
16use kimberlite_types::CompressionKind;
17
18/// Configuration for log compaction.
19#[derive(Debug, Clone)]
20pub struct CompactionConfig {
21    /// Minimum number of completed segments before compaction triggers.
22    pub min_segments: usize,
23    /// Maximum total size of segments to merge in one pass (bytes).
24    pub merge_threshold_bytes: u64,
25    /// Whether to compress records during compaction.
26    pub compress_on_compact: bool,
27    /// Compression algorithm to use when `compress_on_compact` is true.
28    pub compression: CompressionKind,
29}
30
31impl Default for CompactionConfig {
32    fn default() -> Self {
33        Self {
34            min_segments: 4,
35            merge_threshold_bytes: 128 * 1024 * 1024, // 128 MB
36            compress_on_compact: false,
37            compression: CompressionKind::None,
38        }
39    }
40}
41
42/// Result of a compaction operation.
43#[derive(Debug, Clone)]
44pub struct CompactionResult {
45    /// Number of segments before compaction.
46    pub segments_before: usize,
47    /// Number of segments after compaction.
48    pub segments_after: usize,
49    /// Total bytes reclaimed by compaction.
50    pub bytes_reclaimed: u64,
51    /// Number of tombstone records removed.
52    pub tombstones_removed: u64,
53}
54
55#[cfg(test)]
56mod tests {
57    use super::*;
58
59    #[test]
60    fn default_config() {
61        let config = CompactionConfig::default();
62        assert_eq!(config.min_segments, 4);
63        assert_eq!(config.merge_threshold_bytes, 128 * 1024 * 1024);
64        assert!(!config.compress_on_compact);
65        assert_eq!(config.compression, CompressionKind::None);
66    }
67
68    #[test]
69    fn compaction_result_fields() {
70        let result = CompactionResult {
71            segments_before: 8,
72            segments_after: 2,
73            bytes_reclaimed: 512_000,
74            tombstones_removed: 42,
75        };
76        assert_eq!(result.segments_before, 8);
77        assert_eq!(result.segments_after, 2);
78        assert_eq!(result.bytes_reclaimed, 512_000);
79        assert_eq!(result.tombstones_removed, 42);
80    }
81}
kimberlite_storage/compaction.rs

kimberlite_storage/
compaction.rs
