oxirs_tdb/compression/
mod.rs

1//! # Advanced Compression Module for TDB Storage
2//!
3//! Provides advanced compression algorithms including column-store optimizations,
4//! bitmap compression, delta encoding, and adaptive compression selection.
5
6/// Adaptive compression selection based on data characteristics
7pub mod adaptive;
8/// Adaptive compression algorithm selection
9pub mod adaptive_selection;
10/// Compression benchmarking — measures throughput and ratio for all algorithms
11pub mod benchmark;
12/// Bitmap compression for boolean and sparse data
13pub mod bitmap;
14/// Bloom filter for membership testing
15pub mod bloom;
16/// Brotli compression algorithm (high compression ratio)
17pub mod brotli_compression;
18/// Column-store compression optimizations
19pub mod column_store;
20/// Delta encoding for numerical sequences
21pub mod delta;
22/// Dictionary compression for repeated values
23pub mod dictionary;
24/// Frame-of-reference encoding
25pub mod frame_of_reference;
26/// LZ4 compression algorithm (very fast)
27pub mod lz4;
28/// Prefix compression for strings
29pub mod prefix;
30/// Run-length encoding for repeated values
31pub mod run_length;
32/// SIMD-accelerated scan operations
33pub mod simd_scan;
34/// Snappy compression algorithm (fast with good ratio)
35pub mod snappy;
36/// Zstandard compression algorithm (excellent ratio)
37pub mod triple_delta;
38/// Unified compression interface with automatic algorithm selection
39pub mod unified;
40pub mod zstd_codec;
41pub mod zstd_compression;
42
43use anyhow::Result;
44use serde::{Deserialize, Serialize};
45use std::collections::HashMap;
46use std::fmt;
47
48// Re-export main compression implementations
49pub use adaptive::AdaptiveCompressor;
50pub use adaptive::{AdaptiveCodec, CodecType};
51pub use adaptive_selection::{AdaptiveCompressionSelector, SelectionStrategy};
52pub use benchmark::{BenchmarkResult, CompressionAlgo, CompressionBenchmark};
53pub use bitmap::{BitmapRoaringEncoder, BitmapWAHEncoder};
54pub use bloom::{BloomFilter, BloomFilterStats};
55pub use brotli_compression::BrotliCompressor;
56pub use column_store::ColumnStoreCompressor;
57pub use delta::DeltaEncoder;
58pub use dictionary::AdaptiveDictionary;
59pub use dictionary::TripleDictionary;
60pub use frame_of_reference::FrameOfReferenceEncoder;
61pub use lz4::Lz4Compressor;
62pub use prefix::{CompressedString, CompressionStats as PrefixCompressionStats, PrefixCompressor};
63pub use run_length::RunLengthEncoder;
64pub use simd_scan::SimdScanner;
65pub use snappy::SnappyCompressor;
66pub use triple_delta::{
67    DeltaStoreConfig, DeltaStoreStats, EncodedDeltaLog, TripleDelta, TripleDeltaStore,
68};
69pub use zstd_codec::{ZstdCodec, ZstdCodecStats};
70pub use zstd_compression::ZstdCompressor;
71
72/// Advanced compression types
73#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
74pub enum AdvancedCompressionType {
75    /// Run-Length Encoding for repetitive data
76    RunLength = 10,
77    /// Word-Aligned Hybrid (WAH) bitmap compression
78    BitmapWAH = 11,
79    /// Roaring bitmap compression
80    BitmapRoaring = 12,
81    /// Delta encoding for sequences
82    Delta = 13,
83    /// Frame of Reference (FOR) encoding
84    FrameOfReference = 14,
85    /// Dictionary with frequency-based Huffman coding
86    AdaptiveDictionary = 15,
87    /// Column-store with different compression per column
88    ColumnStore = 16,
89    /// Hybrid compression choosing best method
90    Adaptive = 17,
91    /// LZ4 compression (very fast)
92    Lz4 = 20,
93    /// Zstandard compression (excellent ratio)
94    Zstd = 21,
95    /// Snappy compression (fast with good ratio)
96    Snappy = 22,
97    /// Brotli compression (high compression ratio)
98    Brotli = 23,
99}
100
101impl fmt::Display for AdvancedCompressionType {
102    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
103        match self {
104            AdvancedCompressionType::RunLength => write!(f, "RunLength"),
105            AdvancedCompressionType::BitmapWAH => write!(f, "BitmapWAH"),
106            AdvancedCompressionType::BitmapRoaring => write!(f, "BitmapRoaring"),
107            AdvancedCompressionType::Delta => write!(f, "Delta"),
108            AdvancedCompressionType::FrameOfReference => write!(f, "FrameOfReference"),
109            AdvancedCompressionType::AdaptiveDictionary => write!(f, "AdaptiveDictionary"),
110            AdvancedCompressionType::ColumnStore => write!(f, "ColumnStore"),
111            AdvancedCompressionType::Adaptive => write!(f, "Adaptive"),
112            AdvancedCompressionType::Lz4 => write!(f, "LZ4"),
113            AdvancedCompressionType::Zstd => write!(f, "Zstd"),
114            AdvancedCompressionType::Snappy => write!(f, "Snappy"),
115            AdvancedCompressionType::Brotli => write!(f, "Brotli"),
116        }
117    }
118}
119
120/// Compression metadata
121#[derive(Debug, Clone, Serialize, Deserialize)]
122pub struct CompressionMetadata {
123    /// Compression algorithm used
124    pub algorithm: AdvancedCompressionType,
125    /// Original size in bytes
126    pub original_size: u64,
127    /// Compressed size in bytes
128    pub compressed_size: u64,
129    /// Compression time in microseconds
130    pub compression_time_us: u64,
131    /// Additional metadata specific to compression type
132    pub metadata: HashMap<String, String>,
133}
134
135impl CompressionMetadata {
136    /// Calculate compression ratio
137    pub fn compression_ratio(&self) -> f64 {
138        if self.original_size == 0 {
139            0.0
140        } else {
141            self.compressed_size as f64 / self.original_size as f64
142        }
143    }
144
145    /// Calculate space savings percentage
146    pub fn space_savings(&self) -> f64 {
147        (1.0 - self.compression_ratio()) * 100.0
148    }
149}
150
151impl Default for CompressionMetadata {
152    fn default() -> Self {
153        Self {
154            algorithm: AdvancedCompressionType::RunLength,
155            original_size: 0,
156            compressed_size: 0,
157            compression_time_us: 0,
158            metadata: HashMap::new(),
159        }
160    }
161}
162
163/// Compressed data with metadata
164#[derive(Debug, Clone, Serialize, Deserialize)]
165pub struct CompressedData {
166    /// Compressed payload
167    pub data: Vec<u8>,
168    /// Compression metadata
169    pub metadata: CompressionMetadata,
170}
171
172/// Maximum allowed data size for compression (100MB) to prevent resource exhaustion
173pub const MAX_COMPRESSION_INPUT_SIZE: usize = 100 * 1024 * 1024;
174
175/// Maximum allowed block count to prevent memory exhaustion
176pub const MAX_BLOCK_COUNT: usize = 10000;
177
178/// Maximum allowed dictionary size to prevent memory exhaustion  
179pub const MAX_DICTIONARY_SIZE: usize = 100000;
180
181/// Trait for compression algorithms
182pub trait CompressionAlgorithm {
183    /// Compress data and return compressed bytes with metadata
184    fn compress(&self, data: &[u8]) -> Result<CompressedData>;
185
186    /// Decompress data
187    fn decompress(&self, compressed: &CompressedData) -> Result<Vec<u8>>;
188
189    /// Get algorithm type
190    fn algorithm_type(&self) -> AdvancedCompressionType;
191}
192
193#[cfg(test)]
194mod tests {
195    use super::*;
196
197    #[test]
198    fn test_compression_metadata() {
199        let metadata = CompressionMetadata {
200            algorithm: AdvancedCompressionType::RunLength,
201            original_size: 1000,
202            compressed_size: 500,
203            compression_time_us: 100,
204            metadata: HashMap::new(),
205        };
206
207        assert_eq!(metadata.compression_ratio(), 0.5);
208        assert_eq!(metadata.space_savings(), 50.0);
209    }
210
211    #[test]
212    fn test_compression_type_display() {
213        assert_eq!(AdvancedCompressionType::RunLength.to_string(), "RunLength");
214        assert_eq!(AdvancedCompressionType::BitmapWAH.to_string(), "BitmapWAH");
215        assert_eq!(AdvancedCompressionType::Lz4.to_string(), "LZ4");
216        assert_eq!(AdvancedCompressionType::Zstd.to_string(), "Zstd");
217        assert_eq!(AdvancedCompressionType::Snappy.to_string(), "Snappy");
218        assert_eq!(AdvancedCompressionType::Brotli.to_string(), "Brotli");
219    }
220}
oxirs_tdb/compression/mod.rs

oxirs_tdb/compression/
mod.rs
