oxirs_tdb/compression/
zstd_compression.rs

1//! Zstandard (Zstd) compression algorithm implementation
2//!
3//! Zstandard is a modern compression algorithm that provides:
4//! - High compression ratios (often better than gzip)
5//! - Fast compression and decompression speeds
6//! - Tunable compression levels (1-22)
7//! - Dictionary support for small data compression
8//!
9//! # Performance Characteristics
10//!
11//! - Compression speed: 200-500 MB/s (level 3)
12//! - Decompression speed: 500-1500 MB/s
13//! - Compression ratio: 2.5-4x on typical data
14//!
15//! # Use Cases
16//!
17//! - Archival storage with high compression ratios
18//! - Network transmission where bandwidth is limited
19//! - General-purpose compression for RDF data
20
21use super::{
22    AdvancedCompressionType, CompressedData, CompressionAlgorithm, CompressionMetadata,
23    MAX_COMPRESSION_INPUT_SIZE,
24};
25use anyhow::{bail, Result};
26use std::collections::HashMap;
27use std::io::{Read, Write};
28use std::time::Instant;
29
30/// Zstandard compression algorithm
31///
32/// Uses the zstd crate for high-performance compression with excellent ratios.
33#[derive(Debug, Clone)]
34pub struct ZstdCompressor {
35    /// Compression level (1-22, higher = better compression but slower)
36    level: i32,
37}
38
39impl ZstdCompressor {
40    /// Create a new Zstd compressor with default compression level (3)
41    pub fn new() -> Self {
42        Self { level: 3 }
43    }
44
45    /// Create a new Zstd compressor with specified compression level
46    ///
47    /// # Arguments
48    ///
49    /// * `level` - Compression level (1-22). Recommended ranges:
50    ///   - 1-3: Fast compression, lower ratios
51    ///   - 4-9: Balanced compression (recommended for most use cases)
52    ///   - 10-19: High compression, slower
53    ///   - 20-22: Maximum compression, very slow
54    pub fn with_level(level: i32) -> Self {
55        Self {
56            level: level.clamp(1, 22),
57        }
58    }
59
60    /// Create a fast compression instance (level 1)
61    pub fn fast() -> Self {
62        Self::with_level(1)
63    }
64
65    /// Create a balanced compression instance (level 3)
66    pub fn balanced() -> Self {
67        Self::with_level(3)
68    }
69
70    /// Create a high compression instance (level 9)
71    pub fn high() -> Self {
72        Self::with_level(9)
73    }
74
75    /// Create a maximum compression instance (level 19)
76    pub fn max() -> Self {
77        Self::with_level(19)
78    }
79
80    /// Create an ultra compression instance (level 22)
81    pub fn ultra() -> Self {
82        Self::with_level(22)
83    }
84}
85
86impl Default for ZstdCompressor {
87    fn default() -> Self {
88        Self::new()
89    }
90}
91
92impl CompressionAlgorithm for ZstdCompressor {
93    fn compress(&self, data: &[u8]) -> Result<CompressedData> {
94        if data.is_empty() {
95            return Ok(CompressedData {
96                data: Vec::new(),
97                metadata: CompressionMetadata {
98                    algorithm: AdvancedCompressionType::Adaptive, // Will add Zstd type
99                    original_size: 0,
100                    compressed_size: 0,
101                    compression_time_us: 0,
102                    metadata: HashMap::new(),
103                },
104            });
105        }
106
107        if data.len() > MAX_COMPRESSION_INPUT_SIZE {
108            bail!(
109                "Input data too large: {} bytes (max: {})",
110                data.len(),
111                MAX_COMPRESSION_INPUT_SIZE
112            );
113        }
114
115        let start = Instant::now();
116
117        // Compress using zstd
118        let mut encoder = oxiarc_zstd::ZstdStreamEncoder::new(Vec::new(), self.level);
119        encoder.write_all(data)?;
120        let compressed = encoder.finish()?;
121
122        let compression_time = start.elapsed();
123
124        let mut metadata_map = HashMap::new();
125        metadata_map.insert("level".to_string(), self.level.to_string());
126        metadata_map.insert(
127            "compression_ratio".to_string(),
128            format!("{:.2}", compressed.len() as f64 / data.len() as f64),
129        );
130
131        let compressed_size = compressed.len() as u64;
132
133        Ok(CompressedData {
134            data: compressed,
135            metadata: CompressionMetadata {
136                algorithm: AdvancedCompressionType::Adaptive, // Will add Zstd type
137                original_size: data.len() as u64,
138                compressed_size,
139                compression_time_us: compression_time.as_micros() as u64,
140                metadata: metadata_map,
141            },
142        })
143    }
144
145    fn decompress(&self, compressed: &CompressedData) -> Result<Vec<u8>> {
146        if compressed.data.is_empty() {
147            return Ok(Vec::new());
148        }
149
150        let start = Instant::now();
151
152        // Decompress using zstd
153        let mut decoder = oxiarc_zstd::ZstdStreamDecoder::new(&compressed.data[..]);
154        let mut decompressed = Vec::new();
155        decoder.read_to_end(&mut decompressed)?;
156
157        let decompression_time = start.elapsed();
158
159        // Verify decompressed size matches metadata
160        if decompressed.len() != compressed.metadata.original_size as usize {
161            bail!(
162                "Decompressed size mismatch: expected {}, got {}",
163                compressed.metadata.original_size,
164                decompressed.len()
165            );
166        }
167
168        log::trace!(
169            "Zstd decompression: {} bytes -> {} bytes in {:?}",
170            compressed.data.len(),
171            decompressed.len(),
172            decompression_time
173        );
174
175        Ok(decompressed)
176    }
177
178    fn algorithm_type(&self) -> AdvancedCompressionType {
179        AdvancedCompressionType::Adaptive // Will add Zstd type
180    }
181}
182
183#[cfg(test)]
184mod tests {
185    use super::*;
186
187    #[test]
188    fn test_zstd_compress_empty() {
189        let compressor = ZstdCompressor::new();
190        let result = compressor.compress(&[]).unwrap();
191        assert_eq!(result.data.len(), 0);
192        assert_eq!(result.metadata.original_size, 0);
193        assert_eq!(result.metadata.compressed_size, 0);
194    }
195
196    #[test]
197    fn test_zstd_compress_decompress() {
198        let compressor = ZstdCompressor::new();
199        let original = b"Hello, World! This is a test of Zstandard compression. ".repeat(100);
200
201        let compressed = compressor.compress(&original).unwrap();
202        assert!(compressed.data.len() < original.len());
203        assert_eq!(compressed.metadata.original_size, original.len() as u64);
204
205        let decompressed = compressor.decompress(&compressed).unwrap();
206        assert_eq!(decompressed, original);
207    }
208
209    #[test]
210    fn test_zstd_compression_levels() {
211        let data = b"The quick brown fox jumps over the lazy dog. ".repeat(50);
212
213        let fast = ZstdCompressor::fast();
214        let balanced = ZstdCompressor::balanced();
215        let high = ZstdCompressor::high();
216        let max = ZstdCompressor::max();
217
218        let fast_result = fast.compress(&data).unwrap();
219        let balanced_result = balanced.compress(&data).unwrap();
220        let high_result = high.compress(&data).unwrap();
221        let max_result = max.compress(&data).unwrap();
222
223        // All should decompress to the same data
224        assert_eq!(fast.decompress(&fast_result).unwrap(), data);
225        assert_eq!(balanced.decompress(&balanced_result).unwrap(), data);
226        assert_eq!(high.decompress(&high_result).unwrap(), data);
227        assert_eq!(max.decompress(&max_result).unwrap(), data);
228
229        // Higher compression levels should produce smaller or equal output
230        println!("Fast (1): {} bytes", fast_result.data.len());
231        println!("Balanced (3): {} bytes", balanced_result.data.len());
232        println!("High (9): {} bytes", high_result.data.len());
233        println!("Max (19): {} bytes", max_result.data.len());
234
235        // Verify compression improves with higher levels
236        assert!(balanced_result.data.len() <= fast_result.data.len());
237        assert!(high_result.data.len() <= balanced_result.data.len());
238        assert!(max_result.data.len() <= high_result.data.len());
239    }
240
241    #[test]
242    fn test_zstd_highly_compressible_data() {
243        let compressor = ZstdCompressor::new();
244        let data = vec![b'A'; 10000]; // Highly repetitive data
245
246        let compressed = compressor.compress(&data).unwrap();
247        let decompressed = compressor.decompress(&compressed).unwrap();
248
249        assert_eq!(decompressed, data);
250        // Should achieve excellent compression on repetitive data
251        assert!(compressed.data.len() < data.len() / 100);
252        println!(
253            "Zstd highly compressible: {} -> {} bytes ({:.2}x)",
254            data.len(),
255            compressed.data.len(),
256            data.len() as f64 / compressed.data.len() as f64
257        );
258    }
259
260    #[test]
261    fn test_zstd_incompressible_data() {
262        let compressor = ZstdCompressor::new();
263        // Use scirs2_core::random for random data generation
264        use scirs2_core::random::rng;
265        use scirs2_core::RngExt;
266
267        let mut rng = rng();
268        let data: Vec<u8> = (0..1000).map(|_| rng.random_range(0..256) as u8).collect();
269
270        let compressed = compressor.compress(&data).unwrap();
271        let decompressed = compressor.decompress(&compressed).unwrap();
272
273        assert_eq!(decompressed, data);
274        // Random data should not compress well, might even expand slightly
275        println!(
276            "Zstd incompressible: {} -> {} bytes",
277            data.len(),
278            compressed.data.len()
279        );
280    }
281
282    #[test]
283    fn test_zstd_too_large_input() {
284        let compressor = ZstdCompressor::new();
285        let data = vec![0u8; MAX_COMPRESSION_INPUT_SIZE + 1];
286
287        let result = compressor.compress(&data);
288        assert!(result.is_err());
289    }
290
291    #[test]
292    fn test_zstd_metadata() {
293        let compressor = ZstdCompressor::with_level(9);
294        let data = b"Testing metadata collection with Zstandard".repeat(10);
295
296        let compressed = compressor.compress(&data).unwrap();
297
298        assert_eq!(compressed.metadata.metadata.get("level").unwrap(), "9");
299        assert!(compressed
300            .metadata
301            .metadata
302            .contains_key("compression_ratio"));
303        assert!(compressed.metadata.compression_time_us > 0);
304    }
305
306    #[test]
307    fn test_zstd_rdf_data_compression() {
308        let compressor = ZstdCompressor::balanced();
309        // Simulate RDF triple data with typical patterns
310        let rdf_data = b"<http://example.org/resource/1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://example.org/Class> .\n"
311            .repeat(100);
312
313        let compressed = compressor.compress(&rdf_data).unwrap();
314        let decompressed = compressor.decompress(&compressed).unwrap();
315
316        assert_eq!(decompressed, rdf_data);
317
318        // RDF data with repetitive URIs should compress very well
319        let ratio = rdf_data.len() as f64 / compressed.data.len() as f64;
320        println!("RDF compression ratio: {:.2}x", ratio);
321        assert!(ratio > 3.0); // Should achieve >3x compression
322    }
323
324    #[test]
325    fn test_zstd_level_clamping() {
326        let too_low = ZstdCompressor::with_level(0);
327        let too_high = ZstdCompressor::with_level(100);
328
329        assert_eq!(too_low.level, 1);
330        assert_eq!(too_high.level, 22);
331    }
332}
oxirs_tdb/compression/zstd_compression.rs

oxirs_tdb/compression/
zstd_compression.rs
