scirs2_sparse/adaptive_memory_compression/
mod.rs

1//! Adaptive Memory Compression for Advanced-Large Sparse Matrices
2//!
3//! This module provides advanced memory management and compression techniques
4//! specifically designed for handling advanced-large sparse matrices that exceed
5//! available system memory.
6//!
7//! ## Architecture
8//!
9//! The adaptive memory compression system consists of several interconnected components:
10//!
11//! - **Configuration**: Flexible configuration system for different compression strategies
12//! - **Cache Management**: Intelligent block caching with LRU eviction and access tracking
13//! - **Access Tracking**: Pattern analysis for optimizing compression and caching decisions
14//! - **Compression**: Multiple compression algorithms optimized for sparse data
15//! - **Out-of-Core Storage**: Seamless handling of matrices larger than available memory
16//! - **Memory Mapping**: Efficient file-based storage with memory mapping support
17//! - **Statistics**: Comprehensive performance tracking and optimization guidance
18//!
19//! ## Usage
20//!
21//! ```rust,ignore
22//! use scirs2_sparse::adaptive_memory_compression::{
23//!     AdaptiveMemoryCompressor, AdaptiveCompressionConfig, CompressionAlgorithm
24//! };
25//!
26//! // Create configuration
27//! let config = AdaptiveCompressionConfig::new()
28//!     .with_memory_budget(8 * 1024 * 1024 * 1024) // 8GB
29//!     .with_compression_algorithm(CompressionAlgorithm::Adaptive)
30//!     .with_out_of_core(true);
31//!
32//! // Create compressor
33//! let mut compressor = AdaptiveMemoryCompressor::new(config)?;
34//!
35//! // Compress a sparse matrix
36//! let compressed = compressor.compress_matrix(
37//!     1,          // matrix_id
38//!     1000,       // rows
39//!     &indptr,    // CSR indptr
40//!     &indices,   // CSR indices
41//!     &data       // CSR data
42//! )?;
43//! ```
44//!
45//! ## Performance Optimization
46//!
47//! The system automatically learns from access patterns and adapts compression strategies:
48//!
49//! ```rust,ignore
50//! // Get performance statistics
51//! let stats = compressor.get_stats();
52//! println!("Compression ratio: {:.2}", stats.compression_ratio);
53//! println!("Cache hit rate: {:.2}%", stats.cache_hit_ratio * 100.0);
54//!
55//! // Manual optimization based on access patterns
56//! compressor.optimize_for_sequential_access();
57//! compressor.optimize_for_random_access();
58//! ```
59
60pub mod access_tracking;
61pub mod cache;
62pub mod compressed_data;
63pub mod compression;
64pub mod compressor;
65pub mod config;
66pub mod memory_mapping;
67pub mod out_of_core;
68pub mod stats;
69
70// Re-export main types for convenience
71pub use cache::BlockId;
72pub use compressed_data::{BlockType, CompressedBlock, CompressedMatrix};
73pub use compressor::AdaptiveMemoryCompressor;
74pub use config::{AdaptiveCompressionConfig, CompressionAlgorithm};
75pub use stats::{CompressionMetadata, CompressionStats, MemoryStats};
76
77// Re-export key internal types that might be useful
78pub use access_tracking::AccessType;
79pub use compression::{CompressionEngine, CompressionResult};
80pub use memory_mapping::MemoryMappedFile;
81pub use out_of_core::OutOfCoreManager;
82pub use stats::AccessPatternType;
83
84#[cfg(test)]
85mod tests {
86    use super::*;
87
88    #[test]
89    fn test_config_creation() {
90        let config = AdaptiveCompressionConfig::new();
91        assert_eq!(config.memory_budget, 8 * 1024 * 1024 * 1024);
92        assert!(matches!(
93            config.compression_algorithm,
94            CompressionAlgorithm::Adaptive
95        ));
96    }
97
98    #[test]
99    fn test_config_builder_pattern() {
100        let config = AdaptiveCompressionConfig::new()
101            .with_memory_budget(4 * 1024 * 1024 * 1024)
102            .with_compression_algorithm(CompressionAlgorithm::LZ77)
103            .with_out_of_core(false);
104
105        assert_eq!(config.memory_budget, 4 * 1024 * 1024 * 1024);
106        assert!(matches!(
107            config.compression_algorithm,
108            CompressionAlgorithm::LZ77
109        ));
110        assert!(!config.out_of_core);
111    }
112
113    #[test]
114    fn test_config_validation() {
115        let mut config = AdaptiveCompressionConfig::new();
116        assert!(config.validate().is_ok());
117
118        config.memory_budget = 0;
119        assert!(config.validate().is_err());
120
121        config.memory_budget = 1024;
122        config.compression_threshold = 1.5;
123        assert!(config.validate().is_err());
124
125        config.compression_threshold = 0.8;
126        config.cache_size = config.memory_budget + 1;
127        assert!(config.validate().is_err());
128    }
129
130    #[test]
131    fn test_predefined_configurations() {
132        let lightweight = AdaptiveCompressionConfig::lightweight();
133        assert_eq!(lightweight.memory_budget, 64 * 1024 * 1024);
134        assert!(!lightweight.hierarchical_compression);
135
136        let high_perf = AdaptiveCompressionConfig::high_performance();
137        assert_eq!(high_perf.memory_budget, 32 * 1024 * 1024 * 1024);
138        assert!(high_perf.hierarchical_compression);
139
140        let mem_efficient = AdaptiveCompressionConfig::memory_efficient();
141        assert_eq!(mem_efficient.memory_budget, 1024 * 1024 * 1024);
142        assert!(matches!(
143            mem_efficient.compression_algorithm,
144            CompressionAlgorithm::LZ77
145        ));
146    }
147
148    #[test]
149    fn test_compression_algorithm_properties() {
150        assert!(CompressionAlgorithm::Adaptive.supports_adaptive());
151        assert!(CompressionAlgorithm::SparseOptimized.supports_adaptive());
152        assert!(!CompressionAlgorithm::RLE.supports_adaptive());
153
154        assert_eq!(CompressionAlgorithm::None.expected_compression_ratio(), 1.0);
155        assert!(CompressionAlgorithm::Adaptive.expected_compression_ratio() < 0.5);
156
157        assert!(
158            CompressionAlgorithm::None.compression_speed()
159                > CompressionAlgorithm::Adaptive.compression_speed()
160        );
161    }
162
163    #[test]
164    fn test_block_id_operations() {
165        let block_id = BlockId::new(123, 10, 20);
166        assert_eq!(block_id.matrixid, 123);
167        assert_eq!(block_id.block_row, 10);
168        assert_eq!(block_id.block_col, 20);
169
170        let as_u64 = block_id.to_u64();
171        let restored = BlockId::from_u64(as_u64);
172        assert_eq!(block_id, restored);
173
174        let as_string = block_id.as_string();
175        let restored_from_string = BlockId::from_string(&as_string).unwrap();
176        assert_eq!(block_id, restored_from_string);
177    }
178
179    #[test]
180    fn test_compressed_block_creation() {
181        let block_id = BlockId::new(1, 0, 0);
182        let data = vec![1, 2, 3, 4, 5];
183        let original_size = 100;
184
185        let block = CompressedBlock::new(
186            block_id.clone(),
187            BlockType::Data,
188            data.clone(),
189            original_size,
190            1,
191        );
192
193        assert_eq!(block.blockid, block_id);
194        assert_eq!(block.block_type, BlockType::Data);
195        assert_eq!(block.compressed_data, data);
196        assert_eq!(block.original_size, original_size);
197        assert!(block.checksum.is_some());
198        assert!(block.verify_integrity());
199    }
200
201    #[test]
202    fn test_compressed_matrix_operations() {
203        let mut matrix =
204            CompressedMatrix::<f64>::new(1, 1000, 1000, CompressionAlgorithm::RLE, 1024);
205
206        let block_id = BlockId::new(1, 0, 0);
207        let block =
208            CompressedBlock::new(block_id.clone(), BlockType::Data, vec![1, 2, 3, 4], 100, 1);
209
210        matrix.add_block(block);
211        assert_eq!(matrix.block_count(), 1);
212        assert!(matrix.get_block(&block_id).is_some());
213
214        let removed = matrix.remove_block(&block_id);
215        assert!(removed.is_some());
216        assert_eq!(matrix.block_count(), 0);
217    }
218
219    #[test]
220    fn test_block_type_properties() {
221        assert_eq!(BlockType::Data.as_str(), "data");
222        assert_eq!(BlockType::from_str("indices").unwrap(), BlockType::Indices);
223
224        assert!(
225            BlockType::Data.compression_priority() > BlockType::Metadata.compression_priority()
226        );
227        assert!(BlockType::Data.benefits_from_compression());
228        assert!(!BlockType::IndPtr.benefits_from_compression());
229    }
230
231    #[test]
232    fn test_compression_stats() {
233        let mut stats = CompressionStats::new();
234
235        stats.update_compression(1000, 500, 0.1);
236        assert_eq!(stats.total_blocks, 1);
237        assert_eq!(stats.compressed_blocks, 1);
238        assert_eq!(stats.compression_ratio, 0.5);
239
240        stats.record_cache_hit();
241        stats.record_cache_miss();
242        assert_eq!(stats.cache_hit_ratio(), 0.5);
243
244        assert_eq!(stats.space_savings(), 500);
245        assert_eq!(stats.space_savings_ratio(), 0.5);
246    }
247
248    #[test]
249    fn test_memory_stats() {
250        let mut stats = MemoryStats::new(1024 * 1024, true);
251
252        stats.update_memory_usage(512 * 1024);
253        assert_eq!(stats.memory_usage_ratio, 0.5);
254        assert!(!stats.has_memory_pressure(0.8));
255        assert!(stats.has_memory_pressure(0.3));
256
257        assert_eq!(stats.available_memory(), 512 * 1024);
258    }
259
260    #[test]
261    fn test_access_pattern_type_display() {
262        assert_eq!(AccessPatternType::Sequential.to_string(), "Sequential");
263        assert_eq!(AccessPatternType::Random.to_string(), "Random");
264        assert_eq!(AccessPatternType::Clustered.to_string(), "Clustered");
265        assert_eq!(AccessPatternType::Mixed.to_string(), "Mixed");
266        assert_eq!(AccessPatternType::Unknown.to_string(), "Unknown");
267    }
268}
scirs2_sparse/adaptive_memory_compression/mod.rs

scirs2_sparse/adaptive_memory_compression/
mod.rs
