ipfrs_storage/
helpers.rs

1//! Helper functions for creating storage stacks
2//!
3//! This module provides convenient functions for setting up common storage configurations.
4
5use crate::{
6    BlockStoreConfig, BloomBlockStore, BloomConfig, CachedBlockStore, MemoryBlockStore,
7    MetricsBlockStore, SledBlockStore,
8};
9use crate::{ChunkingConfig, DedupBlockStore};
10#[cfg(feature = "encryption")]
11use crate::{Cipher, EncryptedBlockStore, EncryptionConfig};
12use crate::{CoalesceConfig, CoalescingBlockStore};
13#[cfg(feature = "compression")]
14use crate::{CompressionAlgorithm, CompressionBlockStore, CompressionConfig};
15use crate::{TtlBlockStore, TtlConfig};
16use ipfrs_core::Result;
17use std::path::PathBuf;
18use std::time::Duration;
19
20// Type aliases for complex storage stacks
21type FullStack = BloomBlockStore<CachedBlockStore<SledBlockStore>>;
22type MonitoredFullStack = MetricsBlockStore<FullStack>;
23
24#[cfg(feature = "compression")]
25type CompressedStack = CompressionBlockStore<FullStack>;
26#[cfg(feature = "compression")]
27type MonitoredCompressedStack = MetricsBlockStore<CompressedStack>;
28
29#[cfg(feature = "encryption")]
30type EncryptedStack = EncryptedBlockStore<FullStack>;
31#[cfg(feature = "encryption")]
32type MonitoredEncryptedStack = MetricsBlockStore<EncryptedStack>;
33
34type DedupStack = DedupBlockStore<FullStack>;
35type MonitoredDedupStack = MetricsBlockStore<DedupStack>;
36
37#[cfg(feature = "compression")]
38type UltimateStack = DedupBlockStore<CompressedStack>;
39#[cfg(feature = "compression")]
40type MonitoredUltimateStack = MetricsBlockStore<UltimateStack>;
41
42/// Storage stack builder for easy configuration
43pub struct StorageStackBuilder {
44    config: BlockStoreConfig,
45    enable_cache: bool,
46    cache_size_mb: usize,
47    enable_bloom: bool,
48    bloom_expected_items: usize,
49    enable_tiering: bool,
50}
51
52impl Default for StorageStackBuilder {
53    fn default() -> Self {
54        Self {
55            config: BlockStoreConfig::default(),
56            enable_cache: true,
57            cache_size_mb: 100,
58            enable_bloom: true,
59            bloom_expected_items: 100_000,
60            enable_tiering: false,
61        }
62    }
63}
64
65impl StorageStackBuilder {
66    /// Create a new storage stack builder
67    pub fn new() -> Self {
68        Self::default()
69    }
70
71    /// Set the base storage configuration
72    pub fn with_config(mut self, config: BlockStoreConfig) -> Self {
73        self.config = config;
74        self
75    }
76
77    /// Set the storage path
78    pub fn with_path(mut self, path: PathBuf) -> Self {
79        self.config.path = path;
80        self
81    }
82
83    /// Enable LRU caching with specified size in MB
84    pub fn with_cache(mut self, size_mb: usize) -> Self {
85        self.enable_cache = true;
86        self.cache_size_mb = size_mb;
87        self
88    }
89
90    /// Disable LRU caching
91    pub fn without_cache(mut self) -> Self {
92        self.enable_cache = false;
93        self
94    }
95
96    /// Enable bloom filter with expected number of items
97    pub fn with_bloom(mut self, expected_items: usize) -> Self {
98        self.enable_bloom = true;
99        self.bloom_expected_items = expected_items;
100        self
101    }
102
103    /// Disable bloom filter
104    pub fn without_bloom(mut self) -> Self {
105        self.enable_bloom = false;
106        self
107    }
108
109    /// Enable hot/cold tiering
110    pub fn with_tiering(mut self) -> Self {
111        self.enable_tiering = true;
112        self
113    }
114
115    /// Build a simple storage stack (base store only)
116    pub fn build_simple(self) -> Result<SledBlockStore> {
117        SledBlockStore::new(self.config)
118    }
119
120    /// Build a cached storage stack
121    pub fn build_cached(self) -> Result<CachedBlockStore<SledBlockStore>> {
122        let base = SledBlockStore::new(self.config)?;
123        let cache_size = self.cache_size_mb * 1024 * 1024;
124        Ok(CachedBlockStore::new(base, cache_size))
125    }
126
127    /// Build a full storage stack with cache and bloom filter
128    pub fn build_full(self) -> Result<BloomBlockStore<CachedBlockStore<SledBlockStore>>> {
129        let base = SledBlockStore::new(self.config)?;
130
131        let cache_size = if self.enable_cache {
132            self.cache_size_mb * 1024 * 1024
133        } else {
134            1024 // Minimal cache if disabled
135        };
136        let cached = CachedBlockStore::new(base, cache_size);
137
138        if self.enable_bloom {
139            let bloom_config = BloomConfig::new(self.bloom_expected_items, 0.01);
140            Ok(BloomBlockStore::with_config(cached, bloom_config))
141        } else {
142            // Return with minimal bloom filter if disabled
143            let bloom_config = BloomConfig::new(100, 0.01);
144            Ok(BloomBlockStore::with_config(cached, bloom_config))
145        }
146    }
147}
148
149/// Quick setup functions for common use cases
150/// Create a development storage stack with caching and bloom filter
151///
152/// - Path: /tmp/ipfrs-dev
153/// - Cache: 50MB
154/// - Bloom filter: 10,000 expected items
155pub fn development_stack() -> Result<BloomBlockStore<CachedBlockStore<SledBlockStore>>> {
156    StorageStackBuilder::new()
157        .with_config(BlockStoreConfig::development())
158        .with_cache(50)
159        .with_bloom(10_000)
160        .build_full()
161}
162
163/// Create a production storage stack with caching and bloom filter
164///
165/// - Path: provided by user
166/// - Cache: 500MB
167/// - Bloom filter: 1,000,000 expected items
168pub fn production_stack(path: PathBuf) -> Result<FullStack> {
169    StorageStackBuilder::new()
170        .with_config(BlockStoreConfig::production(path))
171        .with_cache(500)
172        .with_bloom(1_000_000)
173        .build_full()
174}
175
176/// Create an embedded storage stack with minimal resource usage
177///
178/// - Path: provided by user
179/// - Cache: 10MB
180/// - Bloom filter: 5,000 expected items
181pub fn embedded_stack(path: PathBuf) -> Result<BloomBlockStore<CachedBlockStore<SledBlockStore>>> {
182    StorageStackBuilder::new()
183        .with_config(BlockStoreConfig::embedded(path))
184        .with_cache(10)
185        .with_bloom(5_000)
186        .build_full()
187}
188
189/// Create a testing storage stack with minimal resources
190///
191/// - Path: temporary directory
192/// - Cache: 5MB
193/// - Bloom filter: 1,000 expected items
194pub fn testing_stack() -> Result<BloomBlockStore<CachedBlockStore<SledBlockStore>>> {
195    StorageStackBuilder::new()
196        .with_config(BlockStoreConfig::testing())
197        .with_cache(5)
198        .with_bloom(1_000)
199        .build_full()
200}
201
202/// Create a production stack with metrics tracking
203///
204/// Adds comprehensive metrics on top of a production storage stack.
205/// Useful for monitoring performance in production deployments.
206pub fn monitored_production_stack(path: PathBuf) -> Result<MonitoredFullStack> {
207    let base = production_stack(path)?;
208    Ok(MetricsBlockStore::new(base))
209}
210
211/// Create a high-performance in-memory stack
212///
213/// Best for:
214/// - Testing and development
215/// - Temporary caching layers
216/// - High-speed operations where persistence isn't needed
217pub fn memory_stack() -> MetricsBlockStore<BloomBlockStore<MemoryBlockStore>> {
218    let base = MemoryBlockStore::new();
219    let bloom_config = BloomConfig::new(100_000, 0.01);
220    let bloom = BloomBlockStore::with_config(base, bloom_config);
221    MetricsBlockStore::new(bloom)
222}
223
224/// Create a compressed production stack (requires "compression" feature)
225///
226/// Uses Zstd compression to reduce storage size.
227/// Best for: Large datasets where storage space is a concern
228#[cfg(feature = "compression")]
229pub fn compressed_production_stack(path: PathBuf) -> Result<MonitoredCompressedStack> {
230    let base = production_stack(path)?;
231    let compression_config = CompressionConfig {
232        algorithm: CompressionAlgorithm::Zstd,
233        level: 3,        // Balanced compression
234        threshold: 1024, // Only compress blocks > 1KB
235        max_ratio: 0.9,  // Only keep if compressed to 90% or less
236    };
237    let compressed = CompressionBlockStore::new(base, compression_config);
238    Ok(MetricsBlockStore::new(compressed))
239}
240
241/// Create an encrypted production stack (requires "encryption" feature)
242///
243/// Uses ChaCha20-Poly1305 encryption for data at rest.
244/// Best for: Sensitive data requiring encryption
245#[cfg(feature = "encryption")]
246pub fn encrypted_production_stack(
247    path: PathBuf,
248    password: &str,
249) -> Result<MonitoredEncryptedStack> {
250    use crate::EncryptionKey;
251
252    let base = production_stack(path)?;
253    let (key, _salt) =
254        EncryptionKey::derive_from_password(Cipher::ChaCha20Poly1305, password.as_bytes(), None)?;
255    let config = EncryptionConfig {
256        cipher: Cipher::ChaCha20Poly1305,
257    };
258    let encrypted = EncryptedBlockStore::new(base, key, config);
259    Ok(MetricsBlockStore::new(encrypted))
260}
261
262/// Create a deduplicated production stack
263///
264/// Uses content-defined chunking for automatic deduplication.
265/// Best for: Datasets with significant redundancy
266pub fn deduplicated_production_stack(path: PathBuf) -> Result<MonitoredDedupStack> {
267    let base = production_stack(path)?;
268    let chunking_config = ChunkingConfig::default();
269    let dedup = DedupBlockStore::new(base, chunking_config);
270    Ok(MetricsBlockStore::new(dedup))
271}
272
273/// Create the ultimate production stack with all optimizations (requires all features)
274///
275/// Combines:
276/// - Compression (Zstd)
277/// - Deduplication
278/// - Caching
279/// - Bloom filters
280/// - Metrics tracking
281///
282/// Best for: Maximum efficiency in production with all features enabled
283#[cfg(feature = "compression")]
284pub fn ultimate_production_stack(path: PathBuf) -> Result<MonitoredUltimateStack> {
285    let base = compressed_production_stack(path)?;
286    // Remove metrics temporarily to add dedup, then re-add metrics
287    let inner = base.into_inner();
288    let chunking_config = ChunkingConfig::default();
289    let dedup = DedupBlockStore::new(inner, chunking_config);
290    Ok(MetricsBlockStore::new(dedup))
291}
292
293/// Create a production stack with TTL support for automatic expiration
294///
295/// Useful for:
296/// - Temporary cache layers
297/// - Time-limited data storage
298/// - Preventing unbounded growth
299///
300/// # Arguments
301/// * `path` - Storage directory path
302/// * `default_ttl` - Default time-to-live for blocks
303pub fn ttl_production_stack(
304    path: PathBuf,
305    default_ttl: Duration,
306) -> Result<MetricsBlockStore<TtlBlockStore<FullStack>>> {
307    let base = production_stack(path)?;
308    let ttl_config = TtlConfig {
309        default_ttl,
310        auto_cleanup: true,
311        cleanup_interval: Duration::from_secs(300), // 5 minutes
312        max_tracked_blocks: 1_000_000,
313    };
314    let ttl_store = TtlBlockStore::new(base, ttl_config);
315    Ok(MetricsBlockStore::new(ttl_store))
316}
317
318/// Create a production stack with automatic expiration for cache use cases
319///
320/// Optimized for cache workloads with:
321/// - 1-hour default TTL
322/// - Automatic cleanup every 5 minutes
323/// - Large cache (500MB)
324/// - Bloom filter for fast negative lookups
325///
326/// Best for: Temporary data caching with automatic expiration
327pub fn cache_stack(path: PathBuf) -> Result<MetricsBlockStore<TtlBlockStore<FullStack>>> {
328    ttl_production_stack(path, Duration::from_secs(3600)) // 1 hour TTL
329}
330
331/// Create a high-performance write-coalescing stack for in-memory operations
332///
333/// Combines:
334/// - In-memory storage (no persistence)
335/// - Write coalescing for batching (1000 writes per batch)
336/// - 100ms flush interval
337/// - Metrics tracking
338///
339/// Best for: Temporary high-throughput write workloads
340pub fn coalescing_memory_stack() -> MetricsBlockStore<CoalescingBlockStore<MemoryBlockStore>> {
341    let base = MemoryBlockStore::new();
342    let coalesce_config = CoalesceConfig::new(1000, Duration::from_millis(100));
343    let coalescing = CoalescingBlockStore::new(base, coalesce_config);
344    MetricsBlockStore::new(coalescing)
345}
346
347/// Create a read-optimized production stack
348///
349/// Optimized for read-heavy workloads with:
350/// - Large cache (1GB) for frequently accessed blocks
351/// - Bloom filter for fast negative lookups
352/// - Metrics tracking
353///
354/// Best for: Content delivery and read-heavy applications
355pub fn read_optimized_stack(path: PathBuf) -> Result<FullStack> {
356    StorageStackBuilder::new()
357        .with_config(BlockStoreConfig::production(path))
358        .with_cache(1024) // 1GB cache
359        .with_bloom(2_000_000) // Support for 2M blocks
360        .build_full()
361}
362
363/// Create a write-optimized production stack with deduplication
364///
365/// Optimized for write-heavy workloads with:
366/// - Deduplication to reduce storage
367/// - Smaller cache (100MB) to favor writes
368/// - Bloom filter for existence checks
369/// - Metrics tracking
370///
371/// Best for: Ingestion pipelines and write-heavy applications
372pub fn write_optimized_stack(path: PathBuf) -> Result<MonitoredDedupStack> {
373    let base = StorageStackBuilder::new()
374        .with_config(BlockStoreConfig::production(path))
375        .with_cache(100) // Smaller cache for writes
376        .with_bloom(1_000_000)
377        .build_full()?;
378
379    let chunking_config = ChunkingConfig::default();
380    let dedup = DedupBlockStore::new(base, chunking_config);
381    Ok(MetricsBlockStore::new(dedup))
382}
383
384/// Create a minimal resource stack for IoT/embedded devices
385///
386/// Ultra-low resource usage with:
387/// - 5MB cache
388/// - Small bloom filter (1000 expected items)
389/// - Minimal batch sizes
390///
391/// Best for: Raspberry Pi, embedded systems, resource-constrained environments
392pub fn iot_stack(path: PathBuf) -> Result<FullStack> {
393    StorageStackBuilder::new()
394        .with_config(BlockStoreConfig::embedded(path))
395        .with_cache(5) // 5MB cache
396        .with_bloom(1_000) // Very small bloom filter
397        .build_full()
398}
399
400/// Create a resilient production stack with all safety features
401///
402/// Combines:
403/// - TTL for automatic cleanup (24 hours default)
404/// - Metrics tracking for monitoring
405/// - Large cache for performance
406/// - Bloom filter for efficiency
407///
408/// Best for: Mission-critical production deployments requiring data lifecycle management
409pub fn resilient_stack(path: PathBuf) -> Result<MetricsBlockStore<TtlBlockStore<FullStack>>> {
410    ttl_production_stack(path, Duration::from_secs(86400)) // 24 hour TTL
411}
412
413/// Create a high-throughput ingestion stack
414///
415/// Optimized for maximum write throughput:
416/// - Deduplication for storage efficiency
417/// - Smaller cache (200MB) to favor writes
418/// - Bloom filter for fast existence checks
419/// - Metrics for monitoring
420///
421/// Best for: Data ingestion pipelines, ETL processes, bulk imports
422pub fn ingestion_stack(path: PathBuf) -> Result<MonitoredDedupStack> {
423    let base = StorageStackBuilder::new()
424        .with_config(BlockStoreConfig::production(path))
425        .with_cache(200) // Smaller cache for write optimization
426        .with_bloom(2_000_000)
427        .build_full()?;
428
429    let chunking_config = ChunkingConfig::default();
430    let dedup = DedupBlockStore::new(base, chunking_config);
431
432    Ok(MetricsBlockStore::new(dedup))
433}
434
435/// Create a CDN edge cache stack
436///
437/// Optimized for content delivery:
438/// - Very large cache (2GB) for hot content
439/// - TTL for automatic expiration (1 hour default)
440/// - Bloom filter for fast negative lookups
441/// - Metrics for monitoring cache effectiveness
442///
443/// Best for: CDN edge nodes, content delivery, proxy caching
444pub fn cdn_edge_stack(path: PathBuf) -> Result<MetricsBlockStore<TtlBlockStore<FullStack>>> {
445    let base = StorageStackBuilder::new()
446        .with_config(BlockStoreConfig::production(path))
447        .with_cache(2048) // 2GB cache
448        .with_bloom(5_000_000) // Support large number of blocks
449        .build_full()?;
450
451    let ttl_config = TtlConfig {
452        default_ttl: Duration::from_secs(3600), // 1 hour
453        auto_cleanup: true,
454        cleanup_interval: Duration::from_secs(600), // 10 minutes
455        max_tracked_blocks: 5_000_000,
456    };
457    let ttl_store = TtlBlockStore::new(base, ttl_config);
458
459    Ok(MetricsBlockStore::new(ttl_store))
460}
461
462/// Create a scientific data archive stack
463///
464/// Optimized for large scientific datasets:
465/// - Compression (Zstd level 5 for better compression)
466/// - Deduplication for redundant data
467/// - Medium cache (256MB)
468/// - Bloom filter
469///
470/// Best for: Scientific data repositories, research archives, large dataset storage
471#[cfg(feature = "compression")]
472pub fn scientific_archive_stack(path: PathBuf) -> Result<MonitoredUltimateStack> {
473    let base = StorageStackBuilder::new()
474        .with_config(BlockStoreConfig::production(path))
475        .with_cache(256)
476        .with_bloom(1_000_000)
477        .build_full()?;
478
479    let compression_config = CompressionConfig {
480        algorithm: CompressionAlgorithm::Zstd,
481        level: 5,        // Higher compression for archives
482        threshold: 512,  // Compress blocks > 512 bytes
483        max_ratio: 0.95, // Keep if compressed to 95% or less
484    };
485    let compressed = CompressionBlockStore::new(base, compression_config);
486
487    let chunking_config = ChunkingConfig::default();
488    let dedup = DedupBlockStore::new(compressed, chunking_config);
489
490    Ok(MetricsBlockStore::new(dedup))
491}
492
493/// Create a blockchain storage stack
494///
495/// Optimized for blockchain data:
496/// - No TTL (permanent storage)
497/// - Large bloom filter for fast lookups
498/// - Medium cache for recent blocks
499/// - Deduplication for transactions
500/// - Metrics for monitoring
501///
502/// Best for: Blockchain nodes, distributed ledgers, immutable data
503pub fn blockchain_stack(path: PathBuf) -> Result<MonitoredDedupStack> {
504    let base = StorageStackBuilder::new()
505        .with_config(BlockStoreConfig::production(path))
506        .with_cache(512) // 512MB for recent blocks
507        .with_bloom(10_000_000) // Large bloom for many blocks
508        .build_full()?;
509
510    let chunking_config = ChunkingConfig::default();
511    let dedup = DedupBlockStore::new(base, chunking_config);
512
513    Ok(MetricsBlockStore::new(dedup))
514}
515
516/// Create a machine learning model storage stack
517///
518/// Optimized for ML model storage:
519/// - Large cache (1GB) for frequently accessed models
520/// - Bloom filter for fast existence checks
521/// - Metrics tracking
522///
523/// Best for: ML model repositories, model versioning, training checkpoints
524pub fn ml_model_stack(path: PathBuf) -> Result<MonitoredFullStack> {
525    let base = StorageStackBuilder::new()
526        .with_config(BlockStoreConfig::production(path))
527        .with_cache(1024) // 1GB cache for models
528        .with_bloom(100_000) // Moderate bloom size
529        .build_full()?;
530
531    Ok(MetricsBlockStore::new(base))
532}
533
534/// Create a media streaming stack
535///
536/// Optimized for video/audio streaming:
537/// - Large cache (3GB) for hot content
538/// - TTL for session-based content (2 hours)
539/// - Bloom filter for catalog lookups
540///
541/// Best for: Video streaming services, audio platforms, media servers
542pub fn media_streaming_stack(path: PathBuf) -> Result<MetricsBlockStore<TtlBlockStore<FullStack>>> {
543    let base = StorageStackBuilder::new()
544        .with_config(BlockStoreConfig::production(path))
545        .with_cache(3072) // 3GB cache
546        .with_bloom(1_000_000)
547        .build_full()?;
548
549    let ttl_config = TtlConfig {
550        default_ttl: Duration::from_secs(7200), // 2 hours
551        auto_cleanup: true,
552        cleanup_interval: Duration::from_secs(300), // 5 minutes
553        max_tracked_blocks: 1_000_000,
554    };
555    let ttl_store = TtlBlockStore::new(base, ttl_config);
556
557    Ok(MetricsBlockStore::new(ttl_store))
558}
559
560/// Create a distributed file system stack
561///
562/// Optimized for distributed filesystems:
563/// - Compression for storage efficiency
564/// - Deduplication for redundant files
565/// - Large cache (1.5GB)
566/// - Large bloom filter
567///
568/// Best for: Distributed filesystems, cluster storage, shared filesystems
569#[cfg(feature = "compression")]
570pub fn distributed_fs_stack(path: PathBuf) -> Result<MonitoredUltimateStack> {
571    let base = StorageStackBuilder::new()
572        .with_config(BlockStoreConfig::production(path))
573        .with_cache(1536) // 1.5GB cache
574        .with_bloom(5_000_000)
575        .build_full()?;
576
577    let compression_config = CompressionConfig {
578        algorithm: CompressionAlgorithm::Lz4, // Fast compression for FS
579        level: 1,                             // Fast compression
580        threshold: 4096,                      // Only compress larger files
581        max_ratio: 0.9,
582    };
583    let compressed = CompressionBlockStore::new(base, compression_config);
584
585    let chunking_config = ChunkingConfig::default();
586    let dedup = DedupBlockStore::new(compressed, chunking_config);
587
588    Ok(MetricsBlockStore::new(dedup))
589}
590
591#[cfg(test)]
592mod tests {
593    use super::*;
594
595    #[test]
596    fn test_builder_simple() {
597        let _stack = StorageStackBuilder::new()
598            .with_path(PathBuf::from("/tmp/test-simple"))
599            .build_simple();
600        assert!(_stack.is_ok());
601    }
602
603    #[test]
604    fn test_builder_cached() {
605        let _stack = StorageStackBuilder::new()
606            .with_path(PathBuf::from("/tmp/test-cached"))
607            .with_cache(10)
608            .build_cached();
609        assert!(_stack.is_ok());
610    }
611
612    #[test]
613    fn test_builder_full() {
614        let _stack = StorageStackBuilder::new()
615            .with_path(PathBuf::from("/tmp/test-full"))
616            .with_cache(10)
617            .with_bloom(1000)
618            .build_full();
619        assert!(_stack.is_ok());
620    }
621
622    #[test]
623    fn test_development_stack() {
624        let _stack = development_stack();
625        assert!(_stack.is_ok());
626    }
627
628    #[test]
629    fn test_testing_stack() {
630        let _stack = testing_stack();
631        assert!(_stack.is_ok());
632    }
633}
ipfrs_storage/helpers.rs

ipfrs_storage/
helpers.rs
