ipfrs_storage/
lib.rs

1//! IPFRS Storage - Block storage and retrieval
2//!
3//! This crate provides the storage layer for IPFRS including:
4//! - Sled, ParityDB, and in-memory block stores
5//! - LRU and tiered caching
6//! - Bloom filter for fast existence checks
7//! - Streaming interface for large blocks
8//! - Content-defined chunking and deduplication
9//! - Transparent block compression (Zstd, Lz4, Snappy)
10//! - Pin management for preventing GC
11//! - Hot/cold tiering with access tracking
12//! - Garbage collection (mark-and-sweep)
13//! - CAR format export/import for backups
14//! - Encryption at rest (ChaCha20-Poly1305, AES-256-GCM)
15//! - Version Control System for differentiable storage (Git for Tensors)
16//! - RAFT consensus protocol for distributed storage
17//! - Network transport abstraction (TCP, QUIC with TLS) for multi-node RAFT clusters
18//! - Cluster coordinator with automatic failover and re-election
19//! - Multi-datacenter support with latency-aware routing and cross-datacenter replication
20//! - Eventual consistency with version vectors and conflict resolution
21//! - GraphQL query interface for flexible metadata querying
22//! - ARM profiler with NEON SIMD optimization and low-power tuning
23//! - Production-grade metrics and observability for monitoring (Prometheus, OpenTelemetry)
24//! - Circuit breaker pattern for fault-tolerant external service calls
25//! - Unified health check system for liveness and readiness monitoring
26//! - TTL support for automatic block expiration
27//! - Advanced retry logic with exponential backoff and jitter
28//! - Optimized S3 multipart uploads for large blocks
29//! - Rate limiting for controlling request rates to backends
30//! - Write coalescing for batching similar writes
31//! - Workload simulation for testing and benchmarking
32//! - Automatic configuration tuning based on workload patterns
33//! - Comprehensive profiling system with comparative analysis and regression detection
34//! - Storage pool manager for multi-backend routing with intelligent load balancing
35//! - Quota management system for per-tenant storage limits and bandwidth control
36//! - Lifecycle policies for automatic data tiering, archival, and expiration
37//! - Predictive prefetching with access pattern learning and adaptive depth control
38//! - Cost analytics and optimization for cloud storage (AWS, Azure, GCP)
39
40pub mod analyzer;
41pub mod arm_profiler;
42pub mod auto_tuner;
43pub mod batch;
44pub mod blockstore;
45pub mod bloom;
46pub mod cache;
47pub mod car;
48pub mod circuit_breaker;
49pub mod cluster;
50pub mod coalesce;
51#[cfg(feature = "compression")]
52pub mod compression;
53pub mod cost_analytics;
54pub mod datacenter;
55pub mod dedup;
56pub mod diagnostics;
57#[cfg(feature = "encryption")]
58pub mod encryption;
59pub mod eventual_consistency;
60pub mod exporters;
61#[cfg(feature = "gateway")]
62pub mod gateway;
63pub mod gc;
64pub mod gradient;
65#[cfg(feature = "graphql")]
66pub mod graphql;
67pub mod health;
68pub mod helpers;
69// pub mod incremental_backup;  // Module file not found - commented out
70pub mod lifecycle;
71pub mod memory;
72pub mod metrics;
73pub mod migration;
74#[cfg(feature = "mmap")]
75pub mod mmap;
76pub mod otel;
77pub mod paritydb;
78pub mod pinning;
79pub mod pool;
80pub mod prefetch;
81pub mod profiler;
82pub mod profiling;
83pub mod prometheus;
84pub mod query_optimizer;
85pub mod quota;
86pub mod raft;
87pub mod rate_limit;
88pub mod replication;
89pub mod retry;
90#[cfg(feature = "s3")]
91pub mod s3;
92pub mod safetensors;
93pub mod streaming;
94pub mod tiering;
95pub mod traits;
96pub mod transport;
97pub mod ttl;
98pub mod utils;
99pub mod vcs;
100pub mod workload;
101
102pub use analyzer::{
103    Category, Difficulty, OperationStats, OptimizationRecommendation, Priority, SizeDistribution,
104    StorageAnalysis, StorageAnalyzer, WorkloadCharacterization, WorkloadType,
105};
106pub use arm_profiler::{
107    hash_block, ArmFeatures, ArmPerfCounter, ArmPerfReport, LowPowerBatcher, PowerProfile,
108    PowerStats,
109};
110pub use auto_tuner::{
111    AutoTuner, AutoTunerConfig, TuningPresets, TuningRecommendation, TuningReport,
112};
113pub use batch::{batch_delete, batch_get, batch_has, batch_put, BatchConfig, BatchResult};
114pub use blockstore::{BlockStoreConfig, SledBlockStore};
115pub use bloom::{BloomBlockStore, BloomConfig, BloomFilter, BloomStats};
116pub use cache::{
117    BlockCache, CacheStats, CachedBlockStore, TieredBlockCache, TieredCacheStats,
118    TieredCachedBlockStore,
119};
120pub use car::{
121    export_to_car, import_from_car, CarHeader, CarReadStats, CarReader, CarWriteStats, CarWriter,
122};
123pub use circuit_breaker::{CircuitBreaker, CircuitState, CircuitStats};
124pub use cluster::{ClusterConfig, ClusterCoordinator, ClusterStats, NodeHealth, NodeInfo};
125pub use coalesce::{CoalesceConfig, CoalesceStats, CoalescingBlockStore};
126#[cfg(feature = "compression")]
127pub use compression::{
128    BlockCompressionStats, CompressionAlgorithm, CompressionBlockStore, CompressionConfig,
129};
130pub use cost_analytics::{
131    CloudProvider, CostAnalyzer, CostBreakdown, CostProjection, CostTier, TierCostModel,
132    TierOption, TierRecommendation,
133};
134pub use datacenter::{
135    CrossDcStats, Datacenter, DatacenterId, LatencyAwareSelector, MultiDatacenterCoordinator,
136    Region, ReplicationPolicy,
137};
138pub use dedup::{ChunkingConfig, DedupBlockStore, DedupStats};
139pub use diagnostics::{
140    BenchmarkComparison, DiagnosticsReport, HealthMetrics, PerformanceMetrics, StorageDiagnostics,
141};
142#[cfg(feature = "encryption")]
143pub use encryption::{Cipher, EncryptedBlockStore, EncryptionConfig, EncryptionKey};
144pub use eventual_consistency::{
145    ConflictResolution, ConsistencyLevel, EventualStore, EventualStoreStats, VersionVector,
146    VersionedValue,
147};
148pub use exporters::{BatchExporter, ExportFormat, MetricExporter};
149#[cfg(feature = "gateway")]
150pub use gateway::{GatewayBlockStore, GatewayConfig, HybridBlockStore};
151pub use gc::{
152    GarbageCollector, GcConfig, GcPolicy, GcResult, GcScheduler, GcStats, GcStatsSnapshot,
153};
154pub use gradient::{
155    CompressionStats, DeltaEncoder, GradientData, GradientStore, ProvenanceMetadata,
156};
157#[cfg(feature = "graphql")]
158pub use graphql::{
159    create_schema, BlockConnection, BlockFilter, BlockMetadata, BlockQuerySchema, BlockStats,
160    QueryRoot, SortField, SortOrder,
161};
162pub use health::{
163    AggregateHealthResult, DetailedHealthStatus, HealthCheck, HealthCheckResult, HealthChecker,
164    HealthStatus, SimpleHealthCheck,
165};
166#[cfg(feature = "encryption")]
167pub use helpers::encrypted_production_stack;
168pub use helpers::{
169    blockchain_stack, cache_stack, cdn_edge_stack, coalescing_memory_stack,
170    deduplicated_production_stack, development_stack, embedded_stack, ingestion_stack, iot_stack,
171    media_streaming_stack, memory_stack, ml_model_stack, monitored_production_stack,
172    production_stack, read_optimized_stack, resilient_stack, testing_stack, ttl_production_stack,
173    write_optimized_stack, StorageStackBuilder,
174};
175#[cfg(feature = "compression")]
176pub use helpers::{compressed_production_stack, ultimate_production_stack};
177#[cfg(feature = "compression")]
178pub use helpers::{distributed_fs_stack, scientific_archive_stack};
179// pub use incremental_backup::{BackupStats, BackupType, IncrementalBackup, Snapshot};  // Module not found
180pub use lifecycle::{
181    BlockMetadata as LifecycleBlockMetadata, LifecycleAction, LifecycleActionResult,
182    LifecycleCondition, LifecyclePolicyConfig, LifecyclePolicyManager, LifecycleRule,
183    LifecycleStatsSnapshot, StorageTier,
184};
185pub use memory::MemoryBlockStore;
186pub use metrics::{MetricsBlockStore, StorageMetrics};
187pub use migration::{
188    estimate_migration, migrate_storage, migrate_storage_batched, migrate_storage_verified,
189    migrate_storage_with_progress, validate_migration, MigrationConfig, MigrationEstimate,
190    MigrationStats, StorageMigrator,
191};
192#[cfg(feature = "mmap")]
193pub use mmap::{MmapBlockStore, MmapConfig};
194pub use otel::OtelBlockStore;
195pub use paritydb::{ParityDbBlockStore, ParityDbConfig, ParityDbPreset};
196pub use pinning::{PinInfo, PinManager, PinSet, PinStatsSnapshot, PinType};
197pub use pool::{BackendConfig, BackendId, PoolConfig, PoolStats, RoutingStrategy, StoragePool};
198pub use prefetch::{
199    AccessPattern, PredictivePrefetcher, PrefetchConfig, PrefetchPrediction, PrefetchStatsSnapshot,
200};
201pub use profiler::{
202    ComparativeProfiler, ComparisonReport, ProfileConfig, ProfileReport, RegressionDetector,
203    RegressionResult, StorageProfiler,
204};
205pub use profiling::{BatchProfiler, LatencyHistogram, PerformanceProfiler, ThroughputTracker};
206pub use prometheus::{PrometheusExporter, PrometheusExporterBuilder};
207pub use query_optimizer::{
208    OptimizerConfig, QueryLogEntry, QueryOptimizer, QueryPlan, QueryStrategy, Recommendation,
209    RecommendationCategory, RecommendationPriority,
210};
211pub use quota::{
212    QuotaBlockStore, QuotaConfig, QuotaManager, QuotaManagerConfig, QuotaReport, QuotaStatus,
213    QuotaUsageSnapshot, TenantId, ViolationType,
214};
215pub use raft::{
216    AppendEntriesRequest, AppendEntriesResponse, Command, LogEntry, LogIndex, NodeId, NodeState,
217    RaftConfig, RaftNode, RaftStats, RequestVoteRequest, RequestVoteResponse, Term,
218};
219pub use rate_limit::{RateLimitAlgorithm, RateLimitConfig, RateLimitStats, RateLimiter};
220pub use replication::{
221    ConflictStrategy, ReplicationManager, ReplicationState, Replicator, SyncResult, SyncStrategy,
222};
223pub use retry::{BackoffStrategy, JitterType, RetryPolicy, RetryStats, Retryable};
224#[cfg(feature = "s3")]
225pub use s3::{S3BlockStore, S3Config};
226pub use safetensors::{
227    ChunkConfig, ChunkedTensor, DType, ModelStats, SafetensorsHeader, SafetensorsManifest,
228    SafetensorsStore, TensorInfo,
229};
230pub use streaming::{
231    BlockReader, ByteRange, PartialBlock, StreamConfig, StreamingBlockStore, StreamingWriter,
232};
233pub use tiering::{AccessStats, AccessTracker, Tier, TierConfig, TierStatsSnapshot, TieredStore};
234pub use traits::BlockStore as BlockStoreTrait;
235#[cfg(feature = "quic")]
236pub use transport::QuicTransport;
237pub use transport::{
238    InMemoryTransport, Message as TransportMessage, TcpTransport, Transport, TransportConfig,
239};
240pub use ttl::{TtlBlockStore, TtlCleanupResult, TtlConfig, TtlStats};
241pub use utils::{
242    compute_cid, compute_total_size, create_block, create_blocks_batch, deduplicate_blocks,
243    estimate_compression_ratio, extract_cids, filter_blocks_by_size, find_duplicates,
244    generate_compressible_blocks, generate_compressible_data, generate_dedup_dataset,
245    generate_incompressible_data, generate_mixed_size_blocks, generate_pattern_blocks,
246    generate_random_block, generate_random_blocks, group_blocks_by_size, sample_blocks,
247    sort_blocks_by_size_asc, sort_blocks_by_size_desc, validate_block_integrity,
248    validate_blocks_batch, BlockStatistics,
249};
250pub use vcs::{
251    Author, Commit, CommitBuilder, MergeResult, MergeStrategy, Ref, RefType, VersionControl,
252};
253pub use workload::{
254    OperationMix, SizeDistribution as WorkloadSizeDistribution, WorkloadConfig, WorkloadPattern,
255    WorkloadPresets, WorkloadResult, WorkloadSimulator,
256};
ipfrs_storage/lib.rs

ipfrs_storage/
lib.rs
