leptos_sync_core/reliability/
mod.rs

1//! Production Reliability Module
2//!
3//! This module provides enterprise-grade reliability features including:
4//! - Advanced error recovery with retry policies
5//! - Data integrity verification and corruption detection
6//! - Comprehensive monitoring and health checks
7//! - Circuit breakers and fault tolerance
8//! - Backup and restore capabilities
9
10pub mod error_recovery;
11pub mod data_integrity;
12pub mod monitoring;
13pub mod health_checks;
14pub mod circuit_breaker;
15pub mod backup_restore;
16
17use std::time::{SystemTime, UNIX_EPOCH};
18
19// Re-export main components
20pub use error_recovery::{
21    ErrorRecovery, RetryPolicy, ExponentialBackoffConfig, CircuitBreakerPolicy,
22    RetryStrategy, RecoveryResult, RecoveryError, RecoveryConfig, RecoveryStats,
23    CircuitBreakerState, CircuitBreakerStatus, ErrorType, LinearBackoffConfig, FixedDelayConfig
24};
25pub use data_integrity::{
26    DataIntegrity, ChecksumVerifier, VersionVerifier, CorruptionDetector,
27    IntegrityResult, IntegrityError, ChecksumAlgorithm, DataMetadata, DataFormat, IntegrityStats
28};
29pub use monitoring::{
30    ReliabilityMonitor, MetricsCollector, AlertManager, HealthReporter,
31    MonitorConfig, AlertConfig, Metric, TimeRange, AggregationType, AggregatedMetric,
32    AlertRule, AlertCondition, ComparisonOperator, AlertSeverity, Alert, AlertStats,
33    MonitoringStats, PerformanceConfig, ResourceConfig, ExtendedMonitorConfig,
34};
35pub use health_checks::{
36    HealthChecker, HealthStatus, HealthCheck, SystemHealth,
37    HealthCheckResult, HealthError, HealthConfig
38};
39pub use circuit_breaker::{
40    CircuitBreaker, CircuitState, BreakerConfig, BreakerError
41};
42pub use backup_restore::{
43    BackupManager, RestoreManager, BackupStrategy, RestoreStrategy,
44    BackupResult, RestoreResult, BackupError
45};
46
47/// Main reliability manager that coordinates all reliability features
48#[derive(Debug, Clone)]
49pub struct ReliabilityManager {
50    /// Error recovery system
51    pub error_recovery: ErrorRecovery,
52    /// Data integrity system
53    pub data_integrity: DataIntegrity,
54    /// Monitoring system
55    pub monitoring: ReliabilityMonitor,
56    /// Health checking system
57    pub health_checks: HealthChecker,
58    /// Circuit breaker system
59    pub circuit_breaker: CircuitBreaker,
60    /// Backup and restore system
61    pub backup_restore: BackupManager,
62}
63
64impl ReliabilityManager {
65    /// Create a new reliability manager with default configuration
66    pub fn new() -> Self {
67        Self {
68            error_recovery: ErrorRecovery::new(),
69            data_integrity: DataIntegrity::new(),
70            monitoring: ReliabilityMonitor::new(),
71            health_checks: HealthChecker::new(),
72            circuit_breaker: CircuitBreaker::new(),
73            backup_restore: BackupManager::new(),
74        }
75    }
76    
77    /// Create a reliability manager with custom configuration
78    pub fn with_config(config: ReliabilityConfig) -> Self {
79        Self {
80            error_recovery: ErrorRecovery::with_config(config.error_recovery),
81            data_integrity: DataIntegrity::with_config(config.data_integrity),
82            monitoring: ReliabilityMonitor::with_config(config.monitoring),
83            health_checks: HealthChecker::with_config(config.health_checks),
84            circuit_breaker: CircuitBreaker::with_config(config.circuit_breaker),
85            backup_restore: BackupManager::with_config(config.backup_restore),
86        }
87    }
88    
89    /// Initialize all reliability systems
90    pub async fn initialize(&mut self) -> Result<(), ReliabilityError> {
91        self.error_recovery.initialize().await?;
92        self.data_integrity.initialize().await?;
93        self.monitoring.initialize().await?;
94        self.health_checks.initialize().await?;
95        self.circuit_breaker.initialize().await?;
96        self.backup_restore.initialize().await?;
97        
98        Ok(())
99    }
100    
101    /// Shutdown all reliability systems
102    pub async fn shutdown(&mut self) -> Result<(), ReliabilityError> {
103        self.error_recovery.shutdown().await?;
104        self.data_integrity.shutdown().await?;
105        self.monitoring.shutdown().await?;
106        self.health_checks.shutdown().await?;
107        self.circuit_breaker.shutdown().await?;
108        self.backup_restore.shutdown().await?;
109        
110        Ok(())
111    }
112    
113    /// Get overall system health
114    pub async fn get_system_health(&self) -> Result<SystemHealth, ReliabilityError> {
115        let health_checks = self.health_checks.check_all().await?;
116        let circuit_breaker_status = self.circuit_breaker.get_state();
117        let monitoring_stats = self.monitoring.get_stats().await?;
118        
119        Ok(SystemHealth {
120            overall_status: self.determine_overall_status(&health_checks, circuit_breaker_status.await),
121            health_checks,
122            last_checked: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(),
123        })
124    }
125    
126    /// Determine overall system status based on component health
127    fn determine_overall_status(
128        &self,
129        health_checks: &[HealthCheckResult],
130        circuit_breaker_status: CircuitState,
131    ) -> HealthStatus {
132        // Check if any health checks are failing
133        let failing_checks = health_checks.iter().any(|check| !check.is_healthy);
134        
135        // Check circuit breaker status
136        let circuit_breaker_healthy = matches!(circuit_breaker_status, CircuitState::Closed);
137        
138        if failing_checks || !circuit_breaker_healthy {
139            HealthStatus::Unhealthy
140        } else {
141            HealthStatus::Healthy
142        }
143    }
144}
145
146/// Configuration for the reliability manager
147#[derive(Debug, Clone)]
148pub struct ReliabilityConfig {
149    /// Error recovery configuration
150    pub error_recovery: RecoveryConfig,
151    /// Data integrity configuration
152    pub data_integrity: data_integrity::IntegrityConfig,
153    /// Monitoring configuration
154    pub monitoring: monitoring::MonitorConfig,
155    /// Health checks configuration
156    pub health_checks: health_checks::HealthConfig,
157    /// Circuit breaker configuration
158    pub circuit_breaker: circuit_breaker::BreakerConfig,
159    /// Backup and restore configuration
160    pub backup_restore: backup_restore::BackupConfig,
161}
162
163impl Default for ReliabilityConfig {
164    fn default() -> Self {
165        Self {
166            error_recovery: RecoveryConfig::default(),
167            data_integrity: data_integrity::IntegrityConfig::default(),
168            monitoring: monitoring::MonitorConfig::default(),
169            health_checks: health_checks::HealthConfig::default(),
170            circuit_breaker: circuit_breaker::BreakerConfig::default(),
171            backup_restore: backup_restore::BackupConfig::default(),
172        }
173    }
174}
175
176/// Error types for reliability operations
177#[derive(Debug, Clone, PartialEq)]
178pub enum ReliabilityError {
179    /// Error recovery failed
180    ErrorRecovery(RecoveryError),
181    /// Data integrity check failed
182    DataIntegrity(IntegrityError),
183    /// Monitoring system error
184    Monitoring(String),
185    /// Health check failed
186    HealthCheck(HealthError),
187    /// Circuit breaker error
188    CircuitBreaker(BreakerError),
189    /// Backup/restore error
190    BackupRestore(BackupError),
191    /// Initialization failed
192    Initialization(String),
193    /// Shutdown failed
194    Shutdown(String),
195}
196
197impl std::fmt::Display for ReliabilityError {
198    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
199        match self {
200            ReliabilityError::ErrorRecovery(e) => write!(f, "Error recovery failed: {}", e),
201            ReliabilityError::DataIntegrity(e) => write!(f, "Data integrity failed: {}", e),
202            ReliabilityError::Monitoring(e) => write!(f, "Monitoring error: {}", e),
203            ReliabilityError::HealthCheck(e) => write!(f, "Health check failed: {}", e),
204            ReliabilityError::CircuitBreaker(e) => write!(f, "Circuit breaker error: {}", e),
205            ReliabilityError::BackupRestore(e) => write!(f, "Backup/restore error: {}", e),
206            ReliabilityError::Initialization(e) => write!(f, "Initialization failed: {}", e),
207            ReliabilityError::Shutdown(e) => write!(f, "Shutdown failed: {}", e),
208        }
209    }
210}
211
212impl std::error::Error for ReliabilityError {}
213
214// From implementations for error conversion
215impl From<RecoveryError> for ReliabilityError {
216    fn from(e: RecoveryError) -> Self {
217        ReliabilityError::ErrorRecovery(e)
218    }
219}
220
221impl From<IntegrityError> for ReliabilityError {
222    fn from(e: IntegrityError) -> Self {
223        ReliabilityError::DataIntegrity(e)
224    }
225}
226
227impl From<HealthError> for ReliabilityError {
228    fn from(e: HealthError) -> Self {
229        ReliabilityError::HealthCheck(e)
230    }
231}
232
233impl From<BreakerError> for ReliabilityError {
234    fn from(e: BreakerError) -> Self {
235        ReliabilityError::CircuitBreaker(e)
236    }
237}
238
239impl From<BackupError> for ReliabilityError {
240    fn from(e: BackupError) -> Self {
241        ReliabilityError::BackupRestore(e)
242    }
243}
244
245impl From<String> for ReliabilityError {
246    fn from(e: String) -> Self {
247        ReliabilityError::Monitoring(e)
248    }
249}
250
251#[cfg(test)]
252mod tests {
253    use super::*;
254    
255    #[tokio::test]
256    async fn test_reliability_manager_creation() {
257        let mut manager = ReliabilityManager::new();
258        
259        // Initialize the manager
260        manager.initialize().await.unwrap();
261        
262        // Verify all components are created and initialized
263        assert!(manager.error_recovery.is_initialized());
264        assert!(manager.data_integrity.is_initialized());
265        // Note: monitoring and other components don't have is_initialized method
266        // They are initialized during the initialize() call
267    }
268    
269    #[tokio::test]
270    async fn test_reliability_manager_with_config() {
271        let config = ReliabilityConfig::default();
272        let mut manager = ReliabilityManager::with_config(config);
273        
274        // Initialize the manager
275        manager.initialize().await.unwrap();
276        
277        // Verify all components are created with config and initialized
278        assert!(manager.error_recovery.is_initialized());
279        assert!(manager.data_integrity.is_initialized());
280        // Note: monitoring and other components don't have is_initialized method
281        // They are initialized during the initialize() call
282    }
283    
284    #[tokio::test]
285    async fn test_reliability_manager_initialization() {
286        let mut manager = ReliabilityManager::new();
287        
288        // Initialize all systems
289        let result = manager.initialize().await;
290        assert!(result.is_ok());
291        
292        // Verify all systems are initialized
293        assert!(manager.error_recovery.is_initialized());
294        assert!(manager.data_integrity.is_initialized());
295        // Note: monitoring and other components don't have is_initialized method
296        // They are initialized during the initialize() call
297    }
298    
299    #[tokio::test]
300    async fn test_reliability_manager_shutdown() {
301        let mut manager = ReliabilityManager::new();
302        
303        // Initialize first
304        manager.initialize().await.unwrap();
305        
306        // Then shutdown
307        let result = manager.shutdown().await;
308        assert!(result.is_ok());
309    }
310    
311    #[tokio::test]
312    async fn test_system_health_check() {
313        let mut manager = ReliabilityManager::new();
314        manager.initialize().await.unwrap();
315        
316        let health = manager.get_system_health().await.unwrap();
317        
318        // Should be healthy by default
319        assert_eq!(health.overall_status, health_checks::HealthStatus::Healthy);
320        assert!(!health.health_checks.is_empty());
321    }
322    
323    #[test]
324    fn test_reliability_config_default() {
325        let config = ReliabilityConfig::default();
326        
327        // Verify all configs are created
328        assert!(config.error_recovery.retry_policy.max_retries > 0);
329        assert!(config.data_integrity.checksum_config.algorithm == ChecksumAlgorithm::Sha256);
330        assert!(config.monitoring.metrics_config.max_metrics_per_name > 0);
331        assert!(config.health_checks.enable_health_checks);
332        assert!(config.circuit_breaker.failure_threshold > 0);
333        assert!(config.backup_restore.enable_backups);
334    }
335    
336    #[test]
337    fn test_reliability_error_display() {
338        let error = ReliabilityError::Initialization("Test error".to_string());
339        let error_string = format!("{}", error);
340        assert!(error_string.contains("Initialization failed"));
341        assert!(error_string.contains("Test error"));
342    }
343    
344    #[test]
345    fn test_reliability_error_from_conversions() {
346        // Test error conversion from RecoveryError
347        let recovery_error = RecoveryError::MaxRetriesExceeded;
348        let reliability_error: ReliabilityError = recovery_error.into();
349        assert!(matches!(reliability_error, ReliabilityError::ErrorRecovery(_)));
350        
351        // Test error conversion from IntegrityError
352        let integrity_error = IntegrityError::ChecksumMismatch;
353        let reliability_error: ReliabilityError = integrity_error.into();
354        assert!(matches!(reliability_error, ReliabilityError::DataIntegrity(_)));
355        
356        // Test error conversion from HealthError
357        let health_error = HealthError::NotInitialized;
358        let reliability_error: ReliabilityError = health_error.into();
359        assert!(matches!(reliability_error, ReliabilityError::HealthCheck(_)));
360        
361        // Test error conversion from BreakerError
362        let breaker_error = BreakerError::CircuitOpen;
363        let reliability_error: ReliabilityError = breaker_error.into();
364        assert!(matches!(reliability_error, ReliabilityError::CircuitBreaker(_)));
365        
366        // Test error conversion from BackupError
367        let backup_error = BackupError::BackupFailed("test".to_string());
368        let reliability_error: ReliabilityError = backup_error.into();
369        assert!(matches!(reliability_error, ReliabilityError::BackupRestore(_)));
370    }
371}
leptos_sync_core/reliability/mod.rs

leptos_sync_core/reliability/
mod.rs
