leptos_sync_core/reliability/
mod.rs

1//! Production Reliability Module
2//!
3//! This module provides enterprise-grade reliability features including:
4//! - Advanced error recovery with retry policies
5//! - Data integrity verification and corruption detection
6//! - Comprehensive monitoring and health checks
7//! - Circuit breakers and fault tolerance
8//! - Backup and restore capabilities
9
10pub mod error_recovery;
11pub mod data_integrity;
12pub mod monitoring;
13pub mod health_checks;
14pub mod circuit_breaker;
15pub mod backup_restore;
16
17use std::time::{SystemTime, UNIX_EPOCH};
18
19// Re-export main components
20pub use error_recovery::{
21    ErrorRecovery, RetryPolicy, ExponentialBackoffConfig, CircuitBreakerPolicy,
22    RetryStrategy, RecoveryResult, RecoveryError
23};
24pub use data_integrity::{
25    DataIntegrity, ChecksumVerifier, VersionVerifier, CorruptionDetector,
26    IntegrityResult, IntegrityError, ChecksumAlgorithm
27};
28pub use monitoring::{
29    ReliabilityMonitor, MetricsCollector, AlertManager, HealthReporter,
30    MonitorConfig, AlertConfig
31};
32pub use health_checks::{
33    HealthChecker, HealthStatus, HealthCheck, SystemHealth,
34    HealthCheckResult, HealthError
35};
36pub use circuit_breaker::{
37    CircuitBreaker, CircuitState, BreakerConfig, BreakerError
38};
39pub use backup_restore::{
40    BackupManager, RestoreManager, BackupStrategy, RestoreStrategy,
41    BackupResult, RestoreResult, BackupError
42};
43
44/// Main reliability manager that coordinates all reliability features
45#[derive(Debug, Clone)]
46pub struct ReliabilityManager {
47    /// Error recovery system
48    pub error_recovery: ErrorRecovery,
49    /// Data integrity system
50    pub data_integrity: DataIntegrity,
51    /// Monitoring system
52    pub monitoring: ReliabilityMonitor,
53    /// Health checking system
54    pub health_checks: HealthChecker,
55    /// Circuit breaker system
56    pub circuit_breaker: CircuitBreaker,
57    /// Backup and restore system
58    pub backup_restore: BackupManager,
59}
60
61impl ReliabilityManager {
62    /// Create a new reliability manager with default configuration
63    pub fn new() -> Self {
64        Self {
65            error_recovery: ErrorRecovery::new(),
66            data_integrity: DataIntegrity::new(),
67            monitoring: ReliabilityMonitor::new(),
68            health_checks: HealthChecker::new(),
69            circuit_breaker: CircuitBreaker::new(),
70            backup_restore: BackupManager::new(),
71        }
72    }
73    
74    /// Create a reliability manager with custom configuration
75    pub fn with_config(config: ReliabilityConfig) -> Self {
76        Self {
77            error_recovery: ErrorRecovery::with_config(config.error_recovery),
78            data_integrity: DataIntegrity::with_config(config.data_integrity),
79            monitoring: ReliabilityMonitor::with_config(config.monitoring),
80            health_checks: HealthChecker::with_config(config.health_checks),
81            circuit_breaker: CircuitBreaker::with_config(config.circuit_breaker),
82            backup_restore: BackupManager::with_config(config.backup_restore),
83        }
84    }
85    
86    /// Initialize all reliability systems
87    pub async fn initialize(&mut self) -> Result<(), ReliabilityError> {
88        self.error_recovery.initialize().await?;
89        self.data_integrity.initialize().await?;
90        self.monitoring.initialize().await?;
91        self.health_checks.initialize().await?;
92        self.circuit_breaker.initialize().await?;
93        self.backup_restore.initialize().await?;
94        
95        Ok(())
96    }
97    
98    /// Shutdown all reliability systems
99    pub async fn shutdown(&mut self) -> Result<(), ReliabilityError> {
100        self.error_recovery.shutdown().await?;
101        self.data_integrity.shutdown().await?;
102        self.monitoring.shutdown().await?;
103        self.health_checks.shutdown().await?;
104        self.circuit_breaker.shutdown().await?;
105        self.backup_restore.shutdown().await?;
106        
107        Ok(())
108    }
109    
110    /// Get overall system health
111    pub async fn get_system_health(&self) -> Result<SystemHealth, ReliabilityError> {
112        let health_checks = self.health_checks.check_all().await?;
113        let circuit_breaker_status = self.circuit_breaker.get_state();
114        let monitoring_status = self.monitoring.get_status().await?;
115        
116        Ok(SystemHealth {
117            overall_status: self.determine_overall_status(&health_checks, circuit_breaker_status.await),
118            health_checks,
119            last_checked: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(),
120        })
121    }
122    
123    /// Determine overall system status based on component health
124    fn determine_overall_status(
125        &self,
126        health_checks: &[HealthCheckResult],
127        circuit_breaker_status: CircuitState,
128    ) -> HealthStatus {
129        // Check if any health checks are failing
130        let failing_checks = health_checks.iter().any(|check| !check.is_healthy);
131        
132        // Check circuit breaker status
133        let circuit_breaker_healthy = matches!(circuit_breaker_status, CircuitState::Closed);
134        
135        if failing_checks || !circuit_breaker_healthy {
136            HealthStatus::Unhealthy
137        } else {
138            HealthStatus::Healthy
139        }
140    }
141}
142
143/// Configuration for the reliability manager
144#[derive(Debug, Clone)]
145pub struct ReliabilityConfig {
146    /// Error recovery configuration
147    pub error_recovery: error_recovery::RecoveryConfig,
148    /// Data integrity configuration
149    pub data_integrity: data_integrity::IntegrityConfig,
150    /// Monitoring configuration
151    pub monitoring: monitoring::MonitorConfig,
152    /// Health checks configuration
153    pub health_checks: health_checks::HealthConfig,
154    /// Circuit breaker configuration
155    pub circuit_breaker: circuit_breaker::BreakerConfig,
156    /// Backup and restore configuration
157    pub backup_restore: backup_restore::BackupConfig,
158}
159
160impl Default for ReliabilityConfig {
161    fn default() -> Self {
162        Self {
163            error_recovery: error_recovery::RecoveryConfig::default(),
164            data_integrity: data_integrity::IntegrityConfig::default(),
165            monitoring: monitoring::MonitorConfig::default(),
166            health_checks: health_checks::HealthConfig::default(),
167            circuit_breaker: circuit_breaker::BreakerConfig::default(),
168            backup_restore: backup_restore::BackupConfig::default(),
169        }
170    }
171}
172
173/// Error types for reliability operations
174#[derive(Debug, Clone, PartialEq)]
175pub enum ReliabilityError {
176    /// Error recovery failed
177    ErrorRecovery(RecoveryError),
178    /// Data integrity check failed
179    DataIntegrity(IntegrityError),
180    /// Monitoring system error
181    Monitoring(String),
182    /// Health check failed
183    HealthCheck(HealthError),
184    /// Circuit breaker error
185    CircuitBreaker(BreakerError),
186    /// Backup/restore error
187    BackupRestore(BackupError),
188    /// Initialization failed
189    Initialization(String),
190    /// Shutdown failed
191    Shutdown(String),
192}
193
194impl std::fmt::Display for ReliabilityError {
195    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
196        match self {
197            ReliabilityError::ErrorRecovery(e) => write!(f, "Error recovery failed: {}", e),
198            ReliabilityError::DataIntegrity(e) => write!(f, "Data integrity failed: {}", e),
199            ReliabilityError::Monitoring(e) => write!(f, "Monitoring error: {}", e),
200            ReliabilityError::HealthCheck(e) => write!(f, "Health check failed: {}", e),
201            ReliabilityError::CircuitBreaker(e) => write!(f, "Circuit breaker error: {}", e),
202            ReliabilityError::BackupRestore(e) => write!(f, "Backup/restore error: {}", e),
203            ReliabilityError::Initialization(e) => write!(f, "Initialization failed: {}", e),
204            ReliabilityError::Shutdown(e) => write!(f, "Shutdown failed: {}", e),
205        }
206    }
207}
208
209impl std::error::Error for ReliabilityError {}
210
211// From implementations for error conversion
212impl From<RecoveryError> for ReliabilityError {
213    fn from(e: RecoveryError) -> Self {
214        ReliabilityError::ErrorRecovery(e)
215    }
216}
217
218impl From<IntegrityError> for ReliabilityError {
219    fn from(e: IntegrityError) -> Self {
220        ReliabilityError::DataIntegrity(e)
221    }
222}
223
224impl From<HealthError> for ReliabilityError {
225    fn from(e: HealthError) -> Self {
226        ReliabilityError::HealthCheck(e)
227    }
228}
229
230impl From<BreakerError> for ReliabilityError {
231    fn from(e: BreakerError) -> Self {
232        ReliabilityError::CircuitBreaker(e)
233    }
234}
235
236impl From<BackupError> for ReliabilityError {
237    fn from(e: BackupError) -> Self {
238        ReliabilityError::BackupRestore(e)
239    }
240}
241
242impl From<String> for ReliabilityError {
243    fn from(e: String) -> Self {
244        ReliabilityError::Monitoring(e)
245    }
246}
247
248#[cfg(test)]
249mod tests {
250    use super::*;
251    
252    #[tokio::test]
253    async fn test_reliability_manager_creation() {
254        let mut manager = ReliabilityManager::new();
255        
256        // Verify all components are created
257        assert!(manager.error_recovery.is_initialized());
258        assert!(manager.data_integrity.is_initialized());
259        assert!(manager.monitoring.is_initialized());
260        assert!(manager.health_checks.is_initialized());
261        assert!(manager.circuit_breaker.is_initialized());
262        assert!(manager.backup_restore.is_initialized());
263    }
264    
265    #[tokio::test]
266    async fn test_reliability_manager_with_config() {
267        let config = ReliabilityConfig::default();
268        let mut manager = ReliabilityManager::with_config(config);
269        
270        // Verify all components are created with config
271        assert!(manager.error_recovery.is_initialized());
272        assert!(manager.data_integrity.is_initialized());
273        assert!(manager.monitoring.is_initialized());
274        assert!(manager.health_checks.is_initialized());
275        assert!(manager.circuit_breaker.is_initialized());
276        assert!(manager.backup_restore.is_initialized());
277    }
278    
279    #[tokio::test]
280    async fn test_reliability_manager_initialization() {
281        let mut manager = ReliabilityManager::new();
282        
283        // Initialize all systems
284        let result = manager.initialize().await;
285        assert!(result.is_ok());
286        
287        // Verify all systems are initialized
288        assert!(manager.error_recovery.is_initialized());
289        assert!(manager.data_integrity.is_initialized());
290        assert!(manager.monitoring.is_initialized());
291        assert!(manager.health_checks.is_initialized());
292        assert!(manager.circuit_breaker.is_initialized());
293        assert!(manager.backup_restore.is_initialized());
294    }
295    
296    #[tokio::test]
297    async fn test_reliability_manager_shutdown() {
298        let mut manager = ReliabilityManager::new();
299        
300        // Initialize first
301        manager.initialize().await.unwrap();
302        
303        // Then shutdown
304        let result = manager.shutdown().await;
305        assert!(result.is_ok());
306    }
307    
308    #[tokio::test]
309    async fn test_system_health_check() {
310        let mut manager = ReliabilityManager::new();
311        manager.initialize().await.unwrap();
312        
313        let health = manager.get_system_health().await.unwrap();
314        
315        // Should be healthy by default
316        assert_eq!(health.overall_status, HealthStatus::Healthy);
317        assert!(!health.health_checks.is_empty());
318    }
319    
320    #[test]
321    fn test_reliability_config_default() {
322        let config = ReliabilityConfig::default();
323        
324        // Verify all configs are created
325        assert!(config.error_recovery.retry_policy.max_retries > 0);
326        assert!(config.data_integrity.checksum_config.algorithm == ChecksumAlgorithm::Sha256);
327        assert!(config.monitoring.metrics_config.max_metrics_per_name > 0);
328        assert!(config.health_checks.enable_health_checks);
329        assert!(config.circuit_breaker.failure_threshold > 0);
330        assert!(config.backup_restore.enable_backups);
331    }
332    
333    #[test]
334    fn test_reliability_error_display() {
335        let error = ReliabilityError::Initialization("Test error".to_string());
336        let error_string = format!("{}", error);
337        assert!(error_string.contains("Initialization failed"));
338        assert!(error_string.contains("Test error"));
339    }
340    
341    #[test]
342    fn test_reliability_error_from_conversions() {
343        // Test error conversion from RecoveryError
344        let recovery_error = RecoveryError::MaxRetriesExceeded;
345        let reliability_error: ReliabilityError = recovery_error.into();
346        assert!(matches!(reliability_error, ReliabilityError::ErrorRecovery(_)));
347        
348        // Test error conversion from IntegrityError
349        let integrity_error = IntegrityError::ChecksumMismatch;
350        let reliability_error: ReliabilityError = integrity_error.into();
351        assert!(matches!(reliability_error, ReliabilityError::DataIntegrity(_)));
352        
353        // Test error conversion from HealthError
354        let health_error = HealthError::NotInitialized;
355        let reliability_error: ReliabilityError = health_error.into();
356        assert!(matches!(reliability_error, ReliabilityError::HealthCheck(_)));
357        
358        // Test error conversion from BreakerError
359        let breaker_error = BreakerError::CircuitOpen;
360        let reliability_error: ReliabilityError = breaker_error.into();
361        assert!(matches!(reliability_error, ReliabilityError::CircuitBreaker(_)));
362        
363        // Test error conversion from BackupError
364        let backup_error = BackupError::BackupFailed("test".to_string());
365        let reliability_error: ReliabilityError = backup_error.into();
366        assert!(matches!(reliability_error, ReliabilityError::BackupRestore(_)));
367    }
368}
leptos_sync_core/reliability/mod.rs

leptos_sync_core/reliability/
mod.rs
