leptos_sync_core/reliability/
mod.rs

1//! Production Reliability Module
2//!
3//! This module provides enterprise-grade reliability features including:
4//! - Advanced error recovery with retry policies
5//! - Data integrity verification and corruption detection
6//! - Comprehensive monitoring and health checks
7//! - Circuit breakers and fault tolerance
8//! - Backup and restore capabilities
9
10pub mod error_recovery;
11pub mod data_integrity;
12pub mod monitoring;
13pub mod health_checks;
14pub mod circuit_breaker;
15pub mod backup_restore;
16
17use std::time::{SystemTime, UNIX_EPOCH};
18
19// Re-export main components
20pub use error_recovery::{
21    ErrorRecovery, RetryPolicy, ExponentialBackoffConfig, CircuitBreakerPolicy,
22    RetryStrategy, RecoveryResult, RecoveryError
23};
24pub use data_integrity::{
25    DataIntegrity, ChecksumVerifier, VersionVerifier, CorruptionDetector,
26    IntegrityResult, IntegrityError, ChecksumAlgorithm
27};
28pub use monitoring::{
29    ReliabilityMonitor, MetricsCollector, AlertManager, HealthReporter,
30    MonitorConfig, AlertConfig
31};
32pub use health_checks::{
33    HealthChecker, HealthStatus, HealthCheck, SystemHealth,
34    HealthCheckResult, HealthError
35};
36pub use circuit_breaker::{
37    CircuitBreaker, CircuitState, BreakerConfig, BreakerError
38};
39pub use backup_restore::{
40    BackupManager, RestoreManager, BackupStrategy, RestoreStrategy,
41    BackupResult, RestoreResult, BackupError
42};
43
44/// Main reliability manager that coordinates all reliability features
45#[derive(Debug, Clone)]
46pub struct ReliabilityManager {
47    /// Error recovery system
48    pub error_recovery: ErrorRecovery,
49    /// Data integrity system
50    pub data_integrity: DataIntegrity,
51    /// Monitoring system
52    pub monitoring: ReliabilityMonitor,
53    /// Health checking system
54    pub health_checks: HealthChecker,
55    /// Circuit breaker system
56    pub circuit_breaker: CircuitBreaker,
57    /// Backup and restore system
58    pub backup_restore: BackupManager,
59}
60
61impl ReliabilityManager {
62    /// Create a new reliability manager with default configuration
63    pub fn new() -> Self {
64        Self {
65            error_recovery: ErrorRecovery::new(),
66            data_integrity: DataIntegrity::new(),
67            monitoring: ReliabilityMonitor::new(),
68            health_checks: HealthChecker::new(),
69            circuit_breaker: CircuitBreaker::new(),
70            backup_restore: BackupManager::new(),
71        }
72    }
73    
74    /// Create a reliability manager with custom configuration
75    pub fn with_config(config: ReliabilityConfig) -> Self {
76        Self {
77            error_recovery: ErrorRecovery::with_config(config.error_recovery),
78            data_integrity: DataIntegrity::with_config(config.data_integrity),
79            monitoring: ReliabilityMonitor::with_config(config.monitoring),
80            health_checks: HealthChecker::with_config(config.health_checks),
81            circuit_breaker: CircuitBreaker::with_config(config.circuit_breaker),
82            backup_restore: BackupManager::with_config(config.backup_restore),
83        }
84    }
85    
86    /// Initialize all reliability systems
87    pub async fn initialize(&mut self) -> Result<(), ReliabilityError> {
88        self.error_recovery.initialize().await?;
89        self.data_integrity.initialize().await?;
90        self.monitoring.initialize().await?;
91        self.health_checks.initialize().await?;
92        self.circuit_breaker.initialize().await?;
93        self.backup_restore.initialize().await?;
94        
95        Ok(())
96    }
97    
98    /// Shutdown all reliability systems
99    pub async fn shutdown(&mut self) -> Result<(), ReliabilityError> {
100        self.error_recovery.shutdown().await?;
101        self.data_integrity.shutdown().await?;
102        self.monitoring.shutdown().await?;
103        self.health_checks.shutdown().await?;
104        self.circuit_breaker.shutdown().await?;
105        self.backup_restore.shutdown().await?;
106        
107        Ok(())
108    }
109    
110    /// Get overall system health
111    pub async fn get_system_health(&self) -> Result<SystemHealth, ReliabilityError> {
112        let health_checks = self.health_checks.check_all().await?;
113        let circuit_breaker_status = self.circuit_breaker.get_state();
114        let monitoring_status = self.monitoring.get_status().await?;
115        
116        Ok(SystemHealth {
117            overall_status: self.determine_overall_status(&health_checks, circuit_breaker_status.await),
118            health_checks,
119            last_checked: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(),
120        })
121    }
122    
123    /// Determine overall system status based on component health
124    fn determine_overall_status(
125        &self,
126        health_checks: &[HealthCheckResult],
127        circuit_breaker_status: CircuitState,
128    ) -> HealthStatus {
129        // Check if any health checks are failing
130        let failing_checks = health_checks.iter().any(|check| !check.is_healthy);
131        
132        // Check circuit breaker status
133        let circuit_breaker_healthy = matches!(circuit_breaker_status, CircuitState::Closed);
134        
135        if failing_checks || !circuit_breaker_healthy {
136            HealthStatus::Unhealthy
137        } else {
138            HealthStatus::Healthy
139        }
140    }
141}
142
143/// Configuration for the reliability manager
144#[derive(Debug, Clone)]
145pub struct ReliabilityConfig {
146    /// Error recovery configuration
147    pub error_recovery: error_recovery::RecoveryConfig,
148    /// Data integrity configuration
149    pub data_integrity: data_integrity::IntegrityConfig,
150    /// Monitoring configuration
151    pub monitoring: monitoring::MonitorConfig,
152    /// Health checks configuration
153    pub health_checks: health_checks::HealthConfig,
154    /// Circuit breaker configuration
155    pub circuit_breaker: circuit_breaker::BreakerConfig,
156    /// Backup and restore configuration
157    pub backup_restore: backup_restore::BackupConfig,
158}
159
160impl Default for ReliabilityConfig {
161    fn default() -> Self {
162        Self {
163            error_recovery: error_recovery::RecoveryConfig::default(),
164            data_integrity: data_integrity::IntegrityConfig::default(),
165            monitoring: monitoring::MonitorConfig::default(),
166            health_checks: health_checks::HealthConfig::default(),
167            circuit_breaker: circuit_breaker::BreakerConfig::default(),
168            backup_restore: backup_restore::BackupConfig::default(),
169        }
170    }
171}
172
173/// Error types for reliability operations
174#[derive(Debug, Clone, PartialEq)]
175pub enum ReliabilityError {
176    /// Error recovery failed
177    ErrorRecovery(RecoveryError),
178    /// Data integrity check failed
179    DataIntegrity(IntegrityError),
180    /// Monitoring system error
181    Monitoring(String),
182    /// Health check failed
183    HealthCheck(HealthError),
184    /// Circuit breaker error
185    CircuitBreaker(BreakerError),
186    /// Backup/restore error
187    BackupRestore(BackupError),
188    /// Initialization failed
189    Initialization(String),
190    /// Shutdown failed
191    Shutdown(String),
192}
193
194impl std::fmt::Display for ReliabilityError {
195    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
196        match self {
197            ReliabilityError::ErrorRecovery(e) => write!(f, "Error recovery failed: {}", e),
198            ReliabilityError::DataIntegrity(e) => write!(f, "Data integrity failed: {}", e),
199            ReliabilityError::Monitoring(e) => write!(f, "Monitoring error: {}", e),
200            ReliabilityError::HealthCheck(e) => write!(f, "Health check failed: {}", e),
201            ReliabilityError::CircuitBreaker(e) => write!(f, "Circuit breaker error: {}", e),
202            ReliabilityError::BackupRestore(e) => write!(f, "Backup/restore error: {}", e),
203            ReliabilityError::Initialization(e) => write!(f, "Initialization failed: {}", e),
204            ReliabilityError::Shutdown(e) => write!(f, "Shutdown failed: {}", e),
205        }
206    }
207}
208
209impl std::error::Error for ReliabilityError {}
210
211// From implementations for error conversion
212impl From<RecoveryError> for ReliabilityError {
213    fn from(e: RecoveryError) -> Self {
214        ReliabilityError::ErrorRecovery(e)
215    }
216}
217
218impl From<IntegrityError> for ReliabilityError {
219    fn from(e: IntegrityError) -> Self {
220        ReliabilityError::DataIntegrity(e)
221    }
222}
223
224impl From<HealthError> for ReliabilityError {
225    fn from(e: HealthError) -> Self {
226        ReliabilityError::HealthCheck(e)
227    }
228}
229
230impl From<BreakerError> for ReliabilityError {
231    fn from(e: BreakerError) -> Self {
232        ReliabilityError::CircuitBreaker(e)
233    }
234}
235
236impl From<BackupError> for ReliabilityError {
237    fn from(e: BackupError) -> Self {
238        ReliabilityError::BackupRestore(e)
239    }
240}
241
242impl From<String> for ReliabilityError {
243    fn from(e: String) -> Self {
244        ReliabilityError::Monitoring(e)
245    }
246}
247
248#[cfg(test)]
249mod tests {
250    use super::*;
251    
252    #[tokio::test]
253    async fn test_reliability_manager_creation() {
254        let mut manager = ReliabilityManager::new();
255        
256        // Initialize the manager
257        manager.initialize().await.unwrap();
258        
259        // Verify all components are created and initialized
260        assert!(manager.error_recovery.is_initialized());
261        assert!(manager.data_integrity.is_initialized());
262        assert!(manager.monitoring.is_initialized());
263        assert!(manager.health_checks.is_initialized());
264        assert!(manager.circuit_breaker.is_initialized());
265        assert!(manager.backup_restore.is_initialized());
266    }
267    
268    #[tokio::test]
269    async fn test_reliability_manager_with_config() {
270        let config = ReliabilityConfig::default();
271        let mut manager = ReliabilityManager::with_config(config);
272        
273        // Initialize the manager
274        manager.initialize().await.unwrap();
275        
276        // Verify all components are created with config and initialized
277        assert!(manager.error_recovery.is_initialized());
278        assert!(manager.data_integrity.is_initialized());
279        assert!(manager.monitoring.is_initialized());
280        assert!(manager.health_checks.is_initialized());
281        assert!(manager.circuit_breaker.is_initialized());
282        assert!(manager.backup_restore.is_initialized());
283    }
284    
285    #[tokio::test]
286    async fn test_reliability_manager_initialization() {
287        let mut manager = ReliabilityManager::new();
288        
289        // Initialize all systems
290        let result = manager.initialize().await;
291        assert!(result.is_ok());
292        
293        // Verify all systems are initialized
294        assert!(manager.error_recovery.is_initialized());
295        assert!(manager.data_integrity.is_initialized());
296        assert!(manager.monitoring.is_initialized());
297        assert!(manager.health_checks.is_initialized());
298        assert!(manager.circuit_breaker.is_initialized());
299        assert!(manager.backup_restore.is_initialized());
300    }
301    
302    #[tokio::test]
303    async fn test_reliability_manager_shutdown() {
304        let mut manager = ReliabilityManager::new();
305        
306        // Initialize first
307        manager.initialize().await.unwrap();
308        
309        // Then shutdown
310        let result = manager.shutdown().await;
311        assert!(result.is_ok());
312    }
313    
314    #[tokio::test]
315    async fn test_system_health_check() {
316        let mut manager = ReliabilityManager::new();
317        manager.initialize().await.unwrap();
318        
319        let health = manager.get_system_health().await.unwrap();
320        
321        // Should be healthy by default
322        assert_eq!(health.overall_status, HealthStatus::Healthy);
323        assert!(!health.health_checks.is_empty());
324    }
325    
326    #[test]
327    fn test_reliability_config_default() {
328        let config = ReliabilityConfig::default();
329        
330        // Verify all configs are created
331        assert!(config.error_recovery.retry_policy.max_retries > 0);
332        assert!(config.data_integrity.checksum_config.algorithm == ChecksumAlgorithm::Sha256);
333        assert!(config.monitoring.metrics_config.max_metrics_per_name > 0);
334        assert!(config.health_checks.enable_health_checks);
335        assert!(config.circuit_breaker.failure_threshold > 0);
336        assert!(config.backup_restore.enable_backups);
337    }
338    
339    #[test]
340    fn test_reliability_error_display() {
341        let error = ReliabilityError::Initialization("Test error".to_string());
342        let error_string = format!("{}", error);
343        assert!(error_string.contains("Initialization failed"));
344        assert!(error_string.contains("Test error"));
345    }
346    
347    #[test]
348    fn test_reliability_error_from_conversions() {
349        // Test error conversion from RecoveryError
350        let recovery_error = RecoveryError::MaxRetriesExceeded;
351        let reliability_error: ReliabilityError = recovery_error.into();
352        assert!(matches!(reliability_error, ReliabilityError::ErrorRecovery(_)));
353        
354        // Test error conversion from IntegrityError
355        let integrity_error = IntegrityError::ChecksumMismatch;
356        let reliability_error: ReliabilityError = integrity_error.into();
357        assert!(matches!(reliability_error, ReliabilityError::DataIntegrity(_)));
358        
359        // Test error conversion from HealthError
360        let health_error = HealthError::NotInitialized;
361        let reliability_error: ReliabilityError = health_error.into();
362        assert!(matches!(reliability_error, ReliabilityError::HealthCheck(_)));
363        
364        // Test error conversion from BreakerError
365        let breaker_error = BreakerError::CircuitOpen;
366        let reliability_error: ReliabilityError = breaker_error.into();
367        assert!(matches!(reliability_error, ReliabilityError::CircuitBreaker(_)));
368        
369        // Test error conversion from BackupError
370        let backup_error = BackupError::BackupFailed("test".to_string());
371        let reliability_error: ReliabilityError = backup_error.into();
372        assert!(matches!(reliability_error, ReliabilityError::BackupRestore(_)));
373    }
374}
leptos_sync_core/reliability/mod.rs

leptos_sync_core/reliability/
mod.rs
