mecha10-core 0.8.1

//! Graceful Degradation and Automatic Recovery Policies
//!
//! This module provides sophisticated failure handling and recovery strategies
//! for robust distributed robot systems.
//!
//! # Features
//!
//! - Recovery policies (restart, exponential backoff, circuit breaker)
//! - Graceful degradation strategies
//! - Dependency-aware recovery
//! - Health-based decision making
//! - Recovery history and analytics
//! - Configurable recovery limits
//!
//! # Usage
//!
//! ```rust
//! use mecha10_core::prelude::*;
//!
//! #[tokio::main]
//! async fn main() {
//!     let policy = RecoveryPolicy::builder()
//!         .max_restarts(5)
//!         .backoff_strategy(BackoffStrategy::Exponential {
//!             initial_delay: Duration::from_secs(1),
//!             max_delay: Duration::from_secs(60),
//!             multiplier: 2.0,
//!         })
//!         .degradation_mode(DegradationMode::Disable)
//!         .build();
//!
//!     let manager = RecoveryManager::new();
//!     manager.set_policy("camera_node", policy).await;
//! }
//! ```

use crate::types::NodeState;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, VecDeque};
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime};
use tokio::sync::RwLock;

#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum BackoffStrategy {
    Immediate,

    Fixed {
        delay: Duration,
    },

    Exponential {
        initial_delay: Duration,
        max_delay: Duration,
        multiplier: f64,
    },

    Linear {
        initial_delay: Duration,
        increment: Duration,
        max_delay: Duration,
    },
}

impl Default for BackoffStrategy {
    fn default() -> Self {
        Self::Exponential {
            initial_delay: Duration::from_secs(1),
            max_delay: Duration::from_secs(60),
            multiplier: 2.0,
        }
    }
}

impl BackoffStrategy {
    /// Calculate delay for nth restart attempt
    pub fn calculate_delay(&self, attempt: u32) -> Duration {
        match self {
            Self::Immediate => Duration::from_secs(0),
            Self::Fixed { delay } => *delay,
            Self::Exponential {
                initial_delay,
                max_delay,
                multiplier,
            } => {
                let delay_secs = initial_delay.as_secs_f64() * multiplier.powi(attempt as i32);
                let clamped = delay_secs.min(max_delay.as_secs_f64());
                Duration::from_secs_f64(clamped)
            }
            Self::Linear {
                initial_delay,
                increment,
                max_delay,
            } => {
                let delay_secs = initial_delay.as_secs_f64() + (increment.as_secs_f64() * attempt as f64);
                let clamped = delay_secs.min(max_delay.as_secs_f64());
                Duration::from_secs_f64(clamped)
            }
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum DegradationMode {
    Disable,

    StopDependents,

    ShutdownSystem,

    ReducedMode,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum RecoveryAction {
    Restart,

    Stop,

    Degrade,

    Monitor,

    Escalate,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RecoveryPolicy {
    pub max_restarts: u32,

    pub restart_window: Duration,

    pub backoff_strategy: BackoffStrategy,

    pub degradation_mode: DegradationMode,

    pub critical: bool,

    pub health_timeout: Duration,

    pub auto_recover: bool,

    pub recovery_script: Option<String>,
}

impl Default for RecoveryPolicy {
    fn default() -> Self {
        Self {
            max_restarts: 3,
            restart_window: Duration::from_secs(300),
            backoff_strategy: BackoffStrategy::default(),
            degradation_mode: DegradationMode::Disable,
            critical: false,
            health_timeout: Duration::from_secs(30),
            auto_recover: true,
            recovery_script: None,
        }
    }
}

impl RecoveryPolicy {
    /// Create a new policy builder
    pub fn builder() -> RecoveryPolicyBuilder {
        RecoveryPolicyBuilder::default()
    }

    /// Check if we should attempt recovery
    pub fn should_recover(&self, history: &RecoveryHistory) -> bool {
        if !self.auto_recover {
            return false;
        }

        let window_start = Instant::now() - self.restart_window;
        let recent_restarts = history
            .attempts
            .iter()
            .filter(|attempt| attempt.timestamp >= window_start)
            .count();

        recent_restarts < self.max_restarts as usize
    }

    /// Calculate next recovery action
    pub fn next_action(&self, history: &RecoveryHistory) -> RecoveryAction {
        if !self.should_recover(history) {
            return match self.degradation_mode {
                DegradationMode::Disable => RecoveryAction::Stop,
                DegradationMode::StopDependents => RecoveryAction::Stop,
                DegradationMode::ShutdownSystem => RecoveryAction::Escalate,
                DegradationMode::ReducedMode => RecoveryAction::Degrade,
            };
        }

        RecoveryAction::Restart
    }

    /// Get delay before next restart attempt
    pub fn get_restart_delay(&self, attempt: u32) -> Duration {
        self.backoff_strategy.calculate_delay(attempt)
    }
}

#[derive(Default)]
pub struct RecoveryPolicyBuilder {
    policy: RecoveryPolicy,
}

impl RecoveryPolicyBuilder {
    crate::builder_setter!(assign, max_restarts, policy.max_restarts, u32);
    crate::builder_setter!(assign, restart_window, policy.restart_window, Duration);
    crate::builder_setter!(assign, backoff_strategy, policy.backoff_strategy, BackoffStrategy);
    crate::builder_setter!(assign, degradation_mode, policy.degradation_mode, DegradationMode);
    crate::builder_setter!(assign, critical, policy.critical, bool);
    crate::builder_setter!(assign, health_timeout, policy.health_timeout, Duration);
    crate::builder_setter!(assign, auto_recover, policy.auto_recover, bool);
    crate::builder_setter!(into_some, recovery_script, policy.recovery_script, impl Into<String>);

    pub fn build(self) -> RecoveryPolicy {
        self.policy
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RecoveryAttempt {
    #[serde(skip, default = "Instant::now")]
    pub timestamp: Instant,

    pub system_time: SystemTime,

    pub action: RecoveryAction,

    pub success: bool,

    pub error: Option<String>,

    pub duration: Duration,

    pub previous_state: NodeState,

    pub new_state: NodeState,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RecoveryHistory {
    pub node_id: String,

    pub attempts: VecDeque<RecoveryAttempt>,

    pub total_restarts: u64,

    pub total_failures: u64,

    #[serde(skip)]
    pub last_success: Option<Instant>,

    #[serde(skip)]
    pub last_failure: Option<Instant>,

    pub consecutive_failures: u32,

    pub mtbf: Option<Duration>,

    pub mttr: Option<Duration>,
}

impl RecoveryHistory {
    /// Create new recovery history
    pub fn new(node_id: String) -> Self {
        Self {
            node_id,
            attempts: VecDeque::with_capacity(100),
            total_restarts: 0,
            total_failures: 0,
            last_success: None,
            last_failure: None,
            consecutive_failures: 0,
            mtbf: None,
            mttr: None,
        }
    }

    /// Record a recovery attempt
    pub fn record_attempt(&mut self, attempt: RecoveryAttempt) {
        if attempt.success {
            self.total_restarts += 1;
            self.last_success = Some(attempt.timestamp);
            self.consecutive_failures = 0;
        } else {
            self.total_failures += 1;
            self.last_failure = Some(attempt.timestamp);
            self.consecutive_failures += 1;
        }

        if self.attempts.len() >= 100 {
            self.attempts.pop_front();
        }
        self.attempts.push_back(attempt);

        self.calculate_metrics();
    }

    /// Calculate MTBF and MTTR
    fn calculate_metrics(&mut self) {
        if self.attempts.len() < 2 {
            return;
        }

        let mut failure_intervals = Vec::new();
        let mut recovery_durations = Vec::new();

        let mut last_success_time: Option<Instant> = None;

        for attempt in &self.attempts {
            if attempt.success {
                if let Some(last_success) = last_success_time {
                    let interval = attempt.timestamp.duration_since(last_success);
                    failure_intervals.push(interval);
                }
                last_success_time = Some(attempt.timestamp);
                recovery_durations.push(attempt.duration);
            }
        }

        if !failure_intervals.is_empty() {
            let total: Duration = failure_intervals.iter().sum();
            self.mtbf = Some(total / failure_intervals.len() as u32);
        }

        if !recovery_durations.is_empty() {
            let total: Duration = recovery_durations.iter().sum();
            self.mttr = Some(total / recovery_durations.len() as u32);
        }
    }

    /// Get success rate (0.0 to 1.0)
    pub fn success_rate(&self) -> f64 {
        let total = self.total_restarts + self.total_failures;
        if total == 0 {
            return 1.0;
        }
        self.total_restarts as f64 / total as f64
    }
}

pub struct RecoveryManager {
    policies: Arc<RwLock<HashMap<String, RecoveryPolicy>>>,

    histories: Arc<RwLock<HashMap<String, RecoveryHistory>>>,

    default_policy: Arc<RwLock<RecoveryPolicy>>,

    degraded_nodes: Arc<RwLock<Vec<String>>>,
}

impl RecoveryManager {
    /// Create new recovery manager
    pub fn new() -> Self {
        Self {
            policies: Arc::new(RwLock::new(HashMap::new())),
            histories: Arc::new(RwLock::new(HashMap::new())),
            default_policy: Arc::new(RwLock::new(RecoveryPolicy::default())),
            degraded_nodes: Arc::new(RwLock::new(Vec::new())),
        }
    }

    /// Set recovery policy for a specific node
    pub async fn set_policy(&self, node_id: &str, policy: RecoveryPolicy) {
        let mut policies = self.policies.write().await;
        policies.insert(node_id.to_string(), policy);
    }

    /// Get recovery policy for a node
    pub async fn get_policy(&self, node_id: &str) -> RecoveryPolicy {
        let policies = self.policies.read().await;
        if let Some(policy) = policies.get(node_id) {
            policy.clone()
        } else {
            self.default_policy.read().await.clone()
        }
    }

    /// Get recovery history for a node
    pub async fn get_history(&self, node_id: &str) -> RecoveryHistory {
        let histories = self.histories.read().await;
        if let Some(history) = histories.get(node_id) {
            history.clone()
        } else {
            RecoveryHistory::new(node_id.to_string())
        }
    }

    /// Record a recovery attempt
    pub async fn record_attempt(&self, node_id: &str, attempt: RecoveryAttempt) {
        let mut histories = self.histories.write().await;
        let history = histories
            .entry(node_id.to_string())
            .or_insert_with(|| RecoveryHistory::new(node_id.to_string()));
        history.record_attempt(attempt);
    }

    /// Determine recovery action for a node
    pub async fn determine_action(&self, node_id: &str, current_state: NodeState) -> RecoveryAction {
        let policy = self.get_policy(node_id).await;
        let history = self.get_history(node_id).await;

        match current_state {
            NodeState::Crashed | NodeState::Unhealthy => policy.next_action(&history),
            _ => RecoveryAction::Monitor,
        }
    }

    /// Calculate delay before next restart
    pub async fn get_restart_delay(&self, node_id: &str) -> Duration {
        let policy = self.get_policy(node_id).await;
        let history = self.get_history(node_id).await;

        let window_start = Instant::now() - policy.restart_window;
        let recent_attempts = history.attempts.iter().filter(|a| a.timestamp >= window_start).count();

        policy.get_restart_delay(recent_attempts as u32)
    }

    /// Mark node as degraded
    pub async fn mark_degraded(&self, node_id: &str) {
        let mut degraded = self.degraded_nodes.write().await;
        if !degraded.contains(&node_id.to_string()) {
            degraded.push(node_id.to_string());
        }
    }

    /// Check if node is degraded
    pub async fn is_degraded(&self, node_id: &str) -> bool {
        let degraded = self.degraded_nodes.read().await;
        degraded.contains(&node_id.to_string())
    }

    /// Get all degraded nodes
    pub async fn list_degraded(&self) -> Vec<String> {
        self.degraded_nodes.read().await.clone()
    }

    /// Clear degraded status
    pub async fn clear_degraded(&self, node_id: &str) {
        let mut degraded = self.degraded_nodes.write().await;
        degraded.retain(|id| id != node_id);
    }

    /// Get recovery statistics
    pub async fn get_stats(&self) -> RecoveryStats {
        let histories = self.histories.read().await;

        let total_nodes = histories.len();
        let total_restarts: u64 = histories.values().map(|h| h.total_restarts).sum();
        let total_failures: u64 = histories.values().map(|h| h.total_failures).sum();

        let nodes_with_failures = histories.values().filter(|h| h.total_failures > 0).count();

        let degraded_count = self.degraded_nodes.read().await.len();

        RecoveryStats {
            total_nodes,
            total_restarts,
            total_failures,
            nodes_with_failures,
            degraded_nodes: degraded_count,
            avg_success_rate: if total_nodes > 0 {
                histories.values().map(|h| h.success_rate()).sum::<f64>() / total_nodes as f64
            } else {
                1.0
            },
        }
    }
}

impl Default for RecoveryManager {
    fn default() -> Self {
        Self::new()
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RecoveryStats {
    pub total_nodes: usize,
    pub total_restarts: u64,
    pub total_failures: u64,
    pub nodes_with_failures: usize,
    pub degraded_nodes: usize,
    pub avg_success_rate: f64,
}