zenith-runtime 0.1.0

//! Supervisor 层级体系
//!
//! 本模块实现 Zenith 运行时的三级 Supervisor 层级结构，
//! 借鉴 Erlang/OTP 的 Supervisor 设计，结合无堆分配的热路径优化：
//!
//! ## 层级结构
//! ```text
//! +--------------------------------------------------+
//! | NodeSupervisor (顶层)                            |
//! |   · 管理所有 DomainSupervisor                    |
//! |   · 处理节点级故障                               |
//! +--------------------------------------------------+
//!                    │
//!         ┌──────────┼──────────┐
//!         ▼          ▼          ▼
//! +-------------+ +-------------+ +-------------+
//! | DomainSup.  | | DomainSup.  | | DomainSup.  |
//! |  (域 A)     | |  (域 B)     | |  (域 C)     |
//! +-------------+ +-------------+ +-------------+
//!    │              │              │
//!    ▼              ▼              ▼
//! +----------+  +----------+  +----------+
//! |QueueSup. |  |QueueSup. |  |QueueSup. |
//! |(队列 0)  |  |(队列 1)  |  |(队列 N)  |
//! +----------+  +----------+  +----------+
//! ```
//!
//! ## 核心特性
//! - `SupervisorState` 状态机：Running / Stopped / Degraded / Failed
//! - 子项管理：`spawn_child`、`stop_child`
//! - 健康检查：`health_check` 返回健康报告
//! - 自动重启：带指数退避的 `restart` 策略
//! - 故障隔离：单点故障不扩散至上级
//! - 热路径零堆分配：使用 const-generics 定长数组预分配

use core::time::Duration;
use std::fmt;

use thiserror::Error;

/// Supervisor 错误
#[derive(Debug, Error, PartialEq, Eq)]
pub enum SupervisorError {
    /// 子项容量已满
    #[error("child capacity exceeded: max={0}")]
    CapacityExceeded(usize),

    /// 子项未找到
    #[error("child not found: id={0}")]
    ChildNotFound(u64),

    /// 非法状态转换
    #[error("invalid state transition: from={from:?} to={to:?}")]
    InvalidTransition {
        /// 当前状态
        from: SupervisorState,
        /// 目标状态
        to: SupervisorState,
    },

    /// 已处于目标状态
    #[error("already in state: {0:?}")]
    AlreadyInState(SupervisorState),

    /// 重启过多
    #[error("restart threshold exceeded: count={count}, max={max}")]
    RestartThresholdExceeded {
        /// 当前重启次数
        count: u32,
        /// 允许的最大次数
        max: u32,
    },

    /// 关闭中
    #[error("supervisor is shutting down")]
    ShuttingDown,
}

/// Supervisor 运行状态
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum SupervisorState {
    /// 正常运行
    Running,
    /// 已停止（正常终止）
    Stopped,
    /// 降级运行（部分子项故障）
    Degraded,
    /// 完全故障
    Failed,
}

impl SupervisorState {
    /// 判断是否允许转换到目标状态
    ///
    /// # Arguments
    /// * `to` - 目标状态
    ///
    /// # Returns
    /// * `true` 允许转换
    pub fn can_transition_to(self, to: SupervisorState) -> bool {
        use SupervisorState::*;
        matches!(
            (self, to),
            (Running, Stopped)
                | (Running, Degraded)
                | (Running, Failed)
                | (Degraded, Running)
                | (Degraded, Stopped)
                | (Degraded, Failed)
                | (Failed, Running)
                | (Stopped, Running)
        )
    }
}

impl fmt::Display for SupervisorState {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            SupervisorState::Running => write!(f, "Running"),
            SupervisorState::Stopped => write!(f, "Stopped"),
            SupervisorState::Degraded => write!(f, "Degraded"),
            SupervisorState::Failed => write!(f, "Failed"),
        }
    }
}

/// 子项类型标识
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ChildKind {
    /// 域 Supervisor
    Domain,
    /// 队列 Supervisor
    Queue,
    /// Worker（叶节点）
    Worker,
}

/// 重启策略
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RestartStrategy {
    /// 子项永久（挂了就重启）
    Permanent,
    /// 仅异常退出时重启
    Transient,
    /// 永不重启
    Temporary,
}

/// 子项退出原因（重启决策入参）
///
/// [`RestartStrategy::Transient`] 据此区分语义：仅 [`ExitReason::Abnormal`] 时重启，
/// [`ExitReason::Normal`] 时停止并移除子项；`Permanent`/`Temporary` 不区分退出原因。
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ExitReason {
    /// 正常退出（任务完成、主动停止）
    Normal,
    /// 异常退出（故障、panic、数据面错误）
    Abnormal,
}

/// 退避策略
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct BackoffConfig {
    /// 初始退避时长
    pub initial: Duration,
    /// 最大退避时长
    pub max: Duration,
    /// 退避倍数
    pub multiplier: u32,
    /// 时间窗口内最大重启次数
    pub max_restarts: u32,
}

impl Default for BackoffConfig {
    fn default() -> Self {
        Self {
            initial: Duration::from_millis(10),
            max: Duration::from_secs(5),
            multiplier: 2,
            max_restarts: 5,
        }
    }
}

/// 子项条目（固定大小槽位）
///
/// 热路径上的子项元数据，使用 Copy 类型以避免堆分配。
#[derive(Debug, Clone, Copy)]
pub struct ChildEntry {
    /// 子项 ID
    pub id: u64,
    /// 子项类型
    pub kind: ChildKind,
    /// 子项状态
    pub state: SupervisorState,
    /// 重启次数
    pub restart_count: u32,
    /// 最近一次健康检查通过
    pub last_healthy: bool,
    /// 已占用
    pub(crate) occupied: bool,
}

impl ChildEntry {
    /// 新的空条目
    const fn empty() -> Self {
        Self {
            id: 0,
            kind: ChildKind::Worker,
            state: SupervisorState::Stopped,
            restart_count: 0,
            last_healthy: true,
            occupied: false,
        }
    }

    /// 是否存活
    pub fn is_alive(&self) -> bool {
        self.occupied
            && matches!(self.state, SupervisorState::Running | SupervisorState::Degraded)
    }
}

/// 健康报告
#[derive(Debug, Clone, Copy)]
pub struct HealthReport {
    /// 总子项数
    pub total: usize,
    /// 健康子项数
    pub healthy: usize,
    /// 降级子项数
    pub degraded: usize,
    /// 故障子项数
    pub failed: usize,
}

impl HealthReport {
    /// 是否完全健康
    pub fn is_healthy(&self) -> bool {
        self.failed == 0 && self.degraded == 0
    }
}

/// 定长子项集合
///
/// 使用 const-generics 在栈上预分配子项槽位，避免堆分配。
#[derive(Debug)]
pub struct ChildSet<const N: usize> {
    slots: [ChildEntry; N],
    count: usize,
}

impl<const N: usize> Default for ChildSet<N> {
    fn default() -> Self {
        Self::new()
    }
}

impl<const N: usize> ChildSet<N> {
    /// 创建空集合
    pub const fn new() -> Self {
        Self {
            slots: [const { ChildEntry::empty() }; N],
            count: 0,
        }
    }

    /// 当前子项数
    #[inline]
    pub fn len(&self) -> usize {
        self.count
    }

    /// 是否为空
    #[inline]
    pub fn is_empty(&self) -> bool {
        self.count == 0
    }

    /// 最大容量
    #[inline]
    pub const fn capacity(&self) -> usize {
        N
    }

    /// 迭代所有子项引用
    pub fn iter(&self) -> impl Iterator<Item = &ChildEntry> {
        self.slots.iter().filter(|s| s.occupied)
    }

    /// 迭代可变引用
    pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut ChildEntry> {
        self.slots.iter_mut().filter(|s| s.occupied)
    }

    /// 按 ID 查找
    pub fn get(&self, id: u64) -> Option<&ChildEntry> {
        self.slots.iter().find(|s| s.occupied && s.id == id)
    }

    /// 按 ID 查找（可变）
    pub fn get_mut(&mut self, id: u64) -> Option<&mut ChildEntry> {
        self.slots.iter_mut().find(|s| s.occupied && s.id == id)
    }

    /// 添加子项
    pub fn spawn(&mut self, entry: ChildEntry) -> Result<(), SupervisorError> {
        if self.count >= N {
            return Err(SupervisorError::CapacityExceeded(N));
        }
        if self.slots.iter().any(|s| s.occupied && s.id == entry.id) {
            return Err(SupervisorError::AlreadyInState(SupervisorState::Running));
        }
        let slot = self
            .slots
            .iter_mut()
            .find(|s| !s.occupied)
            .ok_or(SupervisorError::CapacityExceeded(N))?;
        *slot = entry;
        self.count += 1;
        Ok(())
    }

    /// 停止子项（标记为空闲）
    pub fn stop(&mut self, id: u64) -> Result<(), SupervisorError> {
        let slot = self
            .slots
            .iter_mut()
            .find(|s| s.occupied && s.id == id)
            .ok_or(SupervisorError::ChildNotFound(id))?;
        slot.state = SupervisorState::Stopped;
        slot.occupied = false;
        slot.restart_count = 0;
        self.count -= 1;
        Ok(())
    }

    /// 汇总健康报告
    pub fn health_report(&self) -> HealthReport {
        let mut report = HealthReport {
            total: self.count,
            healthy: 0,
            degraded: 0,
            failed: 0,
        };
        for slot in self.slots.iter().filter(|s| s.occupied) {
            match slot.state {
                SupervisorState::Running if slot.last_healthy => report.healthy += 1,
                SupervisorState::Running => report.degraded += 1,
                SupervisorState::Degraded => report.degraded += 1,
                SupervisorState::Failed => report.failed += 1,
                SupervisorState::Stopped => {}
            }
        }
        report
    }

    /// 所有子项是否健康
    #[inline]
    pub fn all_healthy(&self) -> bool {
        self.slots
            .iter()
            .filter(|s| s.occupied)
            .all(|s| s.state == SupervisorState::Running && s.last_healthy)
    }
}

// ---------------------------------------------------------------------------
// 通用 Supervisor 逻辑
// ---------------------------------------------------------------------------

/// 通用 Supervisor 配置
#[derive(Debug, Clone, Copy)]
pub struct SupervisorConfig {
    /// 初始状态
    pub initial_state: SupervisorState,
    /// 重启策略
    pub restart_strategy: RestartStrategy,
    /// 退避配置
    pub backoff: BackoffConfig,
}

impl Default for SupervisorConfig {
    fn default() -> Self {
        Self {
            initial_state: SupervisorState::Running,
            restart_strategy: RestartStrategy::Permanent,
            backoff: BackoffConfig::default(),
        }
    }
}

/// 通用 Supervisor 核心
///
/// 使用 const-generics 支持不同容量的子项集合。
/// 所有热路径方法均为 `#[inline]` 且无堆分配。
#[derive(Debug)]
pub struct SupervisorCore<const N: usize> {
    /// 当前状态
    state: SupervisorState,
    /// 配置
    config: SupervisorConfig,
    /// 子项集合
    children: ChildSet<N>,
    /// 累计重启次数（用于全局熔断）
    total_restarts: u64,
    /// 熔断阈值
    fuse_threshold: u64,
}

impl<const N: usize> SupervisorCore<N> {
    /// 创建新的 Supervisor
    pub fn new(config: SupervisorConfig) -> Self {
        Self {
            state: config.initial_state,
            config,
            children: ChildSet::new(),
            total_restarts: 0,
            fuse_threshold: 100,
        }
    }

    /// 当前状态
    #[inline]
    pub fn state(&self) -> SupervisorState {
        self.state
    }

    /// 子项数量
    #[inline]
    pub fn child_count(&self) -> usize {
        self.children.len()
    }

    /// 设置熔断阈值
    pub fn with_fuse_threshold(mut self, threshold: u64) -> Self {
        self.fuse_threshold = threshold;
        self
    }

    /// 获取子项引用
    #[inline]
    pub fn get_child(&self, id: u64) -> Option<&ChildEntry> {
        self.children.get(id)
    }

    /// 生成下一个退避时长（指数退避 + 全抖动）
    ///
    /// 统一委托 `zenith_foundation::backoff::exponential_backoff_with_jitter`：
    /// `initial * multiplier^restart_count` 为确定性上界（封顶 `max`），
    /// 再取 `[0, 上界]` 全抖动，打散多子项同时重启的惊群。
    fn next_backoff(&self, restart_count: u32) -> Duration {
        let base_ns = self.config.backoff.initial.as_nanos() as u64;
        let mult = u64::from(self.config.backoff.multiplier);
        let max_ns = self.config.backoff.max.as_nanos() as u64;
        Duration::from_nanos(zenith_foundation::backoff::exponential_backoff_with_jitter(
            base_ns,
            mult,
            restart_count,
            max_ns,
        ))
    }

    /// 启动子项
    pub fn spawn_child(
        &mut self,
        id: u64,
        kind: ChildKind,
    ) -> Result<(), SupervisorError> {
        if self.state == SupervisorState::Failed {
            return Err(SupervisorError::ShuttingDown);
        }
        if self.state == SupervisorState::Stopped {
            return Err(SupervisorError::ShuttingDown);
        }
        let entry = ChildEntry {
            id,
            kind,
            state: SupervisorState::Running,
            restart_count: 0,
            last_healthy: true,
            occupied: true,
        };
        self.children.spawn(entry)?;
        Ok(())
    }

    /// 停止子项
    pub fn stop_child(&mut self, id: u64) -> Result<(), SupervisorError> {
        self.children.stop(id)?;
        Ok(())
    }

    /// 对指定子项执行健康检查
    pub fn health_check_child(&mut self, id: u64, healthy: bool) -> Result<(), SupervisorError> {
        {
            let child = self
                .children
                .get_mut(id)
                .ok_or(SupervisorError::ChildNotFound(id))?;
            child.last_healthy = healthy;
            if healthy {
                // 健康检查恢复：若之前为 Degraded 则回到 Running
                if child.state == SupervisorState::Degraded {
                    child.state = SupervisorState::Running;
                }
            } else if child.state == SupervisorState::Running {
                child.state = SupervisorState::Degraded;
            }
        }
        self.update_self_state();
        Ok(())
    }

    /// 整体健康检查
    pub fn health_check(&mut self) -> HealthReport {
        self.update_self_state();
        self.children.health_report()
    }

    /// 根据子项状态更新自身状态
    fn update_self_state(&mut self) {
        if self.state == SupervisorState::Stopped || self.state == SupervisorState::Failed {
            return;
        }
        let report = self.children.health_report();
        if report.failed == report.total && report.total > 0 {
            let _ = self.transition(SupervisorState::Failed);
        } else if report.failed > 0 || report.degraded > 0 {
            let _ = self.transition(SupervisorState::Degraded);
        } else if report.total == 0 || report.healthy == report.total {
            let _ = self.transition(SupervisorState::Running);
        }
    }

    /// 转换状态
    fn transition(&mut self, to: SupervisorState) -> Result<(), SupervisorError> {
        if self.state == to {
            return Ok(());
        }
        if !self.state.can_transition_to(to) {
            return Err(SupervisorError::InvalidTransition { from: self.state, to });
        }
        self.state = to;
        Ok(())
    }

    /// 重启子项（含退避），按退出原因落实策略语义
    ///
    /// - `Permanent`：无论退出原因一律重启
    /// - `Transient`：仅 [`ExitReason::Abnormal`] 重启；[`ExitReason::Normal`] 停止并移除
    /// - `Temporary`：不重启，停止并移除
    ///
    /// 返回退避时长；`Duration::ZERO` 表示不重启（子项已被移除）。
    pub fn restart(&mut self, id: u64, reason: ExitReason) -> Result<Duration, SupervisorError> {
        if self.state == SupervisorState::Failed {
            return Err(SupervisorError::ShuttingDown);
        }
        // 子项存在性已由后续 children.stop(id) / get_mut(id) 校验，无需在此借用
        match self.config.restart_strategy {
            RestartStrategy::Temporary => {
                // 永不重启：停止并移除（先调 stop 递减 count，再清理字段）
                self.children.stop(id)?;
                return Ok(Duration::ZERO);
            }
            RestartStrategy::Transient if reason == ExitReason::Normal => {
                // Transient 仅异常退出时重启：正常退出停止并移除（先调 stop 递减 count，再清理字段）
                self.children.stop(id)?;
                return Ok(Duration::ZERO);
            }
            RestartStrategy::Permanent | RestartStrategy::Transient => {
                // Permanent 始终重启；Transient 异常退出重启（继续执行重启流程）
            }
        }

        // 提前释放 child 的可变借用，再调用 next_backoff（通过作用域）
        let current_restart_count = {
            let child = self
                .children
                .get_mut(id)
                .ok_or(SupervisorError::ChildNotFound(id))?;
            let current = child.restart_count;
            child.restart_count = child.restart_count.saturating_add(1);
            self.total_restarts = self.total_restarts.saturating_add(1);

            if self.total_restarts > self.fuse_threshold {
                let _ = self.transition(SupervisorState::Failed);
                return Err(SupervisorError::RestartThresholdExceeded {
                    count: self.total_restarts as u32,
                    max: self.fuse_threshold as u32,
                });
            }
            current
        };

        let backoff = self.next_backoff(current_restart_count);
        if let Some(c) = self.children.get_mut(id) {
            c.state = SupervisorState::Running;
            c.last_healthy = true;
        }
        Ok(backoff)
    }

    /// 关闭整个 Supervisor（递归关闭子项）
    pub fn shutdown(&mut self) {
        // 标记所有子项停止
        for slot in self.children.iter_mut() {
            slot.state = SupervisorState::Stopped;
            slot.occupied = false;
        }
        // 重置 count：shutdown 后所有槽位均被释放，
        // 若不重置 count，后续 spawn 会误报 CapacityExceeded
        self.children.count = 0;
        let _ = self.transition(SupervisorState::Stopped);
    }

    /// 恢复到 Running
    pub fn resume(&mut self) -> Result<(), SupervisorError> {
        self.transition(SupervisorState::Running)
    }
}

// ---------------------------------------------------------------------------
// 三层 Supervisor 具体类型
// ---------------------------------------------------------------------------

/// 队列 Supervisor（底层）
///
/// 管理单个 Worker 队列，最多 8 个 Worker 槽位。
#[derive(Debug)]
pub struct QueueSupervisor {
    core: SupervisorCore<8>,
    queue_id: u32,
}

impl QueueSupervisor {
    /// 创建队列 Supervisor
    pub fn new(queue_id: u32) -> Self {
        Self {
            core: SupervisorCore::new(SupervisorConfig::default()),
            queue_id,
        }
    }

    /// 使用指定配置创建队列 Supervisor（如自定义重启策略/退避）
    pub fn with_config(queue_id: u32, config: SupervisorConfig) -> Self {
        Self {
            core: SupervisorCore::new(config),
            queue_id,
        }
    }

    /// 队列 ID
    #[inline]
    pub fn queue_id(&self) -> u32 {
        self.queue_id
    }

    /// 当前在管 Worker 子项数量
    #[inline]
    pub fn worker_count(&self) -> usize {
        self.core.child_count()
    }

    /// 当前状态
    #[inline]
    pub fn state(&self) -> SupervisorState {
        self.core.state()
    }

    /// 启动 Worker 子项
    pub fn spawn_worker(&mut self, id: u64) -> Result<(), SupervisorError> {
        self.core.spawn_child(id, ChildKind::Worker)
    }

    /// 停止 Worker
    pub fn stop_worker(&mut self, id: u64) -> Result<(), SupervisorError> {
        self.core.stop_child(id)
    }

    /// 对 Worker 做健康检查
    pub fn health_check_worker(
        &mut self,
        id: u64,
        healthy: bool,
    ) -> Result<(), SupervisorError> {
        self.core.health_check_child(id, healthy)
    }

    /// 队列健康检查
    pub fn health_check(&mut self) -> HealthReport {
        self.core.health_check()
    }

    /// 重启 Worker（按退出原因落实 Transient 语义）
    pub fn restart_worker(&mut self, id: u64, reason: ExitReason) -> Result<Duration, SupervisorError> {
        self.core.restart(id, reason)
    }

    /// 关闭
    pub fn shutdown(&mut self) {
        self.core.shutdown();
    }
}

/// 域 Supervisor（中层）
///
/// 管理一个域内的多个 QueueSupervisor。
/// 最多 16 个队列。
#[derive(Debug)]
pub struct DomainSupervisor {
    core: SupervisorCore<16>,
    queues: ChildSet<16>,
    domain_id: u32,
}

impl DomainSupervisor {
    /// 创建域 Supervisor
    pub fn new(domain_id: u32) -> Self {
        Self {
            core: SupervisorCore::new(SupervisorConfig::default()),
            queues: ChildSet::new(),
            domain_id,
        }
    }

    /// 域 ID
    #[inline]
    pub fn domain_id(&self) -> u32 {
        self.domain_id
    }

    /// 当前状态
    #[inline]
    pub fn state(&self) -> SupervisorState {
        self.core.state()
    }

    /// 添加一个 QueueSupervisor（通过其 ID 注册）
    ///
    /// 先在 core 账本注册，再写入旁路 `queues` 账本；若旁路写入失败则回滚 core，
    /// 保证双账本不会出现"core 无 / queues 有"的不一致残留。
    pub fn spawn_queue(&mut self, queue_id: u64) -> Result<(), SupervisorError> {
        let entry = ChildEntry {
            id: queue_id,
            kind: ChildKind::Queue,
            state: SupervisorState::Running,
            restart_count: 0,
            last_healthy: true,
            occupied: true,
        };
        self.core.spawn_child(queue_id, ChildKind::Queue)?;
        if let Err(e) = self.queues.spawn(entry) {
            let _ = self.core.stop_child(queue_id);
            return Err(e);
        }
        Ok(())
    }

    /// 移除一个 QueueSupervisor
    ///
    /// 先在 core 账本注销，再清理旁路 `queues` 账本；若旁路清理失败则回滚 core，
    /// 保证双账本不会出现"core 有 / queues 无"的不一致残留。
    pub fn stop_queue(&mut self, queue_id: u64) -> Result<(), SupervisorError> {
        self.core.stop_child(queue_id)?;
        if let Err(e) = self.queues.stop(queue_id) {
            let _ = self.core.spawn_child(queue_id, ChildKind::Queue);
            return Err(e);
        }
        Ok(())
    }

    /// 更新队列健康状态
    pub fn mark_queue_healthy(&mut self, queue_id: u64, healthy: bool) -> Result<(), SupervisorError> {
        let q = self
            .queues
            .get_mut(queue_id)
            .ok_or(SupervisorError::ChildNotFound(queue_id))?;
        q.last_healthy = healthy;
        q.state = if healthy {
            SupervisorState::Running
        } else {
            SupervisorState::Degraded
        };
        // 同步更新 core 中的子项健康状态，并触发自身状态重计算
        self.core.health_check_child(queue_id, healthy)?;
        Ok(())
    }

    /// 域健康检查
    pub fn health_check(&mut self) -> HealthReport {
        self.core.health_check()
    }

    /// 重启队列（按退出原因落实 Transient 语义）
    pub fn restart_queue(&mut self, queue_id: u64, reason: ExitReason) -> Result<Duration, SupervisorError> {
        self.core.restart(queue_id, reason)
    }

    /// 关闭
    pub fn shutdown(&mut self) {
        self.core.shutdown();
        // 与 NodeSupervisor::shutdown 对称：显式清理 queues ChildSet，
        // 防止陈旧条目残留导致后续 spawn_queue 误报 AlreadyInState。
        // 与 ChildSet::stop() 对称：同时重置 restart_count 和 last_healthy，
        // 确保下次 spawn 获得完全干净的槽位（数据卫生一致性）
        for slot in self.queues.iter_mut() {
            slot.state = SupervisorState::Stopped;
            slot.occupied = false;
            slot.restart_count = 0;
            slot.last_healthy = true;
        }
        self.queues.count = 0;
    }
}

/// 节点 Supervisor（顶层）
///
/// 管理所有 DomainSupervisor，处理节点级故障。
/// 最多 8 个域。
#[derive(Debug)]
pub struct NodeSupervisor {
    core: SupervisorCore<8>,
    domains: ChildSet<8>,
    node_id: u64,
}

impl NodeSupervisor {
    /// 创建节点 Supervisor
    pub fn new(node_id: u64) -> Self {
        Self {
            core: SupervisorCore::new(SupervisorConfig::default())
                .with_fuse_threshold(1000),
            domains: ChildSet::new(),
            node_id,
        }
    }

    /// 节点 ID
    #[inline]
    pub fn node_id(&self) -> u64 {
        self.node_id
    }

    /// 当前状态
    #[inline]
    pub fn state(&self) -> SupervisorState {
        self.core.state()
    }

    /// 启动一个域
    ///
    /// 先在 core 账本注册，再写入旁路 `domains` 账本；若旁路写入失败则回滚 core，
    /// 保证双账本不会出现"core 无 / domains 有"的不一致残留。
    pub fn spawn_domain(&mut self, domain_id: u64) -> Result<(), SupervisorError> {
        let entry = ChildEntry {
            id: domain_id,
            kind: ChildKind::Domain,
            state: SupervisorState::Running,
            restart_count: 0,
            last_healthy: true,
            occupied: true,
        };
        self.core.spawn_child(domain_id, ChildKind::Domain)?;
        if let Err(e) = self.domains.spawn(entry) {
            let _ = self.core.stop_child(domain_id);
            return Err(e);
        }
        Ok(())
    }

    /// 停止一个域
    ///
    /// 先在 core 账本注销，再清理旁路 `domains` 账本；若旁路清理失败则回滚 core，
    /// 保证双账本不会出现"core 有 / domains 无"的不一致残留。
    pub fn stop_domain(&mut self, domain_id: u64) -> Result<(), SupervisorError> {
        self.core.stop_child(domain_id)?;
        if let Err(e) = self.domains.stop(domain_id) {
            let _ = self.core.spawn_child(domain_id, ChildKind::Domain);
            return Err(e);
        }
        Ok(())
    }

    /// 标记域健康状态
    pub fn mark_domain_healthy(
        &mut self,
        domain_id: u64,
        healthy: bool,
    ) -> Result<(), SupervisorError> {
        let d = self
            .domains
            .get_mut(domain_id)
            .ok_or(SupervisorError::ChildNotFound(domain_id))?;
        d.last_healthy = healthy;
        d.state = if healthy {
            SupervisorState::Running
        } else {
            SupervisorState::Degraded
        };
        self.core.health_check_child(domain_id, healthy)?;
        Ok(())
    }

    /// 节点级健康检查
    pub fn health_check(&mut self) -> HealthReport {
        self.core.health_check()
    }

    /// 重启域（按退出原因落实 Transient 语义）
    pub fn restart_domain(&mut self, domain_id: u64, reason: ExitReason) -> Result<Duration, SupervisorError> {
        self.core.restart(domain_id, reason)
    }

    /// 全量关闭
    pub fn shutdown(&mut self) {
        self.core.shutdown();
        for slot in self.domains.iter_mut() {
            slot.state = SupervisorState::Stopped;
            slot.occupied = false;
            slot.restart_count = 0;
            slot.last_healthy = true;
        }
        self.domains.count = 0;
    }

    /// 是否完全健康
    #[inline]
    pub fn is_healthy(&self) -> bool {
        self.domains.all_healthy()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_supervisor_state_transitions() {
        assert!(SupervisorState::Running.can_transition_to(SupervisorState::Stopped));
        assert!(SupervisorState::Running.can_transition_to(SupervisorState::Degraded));
        assert!(SupervisorState::Degraded.can_transition_to(SupervisorState::Running));
        assert!(SupervisorState::Degraded.can_transition_to(SupervisorState::Failed));
        assert!(SupervisorState::Stopped.can_transition_to(SupervisorState::Running));
        assert!(!SupervisorState::Running.can_transition_to(SupervisorState::Running));
    }

    #[test]
    fn test_child_set_spawn_and_stop() {
        let mut set: ChildSet<4> = ChildSet::new();
        assert_eq!(set.len(), 0);
        let entry = ChildEntry {
            id: 1,
            kind: ChildKind::Worker,
            state: SupervisorState::Running,
            restart_count: 0,
            last_healthy: true,
            occupied: true,
        };
        set.spawn(entry).unwrap();
        assert_eq!(set.len(), 1);
        assert!(set.get(1).is_some());
        set.stop(1).unwrap();
        assert_eq!(set.len(), 0);
    }

    #[test]
    fn test_child_set_capacity_exceeded() {
        let mut set: ChildSet<2> = ChildSet::new();
        for id in 0..2 {
            let entry = ChildEntry {
                id,
                kind: ChildKind::Worker,
                state: SupervisorState::Running,
                restart_count: 0,
                last_healthy: true,
                occupied: true,
            };
            set.spawn(entry).unwrap();
        }
        let overflow = ChildEntry {
            id: 99,
            kind: ChildKind::Worker,
            state: SupervisorState::Running,
            restart_count: 0,
            last_healthy: true,
            occupied: true,
        };
        assert!(set.spawn(overflow).is_err());
    }

    #[test]
    fn test_child_set_health_report() {
        let mut set: ChildSet<4> = ChildSet::new();
        for id in 0..3 {
            set.spawn(ChildEntry {
                id,
                kind: ChildKind::Worker,
                state: SupervisorState::Running,
                restart_count: 0,
                last_healthy: true,
                occupied: true,
            })
            .unwrap();
        }
        let mut report = set.health_report();
        assert_eq!(report.total, 3);
        assert_eq!(report.healthy, 3);

        // 标记一个为故障
        set.get_mut(1).unwrap().state = SupervisorState::Failed;
        report = set.health_report();
        assert_eq!(report.failed, 1);
        assert_eq!(report.healthy, 2);
    }

    #[test]
    fn test_queue_supervisor_spawn_and_health() {
        let mut q = QueueSupervisor::new(1);
        assert_eq!(q.state(), SupervisorState::Running);
        q.spawn_worker(10).unwrap();
        q.spawn_worker(11).unwrap();
        let report = q.health_check();
        assert!(report.is_healthy());

        q.health_check_worker(11, false).unwrap();
        let report = q.health_check();
        assert_eq!(report.degraded, 1);
    }

    #[test]
    fn test_queue_supervisor_restart() {
        let mut q = QueueSupervisor::new(2);
        q.spawn_worker(20).unwrap();
        // 全抖动退避：返回值 ∈ [0, 确定性上界]（范围断言，不断言精确值）
        let backoff = q.restart_worker(20, ExitReason::Abnormal).unwrap();
        // 第一次重启：确定性上界 = initial = 10ms
        assert!(backoff <= Duration::from_millis(10));
        let backoff = q.restart_worker(20, ExitReason::Abnormal).unwrap();
        // 第二次：上界 = 20ms
        assert!(backoff <= Duration::from_millis(20));
        let backoff = q.restart_worker(20, ExitReason::Abnormal).unwrap();
        // 第三次：上界 = 40ms
        assert!(backoff <= Duration::from_millis(40));
    }

    #[test]
    fn test_queue_supervisor_shutdown() {
        let mut q = QueueSupervisor::new(3);
        q.spawn_worker(30).unwrap();
        q.spawn_worker(31).unwrap();
        q.shutdown();
        assert_eq!(q.state(), SupervisorState::Stopped);
        // 关闭后不能再 spawn
        assert!(q.spawn_worker(32).is_err());
    }

    #[test]
    fn test_domain_supervisor_lifecycle() {
        let mut d = DomainSupervisor::new(100);
        d.spawn_queue(1).unwrap();
        d.spawn_queue(2).unwrap();
        assert_eq!(d.state(), SupervisorState::Running);

        d.mark_queue_healthy(2, false).unwrap();
        let report = d.health_check();
        assert_eq!(report.degraded, 1);

        d.mark_queue_healthy(2, true).unwrap();
        let report = d.health_check();
        assert!(report.is_healthy());

        d.stop_queue(1).unwrap();
        assert_eq!(d.state(), SupervisorState::Running);
    }

    #[test]
    fn test_node_supervisor_lifecycle() {
        let mut n = NodeSupervisor::new(1);
        n.spawn_domain(10).unwrap();
        n.spawn_domain(20).unwrap();
        assert!(n.is_healthy());

        n.mark_domain_healthy(20, false).unwrap();
        let report = n.health_check();
        assert_eq!(report.degraded, 1);

        n.mark_domain_healthy(20, true).unwrap();
        assert!(n.is_healthy());
    }

    #[test]
    fn test_restart_threshold_fuse() {
        let config = SupervisorConfig {
            initial_state: SupervisorState::Running,
            restart_strategy: RestartStrategy::Permanent,
            backoff: BackoffConfig::default(),
        };
        let mut core: SupervisorCore<4> = SupervisorCore::new(config).with_fuse_threshold(2);
        core.spawn_child(1, ChildKind::Worker).unwrap();
        let _ = core.restart(1, ExitReason::Abnormal).unwrap();
        let _ = core.restart(1, ExitReason::Abnormal).unwrap();
        // 第三次应触发熔断
        let res = core.restart(1, ExitReason::Abnormal);
        assert!(res.is_err());
        assert_eq!(res.unwrap_err(), SupervisorError::RestartThresholdExceeded { count: 3, max: 2 });
        assert_eq!(core.state(), SupervisorState::Failed);
    }

    #[test]
    fn test_temporary_strategy_no_restart() {
        let config = SupervisorConfig {
            initial_state: SupervisorState::Running,
            restart_strategy: RestartStrategy::Temporary,
            backoff: BackoffConfig::default(),
        };
        let mut core: SupervisorCore<4> = SupervisorCore::new(config);
        core.spawn_child(1, ChildKind::Worker).unwrap();
        let backoff = core.restart(1, ExitReason::Abnormal).unwrap();
        assert_eq!(backoff, Duration::ZERO);
        // 子项已被移除
        assert!(core.get_child(1).is_none());
        // 计数必须递减（count 漂移修复验证）
        assert_eq!(core.child_count(), 0);
        // 间接验证：再次 spawn 新子项成功，证明 slot 已释放
        core.spawn_child(2, ChildKind::Worker).unwrap();
        assert_eq!(core.child_count(), 1);
    }

    #[test]
    fn test_transient_strategy_restart_on_abnormal_exit() {
        // Transient + 异常退出：必须重启（退避非零，子项保留并恢复健康）
        let config = SupervisorConfig {
            initial_state: SupervisorState::Running,
            restart_strategy: RestartStrategy::Transient,
            backoff: BackoffConfig::default(),
        };
        let mut core: SupervisorCore<4> = SupervisorCore::new(config);
        core.spawn_child(1, ChildKind::Worker).unwrap();
        let backoff = core.restart(1, ExitReason::Abnormal).unwrap();
        // 首次重启确定性上界 = initial = 10ms（全抖动后 ∈ [0, 10ms]）
        assert!(backoff <= Duration::from_millis(10));
        let child = core.get_child(1).expect("Transient 异常退出后子项必须保留");
        assert_eq!(child.state, SupervisorState::Running);
        assert_eq!(child.restart_count, 1);
    }

    #[test]
    fn test_transient_strategy_no_restart_on_normal_exit() {
        // Transient + 正常退出：不重启（ZERO 退避，子项停止并移除）
        let config = SupervisorConfig {
            initial_state: SupervisorState::Running,
            restart_strategy: RestartStrategy::Transient,
            backoff: BackoffConfig::default(),
        };
        let mut core: SupervisorCore<4> = SupervisorCore::new(config);
        core.spawn_child(1, ChildKind::Worker).unwrap();
        let backoff = core.restart(1, ExitReason::Normal).unwrap();
        assert_eq!(backoff, Duration::ZERO);
        assert!(core.get_child(1).is_none(), "Transient 正常退出后子项必须移除");
    }

    #[test]
    fn test_permanent_strategy_restart_on_normal_exit() {
        // Permanent 不区分退出原因：正常退出也重启
        let config = SupervisorConfig {
            initial_state: SupervisorState::Running,
            restart_strategy: RestartStrategy::Permanent,
            backoff: BackoffConfig::default(),
        };
        let mut core: SupervisorCore<4> = SupervisorCore::new(config);
        core.spawn_child(1, ChildKind::Worker).unwrap();
        let backoff = core.restart(1, ExitReason::Normal).unwrap();
        assert!(backoff <= Duration::from_millis(10));
        assert!(core.get_child(1).is_some(), "Permanent 正常退出后子项必须保留");
    }

    #[test]
    fn test_fault_isolation() {
        let mut d = DomainSupervisor::new(7);
        d.spawn_queue(1).unwrap();
        d.spawn_queue(2).unwrap();
        // 一个队列故障不应直接让整个域失败（降级即可）
        d.mark_queue_healthy(1, false).unwrap();
        assert_eq!(d.state(), SupervisorState::Degraded);
        // 另一个仍正常
        d.mark_queue_healthy(1, true).unwrap();
        assert_eq!(d.state(), SupervisorState::Running);
    }

    #[test]
    fn test_invalid_state_transition() {
        let mut core: SupervisorCore<2> = SupervisorCore::new(SupervisorConfig::default());
        core.shutdown();
        // 从 Stopped 不能直接转到 Degraded
        let err = core.transition(SupervisorState::Degraded).unwrap_err();
        assert!(matches!(
            err,
            SupervisorError::InvalidTransition {
                from: SupervisorState::Stopped,
                to: SupervisorState::Degraded,
            }
        ));
    }
}