Skip to main content

spacedb_console/
fleet.rs

1//! Board 1 — Fleet Health. "Is anything on fire or about to lose data?"
2
3use serde::{Deserialize, Serialize};
4
5use crate::observe::{HomeObs, LagObs, ShardObs, StrongObs};
6
7/// The single-glance rollup.
8#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
9pub enum HealthStatus {
10    /// Everything reachable and durable.
11    Green,
12    /// Degraded — redundancy or freshness is below target, but nothing is lost.
13    Amber,
14    /// Data is lost, or a strong-tier collection can no longer serve.
15    Red,
16}
17
18/// The durability + availability picture for the whole fleet.
19#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
20pub struct FleetHealth {
21    pub homes_total: usize,
22    pub homes_online: usize,
23    pub shards_total: usize,
24    pub shards_under_replicated: usize,
25    pub shards_at_risk: usize,
26    pub shards_lost: usize,
27    /// Surplus copies above target — reclaimable cost, not a durability danger.
28    pub shards_over_replicated: usize,
29    pub strong_collections: usize,
30    pub strong_without_quorum: usize,
31    pub stale_collections: usize,
32    pub worst_lag_ops: u64,
33    pub bytes_stored: u64,
34    pub status: HealthStatus,
35}
36
37/// Assess fleet health from observations. A collection is "stale" if its lag
38/// exceeds `lag_ops_warn`.
39pub fn assess_fleet(
40    homes: &[HomeObs],
41    shards: &[ShardObs],
42    strong: &[StrongObs],
43    lags: &[LagObs],
44    lag_ops_warn: u64,
45) -> FleetHealth {
46    let homes_online = homes.iter().filter(|h| h.online).count();
47
48    let mut under = 0;
49    let mut at_risk = 0;
50    let mut lost = 0;
51    let mut over = 0;
52    let mut bytes_stored = 0u64;
53    for s in shards {
54        bytes_stored += s.size_bytes;
55        if s.lost() {
56            lost += 1;
57        } else if s.at_risk() {
58            at_risk += 1;
59        } else if s.under_replicated() {
60            under += 1;
61        }
62        // Orthogonal to under-replication: a shard can be over its target while
63        // every other shard is healthy.
64        if s.over_replicated() {
65            over += 1;
66        }
67    }
68
69    let strong_without_quorum = strong.iter().filter(|s| !s.has_quorum()).count();
70    let stale_collections = lags.iter().filter(|l| l.lag_ops > lag_ops_warn).count();
71    let worst_lag_ops = lags.iter().map(|l| l.lag_ops).max().unwrap_or(0);
72
73    let status = if lost > 0 || strong_without_quorum > 0 {
74        HealthStatus::Red
75    } else if under > 0
76        || at_risk > 0
77        || stale_collections > 0
78        || homes_online < homes.len()
79    {
80        HealthStatus::Amber
81    } else {
82        HealthStatus::Green
83    };
84
85    FleetHealth {
86        homes_total: homes.len(),
87        homes_online,
88        shards_total: shards.len(),
89        shards_under_replicated: under,
90        shards_at_risk: at_risk,
91        shards_lost: lost,
92        shards_over_replicated: over,
93        strong_collections: strong.len(),
94        strong_without_quorum,
95        stale_collections,
96        worst_lag_ops,
97        bytes_stored,
98        status,
99    }
100}