Skip to main content

spacedb_console/
alerts.rs

1//! The page-me surface — alerts ranked by severity.
2//!
3//! Everything an operator should be interrupted for, derived from the same
4//! observations, sorted worst-first.
5
6use serde::{Deserialize, Serialize};
7
8use crate::observe::{AgentBudgetObs, HomeObs, LagObs, ShardObs, StrongObs};
9
10/// Sort order is the variant order: `Critical` < `Warning` < `Info`, so sorting
11/// ascending puts the worst first.
12#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
13pub enum Severity {
14    Critical,
15    Warning,
16    Info,
17}
18
19#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
20pub enum AlertKind {
21    ShardLost,
22    QuorumLost,
23    ShardAtRisk,
24    ShardUnderReplicated,
25    ShardOverReplicated,
26    HomeOffline,
27    ReplicaLagHigh,
28    AgentBudgetExhausted,
29    AgentBudgetLow,
30}
31
32#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
33pub struct Alert {
34    pub severity: Severity,
35    pub kind: AlertKind,
36    pub subject: String,
37    pub detail: String,
38}
39
40/// Thresholds for what counts as alert-worthy.
41#[derive(Clone, Copy, Debug)]
42pub struct AlertThresholds {
43    /// Lag (ops) at or above which a replica is "behind".
44    pub lag_ops_warn: u64,
45    /// Remaining-budget percentage at or below which an agent is "low".
46    pub budget_low_pct: u8,
47}
48
49impl Default for AlertThresholds {
50    fn default() -> Self {
51        Self {
52            lag_ops_warn: 100,
53            budget_low_pct: 10,
54        }
55    }
56}
57
58/// Derive the full alert list, sorted worst-first.
59pub fn derive_alerts(
60    homes: &[HomeObs],
61    shards: &[ShardObs],
62    strong: &[StrongObs],
63    lags: &[LagObs],
64    budgets: &[AgentBudgetObs],
65    thresholds: &AlertThresholds,
66) -> Vec<Alert> {
67    let mut alerts = Vec::new();
68
69    for s in shards {
70        if s.lost() {
71            alerts.push(Alert {
72                severity: Severity::Critical,
73                kind: AlertKind::ShardLost,
74                subject: s.id.clone(),
75                detail: format!(
76                    "{}: {} reachable < {} required to reconstruct",
77                    s.collection, s.reachable_replicas, s.durable_floor
78                ),
79            });
80        } else if s.at_risk() {
81            alerts.push(Alert {
82                severity: Severity::Critical,
83                kind: AlertKind::ShardAtRisk,
84                subject: s.id.clone(),
85                detail: format!(
86                    "{}: {} reachable, at the durability floor — one more loss is data loss",
87                    s.collection, s.reachable_replicas
88                ),
89            });
90        } else if s.under_replicated() {
91            alerts.push(Alert {
92                severity: Severity::Warning,
93                kind: AlertKind::ShardUnderReplicated,
94                subject: s.id.clone(),
95                detail: format!(
96                    "{}: {}/{} replicas",
97                    s.collection, s.reachable_replicas, s.target_replicas
98                ),
99            });
100        }
101        // Cost, not danger: surplus copies an operator can reclaim.
102        if s.over_replicated() {
103            alerts.push(Alert {
104                severity: Severity::Warning,
105                kind: AlertKind::ShardOverReplicated,
106                subject: s.id.clone(),
107                detail: format!(
108                    "{}: {} surplus replicas ({}/{}) — reclaimable",
109                    s.collection,
110                    s.excess(),
111                    s.reachable_replicas,
112                    s.target_replicas
113                ),
114            });
115        }
116    }
117
118    for s in strong {
119        if !s.has_quorum() {
120            alerts.push(Alert {
121                severity: Severity::Critical,
122                kind: AlertKind::QuorumLost,
123                subject: s.collection.clone(),
124                detail: format!(
125                    "{}/{} quorum members online — strong writes are Unavailable",
126                    s.members_online, s.members_total
127                ),
128            });
129        }
130    }
131
132    for h in homes.iter().filter(|h| !h.online) {
133        alerts.push(Alert {
134            severity: Severity::Warning,
135            kind: AlertKind::HomeOffline,
136            subject: h.id.clone(),
137            detail: format!("home in {} is offline", h.region),
138        });
139    }
140
141    for l in lags.iter().filter(|l| l.lag_ops >= thresholds.lag_ops_warn) {
142        alerts.push(Alert {
143            severity: Severity::Warning,
144            kind: AlertKind::ReplicaLagHigh,
145            subject: l.collection.clone(),
146            detail: format!("{} ops behind", l.lag_ops),
147        });
148    }
149
150    for b in budgets {
151        if b.remaining == 0 {
152            alerts.push(Alert {
153                severity: Severity::Warning,
154                kind: AlertKind::AgentBudgetExhausted,
155                subject: b.agent.clone(),
156                detail: "budget exhausted — further priced ops will be refused".into(),
157            });
158        } else if b.limit > 0 && b.remaining * 100 / b.limit <= thresholds.budget_low_pct as u64 {
159            alerts.push(Alert {
160                severity: Severity::Info,
161                kind: AlertKind::AgentBudgetLow,
162                subject: b.agent.clone(),
163                detail: format!("{}% of budget remaining", b.remaining * 100 / b.limit),
164            });
165        }
166    }
167
168    alerts.sort_by(|a, b| a.severity.cmp(&b.severity).then(a.subject.cmp(&b.subject)));
169    alerts
170}