1use serde::{Deserialize, Serialize};
7
8use crate::observe::{AgentBudgetObs, HomeObs, LagObs, ShardObs, StrongObs};
9
10#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
13pub enum Severity {
14 Critical,
15 Warning,
16 Info,
17}
18
19#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
20pub enum AlertKind {
21 ShardLost,
22 QuorumLost,
23 ShardAtRisk,
24 ShardUnderReplicated,
25 ShardOverReplicated,
26 HomeOffline,
27 ReplicaLagHigh,
28 AgentBudgetExhausted,
29 AgentBudgetLow,
30}
31
32#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
33pub struct Alert {
34 pub severity: Severity,
35 pub kind: AlertKind,
36 pub subject: String,
37 pub detail: String,
38}
39
40#[derive(Clone, Copy, Debug)]
42pub struct AlertThresholds {
43 pub lag_ops_warn: u64,
45 pub budget_low_pct: u8,
47}
48
49impl Default for AlertThresholds {
50 fn default() -> Self {
51 Self {
52 lag_ops_warn: 100,
53 budget_low_pct: 10,
54 }
55 }
56}
57
58pub fn derive_alerts(
60 homes: &[HomeObs],
61 shards: &[ShardObs],
62 strong: &[StrongObs],
63 lags: &[LagObs],
64 budgets: &[AgentBudgetObs],
65 thresholds: &AlertThresholds,
66) -> Vec<Alert> {
67 let mut alerts = Vec::new();
68
69 for s in shards {
70 if s.lost() {
71 alerts.push(Alert {
72 severity: Severity::Critical,
73 kind: AlertKind::ShardLost,
74 subject: s.id.clone(),
75 detail: format!(
76 "{}: {} reachable < {} required to reconstruct",
77 s.collection, s.reachable_replicas, s.durable_floor
78 ),
79 });
80 } else if s.at_risk() {
81 alerts.push(Alert {
82 severity: Severity::Critical,
83 kind: AlertKind::ShardAtRisk,
84 subject: s.id.clone(),
85 detail: format!(
86 "{}: {} reachable, at the durability floor — one more loss is data loss",
87 s.collection, s.reachable_replicas
88 ),
89 });
90 } else if s.under_replicated() {
91 alerts.push(Alert {
92 severity: Severity::Warning,
93 kind: AlertKind::ShardUnderReplicated,
94 subject: s.id.clone(),
95 detail: format!(
96 "{}: {}/{} replicas",
97 s.collection, s.reachable_replicas, s.target_replicas
98 ),
99 });
100 }
101 if s.over_replicated() {
103 alerts.push(Alert {
104 severity: Severity::Warning,
105 kind: AlertKind::ShardOverReplicated,
106 subject: s.id.clone(),
107 detail: format!(
108 "{}: {} surplus replicas ({}/{}) — reclaimable",
109 s.collection,
110 s.excess(),
111 s.reachable_replicas,
112 s.target_replicas
113 ),
114 });
115 }
116 }
117
118 for s in strong {
119 if !s.has_quorum() {
120 alerts.push(Alert {
121 severity: Severity::Critical,
122 kind: AlertKind::QuorumLost,
123 subject: s.collection.clone(),
124 detail: format!(
125 "{}/{} quorum members online — strong writes are Unavailable",
126 s.members_online, s.members_total
127 ),
128 });
129 }
130 }
131
132 for h in homes.iter().filter(|h| !h.online) {
133 alerts.push(Alert {
134 severity: Severity::Warning,
135 kind: AlertKind::HomeOffline,
136 subject: h.id.clone(),
137 detail: format!("home in {} is offline", h.region),
138 });
139 }
140
141 for l in lags.iter().filter(|l| l.lag_ops >= thresholds.lag_ops_warn) {
142 alerts.push(Alert {
143 severity: Severity::Warning,
144 kind: AlertKind::ReplicaLagHigh,
145 subject: l.collection.clone(),
146 detail: format!("{} ops behind", l.lag_ops),
147 });
148 }
149
150 for b in budgets {
151 if b.remaining == 0 {
152 alerts.push(Alert {
153 severity: Severity::Warning,
154 kind: AlertKind::AgentBudgetExhausted,
155 subject: b.agent.clone(),
156 detail: "budget exhausted — further priced ops will be refused".into(),
157 });
158 } else if b.limit > 0 && b.remaining * 100 / b.limit <= thresholds.budget_low_pct as u64 {
159 alerts.push(Alert {
160 severity: Severity::Info,
161 kind: AlertKind::AgentBudgetLow,
162 subject: b.agent.clone(),
163 detail: format!("{}% of budget remaining", b.remaining * 100 / b.limit),
164 });
165 }
166 }
167
168 alerts.sort_by(|a, b| a.severity.cmp(&b.severity).then(a.subject.cmp(&b.subject)));
169 alerts
170}