edgeguard/alert.rs
1//! Outbound alerting (gateway L4) — threshold/regression alerts to Slack/webhook, in your own VPC.
2//!
3//! When `[alerts]` is enabled, EdgeGuard fires a Slack-compatible alert (`{ "text": … }`) when a
4//! hard-budget's consumed ratio crosses a threshold — cost-regression alerting with **no SaaS
5//! alerting plane** (the confirmed Phoenix gap: alerting gated behind the paid Arize AX cloud; teams
6//! otherwise pipe to Datadog/Grafana). Fire-and-forget: a background POST that never blocks or fails
7//! the request. **Edge-triggered**: exactly one alert per crossing into the alert zone (tracked per
8//! budget), so a busy proxy over-budget for a while doesn't spam the channel.
9//!
10//! First cut on budget breaches; the same shape extends to latency-percentile / error-rate / (via a
11//! trace store) eval-drift rules — the rule decision ([`AlertRuntime::decide_budget`]) is pure and
12//! the delivery is a generic webhook POST.
13
14use std::collections::HashMap;
15use std::sync::Mutex;
16use std::time::Duration;
17
18use crate::config::AlertsCfg;
19
20/// Compiled alerting runtime, carried on the proxy [`Runtime`](crate::proxy::Runtime).
21pub struct AlertRuntime {
22 pub enabled: bool,
23 webhook_url: String,
24 budget_threshold: f64,
25 client: reqwest::Client,
26 /// Per-budget "currently in the alert zone" flag, for edge-triggering (fire on false→true only).
27 alerting: Mutex<HashMap<String, bool>>,
28}
29
30impl AlertRuntime {
31 /// Compile from config. `enabled` folds in "has a non-empty webhook_url" so a misconfigured
32 /// switch (on, but no URL) is inert rather than trying to POST nowhere on every crossing.
33 pub fn build(cfg: &AlertsCfg) -> Self {
34 let client = reqwest::Client::builder()
35 .timeout(Duration::from_millis(cfg.timeout_ms.max(1)))
36 .build()
37 .unwrap_or_default();
38 AlertRuntime {
39 enabled: cfg.enabled && !cfg.webhook_url.trim().is_empty(),
40 webhook_url: cfg.webhook_url.trim().to_string(),
41 budget_threshold: cfg.budget_consumed_threshold,
42 client,
43 alerting: Mutex::new(HashMap::new()),
44 }
45 }
46
47 /// An inert runtime (alerting off) — the default when `[alerts]` is absent.
48 pub fn disabled() -> Self {
49 Self::build(&AlertsCfg::default())
50 }
51
52 /// Decide whether a budget's `ratio` should fire an alert **now**, updating the edge-trigger
53 /// state: fire only on the transition into the alert zone (was-below → now at/above the
54 /// threshold); reset when it drops back below so a later crossing re-alerts. Pure of IO, so the
55 /// dedup logic is directly testable.
56 pub fn decide_budget(&self, budget: &str, ratio: f64) -> bool {
57 if !self.enabled || !ratio.is_finite() {
58 return false;
59 }
60 let over = ratio >= self.budget_threshold;
61 let mut state = self.alerting.lock().expect("alert state mutex poisoned");
62 let was_over = state.get(budget).copied().unwrap_or(false);
63 if over {
64 if was_over {
65 false // already alerting for this budget — don't spam
66 } else {
67 state.insert(budget.to_string(), true);
68 true
69 }
70 } else {
71 state.insert(budget.to_string(), false); // reset so a future crossing re-alerts
72 false
73 }
74 }
75
76 /// Fire a Slack-compatible budget alert when the ratio first crosses the threshold. No-op unless
77 /// enabled and the crossing is fresh. Fire-and-forget — any webhook error is swallowed at debug.
78 pub fn fire_budget_alert(&self, budget: &str, ratio: f64) {
79 if !self.decide_budget(budget, ratio) {
80 return;
81 }
82 let text = format!(
83 "⚠️ EdgeGuard: LLM budget \"{budget}\" at {:.0}% of its limit (alert threshold {:.0}%).",
84 ratio * 100.0,
85 self.budget_threshold * 100.0
86 );
87 let body = serde_json::json!({ "text": text });
88 let client = self.client.clone();
89 let url = self.webhook_url.clone();
90 tokio::spawn(async move {
91 match client.post(&url).json(&body).send().await {
92 Ok(resp) if resp.status().is_success() => {}
93 Ok(resp) => tracing::debug!(status = %resp.status(), "alert webhook rejected"),
94 Err(e) => tracing::debug!(error = %e, "alert webhook failed"),
95 }
96 });
97 }
98}
99
100#[cfg(test)]
101mod tests {
102 use super::*;
103
104 fn rt(threshold: f64) -> AlertRuntime {
105 AlertRuntime::build(&AlertsCfg {
106 enabled: true,
107 webhook_url: "http://127.0.0.1:1/hook".into(),
108 budget_consumed_threshold: threshold,
109 ..AlertsCfg::default()
110 })
111 }
112
113 #[test]
114 fn budget_alert_is_edge_triggered_per_crossing() {
115 let a = rt(0.9);
116 // First time over the threshold → fire.
117 assert!(a.decide_budget("monthly", 0.95));
118 // Still over on subsequent requests → no repeat (no spam).
119 assert!(!a.decide_budget("monthly", 0.96));
120 assert!(!a.decide_budget("monthly", 1.20));
121 // Drops back below → reset (no fire), then a fresh crossing fires again.
122 assert!(!a.decide_budget("monthly", 0.50));
123 assert!(a.decide_budget("monthly", 0.91));
124 // A different budget name tracks its own edge.
125 assert!(a.decide_budget("daily", 0.90)); // exactly at threshold counts as over
126 }
127
128 #[test]
129 fn disabled_or_missing_url_never_fires() {
130 let off = AlertRuntime::disabled();
131 assert!(!off.decide_budget("b", 5.0));
132 let no_url = AlertRuntime::build(&AlertsCfg {
133 enabled: true,
134 webhook_url: " ".into(), // whitespace-only → treated as unset → inert
135 ..AlertsCfg::default()
136 });
137 assert!(!no_url.enabled);
138 assert!(!no_url.decide_budget("b", 5.0));
139 }
140
141 #[test]
142 fn non_finite_ratio_never_fires() {
143 let a = rt(0.9);
144 assert!(!a.decide_budget("b", f64::NAN));
145 assert!(!a.decide_budget("b", f64::INFINITY)); // inf is not finite → ignored, not a crossing
146 }
147}