loopsmith_core/config/
alerts.rs

1//! What a run measures about itself, and when a number should get a human's
2//! attention.
3//!
4//! A stop gate ends a run; an alert does not. It exists for the numbers that
5//! are worth knowing about long before they are worth stopping for — spend
6//! running at twice the usual rate, retries climbing, the pass rate sliding
7//! after a config change. Each alert fires at most once per run and is written
8//! to the ledger, the run log, and the run's outcome, so a scheduler's email or
9//! a CI step can act on it without parsing prose.
10//!
11//! Alerts are covered by the `audit` protected component: a loop that could
12//! edit its own alert thresholds could silence the one signal meant to catch
13//! it drifting.
14
15use schemars::JsonSchema;
16use serde::{Deserialize, Serialize};
17
18/// A number the engine keeps for every run.
19#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema)]
20#[serde(rename_all = "snake_case")]
21pub enum Metric {
22    /// Iterations completed so far.
23    Iterations,
24    /// Tokens charged so far, estimated where a provider reported none.
25    TokensUsed,
26    /// Dollars charged so far.
27    CostUsd,
28    /// Seconds since the run started.
29    WallClockSeconds,
30    /// Node dispatches that failed, after recovery had its say.
31    FailedDispatches,
32    /// Dispatches recovery sent round again: retries and revisions.
33    Retries,
34    /// Consecutive iterations in which no ruling moved.
35    StaleIterations,
36    /// Fraction of blocking checks passing at the latest ruling, 0 to 1.
37    ValidationPassRate,
38}
39
40impl Metric {
41    pub fn as_str(self) -> &'static str {
42        match self {
43            Metric::Iterations => "iterations",
44            Metric::TokensUsed => "tokens_used",
45            Metric::CostUsd => "cost_usd",
46            Metric::WallClockSeconds => "wall_clock_seconds",
47            Metric::FailedDispatches => "failed_dispatches",
48            Metric::Retries => "retries",
49            Metric::StaleIterations => "stale_iterations",
50            Metric::ValidationPassRate => "validation_pass_rate",
51        }
52    }
53}
54
55/// One threshold on one metric.
56#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
57#[serde(deny_unknown_fields)]
58pub struct Alert {
59    /// Stable name, used in the ledger and the run's outcome.
60    pub id: String,
61    pub metric: Metric,
62    /// Fire when the metric rises above this.
63    #[serde(default)]
64    pub above: Option<f64>,
65    /// Fire when the metric falls below this.
66    #[serde(default)]
67    pub below: Option<f64>,
68    /// What to tell the human, in their words. Defaults to a sentence built
69    /// from the metric and the threshold.
70    #[serde(default)]
71    pub message: Option<String>,
72}
73
74impl Alert {
75    /// Whether `value` crosses this alert's threshold.
76    pub fn fires_at(&self, value: f64) -> bool {
77        self.above.is_some_and(|t| value > t) || self.below.is_some_and(|t| value < t)
78    }
79
80    /// The line written when it fires.
81    pub fn describe(&self, value: f64) -> String {
82        if let Some(m) = &self.message {
83            return format!("{} ({} = {})", m, self.metric.as_str(), trim(value));
84        }
85        let bound = match (self.above, self.below) {
86            (Some(t), _) if value > t => format!("above {}", trim(t)),
87            (_, Some(t)) if value < t => format!("below {}", trim(t)),
88            _ => "outside its bounds".to_string(),
89        };
90        format!("{} is {} ({bound})", self.metric.as_str(), trim(value))
91    }
92}
93
94fn trim(v: f64) -> String {
95    if v.fract() == 0.0 && v.abs() < 1e15 {
96        format!("{}", v as i64)
97    } else {
98        format!("{v:.4}")
99            .trim_end_matches('0')
100            .trim_end_matches('.')
101            .to_string()
102    }
103}
104
105#[cfg(test)]
106mod tests {
107    use super::*;
108
109    fn alert(above: Option<f64>, below: Option<f64>) -> Alert {
110        Alert {
111            id: "a".into(),
112            metric: Metric::CostUsd,
113            above,
114            below,
115            message: None,
116        }
117    }
118
119    #[test]
120    fn an_alert_fires_strictly_past_its_threshold() {
121        let a = alert(Some(2.0), None);
122        assert!(!a.fires_at(2.0), "at the threshold is not past it");
123        assert!(a.fires_at(2.01));
124        let b = alert(None, Some(0.5));
125        assert!(b.fires_at(0.4));
126        assert!(!b.fires_at(0.5));
127    }
128
129    #[test]
130    fn the_default_message_names_the_metric_and_the_bound() {
131        assert_eq!(
132            alert(Some(2.0), None).describe(3.5),
133            "cost_usd is 3.5 (above 2)"
134        );
135    }
136}
loopsmith_core/config/alerts.rs

loopsmith_core/config/
alerts.rs
