chio_http_core/
metrics.rs

1//! HTTP-core verdict-edge metrics surfaced through the workspace
2//! `chio-metrics-spec` registry. The HTTP edge is wired into the workspace
3//! registry: every authority dispatch through
4//! `HttpAuthority::evaluate` increments
5//! [`CHIO_GUARD_EVALUATIONS_TOTAL`] with a `(guard, outcome)` label
6//! pair and observes the dispatch latency under
7//! [`CHIO_KERNEL_DECISION_LATENCY_SECONDS`].
8
9use std::sync::atomic::{AtomicU64, Ordering};
10
11pub use chio_metrics_spec::{
12    CHIO_GUARD_EVALUATIONS_TOTAL, CHIO_KERNEL_DECISION_LATENCY_SECONDS,
13    DECISION_LATENCY_BUCKETS_SECONDS,
14};
15
16pub const GUARD_LABEL_HTTP_AUTHORITY: &str = "http_authority";
17
18pub const GUARD_OUTCOME_ALLOW: &str = "allow";
19pub const GUARD_OUTCOME_DENY: &str = "deny";
20pub const GUARD_OUTCOME_ERROR: &str = "error";
21
22static GUARD_EVAL_ALLOW: AtomicU64 = AtomicU64::new(0);
23static GUARD_EVAL_DENY: AtomicU64 = AtomicU64::new(0);
24static GUARD_EVAL_ERROR: AtomicU64 = AtomicU64::new(0);
25static DECISION_LATENCY_ALLOW_NS_SUM: AtomicU64 = AtomicU64::new(0);
26static DECISION_LATENCY_DENY_NS_SUM: AtomicU64 = AtomicU64::new(0);
27static DECISION_LATENCY_ERROR_NS_SUM: AtomicU64 = AtomicU64::new(0);
28static DECISION_LATENCY_ALLOW_COUNT: AtomicU64 = AtomicU64::new(0);
29static DECISION_LATENCY_DENY_COUNT: AtomicU64 = AtomicU64::new(0);
30static DECISION_LATENCY_ERROR_COUNT: AtomicU64 = AtomicU64::new(0);
31static DECISION_LATENCY_ALLOW_BUCKETS: [AtomicU64; DECISION_LATENCY_BUCKET_COUNT] = [
32    AtomicU64::new(0),
33    AtomicU64::new(0),
34    AtomicU64::new(0),
35    AtomicU64::new(0),
36    AtomicU64::new(0),
37    AtomicU64::new(0),
38    AtomicU64::new(0),
39    AtomicU64::new(0),
40    AtomicU64::new(0),
41];
42static DECISION_LATENCY_DENY_BUCKETS: [AtomicU64; DECISION_LATENCY_BUCKET_COUNT] = [
43    AtomicU64::new(0),
44    AtomicU64::new(0),
45    AtomicU64::new(0),
46    AtomicU64::new(0),
47    AtomicU64::new(0),
48    AtomicU64::new(0),
49    AtomicU64::new(0),
50    AtomicU64::new(0),
51    AtomicU64::new(0),
52];
53static DECISION_LATENCY_ERROR_BUCKETS: [AtomicU64; DECISION_LATENCY_BUCKET_COUNT] = [
54    AtomicU64::new(0),
55    AtomicU64::new(0),
56    AtomicU64::new(0),
57    AtomicU64::new(0),
58    AtomicU64::new(0),
59    AtomicU64::new(0),
60    AtomicU64::new(0),
61    AtomicU64::new(0),
62    AtomicU64::new(0),
63];
64
65const NANOS_PER_SECOND: u64 = 1_000_000_000;
66const DECISION_LATENCY_BUCKET_COUNT: usize = DECISION_LATENCY_BUCKET_UPPER_NANOS.len() + 1;
67const DECISION_LATENCY_BUCKET_UPPER_NANOS: [u64; 8] = [
68    25_000_000,
69    50_000_000,
70    75_000_000,
71    100_000_000,
72    250_000_000,
73    500_000_000,
74    1_000_000_000,
75    2_500_000_000,
76];
77
78struct DecisionLatencySeries {
79    sum: &'static AtomicU64,
80    count: &'static AtomicU64,
81    buckets: &'static [AtomicU64; DECISION_LATENCY_BUCKET_COUNT],
82}
83
84/// Record the outcome of an HTTP authority evaluation. The `outcome`
85/// argument should be one of [`GUARD_OUTCOME_ALLOW`],
86/// [`GUARD_OUTCOME_DENY`], or [`GUARD_OUTCOME_ERROR`].
87pub fn record_guard_evaluation(outcome: &str) {
88    match outcome {
89        GUARD_OUTCOME_ALLOW => {
90            GUARD_EVAL_ALLOW.fetch_add(1, Ordering::Relaxed);
91        }
92        GUARD_OUTCOME_DENY => {
93            GUARD_EVAL_DENY.fetch_add(1, Ordering::Relaxed);
94        }
95        _ => {
96            GUARD_EVAL_ERROR.fetch_add(1, Ordering::Relaxed);
97        }
98    }
99}
100
101/// +1 on chio_dispatch_failure_total for a genuine mediation-edge failure: the
102/// request could not be evaluated (a fail-open/dispatch condition). `outcome` is
103/// "error". A normal policy/capability deny is an expected fail-closed decision,
104/// NOT a dispatch failure, and must never feed this counter or it would page the
105/// P0 alert on every rejected request. Denies are tracked by the guard-verdict
106/// metrics instead. Never called on allow.
107pub fn record_dispatch_failure(surface: &str, outcome: &str) {
108    chio_metrics_spec::runtime::families::DISPATCH_FAILURE.incr(&[surface, outcome]);
109}
110
111/// Observe a kernel decision latency sample in nanoseconds.
112pub fn observe_decision_latency_nanos(nanos: u64) {
113    observe_decision_latency_nanos_for_outcome(GUARD_OUTCOME_ALLOW, nanos);
114}
115
116/// Observe a kernel decision latency sample in nanoseconds for a verdict
117/// outcome. Unknown labels fail closed into the error series.
118pub fn observe_decision_latency_nanos_for_outcome(outcome: &str, nanos: u64) {
119    let series = decision_latency_series(outcome);
120    series.sum.fetch_add(nanos, Ordering::Relaxed);
121    series.count.fetch_add(1, Ordering::Relaxed);
122    for (index, upper) in DECISION_LATENCY_BUCKET_UPPER_NANOS.iter().enumerate() {
123        if nanos <= *upper {
124            series.buckets[index].fetch_add(1, Ordering::Relaxed);
125        }
126    }
127    series.buckets[DECISION_LATENCY_BUCKET_UPPER_NANOS.len()].fetch_add(1, Ordering::Relaxed);
128}
129
130#[must_use]
131pub fn guard_evaluations_total(outcome: &str) -> u64 {
132    match outcome {
133        GUARD_OUTCOME_ALLOW => GUARD_EVAL_ALLOW.load(Ordering::Relaxed),
134        GUARD_OUTCOME_DENY => GUARD_EVAL_DENY.load(Ordering::Relaxed),
135        _ => GUARD_EVAL_ERROR.load(Ordering::Relaxed),
136    }
137}
138
139#[must_use]
140pub fn decision_latency_count() -> u64 {
141    DECISION_LATENCY_ALLOW_COUNT.load(Ordering::Relaxed)
142        + DECISION_LATENCY_DENY_COUNT.load(Ordering::Relaxed)
143        + DECISION_LATENCY_ERROR_COUNT.load(Ordering::Relaxed)
144}
145
146#[must_use]
147pub fn render_http_core_metrics_prometheus() -> String {
148    let mut output = String::new();
149
150    output.push_str("# HELP ");
151    output.push_str(CHIO_GUARD_EVALUATIONS_TOTAL);
152    output.push_str(" Total guard evaluation outcomes across native and WASM guards.\n");
153    output.push_str("# TYPE ");
154    output.push_str(CHIO_GUARD_EVALUATIONS_TOTAL);
155    output.push_str(" counter\n");
156    for outcome in [GUARD_OUTCOME_ALLOW, GUARD_OUTCOME_DENY, GUARD_OUTCOME_ERROR] {
157        output.push_str(CHIO_GUARD_EVALUATIONS_TOTAL);
158        output.push_str("{guard=\"");
159        output.push_str(GUARD_LABEL_HTTP_AUTHORITY);
160        output.push_str("\",outcome=\"");
161        output.push_str(outcome);
162        output.push_str("\"} ");
163        output.push_str(&guard_evaluations_total(outcome).to_string());
164        output.push('\n');
165    }
166
167    output.push_str("# HELP ");
168    output.push_str(CHIO_KERNEL_DECISION_LATENCY_SECONDS);
169    output.push_str(" Kernel mediation decision latency in seconds.\n");
170    output.push_str("# TYPE ");
171    output.push_str(CHIO_KERNEL_DECISION_LATENCY_SECONDS);
172    output.push_str(" histogram\n");
173    for outcome in [GUARD_OUTCOME_ALLOW, GUARD_OUTCOME_DENY, GUARD_OUTCOME_ERROR] {
174        render_decision_latency_histogram(&mut output, outcome);
175    }
176
177    output
178}
179
180fn decision_latency_series(outcome: &str) -> DecisionLatencySeries {
181    match outcome {
182        GUARD_OUTCOME_ALLOW => DecisionLatencySeries {
183            sum: &DECISION_LATENCY_ALLOW_NS_SUM,
184            count: &DECISION_LATENCY_ALLOW_COUNT,
185            buckets: &DECISION_LATENCY_ALLOW_BUCKETS,
186        },
187        GUARD_OUTCOME_DENY => DecisionLatencySeries {
188            sum: &DECISION_LATENCY_DENY_NS_SUM,
189            count: &DECISION_LATENCY_DENY_COUNT,
190            buckets: &DECISION_LATENCY_DENY_BUCKETS,
191        },
192        _ => DecisionLatencySeries {
193            sum: &DECISION_LATENCY_ERROR_NS_SUM,
194            count: &DECISION_LATENCY_ERROR_COUNT,
195            buckets: &DECISION_LATENCY_ERROR_BUCKETS,
196        },
197    }
198}
199
200fn render_decision_latency_histogram(output: &mut String, outcome: &str) {
201    let series = decision_latency_series(outcome);
202    for (index, le) in DECISION_LATENCY_BUCKETS_SECONDS.iter().enumerate() {
203        render_decision_latency_bucket(
204            output,
205            outcome,
206            le,
207            series.buckets[index].load(Ordering::Relaxed),
208        );
209    }
210    render_decision_latency_bucket(
211        output,
212        outcome,
213        "+Inf",
214        series.buckets[DECISION_LATENCY_BUCKET_UPPER_NANOS.len()].load(Ordering::Relaxed),
215    );
216    output.push_str(CHIO_KERNEL_DECISION_LATENCY_SECONDS);
217    output.push_str("_sum{surface=\"");
218    output.push_str(GUARD_LABEL_HTTP_AUTHORITY);
219    output.push_str("\",outcome=\"");
220    output.push_str(outcome);
221    output.push_str("\"} ");
222    output.push_str(&format_seconds(series.sum.load(Ordering::Relaxed)));
223    output.push('\n');
224    output.push_str(CHIO_KERNEL_DECISION_LATENCY_SECONDS);
225    output.push_str("_count{surface=\"");
226    output.push_str(GUARD_LABEL_HTTP_AUTHORITY);
227    output.push_str("\",outcome=\"");
228    output.push_str(outcome);
229    output.push_str("\"} ");
230    output.push_str(&series.count.load(Ordering::Relaxed).to_string());
231    output.push('\n');
232}
233
234fn render_decision_latency_bucket(output: &mut String, outcome: &str, le: &str, count: u64) {
235    output.push_str(CHIO_KERNEL_DECISION_LATENCY_SECONDS);
236    output.push_str("_bucket{surface=\"");
237    output.push_str(GUARD_LABEL_HTTP_AUTHORITY);
238    output.push_str("\",outcome=\"");
239    output.push_str(outcome);
240    output.push_str("\",le=\"");
241    output.push_str(le);
242    output.push_str("\"} ");
243    output.push_str(&count.to_string());
244    output.push('\n');
245}
246
247fn format_seconds(nanos: u64) -> String {
248    format!("{:.9}", (nanos as f64) / (NANOS_PER_SECOND as f64))
249}
250
251#[cfg(test)]
252mod tests {
253    use super::*;
254
255    #[test]
256    fn registry_constants_match_spec() {
257        assert_eq!(CHIO_GUARD_EVALUATIONS_TOTAL, "chio_guard_evaluations_total");
258        assert_eq!(
259            CHIO_KERNEL_DECISION_LATENCY_SECONDS,
260            "chio_kernel_decision_latency_seconds"
261        );
262    }
263
264    #[test]
265    fn dispatch_failure_records_error_never_deny_or_allow() {
266        // Only a genuine evaluation error feeds the paging counter. A normal
267        // deny is NOT a dispatch failure, so the "denied" outcome is not
268        // produced anywhere.
269        record_dispatch_failure(GUARD_LABEL_HTTP_AUTHORITY, "error");
270        let mut body = String::new();
271        chio_metrics_spec::runtime::families::DISPATCH_FAILURE.render(&mut body);
272        assert!(
273            body.contains(
274                "chio_dispatch_failure_total{surface=\"http_authority\",outcome=\"error\"}"
275            ),
276            "error series missing: {body}"
277        );
278        // Neither a deny nor an allow outcome exists for this family.
279        assert!(
280            !body.contains("outcome=\"denied\""),
281            "a deny must not be recorded as a dispatch failure: {body}"
282        );
283        assert!(
284            !body.contains("outcome=\"allow\""),
285            "must not record allow: {body}"
286        );
287    }
288}
chio_http_core/metrics.rs

chio_http_core/
metrics.rs
