chio_kernel/observability/
metrics.rs

1//! Prometheus text exposition for guard metrics.
2
3use chio_metrics_spec::{
4    CHIO_AMBIGUOUS_DISPATCH_RETAINED_HOLD_TOTAL, CHIO_GUARD_DENY_TOTAL,
5    CHIO_GUARD_EVAL_DURATION_SECONDS, CHIO_GUARD_FUEL_CONSUMED_TOTAL,
6    CHIO_GUARD_HOST_CALL_DURATION_SECONDS, CHIO_GUARD_MODULE_BYTES, CHIO_GUARD_RELOAD_TOTAL,
7    CHIO_GUARD_VERDICT_TOTAL, CHIO_SETTLEMENT_UNRESOLVED_TOTAL, CHIO_SIGNING_QUEUE_BLOCK_TOTAL,
8    GUARD_EVAL_DURATION_BUCKETS_SECONDS, GUARD_HOST_CALL_DURATION_BUCKETS_SECONDS,
9};
10
11pub use chio_metrics_spec::MetricKind as PrometheusMetricKind;
12
13pub const GUARD_METRICS_PATH: &str = "/metrics";
14pub const PROMETHEUS_TEXT_CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8";
15
16#[derive(Debug, Clone, Copy, PartialEq)]
17pub struct GuardMetricFamily {
18    pub name: &'static str,
19    pub help: &'static str,
20    pub kind: PrometheusMetricKind,
21    pub labels: &'static [&'static str],
22    pub buckets: &'static [&'static str],
23}
24
25const LABELS_GUARD_VERDICT: &[&str] = &["guard_id", "verdict"];
26const LABELS_GUARD_ONLY: &[&str] = &["guard_id"];
27const LABELS_GUARD_REASON_CLASS: &[&str] = &["guard_id", "reason_class"];
28const LABELS_GUARD_OUTCOME: &[&str] = &["guard_id", "outcome"];
29const LABELS_GUARD_HOST_FN: &[&str] = &["guard_id", "host_fn"];
30const LABELS_GUARD_EPOCH: &[&str] = &["guard_id", "epoch"];
31
32pub const GUARD_METRIC_FAMILIES: &[GuardMetricFamily] = &[
33    GuardMetricFamily {
34        name: CHIO_GUARD_EVAL_DURATION_SECONDS,
35        help: "WASM guard evaluation duration in seconds.",
36        kind: PrometheusMetricKind::Histogram,
37        labels: LABELS_GUARD_VERDICT,
38        buckets: GUARD_EVAL_DURATION_BUCKETS_SECONDS,
39    },
40    GuardMetricFamily {
41        name: CHIO_GUARD_FUEL_CONSUMED_TOTAL,
42        help: "Total WASM guard fuel units consumed.",
43        kind: PrometheusMetricKind::Counter,
44        labels: LABELS_GUARD_ONLY,
45        buckets: &[],
46    },
47    GuardMetricFamily {
48        name: CHIO_GUARD_VERDICT_TOTAL,
49        help: "Total WASM guard verdicts by guard and verdict.",
50        kind: PrometheusMetricKind::Counter,
51        labels: LABELS_GUARD_VERDICT,
52        buckets: &[],
53    },
54    GuardMetricFamily {
55        name: CHIO_GUARD_DENY_TOTAL,
56        help: "Total WASM guard denies by reason class.",
57        kind: PrometheusMetricKind::Counter,
58        labels: LABELS_GUARD_REASON_CLASS,
59        buckets: &[],
60    },
61    GuardMetricFamily {
62        name: CHIO_GUARD_RELOAD_TOTAL,
63        help: "Total WASM guard reload outcomes.",
64        kind: PrometheusMetricKind::Counter,
65        labels: LABELS_GUARD_OUTCOME,
66        buckets: &[],
67    },
68    GuardMetricFamily {
69        name: CHIO_GUARD_HOST_CALL_DURATION_SECONDS,
70        help: "WASM guard host-call duration in seconds.",
71        kind: PrometheusMetricKind::Histogram,
72        labels: LABELS_GUARD_HOST_FN,
73        buckets: GUARD_HOST_CALL_DURATION_BUCKETS_SECONDS,
74    },
75    GuardMetricFamily {
76        name: CHIO_GUARD_MODULE_BYTES,
77        help: "Loaded WASM guard module size in bytes.",
78        kind: PrometheusMetricKind::Gauge,
79        labels: LABELS_GUARD_EPOCH,
80        buckets: &[],
81    },
82];
83
84pub use chio_metrics_spec::CHIO_OTEL_INGRESS_DROP_TOTAL as METRIC_CHIO_OTEL_INGRESS_DROP_TOTAL;
85pub use chio_metrics_spec::CHIO_OTEL_SINK_DROP_TOTAL as METRIC_CHIO_OTEL_SINK_DROP_TOTAL;
86
87// Advertised runtime (non-guard) families the /metrics endpoint renders via the
88// chio-metrics-spec runtime families. Retained as endpoint documentation; the
89// actual samples are produced by render_otel_drop_families and the signing
90// family render, so this table is not iterated by the renderer.
91#[allow(dead_code)]
92const RUNTIME_METRIC_FAMILIES: &[GuardMetricFamily] = &[
93    GuardMetricFamily {
94        name: CHIO_SIGNING_QUEUE_BLOCK_TOTAL,
95        help: "Total receipt signing requests blocked by bounded queue capacity or byte budget.",
96        kind: PrometheusMetricKind::Counter,
97        labels: &["reason"],
98        buckets: &[],
99    },
100    GuardMetricFamily {
101        name: METRIC_CHIO_OTEL_INGRESS_DROP_TOTAL,
102        help: "Total OTEL ingress batches dropped by bounded queue admission.",
103        kind: PrometheusMetricKind::Counter,
104        labels: &[],
105        buckets: &[],
106    },
107    GuardMetricFamily {
108        name: METRIC_CHIO_OTEL_SINK_DROP_TOTAL,
109        help: "Total OTEL receipt sink batches dropped before append.",
110        kind: PrometheusMetricKind::Counter,
111        labels: &[],
112        buckets: &[],
113    },
114    GuardMetricFamily {
115        name: CHIO_SETTLEMENT_UNRESOLVED_TOTAL,
116        help: "Total settlement observer routing invocations with an unresolved outcome.",
117        kind: PrometheusMetricKind::Counter,
118        labels: &[],
119        buckets: &[],
120    },
121    GuardMetricFamily {
122        name: CHIO_AMBIGUOUS_DISPATCH_RETAINED_HOLD_TOTAL,
123        help: "Total budget or payment holds retained after an ambiguous post-dispatch outcome, labeled by whether durable reconciliation is available.",
124        kind: PrometheusMetricKind::Counter,
125        labels: &["reconciliation"],
126        buckets: &[],
127    },
128];
129
130#[derive(Debug, Clone, PartialEq, Eq)]
131pub struct MetricsEndpointResponse {
132    pub status: u16,
133    pub content_type: &'static str,
134    pub body: String,
135}
136
137#[must_use]
138pub fn guard_metrics_endpoint(path: &str) -> Option<MetricsEndpointResponse> {
139    if path != GUARD_METRICS_PATH {
140        return None;
141    }
142
143    Some(MetricsEndpointResponse {
144        status: 200,
145        content_type: PROMETHEUS_TEXT_CONTENT_TYPE,
146        body: render_guard_metrics_prometheus(),
147    })
148}
149
150/// Render the kernel `/metrics` body from the chio-metrics-spec runtime
151/// families. The kernel renders the guard families and the two OTEL-drop
152/// families (whose sole producers are chio-wasm-guards and the OTLP ingress,
153/// which cannot be depended on by the kernel), the signing-queue block family,
154/// the settlement unresolved family, the ambiguous-dispatch retained-hold
155/// family, and receipt watchdog gauges. Every sample comes from its shared
156/// runtime family rather than a fixed zero.
157#[must_use]
158pub fn render_guard_metrics_prometheus() -> String {
159    let mut output = String::new();
160    chio_metrics_spec::runtime::render_guard_families(&mut output);
161    chio_metrics_spec::runtime::render_otel_drop_families(&mut output);
162    chio_metrics_spec::runtime::families::SIGNING_QUEUE_BLOCK.render(&mut output);
163    chio_metrics_spec::runtime::families::SETTLEMENT_UNRESOLVED.render(&mut output);
164    chio_metrics_spec::runtime::families::AMBIGUOUS_DISPATCH_RETAINED_HOLD.render(&mut output);
165    chio_metrics_spec::runtime::render_receipt_watchdog_gauges(&mut output);
166    output
167}
168
169/// Turn a receipt-store health report into the watchdog gauges: the
170/// uncheckpointed entry_seq range and the seconds since the last commit. A
171/// serve-mode watchdog loop calls this on an interval so uncheckpointed growth
172/// and checkpoint staleness are observable without a human `chio receipt health`
173/// run.
174pub fn record_receipt_health_gauges(
175    report: &crate::receipt_store::ReceiptStoreHealthReport,
176    now_unix_ms: u64,
177) {
178    let range = match (
179        report.uncheckpointed_start_seq,
180        report.uncheckpointed_end_seq,
181    ) {
182        (Some(start), Some(end)) => end.saturating_sub(start),
183        _ => 0,
184    };
185    chio_metrics_spec::runtime::families::RECEIPT_UNCHECKPOINTED_RANGE.set(&[], range);
186
187    // Base `chio_receipt_seconds_since_last_checkpoint` on checkpoint PROGRESS,
188    // not on `writer.last_commit_unix_ms`. In an active store where writes keep
189    // committing while checkpointing stalls, the last-commit timestamp stays
190    // fresh, so a commit-based gauge would read near zero and the
191    // ChioReceiptCheckpointStale alert would never fire even as the
192    // uncheckpointed range grows. Track when the checkpointed high-water mark
193    // last advanced while a backlog was pending instead.
194    let has_backlog = report.latest_committed_entry_seq > report.latest_checkpointed_entry_seq;
195    let age_seconds = match CHECKPOINT_PROGRESS.lock() {
196        Ok(mut state) => checkpoint_staleness_seconds(
197            &mut state,
198            report.latest_checkpointed_entry_seq,
199            has_backlog,
200            now_unix_ms,
201        ),
202        // Fail-closed observability: a poisoned lock drops the sample (0) rather
203        // than unwinding the watchdog loop.
204        Err(_) => 0,
205    };
206    chio_metrics_spec::runtime::families::RECEIPT_CHECKPOINT_AGE_SECONDS.set(&[], age_seconds);
207}
208
209/// Process-global record of when the receipt checkpoint high-water mark last
210/// advanced, so `chio_receipt_seconds_since_last_checkpoint` measures
211/// checkpoint staleness rather than write-commit freshness.
212struct CheckpointProgress {
213    /// The latest checkpointed entry seq observed on the previous sample.
214    checkpointed_entry_seq: u64,
215    /// Wall-clock (unix ms) at which that seq was last observed to advance, i.e.
216    /// the start of the current staleness interval.
217    advanced_at_unix_ms: u64,
218}
219
220static CHECKPOINT_PROGRESS: std::sync::Mutex<Option<CheckpointProgress>> =
221    std::sync::Mutex::new(None);
222
223/// Seconds since the checkpoint high-water mark last advanced while a backlog is
224/// pending. Resets to 0 whenever the checkpoint advances OR there is no
225/// uncheckpointed backlog (a healthy, quiet store is never "stale"). Grows only
226/// while committed data sits uncheckpointed and the checkpoint seq does not move.
227/// Pure over `state` so it is unit-testable without the process-global.
228fn checkpoint_staleness_seconds(
229    state: &mut Option<CheckpointProgress>,
230    checkpointed_entry_seq: u64,
231    has_backlog: bool,
232    now_unix_ms: u64,
233) -> u64 {
234    let progressed = state
235        .as_ref()
236        .map(|prev| prev.checkpointed_entry_seq != checkpointed_entry_seq)
237        .unwrap_or(true);
238
239    if progressed || !has_backlog {
240        // Checkpoint advanced this sample, or nothing is pending to checkpoint:
241        // the store is healthy, so reset the staleness clock.
242        *state = Some(CheckpointProgress {
243            checkpointed_entry_seq,
244            advanced_at_unix_ms: now_unix_ms,
245        });
246        return 0;
247    }
248
249    // A backlog is present and the checkpoint high-water mark has not moved since
250    // the last sample: report how long it has been stalled.
251    state
252        .as_ref()
253        .map(|prev| now_unix_ms.saturating_sub(prev.advanced_at_unix_ms) / 1000)
254        .unwrap_or(0)
255}
256
257#[cfg(test)]
258mod checkpoint_staleness_tests {
259    use super::{checkpoint_staleness_seconds, CheckpointProgress};
260
261    #[test]
262    fn resets_when_checkpoint_advances_even_as_time_passes() {
263        let mut state: Option<CheckpointProgress> = None;
264        // First sample seeds the clock at 0.
265        assert_eq!(checkpoint_staleness_seconds(&mut state, 5, true, 1_000), 0);
266        // Checkpoint advanced 5 -> 9: the clock resets despite 60s elapsing.
267        assert_eq!(checkpoint_staleness_seconds(&mut state, 9, true, 61_000), 0);
268    }
269
270    #[test]
271    fn grows_while_backlog_stays_uncheckpointed() {
272        let mut state: Option<CheckpointProgress> = None;
273        assert_eq!(checkpoint_staleness_seconds(&mut state, 5, true, 1_000), 0);
274        // Same checkpoint seq, backlog still pending, 90s later: staleness grows.
275        // A commit-based gauge would read ~0 here because writes keep committing.
276        assert_eq!(
277            checkpoint_staleness_seconds(&mut state, 5, true, 91_000),
278            90
279        );
280    }
281
282    #[test]
283    fn stays_zero_for_a_healthy_quiet_store_without_backlog() {
284        let mut state: Option<CheckpointProgress> = None;
285        assert_eq!(checkpoint_staleness_seconds(&mut state, 5, false, 1_000), 0);
286        // No backlog: even a long-idle store must never look stale.
287        assert_eq!(
288            checkpoint_staleness_seconds(&mut state, 5, false, 999_000),
289            0
290        );
291    }
292}
chio_kernel/observability/metrics.rs

chio_kernel/observability/
metrics.rs
