astra_core/
metrics.rs

1use std::fmt::Write as _;
2use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
3use std::sync::OnceLock;
4
5const QUEUE_WAIT_BUCKETS_MS: &[u64] = &[
6    1, 2, 5, 10, 20, 50, 75, 100, 150, 200, 300, 500, 750, 1_000, 1_500, 2_000, 5_000,
7];
8const CLIENT_WRITE_BUCKETS_MS: &[u64] = &[
9    1, 2, 5, 10, 20, 50, 75, 100, 150, 200, 300, 500, 750, 1_000, 1_500, 2_000, 5_000, 10_000,
10];
11const QUORUM_ACK_BUCKETS_MS: &[u64] = &[
12    1, 2, 5, 10, 20, 50, 75, 100, 150, 200, 300, 500, 750, 1_000, 1_500, 2_000, 5_000, 10_000,
13];
14const BATCH_SIZE_BUCKETS: &[u64] = &[1, 2, 4, 8, 16, 32, 64, 96, 128, 192, 256, 384, 512, 1024];
15const BG_IO_WAIT_BUCKETS_MS: &[u64] = &[
16    1, 2, 5, 10, 20, 50, 75, 100, 150, 200, 300, 500, 750, 1_000, 1_500, 2_000, 5_000, 10_000,
17];
18const WRITE_STALL_BUCKETS_MS: &[u64] = &[
19    1, 2, 5, 10, 20, 50, 75, 100, 150, 200, 300, 500, 750, 1_000, 1_500, 2_000, 5_000, 10_000,
20];
21
22#[derive(Clone, Copy, Debug)]
23enum HistogramMode {
24    SecondsFromMillis,
25    Raw,
26}
27
28#[derive(Debug)]
29struct Histogram {
30    bounds: &'static [u64],
31    mode: HistogramMode,
32    buckets: Vec<AtomicU64>,
33    count: AtomicU64,
34    sum: AtomicU64,
35}
36
37impl Histogram {
38    fn new(bounds: &'static [u64], mode: HistogramMode) -> Self {
39        let buckets = (0..=bounds.len()).map(|_| AtomicU64::new(0)).collect();
40        Self {
41            bounds,
42            mode,
43            buckets,
44            count: AtomicU64::new(0),
45            sum: AtomicU64::new(0),
46        }
47    }
48
49    fn observe(&self, value: u64) {
50        self.count.fetch_add(1, Ordering::Relaxed);
51        self.sum.fetch_add(value, Ordering::Relaxed);
52
53        let bucket_idx = self
54            .bounds
55            .iter()
56            .position(|bound| value <= *bound)
57            .unwrap_or(self.bounds.len());
58        self.buckets[bucket_idx].fetch_add(1, Ordering::Relaxed);
59    }
60
61    fn render(&self, out: &mut String, metric_name: &str, help: &str) {
62        let _ = writeln!(out, "# HELP {metric_name} {help}");
63        let _ = writeln!(out, "# TYPE {metric_name} histogram");
64
65        let mut cumulative = 0_u64;
66        for (idx, bound) in self.bounds.iter().enumerate() {
67            cumulative = cumulative.saturating_add(self.buckets[idx].load(Ordering::Relaxed));
68            let bound_label = match self.mode {
69                HistogramMode::SecondsFromMillis => format!("{:.6}", (*bound as f64) / 1_000.0),
70                HistogramMode::Raw => bound.to_string(),
71            };
72            let _ = writeln!(
73                out,
74                "{metric_name}_bucket{{le=\"{bound_label}\"}} {cumulative}"
75            );
76        }
77
78        cumulative =
79            cumulative.saturating_add(self.buckets[self.bounds.len()].load(Ordering::Relaxed));
80        let _ = writeln!(out, "{metric_name}_bucket{{le=\"+Inf\"}} {cumulative}");
81
82        let sum_value = self.sum.load(Ordering::Relaxed);
83        let sum_label = match self.mode {
84            HistogramMode::SecondsFromMillis => format!("{:.6}", (sum_value as f64) / 1_000.0),
85            HistogramMode::Raw => sum_value.to_string(),
86        };
87        let _ = writeln!(out, "{metric_name}_sum {sum_label}");
88        let _ = writeln!(
89            out,
90            "{metric_name}_count {}",
91            self.count.load(Ordering::Relaxed)
92        );
93    }
94}
95
96#[derive(Debug)]
97struct AstraMetrics {
98    enabled: AtomicBool,
99    put_queue_wait_ms: Histogram,
100    put_queue_wait_tier0_ms: Histogram,
101    put_queue_wait_normal_ms: Histogram,
102    put_raft_write_ms: Histogram,
103    put_quorum_ack_ms: Histogram,
104    put_batch_size: Histogram,
105    bg_io_throttle_wait_ms: Histogram,
106    put_batches_total: AtomicU64,
107    put_batch_requests_total: AtomicU64,
108    put_tier0_enqueued_total: AtomicU64,
109    put_tier0_dispatch_total: AtomicU64,
110    put_tier0_bypass_write_pressure_total: AtomicU64,
111    quorum_ack_observations: AtomicU64,
112    latest_quorum_ack_ms: AtomicU64,
113    put_inflight_requests: AtomicU64,
114    put_tier0_queue_depth: AtomicU64,
115    put_normal_queue_depth: AtomicU64,
116    wal_effective_linger_us: AtomicU64,
117    wal_queue_depth: AtomicU64,
118    wal_queue_bytes: AtomicU64,
119    lsm_synth_l0_files: AtomicU64,
120    write_stall_delay_ms: Histogram,
121    write_stall_events_total: AtomicU64,
122    write_stall_band_5_events_total: AtomicU64,
123    write_stall_band_6_events_total: AtomicU64,
124    write_stall_band_7_plus_events_total: AtomicU64,
125    write_reject_events_total: AtomicU64,
126    bg_io_tokens_available: AtomicU64,
127    bg_io_sqe_tokens_available: AtomicU64,
128    bg_io_throttle_events_total: AtomicU64,
129    bg_io_sqe_wait_ms: Histogram,
130    bg_io_sqe_throttle_events_total: AtomicU64,
131    list_prefix_filter_skips_total: AtomicU64,
132    list_prefix_filter_hits_total: AtomicU64,
133    list_revision_filter_skips_total: AtomicU64,
134    list_revision_filter_hits_total: AtomicU64,
135    list_prefetch_hits_total: AtomicU64,
136    list_prefetch_misses_total: AtomicU64,
137    read_isolation_dispatch_total: AtomicU64,
138    read_isolation_failures_total: AtomicU64,
139    read_quorum_checks_total: AtomicU64,
140    read_quorum_failures_total: AtomicU64,
141    read_quorum_internal_total: AtomicU64,
142    transport_closed_conn_unavailable_total: AtomicU64,
143    forward_retry_attempts_total: AtomicU64,
144    gateway_read_ticket_hits_total: AtomicU64,
145    gateway_read_ticket_misses_total: AtomicU64,
146    gateway_singleflight_leader_total: AtomicU64,
147    gateway_singleflight_waiter_total: AtomicU64,
148    gateway_singleflight_overflow_total: AtomicU64,
149    gateway_singleflight_waiter_timeouts_total: AtomicU64,
150    request_get_total: AtomicU64,
151    request_list_total: AtomicU64,
152    request_put_total: AtomicU64,
153    request_delete_total: AtomicU64,
154    request_txn_total: AtomicU64,
155    request_lease_total: AtomicU64,
156    request_watch_total: AtomicU64,
157    request_tier0_total: AtomicU64,
158    profile_switch_total: AtomicU64,
159    profile_active_kubernetes: AtomicU64,
160    profile_active_omni: AtomicU64,
161    profile_active_gateway: AtomicU64,
162    profile_active_auto: AtomicU64,
163    profile_applied_put_max_requests: AtomicU64,
164    profile_applied_put_linger_us: AtomicU64,
165    profile_applied_prefetch_entries: AtomicU64,
166    profile_applied_bg_io_tokens_per_sec: AtomicU64,
167}
168
169impl AstraMetrics {
170    fn new() -> Self {
171        Self {
172            enabled: AtomicBool::new(true),
173            put_queue_wait_ms: Histogram::new(
174                QUEUE_WAIT_BUCKETS_MS,
175                HistogramMode::SecondsFromMillis,
176            ),
177            put_queue_wait_tier0_ms: Histogram::new(
178                QUEUE_WAIT_BUCKETS_MS,
179                HistogramMode::SecondsFromMillis,
180            ),
181            put_queue_wait_normal_ms: Histogram::new(
182                QUEUE_WAIT_BUCKETS_MS,
183                HistogramMode::SecondsFromMillis,
184            ),
185            put_raft_write_ms: Histogram::new(
186                CLIENT_WRITE_BUCKETS_MS,
187                HistogramMode::SecondsFromMillis,
188            ),
189            put_quorum_ack_ms: Histogram::new(
190                QUORUM_ACK_BUCKETS_MS,
191                HistogramMode::SecondsFromMillis,
192            ),
193            put_batch_size: Histogram::new(BATCH_SIZE_BUCKETS, HistogramMode::Raw),
194            bg_io_throttle_wait_ms: Histogram::new(
195                BG_IO_WAIT_BUCKETS_MS,
196                HistogramMode::SecondsFromMillis,
197            ),
198            put_batches_total: AtomicU64::new(0),
199            put_batch_requests_total: AtomicU64::new(0),
200            put_tier0_enqueued_total: AtomicU64::new(0),
201            put_tier0_dispatch_total: AtomicU64::new(0),
202            put_tier0_bypass_write_pressure_total: AtomicU64::new(0),
203            quorum_ack_observations: AtomicU64::new(0),
204            latest_quorum_ack_ms: AtomicU64::new(0),
205            put_inflight_requests: AtomicU64::new(0),
206            put_tier0_queue_depth: AtomicU64::new(0),
207            put_normal_queue_depth: AtomicU64::new(0),
208            wal_effective_linger_us: AtomicU64::new(0),
209            wal_queue_depth: AtomicU64::new(0),
210            wal_queue_bytes: AtomicU64::new(0),
211            lsm_synth_l0_files: AtomicU64::new(0),
212            write_stall_delay_ms: Histogram::new(
213                WRITE_STALL_BUCKETS_MS,
214                HistogramMode::SecondsFromMillis,
215            ),
216            write_stall_events_total: AtomicU64::new(0),
217            write_stall_band_5_events_total: AtomicU64::new(0),
218            write_stall_band_6_events_total: AtomicU64::new(0),
219            write_stall_band_7_plus_events_total: AtomicU64::new(0),
220            write_reject_events_total: AtomicU64::new(0),
221            bg_io_tokens_available: AtomicU64::new(0),
222            bg_io_sqe_tokens_available: AtomicU64::new(0),
223            bg_io_throttle_events_total: AtomicU64::new(0),
224            bg_io_sqe_wait_ms: Histogram::new(
225                BG_IO_WAIT_BUCKETS_MS,
226                HistogramMode::SecondsFromMillis,
227            ),
228            bg_io_sqe_throttle_events_total: AtomicU64::new(0),
229            list_prefix_filter_skips_total: AtomicU64::new(0),
230            list_prefix_filter_hits_total: AtomicU64::new(0),
231            list_revision_filter_skips_total: AtomicU64::new(0),
232            list_revision_filter_hits_total: AtomicU64::new(0),
233            list_prefetch_hits_total: AtomicU64::new(0),
234            list_prefetch_misses_total: AtomicU64::new(0),
235            read_isolation_dispatch_total: AtomicU64::new(0),
236            read_isolation_failures_total: AtomicU64::new(0),
237            read_quorum_checks_total: AtomicU64::new(0),
238            read_quorum_failures_total: AtomicU64::new(0),
239            read_quorum_internal_total: AtomicU64::new(0),
240            transport_closed_conn_unavailable_total: AtomicU64::new(0),
241            forward_retry_attempts_total: AtomicU64::new(0),
242            gateway_read_ticket_hits_total: AtomicU64::new(0),
243            gateway_read_ticket_misses_total: AtomicU64::new(0),
244            gateway_singleflight_leader_total: AtomicU64::new(0),
245            gateway_singleflight_waiter_total: AtomicU64::new(0),
246            gateway_singleflight_overflow_total: AtomicU64::new(0),
247            gateway_singleflight_waiter_timeouts_total: AtomicU64::new(0),
248            request_get_total: AtomicU64::new(0),
249            request_list_total: AtomicU64::new(0),
250            request_put_total: AtomicU64::new(0),
251            request_delete_total: AtomicU64::new(0),
252            request_txn_total: AtomicU64::new(0),
253            request_lease_total: AtomicU64::new(0),
254            request_watch_total: AtomicU64::new(0),
255            request_tier0_total: AtomicU64::new(0),
256            profile_switch_total: AtomicU64::new(0),
257            profile_active_kubernetes: AtomicU64::new(0),
258            profile_active_omni: AtomicU64::new(0),
259            profile_active_gateway: AtomicU64::new(0),
260            profile_active_auto: AtomicU64::new(1),
261            profile_applied_put_max_requests: AtomicU64::new(0),
262            profile_applied_put_linger_us: AtomicU64::new(0),
263            profile_applied_prefetch_entries: AtomicU64::new(0),
264            profile_applied_bg_io_tokens_per_sec: AtomicU64::new(0),
265        }
266    }
267
268    fn enabled(&self) -> bool {
269        self.enabled.load(Ordering::Relaxed)
270    }
271}
272
273static METRICS: OnceLock<AstraMetrics> = OnceLock::new();
274
275fn metrics() -> &'static AstraMetrics {
276    METRICS.get_or_init(AstraMetrics::new)
277}
278
279#[derive(Debug, Clone, Copy, Default)]
280pub struct RequestMixSnapshot {
281    pub get: u64,
282    pub list: u64,
283    pub put: u64,
284    pub delete: u64,
285    pub txn: u64,
286    pub lease: u64,
287    pub watch: u64,
288    pub tier0: u64,
289}
290
291impl RequestMixSnapshot {
292    pub fn total(self) -> u64 {
293        self.get
294            .saturating_add(self.list)
295            .saturating_add(self.put)
296            .saturating_add(self.delete)
297            .saturating_add(self.txn)
298            .saturating_add(self.lease)
299            .saturating_add(self.watch)
300    }
301
302    pub fn delta_since(self, previous: Self) -> Self {
303        Self {
304            get: self.get.saturating_sub(previous.get),
305            list: self.list.saturating_sub(previous.list),
306            put: self.put.saturating_sub(previous.put),
307            delete: self.delete.saturating_sub(previous.delete),
308            txn: self.txn.saturating_sub(previous.txn),
309            lease: self.lease.saturating_sub(previous.lease),
310            watch: self.watch.saturating_sub(previous.watch),
311            tier0: self.tier0.saturating_sub(previous.tier0),
312        }
313    }
314}
315
316pub fn request_mix_snapshot() -> RequestMixSnapshot {
317    let reg = metrics();
318    if !reg.enabled() {
319        return RequestMixSnapshot::default();
320    }
321    RequestMixSnapshot {
322        get: reg.request_get_total.load(Ordering::Relaxed),
323        list: reg.request_list_total.load(Ordering::Relaxed),
324        put: reg.request_put_total.load(Ordering::Relaxed),
325        delete: reg.request_delete_total.load(Ordering::Relaxed),
326        txn: reg.request_txn_total.load(Ordering::Relaxed),
327        lease: reg.request_lease_total.load(Ordering::Relaxed),
328        watch: reg.request_watch_total.load(Ordering::Relaxed),
329        tier0: reg.request_tier0_total.load(Ordering::Relaxed),
330    }
331}
332
333pub fn set_enabled(enabled: bool) {
334    metrics().enabled.store(enabled, Ordering::Relaxed);
335}
336
337pub fn observe_put_queue_wait_ms(value_ms: u64) {
338    let reg = metrics();
339    if !reg.enabled() {
340        return;
341    }
342    reg.put_queue_wait_ms.observe(value_ms);
343}
344
345pub fn observe_put_queue_wait_ms_by_priority(value_ms: u64, tier0: bool) {
346    let reg = metrics();
347    if !reg.enabled() {
348        return;
349    }
350    if tier0 {
351        reg.put_queue_wait_tier0_ms.observe(value_ms);
352    } else {
353        reg.put_queue_wait_normal_ms.observe(value_ms);
354    }
355}
356
357pub fn observe_put_raft_client_write_ms(value_ms: u64) {
358    let reg = metrics();
359    if !reg.enabled() {
360        return;
361    }
362    reg.put_raft_write_ms.observe(value_ms);
363}
364
365pub fn observe_put_quorum_ack_ms(value_ms: u64) {
366    let reg = metrics();
367    if !reg.enabled() {
368        return;
369    }
370    reg.put_quorum_ack_ms.observe(value_ms);
371    reg.latest_quorum_ack_ms.store(value_ms, Ordering::Relaxed);
372    reg.quorum_ack_observations.fetch_add(1, Ordering::Relaxed);
373}
374
375pub fn observe_put_batch_size(size: usize) {
376    let reg = metrics();
377    if !reg.enabled() {
378        return;
379    }
380    reg.put_batch_size.observe(size as u64);
381}
382
383pub fn inc_put_batches() {
384    let reg = metrics();
385    if !reg.enabled() {
386        return;
387    }
388    reg.put_batches_total.fetch_add(1, Ordering::Relaxed);
389}
390
391pub fn add_put_batch_requests(count: usize) {
392    let reg = metrics();
393    if !reg.enabled() {
394        return;
395    }
396    reg.put_batch_requests_total
397        .fetch_add(count as u64, Ordering::Relaxed);
398}
399
400pub fn inc_put_tier0_enqueued() {
401    let reg = metrics();
402    if !reg.enabled() {
403        return;
404    }
405    reg.put_tier0_enqueued_total.fetch_add(1, Ordering::Relaxed);
406}
407
408pub fn inc_put_tier0_dispatch_events() {
409    let reg = metrics();
410    if !reg.enabled() {
411        return;
412    }
413    reg.put_tier0_dispatch_total.fetch_add(1, Ordering::Relaxed);
414}
415
416pub fn inc_put_tier0_bypass_write_pressure() {
417    let reg = metrics();
418    if !reg.enabled() {
419        return;
420    }
421    reg.put_tier0_bypass_write_pressure_total
422        .fetch_add(1, Ordering::Relaxed);
423}
424
425pub fn set_put_inflight_requests(count: usize) {
426    let reg = metrics();
427    if !reg.enabled() {
428        return;
429    }
430    reg.put_inflight_requests
431        .store(count as u64, Ordering::Relaxed);
432}
433
434pub fn set_put_tier0_queue_depth(count: usize) {
435    let reg = metrics();
436    if !reg.enabled() {
437        return;
438    }
439    reg.put_tier0_queue_depth
440        .store(count as u64, Ordering::Relaxed);
441}
442
443pub fn set_put_normal_queue_depth(count: usize) {
444    let reg = metrics();
445    if !reg.enabled() {
446        return;
447    }
448    reg.put_normal_queue_depth
449        .store(count as u64, Ordering::Relaxed);
450}
451
452pub fn current_put_inflight_requests() -> u64 {
453    let reg = metrics();
454    if !reg.enabled() {
455        return 0;
456    }
457    reg.put_inflight_requests.load(Ordering::Relaxed)
458}
459
460pub fn set_wal_effective_linger_us(value: u64) {
461    let reg = metrics();
462    if !reg.enabled() {
463        return;
464    }
465    reg.wal_effective_linger_us.store(value, Ordering::Relaxed);
466}
467
468pub fn set_wal_queue_depth(depth: usize) {
469    let reg = metrics();
470    reg.wal_queue_depth.store(depth as u64, Ordering::Relaxed);
471}
472
473pub fn set_wal_queue_bytes(bytes: usize) {
474    let reg = metrics();
475    reg.wal_queue_bytes.store(bytes as u64, Ordering::Relaxed);
476}
477
478pub fn current_wal_queue_depth() -> u64 {
479    metrics().wal_queue_depth.load(Ordering::Relaxed)
480}
481
482pub fn current_wal_queue_bytes() -> u64 {
483    metrics().wal_queue_bytes.load(Ordering::Relaxed)
484}
485
486pub fn set_lsm_synth_l0_files(value: u64) {
487    let reg = metrics();
488    if !reg.enabled() {
489        return;
490    }
491    reg.lsm_synth_l0_files.store(value, Ordering::Relaxed);
492}
493
494pub fn observe_write_stall_delay_ms(value_ms: u64) {
495    let reg = metrics();
496    if !reg.enabled() {
497        return;
498    }
499    reg.write_stall_delay_ms.observe(value_ms);
500    reg.write_stall_events_total.fetch_add(1, Ordering::Relaxed);
501}
502
503pub fn inc_write_stall_band_5_events() {
504    let reg = metrics();
505    if !reg.enabled() {
506        return;
507    }
508    reg.write_stall_band_5_events_total
509        .fetch_add(1, Ordering::Relaxed);
510}
511
512pub fn inc_write_stall_band_6_events() {
513    let reg = metrics();
514    if !reg.enabled() {
515        return;
516    }
517    reg.write_stall_band_6_events_total
518        .fetch_add(1, Ordering::Relaxed);
519}
520
521pub fn inc_write_stall_band_7_plus_events() {
522    let reg = metrics();
523    if !reg.enabled() {
524        return;
525    }
526    reg.write_stall_band_7_plus_events_total
527        .fetch_add(1, Ordering::Relaxed);
528}
529
530pub fn inc_write_reject_events() {
531    let reg = metrics();
532    if !reg.enabled() {
533        return;
534    }
535    reg.write_reject_events_total
536        .fetch_add(1, Ordering::Relaxed);
537}
538
539pub fn set_bg_io_tokens_available(tokens: u64) {
540    let reg = metrics();
541    if !reg.enabled() {
542        return;
543    }
544    reg.bg_io_tokens_available.store(tokens, Ordering::Relaxed);
545}
546
547pub fn set_bg_io_sqe_tokens_available(tokens: u64) {
548    let reg = metrics();
549    if !reg.enabled() {
550        return;
551    }
552    reg.bg_io_sqe_tokens_available
553        .store(tokens, Ordering::Relaxed);
554}
555
556pub fn observe_bg_io_throttle_wait_ms(value_ms: u64) {
557    let reg = metrics();
558    if !reg.enabled() {
559        return;
560    }
561    reg.bg_io_throttle_wait_ms.observe(value_ms);
562    reg.bg_io_throttle_events_total
563        .fetch_add(1, Ordering::Relaxed);
564}
565
566pub fn observe_bg_io_sqe_wait_ms(value_ms: u64) {
567    let reg = metrics();
568    if !reg.enabled() {
569        return;
570    }
571    reg.bg_io_sqe_wait_ms.observe(value_ms);
572    reg.bg_io_sqe_throttle_events_total
573        .fetch_add(1, Ordering::Relaxed);
574}
575
576pub fn inc_list_prefix_filter_skips() {
577    let reg = metrics();
578    if !reg.enabled() {
579        return;
580    }
581    reg.list_prefix_filter_skips_total
582        .fetch_add(1, Ordering::Relaxed);
583}
584
585pub fn inc_list_prefix_filter_hits() {
586    let reg = metrics();
587    if !reg.enabled() {
588        return;
589    }
590    reg.list_prefix_filter_hits_total
591        .fetch_add(1, Ordering::Relaxed);
592}
593
594pub fn inc_list_revision_filter_skips() {
595    let reg = metrics();
596    if !reg.enabled() {
597        return;
598    }
599    reg.list_revision_filter_skips_total
600        .fetch_add(1, Ordering::Relaxed);
601}
602
603pub fn inc_list_revision_filter_hits() {
604    let reg = metrics();
605    if !reg.enabled() {
606        return;
607    }
608    reg.list_revision_filter_hits_total
609        .fetch_add(1, Ordering::Relaxed);
610}
611
612pub fn inc_list_prefetch_hits() {
613    let reg = metrics();
614    if !reg.enabled() {
615        return;
616    }
617    reg.list_prefetch_hits_total.fetch_add(1, Ordering::Relaxed);
618}
619
620pub fn inc_list_prefetch_misses() {
621    let reg = metrics();
622    if !reg.enabled() {
623        return;
624    }
625    reg.list_prefetch_misses_total
626        .fetch_add(1, Ordering::Relaxed);
627}
628
629pub fn inc_read_isolation_dispatch() {
630    let reg = metrics();
631    if !reg.enabled() {
632        return;
633    }
634    reg.read_isolation_dispatch_total
635        .fetch_add(1, Ordering::Relaxed);
636}
637
638pub fn inc_read_isolation_failures() {
639    let reg = metrics();
640    if !reg.enabled() {
641        return;
642    }
643    reg.read_isolation_failures_total
644        .fetch_add(1, Ordering::Relaxed);
645}
646
647pub fn inc_read_quorum_checks() {
648    let reg = metrics();
649    if !reg.enabled() {
650        return;
651    }
652    reg.read_quorum_checks_total.fetch_add(1, Ordering::Relaxed);
653}
654
655pub fn inc_read_quorum_failures() {
656    let reg = metrics();
657    if !reg.enabled() {
658        return;
659    }
660    reg.read_quorum_failures_total
661        .fetch_add(1, Ordering::Relaxed);
662}
663
664pub fn inc_read_quorum_internal() {
665    let reg = metrics();
666    if !reg.enabled() {
667        return;
668    }
669    reg.read_quorum_internal_total
670        .fetch_add(1, Ordering::Relaxed);
671}
672
673pub fn inc_transport_closed_conn_unavailable() {
674    let reg = metrics();
675    if !reg.enabled() {
676        return;
677    }
678    reg.transport_closed_conn_unavailable_total
679        .fetch_add(1, Ordering::Relaxed);
680}
681
682pub fn inc_forward_retry_attempts() {
683    let reg = metrics();
684    if !reg.enabled() {
685        return;
686    }
687    reg.forward_retry_attempts_total
688        .fetch_add(1, Ordering::Relaxed);
689}
690
691pub fn inc_gateway_read_ticket_hits() {
692    let reg = metrics();
693    if !reg.enabled() {
694        return;
695    }
696    reg.gateway_read_ticket_hits_total
697        .fetch_add(1, Ordering::Relaxed);
698}
699
700pub fn inc_gateway_read_ticket_misses() {
701    let reg = metrics();
702    if !reg.enabled() {
703        return;
704    }
705    reg.gateway_read_ticket_misses_total
706        .fetch_add(1, Ordering::Relaxed);
707}
708
709pub fn inc_gateway_singleflight_leader() {
710    let reg = metrics();
711    if !reg.enabled() {
712        return;
713    }
714    reg.gateway_singleflight_leader_total
715        .fetch_add(1, Ordering::Relaxed);
716}
717
718pub fn inc_gateway_singleflight_waiter() {
719    let reg = metrics();
720    if !reg.enabled() {
721        return;
722    }
723    reg.gateway_singleflight_waiter_total
724        .fetch_add(1, Ordering::Relaxed);
725}
726
727pub fn inc_gateway_singleflight_overflow() {
728    let reg = metrics();
729    if !reg.enabled() {
730        return;
731    }
732    reg.gateway_singleflight_overflow_total
733        .fetch_add(1, Ordering::Relaxed);
734}
735
736pub fn inc_gateway_singleflight_waiter_timeout() {
737    let reg = metrics();
738    if !reg.enabled() {
739        return;
740    }
741    reg.gateway_singleflight_waiter_timeouts_total
742        .fetch_add(1, Ordering::Relaxed);
743}
744
745pub fn inc_request_get_total() {
746    let reg = metrics();
747    if !reg.enabled() {
748        return;
749    }
750    reg.request_get_total.fetch_add(1, Ordering::Relaxed);
751}
752
753pub fn inc_request_list_total() {
754    let reg = metrics();
755    if !reg.enabled() {
756        return;
757    }
758    reg.request_list_total.fetch_add(1, Ordering::Relaxed);
759}
760
761pub fn inc_request_put_total() {
762    let reg = metrics();
763    if !reg.enabled() {
764        return;
765    }
766    reg.request_put_total.fetch_add(1, Ordering::Relaxed);
767}
768
769pub fn inc_request_delete_total() {
770    let reg = metrics();
771    if !reg.enabled() {
772        return;
773    }
774    reg.request_delete_total.fetch_add(1, Ordering::Relaxed);
775}
776
777pub fn inc_request_txn_total() {
778    let reg = metrics();
779    if !reg.enabled() {
780        return;
781    }
782    reg.request_txn_total.fetch_add(1, Ordering::Relaxed);
783}
784
785pub fn inc_request_lease_total() {
786    let reg = metrics();
787    if !reg.enabled() {
788        return;
789    }
790    reg.request_lease_total.fetch_add(1, Ordering::Relaxed);
791}
792
793pub fn inc_request_watch_total() {
794    let reg = metrics();
795    if !reg.enabled() {
796        return;
797    }
798    reg.request_watch_total.fetch_add(1, Ordering::Relaxed);
799}
800
801pub fn inc_request_tier0_total() {
802    let reg = metrics();
803    if !reg.enabled() {
804        return;
805    }
806    reg.request_tier0_total.fetch_add(1, Ordering::Relaxed);
807}
808
809pub fn inc_profile_switch_total() {
810    let reg = metrics();
811    if !reg.enabled() {
812        return;
813    }
814    reg.profile_switch_total.fetch_add(1, Ordering::Relaxed);
815}
816
817pub fn set_profile_active(profile: &str) {
818    let reg = metrics();
819    if !reg.enabled() {
820        return;
821    }
822    let (k8s, omni, gateway, auto) = match profile {
823        "kubernetes" => (1, 0, 0, 0),
824        "omni" => (0, 1, 0, 0),
825        "gateway" => (0, 0, 1, 0),
826        _ => (0, 0, 0, 1),
827    };
828    reg.profile_active_kubernetes.store(k8s, Ordering::Relaxed);
829    reg.profile_active_omni.store(omni, Ordering::Relaxed);
830    reg.profile_active_gateway.store(gateway, Ordering::Relaxed);
831    reg.profile_active_auto.store(auto, Ordering::Relaxed);
832}
833
834pub fn set_profile_applied_put_max_requests(value: usize) {
835    let reg = metrics();
836    if !reg.enabled() {
837        return;
838    }
839    reg.profile_applied_put_max_requests
840        .store(value as u64, Ordering::Relaxed);
841}
842
843pub fn set_profile_applied_put_linger_us(value: u64) {
844    let reg = metrics();
845    if !reg.enabled() {
846        return;
847    }
848    reg.profile_applied_put_linger_us
849        .store(value, Ordering::Relaxed);
850}
851
852pub fn set_profile_applied_prefetch_entries(value: usize) {
853    let reg = metrics();
854    if !reg.enabled() {
855        return;
856    }
857    reg.profile_applied_prefetch_entries
858        .store(value as u64, Ordering::Relaxed);
859}
860
861pub fn set_profile_applied_bg_io_tokens_per_sec(value: u64) {
862    let reg = metrics();
863    if !reg.enabled() {
864        return;
865    }
866    reg.profile_applied_bg_io_tokens_per_sec
867        .store(value, Ordering::Relaxed);
868}
869
870pub fn latest_put_quorum_ack_ms() -> Option<u64> {
871    let reg = metrics();
872    if !reg.enabled() {
873        return None;
874    }
875    if reg.quorum_ack_observations.load(Ordering::Relaxed) == 0 {
876        None
877    } else {
878        Some(reg.latest_quorum_ack_ms.load(Ordering::Relaxed))
879    }
880}
881
882pub fn render_prometheus() -> String {
883    let reg = metrics();
884    let mut out = String::new();
885    if !reg.enabled() {
886        let _ = writeln!(&mut out, "# astra metrics disabled");
887        return out;
888    }
889
890    reg.put_queue_wait_ms.render(
891        &mut out,
892        "astra_put_queue_wait_seconds",
893        "Queue wait time between enqueue and batch dispatch for put requests.",
894    );
895    reg.put_queue_wait_tier0_ms.render(
896        &mut out,
897        "astra_put_queue_wait_tier0_seconds",
898        "Queue wait time between enqueue and batch dispatch for tier0 put requests.",
899    );
900    reg.put_queue_wait_normal_ms.render(
901        &mut out,
902        "astra_put_queue_wait_normal_seconds",
903        "Queue wait time between enqueue and batch dispatch for normal put requests.",
904    );
905    reg.put_raft_write_ms.render(
906        &mut out,
907        "astra_put_raft_client_write_seconds",
908        "Duration of raft client_write for put batches.",
909    );
910    reg.put_quorum_ack_ms.render(
911        &mut out,
912        "astra_put_quorum_ack_seconds",
913        "Time from batch submit to commit advancement signal.",
914    );
915    reg.put_batch_size.render(
916        &mut out,
917        "astra_put_batch_size",
918        "Number of put operations per dispatched batch.",
919    );
920    reg.bg_io_throttle_wait_ms.render(
921        &mut out,
922        "astra_bg_io_throttle_wait_seconds",
923        "Time spent waiting for background IO token-bucket budget.",
924    );
925    reg.bg_io_sqe_wait_ms.render(
926        &mut out,
927        "astra_bg_io_sqe_wait_seconds",
928        "Time spent waiting for background SQE operation budget.",
929    );
930    reg.write_stall_delay_ms.render(
931        &mut out,
932        "astra_write_stall_delay_seconds",
933        "Write stall delay injected by L0 pressure controller.",
934    );
935
936    let _ = writeln!(
937        &mut out,
938        "# HELP astra_put_batches_total Total dispatched put batches."
939    );
940    let _ = writeln!(&mut out, "# TYPE astra_put_batches_total counter");
941    let _ = writeln!(
942        &mut out,
943        "astra_put_batches_total {}",
944        reg.put_batches_total.load(Ordering::Relaxed)
945    );
946
947    let _ = writeln!(
948        &mut out,
949        "# HELP astra_put_batch_requests_total Total put requests dispatched through the batcher."
950    );
951    let _ = writeln!(&mut out, "# TYPE astra_put_batch_requests_total counter");
952    let _ = writeln!(
953        &mut out,
954        "astra_put_batch_requests_total {}",
955        reg.put_batch_requests_total.load(Ordering::Relaxed)
956    );
957    let _ = writeln!(
958        &mut out,
959        "# HELP astra_put_tier0_enqueued_total Total tier0 put requests enqueued."
960    );
961    let _ = writeln!(&mut out, "# TYPE astra_put_tier0_enqueued_total counter");
962    let _ = writeln!(
963        &mut out,
964        "astra_put_tier0_enqueued_total {}",
965        reg.put_tier0_enqueued_total.load(Ordering::Relaxed)
966    );
967    let _ = writeln!(
968        &mut out,
969        "# HELP astra_put_tier0_dispatch_total Total tier0 batches dispatched."
970    );
971    let _ = writeln!(&mut out, "# TYPE astra_put_tier0_dispatch_total counter");
972    let _ = writeln!(
973        &mut out,
974        "astra_put_tier0_dispatch_total {}",
975        reg.put_tier0_dispatch_total.load(Ordering::Relaxed)
976    );
977    let _ = writeln!(
978        &mut out,
979        "# HELP astra_put_tier0_bypass_write_pressure_total Total tier0 requests bypassing write-pressure control."
980    );
981    let _ = writeln!(
982        &mut out,
983        "# TYPE astra_put_tier0_bypass_write_pressure_total counter"
984    );
985    let _ = writeln!(
986        &mut out,
987        "astra_put_tier0_bypass_write_pressure_total {}",
988        reg.put_tier0_bypass_write_pressure_total
989            .load(Ordering::Relaxed)
990    );
991
992    let _ = writeln!(
993        &mut out,
994        "# HELP astra_put_inflight_requests Current estimated in-flight put requests (queued + dispatched)."
995    );
996    let _ = writeln!(&mut out, "# TYPE astra_put_inflight_requests gauge");
997    let _ = writeln!(
998        &mut out,
999        "astra_put_inflight_requests {}",
1000        reg.put_inflight_requests.load(Ordering::Relaxed)
1001    );
1002    let _ = writeln!(
1003        &mut out,
1004        "# HELP astra_put_tier0_queue_depth Current estimated queued tier0 put requests."
1005    );
1006    let _ = writeln!(&mut out, "# TYPE astra_put_tier0_queue_depth gauge");
1007    let _ = writeln!(
1008        &mut out,
1009        "astra_put_tier0_queue_depth {}",
1010        reg.put_tier0_queue_depth.load(Ordering::Relaxed)
1011    );
1012    let _ = writeln!(
1013        &mut out,
1014        "# HELP astra_put_normal_queue_depth Current estimated queued normal put requests."
1015    );
1016    let _ = writeln!(&mut out, "# TYPE astra_put_normal_queue_depth gauge");
1017    let _ = writeln!(
1018        &mut out,
1019        "astra_put_normal_queue_depth {}",
1020        reg.put_normal_queue_depth.load(Ordering::Relaxed)
1021    );
1022
1023    let _ = writeln!(
1024        &mut out,
1025        "# HELP astra_wal_effective_linger_microseconds Effective WAL linger currently applied."
1026    );
1027    let _ = writeln!(
1028        &mut out,
1029        "# TYPE astra_wal_effective_linger_microseconds gauge"
1030    );
1031    let _ = writeln!(
1032        &mut out,
1033        "astra_wal_effective_linger_microseconds {}",
1034        reg.wal_effective_linger_us.load(Ordering::Relaxed)
1035    );
1036
1037    let _ = writeln!(
1038        &mut out,
1039        "# HELP astra_wal_queue_depth Current queued WAL frames waiting for flush."
1040    );
1041    let _ = writeln!(&mut out, "# TYPE astra_wal_queue_depth gauge");
1042    let _ = writeln!(
1043        &mut out,
1044        "astra_wal_queue_depth {}",
1045        reg.wal_queue_depth.load(Ordering::Relaxed)
1046    );
1047
1048    let _ = writeln!(
1049        &mut out,
1050        "# HELP astra_wal_queue_bytes Current queued WAL payload bytes waiting for flush."
1051    );
1052    let _ = writeln!(&mut out, "# TYPE astra_wal_queue_bytes gauge");
1053    let _ = writeln!(
1054        &mut out,
1055        "astra_wal_queue_bytes {}",
1056        reg.wal_queue_bytes.load(Ordering::Relaxed)
1057    );
1058
1059    let _ = writeln!(
1060        &mut out,
1061        "# HELP astra_lsm_synth_l0_files Synthetic L0 file pressure estimate derived from WAL queues."
1062    );
1063    let _ = writeln!(&mut out, "# TYPE astra_lsm_synth_l0_files gauge");
1064    let _ = writeln!(
1065        &mut out,
1066        "astra_lsm_synth_l0_files {}",
1067        reg.lsm_synth_l0_files.load(Ordering::Relaxed)
1068    );
1069
1070    let _ = writeln!(
1071        &mut out,
1072        "# HELP astra_bg_io_tokens_available Current available tokens in the background IO token bucket."
1073    );
1074    let _ = writeln!(&mut out, "# TYPE astra_bg_io_tokens_available gauge");
1075    let _ = writeln!(
1076        &mut out,
1077        "astra_bg_io_tokens_available {}",
1078        reg.bg_io_tokens_available.load(Ordering::Relaxed)
1079    );
1080
1081    let _ = writeln!(
1082        &mut out,
1083        "# HELP astra_bg_io_sqe_tokens_available Current available tokens in the background SQE token bucket."
1084    );
1085    let _ = writeln!(&mut out, "# TYPE astra_bg_io_sqe_tokens_available gauge");
1086    let _ = writeln!(
1087        &mut out,
1088        "astra_bg_io_sqe_tokens_available {}",
1089        reg.bg_io_sqe_tokens_available.load(Ordering::Relaxed)
1090    );
1091
1092    let _ = writeln!(
1093        &mut out,
1094        "# HELP astra_bg_io_throttle_events_total Total background IO throttle wait events."
1095    );
1096    let _ = writeln!(&mut out, "# TYPE astra_bg_io_throttle_events_total counter");
1097    let _ = writeln!(
1098        &mut out,
1099        "astra_bg_io_throttle_events_total {}",
1100        reg.bg_io_throttle_events_total.load(Ordering::Relaxed)
1101    );
1102
1103    let _ = writeln!(
1104        &mut out,
1105        "# HELP astra_bg_io_sqe_throttle_events_total Total background SQE throttle wait events."
1106    );
1107    let _ = writeln!(
1108        &mut out,
1109        "# TYPE astra_bg_io_sqe_throttle_events_total counter"
1110    );
1111    let _ = writeln!(
1112        &mut out,
1113        "astra_bg_io_sqe_throttle_events_total {}",
1114        reg.bg_io_sqe_throttle_events_total.load(Ordering::Relaxed)
1115    );
1116
1117    let _ = writeln!(
1118        &mut out,
1119        "# HELP astra_write_stall_events_total Total write requests delayed by L0 pressure control."
1120    );
1121    let _ = writeln!(&mut out, "# TYPE astra_write_stall_events_total counter");
1122    let _ = writeln!(
1123        &mut out,
1124        "astra_write_stall_events_total {}",
1125        reg.write_stall_events_total.load(Ordering::Relaxed)
1126    );
1127
1128    let _ = writeln!(
1129        &mut out,
1130        "# HELP astra_write_reject_events_total Total write requests rejected by L0 pressure control."
1131    );
1132    let _ = writeln!(&mut out, "# TYPE astra_write_reject_events_total counter");
1133    let _ = writeln!(
1134        &mut out,
1135        "astra_write_reject_events_total {}",
1136        reg.write_reject_events_total.load(Ordering::Relaxed)
1137    );
1138
1139    let _ = writeln!(
1140        &mut out,
1141        "# HELP astra_write_stall_band_5_events_total Total writes delayed by the L0==5 pressure band."
1142    );
1143    let _ = writeln!(
1144        &mut out,
1145        "# TYPE astra_write_stall_band_5_events_total counter"
1146    );
1147    let _ = writeln!(
1148        &mut out,
1149        "astra_write_stall_band_5_events_total {}",
1150        reg.write_stall_band_5_events_total.load(Ordering::Relaxed)
1151    );
1152
1153    let _ = writeln!(
1154        &mut out,
1155        "# HELP astra_write_stall_band_6_events_total Total writes delayed by the L0==6 pressure band."
1156    );
1157    let _ = writeln!(
1158        &mut out,
1159        "# TYPE astra_write_stall_band_6_events_total counter"
1160    );
1161    let _ = writeln!(
1162        &mut out,
1163        "astra_write_stall_band_6_events_total {}",
1164        reg.write_stall_band_6_events_total.load(Ordering::Relaxed)
1165    );
1166
1167    let _ = writeln!(
1168        &mut out,
1169        "# HELP astra_write_stall_band_7_plus_events_total Total writes delayed by the L0>=7 pressure band."
1170    );
1171    let _ = writeln!(
1172        &mut out,
1173        "# TYPE astra_write_stall_band_7_plus_events_total counter"
1174    );
1175    let _ = writeln!(
1176        &mut out,
1177        "astra_write_stall_band_7_plus_events_total {}",
1178        reg.write_stall_band_7_plus_events_total
1179            .load(Ordering::Relaxed)
1180    );
1181
1182    let _ = writeln!(
1183        &mut out,
1184        "# HELP astra_list_prefix_filter_skips_total LIST requests skipped by prefix negative filter."
1185    );
1186    let _ = writeln!(
1187        &mut out,
1188        "# TYPE astra_list_prefix_filter_skips_total counter"
1189    );
1190    let _ = writeln!(
1191        &mut out,
1192        "astra_list_prefix_filter_skips_total {}",
1193        reg.list_prefix_filter_skips_total.load(Ordering::Relaxed)
1194    );
1195
1196    let _ = writeln!(
1197        &mut out,
1198        "# HELP astra_list_prefix_filter_hits_total LIST requests processed after prefix filter positive hit."
1199    );
1200    let _ = writeln!(
1201        &mut out,
1202        "# TYPE astra_list_prefix_filter_hits_total counter"
1203    );
1204    let _ = writeln!(
1205        &mut out,
1206        "astra_list_prefix_filter_hits_total {}",
1207        reg.list_prefix_filter_hits_total.load(Ordering::Relaxed)
1208    );
1209
1210    let _ = writeln!(
1211        &mut out,
1212        "# HELP astra_list_revision_filter_skips_total LIST requests skipped by prefix+revision filter."
1213    );
1214    let _ = writeln!(
1215        &mut out,
1216        "# TYPE astra_list_revision_filter_skips_total counter"
1217    );
1218    let _ = writeln!(
1219        &mut out,
1220        "astra_list_revision_filter_skips_total {}",
1221        reg.list_revision_filter_skips_total.load(Ordering::Relaxed)
1222    );
1223
1224    let _ = writeln!(
1225        &mut out,
1226        "# HELP astra_list_revision_filter_hits_total LIST requests processed after prefix+revision filter hit."
1227    );
1228    let _ = writeln!(
1229        &mut out,
1230        "# TYPE astra_list_revision_filter_hits_total counter"
1231    );
1232    let _ = writeln!(
1233        &mut out,
1234        "astra_list_revision_filter_hits_total {}",
1235        reg.list_revision_filter_hits_total.load(Ordering::Relaxed)
1236    );
1237
1238    let _ = writeln!(
1239        &mut out,
1240        "# HELP astra_list_prefetch_hits_total LIST prefetch cache hits."
1241    );
1242    let _ = writeln!(&mut out, "# TYPE astra_list_prefetch_hits_total counter");
1243    let _ = writeln!(
1244        &mut out,
1245        "astra_list_prefetch_hits_total {}",
1246        reg.list_prefetch_hits_total.load(Ordering::Relaxed)
1247    );
1248
1249    let _ = writeln!(
1250        &mut out,
1251        "# HELP astra_list_prefetch_misses_total LIST prefetch cache misses."
1252    );
1253    let _ = writeln!(&mut out, "# TYPE astra_list_prefetch_misses_total counter");
1254    let _ = writeln!(
1255        &mut out,
1256        "astra_list_prefetch_misses_total {}",
1257        reg.list_prefetch_misses_total.load(Ordering::Relaxed)
1258    );
1259
1260    let _ = writeln!(
1261        &mut out,
1262        "# HELP astra_read_isolation_dispatch_total Total read requests routed through isolated read lane."
1263    );
1264    let _ = writeln!(
1265        &mut out,
1266        "# TYPE astra_read_isolation_dispatch_total counter"
1267    );
1268    let _ = writeln!(
1269        &mut out,
1270        "astra_read_isolation_dispatch_total {}",
1271        reg.read_isolation_dispatch_total.load(Ordering::Relaxed)
1272    );
1273
1274    let _ = writeln!(
1275        &mut out,
1276        "# HELP astra_read_isolation_failures_total Total isolated read dispatch failures that required inline fallback."
1277    );
1278    let _ = writeln!(
1279        &mut out,
1280        "# TYPE astra_read_isolation_failures_total counter"
1281    );
1282    let _ = writeln!(
1283        &mut out,
1284        "astra_read_isolation_failures_total {}",
1285        reg.read_isolation_failures_total.load(Ordering::Relaxed)
1286    );
1287    let _ = writeln!(
1288        &mut out,
1289        "# HELP astra_read_quorum_checks_total Total linearizable read quorum checks attempted."
1290    );
1291    let _ = writeln!(&mut out, "# TYPE astra_read_quorum_checks_total counter");
1292    let _ = writeln!(
1293        &mut out,
1294        "astra_read_quorum_checks_total {}",
1295        reg.read_quorum_checks_total.load(Ordering::Relaxed)
1296    );
1297    let _ = writeln!(
1298        &mut out,
1299        "# HELP astra_read_quorum_failures_total Total linearizable read quorum check failures."
1300    );
1301    let _ = writeln!(&mut out, "# TYPE astra_read_quorum_failures_total counter");
1302    let _ = writeln!(
1303        &mut out,
1304        "astra_read_quorum_failures_total {}",
1305        reg.read_quorum_failures_total.load(Ordering::Relaxed)
1306    );
1307    let _ = writeln!(
1308        &mut out,
1309        "# HELP astra_read_quorum_internal_total Total quorum-related internal read errors."
1310    );
1311    let _ = writeln!(&mut out, "# TYPE astra_read_quorum_internal_total counter");
1312    let _ = writeln!(
1313        &mut out,
1314        "astra_read_quorum_internal_total {}",
1315        reg.read_quorum_internal_total.load(Ordering::Relaxed)
1316    );
1317    let _ = writeln!(
1318        &mut out,
1319        "# HELP astra_transport_closed_conn_unavailable_total Total unavailable responses attributed to closed transport connections."
1320    );
1321    let _ = writeln!(
1322        &mut out,
1323        "# TYPE astra_transport_closed_conn_unavailable_total counter"
1324    );
1325    let _ = writeln!(
1326        &mut out,
1327        "astra_transport_closed_conn_unavailable_total {}",
1328        reg.transport_closed_conn_unavailable_total
1329            .load(Ordering::Relaxed)
1330    );
1331    let _ = writeln!(
1332        &mut out,
1333        "# HELP astra_forward_retry_attempts_total Total forwarded RPC retry attempts."
1334    );
1335    let _ = writeln!(
1336        &mut out,
1337        "# TYPE astra_forward_retry_attempts_total counter"
1338    );
1339    let _ = writeln!(
1340        &mut out,
1341        "astra_forward_retry_attempts_total {}",
1342        reg.forward_retry_attempts_total.load(Ordering::Relaxed)
1343    );
1344    let _ = writeln!(
1345        &mut out,
1346        "# HELP astra_gateway_read_ticket_hits_total Total gateway read requests that reused an active read ticket."
1347    );
1348    let _ = writeln!(
1349        &mut out,
1350        "# TYPE astra_gateway_read_ticket_hits_total counter"
1351    );
1352    let _ = writeln!(
1353        &mut out,
1354        "astra_gateway_read_ticket_hits_total {}",
1355        reg.gateway_read_ticket_hits_total.load(Ordering::Relaxed)
1356    );
1357    let _ = writeln!(
1358        &mut out,
1359        "# HELP astra_gateway_read_ticket_misses_total Total gateway read requests that required a fresh quorum check."
1360    );
1361    let _ = writeln!(
1362        &mut out,
1363        "# TYPE astra_gateway_read_ticket_misses_total counter"
1364    );
1365    let _ = writeln!(
1366        &mut out,
1367        "astra_gateway_read_ticket_misses_total {}",
1368        reg.gateway_read_ticket_misses_total.load(Ordering::Relaxed)
1369    );
1370    let _ = writeln!(
1371        &mut out,
1372        "# HELP astra_gateway_singleflight_leader_total Total gateway read requests elected as singleflight leaders."
1373    );
1374    let _ = writeln!(
1375        &mut out,
1376        "# TYPE astra_gateway_singleflight_leader_total counter"
1377    );
1378    let _ = writeln!(
1379        &mut out,
1380        "astra_gateway_singleflight_leader_total {}",
1381        reg.gateway_singleflight_leader_total
1382            .load(Ordering::Relaxed)
1383    );
1384    let _ = writeln!(
1385        &mut out,
1386        "# HELP astra_gateway_singleflight_waiter_total Total gateway read requests collapsed behind an in-flight leader request."
1387    );
1388    let _ = writeln!(
1389        &mut out,
1390        "# TYPE astra_gateway_singleflight_waiter_total counter"
1391    );
1392    let _ = writeln!(
1393        &mut out,
1394        "astra_gateway_singleflight_waiter_total {}",
1395        reg.gateway_singleflight_waiter_total
1396            .load(Ordering::Relaxed)
1397    );
1398    let _ = writeln!(
1399        &mut out,
1400        "# HELP astra_gateway_singleflight_overflow_total Total gateway read requests that bypassed singleflight due to waiter cap."
1401    );
1402    let _ = writeln!(
1403        &mut out,
1404        "# TYPE astra_gateway_singleflight_overflow_total counter"
1405    );
1406    let _ = writeln!(
1407        &mut out,
1408        "astra_gateway_singleflight_overflow_total {}",
1409        reg.gateway_singleflight_overflow_total
1410            .load(Ordering::Relaxed)
1411    );
1412    let _ = writeln!(
1413        &mut out,
1414        "# HELP astra_gateway_singleflight_waiter_timeouts_total Total gateway waiter requests that timed out waiting for singleflight leader completion."
1415    );
1416    let _ = writeln!(
1417        &mut out,
1418        "# TYPE astra_gateway_singleflight_waiter_timeouts_total counter"
1419    );
1420    let _ = writeln!(
1421        &mut out,
1422        "astra_gateway_singleflight_waiter_timeouts_total {}",
1423        reg.gateway_singleflight_waiter_timeouts_total
1424            .load(Ordering::Relaxed)
1425    );
1426
1427    let _ = writeln!(
1428        &mut out,
1429        "# HELP astra_request_get_total Total GET-like range requests."
1430    );
1431    let _ = writeln!(&mut out, "# TYPE astra_request_get_total counter");
1432    let _ = writeln!(
1433        &mut out,
1434        "astra_request_get_total {}",
1435        reg.request_get_total.load(Ordering::Relaxed)
1436    );
1437    let _ = writeln!(
1438        &mut out,
1439        "# HELP astra_request_list_total Total LIST/prefix range requests."
1440    );
1441    let _ = writeln!(&mut out, "# TYPE astra_request_list_total counter");
1442    let _ = writeln!(
1443        &mut out,
1444        "astra_request_list_total {}",
1445        reg.request_list_total.load(Ordering::Relaxed)
1446    );
1447    let _ = writeln!(
1448        &mut out,
1449        "# HELP astra_request_put_total Total put requests."
1450    );
1451    let _ = writeln!(&mut out, "# TYPE astra_request_put_total counter");
1452    let _ = writeln!(
1453        &mut out,
1454        "astra_request_put_total {}",
1455        reg.request_put_total.load(Ordering::Relaxed)
1456    );
1457    let _ = writeln!(
1458        &mut out,
1459        "# HELP astra_request_delete_total Total delete range requests."
1460    );
1461    let _ = writeln!(&mut out, "# TYPE astra_request_delete_total counter");
1462    let _ = writeln!(
1463        &mut out,
1464        "astra_request_delete_total {}",
1465        reg.request_delete_total.load(Ordering::Relaxed)
1466    );
1467    let _ = writeln!(
1468        &mut out,
1469        "# HELP astra_request_txn_total Total txn requests."
1470    );
1471    let _ = writeln!(&mut out, "# TYPE astra_request_txn_total counter");
1472    let _ = writeln!(
1473        &mut out,
1474        "astra_request_txn_total {}",
1475        reg.request_txn_total.load(Ordering::Relaxed)
1476    );
1477    let _ = writeln!(
1478        &mut out,
1479        "# HELP astra_request_lease_total Total lease API requests."
1480    );
1481    let _ = writeln!(&mut out, "# TYPE astra_request_lease_total counter");
1482    let _ = writeln!(
1483        &mut out,
1484        "astra_request_lease_total {}",
1485        reg.request_lease_total.load(Ordering::Relaxed)
1486    );
1487    let _ = writeln!(
1488        &mut out,
1489        "# HELP astra_request_watch_total Total watch stream requests."
1490    );
1491    let _ = writeln!(&mut out, "# TYPE astra_request_watch_total counter");
1492    let _ = writeln!(
1493        &mut out,
1494        "astra_request_watch_total {}",
1495        reg.request_watch_total.load(Ordering::Relaxed)
1496    );
1497    let _ = writeln!(
1498        &mut out,
1499        "# HELP astra_request_tier0_total Total requests classified as tier0 critical."
1500    );
1501    let _ = writeln!(&mut out, "# TYPE astra_request_tier0_total counter");
1502    let _ = writeln!(
1503        &mut out,
1504        "astra_request_tier0_total {}",
1505        reg.request_tier0_total.load(Ordering::Relaxed)
1506    );
1507
1508    let _ = writeln!(
1509        &mut out,
1510        "# HELP astra_profile_switch_total Total profile transitions applied by the governor."
1511    );
1512    let _ = writeln!(&mut out, "# TYPE astra_profile_switch_total counter");
1513    let _ = writeln!(
1514        &mut out,
1515        "astra_profile_switch_total {}",
1516        reg.profile_switch_total.load(Ordering::Relaxed)
1517    );
1518    let _ = writeln!(
1519        &mut out,
1520        "# HELP astra_profile_active_kubernetes Active profile gauge (1 when active)."
1521    );
1522    let _ = writeln!(&mut out, "# TYPE astra_profile_active_kubernetes gauge");
1523    let _ = writeln!(
1524        &mut out,
1525        "astra_profile_active_kubernetes {}",
1526        reg.profile_active_kubernetes.load(Ordering::Relaxed)
1527    );
1528    let _ = writeln!(
1529        &mut out,
1530        "# HELP astra_profile_active_omni Active profile gauge (1 when active)."
1531    );
1532    let _ = writeln!(&mut out, "# TYPE astra_profile_active_omni gauge");
1533    let _ = writeln!(
1534        &mut out,
1535        "astra_profile_active_omni {}",
1536        reg.profile_active_omni.load(Ordering::Relaxed)
1537    );
1538    let _ = writeln!(
1539        &mut out,
1540        "# HELP astra_profile_active_gateway Active profile gauge (1 when active)."
1541    );
1542    let _ = writeln!(&mut out, "# TYPE astra_profile_active_gateway gauge");
1543    let _ = writeln!(
1544        &mut out,
1545        "astra_profile_active_gateway {}",
1546        reg.profile_active_gateway.load(Ordering::Relaxed)
1547    );
1548    let _ = writeln!(
1549        &mut out,
1550        "# HELP astra_profile_active_auto Active profile gauge (1 when active)."
1551    );
1552    let _ = writeln!(&mut out, "# TYPE astra_profile_active_auto gauge");
1553    let _ = writeln!(
1554        &mut out,
1555        "astra_profile_active_auto {}",
1556        reg.profile_active_auto.load(Ordering::Relaxed)
1557    );
1558    let _ = writeln!(
1559        &mut out,
1560        "# HELP astra_profile_applied_put_max_requests Last profile-applied put max_requests."
1561    );
1562    let _ = writeln!(
1563        &mut out,
1564        "# TYPE astra_profile_applied_put_max_requests gauge"
1565    );
1566    let _ = writeln!(
1567        &mut out,
1568        "astra_profile_applied_put_max_requests {}",
1569        reg.profile_applied_put_max_requests.load(Ordering::Relaxed)
1570    );
1571    let _ = writeln!(
1572        &mut out,
1573        "# HELP astra_profile_applied_put_linger_microseconds Last profile-applied put linger in microseconds."
1574    );
1575    let _ = writeln!(
1576        &mut out,
1577        "# TYPE astra_profile_applied_put_linger_microseconds gauge"
1578    );
1579    let _ = writeln!(
1580        &mut out,
1581        "astra_profile_applied_put_linger_microseconds {}",
1582        reg.profile_applied_put_linger_us.load(Ordering::Relaxed)
1583    );
1584    let _ = writeln!(
1585        &mut out,
1586        "# HELP astra_profile_applied_prefetch_entries Last profile-applied list prefetch cache entry cap."
1587    );
1588    let _ = writeln!(
1589        &mut out,
1590        "# TYPE astra_profile_applied_prefetch_entries gauge"
1591    );
1592    let _ = writeln!(
1593        &mut out,
1594        "astra_profile_applied_prefetch_entries {}",
1595        reg.profile_applied_prefetch_entries.load(Ordering::Relaxed)
1596    );
1597    let _ = writeln!(
1598        &mut out,
1599        "# HELP astra_profile_applied_bg_io_tokens_per_sec Last profile-applied background IO token rate."
1600    );
1601    let _ = writeln!(
1602        &mut out,
1603        "# TYPE astra_profile_applied_bg_io_tokens_per_sec gauge"
1604    );
1605    let _ = writeln!(
1606        &mut out,
1607        "astra_profile_applied_bg_io_tokens_per_sec {}",
1608        reg.profile_applied_bg_io_tokens_per_sec
1609            .load(Ordering::Relaxed)
1610    );
1611
1612    out
1613}
astra_core/metrics.rs

astra_core/
metrics.rs
