mako_engine/metrics.rs
1//! [`EngineMetrics`] — process-level event counters for Prometheus export.
2//!
3//! Provides a **process-global** set of [`std::sync::atomic::AtomicU64`]
4//! counters that the engine and domain handlers increment at runtime. The
5//! [`metrics_api`] handler reads them via [`EngineMetrics::global()`] without
6//! any I/O and renders them in Prometheus text format.
7//!
8//! ## Design rationale
9//!
10//! The mako-engine is a single-process daemon (`makod`). A process-global
11//! static is the simplest, lowest-overhead counter mechanism that:
12//!
13//! - requires **zero allocations** on the hot path (every command dispatch),
14//! - is **async-safe** (atomics need no async context),
15//! - imposes **no external dependency** (no `prometheus` crate in the engine),
16//! - is **observable** from `metrics_api` via a simple method call.
17//!
18//! The trade-off: counters reset on process restart (they are not persisted).
19//! For a single-process daemon this is acceptable — Prometheus's `rate()`
20//! function handles counter resets automatically.
21//!
22//! ## Usage
23//!
24//! ### Incrementing a counter
25//!
26//! ```rust
27//! use mako_engine::metrics::{EngineMetrics, ProcessOutcome};
28//!
29//! // In a workflow handle() or apply() implementation:
30//! EngineMetrics::global().process_initiated("gpke");
31//! EngineMetrics::global().process_completed("gpke", ProcessOutcome::Accepted);
32//! EngineMetrics::global().validation_failed("utilmd", "S2.1");
33//! ```
34//!
35//! ### Reading counters (metrics endpoint)
36//!
37//! ```rust,ignore
38//! let metrics = mako_engine::metrics::EngineMetrics::global();
39//! let snapshot = metrics.snapshot();
40//! // Render snapshot to Prometheus text format.
41//! ```
42//!
43//! [`metrics_api`]: https://docs.rs/makod
44
45use std::{
46 collections::HashMap,
47 sync::{
48 Arc, OnceLock,
49 atomic::{AtomicU64, Ordering},
50 },
51};
52
53// ── ProcessOutcome ────────────────────────────────────────────────────────────
54
55/// Terminal outcome of a MaKo process instance.
56///
57/// Used as the `result` label on [`EngineMetrics::process_completed`].
58#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
59pub enum ProcessOutcome {
60 /// The counterparty accepted the request (Bestätigung / positive APERAK).
61 Accepted,
62 /// The counterparty rejected the request (Ablehnung / negative APERAK).
63 Rejected,
64 /// The process timed out before a response arrived (24h / 5 WD / 10 WD).
65 Timeout,
66 /// The process was cancelled by the originating ERP before completion.
67 Cancelled,
68}
69
70impl ProcessOutcome {
71 /// Prometheus label value for this outcome.
72 #[must_use]
73 pub fn label(self) -> &'static str {
74 match self {
75 Self::Accepted => "accepted",
76 Self::Rejected => "rejected",
77 Self::Timeout => "timeout",
78 Self::Cancelled => "cancelled",
79 }
80 }
81
82 /// All variants in a fixed order, for metric exposition.
83 pub const ALL: &'static [Self] = &[
84 Self::Accepted,
85 Self::Rejected,
86 Self::Timeout,
87 Self::Cancelled,
88 ];
89}
90
91// ── MetricVec ─────────────────────────────────────────────────────────────────
92
93/// A map of label strings → `AtomicU64` counters.
94///
95/// `MetricVec` is append-only: new label combinations are registered on first
96/// increment and are never removed (counters remain at 0 once created).
97#[derive(Default)]
98struct MetricVec {
99 inner: std::sync::RwLock<HashMap<Box<str>, Arc<AtomicU64>>>,
100}
101
102impl MetricVec {
103 fn increment(&self, label: &str) {
104 // Fast path: label already registered — just increment.
105 {
106 let guard = self.inner.read().expect("MetricVec RwLock poisoned");
107 if let Some(counter) = guard.get(label) {
108 counter.fetch_add(1, Ordering::Relaxed);
109 return;
110 }
111 }
112 // Slow path: first increment for this label — register + increment.
113 let mut guard = self.inner.write().expect("MetricVec RwLock poisoned");
114 let counter = guard
115 .entry(label.into())
116 .or_insert_with(|| Arc::new(AtomicU64::new(0)));
117 counter.fetch_add(1, Ordering::Relaxed);
118 }
119
120 /// Snapshot all label → value pairs, sorted by label for deterministic output.
121 fn snapshot(&self) -> Vec<(Box<str>, u64)> {
122 let guard = self.inner.read().expect("MetricVec RwLock poisoned");
123 let mut pairs: Vec<(Box<str>, u64)> = guard
124 .iter()
125 .map(|(k, v)| (k.clone(), v.load(Ordering::Relaxed)))
126 .collect();
127 pairs.sort_unstable_by(|(a, _), (b, _)| a.cmp(b));
128 pairs
129 }
130}
131
132// ── EngineMetrics ─────────────────────────────────────────────────────────────
133
134/// Process-global engine metrics counters.
135///
136/// Access via [`EngineMetrics::global()`]. The global instance is initialised
137/// once on first access using [`OnceLock`] and lives for the process lifetime.
138///
139/// ## Counter naming (maps 1:1 to Prometheus metric names)
140///
141/// | Method | Prometheus metric | Labels |
142/// |---|---|---|
143/// | [`process_initiated`] | `makod_process_initiated_total` | `family` |
144/// | [`process_completed`] | `makod_process_completed_total` | `family`, `result` |
145/// | [`validation_failed`] | `makod_validation_failed_total` | `message_type`, `release` |
146/// | [`outbox_delivery_attempted`] | `makod_outbox_delivery_attempts_total` | `result` |
147/// | [`deadline_fired`] | `makod_deadline_fired_total` | `family` |
148/// | [`dead_letter_recorded`] | `makod_dead_letter_recorded_total` | `reason` |
149/// | [`aperak_missed`] | `makod_aperak_missed_total` | `label` |
150///
151/// For `makod_dead_letter_recorded_total`, the `reason` label is:
152/// - `unknown_pid:<N>` when `DeadLetterReason::UnknownPid { pid: N, .. }` — one label per
153/// distinct PID, enabling per-PID alerting
154/// - a short category string (`unknown_conversation`, `version_mismatch`, etc.)
155/// for all other reason variants
156///
157/// [`process_initiated`]: EngineMetrics::process_initiated
158/// [`process_completed`]: EngineMetrics::process_completed
159/// [`validation_failed`]: EngineMetrics::validation_failed
160/// [`outbox_delivery_attempted`]: EngineMetrics::outbox_delivery_attempted
161/// [`deadline_fired`]: EngineMetrics::deadline_fired
162/// [`dead_letter_recorded`]: EngineMetrics::dead_letter_recorded
163/// [`aperak_missed`]: EngineMetrics::aperak_missed
164pub struct EngineMetrics {
165 /// `makod_process_initiated_total{family}` — incremented when a new
166 /// process is spawned via `Process::execute(InitiateXxx)`.
167 process_initiated: MetricVec,
168
169 /// `makod_process_completed_total{family,result}` — incremented when a
170 /// process reaches a terminal state.
171 process_completed: MetricVec,
172
173 /// `makod_validation_failed_total{message_type,release}` — incremented
174 /// when an inbound EDIFACT message fails AHB validation.
175 validation_failed: MetricVec,
176
177 /// `makod_outbox_delivery_attempts_total{result}` — incremented by the
178 /// AS4 sender on every delivery attempt.
179 outbox_delivery_attempts: MetricVec,
180
181 /// `makod_deadline_fired_total{family}` — incremented when a deadline
182 /// scheduler fires a `TimeoutExpired` command.
183 deadline_fired: MetricVec,
184
185 /// `makod_dead_letter_recorded_total{reason}` — incremented when a message
186 /// is sent to the dead-letter sink.
187 dead_letter_recorded: MetricVec,
188
189 /// `makod_inbound_messages_total{pid,result}` — incremented for every
190 /// inbound EDIFACT message that enters the dispatch pipeline.
191 ///
192 /// - `pid`: the 5-digit EDIFACT Prüfidentifikator (e.g. `"55001"`)
193 /// - `result`: `"dispatched"`, `"skipped"`, or `"error"`
194 inbound_received: MetricVec,
195
196 /// `makod_aperak_missed_total{label}` — incremented when the deadline
197 /// scheduler fires an APERAK deadline after its due-at time has passed.
198 ///
199 /// A non-zero value means an APERAK was dispatched late; this is a
200 /// regulatory violation under APERAK AHB 1.0 §2.4.1 (Strom) / §2.3.1
201 /// (Gas). Alert on `makod_aperak_missed_total > 0` in Alertmanager.
202 aperak_missed: MetricVec,
203}
204
205impl EngineMetrics {
206 fn new() -> Self {
207 Self {
208 process_initiated: MetricVec::default(),
209 process_completed: MetricVec::default(),
210 validation_failed: MetricVec::default(),
211 outbox_delivery_attempts: MetricVec::default(),
212 deadline_fired: MetricVec::default(),
213 dead_letter_recorded: MetricVec::default(),
214 inbound_received: MetricVec::default(),
215 aperak_missed: MetricVec::default(),
216 }
217 }
218
219 /// Return the process-global [`EngineMetrics`] instance.
220 ///
221 /// The instance is initialised lazily on first call. Subsequent calls
222 /// return the same instance with zero allocation.
223 #[must_use]
224 pub fn global() -> &'static Self {
225 static GLOBAL: OnceLock<EngineMetrics> = OnceLock::new();
226 GLOBAL.get_or_init(Self::new)
227 }
228
229 // ── Increment methods ─────────────────────────────────────────────────────
230
231 /// Increment `makod_process_initiated_total{family=<family>}`.
232 ///
233 /// Call once when a domain workflow receives its first initiating command
234 /// (e.g. `LfAnmeldungCommand::InitiateAnmeldung`).
235 ///
236 /// `family` is the [`EngineModule::name`] value (`"gpke"`, `"wim"`, etc.).
237 ///
238 /// [`EngineModule::name`]: crate::builder::EngineModule::name
239 pub fn process_initiated(&self, family: &str) {
240 self.process_initiated.increment(family);
241 }
242
243 /// Increment `makod_process_completed_total{family=<family>,result=<result>}`.
244 ///
245 /// Call once when a workflow transitions to a **terminal state**
246 /// (`Active`, `Rejected`, timeout, or cancellation).
247 pub fn process_completed(&self, family: &str, outcome: ProcessOutcome) {
248 let label = format!("{family},{}", outcome.label());
249 self.process_completed.increment(&label);
250 }
251
252 /// Increment `makod_validation_failed_total{message_type=<type>,release=<rel>}`.
253 ///
254 /// Call when an inbound message fails `validate()` or `validate_against()`.
255 pub fn validation_failed(&self, message_type: &str, release: &str) {
256 let label = format!("{message_type},{release}");
257 self.validation_failed.increment(&label);
258 }
259
260 /// Increment `makod_outbox_delivery_attempts_total{result=<result>}`.
261 ///
262 /// Call in the AS4 sender after every delivery attempt.
263 /// `result` should be one of `"ok"`, `"transport_error"`, `"partner_unknown"`.
264 pub fn outbox_delivery_attempted(&self, result: &str) {
265 self.outbox_delivery_attempts.increment(result);
266 }
267
268 /// Increment `makod_deadline_fired_total{family=<family>}`.
269 ///
270 /// Call in the deadline scheduler when it dispatches a `TimeoutExpired`.
271 pub fn deadline_fired(&self, family: &str) {
272 self.deadline_fired.increment(family);
273 }
274
275 /// Increment `makod_dead_letter_recorded_total{reason=<reason>}`.
276 ///
277 /// Call in the dead-letter sink when `reject()` is invoked.
278 /// `reason` should match [`DeadLetterReason`]'s label string.
279 ///
280 /// [`DeadLetterReason`]: crate::dead_letter::DeadLetterReason
281 pub fn dead_letter_recorded(&self, reason: &str) {
282 self.dead_letter_recorded.increment(reason);
283 }
284
285 /// Increment `makod_aperak_missed_total{label=<label>}`.
286 ///
287 /// Call in the deadline scheduler when an APERAK deadline fires **after** its
288 /// `due_at` timestamp has already passed, indicating a late dispatch.
289 /// `label` should be the APERAK deadline label constant from `fristen::`
290 /// (e.g. `APERAK_STROM_WINDOW_LABEL`, `APERAK_GAS_FOLGEPROZESS_LABEL`).
291 pub fn aperak_missed(&self, label: &str) {
292 self.aperak_missed.increment(label);
293 }
294
295 /// Increment `makod_inbound_messages_total{pid=<pid>,result=<result>}`.
296 ///
297 /// Call once per inbound EDIFACT message after the dispatch pipeline
298 /// completes (whether it succeeded or failed).
299 ///
300 /// - `pid` — the EDIFACT Prüfidentifikator (e.g. `55001`)
301 /// - `result` — `"dispatched"`, `"skipped"`, or `"error"`
302 pub fn inbound_received(&self, pid: u32, result: &str) {
303 let label = format!("{pid},{result}");
304 self.inbound_received.increment(&label);
305 }
306
307 // ── Snapshot ──────────────────────────────────────────────────────────────
308
309 /// Return a snapshot of all counters as a [`MetricsSnapshot`].
310 ///
311 /// This is a **read-only** operation that does not reset any counters.
312 /// Counters are monotonically increasing; Prometheus's `rate()` handles
313 /// counter resets on process restart automatically.
314 #[must_use]
315 pub fn snapshot(&self) -> MetricsSnapshot {
316 MetricsSnapshot {
317 process_initiated: self.process_initiated.snapshot(),
318 process_completed: self.process_completed.snapshot(),
319 validation_failed: self.validation_failed.snapshot(),
320 outbox_delivery_attempts: self.outbox_delivery_attempts.snapshot(),
321 deadline_fired: self.deadline_fired.snapshot(),
322 dead_letter_recorded: self.dead_letter_recorded.snapshot(),
323 inbound_received: self.inbound_received.snapshot(),
324 aperak_missed: self.aperak_missed.snapshot(),
325 }
326 }
327}
328
329// ── MetricsSnapshot ───────────────────────────────────────────────────────────
330
331/// A point-in-time snapshot of all [`EngineMetrics`] counters.
332///
333/// Obtained via [`EngineMetrics::snapshot()`]. All fields are `Vec` of
334/// `(label, count)` pairs sorted by label for deterministic Prometheus output.
335///
336/// The `label` field uses a `","` separator for multi-label metrics
337/// (e.g. `"gpke,accepted"` for `{family="gpke",result="accepted"}`).
338/// The [`render_prometheus`] function splits them appropriately.
339///
340/// [`render_prometheus`]: MetricsSnapshot::render_prometheus
341#[derive(Debug, Clone)]
342pub struct MetricsSnapshot {
343 /// `(family, count)` pairs for `makod_process_initiated_total`.
344 pub process_initiated: Vec<(Box<str>, u64)>,
345 /// `("family,result", count)` pairs for `makod_process_completed_total`.
346 pub process_completed: Vec<(Box<str>, u64)>,
347 /// `("message_type,release", count)` pairs for `makod_validation_failed_total`.
348 pub validation_failed: Vec<(Box<str>, u64)>,
349 /// `(result, count)` pairs for `makod_outbox_delivery_attempts_total`.
350 pub outbox_delivery_attempts: Vec<(Box<str>, u64)>,
351 /// `(family, count)` pairs for `makod_deadline_fired_total`.
352 pub deadline_fired: Vec<(Box<str>, u64)>,
353 /// `(reason, count)` pairs for `makod_dead_letter_recorded_total`.
354 pub dead_letter_recorded: Vec<(Box<str>, u64)>,
355 /// `("pid,result", count)` pairs for `makod_inbound_messages_total`.
356 pub inbound_received: Vec<(Box<str>, u64)>,
357 /// `(label, count)` pairs for `makod_aperak_missed_total`.
358 pub aperak_missed: Vec<(Box<str>, u64)>,
359}
360
361impl MetricsSnapshot {
362 /// Render this snapshot to Prometheus text exposition format (v0.0.4).
363 ///
364 /// The output follows the format:
365 /// ```text
366 /// # HELP <metric_name> <description>
367 /// # TYPE <metric_name> counter
368 /// <metric_name>{<labels>} <value>
369 /// ```
370 ///
371 /// Multi-label metrics use a `","` separator in the internal label string,
372 /// which is split into separate `key="value"` pairs in the output.
373 #[must_use]
374 pub fn render_prometheus(&self) -> String {
375 let mut out = String::with_capacity(4096);
376
377 Self::write_counter_vec(
378 &mut out,
379 "makod_process_initiated_total",
380 "Total number of MaKo process instances initiated, by process family.",
381 &["family"],
382 &self.process_initiated,
383 );
384 Self::write_counter_vec(
385 &mut out,
386 "makod_process_completed_total",
387 "Total number of MaKo process instances that reached a terminal state.",
388 &["family", "result"],
389 &self.process_completed,
390 );
391 Self::write_counter_vec(
392 &mut out,
393 "makod_validation_failed_total",
394 "Total number of inbound EDIFACT messages that failed AHB validation.",
395 &["message_type", "release"],
396 &self.validation_failed,
397 );
398 Self::write_counter_vec(
399 &mut out,
400 "makod_outbox_delivery_attempts_total",
401 "Total number of AS4 outbox delivery attempts.",
402 &["result"],
403 &self.outbox_delivery_attempts,
404 );
405 Self::write_counter_vec(
406 &mut out,
407 "makod_deadline_fired_total",
408 "Total number of regulatory deadlines fired (TimeoutExpired dispatched).",
409 &["family"],
410 &self.deadline_fired,
411 );
412 Self::write_counter_vec(
413 &mut out,
414 "makod_dead_letter_recorded_total",
415 "Total number of messages sent to the durable dead-letter sink.",
416 &["reason"],
417 &self.dead_letter_recorded,
418 );
419 Self::write_counter_vec(
420 &mut out,
421 "makod_inbound_messages_total",
422 "Total number of inbound EDIFACT messages that entered the dispatch pipeline, \
423 by PID and outcome.",
424 &["pid", "result"],
425 &self.inbound_received,
426 );
427 Self::write_counter_vec(
428 &mut out,
429 "makod_aperak_missed_total",
430 "Total number of APERAK deadlines fired after their due-at time — a regulatory \
431 violation under APERAK AHB 1.0 §2.4.1 (Strom) / §2.3.1 (Gas). \
432 Alert when this counter is non-zero.",
433 &["label"],
434 &self.aperak_missed,
435 );
436
437 out
438 }
439
440 /// Write a `counter` metric family to `out`.
441 ///
442 /// `label_names` specifies the label key names in order. Each entry in
443 /// `pairs` has a label value that is either a bare string (single-label
444 /// metrics) or a `","` separated string (multi-label metrics, split in
445 /// order of `label_names`).
446 fn write_counter_vec(
447 out: &mut String,
448 name: &str,
449 help: &str,
450 label_names: &[&str],
451 pairs: &[(Box<str>, u64)],
452 ) {
453 if pairs.is_empty() {
454 return;
455 }
456 out.push_str("# HELP ");
457 out.push_str(name);
458 out.push(' ');
459 out.push_str(help);
460 out.push('\n');
461 out.push_str("# TYPE ");
462 out.push_str(name);
463 out.push_str(" counter\n");
464
465 for (label_str, count) in pairs {
466 let values: Vec<&str> = label_str.splitn(label_names.len(), ',').collect();
467 out.push_str(name);
468 out.push('{');
469 for (i, (key, val)) in label_names.iter().zip(values.iter()).enumerate() {
470 if i > 0 {
471 out.push(',');
472 }
473 out.push_str(key);
474 out.push_str("=\"");
475 // Escape backslash, double-quote, and newline per Prometheus spec.
476 for ch in val.chars() {
477 match ch {
478 '\\' => out.push_str(r"\\"),
479 '"' => out.push_str(r#"\""#),
480 '\n' => out.push_str(r"\n"),
481 _ => out.push(ch),
482 }
483 }
484 out.push('"');
485 }
486 out.push_str("} ");
487 let _ = std::fmt::Write::write_fmt(out, format_args!("{count}"));
488 out.push('\n');
489 }
490 }
491}
492
493// ── Tests ─────────────────────────────────────────────────────────────────────
494
495#[cfg(test)]
496mod tests {
497 use super::*;
498
499 fn fresh_metrics() -> EngineMetrics {
500 EngineMetrics::new()
501 }
502
503 #[test]
504 fn process_initiated_increments_by_family() {
505 let m = fresh_metrics();
506 m.process_initiated("gpke");
507 m.process_initiated("gpke");
508 m.process_initiated("wim");
509
510 let snap = m.snapshot();
511 assert_eq!(snap.process_initiated.len(), 2);
512
513 let gpke = snap
514 .process_initiated
515 .iter()
516 .find(|(k, _)| k.as_ref() == "gpke");
517 assert_eq!(gpke.map(|(_, v)| *v), Some(2));
518
519 let wim = snap
520 .process_initiated
521 .iter()
522 .find(|(k, _)| k.as_ref() == "wim");
523 assert_eq!(wim.map(|(_, v)| *v), Some(1));
524 }
525
526 #[test]
527 fn process_completed_uses_composite_label() {
528 let m = fresh_metrics();
529 m.process_completed("gpke", ProcessOutcome::Accepted);
530 m.process_completed("gpke", ProcessOutcome::Rejected);
531 m.process_completed("gpke", ProcessOutcome::Accepted);
532 m.process_completed("wim", ProcessOutcome::Timeout);
533
534 let snap = m.snapshot();
535 let accepted = snap
536 .process_completed
537 .iter()
538 .find(|(k, _)| k.as_ref() == "gpke,accepted");
539 assert_eq!(accepted.map(|(_, v)| *v), Some(2));
540
541 let timeout = snap
542 .process_completed
543 .iter()
544 .find(|(k, _)| k.as_ref() == "wim,timeout");
545 assert_eq!(timeout.map(|(_, v)| *v), Some(1));
546 }
547
548 #[test]
549 fn snapshot_returns_zero_for_unincremented_metric() {
550 let m = fresh_metrics();
551 // No increments — snapshot should be empty.
552 let snap = m.snapshot();
553 assert!(snap.process_initiated.is_empty());
554 assert!(snap.process_completed.is_empty());
555 }
556
557 #[test]
558 fn render_prometheus_omits_empty_metric_families() {
559 let m = fresh_metrics();
560 m.process_initiated("gpke");
561
562 let output = m.snapshot().render_prometheus();
563
564 // Only the incremented family should appear.
565 assert!(
566 output.contains("makod_process_initiated_total"),
567 "initiated must appear"
568 );
569 assert!(
570 !output.contains("makod_process_completed_total"),
571 "completed must be absent"
572 );
573 assert!(
574 !output.contains("makod_validation_failed_total"),
575 "validation must be absent"
576 );
577 }
578
579 #[test]
580 fn render_prometheus_formats_labels_correctly() {
581 let m = fresh_metrics();
582 m.process_initiated("gpke");
583 m.process_completed("gpke", ProcessOutcome::Accepted);
584 m.validation_failed("utilmd", "S2.1");
585
586 let output = m.snapshot().render_prometheus();
587
588 assert!(
589 output.contains(r#"makod_process_initiated_total{family="gpke"} 1"#),
590 "single-label format must match; output:\n{output}"
591 );
592 assert!(
593 output.contains(r#"makod_process_completed_total{family="gpke",result="accepted"} 1"#),
594 "two-label format must match; output:\n{output}"
595 );
596 assert!(
597 output.contains(
598 r#"makod_validation_failed_total{message_type="utilmd",release="S2.1"} 1"#
599 ),
600 "message_type+release format must match; output:\n{output}"
601 );
602 }
603
604 #[test]
605 fn render_prometheus_escapes_special_chars_in_label_values() {
606 let m = fresh_metrics();
607 // Inject a label value with a backslash and a double-quote.
608 m.outbox_delivery_attempted("ok");
609 m.dead_letter_recorded("unknown_pid:13002");
610
611 let output = m.snapshot().render_prometheus();
612 assert!(
613 output.contains(r#"result="ok""#),
614 "plain label must survive; output:\n{output}"
615 );
616 assert!(
617 output.contains(r#"reason="unknown_pid:13002""#),
618 "reason label must survive; output:\n{output}"
619 );
620 }
621
622 #[test]
623 fn counters_are_monotonically_increasing() {
624 let m = fresh_metrics();
625 for _ in 0..100 {
626 m.deadline_fired("gpke");
627 }
628 let snap = m.snapshot();
629 let gpke = snap
630 .deadline_fired
631 .iter()
632 .find(|(k, _)| k.as_ref() == "gpke");
633 assert_eq!(gpke.map(|(_, v)| *v), Some(100));
634 }
635
636 #[test]
637 fn snapshot_sorted_by_label() {
638 let m = fresh_metrics();
639 // Insert in reverse order to verify sort.
640 m.process_initiated("wim");
641 m.process_initiated("mabis");
642 m.process_initiated("geli-gas");
643 m.process_initiated("gpke");
644
645 let snap = m.snapshot();
646 let labels: Vec<&str> = snap
647 .process_initiated
648 .iter()
649 .map(|(k, _)| k.as_ref())
650 .collect();
651 let mut sorted = labels.clone();
652 sorted.sort_unstable();
653 assert_eq!(labels, sorted, "snapshot must be sorted by label");
654 }
655}