Skip to main content

subc_daemon/
observability.rs

1use std::{
2    collections::{BTreeMap, HashMap, VecDeque},
3    sync::{
4        atomic::{AtomicU64, Ordering},
5        Arc, Mutex, MutexGuard,
6    },
7    time::Duration,
8};
9
10use serde_json::{json, Value};
11use tracing::{debug, warn};
12
13use crate::registry::ConnectionId;
14
15/// The slash prevents collision with a module ID: `module_id_path_hazard`
16/// rejects separators in `registry.rs:855-875`.
17pub(crate) const CONTROL_WAIT_BUCKET: &str = "daemon/control";
18
19/// The only keys the route.open refusal counter may carry. A module's own
20/// error code on a rejected bind is counted under `module_rejected` and rides
21/// the log event as a separate field: if the module's string were the key, a
22/// module could grow this map for the daemon's lifetime and push terminal
23/// control sequences through `ck daemon` to an operator's screen. The
24/// `&'static str` increment signature plus the debug assertion keep the set
25/// closed at the call site, not just here.
26const ROUTE_OPEN_REFUSAL_COUNTER_CODES: &[&str] = &[
27    "module_warming",
28    ROUTE_OPEN_REFUSED_DECLARED_NOT_READY,
29    ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
30    "target_unavailable",
31    subc_protocol::error_codes::TARGET_FLOW_UNSUPPORTED,
32    subc_protocol::error_codes::TARGET_AGENT_RUN_UNSUPPORTED,
33    "module_removed",
34    "module_no_protocol",
35    "unknown_module",
36    "module_reloading",
37    "op_not_allowed",
38    "bad_consumer_identity",
39    "capability_forbidden",
40    "admission_facts_not_permitted",
41    "admission_facts_target_not_allowed",
42    "route_limit",
43    "forwarding_error",
44    "module_timeout",
45    ROUTE_OPEN_REFUSED_BREAKER_OPEN,
46    "module_rejected",
47    subc_protocol::error_codes::SCOPE_EPOCH_REQUIRED,
48    subc_protocol::error_codes::SCOPE_NOT_SYNCED,
49    subc_protocol::error_codes::SCOPE_NOT_LIVE,
50    subc_protocol::error_codes::SCOPE_ENDED,
51    subc_protocol::error_codes::SCOPE_NOT_CARRIER,
52    subc_protocol::error_codes::SCOPE_BIND_PROOF_REQUIRED,
53    subc_protocol::error_codes::SCOPE_BIND_PROOF_MISMATCH,
54    subc_protocol::error_codes::SCOPE_CHANGED,
55    subc_protocol::error_codes::INVALID_REQUEST,
56];
57
58/// Counter key for a `route.open` refused by the per-module bind-relay breaker
59/// before any relay was attempted.
60///
61/// The frame the caller receives carries `module_timeout`, because both SDKs
62/// already classify that as retryable with capped backoff and inventing a new
63/// wire code would need a change in each of them. The COUNTER is deliberately a
64/// different key: "this module burned the full bind budget" and "this module is
65/// being refused in microseconds because it already did that repeatedly" are
66/// the two states an operator most needs to tell apart, and they are
67/// indistinguishable from the client side, where both look like one retryable
68/// error that the next attempt may well satisfy.
69pub(crate) const ROUTE_OPEN_REFUSED_BREAKER_OPEN: &str = "module_timeout_breaker_open";
70
71/// Counter key for a registered module that declared itself not ready.
72///
73/// The caller still receives `module_warming`, but operators must be able to
74/// distinguish declared readiness from a supervised process that has not
75/// registered yet.
76pub(crate) const ROUTE_OPEN_REFUSED_DECLARED_NOT_READY: &str = "module_warming_declared_not_ready";
77
78/// Counter key for a registered, declared-ready module held not-ready because
79/// a capability it declares `need: required` has no registered provider.
80///
81/// The caller still receives `module_warming`; a separate key lets an operator
82/// tell "the module says it is warming" from "the module is waiting on a
83/// provider that is not running", which point at different fixes.
84pub(crate) const ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED: &str =
85    "module_warming_required_capability_unprovided";
86
87/// Shared count of authenticated socket connections accepted by the daemon.
88#[derive(Debug, Clone, Default)]
89pub struct ConnectedClients {
90    count: Arc<AtomicU64>,
91}
92
93impl ConnectedClients {
94    pub fn new() -> Self {
95        Self::default()
96    }
97
98    pub fn count(&self) -> u64 {
99        self.count.load(Ordering::SeqCst)
100    }
101
102    pub(crate) fn open(&self, connection_id: ConnectionId) -> ConnectedClientGuard {
103        let previous = self.count.fetch_add(1, Ordering::SeqCst);
104        let current = previous + 1;
105        // DEBUG, not INFO: this pair fired on every connect and disconnect and was
106        // measured at 43-74% of the daemon's own log (issue #114), burying the
107        // lines an operator opens the file for. The count itself is not lost: it
108        // is served live as `connected_clients` on server.describe (`ck daemon`),
109        // and a connection that opens a route still names its connection_id on the
110        // `route.open accepted` line, so per-connection forensics survive where
111        // they matter. Raise the filter (`CK_LOG=subc=debug`) to see every churn.
112        debug!(
113            connection_id = connection_id.get(),
114            connected_clients = current,
115            previous_connected_clients = previous,
116            "authenticated connection count changed"
117        );
118        ConnectedClientGuard {
119            clients: self.clone(),
120            connection_id,
121        }
122    }
123}
124
125pub(crate) struct ConnectedClientGuard {
126    clients: ConnectedClients,
127    connection_id: ConnectionId,
128}
129
130impl Drop for ConnectedClientGuard {
131    fn drop(&mut self) {
132        let previous = self.clients.count.fetch_sub(1, Ordering::SeqCst);
133        let current = previous.saturating_sub(1);
134        // DEBUG for the same reason as the open side above.
135        debug!(
136            connection_id = self.connection_id.get(),
137            connected_clients = current,
138            previous_connected_clients = previous,
139            "authenticated connection count changed"
140        );
141    }
142}
143
144/// Counters for route lifecycle drops, delivery failures and reader waits.
145#[derive(Debug, Clone, Default)]
146pub struct DaemonCounters {
147    reader_diagnostics: Arc<Mutex<ReaderDiagnostics>>,
148    carried_containment_anchor_lost: Arc<AtomicU64>,
149    /// Every non-request frame a module sent on a (channel, epoch) the daemon
150    /// holds no bound route for, dropped. Counts both kinds below: orphan
151    /// traffic on a route the daemon released, and frames on a (channel,
152    /// epoch) that was never allocated on that module connection.
153    module_frames_dropped_no_route: Arc<AtomicU64>,
154    /// The orphan-traffic part of `module_frames_dropped_no_route`: the daemon
155    /// had allocated that (channel, epoch) on the sending module connection and
156    /// has since released it, so the module is still holding a route the
157    /// daemon closed. `module_frames_dropped_no_route` minus this is the count
158    /// of frames on a (channel, epoch) that never existed on the connection.
159    module_frames_dropped_released_route: Arc<AtomicU64>,
160    module_frames_dropped_released_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
161    /// Route GOODBYEs the daemon enqueued to a module in answer to its frames
162    /// on a (channel, epoch) the daemon holds no route for, telling the module
163    /// to drop the route. Rate-limited per module connection and channel, so
164    /// this counts answers, not orphan frames.
165    module_orphan_route_goodbyes_sent: Arc<AtomicU64>,
166    // Per-module maps and the rate window are daemon-lifetime diagnostics only:
167    // they deliberately reset on restart instead of becoming durable daemon state.
168    module_frames_dropped_no_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
169    route_open_refused_by_code: Arc<Mutex<HashMap<String, u64>>>,
170    route_open_accepted_by_principal: Arc<Mutex<HashMap<String, u64>>>,
171    module_frames_dropped_no_route_window: Arc<Mutex<DropWindow>>,
172    module_requests_dropped_stale_route: Arc<AtomicU64>,
173    client_frames_dropped_stale_route: Arc<AtomicU64>,
174    client_egress_close_delivery_failed: Arc<AtomicU64>,
175    goodbye_relay_client_failed: Arc<AtomicU64>,
176    goodbye_relay_module_dropped: Arc<AtomicU64>,
177    goodbye_relay_module_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
178    route_released_epoch_fenced: Arc<AtomicU64>,
179    route_release_stale_skipped: Arc<AtomicU64>,
180    drains_with_undeclared_gauge: Arc<AtomicU64>,
181}
182
183#[derive(Debug, Default)]
184struct ReaderDiagnostics {
185    busy: BTreeMap<String, BTreeMap<String, u64>>,
186    waits: BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>>,
187    warned_waits: HashMap<(String, String), tokio::time::Instant>,
188}
189
190/// Ten one-minute buckets make sustained module-to-client route drops visible
191/// without retaining one record for every dropped frame.
192#[derive(Debug)]
193struct DropWindow {
194    started_at: tokio::time::Instant,
195    buckets: VecDeque<DropBucket>,
196}
197
198#[derive(Debug)]
199struct DropBucket {
200    minute: u64,
201    count: u64,
202}
203
204impl Default for DropWindow {
205    fn default() -> Self {
206        Self {
207            started_at: tokio::time::Instant::now(),
208            buckets: VecDeque::new(),
209        }
210    }
211}
212
213impl DropWindow {
214    const MINUTE: Duration = Duration::from_secs(60);
215    const BUCKETS: u64 = 10;
216
217    fn record(&mut self, now: tokio::time::Instant) {
218        let minute = self.minute_at(now);
219        self.prune_before(minute);
220        match self.buckets.back_mut() {
221            Some(bucket) if bucket.minute == minute => bucket.count += 1,
222            _ => self.buckets.push_back(DropBucket { minute, count: 1 }),
223        }
224    }
225
226    fn count_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
227        let minute = self.minute_at(now);
228        self.prune_before(minute);
229        self.buckets.iter().map(|bucket| bucket.count).sum()
230    }
231
232    fn nonzero_minutes_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
233        let minute = self.minute_at(now);
234        self.prune_before(minute);
235        self.buckets.len() as u64
236    }
237
238    fn minute_at(&self, now: tokio::time::Instant) -> u64 {
239        now.saturating_duration_since(self.started_at).as_secs() / Self::MINUTE.as_secs()
240    }
241
242    fn prune_before(&mut self, current_minute: u64) {
243        while self
244            .buckets
245            .front()
246            .is_some_and(|bucket| current_minute.saturating_sub(bucket.minute) >= Self::BUCKETS)
247        {
248            self.buckets.pop_front();
249        }
250    }
251}
252
253impl DaemonCounters {
254    pub fn new() -> Self {
255        Self::default()
256    }
257
258    /// Returns a JSON snapshot whose stable, additive schema keeps the
259    /// `server.describe` diagnostic endpoint backward-compatible.
260    pub fn snapshot(&self) -> Value {
261        let mut snapshot = serde_json::Map::new();
262        let diagnostics = self
263            .reader_diagnostics
264            .lock()
265            .unwrap_or_else(|p| p.into_inner());
266        if !diagnostics.busy.is_empty() {
267            snapshot.insert(
268                "route_busy_by_module_principal".into(),
269                json!(diagnostics.busy),
270            );
271        }
272        if !diagnostics.waits.is_empty() {
273            snapshot.insert(
274                "reader_wait_over_5s_by_module_principal".into(),
275                json!(diagnostics.waits),
276            );
277        }
278        drop(diagnostics);
279        snapshot.insert(
280            "carried_containment_anchor_lost".into(),
281            self.carried_containment_anchor_lost
282                .load(Ordering::Relaxed)
283                .into(),
284        );
285        snapshot.insert(
286            "module_frames_dropped_no_route".into(),
287            self.module_frames_dropped_no_route
288                .load(Ordering::Relaxed)
289                .into(),
290        );
291        let mut drop_window = self
292            .module_frames_dropped_no_route_window
293            .lock()
294            .expect("drop-rate window mutex poisoned");
295        let now = tokio::time::Instant::now();
296        snapshot.insert(
297            "module_frames_dropped_no_route_last_10m".into(),
298            drop_window.count_last_10m(now).into(),
299        );
300        snapshot.insert(
301            "module_frames_dropped_no_route_nonzero_minutes_last_10m".into(),
302            drop_window.nonzero_minutes_last_10m(now).into(),
303        );
304        insert_nonempty_counts(
305            &mut snapshot,
306            "module_frames_dropped_no_route_by_module",
307            &self.module_frames_dropped_no_route_by_module,
308        );
309        snapshot.insert(
310            "module_frames_dropped_released_route".into(),
311            self.module_frames_dropped_released_route
312                .load(Ordering::Relaxed)
313                .into(),
314        );
315        insert_nonempty_counts(
316            &mut snapshot,
317            "module_frames_dropped_released_route_by_module",
318            &self.module_frames_dropped_released_route_by_module,
319        );
320        snapshot.insert(
321            "module_orphan_route_goodbyes_sent".into(),
322            self.module_orphan_route_goodbyes_sent
323                .load(Ordering::Relaxed)
324                .into(),
325        );
326        insert_nonempty_counts(
327            &mut snapshot,
328            "route_open_refused_by_code",
329            &self.route_open_refused_by_code,
330        );
331        insert_nonempty_counts(
332            &mut snapshot,
333            "route_open_accepted_by_principal",
334            &self.route_open_accepted_by_principal,
335        );
336        snapshot.insert(
337            "module_requests_dropped_stale_route".into(),
338            self.module_requests_dropped_stale_route
339                .load(Ordering::Relaxed)
340                .into(),
341        );
342        snapshot.insert(
343            "client_frames_dropped_stale_route".into(),
344            self.client_frames_dropped_stale_route
345                .load(Ordering::Relaxed)
346                .into(),
347        );
348        snapshot.insert(
349            "client_egress_close_delivery_failed".into(),
350            self.client_egress_close_delivery_failed
351                .load(Ordering::Relaxed)
352                .into(),
353        );
354        snapshot.insert(
355            "goodbye_relay_client_failed".into(),
356            self.goodbye_relay_client_failed
357                .load(Ordering::Relaxed)
358                .into(),
359        );
360        snapshot.insert(
361            "goodbye_relay_module_dropped".into(),
362            self.goodbye_relay_module_dropped
363                .load(Ordering::Relaxed)
364                .into(),
365        );
366        insert_nonempty_counts(
367            &mut snapshot,
368            "goodbye_relay_module_dropped_by_module",
369            &self.goodbye_relay_module_dropped_by_module,
370        );
371        snapshot.insert(
372            "route_released_epoch_fenced".into(),
373            self.route_released_epoch_fenced
374                .load(Ordering::Relaxed)
375                .into(),
376        );
377        snapshot.insert(
378            "route_release_stale_skipped".into(),
379            self.route_release_stale_skipped
380                .load(Ordering::Relaxed)
381                .into(),
382        );
383        snapshot.insert(
384            "drains_with_undeclared_gauge".into(),
385            self.drains_with_undeclared_gauge
386                .load(Ordering::Relaxed)
387                .into(),
388        );
389        Value::Object(snapshot)
390    }
391
392    pub(crate) fn increment_route_busy(&self, module: &str, principal: &str) {
393        let mut diagnostics = self
394            .reader_diagnostics
395            .lock()
396            .unwrap_or_else(|p| p.into_inner());
397        *diagnostics
398            .busy
399            .entry(module.into())
400            .or_default()
401            .entry(principal.into())
402            .or_default() += 1;
403    }
404
405    /// Observe a wait without imposing a deadline or cancelling its future.
406    /// Counts advance once at five seconds, even if the peer never recovers.
407    pub(crate) async fn observe_reader_wait<F: std::future::Future>(
408        &self,
409        labels: impl FnOnce() -> (String, String),
410        kind: &'static str,
411        future: F,
412    ) -> F::Output {
413        tokio::pin!(future);
414        // Ready sends need neither a timer nor diagnostic labels. Poll before
415        // constructing Sleep, which would otherwise touch the runtime clock.
416        if let std::task::Poll::Ready(result) =
417            std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
418        {
419            return result;
420        }
421        tokio::select! {
422            biased;
423            result = &mut future => return result,
424            _ = tokio::time::sleep(Duration::from_secs(5)) => {}
425        }
426        let (module, principal) = labels();
427        let should_warn = {
428            let mut diagnostics = self
429                .reader_diagnostics
430                .lock()
431                .unwrap_or_else(|p| p.into_inner());
432            *diagnostics
433                .waits
434                .entry(module.clone())
435                .or_default()
436                .entry(principal.clone())
437                .or_default()
438                .entry(kind.into())
439                .or_default() += 1;
440            let now = tokio::time::Instant::now();
441            let key = (module.to_owned(), principal.to_owned());
442            let should_warn = diagnostics
443                .warned_waits
444                .get(&key)
445                .is_none_or(|last| now.duration_since(*last) >= Duration::from_secs(60));
446            if should_warn {
447                diagnostics.warned_waits.insert(key, now);
448            }
449            should_warn
450        };
451        if should_warn {
452            warn!(
453                module_id = module,
454                client_principal = principal,
455                wait_kind = kind,
456                "connection reader wait exceeded 5 seconds"
457            );
458        }
459        future.await
460    }
461
462    pub(crate) fn increment_module_frames_dropped_no_route(&self, module_id: Option<&str>) {
463        self.module_frames_dropped_no_route
464            .fetch_add(1, Ordering::Relaxed);
465        if let Some(module_id) = module_id {
466            increment_keyed_count(&self.module_frames_dropped_no_route_by_module, module_id);
467        }
468        self.module_frames_dropped_no_route_window
469            .lock()
470            .expect("drop-rate window mutex poisoned")
471            .record(tokio::time::Instant::now());
472    }
473
474    /// Count a dropped module frame whose (channel, epoch) the daemon had
475    /// allocated on that connection and since released. Called in addition to
476    /// [`Self::increment_module_frames_dropped_no_route`], never instead of it.
477    pub(crate) fn increment_module_frames_dropped_released_route(&self, module_id: Option<&str>) {
478        self.module_frames_dropped_released_route
479            .fetch_add(1, Ordering::Relaxed);
480        if let Some(module_id) = module_id {
481            increment_keyed_count(
482                &self.module_frames_dropped_released_route_by_module,
483                module_id,
484            );
485        }
486    }
487
488    pub(crate) fn increment_module_orphan_route_goodbyes_sent(&self) {
489        self.module_orphan_route_goodbyes_sent
490            .fetch_add(1, Ordering::Relaxed);
491    }
492
493    pub(crate) fn increment_route_open_refused(&self, code: &'static str) {
494        debug_assert!(ROUTE_OPEN_REFUSAL_COUNTER_CODES.contains(&code));
495        increment_keyed_count(&self.route_open_refused_by_code, code);
496    }
497
498    /// Count an accepted route.open by the principal the daemon stamped.
499    ///
500    /// THE KEY SPACE IS CLOSED BY CONSTRUCTION, unlike the refusal counter which
501    /// needs an explicit allowlist: a principal is `direct` or
502    /// `reserved:<module_id>`, and a module id was already refused at HELLO
503    /// unless it is a single path component free of control characters. So an
504    /// untrusted string cannot expand this map without first passing module-id
505    /// validation, and the bound is the number of modules rather than the number
506    /// of distinct strings a caller can invent.
507    pub(crate) fn increment_route_open_accepted(&self, principal: &str) {
508        increment_keyed_count(&self.route_open_accepted_by_principal, principal);
509    }
510
511    pub(crate) fn increment_module_requests_dropped_stale_route(&self) {
512        self.module_requests_dropped_stale_route
513            .fetch_add(1, Ordering::Relaxed);
514    }
515
516    pub(crate) fn increment_client_frames_dropped_stale_route(&self) {
517        self.client_frames_dropped_stale_route
518            .fetch_add(1, Ordering::Relaxed);
519    }
520
521    pub(crate) fn increment_client_egress_close_delivery_failed(&self) {
522        self.client_egress_close_delivery_failed
523            .fetch_add(1, Ordering::Relaxed);
524    }
525
526    pub(crate) fn increment_goodbye_relay_client_failed(&self) {
527        self.goodbye_relay_client_failed
528            .fetch_add(1, Ordering::Relaxed);
529    }
530
531    pub(crate) fn increment_goodbye_relay_module_dropped(&self, module_id: Option<&str>) {
532        self.goodbye_relay_module_dropped
533            .fetch_add(1, Ordering::Relaxed);
534        if let Some(module_id) = module_id {
535            increment_keyed_count(&self.goodbye_relay_module_dropped_by_module, module_id);
536        }
537    }
538
539    pub(crate) fn increment_route_released_epoch_fenced(&self) {
540        self.route_released_epoch_fenced
541            .fetch_add(1, Ordering::Relaxed);
542    }
543
544    pub(crate) fn increment_route_release_stale_skipped(&self) {
545        self.route_release_stale_skipped
546            .fetch_add(1, Ordering::Relaxed);
547    }
548
549    pub(crate) fn increment_drains_with_undeclared_gauge(&self) {
550        self.drains_with_undeclared_gauge
551            .fetch_add(1, Ordering::Relaxed);
552    }
553
554    #[cfg(unix)]
555    pub(crate) fn increment_carried_containment_anchor_lost(&self) {
556        self.carried_containment_anchor_lost
557            .fetch_add(1, Ordering::Relaxed);
558    }
559}
560
561fn increment_keyed_count(counts: &Mutex<HashMap<String, u64>>, key: &str) {
562    *counts
563        .lock()
564        .expect("keyed counter mutex poisoned")
565        .entry(key.to_string())
566        .or_default() += 1;
567}
568
569fn insert_nonempty_counts(
570    snapshot: &mut serde_json::Map<String, Value>,
571    key: &str,
572    counts: &Mutex<HashMap<String, u64>>,
573) {
574    let counts: MutexGuard<'_, HashMap<String, u64>> =
575        counts.lock().expect("keyed counter mutex poisoned");
576    if !counts.is_empty() {
577        snapshot.insert(key.to_string(), json!(&*counts));
578    }
579}
580
581#[cfg(test)]
582mod tests {
583    use super::*;
584
585    #[test]
586    fn ready_reader_wait_needs_no_runtime_clock_or_labels() {
587        use std::{
588            future::Future,
589            task::{Context, Poll, Waker},
590        };
591
592        let counters = DaemonCounters::new();
593        let wait = counters.observe_reader_wait(
594            || panic!("ready waits must not resolve diagnostic labels"),
595            "module_queue",
596            std::future::ready(7),
597        );
598        let mut wait = std::pin::pin!(wait);
599        let mut cx = Context::from_waker(Waker::noop());
600        // No Tokio runtime exists here: even constructing Sleep would panic.
601        assert_eq!(wait.as_mut().poll(&mut cx), Poll::Ready(7));
602        assert!(counters.reader_diagnostics.lock().unwrap().waits.is_empty());
603    }
604
605    #[tokio::test(start_paused = true)]
606    async fn reader_wait_labels_are_resolved_once_only_after_five_seconds() {
607        use std::{
608            cell::Cell,
609            future::{poll_fn, Future},
610            task::Poll,
611        };
612
613        let counters = DaemonCounters::new();
614        let calls = Cell::new(0);
615        let (sender, receiver) = tokio::sync::oneshot::channel();
616        let wait = counters.observe_reader_wait(
617            || {
618                calls.set(calls.get() + 1);
619                ("alpha".into(), "direct".into())
620            },
621            "module_queue",
622            receiver,
623        );
624        tokio::pin!(wait);
625        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
626            .await
627            .is_pending());
628        tokio::time::advance(Duration::from_secs(4)).await;
629        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
630            .await
631            .is_pending());
632        assert_eq!(calls.get(), 0);
633        tokio::time::advance(Duration::from_secs(1)).await;
634        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
635            .await
636            .is_pending());
637        assert_eq!(calls.get(), 1);
638        assert_eq!(
639            counters.snapshot()["reader_wait_over_5s_by_module_principal"]["alpha"]["direct"]
640                ["module_queue"],
641            1
642        );
643        sender.send(()).unwrap();
644        wait.await.unwrap();
645        assert_eq!(calls.get(), 1);
646    }
647
648    #[test]
649    fn counter_snapshot_includes_zero_rate_and_omits_empty_module_maps() {
650        let counters = DaemonCounters::new();
651        let snapshot = counters.snapshot();
652
653        assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 0);
654        assert_eq!(
655            snapshot["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
656            0
657        );
658        assert!(snapshot
659            .get("module_frames_dropped_no_route_by_module")
660            .is_none());
661        assert!(snapshot
662            .get("goodbye_relay_module_dropped_by_module")
663            .is_none());
664    }
665
666    #[tokio::test(start_paused = true)]
667    async fn module_frame_drops_are_attributed_to_the_emitting_module() {
668        let counters = DaemonCounters::new();
669        counters.increment_module_frames_dropped_no_route(Some("alpha"));
670        counters.increment_module_frames_dropped_no_route(Some("alpha"));
671
672        let snapshot = counters.snapshot();
673        assert_eq!(snapshot["module_frames_dropped_no_route"], 2);
674        assert_eq!(
675            snapshot["module_frames_dropped_no_route_by_module"],
676            json!({ "alpha": 2 })
677        );
678        assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 2);
679    }
680
681    #[tokio::test(start_paused = true)]
682    async fn frame_drop_rate_ages_out_after_ten_minute_buckets() {
683        let counters = DaemonCounters::new();
684        counters.increment_module_frames_dropped_no_route(Some("alpha"));
685
686        tokio::time::advance(Duration::from_secs(9 * 60)).await;
687        assert_eq!(
688            counters.snapshot()["module_frames_dropped_no_route_last_10m"],
689            1
690        );
691
692        tokio::time::advance(Duration::from_secs(60)).await;
693        assert_eq!(
694            counters.snapshot()["module_frames_dropped_no_route_last_10m"],
695            0
696        );
697    }
698
699    #[tokio::test(start_paused = true)]
700    async fn frame_drop_window_counts_only_nonzero_minutes() {
701        let counters = DaemonCounters::new();
702        for minute in 0..10 {
703            if minute > 0 {
704                tokio::time::advance(Duration::from_secs(60)).await;
705            }
706            if minute != 4 {
707                counters.increment_module_frames_dropped_no_route(Some("alpha"));
708            }
709        }
710
711        assert_eq!(
712            counters.snapshot()["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
713            9
714        );
715    }
716
717    #[test]
718    fn goodbye_relay_drops_are_attributed_to_the_target_module() {
719        let counters = DaemonCounters::new();
720        counters.increment_goodbye_relay_module_dropped(Some("alpha"));
721
722        assert_eq!(
723            counters.snapshot()["goodbye_relay_module_dropped_by_module"],
724            json!({ "alpha": 1 })
725        );
726    }
727}