Skip to main content

subc_daemon/
observability.rs

1use std::{
2    collections::{BTreeMap, HashMap, VecDeque},
3    sync::{
4        atomic::{AtomicU64, Ordering},
5        Arc, Mutex, MutexGuard,
6    },
7    time::Duration,
8};
9
10use serde_json::{json, Value};
11use tracing::{debug, warn};
12
13use crate::registry::ConnectionId;
14
15/// The slash prevents collision with a module ID: `module_id_path_hazard`
16/// rejects separators in `registry.rs:855-875`.
17pub(crate) const CONTROL_WAIT_BUCKET: &str = "daemon/control";
18
19/// The only keys the route.open refusal counter may carry. A module's own
20/// error code on a rejected bind is counted under `module_rejected` and rides
21/// the log event as a separate field: if the module's string were the key, a
22/// module could grow this map for the daemon's lifetime and push terminal
23/// control sequences through `ck daemon` to an operator's screen. The
24/// `&'static str` increment signature plus the debug assertion keep the set
25/// closed at the call site, not just here.
26const ROUTE_OPEN_REFUSAL_COUNTER_CODES: &[&str] = &[
27    "module_warming",
28    ROUTE_OPEN_REFUSED_DECLARED_NOT_READY,
29    ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
30    "target_unavailable",
31    subc_protocol::error_codes::TARGET_FLOW_UNSUPPORTED,
32    subc_protocol::error_codes::TARGET_AGENT_RUN_UNSUPPORTED,
33    "module_removed",
34    "module_no_protocol",
35    "unknown_module",
36    "module_reloading",
37    "op_not_allowed",
38    "bad_consumer_identity",
39    "capability_forbidden",
40    "admission_facts_not_permitted",
41    "admission_facts_target_not_allowed",
42    "route_limit",
43    "forwarding_error",
44    "module_timeout",
45    ROUTE_OPEN_REFUSED_BREAKER_OPEN,
46    "module_rejected",
47    subc_protocol::error_codes::SCOPE_EPOCH_REQUIRED,
48    subc_protocol::error_codes::SCOPE_NOT_SYNCED,
49    subc_protocol::error_codes::SCOPE_NOT_LIVE,
50    subc_protocol::error_codes::SCOPE_ENDED,
51    subc_protocol::error_codes::SCOPE_NOT_CARRIER,
52    subc_protocol::error_codes::SCOPE_BIND_PROOF_REQUIRED,
53    subc_protocol::error_codes::SCOPE_BIND_PROOF_MISMATCH,
54    subc_protocol::error_codes::SCOPE_CHANGED,
55    subc_protocol::error_codes::INVALID_REQUEST,
56];
57
58/// Counter key for a `route.open` refused by the per-module bind-relay breaker
59/// before any relay was attempted.
60///
61/// The frame the caller receives carries `module_timeout`, because both SDKs
62/// already classify that as retryable with capped backoff and inventing a new
63/// wire code would need a change in each of them. The COUNTER is deliberately a
64/// different key: "this module burned the full bind budget" and "this module is
65/// being refused in microseconds because it already did that repeatedly" are
66/// the two states an operator most needs to tell apart, and they are
67/// indistinguishable from the client side, where both look like one retryable
68/// error that the next attempt may well satisfy.
69pub(crate) const ROUTE_OPEN_REFUSED_BREAKER_OPEN: &str = "module_timeout_breaker_open";
70
71/// Counter key for a registered module that declared itself not ready.
72///
73/// The caller still receives `module_warming`, but operators must be able to
74/// distinguish declared readiness from a supervised process that has not
75/// registered yet.
76pub(crate) const ROUTE_OPEN_REFUSED_DECLARED_NOT_READY: &str = "module_warming_declared_not_ready";
77
78/// Counter key for a registered, declared-ready module held not-ready because
79/// a capability it declares `need: required` has no registered provider.
80///
81/// The caller still receives `module_warming`; a separate key lets an operator
82/// tell "the module says it is warming" from "the module is waiting on a
83/// provider that is not running", which point at different fixes.
84pub(crate) const ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED: &str =
85    "module_warming_required_capability_unprovided";
86
87/// Shared count of authenticated socket connections accepted by the daemon.
88#[derive(Debug, Clone, Default)]
89pub struct ConnectedClients {
90    count: Arc<AtomicU64>,
91}
92
93impl ConnectedClients {
94    pub fn new() -> Self {
95        Self::default()
96    }
97
98    pub fn count(&self) -> u64 {
99        self.count.load(Ordering::SeqCst)
100    }
101
102    pub(crate) fn open(&self, connection_id: ConnectionId) -> ConnectedClientGuard {
103        let previous = self.count.fetch_add(1, Ordering::SeqCst);
104        let current = previous + 1;
105        // DEBUG, not INFO: this pair fired on every connect and disconnect and was
106        // measured at 43-74% of the daemon's own log (issue #114), burying the
107        // lines an operator opens the file for. The count itself is not lost: it
108        // is served live as `connected_clients` on server.describe (`ck daemon`),
109        // and a connection that opens a route still names its connection_id on the
110        // `route.open accepted` line, so per-connection forensics survive where
111        // they matter. Raise the filter (`CK_LOG=subc=debug`) to see every churn.
112        debug!(
113            connection_id = connection_id.get(),
114            connected_clients = current,
115            previous_connected_clients = previous,
116            "authenticated connection count changed"
117        );
118        ConnectedClientGuard {
119            clients: self.clone(),
120            connection_id,
121        }
122    }
123}
124
125pub(crate) struct ConnectedClientGuard {
126    clients: ConnectedClients,
127    connection_id: ConnectionId,
128}
129
130impl Drop for ConnectedClientGuard {
131    fn drop(&mut self) {
132        let previous = self.clients.count.fetch_sub(1, Ordering::SeqCst);
133        let current = previous.saturating_sub(1);
134        // DEBUG for the same reason as the open side above.
135        debug!(
136            connection_id = self.connection_id.get(),
137            connected_clients = current,
138            previous_connected_clients = previous,
139            "authenticated connection count changed"
140        );
141    }
142}
143
144/// Counters for route lifecycle drops, delivery failures and reader waits.
145#[derive(Debug, Clone, Default)]
146pub struct DaemonCounters {
147    reader_diagnostics: Arc<Mutex<ReaderDiagnostics>>,
148    /// Every non-request frame a module sent on a (channel, epoch) the daemon
149    /// holds no bound route for, dropped. Counts both kinds below: orphan
150    /// traffic on a route the daemon released, and frames on a (channel,
151    /// epoch) that was never allocated on that module connection.
152    module_frames_dropped_no_route: Arc<AtomicU64>,
153    /// The orphan-traffic part of `module_frames_dropped_no_route`: the daemon
154    /// had allocated that (channel, epoch) on the sending module connection and
155    /// has since released it, so the module is still holding a route the
156    /// daemon closed. `module_frames_dropped_no_route` minus this is the count
157    /// of frames on a (channel, epoch) that never existed on the connection.
158    module_frames_dropped_released_route: Arc<AtomicU64>,
159    module_frames_dropped_released_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
160    /// Route GOODBYEs the daemon enqueued to a module in answer to its frames
161    /// on a (channel, epoch) the daemon holds no route for, telling the module
162    /// to drop the route. Rate-limited per module connection and channel, so
163    /// this counts answers, not orphan frames.
164    module_orphan_route_goodbyes_sent: Arc<AtomicU64>,
165    // Per-module maps and the rate window are daemon-lifetime diagnostics only:
166    // they deliberately reset on restart instead of becoming durable daemon state.
167    module_frames_dropped_no_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
168    route_open_refused_by_code: Arc<Mutex<HashMap<String, u64>>>,
169    route_open_accepted_by_principal: Arc<Mutex<HashMap<String, u64>>>,
170    module_frames_dropped_no_route_window: Arc<Mutex<DropWindow>>,
171    module_requests_dropped_stale_route: Arc<AtomicU64>,
172    client_frames_dropped_stale_route: Arc<AtomicU64>,
173    client_egress_close_delivery_failed: Arc<AtomicU64>,
174    goodbye_relay_client_failed: Arc<AtomicU64>,
175    goodbye_relay_module_dropped: Arc<AtomicU64>,
176    goodbye_relay_module_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
177    route_released_epoch_fenced: Arc<AtomicU64>,
178    route_release_stale_skipped: Arc<AtomicU64>,
179    drains_with_undeclared_gauge: Arc<AtomicU64>,
180}
181
182#[derive(Debug, Default)]
183struct ReaderDiagnostics {
184    busy: BTreeMap<String, BTreeMap<String, u64>>,
185    waits: BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>>,
186    warned_waits: HashMap<(String, String), tokio::time::Instant>,
187}
188
189/// Ten one-minute buckets make sustained module-to-client route drops visible
190/// without retaining one record for every dropped frame.
191#[derive(Debug)]
192struct DropWindow {
193    started_at: tokio::time::Instant,
194    buckets: VecDeque<DropBucket>,
195}
196
197#[derive(Debug)]
198struct DropBucket {
199    minute: u64,
200    count: u64,
201}
202
203impl Default for DropWindow {
204    fn default() -> Self {
205        Self {
206            started_at: tokio::time::Instant::now(),
207            buckets: VecDeque::new(),
208        }
209    }
210}
211
212impl DropWindow {
213    const MINUTE: Duration = Duration::from_secs(60);
214    const BUCKETS: u64 = 10;
215
216    fn record(&mut self, now: tokio::time::Instant) {
217        let minute = self.minute_at(now);
218        self.prune_before(minute);
219        match self.buckets.back_mut() {
220            Some(bucket) if bucket.minute == minute => bucket.count += 1,
221            _ => self.buckets.push_back(DropBucket { minute, count: 1 }),
222        }
223    }
224
225    fn count_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
226        let minute = self.minute_at(now);
227        self.prune_before(minute);
228        self.buckets.iter().map(|bucket| bucket.count).sum()
229    }
230
231    fn nonzero_minutes_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
232        let minute = self.minute_at(now);
233        self.prune_before(minute);
234        self.buckets.len() as u64
235    }
236
237    fn minute_at(&self, now: tokio::time::Instant) -> u64 {
238        now.saturating_duration_since(self.started_at).as_secs() / Self::MINUTE.as_secs()
239    }
240
241    fn prune_before(&mut self, current_minute: u64) {
242        while self
243            .buckets
244            .front()
245            .is_some_and(|bucket| current_minute.saturating_sub(bucket.minute) >= Self::BUCKETS)
246        {
247            self.buckets.pop_front();
248        }
249    }
250}
251
252impl DaemonCounters {
253    pub fn new() -> Self {
254        Self::default()
255    }
256
257    /// Returns a JSON snapshot whose stable, additive schema keeps the
258    /// `server.describe` diagnostic endpoint backward-compatible.
259    pub fn snapshot(&self) -> Value {
260        let mut snapshot = serde_json::Map::new();
261        let diagnostics = self
262            .reader_diagnostics
263            .lock()
264            .unwrap_or_else(|p| p.into_inner());
265        if !diagnostics.busy.is_empty() {
266            snapshot.insert(
267                "route_busy_by_module_principal".into(),
268                json!(diagnostics.busy),
269            );
270        }
271        if !diagnostics.waits.is_empty() {
272            snapshot.insert(
273                "reader_wait_over_5s_by_module_principal".into(),
274                json!(diagnostics.waits),
275            );
276        }
277        drop(diagnostics);
278        snapshot.insert(
279            "module_frames_dropped_no_route".into(),
280            self.module_frames_dropped_no_route
281                .load(Ordering::Relaxed)
282                .into(),
283        );
284        let mut drop_window = self
285            .module_frames_dropped_no_route_window
286            .lock()
287            .expect("drop-rate window mutex poisoned");
288        let now = tokio::time::Instant::now();
289        snapshot.insert(
290            "module_frames_dropped_no_route_last_10m".into(),
291            drop_window.count_last_10m(now).into(),
292        );
293        snapshot.insert(
294            "module_frames_dropped_no_route_nonzero_minutes_last_10m".into(),
295            drop_window.nonzero_minutes_last_10m(now).into(),
296        );
297        insert_nonempty_counts(
298            &mut snapshot,
299            "module_frames_dropped_no_route_by_module",
300            &self.module_frames_dropped_no_route_by_module,
301        );
302        snapshot.insert(
303            "module_frames_dropped_released_route".into(),
304            self.module_frames_dropped_released_route
305                .load(Ordering::Relaxed)
306                .into(),
307        );
308        insert_nonempty_counts(
309            &mut snapshot,
310            "module_frames_dropped_released_route_by_module",
311            &self.module_frames_dropped_released_route_by_module,
312        );
313        snapshot.insert(
314            "module_orphan_route_goodbyes_sent".into(),
315            self.module_orphan_route_goodbyes_sent
316                .load(Ordering::Relaxed)
317                .into(),
318        );
319        insert_nonempty_counts(
320            &mut snapshot,
321            "route_open_refused_by_code",
322            &self.route_open_refused_by_code,
323        );
324        insert_nonempty_counts(
325            &mut snapshot,
326            "route_open_accepted_by_principal",
327            &self.route_open_accepted_by_principal,
328        );
329        snapshot.insert(
330            "module_requests_dropped_stale_route".into(),
331            self.module_requests_dropped_stale_route
332                .load(Ordering::Relaxed)
333                .into(),
334        );
335        snapshot.insert(
336            "client_frames_dropped_stale_route".into(),
337            self.client_frames_dropped_stale_route
338                .load(Ordering::Relaxed)
339                .into(),
340        );
341        snapshot.insert(
342            "client_egress_close_delivery_failed".into(),
343            self.client_egress_close_delivery_failed
344                .load(Ordering::Relaxed)
345                .into(),
346        );
347        snapshot.insert(
348            "goodbye_relay_client_failed".into(),
349            self.goodbye_relay_client_failed
350                .load(Ordering::Relaxed)
351                .into(),
352        );
353        snapshot.insert(
354            "goodbye_relay_module_dropped".into(),
355            self.goodbye_relay_module_dropped
356                .load(Ordering::Relaxed)
357                .into(),
358        );
359        insert_nonempty_counts(
360            &mut snapshot,
361            "goodbye_relay_module_dropped_by_module",
362            &self.goodbye_relay_module_dropped_by_module,
363        );
364        snapshot.insert(
365            "route_released_epoch_fenced".into(),
366            self.route_released_epoch_fenced
367                .load(Ordering::Relaxed)
368                .into(),
369        );
370        snapshot.insert(
371            "route_release_stale_skipped".into(),
372            self.route_release_stale_skipped
373                .load(Ordering::Relaxed)
374                .into(),
375        );
376        snapshot.insert(
377            "drains_with_undeclared_gauge".into(),
378            self.drains_with_undeclared_gauge
379                .load(Ordering::Relaxed)
380                .into(),
381        );
382        Value::Object(snapshot)
383    }
384
385    pub(crate) fn increment_route_busy(&self, module: &str, principal: &str) {
386        let mut diagnostics = self
387            .reader_diagnostics
388            .lock()
389            .unwrap_or_else(|p| p.into_inner());
390        *diagnostics
391            .busy
392            .entry(module.into())
393            .or_default()
394            .entry(principal.into())
395            .or_default() += 1;
396    }
397
398    /// Observe a wait without imposing a deadline or cancelling its future.
399    /// Counts advance once at five seconds, even if the peer never recovers.
400    pub(crate) async fn observe_reader_wait<F: std::future::Future>(
401        &self,
402        labels: impl FnOnce() -> (String, String),
403        kind: &'static str,
404        future: F,
405    ) -> F::Output {
406        tokio::pin!(future);
407        // Ready sends need neither a timer nor diagnostic labels. Poll before
408        // constructing Sleep, which would otherwise touch the runtime clock.
409        if let std::task::Poll::Ready(result) =
410            std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
411        {
412            return result;
413        }
414        tokio::select! {
415            biased;
416            result = &mut future => return result,
417            _ = tokio::time::sleep(Duration::from_secs(5)) => {}
418        }
419        let (module, principal) = labels();
420        let should_warn = {
421            let mut diagnostics = self
422                .reader_diagnostics
423                .lock()
424                .unwrap_or_else(|p| p.into_inner());
425            *diagnostics
426                .waits
427                .entry(module.clone())
428                .or_default()
429                .entry(principal.clone())
430                .or_default()
431                .entry(kind.into())
432                .or_default() += 1;
433            let now = tokio::time::Instant::now();
434            let key = (module.to_owned(), principal.to_owned());
435            let should_warn = diagnostics
436                .warned_waits
437                .get(&key)
438                .is_none_or(|last| now.duration_since(*last) >= Duration::from_secs(60));
439            if should_warn {
440                diagnostics.warned_waits.insert(key, now);
441            }
442            should_warn
443        };
444        if should_warn {
445            warn!(
446                module_id = module,
447                client_principal = principal,
448                wait_kind = kind,
449                "connection reader wait exceeded 5 seconds"
450            );
451        }
452        future.await
453    }
454
455    pub(crate) fn increment_module_frames_dropped_no_route(&self, module_id: Option<&str>) {
456        self.module_frames_dropped_no_route
457            .fetch_add(1, Ordering::Relaxed);
458        if let Some(module_id) = module_id {
459            increment_keyed_count(&self.module_frames_dropped_no_route_by_module, module_id);
460        }
461        self.module_frames_dropped_no_route_window
462            .lock()
463            .expect("drop-rate window mutex poisoned")
464            .record(tokio::time::Instant::now());
465    }
466
467    /// Count a dropped module frame whose (channel, epoch) the daemon had
468    /// allocated on that connection and since released. Called in addition to
469    /// [`Self::increment_module_frames_dropped_no_route`], never instead of it.
470    pub(crate) fn increment_module_frames_dropped_released_route(&self, module_id: Option<&str>) {
471        self.module_frames_dropped_released_route
472            .fetch_add(1, Ordering::Relaxed);
473        if let Some(module_id) = module_id {
474            increment_keyed_count(
475                &self.module_frames_dropped_released_route_by_module,
476                module_id,
477            );
478        }
479    }
480
481    pub(crate) fn increment_module_orphan_route_goodbyes_sent(&self) {
482        self.module_orphan_route_goodbyes_sent
483            .fetch_add(1, Ordering::Relaxed);
484    }
485
486    pub(crate) fn increment_route_open_refused(&self, code: &'static str) {
487        debug_assert!(ROUTE_OPEN_REFUSAL_COUNTER_CODES.contains(&code));
488        increment_keyed_count(&self.route_open_refused_by_code, code);
489    }
490
491    /// Count an accepted route.open by the principal the daemon stamped.
492    ///
493    /// THE KEY SPACE IS CLOSED BY CONSTRUCTION, unlike the refusal counter which
494    /// needs an explicit allowlist: a principal is `direct` or
495    /// `reserved:<module_id>`, and a module id was already refused at HELLO
496    /// unless it is a single path component free of control characters. So an
497    /// untrusted string cannot expand this map without first passing module-id
498    /// validation, and the bound is the number of modules rather than the number
499    /// of distinct strings a caller can invent.
500    pub(crate) fn increment_route_open_accepted(&self, principal: &str) {
501        increment_keyed_count(&self.route_open_accepted_by_principal, principal);
502    }
503
504    pub(crate) fn increment_module_requests_dropped_stale_route(&self) {
505        self.module_requests_dropped_stale_route
506            .fetch_add(1, Ordering::Relaxed);
507    }
508
509    pub(crate) fn increment_client_frames_dropped_stale_route(&self) {
510        self.client_frames_dropped_stale_route
511            .fetch_add(1, Ordering::Relaxed);
512    }
513
514    pub(crate) fn increment_client_egress_close_delivery_failed(&self) {
515        self.client_egress_close_delivery_failed
516            .fetch_add(1, Ordering::Relaxed);
517    }
518
519    pub(crate) fn increment_goodbye_relay_client_failed(&self) {
520        self.goodbye_relay_client_failed
521            .fetch_add(1, Ordering::Relaxed);
522    }
523
524    pub(crate) fn increment_goodbye_relay_module_dropped(&self, module_id: Option<&str>) {
525        self.goodbye_relay_module_dropped
526            .fetch_add(1, Ordering::Relaxed);
527        if let Some(module_id) = module_id {
528            increment_keyed_count(&self.goodbye_relay_module_dropped_by_module, module_id);
529        }
530    }
531
532    pub(crate) fn increment_route_released_epoch_fenced(&self) {
533        self.route_released_epoch_fenced
534            .fetch_add(1, Ordering::Relaxed);
535    }
536
537    pub(crate) fn increment_route_release_stale_skipped(&self) {
538        self.route_release_stale_skipped
539            .fetch_add(1, Ordering::Relaxed);
540    }
541
542    pub(crate) fn increment_drains_with_undeclared_gauge(&self) {
543        self.drains_with_undeclared_gauge
544            .fetch_add(1, Ordering::Relaxed);
545    }
546}
547
548fn increment_keyed_count(counts: &Mutex<HashMap<String, u64>>, key: &str) {
549    *counts
550        .lock()
551        .expect("keyed counter mutex poisoned")
552        .entry(key.to_string())
553        .or_default() += 1;
554}
555
556fn insert_nonempty_counts(
557    snapshot: &mut serde_json::Map<String, Value>,
558    key: &str,
559    counts: &Mutex<HashMap<String, u64>>,
560) {
561    let counts: MutexGuard<'_, HashMap<String, u64>> =
562        counts.lock().expect("keyed counter mutex poisoned");
563    if !counts.is_empty() {
564        snapshot.insert(key.to_string(), json!(&*counts));
565    }
566}
567
568#[cfg(test)]
569mod tests {
570    use super::*;
571
572    #[test]
573    fn ready_reader_wait_needs_no_runtime_clock_or_labels() {
574        use std::{
575            future::Future,
576            task::{Context, Poll, Waker},
577        };
578
579        let counters = DaemonCounters::new();
580        let wait = counters.observe_reader_wait(
581            || panic!("ready waits must not resolve diagnostic labels"),
582            "module_queue",
583            std::future::ready(7),
584        );
585        let mut wait = std::pin::pin!(wait);
586        let mut cx = Context::from_waker(Waker::noop());
587        // No Tokio runtime exists here: even constructing Sleep would panic.
588        assert_eq!(wait.as_mut().poll(&mut cx), Poll::Ready(7));
589        assert!(counters.reader_diagnostics.lock().unwrap().waits.is_empty());
590    }
591
592    #[tokio::test(start_paused = true)]
593    async fn reader_wait_labels_are_resolved_once_only_after_five_seconds() {
594        use std::{
595            cell::Cell,
596            future::{poll_fn, Future},
597            task::Poll,
598        };
599
600        let counters = DaemonCounters::new();
601        let calls = Cell::new(0);
602        let (sender, receiver) = tokio::sync::oneshot::channel();
603        let wait = counters.observe_reader_wait(
604            || {
605                calls.set(calls.get() + 1);
606                ("alpha".into(), "direct".into())
607            },
608            "module_queue",
609            receiver,
610        );
611        tokio::pin!(wait);
612        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
613            .await
614            .is_pending());
615        tokio::time::advance(Duration::from_secs(4)).await;
616        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
617            .await
618            .is_pending());
619        assert_eq!(calls.get(), 0);
620        tokio::time::advance(Duration::from_secs(1)).await;
621        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
622            .await
623            .is_pending());
624        assert_eq!(calls.get(), 1);
625        assert_eq!(
626            counters.snapshot()["reader_wait_over_5s_by_module_principal"]["alpha"]["direct"]
627                ["module_queue"],
628            1
629        );
630        sender.send(()).unwrap();
631        wait.await.unwrap();
632        assert_eq!(calls.get(), 1);
633    }
634
635    #[test]
636    fn counter_snapshot_includes_zero_rate_and_omits_empty_module_maps() {
637        let counters = DaemonCounters::new();
638        let snapshot = counters.snapshot();
639
640        assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 0);
641        assert_eq!(
642            snapshot["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
643            0
644        );
645        assert!(snapshot
646            .get("module_frames_dropped_no_route_by_module")
647            .is_none());
648        assert!(snapshot
649            .get("goodbye_relay_module_dropped_by_module")
650            .is_none());
651    }
652
653    #[tokio::test(start_paused = true)]
654    async fn module_frame_drops_are_attributed_to_the_emitting_module() {
655        let counters = DaemonCounters::new();
656        counters.increment_module_frames_dropped_no_route(Some("alpha"));
657        counters.increment_module_frames_dropped_no_route(Some("alpha"));
658
659        let snapshot = counters.snapshot();
660        assert_eq!(snapshot["module_frames_dropped_no_route"], 2);
661        assert_eq!(
662            snapshot["module_frames_dropped_no_route_by_module"],
663            json!({ "alpha": 2 })
664        );
665        assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 2);
666    }
667
668    #[tokio::test(start_paused = true)]
669    async fn frame_drop_rate_ages_out_after_ten_minute_buckets() {
670        let counters = DaemonCounters::new();
671        counters.increment_module_frames_dropped_no_route(Some("alpha"));
672
673        tokio::time::advance(Duration::from_secs(9 * 60)).await;
674        assert_eq!(
675            counters.snapshot()["module_frames_dropped_no_route_last_10m"],
676            1
677        );
678
679        tokio::time::advance(Duration::from_secs(60)).await;
680        assert_eq!(
681            counters.snapshot()["module_frames_dropped_no_route_last_10m"],
682            0
683        );
684    }
685
686    #[tokio::test(start_paused = true)]
687    async fn frame_drop_window_counts_only_nonzero_minutes() {
688        let counters = DaemonCounters::new();
689        for minute in 0..10 {
690            if minute > 0 {
691                tokio::time::advance(Duration::from_secs(60)).await;
692            }
693            if minute != 4 {
694                counters.increment_module_frames_dropped_no_route(Some("alpha"));
695            }
696        }
697
698        assert_eq!(
699            counters.snapshot()["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
700            9
701        );
702    }
703
704    #[test]
705    fn goodbye_relay_drops_are_attributed_to_the_target_module() {
706        let counters = DaemonCounters::new();
707        counters.increment_goodbye_relay_module_dropped(Some("alpha"));
708
709        assert_eq!(
710            counters.snapshot()["goodbye_relay_module_dropped_by_module"],
711            json!({ "alpha": 1 })
712        );
713    }
714}