Skip to main content

subc_daemon/
observability.rs

1use std::{
2    collections::{BTreeMap, HashMap, VecDeque},
3    sync::{
4        atomic::{AtomicU64, Ordering},
5        Arc, Mutex, MutexGuard,
6    },
7    time::Duration,
8};
9
10use serde_json::{json, Value};
11use tracing::{debug, warn};
12
13use crate::registry::ConnectionId;
14
15/// The slash prevents collision with a module ID: `module_id_path_hazard`
16/// rejects separators in `registry.rs:855-875`.
17pub(crate) const CONTROL_WAIT_BUCKET: &str = "daemon/control";
18
19/// The only keys the route.open refusal counter may carry. A module's own
20/// error code on a rejected bind is counted under `module_rejected` and rides
21/// the log event as a separate field: if the module's string were the key, a
22/// module could grow this map for the daemon's lifetime and push terminal
23/// control sequences through `ck daemon` to an operator's screen. The
24/// `&'static str` increment signature plus the debug assertion keep the set
25/// closed at the call site, not just here.
26const ROUTE_OPEN_REFUSAL_COUNTER_CODES: &[&str] = &[
27    "module_warming",
28    ROUTE_OPEN_REFUSED_DECLARED_NOT_READY,
29    ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
30    "target_unavailable",
31    subc_protocol::error_codes::TARGET_FLOW_UNSUPPORTED,
32    subc_protocol::error_codes::TARGET_FEDERATED_UNSUPPORTED,
33    subc_protocol::error_codes::TARGET_AGENT_RUN_UNSUPPORTED,
34    "module_removed",
35    "module_no_protocol",
36    "unknown_module",
37    "module_reloading",
38    "op_not_allowed",
39    "bad_consumer_identity",
40    "capability_forbidden",
41    "admission_facts_not_permitted",
42    "admission_facts_target_not_allowed",
43    "route_limit",
44    "forwarding_error",
45    "module_timeout",
46    ROUTE_OPEN_REFUSED_BREAKER_OPEN,
47    "module_rejected",
48    subc_protocol::error_codes::SCOPE_EPOCH_REQUIRED,
49    subc_protocol::error_codes::SCOPE_NOT_SYNCED,
50    subc_protocol::error_codes::SCOPE_NOT_LIVE,
51    subc_protocol::error_codes::SCOPE_ENDED,
52    subc_protocol::error_codes::SCOPE_NOT_CARRIER,
53    subc_protocol::error_codes::SCOPE_BIND_PROOF_REQUIRED,
54    subc_protocol::error_codes::SCOPE_BIND_PROOF_MISMATCH,
55    subc_protocol::error_codes::SCOPE_CHANGED,
56    subc_protocol::error_codes::INVALID_REQUEST,
57];
58
59/// Counter key for a `route.open` refused by the per-module bind-relay breaker
60/// before any relay was attempted.
61///
62/// The frame the caller receives carries `module_timeout`, because both SDKs
63/// already classify that as retryable with capped backoff and inventing a new
64/// wire code would need a change in each of them. The COUNTER is deliberately a
65/// different key: "this module burned the full bind budget" and "this module is
66/// being refused in microseconds because it already did that repeatedly" are
67/// the two states an operator most needs to tell apart, and they are
68/// indistinguishable from the client side, where both look like one retryable
69/// error that the next attempt may well satisfy.
70pub(crate) const ROUTE_OPEN_REFUSED_BREAKER_OPEN: &str = "module_timeout_breaker_open";
71
72/// Counter key for a registered module that declared itself not ready.
73///
74/// The caller still receives `module_warming`, but operators must be able to
75/// distinguish declared readiness from a supervised process that has not
76/// registered yet.
77pub(crate) const ROUTE_OPEN_REFUSED_DECLARED_NOT_READY: &str = "module_warming_declared_not_ready";
78
79/// Counter key for a registered, declared-ready module held not-ready because
80/// a capability it declares `need: required` has no registered provider.
81///
82/// The caller still receives `module_warming`; a separate key lets an operator
83/// tell "the module says it is warming" from "the module is waiting on a
84/// provider that is not running", which point at different fixes.
85pub(crate) const ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED: &str =
86    "module_warming_required_capability_unprovided";
87
88/// Shared count of authenticated socket connections accepted by the daemon.
89#[derive(Debug, Clone, Default)]
90pub struct ConnectedClients {
91    count: Arc<AtomicU64>,
92}
93
94impl ConnectedClients {
95    pub fn new() -> Self {
96        Self::default()
97    }
98
99    pub fn count(&self) -> u64 {
100        self.count.load(Ordering::SeqCst)
101    }
102
103    pub(crate) fn open(&self, connection_id: ConnectionId) -> ConnectedClientGuard {
104        let previous = self.count.fetch_add(1, Ordering::SeqCst);
105        let current = previous + 1;
106        // DEBUG, not INFO: this pair fired on every connect and disconnect and was
107        // measured at 43-74% of the daemon's own log (issue #114), burying the
108        // lines an operator opens the file for. The count itself is not lost: it
109        // is served live as `connected_clients` on server.describe (`ck daemon`),
110        // and a connection that opens a route still names its connection_id on the
111        // `route.open accepted` line, so per-connection forensics survive where
112        // they matter. Raise the filter (`CK_LOG=subc=debug`) to see every churn.
113        debug!(
114            connection_id = connection_id.get(),
115            connected_clients = current,
116            previous_connected_clients = previous,
117            "authenticated connection count changed"
118        );
119        ConnectedClientGuard {
120            clients: self.clone(),
121            connection_id,
122        }
123    }
124}
125
126pub(crate) struct ConnectedClientGuard {
127    clients: ConnectedClients,
128    connection_id: ConnectionId,
129}
130
131impl Drop for ConnectedClientGuard {
132    fn drop(&mut self) {
133        let previous = self.clients.count.fetch_sub(1, Ordering::SeqCst);
134        let current = previous.saturating_sub(1);
135        // DEBUG for the same reason as the open side above.
136        debug!(
137            connection_id = self.connection_id.get(),
138            connected_clients = current,
139            previous_connected_clients = previous,
140            "authenticated connection count changed"
141        );
142    }
143}
144
145/// Counters for route lifecycle drops, delivery failures and reader waits.
146#[derive(Debug, Clone, Default)]
147pub struct DaemonCounters {
148    reader_diagnostics: Arc<Mutex<ReaderDiagnostics>>,
149    carried_containment_anchor_lost: Arc<AtomicU64>,
150    /// Every non-request frame a module sent on a (channel, epoch) the daemon
151    /// holds no bound route for, dropped. Counts both kinds below: orphan
152    /// traffic on a route the daemon released, and frames on a (channel,
153    /// epoch) that was never allocated on that module connection.
154    module_frames_dropped_no_route: Arc<AtomicU64>,
155    /// The orphan-traffic part of `module_frames_dropped_no_route`: the daemon
156    /// had allocated that (channel, epoch) on the sending module connection and
157    /// has since released it, so the module is still holding a route the
158    /// daemon closed. `module_frames_dropped_no_route` minus this is the count
159    /// of frames on a (channel, epoch) that never existed on the connection.
160    module_frames_dropped_released_route: Arc<AtomicU64>,
161    module_frames_dropped_released_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
162    /// Route GOODBYEs the daemon enqueued to a module in answer to its frames
163    /// on a (channel, epoch) the daemon holds no route for, telling the module
164    /// to drop the route. Rate-limited per module connection and channel, so
165    /// this counts answers, not orphan frames.
166    module_orphan_route_goodbyes_sent: Arc<AtomicU64>,
167    // Per-module maps and the rate window are daemon-lifetime diagnostics only:
168    // they deliberately reset on restart instead of becoming durable daemon state.
169    module_frames_dropped_no_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
170    agent_command_refusals_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
171    agent_command_refusals_dropped_unlabelled: Arc<std::sync::atomic::AtomicU64>,
172    route_open_refused_by_code: Arc<Mutex<HashMap<String, u64>>>,
173    route_open_accepted_by_principal: Arc<Mutex<HashMap<String, u64>>>,
174    module_frames_dropped_no_route_window: Arc<Mutex<DropWindow>>,
175    module_requests_dropped_stale_route: Arc<AtomicU64>,
176    client_frames_dropped_stale_route: Arc<AtomicU64>,
177    client_egress_close_delivery_failed: Arc<AtomicU64>,
178    goodbye_relay_client_failed: Arc<AtomicU64>,
179    goodbye_relay_module_dropped: Arc<AtomicU64>,
180    goodbye_relay_module_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
181    route_released_epoch_fenced: Arc<AtomicU64>,
182    route_release_stale_skipped: Arc<AtomicU64>,
183    drains_with_undeclared_gauge: Arc<AtomicU64>,
184}
185
186#[derive(Debug, Default)]
187struct ReaderDiagnostics {
188    busy: BTreeMap<String, BTreeMap<String, u64>>,
189    waits: BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>>,
190    warned_waits: HashMap<(String, String), tokio::time::Instant>,
191}
192
193/// Ten one-minute buckets make sustained module-to-client route drops visible
194/// without retaining one record for every dropped frame.
195#[derive(Debug)]
196struct DropWindow {
197    started_at: tokio::time::Instant,
198    buckets: VecDeque<DropBucket>,
199}
200
201#[derive(Debug)]
202struct DropBucket {
203    minute: u64,
204    count: u64,
205}
206
207impl Default for DropWindow {
208    fn default() -> Self {
209        Self {
210            started_at: tokio::time::Instant::now(),
211            buckets: VecDeque::new(),
212        }
213    }
214}
215
216impl DropWindow {
217    const MINUTE: Duration = Duration::from_secs(60);
218    const BUCKETS: u64 = 10;
219
220    fn record(&mut self, now: tokio::time::Instant) {
221        let minute = self.minute_at(now);
222        self.prune_before(minute);
223        match self.buckets.back_mut() {
224            Some(bucket) if bucket.minute == minute => bucket.count += 1,
225            _ => self.buckets.push_back(DropBucket { minute, count: 1 }),
226        }
227    }
228
229    fn count_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
230        let minute = self.minute_at(now);
231        self.prune_before(minute);
232        self.buckets.iter().map(|bucket| bucket.count).sum()
233    }
234
235    fn nonzero_minutes_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
236        let minute = self.minute_at(now);
237        self.prune_before(minute);
238        self.buckets.len() as u64
239    }
240
241    fn minute_at(&self, now: tokio::time::Instant) -> u64 {
242        now.saturating_duration_since(self.started_at).as_secs() / Self::MINUTE.as_secs()
243    }
244
245    fn prune_before(&mut self, current_minute: u64) {
246        while self
247            .buckets
248            .front()
249            .is_some_and(|bucket| current_minute.saturating_sub(bucket.minute) >= Self::BUCKETS)
250        {
251            self.buckets.pop_front();
252        }
253    }
254}
255
256impl DaemonCounters {
257    pub fn new() -> Self {
258        Self::default()
259    }
260
261    /// Returns a JSON snapshot whose stable, additive schema keeps the
262    /// `server.describe` diagnostic endpoint backward-compatible.
263    pub fn snapshot(&self) -> Value {
264        let mut snapshot = serde_json::Map::new();
265        let diagnostics = self
266            .reader_diagnostics
267            .lock()
268            .unwrap_or_else(|p| p.into_inner());
269        if !diagnostics.busy.is_empty() {
270            snapshot.insert(
271                "route_busy_by_module_principal".into(),
272                json!(diagnostics.busy),
273            );
274        }
275        if !diagnostics.waits.is_empty() {
276            snapshot.insert(
277                "reader_wait_over_5s_by_module_principal".into(),
278                json!(diagnostics.waits),
279            );
280        }
281        drop(diagnostics);
282        snapshot.insert(
283            "carried_containment_anchor_lost".into(),
284            self.carried_containment_anchor_lost
285                .load(Ordering::Relaxed)
286                .into(),
287        );
288        snapshot.insert(
289            "module_frames_dropped_no_route".into(),
290            self.module_frames_dropped_no_route
291                .load(Ordering::Relaxed)
292                .into(),
293        );
294        let mut drop_window = self
295            .module_frames_dropped_no_route_window
296            .lock()
297            .expect("drop-rate window mutex poisoned");
298        let now = tokio::time::Instant::now();
299        snapshot.insert(
300            "module_frames_dropped_no_route_last_10m".into(),
301            drop_window.count_last_10m(now).into(),
302        );
303        snapshot.insert(
304            "module_frames_dropped_no_route_nonzero_minutes_last_10m".into(),
305            drop_window.nonzero_minutes_last_10m(now).into(),
306        );
307        insert_nonempty_counts(
308            &mut snapshot,
309            "module_frames_dropped_no_route_by_module",
310            &self.module_frames_dropped_no_route_by_module,
311        );
312        insert_nonempty_counts(
313            &mut snapshot,
314            "agent_command_refusals_dropped_by_module",
315            &self.agent_command_refusals_dropped_by_module,
316        );
317        let unlabelled = self
318            .agent_command_refusals_dropped_unlabelled
319            .load(Ordering::Relaxed);
320        if unlabelled != 0 {
321            snapshot.insert(
322                "agent_command_refusals_dropped_unlabelled".into(),
323                unlabelled.into(),
324            );
325        }
326        snapshot.insert(
327            "module_frames_dropped_released_route".into(),
328            self.module_frames_dropped_released_route
329                .load(Ordering::Relaxed)
330                .into(),
331        );
332        insert_nonempty_counts(
333            &mut snapshot,
334            "module_frames_dropped_released_route_by_module",
335            &self.module_frames_dropped_released_route_by_module,
336        );
337        snapshot.insert(
338            "module_orphan_route_goodbyes_sent".into(),
339            self.module_orphan_route_goodbyes_sent
340                .load(Ordering::Relaxed)
341                .into(),
342        );
343        insert_nonempty_counts(
344            &mut snapshot,
345            "route_open_refused_by_code",
346            &self.route_open_refused_by_code,
347        );
348        insert_nonempty_counts(
349            &mut snapshot,
350            "route_open_accepted_by_principal",
351            &self.route_open_accepted_by_principal,
352        );
353        snapshot.insert(
354            "module_requests_dropped_stale_route".into(),
355            self.module_requests_dropped_stale_route
356                .load(Ordering::Relaxed)
357                .into(),
358        );
359        snapshot.insert(
360            "client_frames_dropped_stale_route".into(),
361            self.client_frames_dropped_stale_route
362                .load(Ordering::Relaxed)
363                .into(),
364        );
365        snapshot.insert(
366            "client_egress_close_delivery_failed".into(),
367            self.client_egress_close_delivery_failed
368                .load(Ordering::Relaxed)
369                .into(),
370        );
371        snapshot.insert(
372            "goodbye_relay_client_failed".into(),
373            self.goodbye_relay_client_failed
374                .load(Ordering::Relaxed)
375                .into(),
376        );
377        snapshot.insert(
378            "goodbye_relay_module_dropped".into(),
379            self.goodbye_relay_module_dropped
380                .load(Ordering::Relaxed)
381                .into(),
382        );
383        insert_nonempty_counts(
384            &mut snapshot,
385            "goodbye_relay_module_dropped_by_module",
386            &self.goodbye_relay_module_dropped_by_module,
387        );
388        snapshot.insert(
389            "route_released_epoch_fenced".into(),
390            self.route_released_epoch_fenced
391                .load(Ordering::Relaxed)
392                .into(),
393        );
394        snapshot.insert(
395            "route_release_stale_skipped".into(),
396            self.route_release_stale_skipped
397                .load(Ordering::Relaxed)
398                .into(),
399        );
400        snapshot.insert(
401            "drains_with_undeclared_gauge".into(),
402            self.drains_with_undeclared_gauge
403                .load(Ordering::Relaxed)
404                .into(),
405        );
406        Value::Object(snapshot)
407    }
408
409    pub(crate) fn increment_route_busy(&self, module: &str, principal: &str) {
410        let mut diagnostics = self
411            .reader_diagnostics
412            .lock()
413            .unwrap_or_else(|p| p.into_inner());
414        *diagnostics
415            .busy
416            .entry(module.into())
417            .or_default()
418            .entry(principal.into())
419            .or_default() += 1;
420    }
421
422    /// Observe a wait without imposing a deadline or cancelling its future.
423    /// Counts advance once at five seconds, even if the peer never recovers.
424    pub(crate) async fn observe_reader_wait<F: std::future::Future>(
425        &self,
426        labels: impl FnOnce() -> (String, String),
427        kind: &'static str,
428        future: F,
429    ) -> F::Output {
430        tokio::pin!(future);
431        // Ready sends need neither a timer nor diagnostic labels. Poll before
432        // constructing Sleep, which would otherwise touch the runtime clock.
433        if let std::task::Poll::Ready(result) =
434            std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
435        {
436            return result;
437        }
438        tokio::select! {
439            biased;
440            result = &mut future => return result,
441            _ = tokio::time::sleep(Duration::from_secs(5)) => {}
442        }
443        let (module, principal) = labels();
444        let should_warn = {
445            let mut diagnostics = self
446                .reader_diagnostics
447                .lock()
448                .unwrap_or_else(|p| p.into_inner());
449            *diagnostics
450                .waits
451                .entry(module.clone())
452                .or_default()
453                .entry(principal.clone())
454                .or_default()
455                .entry(kind.into())
456                .or_default() += 1;
457            let now = tokio::time::Instant::now();
458            let key = (module.to_owned(), principal.to_owned());
459            let should_warn = diagnostics
460                .warned_waits
461                .get(&key)
462                .is_none_or(|last| now.duration_since(*last) >= Duration::from_secs(60));
463            if should_warn {
464                diagnostics.warned_waits.insert(key, now);
465            }
466            should_warn
467        };
468        if should_warn {
469            warn!(
470                module_id = module,
471                client_principal = principal,
472                wait_kind = kind,
473                "connection reader wait exceeded 5 seconds"
474            );
475        }
476        future.await
477    }
478
479    pub(crate) fn increment_module_frames_dropped_no_route(&self, module_id: Option<&str>) {
480        self.module_frames_dropped_no_route
481            .fetch_add(1, Ordering::Relaxed);
482        if let Some(module_id) = module_id {
483            increment_keyed_count(&self.module_frames_dropped_no_route_by_module, module_id);
484        }
485        self.module_frames_dropped_no_route_window
486            .lock()
487            .expect("drop-rate window mutex poisoned")
488            .record(tokio::time::Instant::now());
489    }
490
491    pub(crate) fn increment_agent_command_refusal_dropped(
492        &self,
493        module_id: Option<&str>,
494        registry: &crate::registry::Registry,
495    ) {
496        let Some(module_id) =
497            module_id.filter(|id| registry.get_module(id).ok().flatten().is_some())
498        else {
499            self.agent_command_refusals_dropped_unlabelled
500                .fetch_add(1, Ordering::Relaxed);
501            return;
502        };
503        let mut counts = self
504            .agent_command_refusals_dropped_by_module
505            .lock()
506            .unwrap_or_else(|p| p.into_inner());
507        if let Some(count) = counts.get_mut(module_id) {
508            *count = count.saturating_add(1);
509        } else if counts.len() < MAX_AGENT_REFUSAL_MODULES {
510            counts.insert(module_id.to_owned(), 1);
511        } else {
512            self.agent_command_refusals_dropped_unlabelled
513                .fetch_add(1, Ordering::Relaxed);
514        }
515    }
516
517    /// Count a dropped module frame whose (channel, epoch) the daemon had
518    /// allocated on that connection and since released. Called in addition to
519    /// [`Self::increment_module_frames_dropped_no_route`], never instead of it.
520    pub(crate) fn increment_module_frames_dropped_released_route(&self, module_id: Option<&str>) {
521        self.module_frames_dropped_released_route
522            .fetch_add(1, Ordering::Relaxed);
523        if let Some(module_id) = module_id {
524            increment_keyed_count(
525                &self.module_frames_dropped_released_route_by_module,
526                module_id,
527            );
528        }
529    }
530
531    pub(crate) fn increment_module_orphan_route_goodbyes_sent(&self) {
532        self.module_orphan_route_goodbyes_sent
533            .fetch_add(1, Ordering::Relaxed);
534    }
535
536    pub(crate) fn increment_route_open_refused(&self, code: &'static str) {
537        debug_assert!(ROUTE_OPEN_REFUSAL_COUNTER_CODES.contains(&code));
538        increment_keyed_count(&self.route_open_refused_by_code, code);
539    }
540
541    /// Count an accepted route.open by the principal the daemon stamped.
542    ///
543    /// THE KEY SPACE IS CLOSED BY CONSTRUCTION, unlike the refusal counter which
544    /// needs an explicit allowlist: a principal is `direct` or
545    /// `reserved:<module_id>`, and a module id was already refused at HELLO
546    /// unless it is a single path component free of control characters. So an
547    /// untrusted string cannot expand this map without first passing module-id
548    /// validation, and the bound is the number of modules rather than the number
549    /// of distinct strings a caller can invent.
550    pub(crate) fn increment_route_open_accepted(&self, principal: &str) {
551        increment_keyed_count(&self.route_open_accepted_by_principal, principal);
552    }
553
554    pub(crate) fn increment_module_requests_dropped_stale_route(&self) {
555        self.module_requests_dropped_stale_route
556            .fetch_add(1, Ordering::Relaxed);
557    }
558
559    pub(crate) fn increment_client_frames_dropped_stale_route(&self) {
560        self.client_frames_dropped_stale_route
561            .fetch_add(1, Ordering::Relaxed);
562    }
563
564    pub(crate) fn increment_client_egress_close_delivery_failed(&self) {
565        self.client_egress_close_delivery_failed
566            .fetch_add(1, Ordering::Relaxed);
567    }
568
569    pub(crate) fn increment_goodbye_relay_client_failed(&self) {
570        self.goodbye_relay_client_failed
571            .fetch_add(1, Ordering::Relaxed);
572    }
573
574    pub(crate) fn increment_goodbye_relay_module_dropped(&self, module_id: Option<&str>) {
575        self.goodbye_relay_module_dropped
576            .fetch_add(1, Ordering::Relaxed);
577        if let Some(module_id) = module_id {
578            increment_keyed_count(&self.goodbye_relay_module_dropped_by_module, module_id);
579        }
580    }
581
582    pub(crate) fn increment_route_released_epoch_fenced(&self) {
583        self.route_released_epoch_fenced
584            .fetch_add(1, Ordering::Relaxed);
585    }
586
587    pub(crate) fn increment_route_release_stale_skipped(&self) {
588        self.route_release_stale_skipped
589            .fetch_add(1, Ordering::Relaxed);
590    }
591
592    pub(crate) fn increment_drains_with_undeclared_gauge(&self) {
593        self.drains_with_undeclared_gauge
594            .fetch_add(1, Ordering::Relaxed);
595    }
596
597    #[cfg(unix)]
598    pub(crate) fn increment_carried_containment_anchor_lost(&self) {
599        self.carried_containment_anchor_lost
600            .fetch_add(1, Ordering::Relaxed);
601    }
602}
603
604const MAX_AGENT_REFUSAL_MODULES: usize = 1_024;
605
606#[cfg(test)]
607mod refusal_label_tests {
608    use super::*;
609
610    #[test]
611    fn dropped_refusal_labels_require_registration_and_stop_at_the_hard_cap() {
612        let registry = crate::Registry::default();
613        let counters = DaemonCounters::default();
614        counters.increment_agent_command_refusal_dropped(Some("unregistered"), &registry);
615        assert!(counters
616            .snapshot()
617            .get("agent_command_refusals_dropped_by_module")
618            .is_none());
619        for index in 0..=1_024 {
620            let id = format!("registered-{index}");
621            registry
622                .register_with_control_ops(
623                    subc_protocol::manifest::ModuleManifest::builder(&id, "0.1.0")
624                        .protocol_ver(1)
625                        .build(),
626                    1,
627                    crate::ConnectionId::new(index as u64 + 1),
628                    Vec::new(),
629                )
630                .unwrap();
631            counters.increment_agent_command_refusal_dropped(Some(&id), &registry);
632        }
633        counters.increment_agent_command_refusal_dropped(Some("registered-0"), &registry);
634        let snapshot = counters.snapshot();
635        let labels = snapshot["agent_command_refusals_dropped_by_module"]
636            .as_object()
637            .unwrap();
638        assert_eq!(labels.len(), 1_024);
639        assert_eq!(labels["registered-0"], 2);
640        assert!(!labels.contains_key("registered-1024"));
641        assert_eq!(snapshot["agent_command_refusals_dropped_unlabelled"], 2);
642    }
643}
644
645fn increment_keyed_count(counts: &Mutex<HashMap<String, u64>>, key: &str) {
646    *counts
647        .lock()
648        .expect("keyed counter mutex poisoned")
649        .entry(key.to_string())
650        .or_default() += 1;
651}
652
653fn insert_nonempty_counts(
654    snapshot: &mut serde_json::Map<String, Value>,
655    key: &str,
656    counts: &Mutex<HashMap<String, u64>>,
657) {
658    let counts: MutexGuard<'_, HashMap<String, u64>> =
659        counts.lock().expect("keyed counter mutex poisoned");
660    if !counts.is_empty() {
661        snapshot.insert(key.to_string(), json!(&*counts));
662    }
663}
664
665#[cfg(test)]
666mod tests {
667    use super::*;
668
669    #[test]
670    fn ready_reader_wait_needs_no_runtime_clock_or_labels() {
671        use std::{
672            future::Future,
673            task::{Context, Poll, Waker},
674        };
675
676        let counters = DaemonCounters::new();
677        let wait = counters.observe_reader_wait(
678            || panic!("ready waits must not resolve diagnostic labels"),
679            "module_queue",
680            std::future::ready(7),
681        );
682        let mut wait = std::pin::pin!(wait);
683        let mut cx = Context::from_waker(Waker::noop());
684        // No Tokio runtime exists here: even constructing Sleep would panic.
685        assert_eq!(wait.as_mut().poll(&mut cx), Poll::Ready(7));
686        assert!(counters.reader_diagnostics.lock().unwrap().waits.is_empty());
687    }
688
689    #[tokio::test(start_paused = true)]
690    async fn reader_wait_labels_are_resolved_once_only_after_five_seconds() {
691        use std::{
692            cell::Cell,
693            future::{poll_fn, Future},
694            task::Poll,
695        };
696
697        let counters = DaemonCounters::new();
698        let calls = Cell::new(0);
699        let (sender, receiver) = tokio::sync::oneshot::channel();
700        let wait = counters.observe_reader_wait(
701            || {
702                calls.set(calls.get() + 1);
703                ("alpha".into(), "direct".into())
704            },
705            "module_queue",
706            receiver,
707        );
708        tokio::pin!(wait);
709        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
710            .await
711            .is_pending());
712        tokio::time::advance(Duration::from_secs(4)).await;
713        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
714            .await
715            .is_pending());
716        assert_eq!(calls.get(), 0);
717        tokio::time::advance(Duration::from_secs(1)).await;
718        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
719            .await
720            .is_pending());
721        assert_eq!(calls.get(), 1);
722        assert_eq!(
723            counters.snapshot()["reader_wait_over_5s_by_module_principal"]["alpha"]["direct"]
724                ["module_queue"],
725            1
726        );
727        sender.send(()).unwrap();
728        wait.await.unwrap();
729        assert_eq!(calls.get(), 1);
730    }
731
732    #[test]
733    fn counter_snapshot_includes_zero_rate_and_omits_empty_module_maps() {
734        let counters = DaemonCounters::new();
735        let snapshot = counters.snapshot();
736
737        assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 0);
738        assert_eq!(
739            snapshot["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
740            0
741        );
742        assert!(snapshot
743            .get("module_frames_dropped_no_route_by_module")
744            .is_none());
745        assert!(snapshot
746            .get("goodbye_relay_module_dropped_by_module")
747            .is_none());
748    }
749
750    #[tokio::test(start_paused = true)]
751    async fn module_frame_drops_are_attributed_to_the_emitting_module() {
752        let counters = DaemonCounters::new();
753        counters.increment_module_frames_dropped_no_route(Some("alpha"));
754        counters.increment_module_frames_dropped_no_route(Some("alpha"));
755
756        let snapshot = counters.snapshot();
757        assert_eq!(snapshot["module_frames_dropped_no_route"], 2);
758        assert_eq!(
759            snapshot["module_frames_dropped_no_route_by_module"],
760            json!({ "alpha": 2 })
761        );
762        assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 2);
763    }
764
765    #[tokio::test(start_paused = true)]
766    async fn frame_drop_rate_ages_out_after_ten_minute_buckets() {
767        let counters = DaemonCounters::new();
768        counters.increment_module_frames_dropped_no_route(Some("alpha"));
769
770        tokio::time::advance(Duration::from_secs(9 * 60)).await;
771        assert_eq!(
772            counters.snapshot()["module_frames_dropped_no_route_last_10m"],
773            1
774        );
775
776        tokio::time::advance(Duration::from_secs(60)).await;
777        assert_eq!(
778            counters.snapshot()["module_frames_dropped_no_route_last_10m"],
779            0
780        );
781    }
782
783    #[tokio::test(start_paused = true)]
784    async fn frame_drop_window_counts_only_nonzero_minutes() {
785        let counters = DaemonCounters::new();
786        for minute in 0..10 {
787            if minute > 0 {
788                tokio::time::advance(Duration::from_secs(60)).await;
789            }
790            if minute != 4 {
791                counters.increment_module_frames_dropped_no_route(Some("alpha"));
792            }
793        }
794
795        assert_eq!(
796            counters.snapshot()["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
797            9
798        );
799    }
800
801    #[test]
802    fn goodbye_relay_drops_are_attributed_to_the_target_module() {
803        let counters = DaemonCounters::new();
804        counters.increment_goodbye_relay_module_dropped(Some("alpha"));
805
806        assert_eq!(
807            counters.snapshot()["goodbye_relay_module_dropped_by_module"],
808            json!({ "alpha": 1 })
809        );
810    }
811}