Skip to main content

subc_daemon/
observability.rs

1use std::{
2    collections::{BTreeMap, HashMap, VecDeque},
3    sync::{
4        atomic::{AtomicU64, Ordering},
5        Arc, Mutex, MutexGuard,
6    },
7    time::Duration,
8};
9
10use serde_json::{json, Value};
11use tracing::{debug, warn};
12
13use crate::registry::ConnectionId;
14
15/// The slash prevents collision with a module ID: `module_id_path_hazard`
16/// rejects separators in `registry.rs:855-875`.
17pub(crate) const CONTROL_WAIT_BUCKET: &str = "daemon/control";
18
19/// The only keys the route.open refusal counter may carry. A module's own
20/// error code on a rejected bind is counted under `module_rejected` and rides
21/// the log event as a separate field: if the module's string were the key, a
22/// module could grow this map for the daemon's lifetime and push terminal
23/// control sequences through `ck daemon` to an operator's screen. The
24/// `&'static str` increment signature plus the debug assertion keep the set
25/// closed at the call site, not just here.
26const ROUTE_OPEN_REFUSAL_COUNTER_CODES: &[&str] = &[
27    "module_warming",
28    ROUTE_OPEN_REFUSED_DECLARED_NOT_READY,
29    ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
30    "target_unavailable",
31    subc_protocol::error_codes::TARGET_FLOW_UNSUPPORTED,
32    subc_protocol::error_codes::TARGET_FEDERATED_UNSUPPORTED,
33    subc_protocol::error_codes::FEDERATED_STAMP_REQUIRED,
34    subc_protocol::error_codes::TARGET_AGENT_RUN_UNSUPPORTED,
35    "module_removed",
36    "module_no_protocol",
37    "unknown_module",
38    "module_reloading",
39    "op_not_allowed",
40    "bad_consumer_identity",
41    "capability_forbidden",
42    "admission_facts_not_permitted",
43    "admission_facts_target_not_allowed",
44    "route_limit",
45    "forwarding_error",
46    "module_timeout",
47    ROUTE_OPEN_REFUSED_BREAKER_OPEN,
48    "module_rejected",
49    subc_protocol::error_codes::SCOPE_EPOCH_REQUIRED,
50    subc_protocol::error_codes::SCOPE_NOT_SYNCED,
51    subc_protocol::error_codes::SCOPE_NOT_LIVE,
52    subc_protocol::error_codes::SCOPE_ENDED,
53    subc_protocol::error_codes::SCOPE_NOT_CARRIER,
54    subc_protocol::error_codes::SCOPE_BIND_PROOF_REQUIRED,
55    subc_protocol::error_codes::SCOPE_BIND_PROOF_MISMATCH,
56    subc_protocol::error_codes::SCOPE_CHANGED,
57    subc_protocol::error_codes::INVALID_REQUEST,
58];
59
60/// Counter key for a `route.open` refused by the per-module bind-relay breaker
61/// before any relay was attempted.
62///
63/// The frame the caller receives carries `module_timeout`, because both SDKs
64/// already classify that as retryable with capped backoff and inventing a new
65/// wire code would need a change in each of them. The COUNTER is deliberately a
66/// different key: "this module burned the full bind budget" and "this module is
67/// being refused in microseconds because it already did that repeatedly" are
68/// the two states an operator most needs to tell apart, and they are
69/// indistinguishable from the client side, where both look like one retryable
70/// error that the next attempt may well satisfy.
71pub(crate) const ROUTE_OPEN_REFUSED_BREAKER_OPEN: &str = "module_timeout_breaker_open";
72
73/// Counter key for a registered module that declared itself not ready.
74///
75/// The caller still receives `module_warming`, but operators must be able to
76/// distinguish declared readiness from a supervised process that has not
77/// registered yet.
78pub(crate) const ROUTE_OPEN_REFUSED_DECLARED_NOT_READY: &str = "module_warming_declared_not_ready";
79
80/// Counter key for a registered, declared-ready module held not-ready because
81/// a capability it declares `need: required` has no registered provider.
82///
83/// The caller still receives `module_warming`; a separate key lets an operator
84/// tell "the module says it is warming" from "the module is waiting on a
85/// provider that is not running", which point at different fixes.
86pub(crate) const ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED: &str =
87    "module_warming_required_capability_unprovided";
88
89/// Shared count of authenticated socket connections accepted by the daemon.
90#[derive(Debug, Clone, Default)]
91pub struct ConnectedClients {
92    count: Arc<AtomicU64>,
93}
94
95impl ConnectedClients {
96    pub fn new() -> Self {
97        Self::default()
98    }
99
100    pub fn count(&self) -> u64 {
101        self.count.load(Ordering::SeqCst)
102    }
103
104    pub(crate) fn open(&self, connection_id: ConnectionId) -> ConnectedClientGuard {
105        let previous = self.count.fetch_add(1, Ordering::SeqCst);
106        let current = previous + 1;
107        // DEBUG, not INFO: this pair fired on every connect and disconnect and was
108        // measured at 43-74% of the daemon's own log (issue #114), burying the
109        // lines an operator opens the file for. The count itself is not lost: it
110        // is served live as `connected_clients` on server.describe (`ck daemon`),
111        // and a connection that opens a route still names its connection_id on the
112        // `route.open accepted` line, so per-connection forensics survive where
113        // they matter. Raise the filter (`CK_LOG=subc=debug`) to see every churn.
114        debug!(
115            connection_id = connection_id.get(),
116            connected_clients = current,
117            previous_connected_clients = previous,
118            "authenticated connection count changed"
119        );
120        ConnectedClientGuard {
121            clients: self.clone(),
122            connection_id,
123        }
124    }
125}
126
127pub(crate) struct ConnectedClientGuard {
128    clients: ConnectedClients,
129    connection_id: ConnectionId,
130}
131
132impl Drop for ConnectedClientGuard {
133    fn drop(&mut self) {
134        let previous = self.clients.count.fetch_sub(1, Ordering::SeqCst);
135        let current = previous.saturating_sub(1);
136        // DEBUG for the same reason as the open side above.
137        debug!(
138            connection_id = self.connection_id.get(),
139            connected_clients = current,
140            previous_connected_clients = previous,
141            "authenticated connection count changed"
142        );
143    }
144}
145
146/// Counters for route lifecycle drops, delivery failures and reader waits.
147#[derive(Debug, Clone, Default)]
148pub struct DaemonCounters {
149    reader_diagnostics: Arc<Mutex<ReaderDiagnostics>>,
150    carried_containment_anchor_lost: Arc<AtomicU64>,
151    /// Every non-request frame a module sent on a (channel, epoch) the daemon
152    /// holds no bound route for, dropped. Counts both kinds below: orphan
153    /// traffic on a route the daemon released, and frames on a (channel,
154    /// epoch) that was never allocated on that module connection.
155    module_frames_dropped_no_route: Arc<AtomicU64>,
156    /// The orphan-traffic part of `module_frames_dropped_no_route`: the daemon
157    /// had allocated that (channel, epoch) on the sending module connection and
158    /// has since released it, so the module is still holding a route the
159    /// daemon closed. `module_frames_dropped_no_route` minus this is the count
160    /// of frames on a (channel, epoch) that never existed on the connection.
161    module_frames_dropped_released_route: Arc<AtomicU64>,
162    module_frames_dropped_released_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
163    /// Route GOODBYEs the daemon enqueued to a module in answer to its frames
164    /// on a (channel, epoch) the daemon holds no route for, telling the module
165    /// to drop the route. Rate-limited per module connection and channel, so
166    /// this counts answers, not orphan frames.
167    module_orphan_route_goodbyes_sent: Arc<AtomicU64>,
168    // Per-module maps and the rate window are daemon-lifetime diagnostics only:
169    // they deliberately reset on restart instead of becoming durable daemon state.
170    module_frames_dropped_no_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
171    agent_command_refusals_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
172    agent_command_refusals_dropped_unlabelled: Arc<std::sync::atomic::AtomicU64>,
173    route_open_refused_by_code: Arc<Mutex<HashMap<String, u64>>>,
174    route_open_accepted_by_principal: Arc<Mutex<HashMap<String, u64>>>,
175    module_frames_dropped_no_route_window: Arc<Mutex<DropWindow>>,
176    module_requests_dropped_stale_route: Arc<AtomicU64>,
177    client_frames_dropped_stale_route: Arc<AtomicU64>,
178    client_egress_close_delivery_failed: Arc<AtomicU64>,
179    goodbye_relay_client_failed: Arc<AtomicU64>,
180    goodbye_relay_module_dropped: Arc<AtomicU64>,
181    goodbye_relay_module_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
182    route_released_epoch_fenced: Arc<AtomicU64>,
183    route_release_stale_skipped: Arc<AtomicU64>,
184    drains_with_undeclared_gauge: Arc<AtomicU64>,
185}
186
187#[derive(Debug, Default)]
188struct ReaderDiagnostics {
189    busy: BTreeMap<String, BTreeMap<String, u64>>,
190    waits: BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>>,
191    warned_waits: HashMap<(String, String), tokio::time::Instant>,
192}
193
194/// Ten one-minute buckets make sustained module-to-client route drops visible
195/// without retaining one record for every dropped frame.
196#[derive(Debug)]
197struct DropWindow {
198    started_at: tokio::time::Instant,
199    buckets: VecDeque<DropBucket>,
200}
201
202#[derive(Debug)]
203struct DropBucket {
204    minute: u64,
205    count: u64,
206}
207
208impl Default for DropWindow {
209    fn default() -> Self {
210        Self {
211            started_at: tokio::time::Instant::now(),
212            buckets: VecDeque::new(),
213        }
214    }
215}
216
217impl DropWindow {
218    const MINUTE: Duration = Duration::from_secs(60);
219    const BUCKETS: u64 = 10;
220
221    fn record(&mut self, now: tokio::time::Instant) {
222        let minute = self.minute_at(now);
223        self.prune_before(minute);
224        match self.buckets.back_mut() {
225            Some(bucket) if bucket.minute == minute => bucket.count += 1,
226            _ => self.buckets.push_back(DropBucket { minute, count: 1 }),
227        }
228    }
229
230    fn count_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
231        let minute = self.minute_at(now);
232        self.prune_before(minute);
233        self.buckets.iter().map(|bucket| bucket.count).sum()
234    }
235
236    fn nonzero_minutes_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
237        let minute = self.minute_at(now);
238        self.prune_before(minute);
239        self.buckets.len() as u64
240    }
241
242    fn minute_at(&self, now: tokio::time::Instant) -> u64 {
243        now.saturating_duration_since(self.started_at).as_secs() / Self::MINUTE.as_secs()
244    }
245
246    fn prune_before(&mut self, current_minute: u64) {
247        while self
248            .buckets
249            .front()
250            .is_some_and(|bucket| current_minute.saturating_sub(bucket.minute) >= Self::BUCKETS)
251        {
252            self.buckets.pop_front();
253        }
254    }
255}
256
257impl DaemonCounters {
258    pub fn new() -> Self {
259        Self::default()
260    }
261
262    /// Returns a JSON snapshot whose stable, additive schema keeps the
263    /// `server.describe` diagnostic endpoint backward-compatible.
264    pub fn snapshot(&self) -> Value {
265        let mut snapshot = serde_json::Map::new();
266        let diagnostics = self
267            .reader_diagnostics
268            .lock()
269            .unwrap_or_else(|p| p.into_inner());
270        if !diagnostics.busy.is_empty() {
271            snapshot.insert(
272                "route_busy_by_module_principal".into(),
273                json!(diagnostics.busy),
274            );
275        }
276        if !diagnostics.waits.is_empty() {
277            snapshot.insert(
278                "reader_wait_over_5s_by_module_principal".into(),
279                json!(diagnostics.waits),
280            );
281        }
282        drop(diagnostics);
283        snapshot.insert(
284            "carried_containment_anchor_lost".into(),
285            self.carried_containment_anchor_lost
286                .load(Ordering::Relaxed)
287                .into(),
288        );
289        snapshot.insert(
290            "module_frames_dropped_no_route".into(),
291            self.module_frames_dropped_no_route
292                .load(Ordering::Relaxed)
293                .into(),
294        );
295        let mut drop_window = self
296            .module_frames_dropped_no_route_window
297            .lock()
298            .expect("drop-rate window mutex poisoned");
299        let now = tokio::time::Instant::now();
300        snapshot.insert(
301            "module_frames_dropped_no_route_last_10m".into(),
302            drop_window.count_last_10m(now).into(),
303        );
304        snapshot.insert(
305            "module_frames_dropped_no_route_nonzero_minutes_last_10m".into(),
306            drop_window.nonzero_minutes_last_10m(now).into(),
307        );
308        insert_nonempty_counts(
309            &mut snapshot,
310            "module_frames_dropped_no_route_by_module",
311            &self.module_frames_dropped_no_route_by_module,
312        );
313        insert_nonempty_counts(
314            &mut snapshot,
315            "agent_command_refusals_dropped_by_module",
316            &self.agent_command_refusals_dropped_by_module,
317        );
318        let unlabelled = self
319            .agent_command_refusals_dropped_unlabelled
320            .load(Ordering::Relaxed);
321        if unlabelled != 0 {
322            snapshot.insert(
323                "agent_command_refusals_dropped_unlabelled".into(),
324                unlabelled.into(),
325            );
326        }
327        snapshot.insert(
328            "module_frames_dropped_released_route".into(),
329            self.module_frames_dropped_released_route
330                .load(Ordering::Relaxed)
331                .into(),
332        );
333        insert_nonempty_counts(
334            &mut snapshot,
335            "module_frames_dropped_released_route_by_module",
336            &self.module_frames_dropped_released_route_by_module,
337        );
338        snapshot.insert(
339            "module_orphan_route_goodbyes_sent".into(),
340            self.module_orphan_route_goodbyes_sent
341                .load(Ordering::Relaxed)
342                .into(),
343        );
344        insert_nonempty_counts(
345            &mut snapshot,
346            "route_open_refused_by_code",
347            &self.route_open_refused_by_code,
348        );
349        insert_nonempty_counts(
350            &mut snapshot,
351            "route_open_accepted_by_principal",
352            &self.route_open_accepted_by_principal,
353        );
354        snapshot.insert(
355            "module_requests_dropped_stale_route".into(),
356            self.module_requests_dropped_stale_route
357                .load(Ordering::Relaxed)
358                .into(),
359        );
360        snapshot.insert(
361            "client_frames_dropped_stale_route".into(),
362            self.client_frames_dropped_stale_route
363                .load(Ordering::Relaxed)
364                .into(),
365        );
366        snapshot.insert(
367            "client_egress_close_delivery_failed".into(),
368            self.client_egress_close_delivery_failed
369                .load(Ordering::Relaxed)
370                .into(),
371        );
372        snapshot.insert(
373            "goodbye_relay_client_failed".into(),
374            self.goodbye_relay_client_failed
375                .load(Ordering::Relaxed)
376                .into(),
377        );
378        snapshot.insert(
379            "goodbye_relay_module_dropped".into(),
380            self.goodbye_relay_module_dropped
381                .load(Ordering::Relaxed)
382                .into(),
383        );
384        insert_nonempty_counts(
385            &mut snapshot,
386            "goodbye_relay_module_dropped_by_module",
387            &self.goodbye_relay_module_dropped_by_module,
388        );
389        snapshot.insert(
390            "route_released_epoch_fenced".into(),
391            self.route_released_epoch_fenced
392                .load(Ordering::Relaxed)
393                .into(),
394        );
395        snapshot.insert(
396            "route_release_stale_skipped".into(),
397            self.route_release_stale_skipped
398                .load(Ordering::Relaxed)
399                .into(),
400        );
401        snapshot.insert(
402            "drains_with_undeclared_gauge".into(),
403            self.drains_with_undeclared_gauge
404                .load(Ordering::Relaxed)
405                .into(),
406        );
407        Value::Object(snapshot)
408    }
409
410    pub(crate) fn increment_route_busy(&self, module: &str, principal: &str) {
411        let mut diagnostics = self
412            .reader_diagnostics
413            .lock()
414            .unwrap_or_else(|p| p.into_inner());
415        *diagnostics
416            .busy
417            .entry(module.into())
418            .or_default()
419            .entry(principal.into())
420            .or_default() += 1;
421    }
422
423    /// Observe a wait without imposing a deadline or cancelling its future.
424    /// Counts advance once at five seconds, even if the peer never recovers.
425    pub(crate) async fn observe_reader_wait<F: std::future::Future>(
426        &self,
427        labels: impl FnOnce() -> (String, String),
428        kind: &'static str,
429        future: F,
430    ) -> F::Output {
431        tokio::pin!(future);
432        // Ready sends need neither a timer nor diagnostic labels. Poll before
433        // constructing Sleep, which would otherwise touch the runtime clock.
434        if let std::task::Poll::Ready(result) =
435            std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
436        {
437            return result;
438        }
439        tokio::select! {
440            biased;
441            result = &mut future => return result,
442            _ = tokio::time::sleep(Duration::from_secs(5)) => {}
443        }
444        let (module, principal) = labels();
445        let should_warn = {
446            let mut diagnostics = self
447                .reader_diagnostics
448                .lock()
449                .unwrap_or_else(|p| p.into_inner());
450            *diagnostics
451                .waits
452                .entry(module.clone())
453                .or_default()
454                .entry(principal.clone())
455                .or_default()
456                .entry(kind.into())
457                .or_default() += 1;
458            let now = tokio::time::Instant::now();
459            let key = (module.to_owned(), principal.to_owned());
460            let should_warn = diagnostics
461                .warned_waits
462                .get(&key)
463                .is_none_or(|last| now.duration_since(*last) >= Duration::from_secs(60));
464            if should_warn {
465                diagnostics.warned_waits.insert(key, now);
466            }
467            should_warn
468        };
469        if should_warn {
470            warn!(
471                module_id = module,
472                client_principal = principal,
473                wait_kind = kind,
474                "connection reader wait exceeded 5 seconds"
475            );
476        }
477        future.await
478    }
479
480    pub(crate) fn increment_module_frames_dropped_no_route(&self, module_id: Option<&str>) {
481        self.module_frames_dropped_no_route
482            .fetch_add(1, Ordering::Relaxed);
483        if let Some(module_id) = module_id {
484            increment_keyed_count(&self.module_frames_dropped_no_route_by_module, module_id);
485        }
486        self.module_frames_dropped_no_route_window
487            .lock()
488            .expect("drop-rate window mutex poisoned")
489            .record(tokio::time::Instant::now());
490    }
491
492    pub(crate) fn increment_agent_command_refusal_dropped(
493        &self,
494        module_id: Option<&str>,
495        registry: &crate::registry::Registry,
496    ) {
497        let Some(module_id) =
498            module_id.filter(|id| registry.get_module(id).ok().flatten().is_some())
499        else {
500            self.agent_command_refusals_dropped_unlabelled
501                .fetch_add(1, Ordering::Relaxed);
502            return;
503        };
504        let mut counts = self
505            .agent_command_refusals_dropped_by_module
506            .lock()
507            .unwrap_or_else(|p| p.into_inner());
508        if let Some(count) = counts.get_mut(module_id) {
509            *count = count.saturating_add(1);
510        } else if counts.len() < MAX_AGENT_REFUSAL_MODULES {
511            counts.insert(module_id.to_owned(), 1);
512        } else {
513            self.agent_command_refusals_dropped_unlabelled
514                .fetch_add(1, Ordering::Relaxed);
515        }
516    }
517
518    /// Count a dropped module frame whose (channel, epoch) the daemon had
519    /// allocated on that connection and since released. Called in addition to
520    /// [`Self::increment_module_frames_dropped_no_route`], never instead of it.
521    pub(crate) fn increment_module_frames_dropped_released_route(&self, module_id: Option<&str>) {
522        self.module_frames_dropped_released_route
523            .fetch_add(1, Ordering::Relaxed);
524        if let Some(module_id) = module_id {
525            increment_keyed_count(
526                &self.module_frames_dropped_released_route_by_module,
527                module_id,
528            );
529        }
530    }
531
532    pub(crate) fn increment_module_orphan_route_goodbyes_sent(&self) {
533        self.module_orphan_route_goodbyes_sent
534            .fetch_add(1, Ordering::Relaxed);
535    }
536
537    pub(crate) fn increment_route_open_refused(&self, code: &'static str) {
538        debug_assert!(ROUTE_OPEN_REFUSAL_COUNTER_CODES.contains(&code));
539        increment_keyed_count(&self.route_open_refused_by_code, code);
540    }
541
542    /// Count an accepted route.open by the principal the daemon stamped.
543    ///
544    /// THE KEY SPACE IS CLOSED BY CONSTRUCTION, unlike the refusal counter which
545    /// needs an explicit allowlist: a principal is `direct` or
546    /// `reserved:<module_id>`, and a module id was already refused at HELLO
547    /// unless it is a single path component free of control characters. So an
548    /// untrusted string cannot expand this map without first passing module-id
549    /// validation, and the bound is the number of modules rather than the number
550    /// of distinct strings a caller can invent.
551    pub(crate) fn increment_route_open_accepted(&self, principal: &str) {
552        increment_keyed_count(&self.route_open_accepted_by_principal, principal);
553    }
554
555    pub(crate) fn increment_module_requests_dropped_stale_route(&self) {
556        self.module_requests_dropped_stale_route
557            .fetch_add(1, Ordering::Relaxed);
558    }
559
560    pub(crate) fn increment_client_frames_dropped_stale_route(&self) {
561        self.client_frames_dropped_stale_route
562            .fetch_add(1, Ordering::Relaxed);
563    }
564
565    pub(crate) fn increment_client_egress_close_delivery_failed(&self) {
566        self.client_egress_close_delivery_failed
567            .fetch_add(1, Ordering::Relaxed);
568    }
569
570    pub(crate) fn increment_goodbye_relay_client_failed(&self) {
571        self.goodbye_relay_client_failed
572            .fetch_add(1, Ordering::Relaxed);
573    }
574
575    pub(crate) fn increment_goodbye_relay_module_dropped(&self, module_id: Option<&str>) {
576        self.goodbye_relay_module_dropped
577            .fetch_add(1, Ordering::Relaxed);
578        if let Some(module_id) = module_id {
579            increment_keyed_count(&self.goodbye_relay_module_dropped_by_module, module_id);
580        }
581    }
582
583    pub(crate) fn increment_route_released_epoch_fenced(&self) {
584        self.route_released_epoch_fenced
585            .fetch_add(1, Ordering::Relaxed);
586    }
587
588    pub(crate) fn increment_route_release_stale_skipped(&self) {
589        self.route_release_stale_skipped
590            .fetch_add(1, Ordering::Relaxed);
591    }
592
593    pub(crate) fn increment_drains_with_undeclared_gauge(&self) {
594        self.drains_with_undeclared_gauge
595            .fetch_add(1, Ordering::Relaxed);
596    }
597
598    #[cfg(unix)]
599    pub(crate) fn increment_carried_containment_anchor_lost(&self) {
600        self.carried_containment_anchor_lost
601            .fetch_add(1, Ordering::Relaxed);
602    }
603}
604
605const MAX_AGENT_REFUSAL_MODULES: usize = 1_024;
606
607#[cfg(test)]
608mod refusal_label_tests {
609    use super::*;
610
611    #[test]
612    fn dropped_refusal_labels_require_registration_and_stop_at_the_hard_cap() {
613        let registry = crate::Registry::default();
614        let counters = DaemonCounters::default();
615        counters.increment_agent_command_refusal_dropped(Some("unregistered"), &registry);
616        assert!(counters
617            .snapshot()
618            .get("agent_command_refusals_dropped_by_module")
619            .is_none());
620        for index in 0..=1_024 {
621            let id = format!("registered-{index}");
622            registry
623                .register_with_control_ops(
624                    subc_protocol::manifest::ModuleManifest::builder(&id, "0.1.0")
625                        .protocol_ver(1)
626                        .build(),
627                    1,
628                    crate::ConnectionId::new(index as u64 + 1),
629                    Vec::new(),
630                )
631                .unwrap();
632            counters.increment_agent_command_refusal_dropped(Some(&id), &registry);
633        }
634        counters.increment_agent_command_refusal_dropped(Some("registered-0"), &registry);
635        let snapshot = counters.snapshot();
636        let labels = snapshot["agent_command_refusals_dropped_by_module"]
637            .as_object()
638            .unwrap();
639        assert_eq!(labels.len(), 1_024);
640        assert_eq!(labels["registered-0"], 2);
641        assert!(!labels.contains_key("registered-1024"));
642        assert_eq!(snapshot["agent_command_refusals_dropped_unlabelled"], 2);
643    }
644}
645
646fn increment_keyed_count(counts: &Mutex<HashMap<String, u64>>, key: &str) {
647    *counts
648        .lock()
649        .expect("keyed counter mutex poisoned")
650        .entry(key.to_string())
651        .or_default() += 1;
652}
653
654fn insert_nonempty_counts(
655    snapshot: &mut serde_json::Map<String, Value>,
656    key: &str,
657    counts: &Mutex<HashMap<String, u64>>,
658) {
659    let counts: MutexGuard<'_, HashMap<String, u64>> =
660        counts.lock().expect("keyed counter mutex poisoned");
661    if !counts.is_empty() {
662        snapshot.insert(key.to_string(), json!(&*counts));
663    }
664}
665
666#[cfg(test)]
667mod tests {
668    use super::*;
669
670    #[test]
671    fn ready_reader_wait_needs_no_runtime_clock_or_labels() {
672        use std::{
673            future::Future,
674            task::{Context, Poll, Waker},
675        };
676
677        let counters = DaemonCounters::new();
678        let wait = counters.observe_reader_wait(
679            || panic!("ready waits must not resolve diagnostic labels"),
680            "module_queue",
681            std::future::ready(7),
682        );
683        let mut wait = std::pin::pin!(wait);
684        let mut cx = Context::from_waker(Waker::noop());
685        // No Tokio runtime exists here: even constructing Sleep would panic.
686        assert_eq!(wait.as_mut().poll(&mut cx), Poll::Ready(7));
687        assert!(counters.reader_diagnostics.lock().unwrap().waits.is_empty());
688    }
689
690    #[tokio::test(start_paused = true)]
691    async fn reader_wait_labels_are_resolved_once_only_after_five_seconds() {
692        use std::{
693            cell::Cell,
694            future::{poll_fn, Future},
695            task::Poll,
696        };
697
698        let counters = DaemonCounters::new();
699        let calls = Cell::new(0);
700        let (sender, receiver) = tokio::sync::oneshot::channel();
701        let wait = counters.observe_reader_wait(
702            || {
703                calls.set(calls.get() + 1);
704                ("alpha".into(), "direct".into())
705            },
706            "module_queue",
707            receiver,
708        );
709        tokio::pin!(wait);
710        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
711            .await
712            .is_pending());
713        tokio::time::advance(Duration::from_secs(4)).await;
714        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
715            .await
716            .is_pending());
717        assert_eq!(calls.get(), 0);
718        tokio::time::advance(Duration::from_secs(1)).await;
719        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
720            .await
721            .is_pending());
722        assert_eq!(calls.get(), 1);
723        assert_eq!(
724            counters.snapshot()["reader_wait_over_5s_by_module_principal"]["alpha"]["direct"]
725                ["module_queue"],
726            1
727        );
728        sender.send(()).unwrap();
729        wait.await.unwrap();
730        assert_eq!(calls.get(), 1);
731    }
732
733    #[test]
734    fn counter_snapshot_includes_zero_rate_and_omits_empty_module_maps() {
735        let counters = DaemonCounters::new();
736        let snapshot = counters.snapshot();
737
738        assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 0);
739        assert_eq!(
740            snapshot["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
741            0
742        );
743        assert!(snapshot
744            .get("module_frames_dropped_no_route_by_module")
745            .is_none());
746        assert!(snapshot
747            .get("goodbye_relay_module_dropped_by_module")
748            .is_none());
749    }
750
751    #[tokio::test(start_paused = true)]
752    async fn module_frame_drops_are_attributed_to_the_emitting_module() {
753        let counters = DaemonCounters::new();
754        counters.increment_module_frames_dropped_no_route(Some("alpha"));
755        counters.increment_module_frames_dropped_no_route(Some("alpha"));
756
757        let snapshot = counters.snapshot();
758        assert_eq!(snapshot["module_frames_dropped_no_route"], 2);
759        assert_eq!(
760            snapshot["module_frames_dropped_no_route_by_module"],
761            json!({ "alpha": 2 })
762        );
763        assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 2);
764    }
765
766    #[tokio::test(start_paused = true)]
767    async fn frame_drop_rate_ages_out_after_ten_minute_buckets() {
768        let counters = DaemonCounters::new();
769        counters.increment_module_frames_dropped_no_route(Some("alpha"));
770
771        tokio::time::advance(Duration::from_secs(9 * 60)).await;
772        assert_eq!(
773            counters.snapshot()["module_frames_dropped_no_route_last_10m"],
774            1
775        );
776
777        tokio::time::advance(Duration::from_secs(60)).await;
778        assert_eq!(
779            counters.snapshot()["module_frames_dropped_no_route_last_10m"],
780            0
781        );
782    }
783
784    #[tokio::test(start_paused = true)]
785    async fn frame_drop_window_counts_only_nonzero_minutes() {
786        let counters = DaemonCounters::new();
787        for minute in 0..10 {
788            if minute > 0 {
789                tokio::time::advance(Duration::from_secs(60)).await;
790            }
791            if minute != 4 {
792                counters.increment_module_frames_dropped_no_route(Some("alpha"));
793            }
794        }
795
796        assert_eq!(
797            counters.snapshot()["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
798            9
799        );
800    }
801
802    #[test]
803    fn goodbye_relay_drops_are_attributed_to_the_target_module() {
804        let counters = DaemonCounters::new();
805        counters.increment_goodbye_relay_module_dropped(Some("alpha"));
806
807        assert_eq!(
808            counters.snapshot()["goodbye_relay_module_dropped_by_module"],
809            json!({ "alpha": 1 })
810        );
811    }
812}