Skip to main content

subc_daemon/
observability.rs

1use std::{
2    collections::{BTreeMap, HashMap, VecDeque},
3    sync::{
4        atomic::{AtomicU64, Ordering},
5        Arc, Mutex, MutexGuard,
6    },
7    time::Duration,
8};
9
10use serde_json::{json, Value};
11use tracing::{debug, warn};
12
13use crate::registry::ConnectionId;
14
15/// The slash prevents collision with a module ID: `module_id_path_hazard`
16/// rejects separators in `registry.rs:855-875`.
17pub(crate) const CONTROL_WAIT_BUCKET: &str = "daemon/control";
18
19/// The only keys the route.open refusal counter may carry. A module's own
20/// error code on a rejected bind is counted under `module_rejected` and rides
21/// the log event as a separate field: if the module's string were the key, a
22/// module could grow this map for the daemon's lifetime and push terminal
23/// control sequences through `ck daemon` to an operator's screen. The
24/// `&'static str` increment signature plus the debug assertion keep the set
25/// closed at the call site, not just here.
26const ROUTE_OPEN_REFUSAL_COUNTER_CODES: &[&str] = &[
27    "module_warming",
28    ROUTE_OPEN_REFUSED_DECLARED_NOT_READY,
29    ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
30    "target_unavailable",
31    subc_protocol::error_codes::TARGET_FLOW_UNSUPPORTED,
32    subc_protocol::error_codes::TARGET_FEDERATED_UNSUPPORTED,
33    subc_protocol::error_codes::FEDERATED_STAMP_REQUIRED,
34    subc_protocol::error_codes::TARGET_AGENT_RUN_UNSUPPORTED,
35    "module_removed",
36    "module_no_protocol",
37    "unknown_module",
38    "module_reloading",
39    "op_not_allowed",
40    "bad_consumer_identity",
41    crate::supervise::REGISTRATION_CHANNEL_CLOSED,
42    "capability_forbidden",
43    "admission_facts_not_permitted",
44    "admission_facts_target_not_allowed",
45    "route_limit",
46    "forwarding_error",
47    "module_timeout",
48    ROUTE_OPEN_REFUSED_BREAKER_OPEN,
49    "module_rejected",
50    subc_protocol::error_codes::SCOPE_EPOCH_REQUIRED,
51    subc_protocol::error_codes::SCOPE_NOT_SYNCED,
52    subc_protocol::error_codes::SCOPE_NOT_LIVE,
53    subc_protocol::error_codes::SCOPE_ENDED,
54    subc_protocol::error_codes::SCOPE_NOT_CARRIER,
55    subc_protocol::error_codes::SCOPE_BIND_PROOF_REQUIRED,
56    subc_protocol::error_codes::SCOPE_BIND_PROOF_MISMATCH,
57    subc_protocol::error_codes::SCOPE_CHANGED,
58    subc_protocol::error_codes::INVALID_REQUEST,
59];
60
61/// Counter key for a `route.open` refused by the per-module bind-relay breaker
62/// before any relay was attempted.
63///
64/// The frame the caller receives carries `module_timeout`, because both SDKs
65/// already classify that as retryable with capped backoff and inventing a new
66/// wire code would need a change in each of them. The COUNTER is deliberately a
67/// different key: "this module burned the full bind budget" and "this module is
68/// being refused in microseconds because it already did that repeatedly" are
69/// the two states an operator most needs to tell apart, and they are
70/// indistinguishable from the client side, where both look like one retryable
71/// error that the next attempt may well satisfy.
72pub(crate) const ROUTE_OPEN_REFUSED_BREAKER_OPEN: &str = "module_timeout_breaker_open";
73
74/// Counter key for a registered module that declared itself not ready.
75///
76/// The caller still receives `module_warming`, but operators must be able to
77/// distinguish declared readiness from a supervised process that has not
78/// registered yet.
79pub(crate) const ROUTE_OPEN_REFUSED_DECLARED_NOT_READY: &str = "module_warming_declared_not_ready";
80
81/// Counter key for a registered, declared-ready module held not-ready because
82/// a capability it declares `need: required` has no registered provider.
83///
84/// The caller still receives `module_warming`; a separate key lets an operator
85/// tell "the module says it is warming" from "the module is waiting on a
86/// provider that is not running", which point at different fixes.
87pub(crate) const ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED: &str =
88    "module_warming_required_capability_unprovided";
89
90/// Shared count of authenticated socket connections accepted by the daemon.
91#[derive(Debug, Clone, Default)]
92pub struct ConnectedClients {
93    count: Arc<AtomicU64>,
94}
95
96impl ConnectedClients {
97    pub fn new() -> Self {
98        Self::default()
99    }
100
101    pub fn count(&self) -> u64 {
102        self.count.load(Ordering::SeqCst)
103    }
104
105    pub(crate) fn open(&self, connection_id: ConnectionId) -> ConnectedClientGuard {
106        let previous = self.count.fetch_add(1, Ordering::SeqCst);
107        let current = previous + 1;
108        // DEBUG, not INFO: this pair fired on every connect and disconnect and was
109        // measured at 43-74% of the daemon's own log (issue #114), burying the
110        // lines an operator opens the file for. The count itself is not lost: it
111        // is served live as `connected_clients` on server.describe (`ck daemon`),
112        // and a connection that opens a route still names its connection_id on the
113        // `route.open accepted` line, so per-connection forensics survive where
114        // they matter. Raise the filter (`CK_LOG=subc=debug`) to see every churn.
115        debug!(
116            connection_id = connection_id.get(),
117            connected_clients = current,
118            previous_connected_clients = previous,
119            "authenticated connection count changed"
120        );
121        ConnectedClientGuard {
122            clients: self.clone(),
123            connection_id,
124        }
125    }
126}
127
128pub(crate) struct ConnectedClientGuard {
129    clients: ConnectedClients,
130    connection_id: ConnectionId,
131}
132
133impl Drop for ConnectedClientGuard {
134    fn drop(&mut self) {
135        let previous = self.clients.count.fetch_sub(1, Ordering::SeqCst);
136        let current = previous.saturating_sub(1);
137        // DEBUG for the same reason as the open side above.
138        debug!(
139            connection_id = self.connection_id.get(),
140            connected_clients = current,
141            previous_connected_clients = previous,
142            "authenticated connection count changed"
143        );
144    }
145}
146
147/// Counters for route lifecycle drops, delivery failures and reader waits.
148#[derive(Debug, Clone, Default)]
149pub struct DaemonCounters {
150    reader_diagnostics: Arc<Mutex<ReaderDiagnostics>>,
151    carried_containment_anchor_lost: Arc<AtomicU64>,
152    /// Every non-request frame a module sent on a (channel, epoch) the daemon
153    /// holds no bound route for, dropped. Counts both kinds below: orphan
154    /// traffic on a route the daemon released, and frames on a (channel,
155    /// epoch) that was never allocated on that module connection.
156    module_frames_dropped_no_route: Arc<AtomicU64>,
157    /// The orphan-traffic part of `module_frames_dropped_no_route`: the daemon
158    /// had allocated that (channel, epoch) on the sending module connection and
159    /// has since released it, so the module is still holding a route the
160    /// daemon closed. `module_frames_dropped_no_route` minus this is the count
161    /// of frames on a (channel, epoch) that never existed on the connection.
162    module_frames_dropped_released_route: Arc<AtomicU64>,
163    module_frames_dropped_released_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
164    /// Route GOODBYEs the daemon enqueued to a module in answer to its frames
165    /// on a (channel, epoch) the daemon holds no route for, telling the module
166    /// to drop the route. Rate-limited per module connection and channel, so
167    /// this counts answers, not orphan frames.
168    module_orphan_route_goodbyes_sent: Arc<AtomicU64>,
169    // Per-module maps and the rate window are daemon-lifetime diagnostics only:
170    // they deliberately reset on restart instead of becoming durable daemon state.
171    module_frames_dropped_no_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
172    agent_command_refusals_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
173    agent_command_refusals_dropped_unlabelled: Arc<std::sync::atomic::AtomicU64>,
174    route_open_refused_by_code: Arc<Mutex<HashMap<String, u64>>>,
175    route_open_accepted_by_principal: Arc<Mutex<HashMap<String, u64>>>,
176    module_frames_dropped_no_route_window: Arc<Mutex<DropWindow>>,
177    module_requests_dropped_stale_route: Arc<AtomicU64>,
178    client_frames_dropped_stale_route: Arc<AtomicU64>,
179    client_egress_close_delivery_failed: Arc<AtomicU64>,
180    goodbye_relay_client_failed: Arc<AtomicU64>,
181    goodbye_relay_module_dropped: Arc<AtomicU64>,
182    goodbye_relay_module_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
183    route_released_epoch_fenced: Arc<AtomicU64>,
184    route_release_stale_skipped: Arc<AtomicU64>,
185    drains_with_undeclared_gauge: Arc<AtomicU64>,
186}
187
188#[derive(Debug, Default)]
189struct ReaderDiagnostics {
190    busy: BTreeMap<String, BTreeMap<String, u64>>,
191    waits: BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>>,
192    warned_waits: HashMap<(String, String), tokio::time::Instant>,
193}
194
195/// Ten one-minute buckets make sustained module-to-client route drops visible
196/// without retaining one record for every dropped frame.
197#[derive(Debug)]
198struct DropWindow {
199    started_at: tokio::time::Instant,
200    buckets: VecDeque<DropBucket>,
201}
202
203#[derive(Debug)]
204struct DropBucket {
205    minute: u64,
206    count: u64,
207}
208
209impl Default for DropWindow {
210    fn default() -> Self {
211        Self {
212            started_at: tokio::time::Instant::now(),
213            buckets: VecDeque::new(),
214        }
215    }
216}
217
218impl DropWindow {
219    const MINUTE: Duration = Duration::from_secs(60);
220    const BUCKETS: u64 = 10;
221
222    fn record(&mut self, now: tokio::time::Instant) {
223        let minute = self.minute_at(now);
224        self.prune_before(minute);
225        match self.buckets.back_mut() {
226            Some(bucket) if bucket.minute == minute => bucket.count += 1,
227            _ => self.buckets.push_back(DropBucket { minute, count: 1 }),
228        }
229    }
230
231    fn count_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
232        let minute = self.minute_at(now);
233        self.prune_before(minute);
234        self.buckets.iter().map(|bucket| bucket.count).sum()
235    }
236
237    fn nonzero_minutes_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
238        let minute = self.minute_at(now);
239        self.prune_before(minute);
240        self.buckets.len() as u64
241    }
242
243    fn minute_at(&self, now: tokio::time::Instant) -> u64 {
244        now.saturating_duration_since(self.started_at).as_secs() / Self::MINUTE.as_secs()
245    }
246
247    fn prune_before(&mut self, current_minute: u64) {
248        while self
249            .buckets
250            .front()
251            .is_some_and(|bucket| current_minute.saturating_sub(bucket.minute) >= Self::BUCKETS)
252        {
253            self.buckets.pop_front();
254        }
255    }
256}
257
258impl DaemonCounters {
259    pub fn new() -> Self {
260        Self::default()
261    }
262
263    /// Returns a JSON snapshot whose stable, additive schema keeps the
264    /// `server.describe` diagnostic endpoint backward-compatible.
265    pub fn snapshot(&self) -> Value {
266        let mut snapshot = serde_json::Map::new();
267        let diagnostics = self
268            .reader_diagnostics
269            .lock()
270            .unwrap_or_else(|p| p.into_inner());
271        if !diagnostics.busy.is_empty() {
272            snapshot.insert(
273                "route_busy_by_module_principal".into(),
274                json!(diagnostics.busy),
275            );
276        }
277        if !diagnostics.waits.is_empty() {
278            snapshot.insert(
279                "reader_wait_over_5s_by_module_principal".into(),
280                json!(diagnostics.waits),
281            );
282        }
283        drop(diagnostics);
284        snapshot.insert(
285            "carried_containment_anchor_lost".into(),
286            self.carried_containment_anchor_lost
287                .load(Ordering::Relaxed)
288                .into(),
289        );
290        snapshot.insert(
291            "module_frames_dropped_no_route".into(),
292            self.module_frames_dropped_no_route
293                .load(Ordering::Relaxed)
294                .into(),
295        );
296        let mut drop_window = self
297            .module_frames_dropped_no_route_window
298            .lock()
299            .expect("drop-rate window mutex poisoned");
300        let now = tokio::time::Instant::now();
301        snapshot.insert(
302            "module_frames_dropped_no_route_last_10m".into(),
303            drop_window.count_last_10m(now).into(),
304        );
305        snapshot.insert(
306            "module_frames_dropped_no_route_nonzero_minutes_last_10m".into(),
307            drop_window.nonzero_minutes_last_10m(now).into(),
308        );
309        insert_nonempty_counts(
310            &mut snapshot,
311            "module_frames_dropped_no_route_by_module",
312            &self.module_frames_dropped_no_route_by_module,
313        );
314        insert_nonempty_counts(
315            &mut snapshot,
316            "agent_command_refusals_dropped_by_module",
317            &self.agent_command_refusals_dropped_by_module,
318        );
319        let unlabelled = self
320            .agent_command_refusals_dropped_unlabelled
321            .load(Ordering::Relaxed);
322        if unlabelled != 0 {
323            snapshot.insert(
324                "agent_command_refusals_dropped_unlabelled".into(),
325                unlabelled.into(),
326            );
327        }
328        snapshot.insert(
329            "module_frames_dropped_released_route".into(),
330            self.module_frames_dropped_released_route
331                .load(Ordering::Relaxed)
332                .into(),
333        );
334        insert_nonempty_counts(
335            &mut snapshot,
336            "module_frames_dropped_released_route_by_module",
337            &self.module_frames_dropped_released_route_by_module,
338        );
339        snapshot.insert(
340            "module_orphan_route_goodbyes_sent".into(),
341            self.module_orphan_route_goodbyes_sent
342                .load(Ordering::Relaxed)
343                .into(),
344        );
345        insert_nonempty_counts(
346            &mut snapshot,
347            "route_open_refused_by_code",
348            &self.route_open_refused_by_code,
349        );
350        insert_nonempty_counts(
351            &mut snapshot,
352            "route_open_accepted_by_principal",
353            &self.route_open_accepted_by_principal,
354        );
355        snapshot.insert(
356            "module_requests_dropped_stale_route".into(),
357            self.module_requests_dropped_stale_route
358                .load(Ordering::Relaxed)
359                .into(),
360        );
361        snapshot.insert(
362            "client_frames_dropped_stale_route".into(),
363            self.client_frames_dropped_stale_route
364                .load(Ordering::Relaxed)
365                .into(),
366        );
367        snapshot.insert(
368            "client_egress_close_delivery_failed".into(),
369            self.client_egress_close_delivery_failed
370                .load(Ordering::Relaxed)
371                .into(),
372        );
373        snapshot.insert(
374            "goodbye_relay_client_failed".into(),
375            self.goodbye_relay_client_failed
376                .load(Ordering::Relaxed)
377                .into(),
378        );
379        snapshot.insert(
380            "goodbye_relay_module_dropped".into(),
381            self.goodbye_relay_module_dropped
382                .load(Ordering::Relaxed)
383                .into(),
384        );
385        insert_nonempty_counts(
386            &mut snapshot,
387            "goodbye_relay_module_dropped_by_module",
388            &self.goodbye_relay_module_dropped_by_module,
389        );
390        snapshot.insert(
391            "route_released_epoch_fenced".into(),
392            self.route_released_epoch_fenced
393                .load(Ordering::Relaxed)
394                .into(),
395        );
396        snapshot.insert(
397            "route_release_stale_skipped".into(),
398            self.route_release_stale_skipped
399                .load(Ordering::Relaxed)
400                .into(),
401        );
402        snapshot.insert(
403            "drains_with_undeclared_gauge".into(),
404            self.drains_with_undeclared_gauge
405                .load(Ordering::Relaxed)
406                .into(),
407        );
408        Value::Object(snapshot)
409    }
410
411    pub(crate) fn increment_route_busy(&self, module: &str, principal: &str) {
412        let mut diagnostics = self
413            .reader_diagnostics
414            .lock()
415            .unwrap_or_else(|p| p.into_inner());
416        *diagnostics
417            .busy
418            .entry(module.into())
419            .or_default()
420            .entry(principal.into())
421            .or_default() += 1;
422    }
423
424    /// Observe a wait without imposing a deadline or cancelling its future.
425    /// Counts advance once at five seconds, even if the peer never recovers.
426    pub(crate) async fn observe_reader_wait<F: std::future::Future>(
427        &self,
428        labels: impl FnOnce() -> (String, String),
429        kind: &'static str,
430        future: F,
431    ) -> F::Output {
432        tokio::pin!(future);
433        // Ready sends need neither a timer nor diagnostic labels. Poll before
434        // constructing Sleep, which would otherwise touch the runtime clock.
435        if let std::task::Poll::Ready(result) =
436            std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
437        {
438            return result;
439        }
440        tokio::select! {
441            biased;
442            result = &mut future => return result,
443            _ = tokio::time::sleep(Duration::from_secs(5)) => {}
444        }
445        let (module, principal) = labels();
446        let should_warn = {
447            let mut diagnostics = self
448                .reader_diagnostics
449                .lock()
450                .unwrap_or_else(|p| p.into_inner());
451            *diagnostics
452                .waits
453                .entry(module.clone())
454                .or_default()
455                .entry(principal.clone())
456                .or_default()
457                .entry(kind.into())
458                .or_default() += 1;
459            let now = tokio::time::Instant::now();
460            let key = (module.to_owned(), principal.to_owned());
461            let should_warn = diagnostics
462                .warned_waits
463                .get(&key)
464                .is_none_or(|last| now.duration_since(*last) >= Duration::from_secs(60));
465            if should_warn {
466                diagnostics.warned_waits.insert(key, now);
467            }
468            should_warn
469        };
470        if should_warn {
471            warn!(
472                module_id = module,
473                client_principal = principal,
474                wait_kind = kind,
475                "connection reader wait exceeded 5 seconds"
476            );
477        }
478        future.await
479    }
480
481    pub(crate) fn increment_module_frames_dropped_no_route(&self, module_id: Option<&str>) {
482        self.module_frames_dropped_no_route
483            .fetch_add(1, Ordering::Relaxed);
484        if let Some(module_id) = module_id {
485            increment_keyed_count(&self.module_frames_dropped_no_route_by_module, module_id);
486        }
487        self.module_frames_dropped_no_route_window
488            .lock()
489            .expect("drop-rate window mutex poisoned")
490            .record(tokio::time::Instant::now());
491    }
492
493    pub(crate) fn increment_agent_command_refusal_dropped(
494        &self,
495        module_id: Option<&str>,
496        registry: &crate::registry::Registry,
497    ) {
498        let Some(module_id) =
499            module_id.filter(|id| registry.get_module(id).ok().flatten().is_some())
500        else {
501            self.agent_command_refusals_dropped_unlabelled
502                .fetch_add(1, Ordering::Relaxed);
503            return;
504        };
505        let mut counts = self
506            .agent_command_refusals_dropped_by_module
507            .lock()
508            .unwrap_or_else(|p| p.into_inner());
509        if let Some(count) = counts.get_mut(module_id) {
510            *count = count.saturating_add(1);
511        } else if counts.len() < MAX_AGENT_REFUSAL_MODULES {
512            counts.insert(module_id.to_owned(), 1);
513        } else {
514            self.agent_command_refusals_dropped_unlabelled
515                .fetch_add(1, Ordering::Relaxed);
516        }
517    }
518
519    /// Count a dropped module frame whose (channel, epoch) the daemon had
520    /// allocated on that connection and since released. Called in addition to
521    /// [`Self::increment_module_frames_dropped_no_route`], never instead of it.
522    pub(crate) fn increment_module_frames_dropped_released_route(&self, module_id: Option<&str>) {
523        self.module_frames_dropped_released_route
524            .fetch_add(1, Ordering::Relaxed);
525        if let Some(module_id) = module_id {
526            increment_keyed_count(
527                &self.module_frames_dropped_released_route_by_module,
528                module_id,
529            );
530        }
531    }
532
533    pub(crate) fn increment_module_orphan_route_goodbyes_sent(&self) {
534        self.module_orphan_route_goodbyes_sent
535            .fetch_add(1, Ordering::Relaxed);
536    }
537
538    pub(crate) fn increment_route_open_refused(&self, code: &'static str) {
539        debug_assert!(ROUTE_OPEN_REFUSAL_COUNTER_CODES.contains(&code));
540        increment_keyed_count(&self.route_open_refused_by_code, code);
541    }
542
543    /// Count an accepted route.open by the principal the daemon stamped.
544    ///
545    /// THE KEY SPACE IS CLOSED BY CONSTRUCTION, unlike the refusal counter which
546    /// needs an explicit allowlist: a principal is `direct` or
547    /// `reserved:<module_id>`, and a module id was already refused at HELLO
548    /// unless it is a single path component free of control characters. So an
549    /// untrusted string cannot expand this map without first passing module-id
550    /// validation, and the bound is the number of modules rather than the number
551    /// of distinct strings a caller can invent.
552    pub(crate) fn increment_route_open_accepted(&self, principal: &str) {
553        increment_keyed_count(&self.route_open_accepted_by_principal, principal);
554    }
555
556    pub(crate) fn increment_module_requests_dropped_stale_route(&self) {
557        self.module_requests_dropped_stale_route
558            .fetch_add(1, Ordering::Relaxed);
559    }
560
561    pub(crate) fn increment_client_frames_dropped_stale_route(&self) {
562        self.client_frames_dropped_stale_route
563            .fetch_add(1, Ordering::Relaxed);
564    }
565
566    pub(crate) fn increment_client_egress_close_delivery_failed(&self) {
567        self.client_egress_close_delivery_failed
568            .fetch_add(1, Ordering::Relaxed);
569    }
570
571    pub(crate) fn increment_goodbye_relay_client_failed(&self) {
572        self.goodbye_relay_client_failed
573            .fetch_add(1, Ordering::Relaxed);
574    }
575
576    pub(crate) fn increment_goodbye_relay_module_dropped(&self, module_id: Option<&str>) {
577        self.goodbye_relay_module_dropped
578            .fetch_add(1, Ordering::Relaxed);
579        if let Some(module_id) = module_id {
580            increment_keyed_count(&self.goodbye_relay_module_dropped_by_module, module_id);
581        }
582    }
583
584    pub(crate) fn increment_route_released_epoch_fenced(&self) {
585        self.route_released_epoch_fenced
586            .fetch_add(1, Ordering::Relaxed);
587    }
588
589    pub(crate) fn increment_route_release_stale_skipped(&self) {
590        self.route_release_stale_skipped
591            .fetch_add(1, Ordering::Relaxed);
592    }
593
594    pub(crate) fn increment_drains_with_undeclared_gauge(&self) {
595        self.drains_with_undeclared_gauge
596            .fetch_add(1, Ordering::Relaxed);
597    }
598
599    #[cfg(unix)]
600    pub(crate) fn increment_carried_containment_anchor_lost(&self) {
601        self.carried_containment_anchor_lost
602            .fetch_add(1, Ordering::Relaxed);
603    }
604}
605
606const MAX_AGENT_REFUSAL_MODULES: usize = 1_024;
607
608#[cfg(test)]
609mod refusal_label_tests {
610    use super::*;
611
612    #[test]
613    fn dropped_refusal_labels_require_registration_and_stop_at_the_hard_cap() {
614        let registry = crate::Registry::default();
615        let counters = DaemonCounters::default();
616        counters.increment_agent_command_refusal_dropped(Some("unregistered"), &registry);
617        assert!(counters
618            .snapshot()
619            .get("agent_command_refusals_dropped_by_module")
620            .is_none());
621        for index in 0..=1_024 {
622            let id = format!("registered-{index}");
623            registry
624                .register_with_control_ops(
625                    subc_protocol::manifest::ModuleManifest::builder(&id, "0.1.0")
626                        .protocol_ver(1)
627                        .build(),
628                    1,
629                    crate::ConnectionId::new(index as u64 + 1),
630                    Vec::new(),
631                )
632                .unwrap();
633            counters.increment_agent_command_refusal_dropped(Some(&id), &registry);
634        }
635        counters.increment_agent_command_refusal_dropped(Some("registered-0"), &registry);
636        let snapshot = counters.snapshot();
637        let labels = snapshot["agent_command_refusals_dropped_by_module"]
638            .as_object()
639            .unwrap();
640        assert_eq!(labels.len(), 1_024);
641        assert_eq!(labels["registered-0"], 2);
642        assert!(!labels.contains_key("registered-1024"));
643        assert_eq!(snapshot["agent_command_refusals_dropped_unlabelled"], 2);
644    }
645}
646
647fn increment_keyed_count(counts: &Mutex<HashMap<String, u64>>, key: &str) {
648    *counts
649        .lock()
650        .expect("keyed counter mutex poisoned")
651        .entry(key.to_string())
652        .or_default() += 1;
653}
654
655fn insert_nonempty_counts(
656    snapshot: &mut serde_json::Map<String, Value>,
657    key: &str,
658    counts: &Mutex<HashMap<String, u64>>,
659) {
660    let counts: MutexGuard<'_, HashMap<String, u64>> =
661        counts.lock().expect("keyed counter mutex poisoned");
662    if !counts.is_empty() {
663        snapshot.insert(key.to_string(), json!(&*counts));
664    }
665}
666
667#[cfg(test)]
668mod tests {
669    use super::*;
670
671    #[test]
672    fn registration_channel_closed_route_refusal_is_counted() {
673        let counters = DaemonCounters::new();
674        counters.increment_route_open_refused(crate::supervise::REGISTRATION_CHANNEL_CLOSED);
675        assert_eq!(
676            counters.snapshot()["route_open_refused_by_code"],
677            serde_json::json!({"registration_channel_closed": 1})
678        );
679    }
680
681    #[test]
682    fn ready_reader_wait_needs_no_runtime_clock_or_labels() {
683        use std::{
684            future::Future,
685            task::{Context, Poll, Waker},
686        };
687
688        let counters = DaemonCounters::new();
689        let wait = counters.observe_reader_wait(
690            || panic!("ready waits must not resolve diagnostic labels"),
691            "module_queue",
692            std::future::ready(7),
693        );
694        let mut wait = std::pin::pin!(wait);
695        let mut cx = Context::from_waker(Waker::noop());
696        // No Tokio runtime exists here: even constructing Sleep would panic.
697        assert_eq!(wait.as_mut().poll(&mut cx), Poll::Ready(7));
698        assert!(counters.reader_diagnostics.lock().unwrap().waits.is_empty());
699    }
700
701    #[tokio::test(start_paused = true)]
702    async fn reader_wait_labels_are_resolved_once_only_after_five_seconds() {
703        use std::{
704            cell::Cell,
705            future::{poll_fn, Future},
706            task::Poll,
707        };
708
709        let counters = DaemonCounters::new();
710        let calls = Cell::new(0);
711        let (sender, receiver) = tokio::sync::oneshot::channel();
712        let wait = counters.observe_reader_wait(
713            || {
714                calls.set(calls.get() + 1);
715                ("alpha".into(), "direct".into())
716            },
717            "module_queue",
718            receiver,
719        );
720        tokio::pin!(wait);
721        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
722            .await
723            .is_pending());
724        tokio::time::advance(Duration::from_secs(4)).await;
725        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
726            .await
727            .is_pending());
728        assert_eq!(calls.get(), 0);
729        tokio::time::advance(Duration::from_secs(1)).await;
730        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
731            .await
732            .is_pending());
733        assert_eq!(calls.get(), 1);
734        assert_eq!(
735            counters.snapshot()["reader_wait_over_5s_by_module_principal"]["alpha"]["direct"]
736                ["module_queue"],
737            1
738        );
739        sender.send(()).unwrap();
740        wait.await.unwrap();
741        assert_eq!(calls.get(), 1);
742    }
743
744    #[test]
745    fn counter_snapshot_includes_zero_rate_and_omits_empty_module_maps() {
746        let counters = DaemonCounters::new();
747        let snapshot = counters.snapshot();
748
749        assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 0);
750        assert_eq!(
751            snapshot["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
752            0
753        );
754        assert!(snapshot
755            .get("module_frames_dropped_no_route_by_module")
756            .is_none());
757        assert!(snapshot
758            .get("goodbye_relay_module_dropped_by_module")
759            .is_none());
760    }
761
762    #[tokio::test(start_paused = true)]
763    async fn module_frame_drops_are_attributed_to_the_emitting_module() {
764        let counters = DaemonCounters::new();
765        counters.increment_module_frames_dropped_no_route(Some("alpha"));
766        counters.increment_module_frames_dropped_no_route(Some("alpha"));
767
768        let snapshot = counters.snapshot();
769        assert_eq!(snapshot["module_frames_dropped_no_route"], 2);
770        assert_eq!(
771            snapshot["module_frames_dropped_no_route_by_module"],
772            json!({ "alpha": 2 })
773        );
774        assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 2);
775    }
776
777    #[tokio::test(start_paused = true)]
778    async fn frame_drop_rate_ages_out_after_ten_minute_buckets() {
779        let counters = DaemonCounters::new();
780        counters.increment_module_frames_dropped_no_route(Some("alpha"));
781
782        tokio::time::advance(Duration::from_secs(9 * 60)).await;
783        assert_eq!(
784            counters.snapshot()["module_frames_dropped_no_route_last_10m"],
785            1
786        );
787
788        tokio::time::advance(Duration::from_secs(60)).await;
789        assert_eq!(
790            counters.snapshot()["module_frames_dropped_no_route_last_10m"],
791            0
792        );
793    }
794
795    #[tokio::test(start_paused = true)]
796    async fn frame_drop_window_counts_only_nonzero_minutes() {
797        let counters = DaemonCounters::new();
798        for minute in 0..10 {
799            if minute > 0 {
800                tokio::time::advance(Duration::from_secs(60)).await;
801            }
802            if minute != 4 {
803                counters.increment_module_frames_dropped_no_route(Some("alpha"));
804            }
805        }
806
807        assert_eq!(
808            counters.snapshot()["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
809            9
810        );
811    }
812
813    #[test]
814    fn goodbye_relay_drops_are_attributed_to_the_target_module() {
815        let counters = DaemonCounters::new();
816        counters.increment_goodbye_relay_module_dropped(Some("alpha"));
817
818        assert_eq!(
819            counters.snapshot()["goodbye_relay_module_dropped_by_module"],
820            json!({ "alpha": 1 })
821        );
822    }
823}