Skip to main content

subc_daemon/
observability.rs

1use std::{
2    collections::{BTreeMap, HashMap, VecDeque},
3    sync::{
4        atomic::{AtomicU64, Ordering},
5        Arc, Mutex, MutexGuard,
6    },
7    time::Duration,
8};
9
10use serde_json::{json, Value};
11use tracing::{debug, warn};
12
13use crate::registry::ConnectionId;
14
15/// The slash prevents collision with a module ID: `module_id_path_hazard`
16/// rejects separators in `registry.rs:855-875`.
17pub(crate) const CONTROL_WAIT_BUCKET: &str = "daemon/control";
18
19/// The only keys the route.open refusal counter may carry. A module's own
20/// error code on a rejected bind is counted under `module_rejected` and rides
21/// the log event as a separate field: if the module's string were the key, a
22/// module could grow this map for the daemon's lifetime and push terminal
23/// control sequences through `ck daemon` to an operator's screen. The
24/// `&'static str` increment signature plus the debug assertion keep the set
25/// closed at the call site, not just here.
26const ROUTE_OPEN_REFUSAL_COUNTER_CODES: &[&str] = &[
27    "module_warming",
28    ROUTE_OPEN_REFUSED_DECLARED_NOT_READY,
29    ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
30    "target_unavailable",
31    subc_protocol::error_codes::TARGET_FLOW_UNSUPPORTED,
32    subc_protocol::error_codes::TARGET_AGENT_RUN_UNSUPPORTED,
33    "module_removed",
34    "module_no_protocol",
35    "unknown_module",
36    "module_reloading",
37    "op_not_allowed",
38    "bad_consumer_identity",
39    "capability_forbidden",
40    "admission_facts_not_permitted",
41    "admission_facts_target_not_allowed",
42    "route_limit",
43    "forwarding_error",
44    "module_timeout",
45    ROUTE_OPEN_REFUSED_BREAKER_OPEN,
46    "module_rejected",
47    subc_protocol::error_codes::SCOPE_EPOCH_REQUIRED,
48    subc_protocol::error_codes::SCOPE_NOT_SYNCED,
49    subc_protocol::error_codes::SCOPE_NOT_LIVE,
50    subc_protocol::error_codes::SCOPE_ENDED,
51    subc_protocol::error_codes::SCOPE_NOT_CARRIER,
52    subc_protocol::error_codes::SCOPE_BIND_PROOF_REQUIRED,
53    subc_protocol::error_codes::SCOPE_BIND_PROOF_MISMATCH,
54    subc_protocol::error_codes::SCOPE_CHANGED,
55    subc_protocol::error_codes::INVALID_REQUEST,
56];
57
58/// Counter key for a `route.open` refused by the per-module bind-relay breaker
59/// before any relay was attempted.
60///
61/// The frame the caller receives carries `module_timeout`, because both SDKs
62/// already classify that as retryable with capped backoff and inventing a new
63/// wire code would need a change in each of them. The COUNTER is deliberately a
64/// different key: "this module burned the full bind budget" and "this module is
65/// being refused in microseconds because it already did that repeatedly" are
66/// the two states an operator most needs to tell apart, and they are
67/// indistinguishable from the client side, where both look like one retryable
68/// error that the next attempt may well satisfy.
69pub(crate) const ROUTE_OPEN_REFUSED_BREAKER_OPEN: &str = "module_timeout_breaker_open";
70
71/// Counter key for a registered module that declared itself not ready.
72///
73/// The caller still receives `module_warming`, but operators must be able to
74/// distinguish declared readiness from a supervised process that has not
75/// registered yet.
76pub(crate) const ROUTE_OPEN_REFUSED_DECLARED_NOT_READY: &str = "module_warming_declared_not_ready";
77
78/// Counter key for a registered, declared-ready module held not-ready because
79/// a capability it declares `need: required` has no registered provider.
80///
81/// The caller still receives `module_warming`; a separate key lets an operator
82/// tell "the module says it is warming" from "the module is waiting on a
83/// provider that is not running", which point at different fixes.
84pub(crate) const ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED: &str =
85    "module_warming_required_capability_unprovided";
86
87/// Shared count of authenticated socket connections accepted by the daemon.
88#[derive(Debug, Clone, Default)]
89pub struct ConnectedClients {
90    count: Arc<AtomicU64>,
91}
92
93impl ConnectedClients {
94    pub fn new() -> Self {
95        Self::default()
96    }
97
98    pub fn count(&self) -> u64 {
99        self.count.load(Ordering::SeqCst)
100    }
101
102    pub(crate) fn open(&self, connection_id: ConnectionId) -> ConnectedClientGuard {
103        let previous = self.count.fetch_add(1, Ordering::SeqCst);
104        let current = previous + 1;
105        // DEBUG, not INFO: this pair fired on every connect and disconnect and was
106        // measured at 43-74% of the daemon's own log (issue #114), burying the
107        // lines an operator opens the file for. The count itself is not lost: it
108        // is served live as `connected_clients` on server.describe (`ck daemon`),
109        // and a connection that opens a route still names its connection_id on the
110        // `route.open accepted` line, so per-connection forensics survive where
111        // they matter. Raise the filter (`CK_LOG=subc=debug`) to see every churn.
112        debug!(
113            connection_id = connection_id.get(),
114            connected_clients = current,
115            previous_connected_clients = previous,
116            "authenticated connection count changed"
117        );
118        ConnectedClientGuard {
119            clients: self.clone(),
120            connection_id,
121        }
122    }
123}
124
125pub(crate) struct ConnectedClientGuard {
126    clients: ConnectedClients,
127    connection_id: ConnectionId,
128}
129
130impl Drop for ConnectedClientGuard {
131    fn drop(&mut self) {
132        let previous = self.clients.count.fetch_sub(1, Ordering::SeqCst);
133        let current = previous.saturating_sub(1);
134        // DEBUG for the same reason as the open side above.
135        debug!(
136            connection_id = self.connection_id.get(),
137            connected_clients = current,
138            previous_connected_clients = previous,
139            "authenticated connection count changed"
140        );
141    }
142}
143
144/// Counters for route lifecycle drops, delivery failures and reader waits.
145#[derive(Debug, Clone, Default)]
146pub struct DaemonCounters {
147    reader_diagnostics: Arc<Mutex<ReaderDiagnostics>>,
148    carried_containment_anchor_lost: Arc<AtomicU64>,
149    /// Every non-request frame a module sent on a (channel, epoch) the daemon
150    /// holds no bound route for, dropped. Counts both kinds below: orphan
151    /// traffic on a route the daemon released, and frames on a (channel,
152    /// epoch) that was never allocated on that module connection.
153    module_frames_dropped_no_route: Arc<AtomicU64>,
154    /// The orphan-traffic part of `module_frames_dropped_no_route`: the daemon
155    /// had allocated that (channel, epoch) on the sending module connection and
156    /// has since released it, so the module is still holding a route the
157    /// daemon closed. `module_frames_dropped_no_route` minus this is the count
158    /// of frames on a (channel, epoch) that never existed on the connection.
159    module_frames_dropped_released_route: Arc<AtomicU64>,
160    module_frames_dropped_released_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
161    /// Route GOODBYEs the daemon enqueued to a module in answer to its frames
162    /// on a (channel, epoch) the daemon holds no route for, telling the module
163    /// to drop the route. Rate-limited per module connection and channel, so
164    /// this counts answers, not orphan frames.
165    module_orphan_route_goodbyes_sent: Arc<AtomicU64>,
166    // Per-module maps and the rate window are daemon-lifetime diagnostics only:
167    // they deliberately reset on restart instead of becoming durable daemon state.
168    module_frames_dropped_no_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
169    agent_command_refusals_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
170    agent_command_refusals_dropped_unlabelled: Arc<std::sync::atomic::AtomicU64>,
171    route_open_refused_by_code: Arc<Mutex<HashMap<String, u64>>>,
172    route_open_accepted_by_principal: Arc<Mutex<HashMap<String, u64>>>,
173    module_frames_dropped_no_route_window: Arc<Mutex<DropWindow>>,
174    module_requests_dropped_stale_route: Arc<AtomicU64>,
175    client_frames_dropped_stale_route: Arc<AtomicU64>,
176    client_egress_close_delivery_failed: Arc<AtomicU64>,
177    goodbye_relay_client_failed: Arc<AtomicU64>,
178    goodbye_relay_module_dropped: Arc<AtomicU64>,
179    goodbye_relay_module_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
180    route_released_epoch_fenced: Arc<AtomicU64>,
181    route_release_stale_skipped: Arc<AtomicU64>,
182    drains_with_undeclared_gauge: Arc<AtomicU64>,
183}
184
185#[derive(Debug, Default)]
186struct ReaderDiagnostics {
187    busy: BTreeMap<String, BTreeMap<String, u64>>,
188    waits: BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>>,
189    warned_waits: HashMap<(String, String), tokio::time::Instant>,
190}
191
192/// Ten one-minute buckets make sustained module-to-client route drops visible
193/// without retaining one record for every dropped frame.
194#[derive(Debug)]
195struct DropWindow {
196    started_at: tokio::time::Instant,
197    buckets: VecDeque<DropBucket>,
198}
199
200#[derive(Debug)]
201struct DropBucket {
202    minute: u64,
203    count: u64,
204}
205
206impl Default for DropWindow {
207    fn default() -> Self {
208        Self {
209            started_at: tokio::time::Instant::now(),
210            buckets: VecDeque::new(),
211        }
212    }
213}
214
215impl DropWindow {
216    const MINUTE: Duration = Duration::from_secs(60);
217    const BUCKETS: u64 = 10;
218
219    fn record(&mut self, now: tokio::time::Instant) {
220        let minute = self.minute_at(now);
221        self.prune_before(minute);
222        match self.buckets.back_mut() {
223            Some(bucket) if bucket.minute == minute => bucket.count += 1,
224            _ => self.buckets.push_back(DropBucket { minute, count: 1 }),
225        }
226    }
227
228    fn count_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
229        let minute = self.minute_at(now);
230        self.prune_before(minute);
231        self.buckets.iter().map(|bucket| bucket.count).sum()
232    }
233
234    fn nonzero_minutes_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
235        let minute = self.minute_at(now);
236        self.prune_before(minute);
237        self.buckets.len() as u64
238    }
239
240    fn minute_at(&self, now: tokio::time::Instant) -> u64 {
241        now.saturating_duration_since(self.started_at).as_secs() / Self::MINUTE.as_secs()
242    }
243
244    fn prune_before(&mut self, current_minute: u64) {
245        while self
246            .buckets
247            .front()
248            .is_some_and(|bucket| current_minute.saturating_sub(bucket.minute) >= Self::BUCKETS)
249        {
250            self.buckets.pop_front();
251        }
252    }
253}
254
255impl DaemonCounters {
256    pub fn new() -> Self {
257        Self::default()
258    }
259
260    /// Returns a JSON snapshot whose stable, additive schema keeps the
261    /// `server.describe` diagnostic endpoint backward-compatible.
262    pub fn snapshot(&self) -> Value {
263        let mut snapshot = serde_json::Map::new();
264        let diagnostics = self
265            .reader_diagnostics
266            .lock()
267            .unwrap_or_else(|p| p.into_inner());
268        if !diagnostics.busy.is_empty() {
269            snapshot.insert(
270                "route_busy_by_module_principal".into(),
271                json!(diagnostics.busy),
272            );
273        }
274        if !diagnostics.waits.is_empty() {
275            snapshot.insert(
276                "reader_wait_over_5s_by_module_principal".into(),
277                json!(diagnostics.waits),
278            );
279        }
280        drop(diagnostics);
281        snapshot.insert(
282            "carried_containment_anchor_lost".into(),
283            self.carried_containment_anchor_lost
284                .load(Ordering::Relaxed)
285                .into(),
286        );
287        snapshot.insert(
288            "module_frames_dropped_no_route".into(),
289            self.module_frames_dropped_no_route
290                .load(Ordering::Relaxed)
291                .into(),
292        );
293        let mut drop_window = self
294            .module_frames_dropped_no_route_window
295            .lock()
296            .expect("drop-rate window mutex poisoned");
297        let now = tokio::time::Instant::now();
298        snapshot.insert(
299            "module_frames_dropped_no_route_last_10m".into(),
300            drop_window.count_last_10m(now).into(),
301        );
302        snapshot.insert(
303            "module_frames_dropped_no_route_nonzero_minutes_last_10m".into(),
304            drop_window.nonzero_minutes_last_10m(now).into(),
305        );
306        insert_nonempty_counts(
307            &mut snapshot,
308            "module_frames_dropped_no_route_by_module",
309            &self.module_frames_dropped_no_route_by_module,
310        );
311        insert_nonempty_counts(
312            &mut snapshot,
313            "agent_command_refusals_dropped_by_module",
314            &self.agent_command_refusals_dropped_by_module,
315        );
316        let unlabelled = self
317            .agent_command_refusals_dropped_unlabelled
318            .load(Ordering::Relaxed);
319        if unlabelled != 0 {
320            snapshot.insert(
321                "agent_command_refusals_dropped_unlabelled".into(),
322                unlabelled.into(),
323            );
324        }
325        snapshot.insert(
326            "module_frames_dropped_released_route".into(),
327            self.module_frames_dropped_released_route
328                .load(Ordering::Relaxed)
329                .into(),
330        );
331        insert_nonempty_counts(
332            &mut snapshot,
333            "module_frames_dropped_released_route_by_module",
334            &self.module_frames_dropped_released_route_by_module,
335        );
336        snapshot.insert(
337            "module_orphan_route_goodbyes_sent".into(),
338            self.module_orphan_route_goodbyes_sent
339                .load(Ordering::Relaxed)
340                .into(),
341        );
342        insert_nonempty_counts(
343            &mut snapshot,
344            "route_open_refused_by_code",
345            &self.route_open_refused_by_code,
346        );
347        insert_nonempty_counts(
348            &mut snapshot,
349            "route_open_accepted_by_principal",
350            &self.route_open_accepted_by_principal,
351        );
352        snapshot.insert(
353            "module_requests_dropped_stale_route".into(),
354            self.module_requests_dropped_stale_route
355                .load(Ordering::Relaxed)
356                .into(),
357        );
358        snapshot.insert(
359            "client_frames_dropped_stale_route".into(),
360            self.client_frames_dropped_stale_route
361                .load(Ordering::Relaxed)
362                .into(),
363        );
364        snapshot.insert(
365            "client_egress_close_delivery_failed".into(),
366            self.client_egress_close_delivery_failed
367                .load(Ordering::Relaxed)
368                .into(),
369        );
370        snapshot.insert(
371            "goodbye_relay_client_failed".into(),
372            self.goodbye_relay_client_failed
373                .load(Ordering::Relaxed)
374                .into(),
375        );
376        snapshot.insert(
377            "goodbye_relay_module_dropped".into(),
378            self.goodbye_relay_module_dropped
379                .load(Ordering::Relaxed)
380                .into(),
381        );
382        insert_nonempty_counts(
383            &mut snapshot,
384            "goodbye_relay_module_dropped_by_module",
385            &self.goodbye_relay_module_dropped_by_module,
386        );
387        snapshot.insert(
388            "route_released_epoch_fenced".into(),
389            self.route_released_epoch_fenced
390                .load(Ordering::Relaxed)
391                .into(),
392        );
393        snapshot.insert(
394            "route_release_stale_skipped".into(),
395            self.route_release_stale_skipped
396                .load(Ordering::Relaxed)
397                .into(),
398        );
399        snapshot.insert(
400            "drains_with_undeclared_gauge".into(),
401            self.drains_with_undeclared_gauge
402                .load(Ordering::Relaxed)
403                .into(),
404        );
405        Value::Object(snapshot)
406    }
407
408    pub(crate) fn increment_route_busy(&self, module: &str, principal: &str) {
409        let mut diagnostics = self
410            .reader_diagnostics
411            .lock()
412            .unwrap_or_else(|p| p.into_inner());
413        *diagnostics
414            .busy
415            .entry(module.into())
416            .or_default()
417            .entry(principal.into())
418            .or_default() += 1;
419    }
420
421    /// Observe a wait without imposing a deadline or cancelling its future.
422    /// Counts advance once at five seconds, even if the peer never recovers.
423    pub(crate) async fn observe_reader_wait<F: std::future::Future>(
424        &self,
425        labels: impl FnOnce() -> (String, String),
426        kind: &'static str,
427        future: F,
428    ) -> F::Output {
429        tokio::pin!(future);
430        // Ready sends need neither a timer nor diagnostic labels. Poll before
431        // constructing Sleep, which would otherwise touch the runtime clock.
432        if let std::task::Poll::Ready(result) =
433            std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
434        {
435            return result;
436        }
437        tokio::select! {
438            biased;
439            result = &mut future => return result,
440            _ = tokio::time::sleep(Duration::from_secs(5)) => {}
441        }
442        let (module, principal) = labels();
443        let should_warn = {
444            let mut diagnostics = self
445                .reader_diagnostics
446                .lock()
447                .unwrap_or_else(|p| p.into_inner());
448            *diagnostics
449                .waits
450                .entry(module.clone())
451                .or_default()
452                .entry(principal.clone())
453                .or_default()
454                .entry(kind.into())
455                .or_default() += 1;
456            let now = tokio::time::Instant::now();
457            let key = (module.to_owned(), principal.to_owned());
458            let should_warn = diagnostics
459                .warned_waits
460                .get(&key)
461                .is_none_or(|last| now.duration_since(*last) >= Duration::from_secs(60));
462            if should_warn {
463                diagnostics.warned_waits.insert(key, now);
464            }
465            should_warn
466        };
467        if should_warn {
468            warn!(
469                module_id = module,
470                client_principal = principal,
471                wait_kind = kind,
472                "connection reader wait exceeded 5 seconds"
473            );
474        }
475        future.await
476    }
477
478    pub(crate) fn increment_module_frames_dropped_no_route(&self, module_id: Option<&str>) {
479        self.module_frames_dropped_no_route
480            .fetch_add(1, Ordering::Relaxed);
481        if let Some(module_id) = module_id {
482            increment_keyed_count(&self.module_frames_dropped_no_route_by_module, module_id);
483        }
484        self.module_frames_dropped_no_route_window
485            .lock()
486            .expect("drop-rate window mutex poisoned")
487            .record(tokio::time::Instant::now());
488    }
489
490    pub(crate) fn increment_agent_command_refusal_dropped(
491        &self,
492        module_id: Option<&str>,
493        registry: &crate::registry::Registry,
494    ) {
495        let Some(module_id) =
496            module_id.filter(|id| registry.get_module(id).ok().flatten().is_some())
497        else {
498            self.agent_command_refusals_dropped_unlabelled
499                .fetch_add(1, Ordering::Relaxed);
500            return;
501        };
502        let mut counts = self
503            .agent_command_refusals_dropped_by_module
504            .lock()
505            .unwrap_or_else(|p| p.into_inner());
506        if let Some(count) = counts.get_mut(module_id) {
507            *count = count.saturating_add(1);
508        } else if counts.len() < MAX_AGENT_REFUSAL_MODULES {
509            counts.insert(module_id.to_owned(), 1);
510        } else {
511            self.agent_command_refusals_dropped_unlabelled
512                .fetch_add(1, Ordering::Relaxed);
513        }
514    }
515
516    /// Count a dropped module frame whose (channel, epoch) the daemon had
517    /// allocated on that connection and since released. Called in addition to
518    /// [`Self::increment_module_frames_dropped_no_route`], never instead of it.
519    pub(crate) fn increment_module_frames_dropped_released_route(&self, module_id: Option<&str>) {
520        self.module_frames_dropped_released_route
521            .fetch_add(1, Ordering::Relaxed);
522        if let Some(module_id) = module_id {
523            increment_keyed_count(
524                &self.module_frames_dropped_released_route_by_module,
525                module_id,
526            );
527        }
528    }
529
530    pub(crate) fn increment_module_orphan_route_goodbyes_sent(&self) {
531        self.module_orphan_route_goodbyes_sent
532            .fetch_add(1, Ordering::Relaxed);
533    }
534
535    pub(crate) fn increment_route_open_refused(&self, code: &'static str) {
536        debug_assert!(ROUTE_OPEN_REFUSAL_COUNTER_CODES.contains(&code));
537        increment_keyed_count(&self.route_open_refused_by_code, code);
538    }
539
540    /// Count an accepted route.open by the principal the daemon stamped.
541    ///
542    /// THE KEY SPACE IS CLOSED BY CONSTRUCTION, unlike the refusal counter which
543    /// needs an explicit allowlist: a principal is `direct` or
544    /// `reserved:<module_id>`, and a module id was already refused at HELLO
545    /// unless it is a single path component free of control characters. So an
546    /// untrusted string cannot expand this map without first passing module-id
547    /// validation, and the bound is the number of modules rather than the number
548    /// of distinct strings a caller can invent.
549    pub(crate) fn increment_route_open_accepted(&self, principal: &str) {
550        increment_keyed_count(&self.route_open_accepted_by_principal, principal);
551    }
552
553    pub(crate) fn increment_module_requests_dropped_stale_route(&self) {
554        self.module_requests_dropped_stale_route
555            .fetch_add(1, Ordering::Relaxed);
556    }
557
558    pub(crate) fn increment_client_frames_dropped_stale_route(&self) {
559        self.client_frames_dropped_stale_route
560            .fetch_add(1, Ordering::Relaxed);
561    }
562
563    pub(crate) fn increment_client_egress_close_delivery_failed(&self) {
564        self.client_egress_close_delivery_failed
565            .fetch_add(1, Ordering::Relaxed);
566    }
567
568    pub(crate) fn increment_goodbye_relay_client_failed(&self) {
569        self.goodbye_relay_client_failed
570            .fetch_add(1, Ordering::Relaxed);
571    }
572
573    pub(crate) fn increment_goodbye_relay_module_dropped(&self, module_id: Option<&str>) {
574        self.goodbye_relay_module_dropped
575            .fetch_add(1, Ordering::Relaxed);
576        if let Some(module_id) = module_id {
577            increment_keyed_count(&self.goodbye_relay_module_dropped_by_module, module_id);
578        }
579    }
580
581    pub(crate) fn increment_route_released_epoch_fenced(&self) {
582        self.route_released_epoch_fenced
583            .fetch_add(1, Ordering::Relaxed);
584    }
585
586    pub(crate) fn increment_route_release_stale_skipped(&self) {
587        self.route_release_stale_skipped
588            .fetch_add(1, Ordering::Relaxed);
589    }
590
591    pub(crate) fn increment_drains_with_undeclared_gauge(&self) {
592        self.drains_with_undeclared_gauge
593            .fetch_add(1, Ordering::Relaxed);
594    }
595
596    #[cfg(unix)]
597    pub(crate) fn increment_carried_containment_anchor_lost(&self) {
598        self.carried_containment_anchor_lost
599            .fetch_add(1, Ordering::Relaxed);
600    }
601}
602
603const MAX_AGENT_REFUSAL_MODULES: usize = 1_024;
604
605#[cfg(test)]
606mod refusal_label_tests {
607    use super::*;
608
609    #[test]
610    fn dropped_refusal_labels_require_registration_and_stop_at_the_hard_cap() {
611        let registry = crate::Registry::default();
612        let counters = DaemonCounters::default();
613        counters.increment_agent_command_refusal_dropped(Some("unregistered"), &registry);
614        assert!(counters
615            .snapshot()
616            .get("agent_command_refusals_dropped_by_module")
617            .is_none());
618        for index in 0..=1_024 {
619            let id = format!("registered-{index}");
620            registry
621                .register_with_control_ops(
622                    subc_protocol::manifest::ModuleManifest::builder(&id, "0.1.0")
623                        .protocol_ver(1)
624                        .build(),
625                    1,
626                    crate::ConnectionId::new(index as u64 + 1),
627                    Vec::new(),
628                )
629                .unwrap();
630            counters.increment_agent_command_refusal_dropped(Some(&id), &registry);
631        }
632        counters.increment_agent_command_refusal_dropped(Some("registered-0"), &registry);
633        let snapshot = counters.snapshot();
634        let labels = snapshot["agent_command_refusals_dropped_by_module"]
635            .as_object()
636            .unwrap();
637        assert_eq!(labels.len(), 1_024);
638        assert_eq!(labels["registered-0"], 2);
639        assert!(!labels.contains_key("registered-1024"));
640        assert_eq!(snapshot["agent_command_refusals_dropped_unlabelled"], 2);
641    }
642}
643
644fn increment_keyed_count(counts: &Mutex<HashMap<String, u64>>, key: &str) {
645    *counts
646        .lock()
647        .expect("keyed counter mutex poisoned")
648        .entry(key.to_string())
649        .or_default() += 1;
650}
651
652fn insert_nonempty_counts(
653    snapshot: &mut serde_json::Map<String, Value>,
654    key: &str,
655    counts: &Mutex<HashMap<String, u64>>,
656) {
657    let counts: MutexGuard<'_, HashMap<String, u64>> =
658        counts.lock().expect("keyed counter mutex poisoned");
659    if !counts.is_empty() {
660        snapshot.insert(key.to_string(), json!(&*counts));
661    }
662}
663
664#[cfg(test)]
665mod tests {
666    use super::*;
667
668    #[test]
669    fn ready_reader_wait_needs_no_runtime_clock_or_labels() {
670        use std::{
671            future::Future,
672            task::{Context, Poll, Waker},
673        };
674
675        let counters = DaemonCounters::new();
676        let wait = counters.observe_reader_wait(
677            || panic!("ready waits must not resolve diagnostic labels"),
678            "module_queue",
679            std::future::ready(7),
680        );
681        let mut wait = std::pin::pin!(wait);
682        let mut cx = Context::from_waker(Waker::noop());
683        // No Tokio runtime exists here: even constructing Sleep would panic.
684        assert_eq!(wait.as_mut().poll(&mut cx), Poll::Ready(7));
685        assert!(counters.reader_diagnostics.lock().unwrap().waits.is_empty());
686    }
687
688    #[tokio::test(start_paused = true)]
689    async fn reader_wait_labels_are_resolved_once_only_after_five_seconds() {
690        use std::{
691            cell::Cell,
692            future::{poll_fn, Future},
693            task::Poll,
694        };
695
696        let counters = DaemonCounters::new();
697        let calls = Cell::new(0);
698        let (sender, receiver) = tokio::sync::oneshot::channel();
699        let wait = counters.observe_reader_wait(
700            || {
701                calls.set(calls.get() + 1);
702                ("alpha".into(), "direct".into())
703            },
704            "module_queue",
705            receiver,
706        );
707        tokio::pin!(wait);
708        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
709            .await
710            .is_pending());
711        tokio::time::advance(Duration::from_secs(4)).await;
712        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
713            .await
714            .is_pending());
715        assert_eq!(calls.get(), 0);
716        tokio::time::advance(Duration::from_secs(1)).await;
717        assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
718            .await
719            .is_pending());
720        assert_eq!(calls.get(), 1);
721        assert_eq!(
722            counters.snapshot()["reader_wait_over_5s_by_module_principal"]["alpha"]["direct"]
723                ["module_queue"],
724            1
725        );
726        sender.send(()).unwrap();
727        wait.await.unwrap();
728        assert_eq!(calls.get(), 1);
729    }
730
731    #[test]
732    fn counter_snapshot_includes_zero_rate_and_omits_empty_module_maps() {
733        let counters = DaemonCounters::new();
734        let snapshot = counters.snapshot();
735
736        assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 0);
737        assert_eq!(
738            snapshot["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
739            0
740        );
741        assert!(snapshot
742            .get("module_frames_dropped_no_route_by_module")
743            .is_none());
744        assert!(snapshot
745            .get("goodbye_relay_module_dropped_by_module")
746            .is_none());
747    }
748
749    #[tokio::test(start_paused = true)]
750    async fn module_frame_drops_are_attributed_to_the_emitting_module() {
751        let counters = DaemonCounters::new();
752        counters.increment_module_frames_dropped_no_route(Some("alpha"));
753        counters.increment_module_frames_dropped_no_route(Some("alpha"));
754
755        let snapshot = counters.snapshot();
756        assert_eq!(snapshot["module_frames_dropped_no_route"], 2);
757        assert_eq!(
758            snapshot["module_frames_dropped_no_route_by_module"],
759            json!({ "alpha": 2 })
760        );
761        assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 2);
762    }
763
764    #[tokio::test(start_paused = true)]
765    async fn frame_drop_rate_ages_out_after_ten_minute_buckets() {
766        let counters = DaemonCounters::new();
767        counters.increment_module_frames_dropped_no_route(Some("alpha"));
768
769        tokio::time::advance(Duration::from_secs(9 * 60)).await;
770        assert_eq!(
771            counters.snapshot()["module_frames_dropped_no_route_last_10m"],
772            1
773        );
774
775        tokio::time::advance(Duration::from_secs(60)).await;
776        assert_eq!(
777            counters.snapshot()["module_frames_dropped_no_route_last_10m"],
778            0
779        );
780    }
781
782    #[tokio::test(start_paused = true)]
783    async fn frame_drop_window_counts_only_nonzero_minutes() {
784        let counters = DaemonCounters::new();
785        for minute in 0..10 {
786            if minute > 0 {
787                tokio::time::advance(Duration::from_secs(60)).await;
788            }
789            if minute != 4 {
790                counters.increment_module_frames_dropped_no_route(Some("alpha"));
791            }
792        }
793
794        assert_eq!(
795            counters.snapshot()["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
796            9
797        );
798    }
799
800    #[test]
801    fn goodbye_relay_drops_are_attributed_to_the_target_module() {
802        let counters = DaemonCounters::new();
803        counters.increment_goodbye_relay_module_dropped(Some("alpha"));
804
805        assert_eq!(
806            counters.snapshot()["goodbye_relay_module_dropped_by_module"],
807            json!({ "alpha": 1 })
808        );
809    }
810}