1use std::{
2 collections::{BTreeMap, HashMap, VecDeque},
3 sync::{
4 atomic::{AtomicU64, Ordering},
5 Arc, Mutex, MutexGuard,
6 },
7 time::Duration,
8};
9
10use serde_json::{json, Value};
11use tracing::{debug, warn};
12
13use crate::registry::ConnectionId;
14
15pub(crate) const CONTROL_WAIT_BUCKET: &str = "daemon/control";
18
19const ROUTE_OPEN_REFUSAL_COUNTER_CODES: &[&str] = &[
27 "module_warming",
28 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY,
29 ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
30 "target_unavailable",
31 subc_protocol::error_codes::TARGET_FLOW_UNSUPPORTED,
32 subc_protocol::error_codes::TARGET_FEDERATED_UNSUPPORTED,
33 subc_protocol::error_codes::FEDERATED_STAMP_REQUIRED,
34 subc_protocol::error_codes::TARGET_AGENT_RUN_UNSUPPORTED,
35 "module_removed",
36 "module_no_protocol",
37 "unknown_module",
38 "module_reloading",
39 "op_not_allowed",
40 "bad_consumer_identity",
41 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
42 "capability_forbidden",
43 "admission_facts_not_permitted",
44 "admission_facts_target_not_allowed",
45 "route_limit",
46 "forwarding_error",
47 "module_timeout",
48 ROUTE_OPEN_REFUSED_BREAKER_OPEN,
49 "module_rejected",
50 subc_protocol::error_codes::SCOPE_EPOCH_REQUIRED,
51 subc_protocol::error_codes::SCOPE_NOT_SYNCED,
52 subc_protocol::error_codes::SCOPE_NOT_LIVE,
53 subc_protocol::error_codes::SCOPE_ENDED,
54 subc_protocol::error_codes::SCOPE_NOT_CARRIER,
55 subc_protocol::error_codes::SCOPE_BIND_PROOF_REQUIRED,
56 subc_protocol::error_codes::SCOPE_BIND_PROOF_MISMATCH,
57 subc_protocol::error_codes::SCOPE_CHANGED,
58 subc_protocol::error_codes::INVALID_REQUEST,
59];
60
61pub(crate) const ROUTE_OPEN_REFUSED_BREAKER_OPEN: &str = "module_timeout_breaker_open";
73
74pub(crate) const ROUTE_OPEN_REFUSED_DECLARED_NOT_READY: &str = "module_warming_declared_not_ready";
80
81pub(crate) const ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED: &str =
88 "module_warming_required_capability_unprovided";
89
90#[derive(Debug, Clone, Default)]
92pub struct ConnectedClients {
93 count: Arc<AtomicU64>,
94}
95
96impl ConnectedClients {
97 pub fn new() -> Self {
98 Self::default()
99 }
100
101 pub fn count(&self) -> u64 {
102 self.count.load(Ordering::SeqCst)
103 }
104
105 pub(crate) fn open(&self, connection_id: ConnectionId) -> ConnectedClientGuard {
106 let previous = self.count.fetch_add(1, Ordering::SeqCst);
107 let current = previous + 1;
108 debug!(
116 connection_id = connection_id.get(),
117 connected_clients = current,
118 previous_connected_clients = previous,
119 "authenticated connection count changed"
120 );
121 ConnectedClientGuard {
122 clients: self.clone(),
123 connection_id,
124 }
125 }
126}
127
128pub(crate) struct ConnectedClientGuard {
129 clients: ConnectedClients,
130 connection_id: ConnectionId,
131}
132
133impl Drop for ConnectedClientGuard {
134 fn drop(&mut self) {
135 let previous = self.clients.count.fetch_sub(1, Ordering::SeqCst);
136 let current = previous.saturating_sub(1);
137 debug!(
139 connection_id = self.connection_id.get(),
140 connected_clients = current,
141 previous_connected_clients = previous,
142 "authenticated connection count changed"
143 );
144 }
145}
146
147#[derive(Debug, Clone, Default)]
149pub struct DaemonCounters {
150 reader_diagnostics: Arc<Mutex<ReaderDiagnostics>>,
151 carried_containment_anchor_lost: Arc<AtomicU64>,
152 module_frames_dropped_no_route: Arc<AtomicU64>,
157 module_frames_dropped_released_route: Arc<AtomicU64>,
163 module_frames_dropped_released_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
164 module_orphan_route_goodbyes_sent: Arc<AtomicU64>,
169 module_frames_dropped_no_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
172 agent_command_refusals_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
173 agent_command_refusals_dropped_unlabelled: Arc<std::sync::atomic::AtomicU64>,
174 route_open_refused_by_code: Arc<Mutex<HashMap<String, u64>>>,
175 route_open_accepted_by_principal: Arc<Mutex<HashMap<String, u64>>>,
176 module_frames_dropped_no_route_window: Arc<Mutex<DropWindow>>,
177 module_requests_dropped_stale_route: Arc<AtomicU64>,
178 client_frames_dropped_stale_route: Arc<AtomicU64>,
179 client_egress_close_delivery_failed: Arc<AtomicU64>,
180 goodbye_relay_client_failed: Arc<AtomicU64>,
181 goodbye_relay_module_dropped: Arc<AtomicU64>,
182 goodbye_relay_module_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
183 route_released_epoch_fenced: Arc<AtomicU64>,
184 route_release_stale_skipped: Arc<AtomicU64>,
185 drains_with_undeclared_gauge: Arc<AtomicU64>,
186}
187
188#[derive(Debug, Default)]
189struct ReaderDiagnostics {
190 busy: BTreeMap<String, BTreeMap<String, u64>>,
191 waits: BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>>,
192 warned_waits: HashMap<(String, String), tokio::time::Instant>,
193}
194
195#[derive(Debug)]
198struct DropWindow {
199 started_at: tokio::time::Instant,
200 buckets: VecDeque<DropBucket>,
201}
202
203#[derive(Debug)]
204struct DropBucket {
205 minute: u64,
206 count: u64,
207}
208
209impl Default for DropWindow {
210 fn default() -> Self {
211 Self {
212 started_at: tokio::time::Instant::now(),
213 buckets: VecDeque::new(),
214 }
215 }
216}
217
218impl DropWindow {
219 const MINUTE: Duration = Duration::from_secs(60);
220 const BUCKETS: u64 = 10;
221
222 fn record(&mut self, now: tokio::time::Instant) {
223 let minute = self.minute_at(now);
224 self.prune_before(minute);
225 match self.buckets.back_mut() {
226 Some(bucket) if bucket.minute == minute => bucket.count += 1,
227 _ => self.buckets.push_back(DropBucket { minute, count: 1 }),
228 }
229 }
230
231 fn count_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
232 let minute = self.minute_at(now);
233 self.prune_before(minute);
234 self.buckets.iter().map(|bucket| bucket.count).sum()
235 }
236
237 fn nonzero_minutes_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
238 let minute = self.minute_at(now);
239 self.prune_before(minute);
240 self.buckets.len() as u64
241 }
242
243 fn minute_at(&self, now: tokio::time::Instant) -> u64 {
244 now.saturating_duration_since(self.started_at).as_secs() / Self::MINUTE.as_secs()
245 }
246
247 fn prune_before(&mut self, current_minute: u64) {
248 while self
249 .buckets
250 .front()
251 .is_some_and(|bucket| current_minute.saturating_sub(bucket.minute) >= Self::BUCKETS)
252 {
253 self.buckets.pop_front();
254 }
255 }
256}
257
258impl DaemonCounters {
259 pub fn new() -> Self {
260 Self::default()
261 }
262
263 pub fn snapshot(&self) -> Value {
266 let mut snapshot = serde_json::Map::new();
267 let diagnostics = self
268 .reader_diagnostics
269 .lock()
270 .unwrap_or_else(|p| p.into_inner());
271 if !diagnostics.busy.is_empty() {
272 snapshot.insert(
273 "route_busy_by_module_principal".into(),
274 json!(diagnostics.busy),
275 );
276 }
277 if !diagnostics.waits.is_empty() {
278 snapshot.insert(
279 "reader_wait_over_5s_by_module_principal".into(),
280 json!(diagnostics.waits),
281 );
282 }
283 drop(diagnostics);
284 snapshot.insert(
285 "carried_containment_anchor_lost".into(),
286 self.carried_containment_anchor_lost
287 .load(Ordering::Relaxed)
288 .into(),
289 );
290 snapshot.insert(
291 "module_frames_dropped_no_route".into(),
292 self.module_frames_dropped_no_route
293 .load(Ordering::Relaxed)
294 .into(),
295 );
296 let mut drop_window = self
297 .module_frames_dropped_no_route_window
298 .lock()
299 .expect("drop-rate window mutex poisoned");
300 let now = tokio::time::Instant::now();
301 snapshot.insert(
302 "module_frames_dropped_no_route_last_10m".into(),
303 drop_window.count_last_10m(now).into(),
304 );
305 snapshot.insert(
306 "module_frames_dropped_no_route_nonzero_minutes_last_10m".into(),
307 drop_window.nonzero_minutes_last_10m(now).into(),
308 );
309 insert_nonempty_counts(
310 &mut snapshot,
311 "module_frames_dropped_no_route_by_module",
312 &self.module_frames_dropped_no_route_by_module,
313 );
314 insert_nonempty_counts(
315 &mut snapshot,
316 "agent_command_refusals_dropped_by_module",
317 &self.agent_command_refusals_dropped_by_module,
318 );
319 let unlabelled = self
320 .agent_command_refusals_dropped_unlabelled
321 .load(Ordering::Relaxed);
322 if unlabelled != 0 {
323 snapshot.insert(
324 "agent_command_refusals_dropped_unlabelled".into(),
325 unlabelled.into(),
326 );
327 }
328 snapshot.insert(
329 "module_frames_dropped_released_route".into(),
330 self.module_frames_dropped_released_route
331 .load(Ordering::Relaxed)
332 .into(),
333 );
334 insert_nonempty_counts(
335 &mut snapshot,
336 "module_frames_dropped_released_route_by_module",
337 &self.module_frames_dropped_released_route_by_module,
338 );
339 snapshot.insert(
340 "module_orphan_route_goodbyes_sent".into(),
341 self.module_orphan_route_goodbyes_sent
342 .load(Ordering::Relaxed)
343 .into(),
344 );
345 insert_nonempty_counts(
346 &mut snapshot,
347 "route_open_refused_by_code",
348 &self.route_open_refused_by_code,
349 );
350 insert_nonempty_counts(
351 &mut snapshot,
352 "route_open_accepted_by_principal",
353 &self.route_open_accepted_by_principal,
354 );
355 snapshot.insert(
356 "module_requests_dropped_stale_route".into(),
357 self.module_requests_dropped_stale_route
358 .load(Ordering::Relaxed)
359 .into(),
360 );
361 snapshot.insert(
362 "client_frames_dropped_stale_route".into(),
363 self.client_frames_dropped_stale_route
364 .load(Ordering::Relaxed)
365 .into(),
366 );
367 snapshot.insert(
368 "client_egress_close_delivery_failed".into(),
369 self.client_egress_close_delivery_failed
370 .load(Ordering::Relaxed)
371 .into(),
372 );
373 snapshot.insert(
374 "goodbye_relay_client_failed".into(),
375 self.goodbye_relay_client_failed
376 .load(Ordering::Relaxed)
377 .into(),
378 );
379 snapshot.insert(
380 "goodbye_relay_module_dropped".into(),
381 self.goodbye_relay_module_dropped
382 .load(Ordering::Relaxed)
383 .into(),
384 );
385 insert_nonempty_counts(
386 &mut snapshot,
387 "goodbye_relay_module_dropped_by_module",
388 &self.goodbye_relay_module_dropped_by_module,
389 );
390 snapshot.insert(
391 "route_released_epoch_fenced".into(),
392 self.route_released_epoch_fenced
393 .load(Ordering::Relaxed)
394 .into(),
395 );
396 snapshot.insert(
397 "route_release_stale_skipped".into(),
398 self.route_release_stale_skipped
399 .load(Ordering::Relaxed)
400 .into(),
401 );
402 snapshot.insert(
403 "drains_with_undeclared_gauge".into(),
404 self.drains_with_undeclared_gauge
405 .load(Ordering::Relaxed)
406 .into(),
407 );
408 Value::Object(snapshot)
409 }
410
411 pub(crate) fn increment_route_busy(&self, module: &str, principal: &str) {
412 let mut diagnostics = self
413 .reader_diagnostics
414 .lock()
415 .unwrap_or_else(|p| p.into_inner());
416 *diagnostics
417 .busy
418 .entry(module.into())
419 .or_default()
420 .entry(principal.into())
421 .or_default() += 1;
422 }
423
424 pub(crate) async fn observe_reader_wait<F: std::future::Future>(
427 &self,
428 labels: impl FnOnce() -> (String, String),
429 kind: &'static str,
430 future: F,
431 ) -> F::Output {
432 tokio::pin!(future);
433 if let std::task::Poll::Ready(result) =
436 std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
437 {
438 return result;
439 }
440 tokio::select! {
441 biased;
442 result = &mut future => return result,
443 _ = tokio::time::sleep(Duration::from_secs(5)) => {}
444 }
445 let (module, principal) = labels();
446 let should_warn = {
447 let mut diagnostics = self
448 .reader_diagnostics
449 .lock()
450 .unwrap_or_else(|p| p.into_inner());
451 *diagnostics
452 .waits
453 .entry(module.clone())
454 .or_default()
455 .entry(principal.clone())
456 .or_default()
457 .entry(kind.into())
458 .or_default() += 1;
459 let now = tokio::time::Instant::now();
460 let key = (module.to_owned(), principal.to_owned());
461 let should_warn = diagnostics
462 .warned_waits
463 .get(&key)
464 .is_none_or(|last| now.duration_since(*last) >= Duration::from_secs(60));
465 if should_warn {
466 diagnostics.warned_waits.insert(key, now);
467 }
468 should_warn
469 };
470 if should_warn {
471 warn!(
472 module_id = module,
473 client_principal = principal,
474 wait_kind = kind,
475 "connection reader wait exceeded 5 seconds"
476 );
477 }
478 future.await
479 }
480
481 pub(crate) fn increment_module_frames_dropped_no_route(&self, module_id: Option<&str>) {
482 self.module_frames_dropped_no_route
483 .fetch_add(1, Ordering::Relaxed);
484 if let Some(module_id) = module_id {
485 increment_keyed_count(&self.module_frames_dropped_no_route_by_module, module_id);
486 }
487 self.module_frames_dropped_no_route_window
488 .lock()
489 .expect("drop-rate window mutex poisoned")
490 .record(tokio::time::Instant::now());
491 }
492
493 pub(crate) fn increment_agent_command_refusal_dropped(
494 &self,
495 module_id: Option<&str>,
496 registry: &crate::registry::Registry,
497 ) {
498 let Some(module_id) =
499 module_id.filter(|id| registry.get_module(id).ok().flatten().is_some())
500 else {
501 self.agent_command_refusals_dropped_unlabelled
502 .fetch_add(1, Ordering::Relaxed);
503 return;
504 };
505 let mut counts = self
506 .agent_command_refusals_dropped_by_module
507 .lock()
508 .unwrap_or_else(|p| p.into_inner());
509 if let Some(count) = counts.get_mut(module_id) {
510 *count = count.saturating_add(1);
511 } else if counts.len() < MAX_AGENT_REFUSAL_MODULES {
512 counts.insert(module_id.to_owned(), 1);
513 } else {
514 self.agent_command_refusals_dropped_unlabelled
515 .fetch_add(1, Ordering::Relaxed);
516 }
517 }
518
519 pub(crate) fn increment_module_frames_dropped_released_route(&self, module_id: Option<&str>) {
523 self.module_frames_dropped_released_route
524 .fetch_add(1, Ordering::Relaxed);
525 if let Some(module_id) = module_id {
526 increment_keyed_count(
527 &self.module_frames_dropped_released_route_by_module,
528 module_id,
529 );
530 }
531 }
532
533 pub(crate) fn increment_module_orphan_route_goodbyes_sent(&self) {
534 self.module_orphan_route_goodbyes_sent
535 .fetch_add(1, Ordering::Relaxed);
536 }
537
538 pub(crate) fn increment_route_open_refused(&self, code: &'static str) {
539 debug_assert!(ROUTE_OPEN_REFUSAL_COUNTER_CODES.contains(&code));
540 increment_keyed_count(&self.route_open_refused_by_code, code);
541 }
542
543 pub(crate) fn increment_route_open_accepted(&self, principal: &str) {
553 increment_keyed_count(&self.route_open_accepted_by_principal, principal);
554 }
555
556 pub(crate) fn increment_module_requests_dropped_stale_route(&self) {
557 self.module_requests_dropped_stale_route
558 .fetch_add(1, Ordering::Relaxed);
559 }
560
561 pub(crate) fn increment_client_frames_dropped_stale_route(&self) {
562 self.client_frames_dropped_stale_route
563 .fetch_add(1, Ordering::Relaxed);
564 }
565
566 pub(crate) fn increment_client_egress_close_delivery_failed(&self) {
567 self.client_egress_close_delivery_failed
568 .fetch_add(1, Ordering::Relaxed);
569 }
570
571 pub(crate) fn increment_goodbye_relay_client_failed(&self) {
572 self.goodbye_relay_client_failed
573 .fetch_add(1, Ordering::Relaxed);
574 }
575
576 pub(crate) fn increment_goodbye_relay_module_dropped(&self, module_id: Option<&str>) {
577 self.goodbye_relay_module_dropped
578 .fetch_add(1, Ordering::Relaxed);
579 if let Some(module_id) = module_id {
580 increment_keyed_count(&self.goodbye_relay_module_dropped_by_module, module_id);
581 }
582 }
583
584 pub(crate) fn increment_route_released_epoch_fenced(&self) {
585 self.route_released_epoch_fenced
586 .fetch_add(1, Ordering::Relaxed);
587 }
588
589 pub(crate) fn increment_route_release_stale_skipped(&self) {
590 self.route_release_stale_skipped
591 .fetch_add(1, Ordering::Relaxed);
592 }
593
594 pub(crate) fn increment_drains_with_undeclared_gauge(&self) {
595 self.drains_with_undeclared_gauge
596 .fetch_add(1, Ordering::Relaxed);
597 }
598
599 #[cfg(unix)]
600 pub(crate) fn increment_carried_containment_anchor_lost(&self) {
601 self.carried_containment_anchor_lost
602 .fetch_add(1, Ordering::Relaxed);
603 }
604}
605
606const MAX_AGENT_REFUSAL_MODULES: usize = 1_024;
607
608#[cfg(test)]
609mod refusal_label_tests {
610 use super::*;
611
612 #[test]
613 fn dropped_refusal_labels_require_registration_and_stop_at_the_hard_cap() {
614 let registry = crate::Registry::default();
615 let counters = DaemonCounters::default();
616 counters.increment_agent_command_refusal_dropped(Some("unregistered"), ®istry);
617 assert!(counters
618 .snapshot()
619 .get("agent_command_refusals_dropped_by_module")
620 .is_none());
621 for index in 0..=1_024 {
622 let id = format!("registered-{index}");
623 registry
624 .register_with_control_ops(
625 subc_protocol::manifest::ModuleManifest::builder(&id, "0.1.0")
626 .protocol_ver(1)
627 .build(),
628 1,
629 crate::ConnectionId::new(index as u64 + 1),
630 Vec::new(),
631 )
632 .unwrap();
633 counters.increment_agent_command_refusal_dropped(Some(&id), ®istry);
634 }
635 counters.increment_agent_command_refusal_dropped(Some("registered-0"), ®istry);
636 let snapshot = counters.snapshot();
637 let labels = snapshot["agent_command_refusals_dropped_by_module"]
638 .as_object()
639 .unwrap();
640 assert_eq!(labels.len(), 1_024);
641 assert_eq!(labels["registered-0"], 2);
642 assert!(!labels.contains_key("registered-1024"));
643 assert_eq!(snapshot["agent_command_refusals_dropped_unlabelled"], 2);
644 }
645}
646
647fn increment_keyed_count(counts: &Mutex<HashMap<String, u64>>, key: &str) {
648 *counts
649 .lock()
650 .expect("keyed counter mutex poisoned")
651 .entry(key.to_string())
652 .or_default() += 1;
653}
654
655fn insert_nonempty_counts(
656 snapshot: &mut serde_json::Map<String, Value>,
657 key: &str,
658 counts: &Mutex<HashMap<String, u64>>,
659) {
660 let counts: MutexGuard<'_, HashMap<String, u64>> =
661 counts.lock().expect("keyed counter mutex poisoned");
662 if !counts.is_empty() {
663 snapshot.insert(key.to_string(), json!(&*counts));
664 }
665}
666
667#[cfg(test)]
668mod tests {
669 use super::*;
670
671 #[test]
672 fn registration_channel_closed_route_refusal_is_counted() {
673 let counters = DaemonCounters::new();
674 counters.increment_route_open_refused(crate::supervise::REGISTRATION_CHANNEL_CLOSED);
675 assert_eq!(
676 counters.snapshot()["route_open_refused_by_code"],
677 serde_json::json!({"registration_channel_closed": 1})
678 );
679 }
680
681 #[test]
682 fn ready_reader_wait_needs_no_runtime_clock_or_labels() {
683 use std::{
684 future::Future,
685 task::{Context, Poll, Waker},
686 };
687
688 let counters = DaemonCounters::new();
689 let wait = counters.observe_reader_wait(
690 || panic!("ready waits must not resolve diagnostic labels"),
691 "module_queue",
692 std::future::ready(7),
693 );
694 let mut wait = std::pin::pin!(wait);
695 let mut cx = Context::from_waker(Waker::noop());
696 assert_eq!(wait.as_mut().poll(&mut cx), Poll::Ready(7));
698 assert!(counters.reader_diagnostics.lock().unwrap().waits.is_empty());
699 }
700
701 #[tokio::test(start_paused = true)]
702 async fn reader_wait_labels_are_resolved_once_only_after_five_seconds() {
703 use std::{
704 cell::Cell,
705 future::{poll_fn, Future},
706 task::Poll,
707 };
708
709 let counters = DaemonCounters::new();
710 let calls = Cell::new(0);
711 let (sender, receiver) = tokio::sync::oneshot::channel();
712 let wait = counters.observe_reader_wait(
713 || {
714 calls.set(calls.get() + 1);
715 ("alpha".into(), "direct".into())
716 },
717 "module_queue",
718 receiver,
719 );
720 tokio::pin!(wait);
721 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
722 .await
723 .is_pending());
724 tokio::time::advance(Duration::from_secs(4)).await;
725 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
726 .await
727 .is_pending());
728 assert_eq!(calls.get(), 0);
729 tokio::time::advance(Duration::from_secs(1)).await;
730 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
731 .await
732 .is_pending());
733 assert_eq!(calls.get(), 1);
734 assert_eq!(
735 counters.snapshot()["reader_wait_over_5s_by_module_principal"]["alpha"]["direct"]
736 ["module_queue"],
737 1
738 );
739 sender.send(()).unwrap();
740 wait.await.unwrap();
741 assert_eq!(calls.get(), 1);
742 }
743
744 #[test]
745 fn counter_snapshot_includes_zero_rate_and_omits_empty_module_maps() {
746 let counters = DaemonCounters::new();
747 let snapshot = counters.snapshot();
748
749 assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 0);
750 assert_eq!(
751 snapshot["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
752 0
753 );
754 assert!(snapshot
755 .get("module_frames_dropped_no_route_by_module")
756 .is_none());
757 assert!(snapshot
758 .get("goodbye_relay_module_dropped_by_module")
759 .is_none());
760 }
761
762 #[tokio::test(start_paused = true)]
763 async fn module_frame_drops_are_attributed_to_the_emitting_module() {
764 let counters = DaemonCounters::new();
765 counters.increment_module_frames_dropped_no_route(Some("alpha"));
766 counters.increment_module_frames_dropped_no_route(Some("alpha"));
767
768 let snapshot = counters.snapshot();
769 assert_eq!(snapshot["module_frames_dropped_no_route"], 2);
770 assert_eq!(
771 snapshot["module_frames_dropped_no_route_by_module"],
772 json!({ "alpha": 2 })
773 );
774 assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 2);
775 }
776
777 #[tokio::test(start_paused = true)]
778 async fn frame_drop_rate_ages_out_after_ten_minute_buckets() {
779 let counters = DaemonCounters::new();
780 counters.increment_module_frames_dropped_no_route(Some("alpha"));
781
782 tokio::time::advance(Duration::from_secs(9 * 60)).await;
783 assert_eq!(
784 counters.snapshot()["module_frames_dropped_no_route_last_10m"],
785 1
786 );
787
788 tokio::time::advance(Duration::from_secs(60)).await;
789 assert_eq!(
790 counters.snapshot()["module_frames_dropped_no_route_last_10m"],
791 0
792 );
793 }
794
795 #[tokio::test(start_paused = true)]
796 async fn frame_drop_window_counts_only_nonzero_minutes() {
797 let counters = DaemonCounters::new();
798 for minute in 0..10 {
799 if minute > 0 {
800 tokio::time::advance(Duration::from_secs(60)).await;
801 }
802 if minute != 4 {
803 counters.increment_module_frames_dropped_no_route(Some("alpha"));
804 }
805 }
806
807 assert_eq!(
808 counters.snapshot()["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
809 9
810 );
811 }
812
813 #[test]
814 fn goodbye_relay_drops_are_attributed_to_the_target_module() {
815 let counters = DaemonCounters::new();
816 counters.increment_goodbye_relay_module_dropped(Some("alpha"));
817
818 assert_eq!(
819 counters.snapshot()["goodbye_relay_module_dropped_by_module"],
820 json!({ "alpha": 1 })
821 );
822 }
823}