1use std::{
2 collections::{BTreeMap, HashMap, VecDeque},
3 sync::{
4 atomic::{AtomicU64, Ordering},
5 Arc, Mutex, MutexGuard,
6 },
7 time::Duration,
8};
9
10use serde_json::{json, Value};
11use tracing::{debug, warn};
12
13use crate::registry::ConnectionId;
14
15pub(crate) const CONTROL_WAIT_BUCKET: &str = "daemon/control";
18
19const ROUTE_OPEN_REFUSAL_COUNTER_CODES: &[&str] = &[
27 "module_warming",
28 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY,
29 ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
30 "target_unavailable",
31 subc_protocol::error_codes::TARGET_FLOW_UNSUPPORTED,
32 subc_protocol::error_codes::TARGET_FEDERATED_UNSUPPORTED,
33 subc_protocol::error_codes::FEDERATED_STAMP_REQUIRED,
34 subc_protocol::error_codes::TARGET_AGENT_RUN_UNSUPPORTED,
35 "module_removed",
36 "module_no_protocol",
37 "unknown_module",
38 "module_reloading",
39 "op_not_allowed",
40 "bad_consumer_identity",
41 "capability_forbidden",
42 "admission_facts_not_permitted",
43 "admission_facts_target_not_allowed",
44 "route_limit",
45 "forwarding_error",
46 "module_timeout",
47 ROUTE_OPEN_REFUSED_BREAKER_OPEN,
48 "module_rejected",
49 subc_protocol::error_codes::SCOPE_EPOCH_REQUIRED,
50 subc_protocol::error_codes::SCOPE_NOT_SYNCED,
51 subc_protocol::error_codes::SCOPE_NOT_LIVE,
52 subc_protocol::error_codes::SCOPE_ENDED,
53 subc_protocol::error_codes::SCOPE_NOT_CARRIER,
54 subc_protocol::error_codes::SCOPE_BIND_PROOF_REQUIRED,
55 subc_protocol::error_codes::SCOPE_BIND_PROOF_MISMATCH,
56 subc_protocol::error_codes::SCOPE_CHANGED,
57 subc_protocol::error_codes::INVALID_REQUEST,
58];
59
60pub(crate) const ROUTE_OPEN_REFUSED_BREAKER_OPEN: &str = "module_timeout_breaker_open";
72
73pub(crate) const ROUTE_OPEN_REFUSED_DECLARED_NOT_READY: &str = "module_warming_declared_not_ready";
79
80pub(crate) const ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED: &str =
87 "module_warming_required_capability_unprovided";
88
89#[derive(Debug, Clone, Default)]
91pub struct ConnectedClients {
92 count: Arc<AtomicU64>,
93}
94
95impl ConnectedClients {
96 pub fn new() -> Self {
97 Self::default()
98 }
99
100 pub fn count(&self) -> u64 {
101 self.count.load(Ordering::SeqCst)
102 }
103
104 pub(crate) fn open(&self, connection_id: ConnectionId) -> ConnectedClientGuard {
105 let previous = self.count.fetch_add(1, Ordering::SeqCst);
106 let current = previous + 1;
107 debug!(
115 connection_id = connection_id.get(),
116 connected_clients = current,
117 previous_connected_clients = previous,
118 "authenticated connection count changed"
119 );
120 ConnectedClientGuard {
121 clients: self.clone(),
122 connection_id,
123 }
124 }
125}
126
127pub(crate) struct ConnectedClientGuard {
128 clients: ConnectedClients,
129 connection_id: ConnectionId,
130}
131
132impl Drop for ConnectedClientGuard {
133 fn drop(&mut self) {
134 let previous = self.clients.count.fetch_sub(1, Ordering::SeqCst);
135 let current = previous.saturating_sub(1);
136 debug!(
138 connection_id = self.connection_id.get(),
139 connected_clients = current,
140 previous_connected_clients = previous,
141 "authenticated connection count changed"
142 );
143 }
144}
145
146#[derive(Debug, Clone, Default)]
148pub struct DaemonCounters {
149 reader_diagnostics: Arc<Mutex<ReaderDiagnostics>>,
150 carried_containment_anchor_lost: Arc<AtomicU64>,
151 module_frames_dropped_no_route: Arc<AtomicU64>,
156 module_frames_dropped_released_route: Arc<AtomicU64>,
162 module_frames_dropped_released_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
163 module_orphan_route_goodbyes_sent: Arc<AtomicU64>,
168 module_frames_dropped_no_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
171 agent_command_refusals_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
172 agent_command_refusals_dropped_unlabelled: Arc<std::sync::atomic::AtomicU64>,
173 route_open_refused_by_code: Arc<Mutex<HashMap<String, u64>>>,
174 route_open_accepted_by_principal: Arc<Mutex<HashMap<String, u64>>>,
175 module_frames_dropped_no_route_window: Arc<Mutex<DropWindow>>,
176 module_requests_dropped_stale_route: Arc<AtomicU64>,
177 client_frames_dropped_stale_route: Arc<AtomicU64>,
178 client_egress_close_delivery_failed: Arc<AtomicU64>,
179 goodbye_relay_client_failed: Arc<AtomicU64>,
180 goodbye_relay_module_dropped: Arc<AtomicU64>,
181 goodbye_relay_module_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
182 route_released_epoch_fenced: Arc<AtomicU64>,
183 route_release_stale_skipped: Arc<AtomicU64>,
184 drains_with_undeclared_gauge: Arc<AtomicU64>,
185}
186
187#[derive(Debug, Default)]
188struct ReaderDiagnostics {
189 busy: BTreeMap<String, BTreeMap<String, u64>>,
190 waits: BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>>,
191 warned_waits: HashMap<(String, String), tokio::time::Instant>,
192}
193
194#[derive(Debug)]
197struct DropWindow {
198 started_at: tokio::time::Instant,
199 buckets: VecDeque<DropBucket>,
200}
201
202#[derive(Debug)]
203struct DropBucket {
204 minute: u64,
205 count: u64,
206}
207
208impl Default for DropWindow {
209 fn default() -> Self {
210 Self {
211 started_at: tokio::time::Instant::now(),
212 buckets: VecDeque::new(),
213 }
214 }
215}
216
217impl DropWindow {
218 const MINUTE: Duration = Duration::from_secs(60);
219 const BUCKETS: u64 = 10;
220
221 fn record(&mut self, now: tokio::time::Instant) {
222 let minute = self.minute_at(now);
223 self.prune_before(minute);
224 match self.buckets.back_mut() {
225 Some(bucket) if bucket.minute == minute => bucket.count += 1,
226 _ => self.buckets.push_back(DropBucket { minute, count: 1 }),
227 }
228 }
229
230 fn count_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
231 let minute = self.minute_at(now);
232 self.prune_before(minute);
233 self.buckets.iter().map(|bucket| bucket.count).sum()
234 }
235
236 fn nonzero_minutes_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
237 let minute = self.minute_at(now);
238 self.prune_before(minute);
239 self.buckets.len() as u64
240 }
241
242 fn minute_at(&self, now: tokio::time::Instant) -> u64 {
243 now.saturating_duration_since(self.started_at).as_secs() / Self::MINUTE.as_secs()
244 }
245
246 fn prune_before(&mut self, current_minute: u64) {
247 while self
248 .buckets
249 .front()
250 .is_some_and(|bucket| current_minute.saturating_sub(bucket.minute) >= Self::BUCKETS)
251 {
252 self.buckets.pop_front();
253 }
254 }
255}
256
257impl DaemonCounters {
258 pub fn new() -> Self {
259 Self::default()
260 }
261
262 pub fn snapshot(&self) -> Value {
265 let mut snapshot = serde_json::Map::new();
266 let diagnostics = self
267 .reader_diagnostics
268 .lock()
269 .unwrap_or_else(|p| p.into_inner());
270 if !diagnostics.busy.is_empty() {
271 snapshot.insert(
272 "route_busy_by_module_principal".into(),
273 json!(diagnostics.busy),
274 );
275 }
276 if !diagnostics.waits.is_empty() {
277 snapshot.insert(
278 "reader_wait_over_5s_by_module_principal".into(),
279 json!(diagnostics.waits),
280 );
281 }
282 drop(diagnostics);
283 snapshot.insert(
284 "carried_containment_anchor_lost".into(),
285 self.carried_containment_anchor_lost
286 .load(Ordering::Relaxed)
287 .into(),
288 );
289 snapshot.insert(
290 "module_frames_dropped_no_route".into(),
291 self.module_frames_dropped_no_route
292 .load(Ordering::Relaxed)
293 .into(),
294 );
295 let mut drop_window = self
296 .module_frames_dropped_no_route_window
297 .lock()
298 .expect("drop-rate window mutex poisoned");
299 let now = tokio::time::Instant::now();
300 snapshot.insert(
301 "module_frames_dropped_no_route_last_10m".into(),
302 drop_window.count_last_10m(now).into(),
303 );
304 snapshot.insert(
305 "module_frames_dropped_no_route_nonzero_minutes_last_10m".into(),
306 drop_window.nonzero_minutes_last_10m(now).into(),
307 );
308 insert_nonempty_counts(
309 &mut snapshot,
310 "module_frames_dropped_no_route_by_module",
311 &self.module_frames_dropped_no_route_by_module,
312 );
313 insert_nonempty_counts(
314 &mut snapshot,
315 "agent_command_refusals_dropped_by_module",
316 &self.agent_command_refusals_dropped_by_module,
317 );
318 let unlabelled = self
319 .agent_command_refusals_dropped_unlabelled
320 .load(Ordering::Relaxed);
321 if unlabelled != 0 {
322 snapshot.insert(
323 "agent_command_refusals_dropped_unlabelled".into(),
324 unlabelled.into(),
325 );
326 }
327 snapshot.insert(
328 "module_frames_dropped_released_route".into(),
329 self.module_frames_dropped_released_route
330 .load(Ordering::Relaxed)
331 .into(),
332 );
333 insert_nonempty_counts(
334 &mut snapshot,
335 "module_frames_dropped_released_route_by_module",
336 &self.module_frames_dropped_released_route_by_module,
337 );
338 snapshot.insert(
339 "module_orphan_route_goodbyes_sent".into(),
340 self.module_orphan_route_goodbyes_sent
341 .load(Ordering::Relaxed)
342 .into(),
343 );
344 insert_nonempty_counts(
345 &mut snapshot,
346 "route_open_refused_by_code",
347 &self.route_open_refused_by_code,
348 );
349 insert_nonempty_counts(
350 &mut snapshot,
351 "route_open_accepted_by_principal",
352 &self.route_open_accepted_by_principal,
353 );
354 snapshot.insert(
355 "module_requests_dropped_stale_route".into(),
356 self.module_requests_dropped_stale_route
357 .load(Ordering::Relaxed)
358 .into(),
359 );
360 snapshot.insert(
361 "client_frames_dropped_stale_route".into(),
362 self.client_frames_dropped_stale_route
363 .load(Ordering::Relaxed)
364 .into(),
365 );
366 snapshot.insert(
367 "client_egress_close_delivery_failed".into(),
368 self.client_egress_close_delivery_failed
369 .load(Ordering::Relaxed)
370 .into(),
371 );
372 snapshot.insert(
373 "goodbye_relay_client_failed".into(),
374 self.goodbye_relay_client_failed
375 .load(Ordering::Relaxed)
376 .into(),
377 );
378 snapshot.insert(
379 "goodbye_relay_module_dropped".into(),
380 self.goodbye_relay_module_dropped
381 .load(Ordering::Relaxed)
382 .into(),
383 );
384 insert_nonempty_counts(
385 &mut snapshot,
386 "goodbye_relay_module_dropped_by_module",
387 &self.goodbye_relay_module_dropped_by_module,
388 );
389 snapshot.insert(
390 "route_released_epoch_fenced".into(),
391 self.route_released_epoch_fenced
392 .load(Ordering::Relaxed)
393 .into(),
394 );
395 snapshot.insert(
396 "route_release_stale_skipped".into(),
397 self.route_release_stale_skipped
398 .load(Ordering::Relaxed)
399 .into(),
400 );
401 snapshot.insert(
402 "drains_with_undeclared_gauge".into(),
403 self.drains_with_undeclared_gauge
404 .load(Ordering::Relaxed)
405 .into(),
406 );
407 Value::Object(snapshot)
408 }
409
410 pub(crate) fn increment_route_busy(&self, module: &str, principal: &str) {
411 let mut diagnostics = self
412 .reader_diagnostics
413 .lock()
414 .unwrap_or_else(|p| p.into_inner());
415 *diagnostics
416 .busy
417 .entry(module.into())
418 .or_default()
419 .entry(principal.into())
420 .or_default() += 1;
421 }
422
423 pub(crate) async fn observe_reader_wait<F: std::future::Future>(
426 &self,
427 labels: impl FnOnce() -> (String, String),
428 kind: &'static str,
429 future: F,
430 ) -> F::Output {
431 tokio::pin!(future);
432 if let std::task::Poll::Ready(result) =
435 std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
436 {
437 return result;
438 }
439 tokio::select! {
440 biased;
441 result = &mut future => return result,
442 _ = tokio::time::sleep(Duration::from_secs(5)) => {}
443 }
444 let (module, principal) = labels();
445 let should_warn = {
446 let mut diagnostics = self
447 .reader_diagnostics
448 .lock()
449 .unwrap_or_else(|p| p.into_inner());
450 *diagnostics
451 .waits
452 .entry(module.clone())
453 .or_default()
454 .entry(principal.clone())
455 .or_default()
456 .entry(kind.into())
457 .or_default() += 1;
458 let now = tokio::time::Instant::now();
459 let key = (module.to_owned(), principal.to_owned());
460 let should_warn = diagnostics
461 .warned_waits
462 .get(&key)
463 .is_none_or(|last| now.duration_since(*last) >= Duration::from_secs(60));
464 if should_warn {
465 diagnostics.warned_waits.insert(key, now);
466 }
467 should_warn
468 };
469 if should_warn {
470 warn!(
471 module_id = module,
472 client_principal = principal,
473 wait_kind = kind,
474 "connection reader wait exceeded 5 seconds"
475 );
476 }
477 future.await
478 }
479
480 pub(crate) fn increment_module_frames_dropped_no_route(&self, module_id: Option<&str>) {
481 self.module_frames_dropped_no_route
482 .fetch_add(1, Ordering::Relaxed);
483 if let Some(module_id) = module_id {
484 increment_keyed_count(&self.module_frames_dropped_no_route_by_module, module_id);
485 }
486 self.module_frames_dropped_no_route_window
487 .lock()
488 .expect("drop-rate window mutex poisoned")
489 .record(tokio::time::Instant::now());
490 }
491
492 pub(crate) fn increment_agent_command_refusal_dropped(
493 &self,
494 module_id: Option<&str>,
495 registry: &crate::registry::Registry,
496 ) {
497 let Some(module_id) =
498 module_id.filter(|id| registry.get_module(id).ok().flatten().is_some())
499 else {
500 self.agent_command_refusals_dropped_unlabelled
501 .fetch_add(1, Ordering::Relaxed);
502 return;
503 };
504 let mut counts = self
505 .agent_command_refusals_dropped_by_module
506 .lock()
507 .unwrap_or_else(|p| p.into_inner());
508 if let Some(count) = counts.get_mut(module_id) {
509 *count = count.saturating_add(1);
510 } else if counts.len() < MAX_AGENT_REFUSAL_MODULES {
511 counts.insert(module_id.to_owned(), 1);
512 } else {
513 self.agent_command_refusals_dropped_unlabelled
514 .fetch_add(1, Ordering::Relaxed);
515 }
516 }
517
518 pub(crate) fn increment_module_frames_dropped_released_route(&self, module_id: Option<&str>) {
522 self.module_frames_dropped_released_route
523 .fetch_add(1, Ordering::Relaxed);
524 if let Some(module_id) = module_id {
525 increment_keyed_count(
526 &self.module_frames_dropped_released_route_by_module,
527 module_id,
528 );
529 }
530 }
531
532 pub(crate) fn increment_module_orphan_route_goodbyes_sent(&self) {
533 self.module_orphan_route_goodbyes_sent
534 .fetch_add(1, Ordering::Relaxed);
535 }
536
537 pub(crate) fn increment_route_open_refused(&self, code: &'static str) {
538 debug_assert!(ROUTE_OPEN_REFUSAL_COUNTER_CODES.contains(&code));
539 increment_keyed_count(&self.route_open_refused_by_code, code);
540 }
541
542 pub(crate) fn increment_route_open_accepted(&self, principal: &str) {
552 increment_keyed_count(&self.route_open_accepted_by_principal, principal);
553 }
554
555 pub(crate) fn increment_module_requests_dropped_stale_route(&self) {
556 self.module_requests_dropped_stale_route
557 .fetch_add(1, Ordering::Relaxed);
558 }
559
560 pub(crate) fn increment_client_frames_dropped_stale_route(&self) {
561 self.client_frames_dropped_stale_route
562 .fetch_add(1, Ordering::Relaxed);
563 }
564
565 pub(crate) fn increment_client_egress_close_delivery_failed(&self) {
566 self.client_egress_close_delivery_failed
567 .fetch_add(1, Ordering::Relaxed);
568 }
569
570 pub(crate) fn increment_goodbye_relay_client_failed(&self) {
571 self.goodbye_relay_client_failed
572 .fetch_add(1, Ordering::Relaxed);
573 }
574
575 pub(crate) fn increment_goodbye_relay_module_dropped(&self, module_id: Option<&str>) {
576 self.goodbye_relay_module_dropped
577 .fetch_add(1, Ordering::Relaxed);
578 if let Some(module_id) = module_id {
579 increment_keyed_count(&self.goodbye_relay_module_dropped_by_module, module_id);
580 }
581 }
582
583 pub(crate) fn increment_route_released_epoch_fenced(&self) {
584 self.route_released_epoch_fenced
585 .fetch_add(1, Ordering::Relaxed);
586 }
587
588 pub(crate) fn increment_route_release_stale_skipped(&self) {
589 self.route_release_stale_skipped
590 .fetch_add(1, Ordering::Relaxed);
591 }
592
593 pub(crate) fn increment_drains_with_undeclared_gauge(&self) {
594 self.drains_with_undeclared_gauge
595 .fetch_add(1, Ordering::Relaxed);
596 }
597
598 #[cfg(unix)]
599 pub(crate) fn increment_carried_containment_anchor_lost(&self) {
600 self.carried_containment_anchor_lost
601 .fetch_add(1, Ordering::Relaxed);
602 }
603}
604
605const MAX_AGENT_REFUSAL_MODULES: usize = 1_024;
606
607#[cfg(test)]
608mod refusal_label_tests {
609 use super::*;
610
611 #[test]
612 fn dropped_refusal_labels_require_registration_and_stop_at_the_hard_cap() {
613 let registry = crate::Registry::default();
614 let counters = DaemonCounters::default();
615 counters.increment_agent_command_refusal_dropped(Some("unregistered"), ®istry);
616 assert!(counters
617 .snapshot()
618 .get("agent_command_refusals_dropped_by_module")
619 .is_none());
620 for index in 0..=1_024 {
621 let id = format!("registered-{index}");
622 registry
623 .register_with_control_ops(
624 subc_protocol::manifest::ModuleManifest::builder(&id, "0.1.0")
625 .protocol_ver(1)
626 .build(),
627 1,
628 crate::ConnectionId::new(index as u64 + 1),
629 Vec::new(),
630 )
631 .unwrap();
632 counters.increment_agent_command_refusal_dropped(Some(&id), ®istry);
633 }
634 counters.increment_agent_command_refusal_dropped(Some("registered-0"), ®istry);
635 let snapshot = counters.snapshot();
636 let labels = snapshot["agent_command_refusals_dropped_by_module"]
637 .as_object()
638 .unwrap();
639 assert_eq!(labels.len(), 1_024);
640 assert_eq!(labels["registered-0"], 2);
641 assert!(!labels.contains_key("registered-1024"));
642 assert_eq!(snapshot["agent_command_refusals_dropped_unlabelled"], 2);
643 }
644}
645
646fn increment_keyed_count(counts: &Mutex<HashMap<String, u64>>, key: &str) {
647 *counts
648 .lock()
649 .expect("keyed counter mutex poisoned")
650 .entry(key.to_string())
651 .or_default() += 1;
652}
653
654fn insert_nonempty_counts(
655 snapshot: &mut serde_json::Map<String, Value>,
656 key: &str,
657 counts: &Mutex<HashMap<String, u64>>,
658) {
659 let counts: MutexGuard<'_, HashMap<String, u64>> =
660 counts.lock().expect("keyed counter mutex poisoned");
661 if !counts.is_empty() {
662 snapshot.insert(key.to_string(), json!(&*counts));
663 }
664}
665
666#[cfg(test)]
667mod tests {
668 use super::*;
669
670 #[test]
671 fn ready_reader_wait_needs_no_runtime_clock_or_labels() {
672 use std::{
673 future::Future,
674 task::{Context, Poll, Waker},
675 };
676
677 let counters = DaemonCounters::new();
678 let wait = counters.observe_reader_wait(
679 || panic!("ready waits must not resolve diagnostic labels"),
680 "module_queue",
681 std::future::ready(7),
682 );
683 let mut wait = std::pin::pin!(wait);
684 let mut cx = Context::from_waker(Waker::noop());
685 assert_eq!(wait.as_mut().poll(&mut cx), Poll::Ready(7));
687 assert!(counters.reader_diagnostics.lock().unwrap().waits.is_empty());
688 }
689
690 #[tokio::test(start_paused = true)]
691 async fn reader_wait_labels_are_resolved_once_only_after_five_seconds() {
692 use std::{
693 cell::Cell,
694 future::{poll_fn, Future},
695 task::Poll,
696 };
697
698 let counters = DaemonCounters::new();
699 let calls = Cell::new(0);
700 let (sender, receiver) = tokio::sync::oneshot::channel();
701 let wait = counters.observe_reader_wait(
702 || {
703 calls.set(calls.get() + 1);
704 ("alpha".into(), "direct".into())
705 },
706 "module_queue",
707 receiver,
708 );
709 tokio::pin!(wait);
710 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
711 .await
712 .is_pending());
713 tokio::time::advance(Duration::from_secs(4)).await;
714 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
715 .await
716 .is_pending());
717 assert_eq!(calls.get(), 0);
718 tokio::time::advance(Duration::from_secs(1)).await;
719 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
720 .await
721 .is_pending());
722 assert_eq!(calls.get(), 1);
723 assert_eq!(
724 counters.snapshot()["reader_wait_over_5s_by_module_principal"]["alpha"]["direct"]
725 ["module_queue"],
726 1
727 );
728 sender.send(()).unwrap();
729 wait.await.unwrap();
730 assert_eq!(calls.get(), 1);
731 }
732
733 #[test]
734 fn counter_snapshot_includes_zero_rate_and_omits_empty_module_maps() {
735 let counters = DaemonCounters::new();
736 let snapshot = counters.snapshot();
737
738 assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 0);
739 assert_eq!(
740 snapshot["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
741 0
742 );
743 assert!(snapshot
744 .get("module_frames_dropped_no_route_by_module")
745 .is_none());
746 assert!(snapshot
747 .get("goodbye_relay_module_dropped_by_module")
748 .is_none());
749 }
750
751 #[tokio::test(start_paused = true)]
752 async fn module_frame_drops_are_attributed_to_the_emitting_module() {
753 let counters = DaemonCounters::new();
754 counters.increment_module_frames_dropped_no_route(Some("alpha"));
755 counters.increment_module_frames_dropped_no_route(Some("alpha"));
756
757 let snapshot = counters.snapshot();
758 assert_eq!(snapshot["module_frames_dropped_no_route"], 2);
759 assert_eq!(
760 snapshot["module_frames_dropped_no_route_by_module"],
761 json!({ "alpha": 2 })
762 );
763 assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 2);
764 }
765
766 #[tokio::test(start_paused = true)]
767 async fn frame_drop_rate_ages_out_after_ten_minute_buckets() {
768 let counters = DaemonCounters::new();
769 counters.increment_module_frames_dropped_no_route(Some("alpha"));
770
771 tokio::time::advance(Duration::from_secs(9 * 60)).await;
772 assert_eq!(
773 counters.snapshot()["module_frames_dropped_no_route_last_10m"],
774 1
775 );
776
777 tokio::time::advance(Duration::from_secs(60)).await;
778 assert_eq!(
779 counters.snapshot()["module_frames_dropped_no_route_last_10m"],
780 0
781 );
782 }
783
784 #[tokio::test(start_paused = true)]
785 async fn frame_drop_window_counts_only_nonzero_minutes() {
786 let counters = DaemonCounters::new();
787 for minute in 0..10 {
788 if minute > 0 {
789 tokio::time::advance(Duration::from_secs(60)).await;
790 }
791 if minute != 4 {
792 counters.increment_module_frames_dropped_no_route(Some("alpha"));
793 }
794 }
795
796 assert_eq!(
797 counters.snapshot()["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
798 9
799 );
800 }
801
802 #[test]
803 fn goodbye_relay_drops_are_attributed_to_the_target_module() {
804 let counters = DaemonCounters::new();
805 counters.increment_goodbye_relay_module_dropped(Some("alpha"));
806
807 assert_eq!(
808 counters.snapshot()["goodbye_relay_module_dropped_by_module"],
809 json!({ "alpha": 1 })
810 );
811 }
812}