1use std::{
2 collections::{BTreeMap, HashMap, VecDeque},
3 sync::{
4 atomic::{AtomicU64, Ordering},
5 Arc, Mutex, MutexGuard,
6 },
7 time::Duration,
8};
9
10use serde_json::{json, Value};
11use tracing::{debug, warn};
12
13use crate::registry::ConnectionId;
14
15pub(crate) const CONTROL_WAIT_BUCKET: &str = "daemon/control";
18
19const ROUTE_OPEN_REFUSAL_COUNTER_CODES: &[&str] = &[
27 "module_warming",
28 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY,
29 ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
30 "target_unavailable",
31 subc_protocol::error_codes::TARGET_FLOW_UNSUPPORTED,
32 subc_protocol::error_codes::TARGET_FEDERATED_UNSUPPORTED,
33 subc_protocol::error_codes::TARGET_AGENT_RUN_UNSUPPORTED,
34 "module_removed",
35 "module_no_protocol",
36 "unknown_module",
37 "module_reloading",
38 "op_not_allowed",
39 "bad_consumer_identity",
40 "capability_forbidden",
41 "admission_facts_not_permitted",
42 "admission_facts_target_not_allowed",
43 "route_limit",
44 "forwarding_error",
45 "module_timeout",
46 ROUTE_OPEN_REFUSED_BREAKER_OPEN,
47 "module_rejected",
48 subc_protocol::error_codes::SCOPE_EPOCH_REQUIRED,
49 subc_protocol::error_codes::SCOPE_NOT_SYNCED,
50 subc_protocol::error_codes::SCOPE_NOT_LIVE,
51 subc_protocol::error_codes::SCOPE_ENDED,
52 subc_protocol::error_codes::SCOPE_NOT_CARRIER,
53 subc_protocol::error_codes::SCOPE_BIND_PROOF_REQUIRED,
54 subc_protocol::error_codes::SCOPE_BIND_PROOF_MISMATCH,
55 subc_protocol::error_codes::SCOPE_CHANGED,
56 subc_protocol::error_codes::INVALID_REQUEST,
57];
58
59pub(crate) const ROUTE_OPEN_REFUSED_BREAKER_OPEN: &str = "module_timeout_breaker_open";
71
72pub(crate) const ROUTE_OPEN_REFUSED_DECLARED_NOT_READY: &str = "module_warming_declared_not_ready";
78
79pub(crate) const ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED: &str =
86 "module_warming_required_capability_unprovided";
87
88#[derive(Debug, Clone, Default)]
90pub struct ConnectedClients {
91 count: Arc<AtomicU64>,
92}
93
94impl ConnectedClients {
95 pub fn new() -> Self {
96 Self::default()
97 }
98
99 pub fn count(&self) -> u64 {
100 self.count.load(Ordering::SeqCst)
101 }
102
103 pub(crate) fn open(&self, connection_id: ConnectionId) -> ConnectedClientGuard {
104 let previous = self.count.fetch_add(1, Ordering::SeqCst);
105 let current = previous + 1;
106 debug!(
114 connection_id = connection_id.get(),
115 connected_clients = current,
116 previous_connected_clients = previous,
117 "authenticated connection count changed"
118 );
119 ConnectedClientGuard {
120 clients: self.clone(),
121 connection_id,
122 }
123 }
124}
125
126pub(crate) struct ConnectedClientGuard {
127 clients: ConnectedClients,
128 connection_id: ConnectionId,
129}
130
131impl Drop for ConnectedClientGuard {
132 fn drop(&mut self) {
133 let previous = self.clients.count.fetch_sub(1, Ordering::SeqCst);
134 let current = previous.saturating_sub(1);
135 debug!(
137 connection_id = self.connection_id.get(),
138 connected_clients = current,
139 previous_connected_clients = previous,
140 "authenticated connection count changed"
141 );
142 }
143}
144
145#[derive(Debug, Clone, Default)]
147pub struct DaemonCounters {
148 reader_diagnostics: Arc<Mutex<ReaderDiagnostics>>,
149 carried_containment_anchor_lost: Arc<AtomicU64>,
150 module_frames_dropped_no_route: Arc<AtomicU64>,
155 module_frames_dropped_released_route: Arc<AtomicU64>,
161 module_frames_dropped_released_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
162 module_orphan_route_goodbyes_sent: Arc<AtomicU64>,
167 module_frames_dropped_no_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
170 agent_command_refusals_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
171 agent_command_refusals_dropped_unlabelled: Arc<std::sync::atomic::AtomicU64>,
172 route_open_refused_by_code: Arc<Mutex<HashMap<String, u64>>>,
173 route_open_accepted_by_principal: Arc<Mutex<HashMap<String, u64>>>,
174 module_frames_dropped_no_route_window: Arc<Mutex<DropWindow>>,
175 module_requests_dropped_stale_route: Arc<AtomicU64>,
176 client_frames_dropped_stale_route: Arc<AtomicU64>,
177 client_egress_close_delivery_failed: Arc<AtomicU64>,
178 goodbye_relay_client_failed: Arc<AtomicU64>,
179 goodbye_relay_module_dropped: Arc<AtomicU64>,
180 goodbye_relay_module_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
181 route_released_epoch_fenced: Arc<AtomicU64>,
182 route_release_stale_skipped: Arc<AtomicU64>,
183 drains_with_undeclared_gauge: Arc<AtomicU64>,
184}
185
186#[derive(Debug, Default)]
187struct ReaderDiagnostics {
188 busy: BTreeMap<String, BTreeMap<String, u64>>,
189 waits: BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>>,
190 warned_waits: HashMap<(String, String), tokio::time::Instant>,
191}
192
193#[derive(Debug)]
196struct DropWindow {
197 started_at: tokio::time::Instant,
198 buckets: VecDeque<DropBucket>,
199}
200
201#[derive(Debug)]
202struct DropBucket {
203 minute: u64,
204 count: u64,
205}
206
207impl Default for DropWindow {
208 fn default() -> Self {
209 Self {
210 started_at: tokio::time::Instant::now(),
211 buckets: VecDeque::new(),
212 }
213 }
214}
215
216impl DropWindow {
217 const MINUTE: Duration = Duration::from_secs(60);
218 const BUCKETS: u64 = 10;
219
220 fn record(&mut self, now: tokio::time::Instant) {
221 let minute = self.minute_at(now);
222 self.prune_before(minute);
223 match self.buckets.back_mut() {
224 Some(bucket) if bucket.minute == minute => bucket.count += 1,
225 _ => self.buckets.push_back(DropBucket { minute, count: 1 }),
226 }
227 }
228
229 fn count_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
230 let minute = self.minute_at(now);
231 self.prune_before(minute);
232 self.buckets.iter().map(|bucket| bucket.count).sum()
233 }
234
235 fn nonzero_minutes_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
236 let minute = self.minute_at(now);
237 self.prune_before(minute);
238 self.buckets.len() as u64
239 }
240
241 fn minute_at(&self, now: tokio::time::Instant) -> u64 {
242 now.saturating_duration_since(self.started_at).as_secs() / Self::MINUTE.as_secs()
243 }
244
245 fn prune_before(&mut self, current_minute: u64) {
246 while self
247 .buckets
248 .front()
249 .is_some_and(|bucket| current_minute.saturating_sub(bucket.minute) >= Self::BUCKETS)
250 {
251 self.buckets.pop_front();
252 }
253 }
254}
255
256impl DaemonCounters {
257 pub fn new() -> Self {
258 Self::default()
259 }
260
261 pub fn snapshot(&self) -> Value {
264 let mut snapshot = serde_json::Map::new();
265 let diagnostics = self
266 .reader_diagnostics
267 .lock()
268 .unwrap_or_else(|p| p.into_inner());
269 if !diagnostics.busy.is_empty() {
270 snapshot.insert(
271 "route_busy_by_module_principal".into(),
272 json!(diagnostics.busy),
273 );
274 }
275 if !diagnostics.waits.is_empty() {
276 snapshot.insert(
277 "reader_wait_over_5s_by_module_principal".into(),
278 json!(diagnostics.waits),
279 );
280 }
281 drop(diagnostics);
282 snapshot.insert(
283 "carried_containment_anchor_lost".into(),
284 self.carried_containment_anchor_lost
285 .load(Ordering::Relaxed)
286 .into(),
287 );
288 snapshot.insert(
289 "module_frames_dropped_no_route".into(),
290 self.module_frames_dropped_no_route
291 .load(Ordering::Relaxed)
292 .into(),
293 );
294 let mut drop_window = self
295 .module_frames_dropped_no_route_window
296 .lock()
297 .expect("drop-rate window mutex poisoned");
298 let now = tokio::time::Instant::now();
299 snapshot.insert(
300 "module_frames_dropped_no_route_last_10m".into(),
301 drop_window.count_last_10m(now).into(),
302 );
303 snapshot.insert(
304 "module_frames_dropped_no_route_nonzero_minutes_last_10m".into(),
305 drop_window.nonzero_minutes_last_10m(now).into(),
306 );
307 insert_nonempty_counts(
308 &mut snapshot,
309 "module_frames_dropped_no_route_by_module",
310 &self.module_frames_dropped_no_route_by_module,
311 );
312 insert_nonempty_counts(
313 &mut snapshot,
314 "agent_command_refusals_dropped_by_module",
315 &self.agent_command_refusals_dropped_by_module,
316 );
317 let unlabelled = self
318 .agent_command_refusals_dropped_unlabelled
319 .load(Ordering::Relaxed);
320 if unlabelled != 0 {
321 snapshot.insert(
322 "agent_command_refusals_dropped_unlabelled".into(),
323 unlabelled.into(),
324 );
325 }
326 snapshot.insert(
327 "module_frames_dropped_released_route".into(),
328 self.module_frames_dropped_released_route
329 .load(Ordering::Relaxed)
330 .into(),
331 );
332 insert_nonempty_counts(
333 &mut snapshot,
334 "module_frames_dropped_released_route_by_module",
335 &self.module_frames_dropped_released_route_by_module,
336 );
337 snapshot.insert(
338 "module_orphan_route_goodbyes_sent".into(),
339 self.module_orphan_route_goodbyes_sent
340 .load(Ordering::Relaxed)
341 .into(),
342 );
343 insert_nonempty_counts(
344 &mut snapshot,
345 "route_open_refused_by_code",
346 &self.route_open_refused_by_code,
347 );
348 insert_nonempty_counts(
349 &mut snapshot,
350 "route_open_accepted_by_principal",
351 &self.route_open_accepted_by_principal,
352 );
353 snapshot.insert(
354 "module_requests_dropped_stale_route".into(),
355 self.module_requests_dropped_stale_route
356 .load(Ordering::Relaxed)
357 .into(),
358 );
359 snapshot.insert(
360 "client_frames_dropped_stale_route".into(),
361 self.client_frames_dropped_stale_route
362 .load(Ordering::Relaxed)
363 .into(),
364 );
365 snapshot.insert(
366 "client_egress_close_delivery_failed".into(),
367 self.client_egress_close_delivery_failed
368 .load(Ordering::Relaxed)
369 .into(),
370 );
371 snapshot.insert(
372 "goodbye_relay_client_failed".into(),
373 self.goodbye_relay_client_failed
374 .load(Ordering::Relaxed)
375 .into(),
376 );
377 snapshot.insert(
378 "goodbye_relay_module_dropped".into(),
379 self.goodbye_relay_module_dropped
380 .load(Ordering::Relaxed)
381 .into(),
382 );
383 insert_nonempty_counts(
384 &mut snapshot,
385 "goodbye_relay_module_dropped_by_module",
386 &self.goodbye_relay_module_dropped_by_module,
387 );
388 snapshot.insert(
389 "route_released_epoch_fenced".into(),
390 self.route_released_epoch_fenced
391 .load(Ordering::Relaxed)
392 .into(),
393 );
394 snapshot.insert(
395 "route_release_stale_skipped".into(),
396 self.route_release_stale_skipped
397 .load(Ordering::Relaxed)
398 .into(),
399 );
400 snapshot.insert(
401 "drains_with_undeclared_gauge".into(),
402 self.drains_with_undeclared_gauge
403 .load(Ordering::Relaxed)
404 .into(),
405 );
406 Value::Object(snapshot)
407 }
408
409 pub(crate) fn increment_route_busy(&self, module: &str, principal: &str) {
410 let mut diagnostics = self
411 .reader_diagnostics
412 .lock()
413 .unwrap_or_else(|p| p.into_inner());
414 *diagnostics
415 .busy
416 .entry(module.into())
417 .or_default()
418 .entry(principal.into())
419 .or_default() += 1;
420 }
421
422 pub(crate) async fn observe_reader_wait<F: std::future::Future>(
425 &self,
426 labels: impl FnOnce() -> (String, String),
427 kind: &'static str,
428 future: F,
429 ) -> F::Output {
430 tokio::pin!(future);
431 if let std::task::Poll::Ready(result) =
434 std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
435 {
436 return result;
437 }
438 tokio::select! {
439 biased;
440 result = &mut future => return result,
441 _ = tokio::time::sleep(Duration::from_secs(5)) => {}
442 }
443 let (module, principal) = labels();
444 let should_warn = {
445 let mut diagnostics = self
446 .reader_diagnostics
447 .lock()
448 .unwrap_or_else(|p| p.into_inner());
449 *diagnostics
450 .waits
451 .entry(module.clone())
452 .or_default()
453 .entry(principal.clone())
454 .or_default()
455 .entry(kind.into())
456 .or_default() += 1;
457 let now = tokio::time::Instant::now();
458 let key = (module.to_owned(), principal.to_owned());
459 let should_warn = diagnostics
460 .warned_waits
461 .get(&key)
462 .is_none_or(|last| now.duration_since(*last) >= Duration::from_secs(60));
463 if should_warn {
464 diagnostics.warned_waits.insert(key, now);
465 }
466 should_warn
467 };
468 if should_warn {
469 warn!(
470 module_id = module,
471 client_principal = principal,
472 wait_kind = kind,
473 "connection reader wait exceeded 5 seconds"
474 );
475 }
476 future.await
477 }
478
479 pub(crate) fn increment_module_frames_dropped_no_route(&self, module_id: Option<&str>) {
480 self.module_frames_dropped_no_route
481 .fetch_add(1, Ordering::Relaxed);
482 if let Some(module_id) = module_id {
483 increment_keyed_count(&self.module_frames_dropped_no_route_by_module, module_id);
484 }
485 self.module_frames_dropped_no_route_window
486 .lock()
487 .expect("drop-rate window mutex poisoned")
488 .record(tokio::time::Instant::now());
489 }
490
491 pub(crate) fn increment_agent_command_refusal_dropped(
492 &self,
493 module_id: Option<&str>,
494 registry: &crate::registry::Registry,
495 ) {
496 let Some(module_id) =
497 module_id.filter(|id| registry.get_module(id).ok().flatten().is_some())
498 else {
499 self.agent_command_refusals_dropped_unlabelled
500 .fetch_add(1, Ordering::Relaxed);
501 return;
502 };
503 let mut counts = self
504 .agent_command_refusals_dropped_by_module
505 .lock()
506 .unwrap_or_else(|p| p.into_inner());
507 if let Some(count) = counts.get_mut(module_id) {
508 *count = count.saturating_add(1);
509 } else if counts.len() < MAX_AGENT_REFUSAL_MODULES {
510 counts.insert(module_id.to_owned(), 1);
511 } else {
512 self.agent_command_refusals_dropped_unlabelled
513 .fetch_add(1, Ordering::Relaxed);
514 }
515 }
516
517 pub(crate) fn increment_module_frames_dropped_released_route(&self, module_id: Option<&str>) {
521 self.module_frames_dropped_released_route
522 .fetch_add(1, Ordering::Relaxed);
523 if let Some(module_id) = module_id {
524 increment_keyed_count(
525 &self.module_frames_dropped_released_route_by_module,
526 module_id,
527 );
528 }
529 }
530
531 pub(crate) fn increment_module_orphan_route_goodbyes_sent(&self) {
532 self.module_orphan_route_goodbyes_sent
533 .fetch_add(1, Ordering::Relaxed);
534 }
535
536 pub(crate) fn increment_route_open_refused(&self, code: &'static str) {
537 debug_assert!(ROUTE_OPEN_REFUSAL_COUNTER_CODES.contains(&code));
538 increment_keyed_count(&self.route_open_refused_by_code, code);
539 }
540
541 pub(crate) fn increment_route_open_accepted(&self, principal: &str) {
551 increment_keyed_count(&self.route_open_accepted_by_principal, principal);
552 }
553
554 pub(crate) fn increment_module_requests_dropped_stale_route(&self) {
555 self.module_requests_dropped_stale_route
556 .fetch_add(1, Ordering::Relaxed);
557 }
558
559 pub(crate) fn increment_client_frames_dropped_stale_route(&self) {
560 self.client_frames_dropped_stale_route
561 .fetch_add(1, Ordering::Relaxed);
562 }
563
564 pub(crate) fn increment_client_egress_close_delivery_failed(&self) {
565 self.client_egress_close_delivery_failed
566 .fetch_add(1, Ordering::Relaxed);
567 }
568
569 pub(crate) fn increment_goodbye_relay_client_failed(&self) {
570 self.goodbye_relay_client_failed
571 .fetch_add(1, Ordering::Relaxed);
572 }
573
574 pub(crate) fn increment_goodbye_relay_module_dropped(&self, module_id: Option<&str>) {
575 self.goodbye_relay_module_dropped
576 .fetch_add(1, Ordering::Relaxed);
577 if let Some(module_id) = module_id {
578 increment_keyed_count(&self.goodbye_relay_module_dropped_by_module, module_id);
579 }
580 }
581
582 pub(crate) fn increment_route_released_epoch_fenced(&self) {
583 self.route_released_epoch_fenced
584 .fetch_add(1, Ordering::Relaxed);
585 }
586
587 pub(crate) fn increment_route_release_stale_skipped(&self) {
588 self.route_release_stale_skipped
589 .fetch_add(1, Ordering::Relaxed);
590 }
591
592 pub(crate) fn increment_drains_with_undeclared_gauge(&self) {
593 self.drains_with_undeclared_gauge
594 .fetch_add(1, Ordering::Relaxed);
595 }
596
597 #[cfg(unix)]
598 pub(crate) fn increment_carried_containment_anchor_lost(&self) {
599 self.carried_containment_anchor_lost
600 .fetch_add(1, Ordering::Relaxed);
601 }
602}
603
604const MAX_AGENT_REFUSAL_MODULES: usize = 1_024;
605
606#[cfg(test)]
607mod refusal_label_tests {
608 use super::*;
609
610 #[test]
611 fn dropped_refusal_labels_require_registration_and_stop_at_the_hard_cap() {
612 let registry = crate::Registry::default();
613 let counters = DaemonCounters::default();
614 counters.increment_agent_command_refusal_dropped(Some("unregistered"), ®istry);
615 assert!(counters
616 .snapshot()
617 .get("agent_command_refusals_dropped_by_module")
618 .is_none());
619 for index in 0..=1_024 {
620 let id = format!("registered-{index}");
621 registry
622 .register_with_control_ops(
623 subc_protocol::manifest::ModuleManifest::builder(&id, "0.1.0")
624 .protocol_ver(1)
625 .build(),
626 1,
627 crate::ConnectionId::new(index as u64 + 1),
628 Vec::new(),
629 )
630 .unwrap();
631 counters.increment_agent_command_refusal_dropped(Some(&id), ®istry);
632 }
633 counters.increment_agent_command_refusal_dropped(Some("registered-0"), ®istry);
634 let snapshot = counters.snapshot();
635 let labels = snapshot["agent_command_refusals_dropped_by_module"]
636 .as_object()
637 .unwrap();
638 assert_eq!(labels.len(), 1_024);
639 assert_eq!(labels["registered-0"], 2);
640 assert!(!labels.contains_key("registered-1024"));
641 assert_eq!(snapshot["agent_command_refusals_dropped_unlabelled"], 2);
642 }
643}
644
645fn increment_keyed_count(counts: &Mutex<HashMap<String, u64>>, key: &str) {
646 *counts
647 .lock()
648 .expect("keyed counter mutex poisoned")
649 .entry(key.to_string())
650 .or_default() += 1;
651}
652
653fn insert_nonempty_counts(
654 snapshot: &mut serde_json::Map<String, Value>,
655 key: &str,
656 counts: &Mutex<HashMap<String, u64>>,
657) {
658 let counts: MutexGuard<'_, HashMap<String, u64>> =
659 counts.lock().expect("keyed counter mutex poisoned");
660 if !counts.is_empty() {
661 snapshot.insert(key.to_string(), json!(&*counts));
662 }
663}
664
665#[cfg(test)]
666mod tests {
667 use super::*;
668
669 #[test]
670 fn ready_reader_wait_needs_no_runtime_clock_or_labels() {
671 use std::{
672 future::Future,
673 task::{Context, Poll, Waker},
674 };
675
676 let counters = DaemonCounters::new();
677 let wait = counters.observe_reader_wait(
678 || panic!("ready waits must not resolve diagnostic labels"),
679 "module_queue",
680 std::future::ready(7),
681 );
682 let mut wait = std::pin::pin!(wait);
683 let mut cx = Context::from_waker(Waker::noop());
684 assert_eq!(wait.as_mut().poll(&mut cx), Poll::Ready(7));
686 assert!(counters.reader_diagnostics.lock().unwrap().waits.is_empty());
687 }
688
689 #[tokio::test(start_paused = true)]
690 async fn reader_wait_labels_are_resolved_once_only_after_five_seconds() {
691 use std::{
692 cell::Cell,
693 future::{poll_fn, Future},
694 task::Poll,
695 };
696
697 let counters = DaemonCounters::new();
698 let calls = Cell::new(0);
699 let (sender, receiver) = tokio::sync::oneshot::channel();
700 let wait = counters.observe_reader_wait(
701 || {
702 calls.set(calls.get() + 1);
703 ("alpha".into(), "direct".into())
704 },
705 "module_queue",
706 receiver,
707 );
708 tokio::pin!(wait);
709 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
710 .await
711 .is_pending());
712 tokio::time::advance(Duration::from_secs(4)).await;
713 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
714 .await
715 .is_pending());
716 assert_eq!(calls.get(), 0);
717 tokio::time::advance(Duration::from_secs(1)).await;
718 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
719 .await
720 .is_pending());
721 assert_eq!(calls.get(), 1);
722 assert_eq!(
723 counters.snapshot()["reader_wait_over_5s_by_module_principal"]["alpha"]["direct"]
724 ["module_queue"],
725 1
726 );
727 sender.send(()).unwrap();
728 wait.await.unwrap();
729 assert_eq!(calls.get(), 1);
730 }
731
732 #[test]
733 fn counter_snapshot_includes_zero_rate_and_omits_empty_module_maps() {
734 let counters = DaemonCounters::new();
735 let snapshot = counters.snapshot();
736
737 assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 0);
738 assert_eq!(
739 snapshot["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
740 0
741 );
742 assert!(snapshot
743 .get("module_frames_dropped_no_route_by_module")
744 .is_none());
745 assert!(snapshot
746 .get("goodbye_relay_module_dropped_by_module")
747 .is_none());
748 }
749
750 #[tokio::test(start_paused = true)]
751 async fn module_frame_drops_are_attributed_to_the_emitting_module() {
752 let counters = DaemonCounters::new();
753 counters.increment_module_frames_dropped_no_route(Some("alpha"));
754 counters.increment_module_frames_dropped_no_route(Some("alpha"));
755
756 let snapshot = counters.snapshot();
757 assert_eq!(snapshot["module_frames_dropped_no_route"], 2);
758 assert_eq!(
759 snapshot["module_frames_dropped_no_route_by_module"],
760 json!({ "alpha": 2 })
761 );
762 assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 2);
763 }
764
765 #[tokio::test(start_paused = true)]
766 async fn frame_drop_rate_ages_out_after_ten_minute_buckets() {
767 let counters = DaemonCounters::new();
768 counters.increment_module_frames_dropped_no_route(Some("alpha"));
769
770 tokio::time::advance(Duration::from_secs(9 * 60)).await;
771 assert_eq!(
772 counters.snapshot()["module_frames_dropped_no_route_last_10m"],
773 1
774 );
775
776 tokio::time::advance(Duration::from_secs(60)).await;
777 assert_eq!(
778 counters.snapshot()["module_frames_dropped_no_route_last_10m"],
779 0
780 );
781 }
782
783 #[tokio::test(start_paused = true)]
784 async fn frame_drop_window_counts_only_nonzero_minutes() {
785 let counters = DaemonCounters::new();
786 for minute in 0..10 {
787 if minute > 0 {
788 tokio::time::advance(Duration::from_secs(60)).await;
789 }
790 if minute != 4 {
791 counters.increment_module_frames_dropped_no_route(Some("alpha"));
792 }
793 }
794
795 assert_eq!(
796 counters.snapshot()["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
797 9
798 );
799 }
800
801 #[test]
802 fn goodbye_relay_drops_are_attributed_to_the_target_module() {
803 let counters = DaemonCounters::new();
804 counters.increment_goodbye_relay_module_dropped(Some("alpha"));
805
806 assert_eq!(
807 counters.snapshot()["goodbye_relay_module_dropped_by_module"],
808 json!({ "alpha": 1 })
809 );
810 }
811}