1use std::{
2 collections::{BTreeMap, HashMap, VecDeque},
3 sync::{
4 atomic::{AtomicU64, Ordering},
5 Arc, Mutex, MutexGuard,
6 },
7 time::Duration,
8};
9
10use serde_json::{json, Value};
11use tracing::{debug, warn};
12
13use crate::registry::ConnectionId;
14
15pub(crate) const CONTROL_WAIT_BUCKET: &str = "daemon/control";
18
19const ROUTE_OPEN_REFUSAL_COUNTER_CODES: &[&str] = &[
27 "module_warming",
28 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY,
29 ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
30 "target_unavailable",
31 subc_protocol::error_codes::TARGET_FLOW_UNSUPPORTED,
32 subc_protocol::error_codes::TARGET_AGENT_RUN_UNSUPPORTED,
33 "module_removed",
34 "module_no_protocol",
35 "unknown_module",
36 "module_reloading",
37 "op_not_allowed",
38 "bad_consumer_identity",
39 "capability_forbidden",
40 "admission_facts_not_permitted",
41 "admission_facts_target_not_allowed",
42 "route_limit",
43 "forwarding_error",
44 "module_timeout",
45 ROUTE_OPEN_REFUSED_BREAKER_OPEN,
46 "module_rejected",
47 subc_protocol::error_codes::SCOPE_EPOCH_REQUIRED,
48 subc_protocol::error_codes::SCOPE_NOT_SYNCED,
49 subc_protocol::error_codes::SCOPE_NOT_LIVE,
50 subc_protocol::error_codes::SCOPE_ENDED,
51 subc_protocol::error_codes::SCOPE_NOT_CARRIER,
52 subc_protocol::error_codes::SCOPE_BIND_PROOF_REQUIRED,
53 subc_protocol::error_codes::SCOPE_BIND_PROOF_MISMATCH,
54 subc_protocol::error_codes::SCOPE_CHANGED,
55 subc_protocol::error_codes::INVALID_REQUEST,
56];
57
58pub(crate) const ROUTE_OPEN_REFUSED_BREAKER_OPEN: &str = "module_timeout_breaker_open";
70
71pub(crate) const ROUTE_OPEN_REFUSED_DECLARED_NOT_READY: &str = "module_warming_declared_not_ready";
77
78pub(crate) const ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED: &str =
85 "module_warming_required_capability_unprovided";
86
87#[derive(Debug, Clone, Default)]
89pub struct ConnectedClients {
90 count: Arc<AtomicU64>,
91}
92
93impl ConnectedClients {
94 pub fn new() -> Self {
95 Self::default()
96 }
97
98 pub fn count(&self) -> u64 {
99 self.count.load(Ordering::SeqCst)
100 }
101
102 pub(crate) fn open(&self, connection_id: ConnectionId) -> ConnectedClientGuard {
103 let previous = self.count.fetch_add(1, Ordering::SeqCst);
104 let current = previous + 1;
105 debug!(
113 connection_id = connection_id.get(),
114 connected_clients = current,
115 previous_connected_clients = previous,
116 "authenticated connection count changed"
117 );
118 ConnectedClientGuard {
119 clients: self.clone(),
120 connection_id,
121 }
122 }
123}
124
125pub(crate) struct ConnectedClientGuard {
126 clients: ConnectedClients,
127 connection_id: ConnectionId,
128}
129
130impl Drop for ConnectedClientGuard {
131 fn drop(&mut self) {
132 let previous = self.clients.count.fetch_sub(1, Ordering::SeqCst);
133 let current = previous.saturating_sub(1);
134 debug!(
136 connection_id = self.connection_id.get(),
137 connected_clients = current,
138 previous_connected_clients = previous,
139 "authenticated connection count changed"
140 );
141 }
142}
143
144#[derive(Debug, Clone, Default)]
146pub struct DaemonCounters {
147 reader_diagnostics: Arc<Mutex<ReaderDiagnostics>>,
148 carried_containment_anchor_lost: Arc<AtomicU64>,
149 module_frames_dropped_no_route: Arc<AtomicU64>,
154 module_frames_dropped_released_route: Arc<AtomicU64>,
160 module_frames_dropped_released_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
161 module_orphan_route_goodbyes_sent: Arc<AtomicU64>,
166 module_frames_dropped_no_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
169 agent_command_refusals_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
170 agent_command_refusals_dropped_unlabelled: Arc<std::sync::atomic::AtomicU64>,
171 route_open_refused_by_code: Arc<Mutex<HashMap<String, u64>>>,
172 route_open_accepted_by_principal: Arc<Mutex<HashMap<String, u64>>>,
173 module_frames_dropped_no_route_window: Arc<Mutex<DropWindow>>,
174 module_requests_dropped_stale_route: Arc<AtomicU64>,
175 client_frames_dropped_stale_route: Arc<AtomicU64>,
176 client_egress_close_delivery_failed: Arc<AtomicU64>,
177 goodbye_relay_client_failed: Arc<AtomicU64>,
178 goodbye_relay_module_dropped: Arc<AtomicU64>,
179 goodbye_relay_module_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
180 route_released_epoch_fenced: Arc<AtomicU64>,
181 route_release_stale_skipped: Arc<AtomicU64>,
182 drains_with_undeclared_gauge: Arc<AtomicU64>,
183}
184
185#[derive(Debug, Default)]
186struct ReaderDiagnostics {
187 busy: BTreeMap<String, BTreeMap<String, u64>>,
188 waits: BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>>,
189 warned_waits: HashMap<(String, String), tokio::time::Instant>,
190}
191
192#[derive(Debug)]
195struct DropWindow {
196 started_at: tokio::time::Instant,
197 buckets: VecDeque<DropBucket>,
198}
199
200#[derive(Debug)]
201struct DropBucket {
202 minute: u64,
203 count: u64,
204}
205
206impl Default for DropWindow {
207 fn default() -> Self {
208 Self {
209 started_at: tokio::time::Instant::now(),
210 buckets: VecDeque::new(),
211 }
212 }
213}
214
215impl DropWindow {
216 const MINUTE: Duration = Duration::from_secs(60);
217 const BUCKETS: u64 = 10;
218
219 fn record(&mut self, now: tokio::time::Instant) {
220 let minute = self.minute_at(now);
221 self.prune_before(minute);
222 match self.buckets.back_mut() {
223 Some(bucket) if bucket.minute == minute => bucket.count += 1,
224 _ => self.buckets.push_back(DropBucket { minute, count: 1 }),
225 }
226 }
227
228 fn count_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
229 let minute = self.minute_at(now);
230 self.prune_before(minute);
231 self.buckets.iter().map(|bucket| bucket.count).sum()
232 }
233
234 fn nonzero_minutes_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
235 let minute = self.minute_at(now);
236 self.prune_before(minute);
237 self.buckets.len() as u64
238 }
239
240 fn minute_at(&self, now: tokio::time::Instant) -> u64 {
241 now.saturating_duration_since(self.started_at).as_secs() / Self::MINUTE.as_secs()
242 }
243
244 fn prune_before(&mut self, current_minute: u64) {
245 while self
246 .buckets
247 .front()
248 .is_some_and(|bucket| current_minute.saturating_sub(bucket.minute) >= Self::BUCKETS)
249 {
250 self.buckets.pop_front();
251 }
252 }
253}
254
255impl DaemonCounters {
256 pub fn new() -> Self {
257 Self::default()
258 }
259
260 pub fn snapshot(&self) -> Value {
263 let mut snapshot = serde_json::Map::new();
264 let diagnostics = self
265 .reader_diagnostics
266 .lock()
267 .unwrap_or_else(|p| p.into_inner());
268 if !diagnostics.busy.is_empty() {
269 snapshot.insert(
270 "route_busy_by_module_principal".into(),
271 json!(diagnostics.busy),
272 );
273 }
274 if !diagnostics.waits.is_empty() {
275 snapshot.insert(
276 "reader_wait_over_5s_by_module_principal".into(),
277 json!(diagnostics.waits),
278 );
279 }
280 drop(diagnostics);
281 snapshot.insert(
282 "carried_containment_anchor_lost".into(),
283 self.carried_containment_anchor_lost
284 .load(Ordering::Relaxed)
285 .into(),
286 );
287 snapshot.insert(
288 "module_frames_dropped_no_route".into(),
289 self.module_frames_dropped_no_route
290 .load(Ordering::Relaxed)
291 .into(),
292 );
293 let mut drop_window = self
294 .module_frames_dropped_no_route_window
295 .lock()
296 .expect("drop-rate window mutex poisoned");
297 let now = tokio::time::Instant::now();
298 snapshot.insert(
299 "module_frames_dropped_no_route_last_10m".into(),
300 drop_window.count_last_10m(now).into(),
301 );
302 snapshot.insert(
303 "module_frames_dropped_no_route_nonzero_minutes_last_10m".into(),
304 drop_window.nonzero_minutes_last_10m(now).into(),
305 );
306 insert_nonempty_counts(
307 &mut snapshot,
308 "module_frames_dropped_no_route_by_module",
309 &self.module_frames_dropped_no_route_by_module,
310 );
311 insert_nonempty_counts(
312 &mut snapshot,
313 "agent_command_refusals_dropped_by_module",
314 &self.agent_command_refusals_dropped_by_module,
315 );
316 let unlabelled = self
317 .agent_command_refusals_dropped_unlabelled
318 .load(Ordering::Relaxed);
319 if unlabelled != 0 {
320 snapshot.insert(
321 "agent_command_refusals_dropped_unlabelled".into(),
322 unlabelled.into(),
323 );
324 }
325 snapshot.insert(
326 "module_frames_dropped_released_route".into(),
327 self.module_frames_dropped_released_route
328 .load(Ordering::Relaxed)
329 .into(),
330 );
331 insert_nonempty_counts(
332 &mut snapshot,
333 "module_frames_dropped_released_route_by_module",
334 &self.module_frames_dropped_released_route_by_module,
335 );
336 snapshot.insert(
337 "module_orphan_route_goodbyes_sent".into(),
338 self.module_orphan_route_goodbyes_sent
339 .load(Ordering::Relaxed)
340 .into(),
341 );
342 insert_nonempty_counts(
343 &mut snapshot,
344 "route_open_refused_by_code",
345 &self.route_open_refused_by_code,
346 );
347 insert_nonempty_counts(
348 &mut snapshot,
349 "route_open_accepted_by_principal",
350 &self.route_open_accepted_by_principal,
351 );
352 snapshot.insert(
353 "module_requests_dropped_stale_route".into(),
354 self.module_requests_dropped_stale_route
355 .load(Ordering::Relaxed)
356 .into(),
357 );
358 snapshot.insert(
359 "client_frames_dropped_stale_route".into(),
360 self.client_frames_dropped_stale_route
361 .load(Ordering::Relaxed)
362 .into(),
363 );
364 snapshot.insert(
365 "client_egress_close_delivery_failed".into(),
366 self.client_egress_close_delivery_failed
367 .load(Ordering::Relaxed)
368 .into(),
369 );
370 snapshot.insert(
371 "goodbye_relay_client_failed".into(),
372 self.goodbye_relay_client_failed
373 .load(Ordering::Relaxed)
374 .into(),
375 );
376 snapshot.insert(
377 "goodbye_relay_module_dropped".into(),
378 self.goodbye_relay_module_dropped
379 .load(Ordering::Relaxed)
380 .into(),
381 );
382 insert_nonempty_counts(
383 &mut snapshot,
384 "goodbye_relay_module_dropped_by_module",
385 &self.goodbye_relay_module_dropped_by_module,
386 );
387 snapshot.insert(
388 "route_released_epoch_fenced".into(),
389 self.route_released_epoch_fenced
390 .load(Ordering::Relaxed)
391 .into(),
392 );
393 snapshot.insert(
394 "route_release_stale_skipped".into(),
395 self.route_release_stale_skipped
396 .load(Ordering::Relaxed)
397 .into(),
398 );
399 snapshot.insert(
400 "drains_with_undeclared_gauge".into(),
401 self.drains_with_undeclared_gauge
402 .load(Ordering::Relaxed)
403 .into(),
404 );
405 Value::Object(snapshot)
406 }
407
408 pub(crate) fn increment_route_busy(&self, module: &str, principal: &str) {
409 let mut diagnostics = self
410 .reader_diagnostics
411 .lock()
412 .unwrap_or_else(|p| p.into_inner());
413 *diagnostics
414 .busy
415 .entry(module.into())
416 .or_default()
417 .entry(principal.into())
418 .or_default() += 1;
419 }
420
421 pub(crate) async fn observe_reader_wait<F: std::future::Future>(
424 &self,
425 labels: impl FnOnce() -> (String, String),
426 kind: &'static str,
427 future: F,
428 ) -> F::Output {
429 tokio::pin!(future);
430 if let std::task::Poll::Ready(result) =
433 std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
434 {
435 return result;
436 }
437 tokio::select! {
438 biased;
439 result = &mut future => return result,
440 _ = tokio::time::sleep(Duration::from_secs(5)) => {}
441 }
442 let (module, principal) = labels();
443 let should_warn = {
444 let mut diagnostics = self
445 .reader_diagnostics
446 .lock()
447 .unwrap_or_else(|p| p.into_inner());
448 *diagnostics
449 .waits
450 .entry(module.clone())
451 .or_default()
452 .entry(principal.clone())
453 .or_default()
454 .entry(kind.into())
455 .or_default() += 1;
456 let now = tokio::time::Instant::now();
457 let key = (module.to_owned(), principal.to_owned());
458 let should_warn = diagnostics
459 .warned_waits
460 .get(&key)
461 .is_none_or(|last| now.duration_since(*last) >= Duration::from_secs(60));
462 if should_warn {
463 diagnostics.warned_waits.insert(key, now);
464 }
465 should_warn
466 };
467 if should_warn {
468 warn!(
469 module_id = module,
470 client_principal = principal,
471 wait_kind = kind,
472 "connection reader wait exceeded 5 seconds"
473 );
474 }
475 future.await
476 }
477
478 pub(crate) fn increment_module_frames_dropped_no_route(&self, module_id: Option<&str>) {
479 self.module_frames_dropped_no_route
480 .fetch_add(1, Ordering::Relaxed);
481 if let Some(module_id) = module_id {
482 increment_keyed_count(&self.module_frames_dropped_no_route_by_module, module_id);
483 }
484 self.module_frames_dropped_no_route_window
485 .lock()
486 .expect("drop-rate window mutex poisoned")
487 .record(tokio::time::Instant::now());
488 }
489
490 pub(crate) fn increment_agent_command_refusal_dropped(
491 &self,
492 module_id: Option<&str>,
493 registry: &crate::registry::Registry,
494 ) {
495 let Some(module_id) =
496 module_id.filter(|id| registry.get_module(id).ok().flatten().is_some())
497 else {
498 self.agent_command_refusals_dropped_unlabelled
499 .fetch_add(1, Ordering::Relaxed);
500 return;
501 };
502 let mut counts = self
503 .agent_command_refusals_dropped_by_module
504 .lock()
505 .unwrap_or_else(|p| p.into_inner());
506 if let Some(count) = counts.get_mut(module_id) {
507 *count = count.saturating_add(1);
508 } else if counts.len() < MAX_AGENT_REFUSAL_MODULES {
509 counts.insert(module_id.to_owned(), 1);
510 } else {
511 self.agent_command_refusals_dropped_unlabelled
512 .fetch_add(1, Ordering::Relaxed);
513 }
514 }
515
516 pub(crate) fn increment_module_frames_dropped_released_route(&self, module_id: Option<&str>) {
520 self.module_frames_dropped_released_route
521 .fetch_add(1, Ordering::Relaxed);
522 if let Some(module_id) = module_id {
523 increment_keyed_count(
524 &self.module_frames_dropped_released_route_by_module,
525 module_id,
526 );
527 }
528 }
529
530 pub(crate) fn increment_module_orphan_route_goodbyes_sent(&self) {
531 self.module_orphan_route_goodbyes_sent
532 .fetch_add(1, Ordering::Relaxed);
533 }
534
535 pub(crate) fn increment_route_open_refused(&self, code: &'static str) {
536 debug_assert!(ROUTE_OPEN_REFUSAL_COUNTER_CODES.contains(&code));
537 increment_keyed_count(&self.route_open_refused_by_code, code);
538 }
539
540 pub(crate) fn increment_route_open_accepted(&self, principal: &str) {
550 increment_keyed_count(&self.route_open_accepted_by_principal, principal);
551 }
552
553 pub(crate) fn increment_module_requests_dropped_stale_route(&self) {
554 self.module_requests_dropped_stale_route
555 .fetch_add(1, Ordering::Relaxed);
556 }
557
558 pub(crate) fn increment_client_frames_dropped_stale_route(&self) {
559 self.client_frames_dropped_stale_route
560 .fetch_add(1, Ordering::Relaxed);
561 }
562
563 pub(crate) fn increment_client_egress_close_delivery_failed(&self) {
564 self.client_egress_close_delivery_failed
565 .fetch_add(1, Ordering::Relaxed);
566 }
567
568 pub(crate) fn increment_goodbye_relay_client_failed(&self) {
569 self.goodbye_relay_client_failed
570 .fetch_add(1, Ordering::Relaxed);
571 }
572
573 pub(crate) fn increment_goodbye_relay_module_dropped(&self, module_id: Option<&str>) {
574 self.goodbye_relay_module_dropped
575 .fetch_add(1, Ordering::Relaxed);
576 if let Some(module_id) = module_id {
577 increment_keyed_count(&self.goodbye_relay_module_dropped_by_module, module_id);
578 }
579 }
580
581 pub(crate) fn increment_route_released_epoch_fenced(&self) {
582 self.route_released_epoch_fenced
583 .fetch_add(1, Ordering::Relaxed);
584 }
585
586 pub(crate) fn increment_route_release_stale_skipped(&self) {
587 self.route_release_stale_skipped
588 .fetch_add(1, Ordering::Relaxed);
589 }
590
591 pub(crate) fn increment_drains_with_undeclared_gauge(&self) {
592 self.drains_with_undeclared_gauge
593 .fetch_add(1, Ordering::Relaxed);
594 }
595
596 #[cfg(unix)]
597 pub(crate) fn increment_carried_containment_anchor_lost(&self) {
598 self.carried_containment_anchor_lost
599 .fetch_add(1, Ordering::Relaxed);
600 }
601}
602
603const MAX_AGENT_REFUSAL_MODULES: usize = 1_024;
604
605#[cfg(test)]
606mod refusal_label_tests {
607 use super::*;
608
609 #[test]
610 fn dropped_refusal_labels_require_registration_and_stop_at_the_hard_cap() {
611 let registry = crate::Registry::default();
612 let counters = DaemonCounters::default();
613 counters.increment_agent_command_refusal_dropped(Some("unregistered"), ®istry);
614 assert!(counters
615 .snapshot()
616 .get("agent_command_refusals_dropped_by_module")
617 .is_none());
618 for index in 0..=1_024 {
619 let id = format!("registered-{index}");
620 registry
621 .register_with_control_ops(
622 subc_protocol::manifest::ModuleManifest::builder(&id, "0.1.0")
623 .protocol_ver(1)
624 .build(),
625 1,
626 crate::ConnectionId::new(index as u64 + 1),
627 Vec::new(),
628 )
629 .unwrap();
630 counters.increment_agent_command_refusal_dropped(Some(&id), ®istry);
631 }
632 counters.increment_agent_command_refusal_dropped(Some("registered-0"), ®istry);
633 let snapshot = counters.snapshot();
634 let labels = snapshot["agent_command_refusals_dropped_by_module"]
635 .as_object()
636 .unwrap();
637 assert_eq!(labels.len(), 1_024);
638 assert_eq!(labels["registered-0"], 2);
639 assert!(!labels.contains_key("registered-1024"));
640 assert_eq!(snapshot["agent_command_refusals_dropped_unlabelled"], 2);
641 }
642}
643
644fn increment_keyed_count(counts: &Mutex<HashMap<String, u64>>, key: &str) {
645 *counts
646 .lock()
647 .expect("keyed counter mutex poisoned")
648 .entry(key.to_string())
649 .or_default() += 1;
650}
651
652fn insert_nonempty_counts(
653 snapshot: &mut serde_json::Map<String, Value>,
654 key: &str,
655 counts: &Mutex<HashMap<String, u64>>,
656) {
657 let counts: MutexGuard<'_, HashMap<String, u64>> =
658 counts.lock().expect("keyed counter mutex poisoned");
659 if !counts.is_empty() {
660 snapshot.insert(key.to_string(), json!(&*counts));
661 }
662}
663
664#[cfg(test)]
665mod tests {
666 use super::*;
667
668 #[test]
669 fn ready_reader_wait_needs_no_runtime_clock_or_labels() {
670 use std::{
671 future::Future,
672 task::{Context, Poll, Waker},
673 };
674
675 let counters = DaemonCounters::new();
676 let wait = counters.observe_reader_wait(
677 || panic!("ready waits must not resolve diagnostic labels"),
678 "module_queue",
679 std::future::ready(7),
680 );
681 let mut wait = std::pin::pin!(wait);
682 let mut cx = Context::from_waker(Waker::noop());
683 assert_eq!(wait.as_mut().poll(&mut cx), Poll::Ready(7));
685 assert!(counters.reader_diagnostics.lock().unwrap().waits.is_empty());
686 }
687
688 #[tokio::test(start_paused = true)]
689 async fn reader_wait_labels_are_resolved_once_only_after_five_seconds() {
690 use std::{
691 cell::Cell,
692 future::{poll_fn, Future},
693 task::Poll,
694 };
695
696 let counters = DaemonCounters::new();
697 let calls = Cell::new(0);
698 let (sender, receiver) = tokio::sync::oneshot::channel();
699 let wait = counters.observe_reader_wait(
700 || {
701 calls.set(calls.get() + 1);
702 ("alpha".into(), "direct".into())
703 },
704 "module_queue",
705 receiver,
706 );
707 tokio::pin!(wait);
708 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
709 .await
710 .is_pending());
711 tokio::time::advance(Duration::from_secs(4)).await;
712 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
713 .await
714 .is_pending());
715 assert_eq!(calls.get(), 0);
716 tokio::time::advance(Duration::from_secs(1)).await;
717 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
718 .await
719 .is_pending());
720 assert_eq!(calls.get(), 1);
721 assert_eq!(
722 counters.snapshot()["reader_wait_over_5s_by_module_principal"]["alpha"]["direct"]
723 ["module_queue"],
724 1
725 );
726 sender.send(()).unwrap();
727 wait.await.unwrap();
728 assert_eq!(calls.get(), 1);
729 }
730
731 #[test]
732 fn counter_snapshot_includes_zero_rate_and_omits_empty_module_maps() {
733 let counters = DaemonCounters::new();
734 let snapshot = counters.snapshot();
735
736 assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 0);
737 assert_eq!(
738 snapshot["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
739 0
740 );
741 assert!(snapshot
742 .get("module_frames_dropped_no_route_by_module")
743 .is_none());
744 assert!(snapshot
745 .get("goodbye_relay_module_dropped_by_module")
746 .is_none());
747 }
748
749 #[tokio::test(start_paused = true)]
750 async fn module_frame_drops_are_attributed_to_the_emitting_module() {
751 let counters = DaemonCounters::new();
752 counters.increment_module_frames_dropped_no_route(Some("alpha"));
753 counters.increment_module_frames_dropped_no_route(Some("alpha"));
754
755 let snapshot = counters.snapshot();
756 assert_eq!(snapshot["module_frames_dropped_no_route"], 2);
757 assert_eq!(
758 snapshot["module_frames_dropped_no_route_by_module"],
759 json!({ "alpha": 2 })
760 );
761 assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 2);
762 }
763
764 #[tokio::test(start_paused = true)]
765 async fn frame_drop_rate_ages_out_after_ten_minute_buckets() {
766 let counters = DaemonCounters::new();
767 counters.increment_module_frames_dropped_no_route(Some("alpha"));
768
769 tokio::time::advance(Duration::from_secs(9 * 60)).await;
770 assert_eq!(
771 counters.snapshot()["module_frames_dropped_no_route_last_10m"],
772 1
773 );
774
775 tokio::time::advance(Duration::from_secs(60)).await;
776 assert_eq!(
777 counters.snapshot()["module_frames_dropped_no_route_last_10m"],
778 0
779 );
780 }
781
782 #[tokio::test(start_paused = true)]
783 async fn frame_drop_window_counts_only_nonzero_minutes() {
784 let counters = DaemonCounters::new();
785 for minute in 0..10 {
786 if minute > 0 {
787 tokio::time::advance(Duration::from_secs(60)).await;
788 }
789 if minute != 4 {
790 counters.increment_module_frames_dropped_no_route(Some("alpha"));
791 }
792 }
793
794 assert_eq!(
795 counters.snapshot()["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
796 9
797 );
798 }
799
800 #[test]
801 fn goodbye_relay_drops_are_attributed_to_the_target_module() {
802 let counters = DaemonCounters::new();
803 counters.increment_goodbye_relay_module_dropped(Some("alpha"));
804
805 assert_eq!(
806 counters.snapshot()["goodbye_relay_module_dropped_by_module"],
807 json!({ "alpha": 1 })
808 );
809 }
810}