1use std::{
2 collections::{BTreeMap, HashMap, VecDeque},
3 sync::{
4 atomic::{AtomicU64, Ordering},
5 Arc, Mutex, MutexGuard,
6 },
7 time::Duration,
8};
9
10use serde_json::{json, Value};
11use tracing::{debug, warn};
12
13use crate::registry::ConnectionId;
14
15pub(crate) const CONTROL_WAIT_BUCKET: &str = "daemon/control";
18
19const ROUTE_OPEN_REFUSAL_COUNTER_CODES: &[&str] = &[
27 "module_warming",
28 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY,
29 ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
30 "target_unavailable",
31 subc_protocol::error_codes::TARGET_FLOW_UNSUPPORTED,
32 subc_protocol::error_codes::TARGET_AGENT_RUN_UNSUPPORTED,
33 "module_removed",
34 "module_no_protocol",
35 "unknown_module",
36 "module_reloading",
37 "op_not_allowed",
38 "bad_consumer_identity",
39 "capability_forbidden",
40 "admission_facts_not_permitted",
41 "admission_facts_target_not_allowed",
42 "route_limit",
43 "forwarding_error",
44 "module_timeout",
45 ROUTE_OPEN_REFUSED_BREAKER_OPEN,
46 "module_rejected",
47 subc_protocol::error_codes::SCOPE_EPOCH_REQUIRED,
48 subc_protocol::error_codes::SCOPE_NOT_SYNCED,
49 subc_protocol::error_codes::SCOPE_NOT_LIVE,
50 subc_protocol::error_codes::SCOPE_ENDED,
51 subc_protocol::error_codes::SCOPE_NOT_CARRIER,
52 subc_protocol::error_codes::SCOPE_BIND_PROOF_REQUIRED,
53 subc_protocol::error_codes::SCOPE_BIND_PROOF_MISMATCH,
54 subc_protocol::error_codes::SCOPE_CHANGED,
55 subc_protocol::error_codes::INVALID_REQUEST,
56];
57
58pub(crate) const ROUTE_OPEN_REFUSED_BREAKER_OPEN: &str = "module_timeout_breaker_open";
70
71pub(crate) const ROUTE_OPEN_REFUSED_DECLARED_NOT_READY: &str = "module_warming_declared_not_ready";
77
78pub(crate) const ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED: &str =
85 "module_warming_required_capability_unprovided";
86
87#[derive(Debug, Clone, Default)]
89pub struct ConnectedClients {
90 count: Arc<AtomicU64>,
91}
92
93impl ConnectedClients {
94 pub fn new() -> Self {
95 Self::default()
96 }
97
98 pub fn count(&self) -> u64 {
99 self.count.load(Ordering::SeqCst)
100 }
101
102 pub(crate) fn open(&self, connection_id: ConnectionId) -> ConnectedClientGuard {
103 let previous = self.count.fetch_add(1, Ordering::SeqCst);
104 let current = previous + 1;
105 debug!(
113 connection_id = connection_id.get(),
114 connected_clients = current,
115 previous_connected_clients = previous,
116 "authenticated connection count changed"
117 );
118 ConnectedClientGuard {
119 clients: self.clone(),
120 connection_id,
121 }
122 }
123}
124
125pub(crate) struct ConnectedClientGuard {
126 clients: ConnectedClients,
127 connection_id: ConnectionId,
128}
129
130impl Drop for ConnectedClientGuard {
131 fn drop(&mut self) {
132 let previous = self.clients.count.fetch_sub(1, Ordering::SeqCst);
133 let current = previous.saturating_sub(1);
134 debug!(
136 connection_id = self.connection_id.get(),
137 connected_clients = current,
138 previous_connected_clients = previous,
139 "authenticated connection count changed"
140 );
141 }
142}
143
144#[derive(Debug, Clone, Default)]
146pub struct DaemonCounters {
147 reader_diagnostics: Arc<Mutex<ReaderDiagnostics>>,
148 carried_containment_anchor_lost: Arc<AtomicU64>,
149 module_frames_dropped_no_route: Arc<AtomicU64>,
154 module_frames_dropped_released_route: Arc<AtomicU64>,
160 module_frames_dropped_released_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
161 module_orphan_route_goodbyes_sent: Arc<AtomicU64>,
166 module_frames_dropped_no_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
169 route_open_refused_by_code: Arc<Mutex<HashMap<String, u64>>>,
170 route_open_accepted_by_principal: Arc<Mutex<HashMap<String, u64>>>,
171 module_frames_dropped_no_route_window: Arc<Mutex<DropWindow>>,
172 module_requests_dropped_stale_route: Arc<AtomicU64>,
173 client_frames_dropped_stale_route: Arc<AtomicU64>,
174 client_egress_close_delivery_failed: Arc<AtomicU64>,
175 goodbye_relay_client_failed: Arc<AtomicU64>,
176 goodbye_relay_module_dropped: Arc<AtomicU64>,
177 goodbye_relay_module_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
178 route_released_epoch_fenced: Arc<AtomicU64>,
179 route_release_stale_skipped: Arc<AtomicU64>,
180 drains_with_undeclared_gauge: Arc<AtomicU64>,
181}
182
183#[derive(Debug, Default)]
184struct ReaderDiagnostics {
185 busy: BTreeMap<String, BTreeMap<String, u64>>,
186 waits: BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>>,
187 warned_waits: HashMap<(String, String), tokio::time::Instant>,
188}
189
190#[derive(Debug)]
193struct DropWindow {
194 started_at: tokio::time::Instant,
195 buckets: VecDeque<DropBucket>,
196}
197
198#[derive(Debug)]
199struct DropBucket {
200 minute: u64,
201 count: u64,
202}
203
204impl Default for DropWindow {
205 fn default() -> Self {
206 Self {
207 started_at: tokio::time::Instant::now(),
208 buckets: VecDeque::new(),
209 }
210 }
211}
212
213impl DropWindow {
214 const MINUTE: Duration = Duration::from_secs(60);
215 const BUCKETS: u64 = 10;
216
217 fn record(&mut self, now: tokio::time::Instant) {
218 let minute = self.minute_at(now);
219 self.prune_before(minute);
220 match self.buckets.back_mut() {
221 Some(bucket) if bucket.minute == minute => bucket.count += 1,
222 _ => self.buckets.push_back(DropBucket { minute, count: 1 }),
223 }
224 }
225
226 fn count_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
227 let minute = self.minute_at(now);
228 self.prune_before(minute);
229 self.buckets.iter().map(|bucket| bucket.count).sum()
230 }
231
232 fn nonzero_minutes_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
233 let minute = self.minute_at(now);
234 self.prune_before(minute);
235 self.buckets.len() as u64
236 }
237
238 fn minute_at(&self, now: tokio::time::Instant) -> u64 {
239 now.saturating_duration_since(self.started_at).as_secs() / Self::MINUTE.as_secs()
240 }
241
242 fn prune_before(&mut self, current_minute: u64) {
243 while self
244 .buckets
245 .front()
246 .is_some_and(|bucket| current_minute.saturating_sub(bucket.minute) >= Self::BUCKETS)
247 {
248 self.buckets.pop_front();
249 }
250 }
251}
252
253impl DaemonCounters {
254 pub fn new() -> Self {
255 Self::default()
256 }
257
258 pub fn snapshot(&self) -> Value {
261 let mut snapshot = serde_json::Map::new();
262 let diagnostics = self
263 .reader_diagnostics
264 .lock()
265 .unwrap_or_else(|p| p.into_inner());
266 if !diagnostics.busy.is_empty() {
267 snapshot.insert(
268 "route_busy_by_module_principal".into(),
269 json!(diagnostics.busy),
270 );
271 }
272 if !diagnostics.waits.is_empty() {
273 snapshot.insert(
274 "reader_wait_over_5s_by_module_principal".into(),
275 json!(diagnostics.waits),
276 );
277 }
278 drop(diagnostics);
279 snapshot.insert(
280 "carried_containment_anchor_lost".into(),
281 self.carried_containment_anchor_lost
282 .load(Ordering::Relaxed)
283 .into(),
284 );
285 snapshot.insert(
286 "module_frames_dropped_no_route".into(),
287 self.module_frames_dropped_no_route
288 .load(Ordering::Relaxed)
289 .into(),
290 );
291 let mut drop_window = self
292 .module_frames_dropped_no_route_window
293 .lock()
294 .expect("drop-rate window mutex poisoned");
295 let now = tokio::time::Instant::now();
296 snapshot.insert(
297 "module_frames_dropped_no_route_last_10m".into(),
298 drop_window.count_last_10m(now).into(),
299 );
300 snapshot.insert(
301 "module_frames_dropped_no_route_nonzero_minutes_last_10m".into(),
302 drop_window.nonzero_minutes_last_10m(now).into(),
303 );
304 insert_nonempty_counts(
305 &mut snapshot,
306 "module_frames_dropped_no_route_by_module",
307 &self.module_frames_dropped_no_route_by_module,
308 );
309 snapshot.insert(
310 "module_frames_dropped_released_route".into(),
311 self.module_frames_dropped_released_route
312 .load(Ordering::Relaxed)
313 .into(),
314 );
315 insert_nonempty_counts(
316 &mut snapshot,
317 "module_frames_dropped_released_route_by_module",
318 &self.module_frames_dropped_released_route_by_module,
319 );
320 snapshot.insert(
321 "module_orphan_route_goodbyes_sent".into(),
322 self.module_orphan_route_goodbyes_sent
323 .load(Ordering::Relaxed)
324 .into(),
325 );
326 insert_nonempty_counts(
327 &mut snapshot,
328 "route_open_refused_by_code",
329 &self.route_open_refused_by_code,
330 );
331 insert_nonempty_counts(
332 &mut snapshot,
333 "route_open_accepted_by_principal",
334 &self.route_open_accepted_by_principal,
335 );
336 snapshot.insert(
337 "module_requests_dropped_stale_route".into(),
338 self.module_requests_dropped_stale_route
339 .load(Ordering::Relaxed)
340 .into(),
341 );
342 snapshot.insert(
343 "client_frames_dropped_stale_route".into(),
344 self.client_frames_dropped_stale_route
345 .load(Ordering::Relaxed)
346 .into(),
347 );
348 snapshot.insert(
349 "client_egress_close_delivery_failed".into(),
350 self.client_egress_close_delivery_failed
351 .load(Ordering::Relaxed)
352 .into(),
353 );
354 snapshot.insert(
355 "goodbye_relay_client_failed".into(),
356 self.goodbye_relay_client_failed
357 .load(Ordering::Relaxed)
358 .into(),
359 );
360 snapshot.insert(
361 "goodbye_relay_module_dropped".into(),
362 self.goodbye_relay_module_dropped
363 .load(Ordering::Relaxed)
364 .into(),
365 );
366 insert_nonempty_counts(
367 &mut snapshot,
368 "goodbye_relay_module_dropped_by_module",
369 &self.goodbye_relay_module_dropped_by_module,
370 );
371 snapshot.insert(
372 "route_released_epoch_fenced".into(),
373 self.route_released_epoch_fenced
374 .load(Ordering::Relaxed)
375 .into(),
376 );
377 snapshot.insert(
378 "route_release_stale_skipped".into(),
379 self.route_release_stale_skipped
380 .load(Ordering::Relaxed)
381 .into(),
382 );
383 snapshot.insert(
384 "drains_with_undeclared_gauge".into(),
385 self.drains_with_undeclared_gauge
386 .load(Ordering::Relaxed)
387 .into(),
388 );
389 Value::Object(snapshot)
390 }
391
392 pub(crate) fn increment_route_busy(&self, module: &str, principal: &str) {
393 let mut diagnostics = self
394 .reader_diagnostics
395 .lock()
396 .unwrap_or_else(|p| p.into_inner());
397 *diagnostics
398 .busy
399 .entry(module.into())
400 .or_default()
401 .entry(principal.into())
402 .or_default() += 1;
403 }
404
405 pub(crate) async fn observe_reader_wait<F: std::future::Future>(
408 &self,
409 labels: impl FnOnce() -> (String, String),
410 kind: &'static str,
411 future: F,
412 ) -> F::Output {
413 tokio::pin!(future);
414 if let std::task::Poll::Ready(result) =
417 std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
418 {
419 return result;
420 }
421 tokio::select! {
422 biased;
423 result = &mut future => return result,
424 _ = tokio::time::sleep(Duration::from_secs(5)) => {}
425 }
426 let (module, principal) = labels();
427 let should_warn = {
428 let mut diagnostics = self
429 .reader_diagnostics
430 .lock()
431 .unwrap_or_else(|p| p.into_inner());
432 *diagnostics
433 .waits
434 .entry(module.clone())
435 .or_default()
436 .entry(principal.clone())
437 .or_default()
438 .entry(kind.into())
439 .or_default() += 1;
440 let now = tokio::time::Instant::now();
441 let key = (module.to_owned(), principal.to_owned());
442 let should_warn = diagnostics
443 .warned_waits
444 .get(&key)
445 .is_none_or(|last| now.duration_since(*last) >= Duration::from_secs(60));
446 if should_warn {
447 diagnostics.warned_waits.insert(key, now);
448 }
449 should_warn
450 };
451 if should_warn {
452 warn!(
453 module_id = module,
454 client_principal = principal,
455 wait_kind = kind,
456 "connection reader wait exceeded 5 seconds"
457 );
458 }
459 future.await
460 }
461
462 pub(crate) fn increment_module_frames_dropped_no_route(&self, module_id: Option<&str>) {
463 self.module_frames_dropped_no_route
464 .fetch_add(1, Ordering::Relaxed);
465 if let Some(module_id) = module_id {
466 increment_keyed_count(&self.module_frames_dropped_no_route_by_module, module_id);
467 }
468 self.module_frames_dropped_no_route_window
469 .lock()
470 .expect("drop-rate window mutex poisoned")
471 .record(tokio::time::Instant::now());
472 }
473
474 pub(crate) fn increment_module_frames_dropped_released_route(&self, module_id: Option<&str>) {
478 self.module_frames_dropped_released_route
479 .fetch_add(1, Ordering::Relaxed);
480 if let Some(module_id) = module_id {
481 increment_keyed_count(
482 &self.module_frames_dropped_released_route_by_module,
483 module_id,
484 );
485 }
486 }
487
488 pub(crate) fn increment_module_orphan_route_goodbyes_sent(&self) {
489 self.module_orphan_route_goodbyes_sent
490 .fetch_add(1, Ordering::Relaxed);
491 }
492
493 pub(crate) fn increment_route_open_refused(&self, code: &'static str) {
494 debug_assert!(ROUTE_OPEN_REFUSAL_COUNTER_CODES.contains(&code));
495 increment_keyed_count(&self.route_open_refused_by_code, code);
496 }
497
498 pub(crate) fn increment_route_open_accepted(&self, principal: &str) {
508 increment_keyed_count(&self.route_open_accepted_by_principal, principal);
509 }
510
511 pub(crate) fn increment_module_requests_dropped_stale_route(&self) {
512 self.module_requests_dropped_stale_route
513 .fetch_add(1, Ordering::Relaxed);
514 }
515
516 pub(crate) fn increment_client_frames_dropped_stale_route(&self) {
517 self.client_frames_dropped_stale_route
518 .fetch_add(1, Ordering::Relaxed);
519 }
520
521 pub(crate) fn increment_client_egress_close_delivery_failed(&self) {
522 self.client_egress_close_delivery_failed
523 .fetch_add(1, Ordering::Relaxed);
524 }
525
526 pub(crate) fn increment_goodbye_relay_client_failed(&self) {
527 self.goodbye_relay_client_failed
528 .fetch_add(1, Ordering::Relaxed);
529 }
530
531 pub(crate) fn increment_goodbye_relay_module_dropped(&self, module_id: Option<&str>) {
532 self.goodbye_relay_module_dropped
533 .fetch_add(1, Ordering::Relaxed);
534 if let Some(module_id) = module_id {
535 increment_keyed_count(&self.goodbye_relay_module_dropped_by_module, module_id);
536 }
537 }
538
539 pub(crate) fn increment_route_released_epoch_fenced(&self) {
540 self.route_released_epoch_fenced
541 .fetch_add(1, Ordering::Relaxed);
542 }
543
544 pub(crate) fn increment_route_release_stale_skipped(&self) {
545 self.route_release_stale_skipped
546 .fetch_add(1, Ordering::Relaxed);
547 }
548
549 pub(crate) fn increment_drains_with_undeclared_gauge(&self) {
550 self.drains_with_undeclared_gauge
551 .fetch_add(1, Ordering::Relaxed);
552 }
553
554 #[cfg(unix)]
555 pub(crate) fn increment_carried_containment_anchor_lost(&self) {
556 self.carried_containment_anchor_lost
557 .fetch_add(1, Ordering::Relaxed);
558 }
559}
560
561fn increment_keyed_count(counts: &Mutex<HashMap<String, u64>>, key: &str) {
562 *counts
563 .lock()
564 .expect("keyed counter mutex poisoned")
565 .entry(key.to_string())
566 .or_default() += 1;
567}
568
569fn insert_nonempty_counts(
570 snapshot: &mut serde_json::Map<String, Value>,
571 key: &str,
572 counts: &Mutex<HashMap<String, u64>>,
573) {
574 let counts: MutexGuard<'_, HashMap<String, u64>> =
575 counts.lock().expect("keyed counter mutex poisoned");
576 if !counts.is_empty() {
577 snapshot.insert(key.to_string(), json!(&*counts));
578 }
579}
580
581#[cfg(test)]
582mod tests {
583 use super::*;
584
585 #[test]
586 fn ready_reader_wait_needs_no_runtime_clock_or_labels() {
587 use std::{
588 future::Future,
589 task::{Context, Poll, Waker},
590 };
591
592 let counters = DaemonCounters::new();
593 let wait = counters.observe_reader_wait(
594 || panic!("ready waits must not resolve diagnostic labels"),
595 "module_queue",
596 std::future::ready(7),
597 );
598 let mut wait = std::pin::pin!(wait);
599 let mut cx = Context::from_waker(Waker::noop());
600 assert_eq!(wait.as_mut().poll(&mut cx), Poll::Ready(7));
602 assert!(counters.reader_diagnostics.lock().unwrap().waits.is_empty());
603 }
604
605 #[tokio::test(start_paused = true)]
606 async fn reader_wait_labels_are_resolved_once_only_after_five_seconds() {
607 use std::{
608 cell::Cell,
609 future::{poll_fn, Future},
610 task::Poll,
611 };
612
613 let counters = DaemonCounters::new();
614 let calls = Cell::new(0);
615 let (sender, receiver) = tokio::sync::oneshot::channel();
616 let wait = counters.observe_reader_wait(
617 || {
618 calls.set(calls.get() + 1);
619 ("alpha".into(), "direct".into())
620 },
621 "module_queue",
622 receiver,
623 );
624 tokio::pin!(wait);
625 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
626 .await
627 .is_pending());
628 tokio::time::advance(Duration::from_secs(4)).await;
629 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
630 .await
631 .is_pending());
632 assert_eq!(calls.get(), 0);
633 tokio::time::advance(Duration::from_secs(1)).await;
634 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
635 .await
636 .is_pending());
637 assert_eq!(calls.get(), 1);
638 assert_eq!(
639 counters.snapshot()["reader_wait_over_5s_by_module_principal"]["alpha"]["direct"]
640 ["module_queue"],
641 1
642 );
643 sender.send(()).unwrap();
644 wait.await.unwrap();
645 assert_eq!(calls.get(), 1);
646 }
647
648 #[test]
649 fn counter_snapshot_includes_zero_rate_and_omits_empty_module_maps() {
650 let counters = DaemonCounters::new();
651 let snapshot = counters.snapshot();
652
653 assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 0);
654 assert_eq!(
655 snapshot["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
656 0
657 );
658 assert!(snapshot
659 .get("module_frames_dropped_no_route_by_module")
660 .is_none());
661 assert!(snapshot
662 .get("goodbye_relay_module_dropped_by_module")
663 .is_none());
664 }
665
666 #[tokio::test(start_paused = true)]
667 async fn module_frame_drops_are_attributed_to_the_emitting_module() {
668 let counters = DaemonCounters::new();
669 counters.increment_module_frames_dropped_no_route(Some("alpha"));
670 counters.increment_module_frames_dropped_no_route(Some("alpha"));
671
672 let snapshot = counters.snapshot();
673 assert_eq!(snapshot["module_frames_dropped_no_route"], 2);
674 assert_eq!(
675 snapshot["module_frames_dropped_no_route_by_module"],
676 json!({ "alpha": 2 })
677 );
678 assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 2);
679 }
680
681 #[tokio::test(start_paused = true)]
682 async fn frame_drop_rate_ages_out_after_ten_minute_buckets() {
683 let counters = DaemonCounters::new();
684 counters.increment_module_frames_dropped_no_route(Some("alpha"));
685
686 tokio::time::advance(Duration::from_secs(9 * 60)).await;
687 assert_eq!(
688 counters.snapshot()["module_frames_dropped_no_route_last_10m"],
689 1
690 );
691
692 tokio::time::advance(Duration::from_secs(60)).await;
693 assert_eq!(
694 counters.snapshot()["module_frames_dropped_no_route_last_10m"],
695 0
696 );
697 }
698
699 #[tokio::test(start_paused = true)]
700 async fn frame_drop_window_counts_only_nonzero_minutes() {
701 let counters = DaemonCounters::new();
702 for minute in 0..10 {
703 if minute > 0 {
704 tokio::time::advance(Duration::from_secs(60)).await;
705 }
706 if minute != 4 {
707 counters.increment_module_frames_dropped_no_route(Some("alpha"));
708 }
709 }
710
711 assert_eq!(
712 counters.snapshot()["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
713 9
714 );
715 }
716
717 #[test]
718 fn goodbye_relay_drops_are_attributed_to_the_target_module() {
719 let counters = DaemonCounters::new();
720 counters.increment_goodbye_relay_module_dropped(Some("alpha"));
721
722 assert_eq!(
723 counters.snapshot()["goodbye_relay_module_dropped_by_module"],
724 json!({ "alpha": 1 })
725 );
726 }
727}