1use std::{
2 collections::{BTreeMap, HashMap, VecDeque},
3 sync::{
4 atomic::{AtomicU64, Ordering},
5 Arc, Mutex, MutexGuard,
6 },
7 time::Duration,
8};
9
10use serde_json::{json, Value};
11use tracing::{debug, warn};
12
13use crate::registry::ConnectionId;
14
15pub(crate) const CONTROL_WAIT_BUCKET: &str = "daemon/control";
18
19const ROUTE_OPEN_REFUSAL_COUNTER_CODES: &[&str] = &[
27 "module_warming",
28 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY,
29 ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
30 "target_unavailable",
31 subc_protocol::error_codes::TARGET_FLOW_UNSUPPORTED,
32 subc_protocol::error_codes::TARGET_AGENT_RUN_UNSUPPORTED,
33 "module_removed",
34 "module_no_protocol",
35 "unknown_module",
36 "module_reloading",
37 "op_not_allowed",
38 "bad_consumer_identity",
39 "capability_forbidden",
40 "admission_facts_not_permitted",
41 "admission_facts_target_not_allowed",
42 "route_limit",
43 "forwarding_error",
44 "module_timeout",
45 ROUTE_OPEN_REFUSED_BREAKER_OPEN,
46 "module_rejected",
47 subc_protocol::error_codes::SCOPE_EPOCH_REQUIRED,
48 subc_protocol::error_codes::SCOPE_NOT_SYNCED,
49 subc_protocol::error_codes::SCOPE_NOT_LIVE,
50 subc_protocol::error_codes::SCOPE_ENDED,
51 subc_protocol::error_codes::SCOPE_NOT_CARRIER,
52 subc_protocol::error_codes::SCOPE_BIND_PROOF_REQUIRED,
53 subc_protocol::error_codes::SCOPE_BIND_PROOF_MISMATCH,
54 subc_protocol::error_codes::SCOPE_CHANGED,
55 subc_protocol::error_codes::INVALID_REQUEST,
56];
57
58pub(crate) const ROUTE_OPEN_REFUSED_BREAKER_OPEN: &str = "module_timeout_breaker_open";
70
71pub(crate) const ROUTE_OPEN_REFUSED_DECLARED_NOT_READY: &str = "module_warming_declared_not_ready";
77
78pub(crate) const ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED: &str =
85 "module_warming_required_capability_unprovided";
86
87#[derive(Debug, Clone, Default)]
89pub struct ConnectedClients {
90 count: Arc<AtomicU64>,
91}
92
93impl ConnectedClients {
94 pub fn new() -> Self {
95 Self::default()
96 }
97
98 pub fn count(&self) -> u64 {
99 self.count.load(Ordering::SeqCst)
100 }
101
102 pub(crate) fn open(&self, connection_id: ConnectionId) -> ConnectedClientGuard {
103 let previous = self.count.fetch_add(1, Ordering::SeqCst);
104 let current = previous + 1;
105 debug!(
113 connection_id = connection_id.get(),
114 connected_clients = current,
115 previous_connected_clients = previous,
116 "authenticated connection count changed"
117 );
118 ConnectedClientGuard {
119 clients: self.clone(),
120 connection_id,
121 }
122 }
123}
124
125pub(crate) struct ConnectedClientGuard {
126 clients: ConnectedClients,
127 connection_id: ConnectionId,
128}
129
130impl Drop for ConnectedClientGuard {
131 fn drop(&mut self) {
132 let previous = self.clients.count.fetch_sub(1, Ordering::SeqCst);
133 let current = previous.saturating_sub(1);
134 debug!(
136 connection_id = self.connection_id.get(),
137 connected_clients = current,
138 previous_connected_clients = previous,
139 "authenticated connection count changed"
140 );
141 }
142}
143
144#[derive(Debug, Clone, Default)]
146pub struct DaemonCounters {
147 reader_diagnostics: Arc<Mutex<ReaderDiagnostics>>,
148 module_frames_dropped_no_route: Arc<AtomicU64>,
153 module_frames_dropped_released_route: Arc<AtomicU64>,
159 module_frames_dropped_released_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
160 module_orphan_route_goodbyes_sent: Arc<AtomicU64>,
165 module_frames_dropped_no_route_by_module: Arc<Mutex<HashMap<String, u64>>>,
168 route_open_refused_by_code: Arc<Mutex<HashMap<String, u64>>>,
169 route_open_accepted_by_principal: Arc<Mutex<HashMap<String, u64>>>,
170 module_frames_dropped_no_route_window: Arc<Mutex<DropWindow>>,
171 module_requests_dropped_stale_route: Arc<AtomicU64>,
172 client_frames_dropped_stale_route: Arc<AtomicU64>,
173 client_egress_close_delivery_failed: Arc<AtomicU64>,
174 goodbye_relay_client_failed: Arc<AtomicU64>,
175 goodbye_relay_module_dropped: Arc<AtomicU64>,
176 goodbye_relay_module_dropped_by_module: Arc<Mutex<HashMap<String, u64>>>,
177 route_released_epoch_fenced: Arc<AtomicU64>,
178 route_release_stale_skipped: Arc<AtomicU64>,
179 drains_with_undeclared_gauge: Arc<AtomicU64>,
180}
181
182#[derive(Debug, Default)]
183struct ReaderDiagnostics {
184 busy: BTreeMap<String, BTreeMap<String, u64>>,
185 waits: BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>>,
186 warned_waits: HashMap<(String, String), tokio::time::Instant>,
187}
188
189#[derive(Debug)]
192struct DropWindow {
193 started_at: tokio::time::Instant,
194 buckets: VecDeque<DropBucket>,
195}
196
197#[derive(Debug)]
198struct DropBucket {
199 minute: u64,
200 count: u64,
201}
202
203impl Default for DropWindow {
204 fn default() -> Self {
205 Self {
206 started_at: tokio::time::Instant::now(),
207 buckets: VecDeque::new(),
208 }
209 }
210}
211
212impl DropWindow {
213 const MINUTE: Duration = Duration::from_secs(60);
214 const BUCKETS: u64 = 10;
215
216 fn record(&mut self, now: tokio::time::Instant) {
217 let minute = self.minute_at(now);
218 self.prune_before(minute);
219 match self.buckets.back_mut() {
220 Some(bucket) if bucket.minute == minute => bucket.count += 1,
221 _ => self.buckets.push_back(DropBucket { minute, count: 1 }),
222 }
223 }
224
225 fn count_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
226 let minute = self.minute_at(now);
227 self.prune_before(minute);
228 self.buckets.iter().map(|bucket| bucket.count).sum()
229 }
230
231 fn nonzero_minutes_last_10m(&mut self, now: tokio::time::Instant) -> u64 {
232 let minute = self.minute_at(now);
233 self.prune_before(minute);
234 self.buckets.len() as u64
235 }
236
237 fn minute_at(&self, now: tokio::time::Instant) -> u64 {
238 now.saturating_duration_since(self.started_at).as_secs() / Self::MINUTE.as_secs()
239 }
240
241 fn prune_before(&mut self, current_minute: u64) {
242 while self
243 .buckets
244 .front()
245 .is_some_and(|bucket| current_minute.saturating_sub(bucket.minute) >= Self::BUCKETS)
246 {
247 self.buckets.pop_front();
248 }
249 }
250}
251
252impl DaemonCounters {
253 pub fn new() -> Self {
254 Self::default()
255 }
256
257 pub fn snapshot(&self) -> Value {
260 let mut snapshot = serde_json::Map::new();
261 let diagnostics = self
262 .reader_diagnostics
263 .lock()
264 .unwrap_or_else(|p| p.into_inner());
265 if !diagnostics.busy.is_empty() {
266 snapshot.insert(
267 "route_busy_by_module_principal".into(),
268 json!(diagnostics.busy),
269 );
270 }
271 if !diagnostics.waits.is_empty() {
272 snapshot.insert(
273 "reader_wait_over_5s_by_module_principal".into(),
274 json!(diagnostics.waits),
275 );
276 }
277 drop(diagnostics);
278 snapshot.insert(
279 "module_frames_dropped_no_route".into(),
280 self.module_frames_dropped_no_route
281 .load(Ordering::Relaxed)
282 .into(),
283 );
284 let mut drop_window = self
285 .module_frames_dropped_no_route_window
286 .lock()
287 .expect("drop-rate window mutex poisoned");
288 let now = tokio::time::Instant::now();
289 snapshot.insert(
290 "module_frames_dropped_no_route_last_10m".into(),
291 drop_window.count_last_10m(now).into(),
292 );
293 snapshot.insert(
294 "module_frames_dropped_no_route_nonzero_minutes_last_10m".into(),
295 drop_window.nonzero_minutes_last_10m(now).into(),
296 );
297 insert_nonempty_counts(
298 &mut snapshot,
299 "module_frames_dropped_no_route_by_module",
300 &self.module_frames_dropped_no_route_by_module,
301 );
302 snapshot.insert(
303 "module_frames_dropped_released_route".into(),
304 self.module_frames_dropped_released_route
305 .load(Ordering::Relaxed)
306 .into(),
307 );
308 insert_nonempty_counts(
309 &mut snapshot,
310 "module_frames_dropped_released_route_by_module",
311 &self.module_frames_dropped_released_route_by_module,
312 );
313 snapshot.insert(
314 "module_orphan_route_goodbyes_sent".into(),
315 self.module_orphan_route_goodbyes_sent
316 .load(Ordering::Relaxed)
317 .into(),
318 );
319 insert_nonempty_counts(
320 &mut snapshot,
321 "route_open_refused_by_code",
322 &self.route_open_refused_by_code,
323 );
324 insert_nonempty_counts(
325 &mut snapshot,
326 "route_open_accepted_by_principal",
327 &self.route_open_accepted_by_principal,
328 );
329 snapshot.insert(
330 "module_requests_dropped_stale_route".into(),
331 self.module_requests_dropped_stale_route
332 .load(Ordering::Relaxed)
333 .into(),
334 );
335 snapshot.insert(
336 "client_frames_dropped_stale_route".into(),
337 self.client_frames_dropped_stale_route
338 .load(Ordering::Relaxed)
339 .into(),
340 );
341 snapshot.insert(
342 "client_egress_close_delivery_failed".into(),
343 self.client_egress_close_delivery_failed
344 .load(Ordering::Relaxed)
345 .into(),
346 );
347 snapshot.insert(
348 "goodbye_relay_client_failed".into(),
349 self.goodbye_relay_client_failed
350 .load(Ordering::Relaxed)
351 .into(),
352 );
353 snapshot.insert(
354 "goodbye_relay_module_dropped".into(),
355 self.goodbye_relay_module_dropped
356 .load(Ordering::Relaxed)
357 .into(),
358 );
359 insert_nonempty_counts(
360 &mut snapshot,
361 "goodbye_relay_module_dropped_by_module",
362 &self.goodbye_relay_module_dropped_by_module,
363 );
364 snapshot.insert(
365 "route_released_epoch_fenced".into(),
366 self.route_released_epoch_fenced
367 .load(Ordering::Relaxed)
368 .into(),
369 );
370 snapshot.insert(
371 "route_release_stale_skipped".into(),
372 self.route_release_stale_skipped
373 .load(Ordering::Relaxed)
374 .into(),
375 );
376 snapshot.insert(
377 "drains_with_undeclared_gauge".into(),
378 self.drains_with_undeclared_gauge
379 .load(Ordering::Relaxed)
380 .into(),
381 );
382 Value::Object(snapshot)
383 }
384
385 pub(crate) fn increment_route_busy(&self, module: &str, principal: &str) {
386 let mut diagnostics = self
387 .reader_diagnostics
388 .lock()
389 .unwrap_or_else(|p| p.into_inner());
390 *diagnostics
391 .busy
392 .entry(module.into())
393 .or_default()
394 .entry(principal.into())
395 .or_default() += 1;
396 }
397
398 pub(crate) async fn observe_reader_wait<F: std::future::Future>(
401 &self,
402 labels: impl FnOnce() -> (String, String),
403 kind: &'static str,
404 future: F,
405 ) -> F::Output {
406 tokio::pin!(future);
407 if let std::task::Poll::Ready(result) =
410 std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
411 {
412 return result;
413 }
414 tokio::select! {
415 biased;
416 result = &mut future => return result,
417 _ = tokio::time::sleep(Duration::from_secs(5)) => {}
418 }
419 let (module, principal) = labels();
420 let should_warn = {
421 let mut diagnostics = self
422 .reader_diagnostics
423 .lock()
424 .unwrap_or_else(|p| p.into_inner());
425 *diagnostics
426 .waits
427 .entry(module.clone())
428 .or_default()
429 .entry(principal.clone())
430 .or_default()
431 .entry(kind.into())
432 .or_default() += 1;
433 let now = tokio::time::Instant::now();
434 let key = (module.to_owned(), principal.to_owned());
435 let should_warn = diagnostics
436 .warned_waits
437 .get(&key)
438 .is_none_or(|last| now.duration_since(*last) >= Duration::from_secs(60));
439 if should_warn {
440 diagnostics.warned_waits.insert(key, now);
441 }
442 should_warn
443 };
444 if should_warn {
445 warn!(
446 module_id = module,
447 client_principal = principal,
448 wait_kind = kind,
449 "connection reader wait exceeded 5 seconds"
450 );
451 }
452 future.await
453 }
454
455 pub(crate) fn increment_module_frames_dropped_no_route(&self, module_id: Option<&str>) {
456 self.module_frames_dropped_no_route
457 .fetch_add(1, Ordering::Relaxed);
458 if let Some(module_id) = module_id {
459 increment_keyed_count(&self.module_frames_dropped_no_route_by_module, module_id);
460 }
461 self.module_frames_dropped_no_route_window
462 .lock()
463 .expect("drop-rate window mutex poisoned")
464 .record(tokio::time::Instant::now());
465 }
466
467 pub(crate) fn increment_module_frames_dropped_released_route(&self, module_id: Option<&str>) {
471 self.module_frames_dropped_released_route
472 .fetch_add(1, Ordering::Relaxed);
473 if let Some(module_id) = module_id {
474 increment_keyed_count(
475 &self.module_frames_dropped_released_route_by_module,
476 module_id,
477 );
478 }
479 }
480
481 pub(crate) fn increment_module_orphan_route_goodbyes_sent(&self) {
482 self.module_orphan_route_goodbyes_sent
483 .fetch_add(1, Ordering::Relaxed);
484 }
485
486 pub(crate) fn increment_route_open_refused(&self, code: &'static str) {
487 debug_assert!(ROUTE_OPEN_REFUSAL_COUNTER_CODES.contains(&code));
488 increment_keyed_count(&self.route_open_refused_by_code, code);
489 }
490
491 pub(crate) fn increment_route_open_accepted(&self, principal: &str) {
501 increment_keyed_count(&self.route_open_accepted_by_principal, principal);
502 }
503
504 pub(crate) fn increment_module_requests_dropped_stale_route(&self) {
505 self.module_requests_dropped_stale_route
506 .fetch_add(1, Ordering::Relaxed);
507 }
508
509 pub(crate) fn increment_client_frames_dropped_stale_route(&self) {
510 self.client_frames_dropped_stale_route
511 .fetch_add(1, Ordering::Relaxed);
512 }
513
514 pub(crate) fn increment_client_egress_close_delivery_failed(&self) {
515 self.client_egress_close_delivery_failed
516 .fetch_add(1, Ordering::Relaxed);
517 }
518
519 pub(crate) fn increment_goodbye_relay_client_failed(&self) {
520 self.goodbye_relay_client_failed
521 .fetch_add(1, Ordering::Relaxed);
522 }
523
524 pub(crate) fn increment_goodbye_relay_module_dropped(&self, module_id: Option<&str>) {
525 self.goodbye_relay_module_dropped
526 .fetch_add(1, Ordering::Relaxed);
527 if let Some(module_id) = module_id {
528 increment_keyed_count(&self.goodbye_relay_module_dropped_by_module, module_id);
529 }
530 }
531
532 pub(crate) fn increment_route_released_epoch_fenced(&self) {
533 self.route_released_epoch_fenced
534 .fetch_add(1, Ordering::Relaxed);
535 }
536
537 pub(crate) fn increment_route_release_stale_skipped(&self) {
538 self.route_release_stale_skipped
539 .fetch_add(1, Ordering::Relaxed);
540 }
541
542 pub(crate) fn increment_drains_with_undeclared_gauge(&self) {
543 self.drains_with_undeclared_gauge
544 .fetch_add(1, Ordering::Relaxed);
545 }
546}
547
548fn increment_keyed_count(counts: &Mutex<HashMap<String, u64>>, key: &str) {
549 *counts
550 .lock()
551 .expect("keyed counter mutex poisoned")
552 .entry(key.to_string())
553 .or_default() += 1;
554}
555
556fn insert_nonempty_counts(
557 snapshot: &mut serde_json::Map<String, Value>,
558 key: &str,
559 counts: &Mutex<HashMap<String, u64>>,
560) {
561 let counts: MutexGuard<'_, HashMap<String, u64>> =
562 counts.lock().expect("keyed counter mutex poisoned");
563 if !counts.is_empty() {
564 snapshot.insert(key.to_string(), json!(&*counts));
565 }
566}
567
568#[cfg(test)]
569mod tests {
570 use super::*;
571
572 #[test]
573 fn ready_reader_wait_needs_no_runtime_clock_or_labels() {
574 use std::{
575 future::Future,
576 task::{Context, Poll, Waker},
577 };
578
579 let counters = DaemonCounters::new();
580 let wait = counters.observe_reader_wait(
581 || panic!("ready waits must not resolve diagnostic labels"),
582 "module_queue",
583 std::future::ready(7),
584 );
585 let mut wait = std::pin::pin!(wait);
586 let mut cx = Context::from_waker(Waker::noop());
587 assert_eq!(wait.as_mut().poll(&mut cx), Poll::Ready(7));
589 assert!(counters.reader_diagnostics.lock().unwrap().waits.is_empty());
590 }
591
592 #[tokio::test(start_paused = true)]
593 async fn reader_wait_labels_are_resolved_once_only_after_five_seconds() {
594 use std::{
595 cell::Cell,
596 future::{poll_fn, Future},
597 task::Poll,
598 };
599
600 let counters = DaemonCounters::new();
601 let calls = Cell::new(0);
602 let (sender, receiver) = tokio::sync::oneshot::channel();
603 let wait = counters.observe_reader_wait(
604 || {
605 calls.set(calls.get() + 1);
606 ("alpha".into(), "direct".into())
607 },
608 "module_queue",
609 receiver,
610 );
611 tokio::pin!(wait);
612 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
613 .await
614 .is_pending());
615 tokio::time::advance(Duration::from_secs(4)).await;
616 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
617 .await
618 .is_pending());
619 assert_eq!(calls.get(), 0);
620 tokio::time::advance(Duration::from_secs(1)).await;
621 assert!(poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx)))
622 .await
623 .is_pending());
624 assert_eq!(calls.get(), 1);
625 assert_eq!(
626 counters.snapshot()["reader_wait_over_5s_by_module_principal"]["alpha"]["direct"]
627 ["module_queue"],
628 1
629 );
630 sender.send(()).unwrap();
631 wait.await.unwrap();
632 assert_eq!(calls.get(), 1);
633 }
634
635 #[test]
636 fn counter_snapshot_includes_zero_rate_and_omits_empty_module_maps() {
637 let counters = DaemonCounters::new();
638 let snapshot = counters.snapshot();
639
640 assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 0);
641 assert_eq!(
642 snapshot["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
643 0
644 );
645 assert!(snapshot
646 .get("module_frames_dropped_no_route_by_module")
647 .is_none());
648 assert!(snapshot
649 .get("goodbye_relay_module_dropped_by_module")
650 .is_none());
651 }
652
653 #[tokio::test(start_paused = true)]
654 async fn module_frame_drops_are_attributed_to_the_emitting_module() {
655 let counters = DaemonCounters::new();
656 counters.increment_module_frames_dropped_no_route(Some("alpha"));
657 counters.increment_module_frames_dropped_no_route(Some("alpha"));
658
659 let snapshot = counters.snapshot();
660 assert_eq!(snapshot["module_frames_dropped_no_route"], 2);
661 assert_eq!(
662 snapshot["module_frames_dropped_no_route_by_module"],
663 json!({ "alpha": 2 })
664 );
665 assert_eq!(snapshot["module_frames_dropped_no_route_last_10m"], 2);
666 }
667
668 #[tokio::test(start_paused = true)]
669 async fn frame_drop_rate_ages_out_after_ten_minute_buckets() {
670 let counters = DaemonCounters::new();
671 counters.increment_module_frames_dropped_no_route(Some("alpha"));
672
673 tokio::time::advance(Duration::from_secs(9 * 60)).await;
674 assert_eq!(
675 counters.snapshot()["module_frames_dropped_no_route_last_10m"],
676 1
677 );
678
679 tokio::time::advance(Duration::from_secs(60)).await;
680 assert_eq!(
681 counters.snapshot()["module_frames_dropped_no_route_last_10m"],
682 0
683 );
684 }
685
686 #[tokio::test(start_paused = true)]
687 async fn frame_drop_window_counts_only_nonzero_minutes() {
688 let counters = DaemonCounters::new();
689 for minute in 0..10 {
690 if minute > 0 {
691 tokio::time::advance(Duration::from_secs(60)).await;
692 }
693 if minute != 4 {
694 counters.increment_module_frames_dropped_no_route(Some("alpha"));
695 }
696 }
697
698 assert_eq!(
699 counters.snapshot()["module_frames_dropped_no_route_nonzero_minutes_last_10m"],
700 9
701 );
702 }
703
704 #[test]
705 fn goodbye_relay_drops_are_attributed_to_the_target_module() {
706 let counters = DaemonCounters::new();
707 counters.increment_goodbye_relay_module_dropped(Some("alpha"));
708
709 assert_eq!(
710 counters.snapshot()["goodbye_relay_module_dropped_by_module"],
711 json!({ "alpha": 1 })
712 );
713 }
714}