liminal_server/console/
counts.rs

1//! The console's counters.
2//!
3//! Relaxed atomics, each moved where its event happens and read at one moment
4//! as [`ConsoleCountsSnapshot`]. Every console bookkeeping failure is one of
5//! these, never a log line alone.
6
7use std::sync::atomic::{AtomicU64, Ordering};
8
9/// The live counters.
10#[derive(Debug, Default)]
11pub struct ConsoleCounts {
12    pub(super) topology_emit_failures: AtomicU64,
13    pub(super) topology_clock_failures: AtomicU64,
14    pub(super) station_collisions: AtomicU64,
15    pub(super) tap_observe_failures: AtomicU64,
16    pub(super) tap_attach_failures: AtomicU64,
17    pub(super) tap_detach_failures: AtomicU64,
18    pub(super) console_wake_failures: AtomicU64,
19    pub(super) console_publish_failures: AtomicU64,
20    pub(super) console_records_lost: AtomicU64,
21    pub(super) tap_records_abandoned: AtomicU64,
22    pub(super) tap_messages_abandoned: AtomicU64,
23    pub(super) console_pending_records: AtomicU64,
24    pub(super) topology_records_published: AtomicU64,
25    pub(super) tap_records_published: AtomicU64,
26    pub(super) console_pump_wakes: AtomicU64,
27    pub(super) taps_attached: AtomicU64,
28    pub(super) tap_suppressed_total: AtomicU64,
29}
30
31/// Every console counter, read at one moment.
32#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
33pub struct ConsoleCountsSnapshot {
34    /// Topology changes the hub could not apply (a poisoned model lock).
35    pub topology_emit_failures: u64,
36    /// Topology documents sent with this device's start missing: the wall
37    /// clock gave no time for it, and the fault is named on the record.
38    pub topology_clock_failures: u64,
39    /// Worker identities or service subjects left off the map because they
40    /// are this server's exchange or depot id: kept apart, never merged
41    /// into those stations (Buttercup's V1, Hermes 04:27).
42    pub station_collisions: u64,
43    /// Tap observe failures: a publish the tap could not observe (a
44    /// poisoned tap lock), a station list refused, or a hop that could not
45    /// be kept as measured; each failure of one observe counts.
46    pub tap_observe_failures: u64,
47    /// Tap subscriptions that could not start.
48    pub tap_attach_failures: u64,
49    /// Tap subscriptions that could not stop.
50    pub tap_detach_failures: u64,
51    /// Wakes that reached no publisher (a poisoned notifier slot).
52    pub console_wake_failures: u64,
53    /// Drops of a tap subscription that could not leave the subscriber list.
54    pub tap_unsubscribe_failures: u64,
55    /// Console records whose publication failed (each kept and retried).
56    pub console_publish_failures: u64,
57    /// Drained console records thrown away. The ring is the only buffer and a
58    /// tap with a pending batch is not drained, so this stays 0: a control
59    /// against a second buffer creeping back in.
60    pub console_records_lost: u64,
61    /// Encoded tap records left unsent when their tap went: its last
62    /// subscriber left, or it was attached anew before they were sent.
63    pub tap_records_abandoned: u64,
64    /// Messages still in a tap's ring when its last subscriber left.
65    pub tap_messages_abandoned: u64,
66    /// Console records held for retry after a failed publication.
67    pub console_pending_records: u64,
68    /// `console.topology` records published.
69    pub topology_records_published: u64,
70    /// `console.tap.<target>` records published.
71    pub tap_records_published: u64,
72    /// Times the console publisher was woken and ran.
73    pub console_pump_wakes: u64,
74    /// Taps live now.
75    pub taps_attached: u64,
76    /// Messages evicted from every tap's ring since start.
77    pub tap_dropped_total: u64,
78    /// Dedup-suppressed re-publishes, which are not tapped.
79    pub tap_suppressed_total: u64,
80}
81
82/// A console bookkeeping failure, by name (SRV-012 R8). Each is a counter
83/// here and the `kind` of `liminal_console_failures_total`.
84#[derive(Clone, Copy, Debug, PartialEq, Eq)]
85pub enum ConsoleFailure {
86    /// A topology change the model could not take.
87    TopologyEmit,
88    /// A publish the tap could not observe.
89    TapObserve,
90    /// A tap subscription that could not start.
91    TapAttach,
92    /// A tap subscription that could not stop.
93    TapDetach,
94    /// A wake that reached no publisher.
95    ConsoleWake,
96    /// A dropped tap subscription that could not leave the list.
97    TapUnsubscribe,
98    /// A console record whose publication failed.
99    ConsolePublish,
100    /// A drained console record thrown away.
101    ConsoleRecordsLost,
102    /// A topology document whose start the wall clock could not give.
103    TopologyClock,
104    /// A worker identity or service subject equal to this server's exchange
105    /// or depot id, left off the map.
106    StationCollision,
107}
108
109impl ConsoleFailure {
110    /// The metric label values, in [`Self::slot`] order.
111    pub const LABELS: [&'static str; 10] = [
112        "topology_emit",
113        "tap_observe",
114        "tap_attach",
115        "tap_detach",
116        "console_wake",
117        "tap_unsubscribe",
118        "console_publish",
119        "console_records_lost",
120        "topology_clock",
121        "station_collision",
122    ];
123
124    /// This failure's index into [`Self::LABELS`].
125    #[must_use]
126    pub const fn slot(self) -> usize {
127        match self {
128            Self::TopologyEmit => 0,
129            Self::TapObserve => 1,
130            Self::TapAttach => 2,
131            Self::TapDetach => 3,
132            Self::ConsoleWake => 4,
133            Self::TapUnsubscribe => 5,
134            Self::ConsolePublish => 6,
135            Self::ConsoleRecordsLost => 7,
136            Self::TopologyClock => 8,
137            Self::StationCollision => 9,
138        }
139    }
140}
141
142/// Counts `amount` failures of `kind` in `counter` and in its metric.
143pub(crate) fn failed(counter: &AtomicU64, amount: u64, kind: ConsoleFailure) {
144    add(counter, amount);
145    crate::metrics::console_failure(kind, amount);
146}
147
148pub(super) fn add(counter: &AtomicU64, amount: u64) {
149    counter.fetch_add(amount, Ordering::Relaxed);
150}
151
152pub(super) fn set(counter: &AtomicU64, value: u64) {
153    counter.store(value, Ordering::Relaxed);
154}
155
156impl ConsoleCounts {
157    /// Every counter as it stands. `tap_dropped_total` and
158    /// `tap_unsubscribe_failures` are the tap's own, passed in.
159    #[must_use]
160    pub fn snapshot(
161        &self,
162        tap_dropped_total: u64,
163        tap_unsubscribe_failures: u64,
164    ) -> ConsoleCountsSnapshot {
165        let read = |counter: &AtomicU64| counter.load(Ordering::Relaxed);
166        ConsoleCountsSnapshot {
167            topology_emit_failures: read(&self.topology_emit_failures),
168            topology_clock_failures: read(&self.topology_clock_failures),
169            station_collisions: read(&self.station_collisions),
170            tap_observe_failures: read(&self.tap_observe_failures),
171            tap_attach_failures: read(&self.tap_attach_failures),
172            tap_detach_failures: read(&self.tap_detach_failures),
173            console_wake_failures: read(&self.console_wake_failures),
174            tap_unsubscribe_failures,
175            console_publish_failures: read(&self.console_publish_failures),
176            console_records_lost: read(&self.console_records_lost),
177            tap_records_abandoned: read(&self.tap_records_abandoned),
178            tap_messages_abandoned: read(&self.tap_messages_abandoned),
179            console_pending_records: read(&self.console_pending_records),
180            topology_records_published: read(&self.topology_records_published),
181            tap_records_published: read(&self.tap_records_published),
182            console_pump_wakes: read(&self.console_pump_wakes),
183            taps_attached: read(&self.taps_attached),
184            tap_dropped_total,
185            tap_suppressed_total: read(&self.tap_suppressed_total),
186        }
187    }
188}
liminal_server/console/counts.rs

liminal_server/console/
counts.rs
