use std::sync::OnceLock;
use liminal::durability::{DedupHeadCounts, dedup_head_counts};
use liminal::metrics::{
CounterHandle, GaugeHandle, MetricRegistrationError, MetricsRegistry, global_registry,
install_global_registry,
};
use crate::console::counts::ConsoleFailure;
use crate::server::connection::refusal::AdmissionRefusal;
use crate::server::connection::websocket::UpgradeRefusal;
use crate::server::mount::MountKind;
const CONNECTIONS_ACTIVE: &str = "liminal_connections_active";
const PUBLISHES_TOTAL: &str = "liminal_publishes_total";
const DELIVERIES_TOTAL: &str = "liminal_deliveries_total";
const TRANSPORT_DELIVERIES_TOTAL: &str = "liminal_transport_deliveries_total";
const SHEDS_TOTAL: &str = "liminal_subscription_sheds_total";
const SELF_STREAM_APPEND_FAILURES_TOTAL: &str = "liminal_self_stream_append_failures_total";
const SELF_STREAMS_RESERVED_PUBLICATIONS_TOTAL: &str =
"liminal_self_streams_reserved_publications_total";
const SELF_STREAMS_DISABLED_TOTAL: &str = "liminal_self_streams_disabled_total";
const CONSOLE_STREAMS_DISABLED_TOTAL: &str = "liminal_console_streams_disabled_total";
const CONSOLE_FAILURES_TOTAL: &str = "liminal_console_failures_total";
const KIND_LABEL: &str = "kind";
const SELF_STREAMS_UNMATCHED_PRESSURE_TOTAL: &str = "liminal_self_streams_unmatched_pressure_total";
const SELF_STREAMS_SUPPRESSED_PRESSURE_TOTAL: &str =
"liminal_self_streams_suppressed_pressure_total";
const SELF_STREAMS_PRESSURE_LEDGER_EVICTIONS_TOTAL: &str =
"liminal_self_streams_pressure_ledger_evictions_total";
const PRESSURE_REFUSED_TOTAL: &str = "liminal_pressure_refused_total";
const TRANSPORT_LABEL: &str = "transport";
const ADMISSION_REFUSALS_TOTAL: &str = "liminal_admission_refusals_total";
const HANDSHAKE_REFUSALS_TOTAL: &str = "liminal_handshake_refusals_total";
const REASON_LABEL: &str = "reason";
const RECEIPT_DISPLACEMENTS_TOTAL: &str = "liminal_receipt_displacements_total";
const RECEIPT_POOL_RUNAWAY_TOTAL: &str = "liminal_receipt_pool_runaway_total";
const CONNECTION_FATES_PARKED: &str = "liminal_connection_fates_parked";
const SCOPE_LABEL: &str = "scope";
const POOL_LABEL: &str = "pool";
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum ReceiptWindowScope {
LiveReceiptParticipant,
ProvenanceParticipant,
}
impl ReceiptWindowScope {
pub(crate) const LABELS: [&'static str; 2] =
["live_receipt_participant", "provenance_participant"];
const fn slot(self) -> usize {
match self {
Self::LiveReceiptParticipant => 0,
Self::ProvenanceParticipant => 1,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum SharedReceiptPool {
LiveReceiptServer,
ProvenanceServer,
ProvenanceConversation,
}
impl SharedReceiptPool {
pub(crate) const LABELS: [&'static str; 3] = [
"live_receipt_server",
"provenance_server",
"provenance_conversation",
];
const fn slot(self) -> usize {
match self {
Self::LiveReceiptServer => 0,
Self::ProvenanceServer => 1,
Self::ProvenanceConversation => 2,
}
}
pub(crate) const fn label(self) -> &'static str {
Self::LABELS[self.slot()]
}
}
static SERVER_METRICS: OnceLock<ServerMetrics> = OnceLock::new();
#[derive(Clone, Debug)]
struct ServerMetrics {
connections_active: GaugeHandle,
connection_fates_parked: GaugeHandle,
publishes_total: CounterHandle,
deliveries_total: CounterHandle,
transport_deliveries: [CounterHandle; MOUNT_KINDS.len()],
sheds_total: CounterHandle,
self_stream_append_failures_total: CounterHandle,
self_streams_disabled_total: CounterHandle,
console_streams_disabled_total: CounterHandle,
console_failures: [CounterHandle; ConsoleFailure::LABELS.len()],
self_streams_reserved_publications_total: CounterHandle,
self_streams_unmatched_pressure_total: CounterHandle,
self_streams_suppressed_pressure_total: CounterHandle,
self_streams_pressure_ledger_evictions_total: CounterHandle,
pressure_refused_total: CounterHandle,
admission_refusals: [CounterHandle; AdmissionRefusal::LABELS.len()],
handshake_refusals: [CounterHandle; UpgradeRefusal::LABELS.len()],
receipt_displacements: [CounterHandle; ReceiptWindowScope::LABELS.len()],
receipt_pool_runaway: [CounterHandle; SharedReceiptPool::LABELS.len()],
}
const MOUNT_KINDS: [MountKind; 3] = [MountKind::Tcp, MountKind::WebSocket, MountKind::Loopback];
const fn transport_slot(transport: MountKind) -> usize {
match transport {
MountKind::Tcp => 0,
MountKind::WebSocket => 1,
MountKind::Loopback => 2,
}
}
pub fn init() {
if SERVER_METRICS.get().is_some() {
return;
}
let Some(registry) = global_or_install() else {
tracing::warn!(
"global metrics registry could not be installed or read back; server metrics stay disabled"
);
return;
};
if let Some(metrics) = ServerMetrics::register(registry) {
let _ = SERVER_METRICS.set(metrics);
}
}
pub fn connection_spawned() {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.connections_active.increment();
}
}
pub fn connection_closed() {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.connections_active.decrement();
}
}
pub(crate) fn connection_fate_parked() {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.connection_fates_parked.increment();
}
}
pub(crate) fn connection_fate_unparked() {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.connection_fates_parked.decrement();
}
}
pub fn publish_accepted() {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.publishes_total.increment();
}
}
pub fn deliveries_recorded(count: u64) {
if count == 0 {
return;
}
if let Some(metrics) = SERVER_METRICS.get() {
metrics.deliveries_total.increment_by(count);
}
}
pub fn transport_deliveries_recorded(transport: MountKind, count: u64) {
if count == 0 {
return;
}
if let Some(metrics) = SERVER_METRICS.get() {
metrics.transport_deliveries[transport_slot(transport)].increment_by(count);
}
}
pub(crate) fn admission_refused(refusal: AdmissionRefusal) {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.admission_refusals[refusal.slot()].increment();
}
}
pub(crate) fn handshake_refused(refusal: &UpgradeRefusal) {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.handshake_refusals[refusal.slot()].increment();
}
}
pub(crate) fn self_stream_append_failed() {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.self_stream_append_failures_total.increment();
}
}
pub(crate) fn self_stream_reserved_publication() {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.self_streams_reserved_publications_total.increment();
}
}
pub(crate) fn self_streams_disabled() {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.self_streams_disabled_total.increment();
}
}
pub(crate) fn console_failure(kind: ConsoleFailure, amount: u64) {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.console_failures[kind.slot()].increment_by(amount);
}
}
pub(crate) fn console_streams_disabled() {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.console_streams_disabled_total.increment();
}
}
pub(crate) fn self_stream_pressure_unmatched() {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.self_streams_unmatched_pressure_total.increment();
}
}
pub(crate) fn self_stream_pressure_suppressed() {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.self_streams_suppressed_pressure_total.increment();
}
}
pub(crate) fn pressure_refused() {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.pressure_refused_total.increment();
}
}
pub(crate) fn pressure_ledger_evicted() {
if let Some(metrics) = SERVER_METRICS.get() {
metrics
.self_streams_pressure_ledger_evictions_total
.increment();
}
}
pub fn subscription_shed() {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.sheds_total.increment();
}
}
pub(crate) fn receipt_entries_displaced(scope: ReceiptWindowScope, count: u64) {
if count == 0 {
return;
}
if let Some(metrics) = SERVER_METRICS.get() {
metrics.receipt_displacements[scope.slot()].increment_by(count);
}
}
pub(crate) fn receipt_pool_runaway_observed(pool: SharedReceiptPool) {
if let Some(metrics) = SERVER_METRICS.get() {
metrics.receipt_pool_runaway[pool.slot()].increment();
}
}
pub(crate) fn render_dedup_head_counters(output: &mut String) {
render_dedup_head_block(output, dedup_head_counts());
}
fn render_dedup_head_block(output: &mut String, counts: DedupHeadCounts) {
for (name, value) in counts.named() {
let value = value.to_string();
for piece in [
"# HELP ",
name,
" ",
name,
"\n# TYPE ",
name,
" counter\n",
name,
" ",
&value,
"\n",
] {
output.push_str(piece);
}
}
}
#[cfg(test)]
pub(crate) fn publishes_total_value() -> Option<u64> {
use liminal::metrics::MetricValue;
let registry = global_registry()?;
registry
.snapshot()
.metrics()
.iter()
.find(|metric| metric.name == PUBLISHES_TOTAL)
.and_then(|metric| match metric.value {
MetricValue::Counter(value) => Some(value),
MetricValue::Gauge(_) | MetricValue::Histogram(_) => None,
})
}
#[cfg(test)]
pub(crate) fn receipt_displacements_value(scope: ReceiptWindowScope) -> Option<u64> {
labelled_counter_value(
RECEIPT_DISPLACEMENTS_TOTAL,
SCOPE_LABEL,
ReceiptWindowScope::LABELS[scope.slot()],
)
}
#[cfg(test)]
pub(crate) fn receipt_pool_runaway_value(pool: SharedReceiptPool) -> Option<u64> {
labelled_counter_value(RECEIPT_POOL_RUNAWAY_TOTAL, POOL_LABEL, pool.label())
}
#[cfg(test)]
pub(crate) fn connection_fates_parked_value() -> Option<i64> {
use liminal::metrics::MetricValue;
let registry = global_registry()?;
registry
.snapshot()
.metrics()
.iter()
.find(|metric| metric.name == CONNECTION_FATES_PARKED)
.and_then(|metric| match metric.value {
MetricValue::Gauge(value) => Some(value),
MetricValue::Counter(_) | MetricValue::Histogram(_) => None,
})
}
#[cfg(test)]
fn labelled_counter_value(name: &str, key: &str, label: &str) -> Option<u64> {
use liminal::metrics::MetricValue;
let registry = global_registry()?;
registry
.snapshot()
.metrics()
.iter()
.find(|metric| {
metric.name == name
&& metric
.labels
.iter()
.any(|(metric_key, value)| metric_key == key && value == label)
})
.and_then(|metric| match metric.value {
MetricValue::Counter(value) => Some(value),
MetricValue::Gauge(_) | MetricValue::Histogram(_) => None,
})
}
impl ServerMetrics {
fn register(registry: &MetricsRegistry) -> Option<Self> {
Some(Self {
connections_active: register_gauge(registry, CONNECTIONS_ACTIVE)?,
connection_fates_parked: register_gauge(registry, CONNECTION_FATES_PARKED)?,
publishes_total: register_counter(registry, PUBLISHES_TOTAL)?,
deliveries_total: register_counter(registry, DELIVERIES_TOTAL)?,
transport_deliveries: register_transport_deliveries(registry)?,
sheds_total: register_counter(registry, SHEDS_TOTAL)?,
self_stream_append_failures_total: register_counter(
registry,
SELF_STREAM_APPEND_FAILURES_TOTAL,
)?,
self_streams_disabled_total: register_counter(registry, SELF_STREAMS_DISABLED_TOTAL)?,
console_streams_disabled_total: register_counter(
registry,
CONSOLE_STREAMS_DISABLED_TOTAL,
)?,
console_failures: register_labelled(
registry,
CONSOLE_FAILURES_TOTAL,
KIND_LABEL,
&ConsoleFailure::LABELS,
)?,
self_streams_reserved_publications_total: register_counter(
registry,
SELF_STREAMS_RESERVED_PUBLICATIONS_TOTAL,
)?,
self_streams_unmatched_pressure_total: register_counter(
registry,
SELF_STREAMS_UNMATCHED_PRESSURE_TOTAL,
)?,
self_streams_suppressed_pressure_total: register_counter(
registry,
SELF_STREAMS_SUPPRESSED_PRESSURE_TOTAL,
)?,
self_streams_pressure_ledger_evictions_total: register_counter(
registry,
SELF_STREAMS_PRESSURE_LEDGER_EVICTIONS_TOTAL,
)?,
pressure_refused_total: register_counter(registry, PRESSURE_REFUSED_TOTAL)?,
admission_refusals: register_labelled(
registry,
ADMISSION_REFUSALS_TOTAL,
REASON_LABEL,
&AdmissionRefusal::LABELS,
)?,
handshake_refusals: register_labelled(
registry,
HANDSHAKE_REFUSALS_TOTAL,
REASON_LABEL,
&UpgradeRefusal::LABELS,
)?,
receipt_displacements: register_labelled(
registry,
RECEIPT_DISPLACEMENTS_TOTAL,
SCOPE_LABEL,
&ReceiptWindowScope::LABELS,
)?,
receipt_pool_runaway: register_labelled(
registry,
RECEIPT_POOL_RUNAWAY_TOTAL,
POOL_LABEL,
&SharedReceiptPool::LABELS,
)?,
})
}
}
fn register_counter(registry: &MetricsRegistry, name: &'static str) -> Option<CounterHandle> {
registry
.register_counter(name, no_labels())
.inspect_err(warn_registration_failure)
.ok()
}
fn register_gauge(registry: &MetricsRegistry, name: &'static str) -> Option<GaugeHandle> {
registry
.register_gauge(name, no_labels())
.inspect_err(warn_registration_failure)
.ok()
}
fn register_transport_deliveries(
registry: &MetricsRegistry,
) -> Option<[CounterHandle; MOUNT_KINDS.len()]> {
let mut transport_deliveries = Vec::with_capacity(MOUNT_KINDS.len());
for transport in MOUNT_KINDS {
transport_deliveries.push(
registry
.register_counter(
TRANSPORT_DELIVERIES_TOTAL,
[(TRANSPORT_LABEL, transport.as_str())],
)
.inspect_err(warn_registration_failure)
.ok()?,
);
}
transport_deliveries.try_into().ok()
}
fn register_labelled<const N: usize>(
registry: &MetricsRegistry,
name: &'static str,
key: &'static str,
labels: &[&'static str; N],
) -> Option<[CounterHandle; N]> {
let mut handles = Vec::with_capacity(N);
for label in labels {
handles.push(
registry
.register_counter(name, [(key, *label)])
.inspect_err(warn_registration_failure)
.ok()?,
);
}
handles.try_into().ok()
}
fn warn_registration_failure(error: &MetricRegistrationError) {
tracing::warn!(
%error,
"server metrics family registration failed; metrics recording stays disabled"
);
}
const fn no_labels() -> std::iter::Empty<(&'static str, &'static str)> {
std::iter::empty()
}
fn global_or_install() -> Option<&'static MetricsRegistry> {
if let Some(registry) = global_registry() {
return Some(registry);
}
let _ = install_global_registry(MetricsRegistry::new());
global_registry()
}
#[cfg(test)]
mod tests {
use super::{
CONNECTIONS_ACTIVE, DELIVERIES_TOTAL, PUBLISHES_TOTAL, ServerMetrics, connection_spawned,
deliveries_recorded, init, no_labels, publish_accepted, render_dedup_head_block,
};
use liminal::durability::DedupHeadCounts;
use liminal::metrics::{MetricsRegistry, global_registry, render};
#[test]
fn the_dedup_head_block_renders_byte_for_byte() {
let mut output = String::new();
render_dedup_head_block(
&mut output,
DedupHeadCounts {
refusals: 7,
record_lost_races: 0,
repairs: 12,
},
);
assert_eq!(
output,
"# HELP liminal_dedup_head_refusals_total liminal_dedup_head_refusals_total\n\
# TYPE liminal_dedup_head_refusals_total counter\n\
liminal_dedup_head_refusals_total 7\n\
# HELP liminal_dedup_head_record_lost_races_total \
liminal_dedup_head_record_lost_races_total\n\
# TYPE liminal_dedup_head_record_lost_races_total counter\n\
liminal_dedup_head_record_lost_races_total 0\n\
# HELP liminal_dedup_head_repairs_total liminal_dedup_head_repairs_total\n\
# TYPE liminal_dedup_head_repairs_total counter\n\
liminal_dedup_head_repairs_total 12\n"
);
}
#[test]
fn init_registers_the_three_server_families_on_the_global_registry()
-> Result<(), Box<dyn std::error::Error>> {
init();
connection_spawned();
publish_accepted();
deliveries_recorded(2);
let registry =
global_registry().ok_or("init must install and enable the global registry")?;
let exposition = render(®istry.snapshot());
assert!(exposition.contains(CONNECTIONS_ACTIVE));
assert!(exposition.contains(PUBLISHES_TOTAL));
assert!(exposition.contains(DELIVERIES_TOTAL));
Ok(())
}
#[test]
fn register_warns_when_a_family_registration_fails() -> Result<(), Box<dyn std::error::Error>> {
let registry = MetricsRegistry::new();
registry.register_counter(CONNECTIONS_ACTIVE, no_labels())?;
let log = crate::test_log::CapturedLog::default();
let metrics = log.capture(|| ServerMetrics::register(®istry));
assert!(
metrics.is_none(),
"a failed registration must still yield None"
);
let output = log.contents();
assert!(
output.contains("WARN") && output.contains(CONNECTIONS_ACTIVE),
"expected a warn carrying the registration error, got: {output}"
);
Ok(())
}
}