use std::collections::BTreeMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Mutex;
use std::time::Duration;
const DURATION_BUCKETS: [f64; 13] = [
0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0,
];
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub(crate) enum UnloadReason {
Idle,
Requested,
Crashed,
Oom,
Evicted,
Reloaded,
}
impl UnloadReason {
fn as_str(self) -> &'static str {
match self {
Self::Idle => "idle",
Self::Requested => "requested",
Self::Crashed => "crashed",
Self::Oom => "oom",
Self::Evicted => "evicted",
Self::Reloaded => "reloaded",
}
}
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub(crate) enum OomRetry {
EvictOthers,
SplitMode,
CtxShrink,
}
impl OomRetry {
fn as_str(self) -> &'static str {
match self {
Self::EvictOthers => "evict_others",
Self::SplitMode => "split_mode",
Self::CtxShrink => "ctx_shrink",
}
}
}
#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Debug)]
struct Labels(String);
impl Labels {
fn new(pairs: &[(&str, &str)]) -> Self {
let mut out = String::new();
for (name, value) in pairs {
if !out.is_empty() {
out.push(',');
}
out.push_str(name);
out.push_str("=\"");
out.push_str(&escape_label_value(value));
out.push('"');
}
Self(out)
}
fn braces(&self) -> String {
if self.0.is_empty() {
String::new()
} else {
format!("{{{}}}", self.0)
}
}
fn with_le(&self, upper: &str) -> String {
if self.0.is_empty() {
format!("{{le=\"{upper}\"}}")
} else {
format!("{{{},le=\"{upper}\"}}", self.0)
}
}
}
type Key = (&'static str, Labels);
#[derive(Clone, Default, PartialEq, Debug)]
struct Histogram {
buckets: [u64; DURATION_BUCKETS.len()],
sum_seconds: f64,
count: u64,
}
impl Histogram {
fn observe(&mut self, elapsed: Duration) {
let seconds = elapsed.as_secs_f64();
self.sum_seconds += seconds;
self.count += 1;
for (bucket, upper) in self.buckets.iter_mut().zip(DURATION_BUCKETS) {
if seconds <= upper {
*bucket += 1;
}
}
}
}
#[derive(Default)]
struct Store {
counters: BTreeMap<Key, u64>,
histograms: BTreeMap<Key, Histogram>,
}
impl Store {
const fn new() -> Self {
Self {
counters: BTreeMap::new(),
histograms: BTreeMap::new(),
}
}
}
static REGISTRY: Mutex<Store> = Mutex::new(Store::new());
static PROCESS_START_SECONDS: AtomicU64 = AtomicU64::new(0);
pub(crate) fn mark_process_start() {
let secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
PROCESS_START_SECONDS.store(secs, Ordering::Relaxed);
}
pub(crate) fn process_start_seconds() -> u64 {
PROCESS_START_SECONDS.load(Ordering::Relaxed)
}
#[cfg(test)]
pub(crate) static GLOBAL_COUNTER_TEST_LOCK: tokio::sync::Mutex<()> =
tokio::sync::Mutex::const_new(());
fn with_store<R>(store: &Mutex<Store>, f: impl FnOnce(&mut Store) -> R) -> R {
let mut guard = match store.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
f(&mut guard)
}
fn incr_into(store: &Mutex<Store>, name: &'static str, labels: Labels) {
with_store(store, |s| {
*s.counters.entry((name, labels)).or_insert(0) += 1
});
}
fn observe_into(store: &Mutex<Store>, name: &'static str, labels: Labels, elapsed: Duration) {
with_store(store, |s| {
s.histograms
.entry((name, labels))
.or_default()
.observe(elapsed)
});
}
fn record_request_into(store: &Mutex<Store>, route: &str, status: u16, ttfb: Duration) {
incr_into(
store,
"llmman_http_requests_total",
Labels::new(&[("route", route), ("status", &status.to_string())]),
);
observe_into(
store,
"llmman_http_request_ttfb_seconds",
Labels::new(&[("route", route)]),
ttfb,
);
}
pub(crate) fn record_request(route: &str, status: u16, ttfb: Duration) {
record_request_into(®ISTRY, route, status, ttfb);
}
fn record_response_body_into(store: &Mutex<Store>, route: &str, elapsed: Duration) {
observe_into(
store,
"llmman_http_request_duration_seconds",
Labels::new(&[("route", route)]),
elapsed,
);
}
pub(crate) fn record_response_body(route: &str, elapsed: Duration) {
record_response_body_into(®ISTRY, route, elapsed);
}
fn record_model_load_into(store: &Mutex<Store>, model: &str, elapsed: Duration) {
let labels = Labels::new(&[("model", model)]);
incr_into(store, "llmman_model_loads_total", labels.clone());
observe_into(store, "llmman_model_load_duration_seconds", labels, elapsed);
}
pub(crate) fn record_model_load(model: &str, elapsed: Duration) {
record_model_load_into(®ISTRY, model, elapsed);
}
fn record_model_unload_into(store: &Mutex<Store>, model: &str, reason: UnloadReason) {
incr_into(
store,
"llmman_model_unloads_total",
Labels::new(&[("model", model), ("reason", reason.as_str())]),
);
}
pub(crate) fn record_model_unload(model: &str, reason: UnloadReason) {
record_model_unload_into(®ISTRY, model, reason);
}
fn record_oom_retry_into(store: &Mutex<Store>, model: &str, strategy: OomRetry) {
incr_into(
store,
"llmman_model_load_oom_retries_total",
Labels::new(&[("model", model), ("strategy", strategy.as_str())]),
);
}
pub(crate) fn record_oom_retry(model: &str, strategy: OomRetry) {
record_oom_retry_into(®ISTRY, model, strategy);
}
fn record_scheduling_rejection_into(store: &Mutex<Store>) {
incr_into(
store,
"llmman_scheduling_rejections_total",
Labels::default(),
);
}
pub(crate) fn record_scheduling_rejection() {
record_scheduling_rejection_into(®ISTRY);
}
pub(crate) struct ModelState {
pub(crate) model: String,
pub(crate) engine: &'static str,
pub(crate) up: bool,
pub(crate) requests_in_flight: u64,
pub(crate) requests_queued: Option<u64>,
}
pub(crate) struct Snapshot {
pub(crate) version: String,
pub(crate) start_time_seconds: u64,
pub(crate) scheduling_requests_in_flight: u64,
pub(crate) scheduling_capacity: u64,
pub(crate) models_loaded: u64,
pub(crate) models_loading: u64,
pub(crate) models: Vec<ModelState>,
}
fn escape_label_value(value: &str) -> String {
let mut escaped = String::with_capacity(value.len());
for c in value.chars() {
match c {
'\\' => escaped.push_str("\\\\"),
'"' => escaped.push_str("\\\""),
'\n' => escaped.push_str("\\n"),
'\r' => {}
_ => escaped.push(c),
}
}
escaped
}
fn header(out: &mut String, name: &str, kind: &str, help: &str) {
out.push_str(&format!("# HELP {name} {help}\n# TYPE {name} {kind}\n"));
}
fn gauge(out: &mut String, name: &str, help: &str, labels: &Labels, value: impl std::fmt::Display) {
header(out, name, "gauge", help);
out.push_str(&format!("{name}{} {value}\n", labels.braces()));
}
fn model_gauge(
out: &mut String,
name: &str,
help: &str,
models: &[ModelState],
series: impl Fn(&ModelState) -> Option<(Labels, u64)>,
) {
header(out, name, "gauge", help);
let samples: BTreeMap<Labels, u64> = models.iter().filter_map(series).collect();
for (labels, value) in &samples {
out.push_str(&format!("{name}{} {value}\n", labels.braces()));
}
}
fn counter(out: &mut String, store: &Store, name: &str, help: &str) {
header(out, name, "counter", help);
for ((_, labels), value) in store.counters.iter().filter(|((f, _), _)| *f == name) {
out.push_str(&format!("{name}{} {value}\n", labels.braces()));
}
}
fn counter_unlabelled(out: &mut String, store: &Store, name: &'static str, help: &str) {
header(out, name, "counter", help);
let value = store
.counters
.get(&(name, Labels::default()))
.copied()
.unwrap_or(0);
out.push_str(&format!("{name} {value}\n"));
}
fn histogram(out: &mut String, store: &Store, name: &str, help: &str) {
header(out, name, "histogram", help);
for ((_, labels), hist) in store.histograms.iter().filter(|((f, _), _)| *f == name) {
for (count, upper) in hist.buckets.iter().zip(DURATION_BUCKETS) {
out.push_str(&format!(
"{name}_bucket{} {count}\n",
labels.with_le(&upper.to_string())
));
}
out.push_str(&format!(
"{name}_bucket{} {}\n",
labels.with_le("+Inf"),
hist.count
));
out.push_str(&format!(
"{name}_sum{} {}\n",
labels.braces(),
hist.sum_seconds
));
out.push_str(&format!("{name}_count{} {}\n", labels.braces(), hist.count));
}
}
fn render_from(store: &Store, snapshot: &Snapshot) -> String {
let mut out = String::new();
let none = Labels::default();
gauge(
&mut out,
"llmman_build_info",
"Build information for this llmman daemon.",
&Labels::new(&[("version", &snapshot.version)]),
1,
);
gauge(
&mut out,
"llmman_start_time_seconds",
"Unix time this daemon started, for uptime and restart detection.",
&none,
snapshot.start_time_seconds,
);
gauge(
&mut out,
"llmman_scheduling_requests_in_flight",
"Requests doing model-scheduling work; a request for an already-loaded model bypasses this.",
&none,
snapshot.scheduling_requests_in_flight,
);
gauge(
&mut out,
"llmman_scheduling_capacity",
"Admission limit for model-scheduling work (LLMMAN_MAX_QUEUE); not a cap on total concurrency.",
&none,
snapshot.scheduling_capacity,
);
counter_unlabelled(
&mut out,
store,
"llmman_scheduling_rejections_total",
"Requests rejected with 503 because LLMMAN_MAX_QUEUE was already full.",
);
gauge(
&mut out,
"llmman_models_loaded",
"Models llmman currently has loaded, the same set /api/ps reports.",
&none,
snapshot.models_loaded,
);
gauge(
&mut out,
"llmman_models_loading",
"Loads in flight: admitted, not yet loaded. Added to llmman_models_loaded this is what LLMMAN_MAX_LOADED_MODELS caps.",
&none,
snapshot.models_loading,
);
model_gauge(
&mut out,
"llmman_model_up",
"1 while a loaded model's backend process is alive, 0 once it has exited and llmman has not yet noticed.",
&snapshot.models,
|m| Some((Labels::new(&[("model", &m.model), ("engine", m.engine)]), u64::from(m.up))),
);
model_gauge(
&mut out,
"llmman_requests_in_flight",
"Requests a loaded model is serving or holding, including responses still being sent.",
&snapshot.models,
|m| Some((Labels::new(&[("model", &m.model)]), m.requests_in_flight)),
);
model_gauge(
&mut out,
"llmman_requests_queued",
"Estimated requests waiting for a free parallel slot; absent where llmman does not know the slot count.",
&snapshot.models,
|m| {
m.requests_queued
.map(|q| (Labels::new(&[("model", &m.model)]), q))
},
);
counter(
&mut out,
store,
"llmman_model_loads_total",
"Models loaded since this daemon started.",
);
histogram(
&mut out,
store,
"llmman_model_load_duration_seconds",
"Cold start: from llmman finding a model unloaded to it answering, including any queue wait, pull and eviction.",
);
counter(
&mut out,
store,
"llmman_model_load_oom_retries_total",
"Load attempts retried after an out-of-memory failure, by which fallback was used.",
);
counter(
&mut out,
store,
"llmman_model_unloads_total",
"Models removed from the loaded set since this daemon started, by cause.",
);
counter(
&mut out,
store,
"llmman_http_requests_total",
"Responses served, by matched route and status code.",
);
histogram(
&mut out,
store,
"llmman_http_request_ttfb_seconds",
"Time from llmman receiving a request to it sending that response's headers.",
);
histogram(
&mut out,
store,
"llmman_http_request_duration_seconds",
"Time from llmman receiving a request to it finishing that response's body, streaming included.",
);
out
}
pub(crate) fn render(snapshot: &Snapshot) -> String {
let store = with_store(®ISTRY, |s| Store {
counters: s.counters.clone(),
histograms: s.histograms.clone(),
});
render_from(&store, snapshot)
}
#[cfg(test)]
mod tests {
use super::*;
fn store() -> Mutex<Store> {
Mutex::new(Store::new())
}
fn rendered(store: &Mutex<Store>, snapshot: &Snapshot) -> String {
with_store(store, |s| render_from(s, snapshot))
}
fn snapshot() -> Snapshot {
Snapshot {
version: "0.1.0".into(),
start_time_seconds: 0,
scheduling_requests_in_flight: 0,
scheduling_capacity: 512,
models_loaded: 0,
models_loading: 0,
models: Vec::new(),
}
}
fn value_of(rendered: &str, prefix: &str) -> u64 {
rendered
.lines()
.find_map(|l| l.strip_prefix(prefix))
.and_then(|rest| rest.trim().parse().ok())
.unwrap_or(0)
}
#[test]
fn the_emitted_metric_names_are_a_fixed_set() {
let rendered = rendered(&store(), &snapshot());
let names: Vec<&str> = rendered
.lines()
.filter_map(|l| l.strip_prefix("# TYPE "))
.filter_map(|l| l.split_whitespace().next())
.collect();
assert_eq!(
names,
[
"llmman_build_info",
"llmman_start_time_seconds",
"llmman_scheduling_requests_in_flight",
"llmman_scheduling_capacity",
"llmman_scheduling_rejections_total",
"llmman_models_loaded",
"llmman_models_loading",
"llmman_model_up",
"llmman_requests_in_flight",
"llmman_requests_queued",
"llmman_model_loads_total",
"llmman_model_load_duration_seconds",
"llmman_model_load_oom_retries_total",
"llmman_model_unloads_total",
"llmman_http_requests_total",
"llmman_http_request_ttfb_seconds",
"llmman_http_request_duration_seconds",
]
);
}
#[test]
fn rendering_the_same_data_twice_is_byte_identical() {
let store = store();
for route in ["/api/chat", "/v1/models", "/api/tags", "/metrics"] {
for status in [200u16, 500, 404] {
record_request_into(&store, route, status, Duration::from_millis(7));
}
}
record_model_load_into(&store, "b:latest", Duration::from_secs(3));
record_model_unload_into(&store, "b:latest", UnloadReason::Idle);
let first = rendered(&store, &snapshot());
let second = rendered(&store, &snapshot());
assert_eq!(first, second);
let chat = first.find("route=\"/api/chat\"").expect(&first);
let tags = first.find("route=\"/api/tags\"").expect(&first);
assert!(tags > chat, "{first}");
let models = first.find("route=\"/v1/models\"").expect(&first);
assert!(models > tags, "{first}");
}
#[test]
fn buckets_are_cumulative_and_their_bounds_are_inclusive() {
let store = store();
record_request_into(&store, "/api/chat", 200, Duration::from_millis(250));
record_request_into(&store, "/api/chat", 200, Duration::from_secs(45));
let rendered = rendered(&store, &snapshot());
for (le, expected) in [
("0.1", 0), ("0.25", 1), ("0.5", 1),
("30", 1),
("60", 2), ("600", 2),
("+Inf", 2),
] {
assert!(
rendered.contains(&format!(
"llmman_http_request_ttfb_seconds_bucket{{route=\"/api/chat\",le=\"{le}\"}} {expected}\n"
)),
"le={le} should be {expected} in:\n{rendered}"
);
}
assert!(
rendered.contains("llmman_http_request_ttfb_seconds_count{route=\"/api/chat\"} 2\n"),
"{rendered}"
);
}
#[test]
fn the_histogram_sum_is_the_total_observed_seconds() {
let store = store();
record_model_load_into(&store, "a:latest", Duration::from_millis(250));
record_model_load_into(&store, "a:latest", Duration::from_millis(750));
let rendered = rendered(&store, &snapshot());
assert!(
rendered.contains("llmman_model_load_duration_seconds_sum{model=\"a:latest\"} 1\n"),
"{rendered}"
);
assert!(
rendered.contains("llmman_model_load_duration_seconds_count{model=\"a:latest\"} 2\n"),
"{rendered}"
);
}
#[test]
fn ttfb_and_total_duration_are_separate_families() {
let store = store();
record_request_into(&store, "/api/chat", 200, Duration::from_millis(250));
record_response_body_into(&store, "/api/chat", Duration::from_secs(45));
let rendered = rendered(&store, &snapshot());
assert!(
rendered.contains("llmman_http_request_ttfb_seconds_sum{route=\"/api/chat\"} 0.25\n"),
"{rendered}"
);
assert!(
rendered.contains("llmman_http_request_duration_seconds_sum{route=\"/api/chat\"} 45\n"),
"{rendered}"
);
}
#[test]
fn every_model_and_reason_counts_separately() {
let store = store();
record_model_unload_into(&store, "a:latest", UnloadReason::Evicted);
record_model_unload_into(&store, "a:latest", UnloadReason::Evicted);
record_model_unload_into(&store, "a:latest", UnloadReason::Crashed);
record_model_unload_into(&store, "b:latest", UnloadReason::Evicted);
let rendered = rendered(&store, &snapshot());
for (model, reason, expected) in [
("a:latest", "evicted", 2),
("a:latest", "crashed", 1),
("b:latest", "evicted", 1),
] {
assert!(
rendered.contains(&format!(
"llmman_model_unloads_total{{model=\"{model}\",reason=\"{reason}\"}} {expected}\n"
)),
"{model}/{reason} should be {expected} in:\n{rendered}"
);
}
assert!(!rendered.contains("reason=\"idle\""), "{rendered}");
}
#[test]
fn oom_retries_count_by_strategy() {
let store = store();
record_oom_retry_into(&store, "a:latest", OomRetry::EvictOthers);
record_oom_retry_into(&store, "a:latest", OomRetry::CtxShrink);
record_oom_retry_into(&store, "a:latest", OomRetry::CtxShrink);
let rendered = rendered(&store, &snapshot());
for (strategy, expected) in [("evict_others", 1), ("ctx_shrink", 2)] {
assert!(
rendered.contains(&format!(
"llmman_model_load_oom_retries_total{{model=\"a:latest\",strategy=\"{strategy}\"}} {expected}\n"
)),
"{strategy} should be {expected} in:\n{rendered}"
);
}
}
#[test]
fn a_waiting_request_reads_as_queued_only_when_the_slots_are_known() {
let mut snapshot = snapshot();
snapshot.models = vec![
ModelState {
model: "a:latest".into(),
engine: "llama-server",
up: true,
requests_in_flight: 2,
requests_queued: Some(1),
},
ModelState {
model: "b:latest".into(),
engine: "llama-server",
up: true,
requests_in_flight: 3,
requests_queued: None,
},
];
let rendered = rendered(&store(), &snapshot);
for expected in [
"llmman_requests_in_flight{model=\"a:latest\"} 2\n",
"llmman_requests_in_flight{model=\"b:latest\"} 3\n",
"llmman_requests_queued{model=\"a:latest\"} 1\n",
] {
assert!(rendered.contains(expected), "{expected} in:\n{rendered}");
}
assert!(!rendered.contains("llmman_requests_queued{model=\"b:latest\"}"));
}
#[test]
fn a_dead_backend_reads_down_while_llmman_still_lists_it() {
let mut snapshot = snapshot();
snapshot.models_loaded = 2;
snapshot.models = vec![
ModelState {
model: "a:latest".into(),
engine: "llama-server",
up: true,
requests_in_flight: 0,
requests_queued: None,
},
ModelState {
model: "b:latest".into(),
engine: "vllm",
up: false,
requests_in_flight: 0,
requests_queued: None,
},
];
let rendered = rendered(&store(), &snapshot);
assert!(
rendered.contains("llmman_model_up{model=\"a:latest\",engine=\"llama-server\"} 1\n"),
"{rendered}"
);
assert!(
rendered.contains("llmman_model_up{model=\"b:latest\",engine=\"vllm\"} 0\n"),
"{rendered}"
);
assert!(rendered.contains("llmman_models_loaded 2\n"), "{rendered}");
}
#[test]
fn an_unlabelled_counter_zero_fills_and_renders_without_empty_braces() {
let store = store();
let cold = rendered(&store, &snapshot());
assert!(
cold.contains("llmman_scheduling_rejections_total 0\n"),
"{cold}"
);
record_scheduling_rejection_into(&store);
record_scheduling_rejection_into(&store);
let rendered = rendered(&store, &snapshot());
assert!(
rendered.contains("llmman_scheduling_rejections_total 2\n"),
"{rendered}"
);
assert!(!rendered.contains("{}"), "{rendered}");
}
#[test]
fn a_model_reference_can_legitimately_contain_a_quote() {
let path = "/models/My \"Model\"/x.gguf";
assert!(
crate::shortnames::validate_reference(path).is_ok(),
"{path} is a reference llmman accepts, so it can reach a label"
);
let store = store();
record_model_load_into(&store, path, Duration::from_secs(1));
let rendered = rendered(&store, &snapshot());
assert!(
rendered.contains(
"llmman_model_loads_total{model=\"/models/My \\\"Model\\\"/x.gguf\"} 1\n"
),
"{rendered}"
);
}
#[test]
fn escape_label_value_covers_the_three_sequences_and_drops_a_carriage_return() {
assert_eq!(escape_label_value("a\\b"), "a\\\\b");
assert_eq!(escape_label_value("a\"b"), "a\\\"b");
assert_eq!(escape_label_value("a\nb"), "a\\nb");
assert_eq!(escape_label_value("1.0\r2.0"), "1.02.0");
assert_eq!(escape_label_value("plain/0.1"), "plain/0.1");
}
#[test]
fn marking_the_process_start_records_a_plausible_unix_time() {
mark_process_start();
let marked = process_start_seconds();
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("the system clock is after the unix epoch")
.as_secs();
assert!(marked > 1_700_000_000, "{marked} is not a plausible time");
assert!(marked <= now, "{marked} is ahead of now ({now})");
}
#[test]
fn the_process_wide_wrappers_reach_the_shared_registry() {
let _serialised = GLOBAL_COUNTER_TEST_LOCK.blocking_lock();
const LOADS: &str = "llmman_model_loads_total{model=\"wrapper-probe\"}";
const UNLOADS: &str = "llmman_model_unloads_total{model=\"wrapper-probe\",reason=\"oom\"}";
const RETRIES: &str =
"llmman_model_load_oom_retries_total{model=\"wrapper-probe\",strategy=\"ctx_shrink\"}";
const REJECTIONS: &str = "llmman_scheduling_rejections_total ";
let before = render(&snapshot());
record_model_load("wrapper-probe", Duration::from_secs(1));
record_model_unload("wrapper-probe", UnloadReason::Oom);
record_oom_retry("wrapper-probe", OomRetry::CtxShrink);
record_scheduling_rejection();
let after = render(&snapshot());
assert_eq!(value_of(&after, LOADS), value_of(&before, LOADS) + 1);
assert_eq!(value_of(&after, UNLOADS), value_of(&before, UNLOADS) + 1);
assert_eq!(value_of(&after, RETRIES), value_of(&before, RETRIES) + 1);
assert_eq!(
value_of(&after, REJECTIONS),
value_of(&before, REJECTIONS) + 1
);
}
#[test]
fn enum_labels_are_stable() {
let unloads: Vec<&str> = [
UnloadReason::Idle,
UnloadReason::Requested,
UnloadReason::Crashed,
UnloadReason::Oom,
UnloadReason::Evicted,
UnloadReason::Reloaded,
]
.iter()
.map(|r| r.as_str())
.collect();
assert_eq!(
unloads,
["idle", "requested", "crashed", "oom", "evicted", "reloaded"]
);
let retries: Vec<&str> = [
OomRetry::EvictOthers,
OomRetry::SplitMode,
OomRetry::CtxShrink,
]
.iter()
.map(|r| r.as_str())
.collect();
assert_eq!(retries, ["evict_others", "split_mode", "ctx_shrink"]);
}
#[test]
fn a_poisoned_lock_does_not_stop_recording() {
let store = store();
record_scheduling_rejection_into(&store);
let poisoned = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _guard = store.lock().unwrap();
panic!("a recorder panicked while holding the lock");
}));
assert!(poisoned.is_err());
assert!(store.is_poisoned());
record_scheduling_rejection_into(&store);
let rendered = rendered(&store, &snapshot());
assert!(
rendered.contains("llmman_scheduling_rejections_total 2\n"),
"{rendered}"
);
}
}