use crate::{ConfigError, EventKind, Notification, NotifyErrorKind};
use nametrics_core::atomic::{add, AtomicHistogram, LATENCY_SECONDS};
use nametrics_core::{
LegacyMetricsSource, MetricDescriptor, MetricKind, MetricSample, MetricValue,
};
use std::collections::BTreeSet;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::mpsc;
static ENQUEUED: MetricDescriptor = MetricDescriptor {
name: "nanotify_enqueued_total",
help: "进入通知队列的离散事件累计数。",
unit: "",
kind: MetricKind::Counter,
label_names: &["provider", "event"],
histogram_bounds: &[],
};
static DROPPED: MetricDescriptor = MetricDescriptor {
name: "nanotify_dropped_total",
help: "因队列或停机门禁丢弃的通知累计数。",
unit: "",
kind: MetricKind::Counter,
label_names: &["provider", "event", "reason"],
histogram_bounds: &[],
};
static DELIVERIES: MetricDescriptor = MetricDescriptor {
name: "nanotify_deliveries_total",
help: "以最终结果分类的通知投递累计数。",
unit: "",
kind: MetricKind::Counter,
label_names: &["provider", "event", "outcome"],
histogram_bounds: &[],
};
static DURATION: MetricDescriptor = MetricDescriptor {
name: "nanotify_delivery_duration_seconds",
help: "包含有限重试和等待的通知投递时长。",
unit: "seconds",
kind: MetricKind::Histogram,
label_names: &["provider", "outcome"],
histogram_bounds: LATENCY_SECONDS,
};
static DEPTH: MetricDescriptor = MetricDescriptor {
name: "nanotify_queue_depth",
help: "等待 worker 领取的通知数量。",
unit: "",
kind: MetricKind::Gauge,
label_names: &[],
histogram_bounds: &[],
};
static CAPACITY: MetricDescriptor = MetricDescriptor {
name: "nanotify_queue_capacity",
help: "当前进程通知队列的固定容量。",
unit: "",
kind: MetricKind::Gauge,
label_names: &[],
histogram_bounds: &[],
};
static DESCRIPTORS: &[&MetricDescriptor] = &[
&ENQUEUED,
&DROPPED,
&DELIVERIES,
&DURATION,
&DEPTH,
&CAPACITY,
];
const EVENTS: [EventKind; 3] = [
EventKind::SlowSql,
EventKind::ExecutionError,
EventKind::AcquireTimeout,
];
const DROP_REASONS: [&str; 3] = ["queue_full", "stopping", "shutdown_timeout"];
const OUTCOMES: [&str; 10] = [
"accepted",
"timeout",
"unavailable",
"rate_limited",
"rejected",
"authentication",
"invalid_request",
"other",
"panic",
"shutdown",
];
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(usize)]
pub enum DeliveryOutcome {
Accepted,
Timeout,
Unavailable,
RateLimited,
Rejected,
Authentication,
InvalidRequest,
Other,
Panic,
Shutdown,
}
impl From<NotifyErrorKind> for DeliveryOutcome {
fn from(kind: NotifyErrorKind) -> Self {
match kind {
NotifyErrorKind::Timeout => Self::Timeout,
NotifyErrorKind::Unavailable => Self::Unavailable,
NotifyErrorKind::RateLimited => Self::RateLimited,
NotifyErrorKind::Rejected => Self::Rejected,
NotifyErrorKind::Authentication => Self::Authentication,
NotifyErrorKind::InvalidRequest => Self::InvalidRequest,
NotifyErrorKind::Other => Self::Other,
}
}
}
struct ProviderMetrics {
id: String,
enqueued: [AtomicU64; 3],
dropped: [[AtomicU64; 3]; 3],
deliveries: [[AtomicU64; 10]; 3],
duration: [AtomicHistogram; 10],
}
pub struct QueueMetrics {
providers: Vec<ProviderMetrics>,
capacity: usize,
depth: AtomicU64,
}
impl QueueMetrics {
pub fn series_budget(&self) -> usize {
self.providers.len() * (3 + 9 + 30 + 10 * (LATENCY_SECONDS.len() + 3)) + 2
}
pub fn record_delivery(
&self,
provider: usize,
event: EventKind,
outcome: DeliveryOutcome,
elapsed: Duration,
) {
if let Some(slot) = self.providers.get(provider) {
add(&slot.deliveries[event as usize][outcome as usize], 1);
slot.duration[outcome as usize].observe(elapsed);
}
}
}
impl LegacyMetricsSource for QueueMetrics {
fn descriptors(&self) -> &'static [&'static MetricDescriptor] {
DESCRIPTORS
}
fn snapshot(&self) -> Option<Vec<MetricSample>> {
let mut samples = Vec::new();
for provider in &self.providers {
for event in EVENTS {
let labels = vec![
("provider", provider.id.clone()),
("event", event.as_str().into()),
];
samples.push(MetricSample {
name: ENQUEUED.name,
labels: labels.clone(),
value: MetricValue::Counter(
provider.enqueued[event as usize].load(Ordering::Relaxed),
),
});
for (index, reason) in DROP_REASONS.iter().enumerate() {
let mut labels = labels.clone();
labels.push(("reason", (*reason).into()));
samples.push(MetricSample {
name: DROPPED.name,
labels,
value: MetricValue::Counter(
provider.dropped[event as usize][index].load(Ordering::Relaxed),
),
});
}
for (index, outcome) in OUTCOMES.iter().enumerate() {
let mut labels = labels.clone();
labels.push(("outcome", (*outcome).into()));
samples.push(MetricSample {
name: DELIVERIES.name,
labels,
value: MetricValue::Counter(
provider.deliveries[event as usize][index].load(Ordering::Relaxed),
),
});
}
}
for (index, outcome) in OUTCOMES.iter().enumerate() {
samples.push(MetricSample {
name: DURATION.name,
labels: vec![
("provider", provider.id.clone()),
("outcome", (*outcome).into()),
],
value: provider.duration[index].snapshot(),
});
}
}
samples.push(MetricSample {
name: DEPTH.name,
labels: vec![],
value: MetricValue::Gauge(self.depth.load(Ordering::Relaxed) as f64),
});
samples.push(MetricSample {
name: CAPACITY.name,
labels: vec![],
value: MetricValue::Gauge(self.capacity as f64),
});
Some(samples)
}
fn render_prometheus(&self, output: &mut String) {
nametrics_core::atomic::render_snapshot(
DESCRIPTORS,
&self.snapshot().unwrap_or_default(),
output,
);
}
}
struct QueueState {
accepting: AtomicBool,
metrics: Arc<QueueMetrics>,
}
#[derive(Clone)]
pub struct AlertRoute {
sender: mpsc::Sender<Envelope>,
state: Arc<QueueState>,
provider: usize,
requires_initialized_notify: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EnqueueOutcome {
Enqueued,
Full,
Stopping,
Ignored,
}
impl AlertRoute {
pub fn try_send(&self, notification: Notification) -> EnqueueOutcome {
if self.requires_initialized_notify && crate::get().is_none() {
return EnqueueOutcome::Ignored;
}
let event = notification.fields().event;
let slot = &self.state.metrics.providers[self.provider];
if !self.state.accepting.load(Ordering::Acquire) {
add(&slot.dropped[event as usize][1], 1);
return EnqueueOutcome::Stopping;
}
match self.sender.try_reserve() {
Ok(permit) => {
if !self.state.accepting.load(Ordering::Acquire) {
add(&slot.dropped[event as usize][1], 1);
return EnqueueOutcome::Stopping;
}
add(&self.state.metrics.depth, 1);
add(&slot.enqueued[event as usize], 1);
permit.send(Envelope {
provider: self.provider,
notification,
queued: true,
metrics: self.state.metrics.clone(),
});
EnqueueOutcome::Enqueued
}
Err(mpsc::error::TrySendError::Full(_)) => {
add(&slot.dropped[event as usize][0], 1);
EnqueueOutcome::Full
}
Err(mpsc::error::TrySendError::Closed(_)) => {
add(&slot.dropped[event as usize][1], 1);
EnqueueOutcome::Stopping
}
}
}
}
#[derive(Clone)]
pub struct NotificationProducer {
sender: mpsc::Sender<Envelope>,
state: Arc<QueueState>,
}
impl NotificationProducer {
pub fn route(&self, provider_ref: &str) -> Option<AlertRoute> {
self.state
.metrics
.providers
.iter()
.position(|slot| slot.id == provider_ref)
.map(|provider| AlertRoute {
sender: self.sender.clone(),
state: self.state.clone(),
provider,
requires_initialized_notify: false,
})
}
pub fn default_route(&self) -> Option<AlertRoute> {
self.route(crate::DEFAULT_PROVIDER_ID).map(|mut route| {
route.requires_initialized_notify = true;
route
})
}
pub fn close(&self) {
self.state.accepting.store(false, Ordering::Release);
}
pub fn metrics(&self) -> Arc<QueueMetrics> {
self.state.metrics.clone()
}
}
pub struct Envelope {
pub provider: usize,
pub notification: Notification,
queued: bool,
metrics: Arc<QueueMetrics>,
}
impl Drop for Envelope {
fn drop(&mut self) {
if self.queued {
self.metrics.depth.fetch_sub(1, Ordering::Relaxed);
add(
&self.metrics.providers[self.provider].dropped
[self.notification.fields().event as usize][2],
1,
);
}
}
}
pub struct NotificationReceiver {
receiver: mpsc::Receiver<Envelope>,
state: Arc<QueueState>,
}
impl NotificationReceiver {
pub async fn recv(&mut self) -> Option<Envelope> {
let mut message = self.receiver.recv().await;
if let Some(message) = &mut message {
message.queued = false;
self.state.metrics.depth.fetch_sub(1, Ordering::Relaxed);
}
message
}
pub fn close(&mut self) {
self.state.accepting.store(false, Ordering::Release);
self.receiver.close();
}
pub fn discard_remaining(&mut self) -> usize {
self.close();
let mut count = 0;
while let Ok(message) = self.receiver.try_recv() {
drop(message);
count += 1;
}
count
}
}
impl Drop for NotificationReceiver {
fn drop(&mut self) {
self.discard_remaining();
}
}
pub struct NotificationQueue;
impl NotificationQueue {
pub fn bounded(
provider_ids: Vec<String>,
capacity: usize,
) -> Result<(NotificationProducer, NotificationReceiver), ConfigError> {
if !(1..=65536).contains(&capacity) || provider_ids.is_empty() || provider_ids.len() > 32 {
return Err(ConfigError("queue"));
}
let mut seen = BTreeSet::new();
if provider_ids
.iter()
.any(|id| !valid_provider_id(id) || !seen.insert(id))
{
return Err(ConfigError("providers"));
}
let providers = provider_ids
.into_iter()
.map(|id| ProviderMetrics {
id,
enqueued: std::array::from_fn(|_| AtomicU64::new(0)),
dropped: std::array::from_fn(|_| std::array::from_fn(|_| AtomicU64::new(0))),
deliveries: std::array::from_fn(|_| std::array::from_fn(|_| AtomicU64::new(0))),
duration: std::array::from_fn(|_| AtomicHistogram::new(LATENCY_SECONDS)),
})
.collect();
let state = Arc::new(QueueState {
accepting: AtomicBool::new(true),
metrics: Arc::new(QueueMetrics {
providers,
capacity,
depth: AtomicU64::new(0),
}),
});
let (sender, receiver) = mpsc::channel(capacity);
Ok((
NotificationProducer {
sender,
state: state.clone(),
},
NotificationReceiver { receiver, state },
))
}
}
pub fn valid_provider_id(id: &str) -> bool {
(1..=64).contains(&id.len())
&& id.as_bytes()[0].is_ascii_alphabetic()
&& id
.bytes()
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, b'_' | b'.' | b'-'))
}