Skip to main content

redis_tower/
metrics_layer.rs

1//! Tower layer for collecting per-command metrics at the Frame level.
2//!
3//! Framework-agnostic: users implement [`MetricsRecorder`] for their
4//! metrics backend (prometheus, metrics crate, etc.).
5//!
6//! # Example
7//!
8//! ```no_run
9//! use std::time::Duration;
10//! use redis_tower::metrics_layer::{MetricsLayer, MetricsRecorder, ErrorKind};
11//!
12//! struct MyRecorder;
13//!
14//! impl MetricsRecorder for MyRecorder {
15//!     fn command_completed(&self, command: &str, duration: Duration, error: Option<ErrorKind>) {
16//!         match error {
17//!             None => println!("{command} took {duration:?} (ok)"),
18//!             Some(kind) => println!("{command} took {duration:?} (error: {kind:?})"),
19//!         }
20//!     }
21//! }
22//!
23//! let layer = MetricsLayer::new(MyRecorder);
24//! # let _ = layer;
25//! ```
26
27use std::future::Future;
28use std::pin::Pin;
29use std::sync::Arc;
30use std::task::{Context, Poll};
31use std::time::{Duration, Instant};
32
33use redis_tower_core::{Frame, RedisError};
34use tower_service::Service;
35
36#[cfg(feature = "metrics")]
37use crate::multiplexed::MultiplexedClient;
38use crate::pool::HealthProbeKind;
39#[cfg(feature = "metrics")]
40use crate::pool::{ConnectionPool, PoolStats};
41
42/// Categorizes a Redis error for metrics labeling.
43///
44/// Used as the `error` argument to [`MetricsRecorder::command_completed`].
45/// Each variant corresponds to a meaningful alert/dashboard category.
46///
47/// # Categories
48///
49/// - `Connection` — transport-level failures: IO errors, closed connections.
50///   These are transient and typically require reconnection.
51/// - `Timeout` — pool acquisition timeout; indicates pool exhaustion.
52/// - `WrongType` — Redis `WRONGTYPE` error; indicates an application bug.
53/// - `CircuitOpen` — circuit breaker is open; request rejected without
54///   touching Redis.
55/// - `QueueFull` — the auto-pipeline channel is full; caller should shed load.
56/// - `Auth` — credential-provider or authentication failure (`AUTH_PROVIDER`,
57///   `NOAUTH`, `WRONGPASS`).
58/// - `Other` — all other errors (generic Redis errors, protocol errors, etc.).
59#[derive(Debug, Clone, Copy, PartialEq, Eq)]
60pub enum ErrorKind {
61    /// Transport-level error (IO error, connection closed).
62    Connection,
63    /// Pool acquisition timed out.
64    Timeout,
65    /// Redis `WRONGTYPE` error.
66    WrongType,
67    /// Circuit breaker is open.
68    CircuitOpen,
69    /// Auto-pipeline queue is full.
70    QueueFull,
71    /// Credential-provider or authentication error.
72    Auth,
73    /// All other errors.
74    Other,
75}
76
77impl ErrorKind {
78    /// Map a [`RedisError`] to the most specific [`ErrorKind`].
79    pub fn from_error(e: &RedisError) -> Self {
80        match e {
81            RedisError::Connection { .. } | RedisError::ConnectionClosed => ErrorKind::Connection,
82            RedisError::PoolAcquisitionTimeout { .. } => ErrorKind::Timeout,
83            RedisError::CircuitOpen => ErrorKind::CircuitOpen,
84            RedisError::QueueFull => ErrorKind::QueueFull,
85            RedisError::Redis(msg) => {
86                if msg.starts_with("WRONGTYPE") {
87                    ErrorKind::WrongType
88                } else if msg.starts_with("AUTH_PROVIDER")
89                    || msg.starts_with("NOAUTH")
90                    || msg.starts_with("WRONGPASS")
91                {
92                    ErrorKind::Auth
93                } else {
94                    ErrorKind::Other
95                }
96            }
97            _ => ErrorKind::Other,
98        }
99    }
100
101    /// Classify a Redis-level error frame (i.e. `Frame::Error`) by its
102    /// error prefix.
103    fn from_frame_error(bytes: &bytes::Bytes) -> Self {
104        let msg = std::str::from_utf8(bytes).unwrap_or("");
105        if msg.starts_with("WRONGTYPE") {
106            ErrorKind::WrongType
107        } else if msg.starts_with("NOAUTH") || msg.starts_with("WRONGPASS") {
108            ErrorKind::Auth
109        } else {
110            ErrorKind::Other
111        }
112    }
113
114    #[cfg(feature = "metrics")]
115    fn as_label(self) -> &'static str {
116        match self {
117            Self::Connection => "connection",
118            Self::Timeout => "timeout",
119            Self::WrongType => "wrong_type",
120            Self::CircuitOpen => "circuit_open",
121            Self::QueueFull => "queue_full",
122            Self::Auth => "auth",
123            Self::Other => "_OTHER",
124        }
125    }
126}
127
128/// The bounded set of redirect responses emitted by Redis Cluster.
129///
130/// Keeping redirect kinds as an enum rather than an arbitrary string prevents
131/// accidental high-cardinality metric labels. The target node and hash slot
132/// belong in tracing events; redirect counters need only distinguish the two
133/// protocol-defined redirect modes.
134#[derive(Debug, Clone, Copy, PartialEq, Eq)]
135pub enum ClusterRedirectKind {
136    /// A permanent slot-owner redirect (`MOVED`).
137    Moved,
138    /// A temporary migrating-slot redirect (`ASK`).
139    Ask,
140}
141
142impl ClusterRedirectKind {
143    #[cfg(feature = "metrics")]
144    fn as_label(self) -> &'static str {
145        match self {
146            Self::Moved => "moved",
147            Self::Ask => "ask",
148        }
149    }
150}
151
152/// The bounded outcome of a Redis Cluster topology refresh attempt.
153///
154/// `Partial` distinguishes useful best-effort progress from a refresh that
155/// reconciled every desired node, without introducing an arbitrary string
156/// label into metrics backends.
157#[derive(Debug, Clone, Copy, PartialEq, Eq)]
158pub enum ClusterTopologyRefreshOutcome {
159    /// Every desired node service was reconciled successfully.
160    Success,
161    /// The topology was updated, but one or more desired node services could
162    /// not be built during this attempt.
163    Partial,
164    /// Topology discovery or reconciliation failed.
165    Error,
166}
167
168/// A bounded client-side cache event suitable for metrics labels.
169///
170/// Cache keys and command arguments are intentionally absent: emitting them
171/// would create unbounded-cardinality metric series and could expose user
172/// data. Applications that need the current aggregate totals can also read a
173/// [`crate::CacheStatistics`] snapshot from a cached client.
174#[derive(Debug, Clone, Copy, PartialEq, Eq)]
175pub enum CacheEvent {
176    /// A response was served from the local cache.
177    Hit,
178    /// A cacheable command was not present (or had expired) locally.
179    Miss,
180    /// Redis or a local write invalidated one or more keys.
181    Invalidation,
182    /// A cached entry was removed by invalidation, expiry, or capacity bounds.
183    Eviction,
184}
185
186impl CacheEvent {
187    #[cfg(feature = "metrics")]
188    fn as_label(self) -> &'static str {
189        match self {
190            Self::Hit => "hit",
191            Self::Miss => "miss",
192            Self::Invalidation => "invalidation",
193            Self::Eviction => "eviction",
194        }
195    }
196}
197
198impl ClusterTopologyRefreshOutcome {
199    #[cfg(feature = "metrics")]
200    fn as_label(self) -> &'static str {
201        match self {
202            Self::Success => "success",
203            Self::Partial => "partial",
204            Self::Error => "error",
205        }
206    }
207}
208
209/// Receives metric events for each Redis command. Users implement this
210/// trait to integrate with their metrics framework (prometheus, metrics
211/// crate, OpenTelemetry, etc.).
212pub trait MetricsRecorder: Send + Sync + 'static {
213    /// Called after each command completes.
214    ///
215    /// - `command`: the Redis command name (e.g. `"GET"`, `"SET"`), or
216    ///   `"UNKNOWN"` if it could not be extracted from the frame.
217    /// - `duration`: wall-clock time from call to completion.
218    /// - `error`: `None` on success; `Some(kind)` on failure. The kind
219    ///   enables labeled counters such as
220    ///   `redis_command_errors_total{command="GET",kind="connection"}`.
221    ///
222    /// Note: `Frame::Error` responses (Redis-level errors such as
223    /// `WRONGTYPE`) are classified as failures, not successes.
224    fn command_completed(&self, command: &str, duration: Duration, error: Option<ErrorKind>);
225
226    /// Called after a command completes with an optional cluster-node label.
227    ///
228    /// Cluster clients pass the node that ultimately handled the command when
229    /// node labeling is enabled. The label is optional because node addresses
230    /// create one metric series per cluster member and applications may prefer
231    /// the lower-cardinality aggregate.
232    ///
233    /// The default implementation discards `node` and delegates to
234    /// [`command_completed`](Self::command_completed), so existing recorder
235    /// implementations automatically receive cluster command metrics without
236    /// an API break.
237    fn command_completed_on_node(
238        &self,
239        command: &str,
240        duration: Duration,
241        error: Option<ErrorKind>,
242        node: Option<&str>,
243    ) {
244        let _ = node;
245        self.command_completed(command, duration, error);
246    }
247
248    /// Called when a Redis Cluster command receives a redirect response.
249    ///
250    /// [`ClusterRedirectKind`] bounds the possible metric label values. Node,
251    /// slot, and retry-attempt details are intentionally left to tracing so
252    /// this counter cannot grow with cluster size or workload keys.
253    ///
254    /// The default implementation is a no-op.
255    fn cluster_redirected(&self, kind: ClusterRedirectKind) {
256        let _ = kind;
257    }
258
259    /// Called after a Redis Cluster topology refresh attempt completes.
260    ///
261    /// `duration` covers the complete refresh attempt and `outcome` supplies a
262    /// bounded result suitable for counters and histograms.
263    ///
264    /// The default implementation is a no-op.
265    fn cluster_topology_refresh_completed(
266        &self,
267        duration: Duration,
268        outcome: ClusterTopologyRefreshOutcome,
269    ) {
270        let _ = (duration, outcome);
271    }
272
273    /// Called after each auto-pipeline flush.
274    ///
275    /// `batch_size` is the number of frames sent in that flush. A histogram
276    /// of this value reveals whether pipelining is effective (`batch_size >
277    /// 1`) or whether individual callers flush immediately (`batch_size ==
278    /// 1`).
279    ///
280    /// Default implementation is a no-op so existing implementors are not
281    /// affected.
282    fn pipeline_flushed(&self, batch_size: usize) {
283        let _ = batch_size;
284    }
285
286    /// Called when the client-side cache records an aggregate event.
287    ///
288    /// `count` is normally one, but may be greater when a single invalidation
289    /// removes several cached command variants. The event kind is deliberately
290    /// bounded and carries no Redis keys or command arguments.
291    ///
292    /// The default implementation is a no-op.
293    fn cache_event(&self, event: CacheEvent, count: u64) {
294        let _ = (event, count);
295    }
296
297    /// Called when a caller finishes waiting to acquire a pool connection.
298    ///
299    /// `duration` includes both immediately successful acquisitions and time
300    /// spent queued behind a busy connection. `timed_out` is `true` when the
301    /// configured acquisition timeout elapsed before a connection became
302    /// available.
303    ///
304    /// The default implementation is a no-op so custom recorders can opt in
305    /// to pool metrics without a breaking trait change.
306    fn pool_acquisition_completed(&self, pool_name: &str, duration: Duration, timed_out: bool) {
307        let _ = (pool_name, duration, timed_out);
308    }
309
310    /// Called when a pooled connection fails its health check.
311    ///
312    /// The default implementation is a no-op.
313    fn pool_health_check_failed(&self, pool_name: &str) {
314        let _ = pool_name;
315    }
316
317    /// Called after a failed pooled connection is successfully replaced.
318    ///
319    /// The default implementation is a no-op.
320    fn pool_connection_replaced(&self, pool_name: &str) {
321        let _ = pool_name;
322    }
323
324    /// Called after one active pool health probe completes.
325    ///
326    /// Probe kinds and outcomes are bounded for safe metric labels. Lag is
327    /// present only for replication-lag probes.
328    fn pool_health_probe_completed(
329        &self,
330        pool_name: &str,
331        kind: HealthProbeKind,
332        duration: Duration,
333        healthy: bool,
334        replication_lag_bytes: Option<u64>,
335    ) {
336        let _ = (pool_name, kind, duration, healthy, replication_lag_bytes);
337    }
338
339    /// Called after an idle reaper removes one or more pool connections.
340    fn pool_connections_reaped(&self, pool_name: &str, count: usize) {
341        let _ = (pool_name, count);
342    }
343}
344
345impl<R> MetricsRecorder for Arc<R>
346where
347    R: MetricsRecorder + ?Sized,
348{
349    fn command_completed(&self, command: &str, duration: Duration, error: Option<ErrorKind>) {
350        (**self).command_completed(command, duration, error);
351    }
352
353    fn command_completed_on_node(
354        &self,
355        command: &str,
356        duration: Duration,
357        error: Option<ErrorKind>,
358        node: Option<&str>,
359    ) {
360        (**self).command_completed_on_node(command, duration, error, node);
361    }
362
363    fn cluster_redirected(&self, kind: ClusterRedirectKind) {
364        (**self).cluster_redirected(kind);
365    }
366
367    fn cluster_topology_refresh_completed(
368        &self,
369        duration: Duration,
370        outcome: ClusterTopologyRefreshOutcome,
371    ) {
372        (**self).cluster_topology_refresh_completed(duration, outcome);
373    }
374
375    fn pipeline_flushed(&self, batch_size: usize) {
376        (**self).pipeline_flushed(batch_size);
377    }
378
379    fn cache_event(&self, event: CacheEvent, count: u64) {
380        (**self).cache_event(event, count);
381    }
382
383    fn pool_acquisition_completed(&self, pool_name: &str, duration: Duration, timed_out: bool) {
384        (**self).pool_acquisition_completed(pool_name, duration, timed_out);
385    }
386
387    fn pool_health_check_failed(&self, pool_name: &str) {
388        (**self).pool_health_check_failed(pool_name);
389    }
390
391    fn pool_connection_replaced(&self, pool_name: &str) {
392        (**self).pool_connection_replaced(pool_name);
393    }
394
395    fn pool_health_probe_completed(
396        &self,
397        pool_name: &str,
398        kind: HealthProbeKind,
399        duration: Duration,
400        healthy: bool,
401        replication_lag_bytes: Option<u64>,
402    ) {
403        (**self).pool_health_probe_completed(
404            pool_name,
405            kind,
406            duration,
407            healthy,
408            replication_lag_bytes,
409        );
410    }
411
412    fn pool_connections_reaped(&self, pool_name: &str, count: usize) {
413        (**self).pool_connections_reaped(pool_name, count);
414    }
415}
416
417/// A [`MetricsRecorder`] backed by the [`metrics`](https://docs.rs/metrics)
418/// facade.
419///
420/// This recorder lets applications choose their metrics exporter at startup
421/// without coupling redis-tower to Prometheus, OpenTelemetry, or another
422/// concrete backend. Enable the `metrics` crate feature, install a compatible
423/// global recorder in the application, and pass this value to
424/// [`MetricsLayer`],
425/// [`crate::AutoPipelineConfig`], or [`crate::PoolConfig`].
426///
427/// Metric labels use command and pool names supplied by the application, plus
428/// bounded success/error outcomes and [`ErrorKind`] variants. Applications
429/// that build commands dynamically should normalize command names before
430/// exposing them to an untrusted caller to avoid high-cardinality series.
431#[cfg(feature = "metrics")]
432#[cfg_attr(docsrs, doc(cfg(feature = "metrics")))]
433#[derive(Debug, Default, Clone, Copy)]
434pub struct MetricsFacadeRecorder;
435
436#[cfg(feature = "metrics")]
437impl MetricsFacadeRecorder {
438    /// Create a metrics-facade recorder.
439    pub const fn new() -> Self {
440        Self
441    }
442}
443
444#[cfg(feature = "metrics")]
445fn record_command_metrics(
446    command: &str,
447    duration: Duration,
448    error: Option<ErrorKind>,
449    node: Option<&str>,
450) {
451    let error = error.map(ErrorKind::as_label);
452    let outcome = if error.is_some() { "error" } else { "success" };
453
454    let mut duration_labels = vec![
455        metrics::Label::new("db.system.name", "redis"),
456        metrics::Label::new("db.operation.name", command.to_owned()),
457    ];
458    let mut counter_labels = vec![
459        metrics::Label::new("db.operation.name", command.to_owned()),
460        metrics::Label::new("outcome", outcome),
461    ];
462
463    if let Some(error) = error {
464        duration_labels.push(metrics::Label::new("error.type", error));
465        counter_labels.push(metrics::Label::new("error.type", error));
466    }
467    if let Some(node) = node {
468        let node = node.to_owned();
469        duration_labels.push(metrics::Label::new(
470            "redis_tower.cluster.node",
471            node.clone(),
472        ));
473        counter_labels.push(metrics::Label::new("redis_tower.cluster.node", node));
474    }
475
476    metrics::histogram!(
477        description: "Duration of Redis client operations",
478        unit: metrics::Unit::Seconds,
479        "db.client.operation.duration",
480        duration_labels,
481    )
482    .record(duration.as_secs_f64());
483    metrics::counter!(
484        description: "Redis commands completed",
485        unit: metrics::Unit::Count,
486        "redis_tower.commands",
487        counter_labels,
488    )
489    .increment(1);
490}
491
492#[cfg(feature = "metrics")]
493impl MetricsRecorder for MetricsFacadeRecorder {
494    fn command_completed(&self, command: &str, duration: Duration, error: Option<ErrorKind>) {
495        record_command_metrics(command, duration, error, None);
496    }
497
498    fn command_completed_on_node(
499        &self,
500        command: &str,
501        duration: Duration,
502        error: Option<ErrorKind>,
503        node: Option<&str>,
504    ) {
505        record_command_metrics(command, duration, error, node);
506    }
507
508    fn cluster_redirected(&self, kind: ClusterRedirectKind) {
509        metrics::counter!(
510            description: "Redis Cluster redirect responses",
511            unit: metrics::Unit::Count,
512            "redis_tower.cluster.redirects",
513            "kind" => kind.as_label(),
514        )
515        .increment(1);
516    }
517
518    fn cluster_topology_refresh_completed(
519        &self,
520        duration: Duration,
521        outcome: ClusterTopologyRefreshOutcome,
522    ) {
523        let outcome = outcome.as_label();
524        metrics::histogram!(
525            description: "Duration of Redis Cluster topology refresh attempts",
526            unit: metrics::Unit::Seconds,
527            "redis_tower.cluster.topology_refresh.duration",
528            "outcome" => outcome,
529        )
530        .record(duration.as_secs_f64());
531        metrics::counter!(
532            description: "Redis Cluster topology refresh attempts",
533            unit: metrics::Unit::Count,
534            "redis_tower.cluster.topology_refreshes",
535            "outcome" => outcome,
536        )
537        .increment(1);
538    }
539
540    fn pipeline_flushed(&self, batch_size: usize) {
541        metrics::histogram!(
542            description: "Redis commands sent in each automatic pipeline flush",
543            unit: metrics::Unit::Count,
544            "redis_tower.pipeline.batch_size",
545        )
546        .record(batch_size as f64);
547    }
548
549    fn cache_event(&self, event: CacheEvent, count: u64) {
550        metrics::counter!(
551            description: "Redis client-side cache events",
552            unit: metrics::Unit::Count,
553            "redis_tower.cache.events",
554            "event" => event.as_label(),
555        )
556        .increment(count);
557    }
558
559    fn pool_acquisition_completed(&self, pool_name: &str, duration: Duration, timed_out: bool) {
560        metrics::histogram!(
561            description: "Time spent waiting to acquire a Redis pool connection",
562            unit: metrics::Unit::Seconds,
563            "db.client.connection.wait_time",
564            "db.client.connection.pool.name" => pool_name.to_owned(),
565        )
566        .record(duration.as_secs_f64());
567
568        if timed_out {
569            metrics::counter!(
570                description: "Redis pool connection acquisition timeouts",
571                unit: metrics::Unit::Count,
572                "db.client.connection.timeouts",
573                "db.client.connection.pool.name" => pool_name.to_owned(),
574            )
575            .increment(1);
576        }
577    }
578
579    fn pool_health_check_failed(&self, pool_name: &str) {
580        metrics::counter!(
581            description: "Redis pool connection health-check failures",
582            unit: metrics::Unit::Count,
583            "redis_tower.pool.health_check_failures",
584            "db.client.connection.pool.name" => pool_name.to_owned(),
585        )
586        .increment(1);
587    }
588
589    fn pool_connection_replaced(&self, pool_name: &str) {
590        metrics::counter!(
591            description: "Redis pool connections replaced after failed health checks",
592            unit: metrics::Unit::Count,
593            "redis_tower.pool.connection_replacements",
594            "db.client.connection.pool.name" => pool_name.to_owned(),
595        )
596        .increment(1);
597    }
598
599    fn pool_health_probe_completed(
600        &self,
601        pool_name: &str,
602        kind: HealthProbeKind,
603        duration: Duration,
604        healthy: bool,
605        replication_lag_bytes: Option<u64>,
606    ) {
607        let outcome = if healthy { "healthy" } else { "unhealthy" };
608        metrics::histogram!(
609            description: "Duration of active Redis pool health probes",
610            unit: metrics::Unit::Seconds,
611            "redis_tower.pool.health_probe.duration",
612            "db.client.connection.pool.name" => pool_name.to_owned(),
613            "probe" => kind.as_str(),
614            "outcome" => outcome,
615        )
616        .record(duration.as_secs_f64());
617        metrics::counter!(
618            description: "Active Redis pool health probe outcomes",
619            unit: metrics::Unit::Count,
620            "redis_tower.pool.health_probes",
621            "db.client.connection.pool.name" => pool_name.to_owned(),
622            "probe" => kind.as_str(),
623            "outcome" => outcome,
624        )
625        .increment(1);
626        metrics::gauge!(
627            description: "Latest Redis replication lag observed by a pool probe",
628            unit: metrics::Unit::Bytes,
629            "redis_tower.pool.replication_lag",
630            "db.client.connection.pool.name" => pool_name.to_owned(),
631        )
632        .set(replication_lag_bytes.unwrap_or(0) as f64);
633        metrics::gauge!(
634            description: "Whether the Redis pool replication-lag gauge has a current observation",
635            unit: metrics::Unit::Count,
636            "redis_tower.pool.replication_lag_observed",
637            "db.client.connection.pool.name" => pool_name.to_owned(),
638        )
639        .set(if replication_lag_bytes.is_some() {
640            1.0
641        } else {
642            0.0
643        });
644    }
645
646    fn pool_connections_reaped(&self, pool_name: &str, count: usize) {
647        metrics::counter!(
648            description: "Redis pool connections removed after becoming idle",
649            unit: metrics::Unit::Count,
650            "redis_tower.pool.connections_reaped",
651            "db.client.connection.pool.name" => pool_name.to_owned(),
652        )
653        .increment(count as u64);
654    }
655}
656
657/// A cancellation handle for a background metrics snapshot exporter.
658///
659/// Dropping the handle cancels the exporter task immediately. Call
660/// [`shutdown`](Self::shutdown) to emit one final snapshot, cancel the task,
661/// and wait for it to finish.
662#[cfg(feature = "metrics")]
663#[cfg_attr(docsrs, doc(cfg(feature = "metrics")))]
664#[must_use = "dropping the handle immediately stops metrics export"]
665pub struct MetricsExporterHandle {
666    shutdown: Option<tokio::sync::oneshot::Sender<()>>,
667    task: Option<tokio::task::JoinHandle<()>>,
668}
669
670#[cfg(feature = "metrics")]
671impl MetricsExporterHandle {
672    fn spawn(interval: Duration, mut record: impl FnMut() + Send + 'static) -> Self {
673        assert!(
674            !interval.is_zero(),
675            "stats export interval must be non-zero"
676        );
677
678        let (shutdown, mut shutdown_rx) = tokio::sync::oneshot::channel();
679        let task = tokio::spawn(async move {
680            let mut ticker = tokio::time::interval(interval);
681            ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
682
683            loop {
684                tokio::select! {
685                    biased;
686                    _ = &mut shutdown_rx => {
687                        record();
688                        break;
689                    },
690                    _ = ticker.tick() => record(),
691                }
692            }
693        });
694
695        Self {
696            shutdown: Some(shutdown),
697            task: Some(task),
698        }
699    }
700
701    fn cancel(&mut self) {
702        if let Some(shutdown) = self.shutdown.take() {
703            let _ = shutdown.send(());
704        }
705    }
706
707    /// Emit a final snapshot, cancel the exporter, and wait for it to finish.
708    pub async fn shutdown(mut self) {
709        self.cancel();
710        if let Some(task) = self.task.take() {
711            let _ = task.await;
712        }
713    }
714}
715
716#[cfg(feature = "metrics")]
717impl Drop for MetricsExporterHandle {
718    fn drop(&mut self) {
719        self.cancel();
720        if let Some(task) = &self.task {
721            task.abort();
722        }
723    }
724}
725
726#[cfg(feature = "metrics")]
727fn pool_gauge_values(stats: &PoolStats) -> (usize, usize) {
728    let used_connections = stats.size.saturating_sub(stats.idle_count);
729    let pending_requests = stats.total_inflight.saturating_sub(used_connections);
730    (used_connections, pending_requests)
731}
732
733#[cfg(feature = "metrics")]
734fn record_pool_stats(pool_name: &str, stats: &PoolStats) {
735    let (used_connections, pending_requests) = pool_gauge_values(stats);
736    let pool_name = pool_name.to_owned();
737
738    metrics::gauge!(
739        description: "Redis pool connections by state",
740        unit: metrics::Unit::Count,
741        "db.client.connection.count",
742        "db.client.connection.pool.name" => pool_name.clone(),
743        "db.client.connection.state" => "idle",
744    )
745    .set(stats.idle_count as f64);
746    metrics::gauge!(
747        description: "Redis pool connections by state",
748        unit: metrics::Unit::Count,
749        "db.client.connection.count",
750        "db.client.connection.pool.name" => pool_name.clone(),
751        "db.client.connection.state" => "used",
752    )
753    .set(used_connections as f64);
754    metrics::gauge!(
755        description: "Maximum Redis pool connection count",
756        unit: metrics::Unit::Count,
757        "db.client.connection.max",
758        "db.client.connection.pool.name" => pool_name.clone(),
759    )
760    .set(stats.max_size as f64);
761    metrics::gauge!(
762        description: "Minimum Redis pool connection count",
763        unit: metrics::Unit::Count,
764        "db.client.connection.min",
765        "db.client.connection.pool.name" => pool_name.clone(),
766    )
767    .set(stats.min_size as f64);
768    metrics::gauge!(
769        description: "Redis requests waiting for a pool connection",
770        unit: metrics::Unit::Count,
771        "db.client.connection.pending_requests",
772        "db.client.connection.pool.name" => pool_name.clone(),
773    )
774    .set(pending_requests as f64);
775    metrics::gauge!(
776        description: "Redis commands active or waiting in the pool",
777        unit: metrics::Unit::Count,
778        "redis_tower.pool.inflight_commands",
779        "db.client.connection.pool.name" => pool_name.clone(),
780    )
781    .set(stats.total_inflight as f64);
782    metrics::gauge!(
783        description: "Highest in-flight command count on one Redis pool connection",
784        unit: metrics::Unit::Count,
785        "redis_tower.pool.max_inflight_per_connection",
786        "db.client.connection.pool.name" => pool_name.clone(),
787    )
788    .set(stats.max_inflight as f64);
789    metrics::gauge!(
790        description: "Redis pool connections by active-probe health state",
791        unit: metrics::Unit::Count,
792        "redis_tower.pool.health_count",
793        "db.client.connection.pool.name" => pool_name.clone(),
794        "state" => "healthy",
795    )
796    .set(stats.healthy_count as f64);
797    metrics::gauge!(
798        description: "Redis pool connections by active-probe health state",
799        unit: metrics::Unit::Count,
800        "redis_tower.pool.health_count",
801        "db.client.connection.pool.name" => pool_name.clone(),
802        "state" => "unhealthy",
803    )
804    .set(stats.unhealthy_count as f64);
805    metrics::gauge!(
806        description: "Redis pool connections by active-probe health state",
807        unit: metrics::Unit::Count,
808        "redis_tower.pool.health_count",
809        "db.client.connection.pool.name" => pool_name.clone(),
810        "state" => "unknown",
811    )
812    .set(stats.unknown_health_count as f64);
813    metrics::gauge!(
814        description: "Cumulative Redis pool connections removed by idle reaping",
815        unit: metrics::Unit::Count,
816        "redis_tower.pool.connections_reaped_total",
817        "db.client.connection.pool.name" => pool_name.clone(),
818    )
819    .set(stats.reaped_connections as f64);
820    metrics::gauge!(
821        description: "Maximum latest Redis replication lag across pool slots",
822        unit: metrics::Unit::Bytes,
823        "redis_tower.pool.max_replication_lag",
824        "db.client.connection.pool.name" => pool_name.clone(),
825    )
826    .set(stats.max_replication_lag_bytes.unwrap_or(0) as f64);
827    metrics::gauge!(
828        description: "Whether the maximum Redis pool replication-lag gauge has a current observation",
829        unit: metrics::Unit::Count,
830        "redis_tower.pool.max_replication_lag_observed",
831        "db.client.connection.pool.name" => pool_name,
832    )
833    .set(if stats.max_replication_lag_bytes.is_some() {
834        1.0
835    } else {
836        0.0
837    });
838}
839
840#[cfg(feature = "metrics")]
841fn record_queue_depth(pipeline_name: &str, queue_depth: usize) {
842    metrics::gauge!(
843        description: "Redis auto-pipeline requests waiting in the internal queue",
844        unit: metrics::Unit::Count,
845        "redis_tower.pipeline.queue_depth",
846        "redis_tower.pipeline.name" => pipeline_name.to_owned(),
847    )
848    .set(queue_depth as f64);
849}
850
851/// Spawn a task that periodically exports a connection pool's utilization.
852///
853/// The first snapshot is emitted immediately. Subsequent snapshots are
854/// emitted every `interval`. The exporter uses the pool's configured name as
855/// the `db.client.connection.pool.name` label and records idle/used connection
856/// counts, configured capacity, pending requests, and in-flight work through
857/// the global `metrics` recorder.
858///
859/// # Panics
860///
861/// Panics if `interval` is zero or if called outside a Tokio runtime.
862#[cfg(feature = "metrics")]
863#[cfg_attr(docsrs, doc(cfg(feature = "metrics")))]
864pub fn spawn_pool_stats_exporter<S>(
865    pool: ConnectionPool<S>,
866    interval: Duration,
867) -> MetricsExporterHandle
868where
869    S: Send + 'static,
870{
871    let pool_name = pool.name().to_owned();
872    MetricsExporterHandle::spawn(interval, move || {
873        record_pool_stats(&pool_name, &pool.stats())
874    })
875}
876
877/// Spawn a task that periodically exports an auto-pipeline's queue depth.
878///
879/// The first snapshot is emitted immediately. Subsequent snapshots are
880/// emitted every `interval` as `redis_tower.pipeline.queue_depth`, labeled by
881/// the supplied stable `pipeline_name`. The exporter keeps a lightweight clone
882/// of the client alive until it is dropped or shut down.
883///
884/// # Panics
885///
886/// Panics if `interval` is zero or if called outside a Tokio runtime.
887#[cfg(feature = "metrics")]
888#[cfg_attr(docsrs, doc(cfg(feature = "metrics")))]
889pub fn spawn_queue_depth_exporter(
890    client: MultiplexedClient,
891    pipeline_name: impl Into<String>,
892    interval: Duration,
893) -> MetricsExporterHandle {
894    let pipeline_name = pipeline_name.into();
895    MetricsExporterHandle::spawn(interval, move || {
896        record_queue_depth(&pipeline_name, client.queue_depth())
897    })
898}
899
900/// Tower `Layer` that produces [`MetricsService`] wrappers.
901///
902/// Wraps each inner service with a [`MetricsService`] that records
903/// per-command latency and error category via the provided
904/// [`MetricsRecorder`].
905///
906/// # Example
907///
908/// ```no_run
909/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
910/// use tower::ServiceBuilder;
911/// use redis_tower::FrameService;
912/// use redis_tower::metrics_layer::MetricsLayer;
913/// #
914/// # use std::time::Duration;
915/// # use redis_tower::metrics_layer::{ErrorKind, MetricsRecorder};
916/// # struct MyRecorder;
917/// # impl MetricsRecorder for MyRecorder {
918/// #     fn command_completed(&self, command: &str, duration: Duration, error: Option<ErrorKind>) {
919/// #         let _ = (command, duration, error);
920/// #     }
921/// # }
922///
923/// let frame_service = FrameService::connect("127.0.0.1:6379").await?;
924/// let layer = MetricsLayer::new(MyRecorder);
925/// let svc = ServiceBuilder::new()
926///     .layer(layer)
927///     .service(frame_service);
928/// # let _ = svc;
929/// # Ok(())
930/// # }
931/// ```
932pub struct MetricsLayer<R> {
933    recorder: Arc<R>,
934}
935
936impl<R: MetricsRecorder> MetricsLayer<R> {
937    /// Create a new metrics layer with the given recorder.
938    pub fn new(recorder: R) -> Self {
939        Self {
940            recorder: Arc::new(recorder),
941        }
942    }
943}
944
945impl<R: MetricsRecorder, S> tower_layer::Layer<S> for MetricsLayer<R> {
946    type Service = MetricsService<S, R>;
947
948    fn layer(&self, inner: S) -> Self::Service {
949        MetricsService {
950            inner,
951            recorder: Arc::clone(&self.recorder),
952        }
953    }
954}
955
956/// Tower `Service` that records per-command metrics via a [`MetricsRecorder`].
957///
958/// Created by [`MetricsLayer`] or directly via [`MetricsService::new`].
959/// Extracts the command name from each request frame and reports the
960/// command name, wall-clock duration, and error category after each call
961/// completes.
962pub struct MetricsService<S, R> {
963    inner: S,
964    recorder: Arc<R>,
965}
966
967impl<S, R> Clone for MetricsService<S, R>
968where
969    S: Clone,
970{
971    fn clone(&self) -> Self {
972        Self {
973            inner: self.inner.clone(),
974            recorder: Arc::clone(&self.recorder),
975        }
976    }
977}
978
979impl<S, R> MetricsService<S, R> {
980    /// Create a new metrics service wrapping an inner Frame service.
981    pub fn new(inner: S, recorder: Arc<R>) -> Self {
982        Self { inner, recorder }
983    }
984}
985
986/// Extract the command name from a Redis command frame.
987///
988/// Expects `Frame::Array(Some(vec))` where the first element is
989/// `Frame::BulkString(Some(bytes))`.
990fn extract_command_name(frame: &Frame) -> Option<&str> {
991    let items = match frame {
992        Frame::Array(Some(items)) if !items.is_empty() => items,
993        _ => return None,
994    };
995    match &items[0] {
996        Frame::BulkString(Some(b)) => std::str::from_utf8(b).ok(),
997        _ => None,
998    }
999}
1000
1001impl<S, R> Service<Frame> for MetricsService<S, R>
1002where
1003    S: Service<Frame, Response = Frame, Error = RedisError>,
1004    S::Future: Send + 'static,
1005    R: MetricsRecorder,
1006{
1007    type Response = Frame;
1008    type Error = RedisError;
1009    type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
1010
1011    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
1012        self.inner.poll_ready(cx)
1013    }
1014
1015    fn call(&mut self, request: Frame) -> Self::Future {
1016        let command_name = extract_command_name(&request)
1017            .unwrap_or("UNKNOWN")
1018            .to_ascii_uppercase();
1019        let start = Instant::now();
1020        let recorder = Arc::clone(&self.recorder);
1021        let future = self.inner.call(request);
1022
1023        Box::pin(async move {
1024            let result = future.await;
1025            let elapsed = start.elapsed();
1026            let error_kind = match &result {
1027                Ok(Frame::Error(bytes)) => Some(ErrorKind::from_frame_error(bytes)),
1028                Ok(_) => None,
1029                Err(e) => Some(ErrorKind::from_error(e)),
1030            };
1031            recorder.command_completed(&command_name, elapsed, error_kind);
1032            result
1033        })
1034    }
1035}
1036
1037#[cfg(test)]
1038mod tests {
1039    use super::*;
1040    use bytes::Bytes;
1041    use std::sync::Mutex;
1042    use std::sync::atomic::{AtomicU64, Ordering};
1043
1044    #[cfg(feature = "metrics")]
1045    #[derive(Debug, Clone, PartialEq, Eq)]
1046    struct CapturedMetric {
1047        name: String,
1048        labels: Vec<(String, String)>,
1049    }
1050
1051    #[cfg(feature = "metrics")]
1052    impl CapturedMetric {
1053        fn from_key(key: &metrics::Key) -> Self {
1054            Self {
1055                name: key.name().to_owned(),
1056                labels: key
1057                    .labels()
1058                    .map(|label| (label.key().to_owned(), label.value().to_owned()))
1059                    .collect(),
1060            }
1061        }
1062
1063        fn label(&self, name: &str) -> Option<&str> {
1064            self.labels
1065                .iter()
1066                .find_map(|(key, value)| (key == name).then_some(value.as_str()))
1067        }
1068    }
1069
1070    #[cfg(feature = "metrics")]
1071    struct CapturedGauge {
1072        metric: CapturedMetric,
1073        value: Arc<Mutex<f64>>,
1074    }
1075
1076    #[cfg(feature = "metrics")]
1077    impl std::ops::Deref for CapturedGauge {
1078        type Target = CapturedMetric;
1079
1080        fn deref(&self) -> &Self::Target {
1081            &self.metric
1082        }
1083    }
1084
1085    #[cfg(feature = "metrics")]
1086    impl CapturedGauge {
1087        fn value(&self) -> f64 {
1088            *self.value.lock().unwrap()
1089        }
1090    }
1091
1092    #[cfg(feature = "metrics")]
1093    struct CapturingGauge {
1094        value: Arc<Mutex<f64>>,
1095    }
1096
1097    #[cfg(feature = "metrics")]
1098    impl metrics::GaugeFn for CapturingGauge {
1099        fn increment(&self, value: f64) {
1100            *self.value.lock().unwrap() += value;
1101        }
1102
1103        fn decrement(&self, value: f64) {
1104            *self.value.lock().unwrap() -= value;
1105        }
1106
1107        fn set(&self, value: f64) {
1108            *self.value.lock().unwrap() = value;
1109        }
1110    }
1111
1112    #[cfg(feature = "metrics")]
1113    #[derive(Default)]
1114    struct CapturingFacadeRecorder {
1115        counters: Mutex<Vec<CapturedMetric>>,
1116        gauges: Mutex<Vec<CapturedGauge>>,
1117        histograms: Mutex<Vec<CapturedMetric>>,
1118    }
1119
1120    #[cfg(feature = "metrics")]
1121    impl metrics::Recorder for CapturingFacadeRecorder {
1122        fn describe_counter(
1123            &self,
1124            _key: metrics::KeyName,
1125            _unit: Option<metrics::Unit>,
1126            _description: metrics::SharedString,
1127        ) {
1128        }
1129
1130        fn describe_gauge(
1131            &self,
1132            _key: metrics::KeyName,
1133            _unit: Option<metrics::Unit>,
1134            _description: metrics::SharedString,
1135        ) {
1136        }
1137
1138        fn describe_histogram(
1139            &self,
1140            _key: metrics::KeyName,
1141            _unit: Option<metrics::Unit>,
1142            _description: metrics::SharedString,
1143        ) {
1144        }
1145
1146        fn register_counter(
1147            &self,
1148            key: &metrics::Key,
1149            _metadata: &metrics::Metadata<'_>,
1150        ) -> metrics::Counter {
1151            self.counters
1152                .lock()
1153                .unwrap()
1154                .push(CapturedMetric::from_key(key));
1155            metrics::Counter::noop()
1156        }
1157
1158        fn register_gauge(
1159            &self,
1160            key: &metrics::Key,
1161            _metadata: &metrics::Metadata<'_>,
1162        ) -> metrics::Gauge {
1163            let value = Arc::new(Mutex::new(0.0));
1164            self.gauges.lock().unwrap().push(CapturedGauge {
1165                metric: CapturedMetric::from_key(key),
1166                value: Arc::clone(&value),
1167            });
1168            metrics::Gauge::from_arc(Arc::new(CapturingGauge { value }))
1169        }
1170
1171        fn register_histogram(
1172            &self,
1173            key: &metrics::Key,
1174            _metadata: &metrics::Metadata<'_>,
1175        ) -> metrics::Histogram {
1176            self.histograms
1177                .lock()
1178                .unwrap()
1179                .push(CapturedMetric::from_key(key));
1180            metrics::Histogram::noop()
1181        }
1182    }
1183
1184    #[test]
1185    fn extract_name_from_get() {
1186        let frame = Frame::Array(Some(vec![
1187            Frame::BulkString(Some(Bytes::from("GET"))),
1188            Frame::BulkString(Some(Bytes::from("key"))),
1189        ]));
1190        assert_eq!(extract_command_name(&frame), Some("GET"));
1191    }
1192
1193    #[test]
1194    fn extract_name_from_set() {
1195        let frame = Frame::Array(Some(vec![
1196            Frame::BulkString(Some(Bytes::from("SET"))),
1197            Frame::BulkString(Some(Bytes::from("key"))),
1198            Frame::BulkString(Some(Bytes::from("value"))),
1199        ]));
1200        assert_eq!(extract_command_name(&frame), Some("SET"));
1201    }
1202
1203    #[test]
1204    fn extract_name_empty_array() {
1205        let frame = Frame::Array(Some(vec![]));
1206        assert_eq!(extract_command_name(&frame), None);
1207    }
1208
1209    #[test]
1210    fn extract_name_null_frame() {
1211        assert_eq!(extract_command_name(&Frame::Null), None);
1212    }
1213
1214    #[test]
1215    fn extract_name_none_array() {
1216        let frame = Frame::Array(None);
1217        assert_eq!(extract_command_name(&frame), None);
1218    }
1219
1220    #[test]
1221    fn error_kind_from_error_connection() {
1222        assert_eq!(
1223            ErrorKind::from_error(&RedisError::ConnectionClosed),
1224            ErrorKind::Connection
1225        );
1226    }
1227
1228    #[test]
1229    fn error_kind_from_error_timeout() {
1230        assert_eq!(
1231            ErrorKind::from_error(&RedisError::PoolAcquisitionTimeout {
1232                waited: std::time::Duration::from_millis(50),
1233                pool_size: 4,
1234            }),
1235            ErrorKind::Timeout
1236        );
1237    }
1238
1239    #[test]
1240    fn error_kind_from_error_wrongtype() {
1241        assert_eq!(
1242            ErrorKind::from_error(&RedisError::Redis(
1243                "WRONGTYPE Operation against a key holding the wrong kind of value".into()
1244            )),
1245            ErrorKind::WrongType
1246        );
1247    }
1248
1249    #[test]
1250    fn error_kind_from_error_circuit_open() {
1251        assert_eq!(
1252            ErrorKind::from_error(&RedisError::CircuitOpen),
1253            ErrorKind::CircuitOpen
1254        );
1255    }
1256
1257    #[test]
1258    fn error_kind_from_error_queue_full() {
1259        assert_eq!(
1260            ErrorKind::from_error(&RedisError::QueueFull),
1261            ErrorKind::QueueFull
1262        );
1263    }
1264
1265    #[test]
1266    fn error_kind_from_error_auth_noauth() {
1267        assert_eq!(
1268            ErrorKind::from_error(&RedisError::Redis("NOAUTH Authentication required".into())),
1269            ErrorKind::Auth
1270        );
1271    }
1272
1273    #[test]
1274    fn error_kind_from_error_auth_wrongpass() {
1275        assert_eq!(
1276            ErrorKind::from_error(&RedisError::Redis(
1277                "WRONGPASS invalid username-password pair".into()
1278            )),
1279            ErrorKind::Auth
1280        );
1281    }
1282
1283    #[test]
1284    fn error_kind_from_error_auth_provider() {
1285        assert_eq!(
1286            ErrorKind::from_error(&RedisError::Redis(
1287                "AUTH_PROVIDER current credential lookup failed".into()
1288            )),
1289            ErrorKind::Auth
1290        );
1291    }
1292
1293    #[test]
1294    fn error_kind_from_error_other() {
1295        assert_eq!(
1296            ErrorKind::from_error(&RedisError::Redis("ERR unknown command".into())),
1297            ErrorKind::Other
1298        );
1299    }
1300
1301    #[test]
1302    fn error_kind_from_frame_error_wrongtype() {
1303        assert_eq!(
1304            ErrorKind::from_frame_error(&Bytes::from("WRONGTYPE value is not a set")),
1305            ErrorKind::WrongType
1306        );
1307    }
1308
1309    #[test]
1310    fn error_kind_from_frame_error_other() {
1311        assert_eq!(
1312            ErrorKind::from_frame_error(&Bytes::from("ERR syntax error")),
1313            ErrorKind::Other
1314        );
1315    }
1316
1317    struct TestRecorder {
1318        error_kind: Mutex<Option<Option<ErrorKind>>>,
1319        duration_ns: AtomicU64,
1320    }
1321
1322    impl TestRecorder {
1323        fn new() -> Self {
1324            Self {
1325                error_kind: Mutex::new(None),
1326                duration_ns: AtomicU64::new(0),
1327            }
1328        }
1329
1330        fn was_called(&self) -> bool {
1331            self.error_kind.lock().unwrap().is_some()
1332        }
1333
1334        fn recorded_error(&self) -> Option<ErrorKind> {
1335            self.error_kind.lock().unwrap().flatten()
1336        }
1337    }
1338
1339    impl MetricsRecorder for TestRecorder {
1340        fn command_completed(&self, _command: &str, duration: Duration, error: Option<ErrorKind>) {
1341            *self.error_kind.lock().unwrap() = Some(error);
1342            self.duration_ns
1343                .store(duration.as_nanos() as u64, Ordering::SeqCst);
1344        }
1345    }
1346
1347    #[test]
1348    fn custom_recorders_inherit_noop_pool_hooks() {
1349        let recorder = TestRecorder::new();
1350        recorder.pool_acquisition_completed("default", Duration::from_millis(2), false);
1351        recorder.pool_health_check_failed("default");
1352        recorder.pool_connection_replaced("default");
1353        assert!(!recorder.was_called());
1354    }
1355
1356    #[test]
1357    fn existing_recorders_receive_node_completions_through_the_original_hook() {
1358        let recorder = TestRecorder::new();
1359        recorder.command_completed_on_node(
1360            "GET",
1361            Duration::from_millis(2),
1362            Some(ErrorKind::Connection),
1363            Some("127.0.0.1:7000"),
1364        );
1365        recorder.cluster_redirected(ClusterRedirectKind::Moved);
1366        recorder.cluster_topology_refresh_completed(
1367            Duration::from_millis(3),
1368            ClusterTopologyRefreshOutcome::Success,
1369        );
1370
1371        assert!(recorder.was_called());
1372        assert_eq!(recorder.recorded_error(), Some(ErrorKind::Connection));
1373    }
1374
1375    #[derive(Default)]
1376    struct ClusterHookRecorder {
1377        node: Mutex<Option<String>>,
1378        redirect: Mutex<Option<ClusterRedirectKind>>,
1379        refresh: Mutex<Option<(Duration, ClusterTopologyRefreshOutcome)>>,
1380    }
1381
1382    impl MetricsRecorder for ClusterHookRecorder {
1383        fn command_completed(
1384            &self,
1385            _command: &str,
1386            _duration: Duration,
1387            _error: Option<ErrorKind>,
1388        ) {
1389        }
1390
1391        fn command_completed_on_node(
1392            &self,
1393            _command: &str,
1394            _duration: Duration,
1395            _error: Option<ErrorKind>,
1396            node: Option<&str>,
1397        ) {
1398            *self.node.lock().unwrap() = node.map(str::to_owned);
1399        }
1400
1401        fn cluster_redirected(&self, kind: ClusterRedirectKind) {
1402            *self.redirect.lock().unwrap() = Some(kind);
1403        }
1404
1405        fn cluster_topology_refresh_completed(
1406            &self,
1407            duration: Duration,
1408            outcome: ClusterTopologyRefreshOutcome,
1409        ) {
1410            *self.refresh.lock().unwrap() = Some((duration, outcome));
1411        }
1412    }
1413
1414    #[test]
1415    fn arc_delegates_cluster_hooks_to_its_metrics_recorder() {
1416        let recorder = Arc::new(ClusterHookRecorder::default());
1417
1418        MetricsRecorder::command_completed_on_node(
1419            &recorder,
1420            "SET",
1421            Duration::from_millis(1),
1422            None,
1423            Some("127.0.0.1:7001"),
1424        );
1425        MetricsRecorder::cluster_redirected(&recorder, ClusterRedirectKind::Ask);
1426        MetricsRecorder::cluster_topology_refresh_completed(
1427            &recorder,
1428            Duration::from_millis(4),
1429            ClusterTopologyRefreshOutcome::Partial,
1430        );
1431
1432        assert_eq!(
1433            recorder.node.lock().unwrap().as_deref(),
1434            Some("127.0.0.1:7001")
1435        );
1436        assert_eq!(
1437            *recorder.redirect.lock().unwrap(),
1438            Some(ClusterRedirectKind::Ask)
1439        );
1440        assert_eq!(
1441            *recorder.refresh.lock().unwrap(),
1442            Some((
1443                Duration::from_millis(4),
1444                ClusterTopologyRefreshOutcome::Partial
1445            ))
1446        );
1447    }
1448
1449    #[test]
1450    fn arc_delegates_to_its_metrics_recorder() {
1451        let recorder = Arc::new(TestRecorder::new());
1452        MetricsRecorder::command_completed(
1453            &recorder,
1454            "GET",
1455            Duration::from_millis(2),
1456            Some(ErrorKind::Connection),
1457        );
1458        assert!(recorder.was_called());
1459        assert_eq!(recorder.recorded_error(), Some(ErrorKind::Connection));
1460    }
1461
1462    #[cfg(feature = "metrics")]
1463    #[test]
1464    fn facade_recorder_uses_semantic_convention_labels() {
1465        let capture = CapturingFacadeRecorder::default();
1466        let recorder = MetricsFacadeRecorder::new();
1467        metrics::with_local_recorder(&capture, || {
1468            recorder.command_completed("GET", Duration::from_millis(2), None);
1469            recorder.command_completed(
1470                "SET",
1471                Duration::from_millis(3),
1472                Some(ErrorKind::Connection),
1473            );
1474            recorder.command_completed("CUSTOM", Duration::from_millis(1), Some(ErrorKind::Other));
1475        });
1476
1477        let histograms = capture.histograms.lock().unwrap();
1478        let success = &histograms[0];
1479        assert_eq!(success.name, "db.client.operation.duration");
1480        assert_eq!(success.label("db.system.name"), Some("redis"));
1481        assert_eq!(success.label("db.operation.name"), Some("GET"));
1482        assert_eq!(success.label("error.type"), None);
1483
1484        let failure = &histograms[1];
1485        assert_eq!(failure.label("db.operation.name"), Some("SET"));
1486        assert_eq!(failure.label("error.type"), Some("connection"));
1487
1488        let fallback = &histograms[2];
1489        assert_eq!(fallback.label("db.operation.name"), Some("CUSTOM"));
1490        assert_eq!(fallback.label("error.type"), Some("_OTHER"));
1491
1492        let counters = capture.counters.lock().unwrap();
1493        assert_eq!(counters[0].label("outcome"), Some("success"));
1494        assert_eq!(counters[1].label("outcome"), Some("error"));
1495        assert_eq!(counters[1].label("error.type"), Some("connection"));
1496        assert_eq!(counters[2].label("error.type"), Some("_OTHER"));
1497    }
1498
1499    #[cfg(feature = "metrics")]
1500    #[test]
1501    fn facade_recorder_adds_the_optional_cluster_node_label() {
1502        let capture = CapturingFacadeRecorder::default();
1503        let recorder = MetricsFacadeRecorder::new();
1504        metrics::with_local_recorder(&capture, || {
1505            recorder.command_completed_on_node(
1506                "GET",
1507                Duration::from_millis(2),
1508                None,
1509                Some("127.0.0.1:7000"),
1510            );
1511            recorder.command_completed_on_node(
1512                "SET",
1513                Duration::from_millis(3),
1514                Some(ErrorKind::Connection),
1515                None,
1516            );
1517        });
1518
1519        let histograms = capture.histograms.lock().unwrap();
1520        assert_eq!(
1521            histograms[0].label("redis_tower.cluster.node"),
1522            Some("127.0.0.1:7000")
1523        );
1524        assert_eq!(histograms[1].label("redis_tower.cluster.node"), None);
1525
1526        let counters = capture.counters.lock().unwrap();
1527        assert_eq!(
1528            counters[0].label("redis_tower.cluster.node"),
1529            Some("127.0.0.1:7000")
1530        );
1531        assert_eq!(counters[1].label("redis_tower.cluster.node"), None);
1532    }
1533
1534    #[cfg(feature = "metrics")]
1535    #[test]
1536    fn facade_recorder_emits_bounded_cluster_event_labels() {
1537        let capture = CapturingFacadeRecorder::default();
1538        let recorder = MetricsFacadeRecorder::new();
1539        metrics::with_local_recorder(&capture, || {
1540            recorder.cluster_redirected(ClusterRedirectKind::Moved);
1541            recorder.cluster_redirected(ClusterRedirectKind::Ask);
1542            recorder.cluster_topology_refresh_completed(
1543                Duration::from_millis(3),
1544                ClusterTopologyRefreshOutcome::Success,
1545            );
1546            recorder.cluster_topology_refresh_completed(
1547                Duration::from_millis(4),
1548                ClusterTopologyRefreshOutcome::Partial,
1549            );
1550            recorder.cluster_topology_refresh_completed(
1551                Duration::from_millis(5),
1552                ClusterTopologyRefreshOutcome::Error,
1553            );
1554        });
1555
1556        let counters = capture.counters.lock().unwrap();
1557        let redirect_kinds: Vec<_> = counters
1558            .iter()
1559            .filter(|metric| metric.name == "redis_tower.cluster.redirects")
1560            .map(|metric| metric.label("kind").unwrap())
1561            .collect();
1562        assert_eq!(redirect_kinds, ["moved", "ask"]);
1563
1564        let refresh_outcomes: Vec<_> = counters
1565            .iter()
1566            .filter(|metric| metric.name == "redis_tower.cluster.topology_refreshes")
1567            .map(|metric| metric.label("outcome").unwrap())
1568            .collect();
1569        assert_eq!(refresh_outcomes, ["success", "partial", "error"]);
1570
1571        let histograms = capture.histograms.lock().unwrap();
1572        let duration_outcomes: Vec<_> = histograms
1573            .iter()
1574            .filter(|metric| metric.name == "redis_tower.cluster.topology_refresh.duration")
1575            .map(|metric| metric.label("outcome").unwrap())
1576            .collect();
1577        assert_eq!(duration_outcomes, ["success", "partial", "error"]);
1578    }
1579
1580    #[cfg(feature = "metrics")]
1581    #[test]
1582    fn pending_requests_exclude_connections_doing_work() {
1583        let stats = PoolStats {
1584            size: 4,
1585            idle_count: 1,
1586            total_inflight: 7,
1587            max_inflight: 3,
1588            min_size: 2,
1589            max_size: 8,
1590            healthy_count: 2,
1591            unhealthy_count: 1,
1592            unknown_health_count: 1,
1593            max_replication_lag_bytes: Some(42),
1594            reaped_connections: 5,
1595        };
1596        assert_eq!(pool_gauge_values(&stats), (3, 4));
1597    }
1598
1599    #[cfg(feature = "metrics")]
1600    #[test]
1601    fn pool_stats_use_canonical_names_and_labels() {
1602        let capture = CapturingFacadeRecorder::default();
1603        let stats = PoolStats {
1604            size: 4,
1605            idle_count: 1,
1606            total_inflight: 7,
1607            max_inflight: 3,
1608            min_size: 2,
1609            max_size: 8,
1610            healthy_count: 2,
1611            unhealthy_count: 1,
1612            unknown_health_count: 1,
1613            max_replication_lag_bytes: Some(42),
1614            reaped_connections: 5,
1615        };
1616        metrics::with_local_recorder(&capture, || record_pool_stats("primary", &stats));
1617
1618        let gauges = capture.gauges.lock().unwrap();
1619        let idle = gauges
1620            .iter()
1621            .find(|metric| {
1622                metric.name == "db.client.connection.count"
1623                    && metric.label("db.client.connection.state") == Some("idle")
1624            })
1625            .unwrap();
1626        assert_eq!(
1627            idle.label("db.client.connection.pool.name"),
1628            Some("primary")
1629        );
1630        assert_eq!(idle.value(), 1.0);
1631        let max = gauges
1632            .iter()
1633            .find(|metric| metric.name == "db.client.connection.max")
1634            .unwrap();
1635        assert_eq!(max.value(), 8.0);
1636        let pending = gauges
1637            .iter()
1638            .find(|metric| metric.name == "db.client.connection.pending_requests")
1639            .unwrap();
1640        assert_eq!(pending.value(), 4.0);
1641    }
1642
1643    #[cfg(feature = "metrics")]
1644    #[test]
1645    fn unknown_replication_lag_resets_value_and_freshness_gauges() {
1646        let capture = CapturingFacadeRecorder::default();
1647        let recorder = MetricsFacadeRecorder::new();
1648        metrics::with_local_recorder(&capture, || {
1649            recorder.pool_health_probe_completed(
1650                "primary",
1651                HealthProbeKind::ReplicationLag,
1652                Duration::from_millis(1),
1653                true,
1654                Some(42),
1655            );
1656            recorder.pool_health_probe_completed(
1657                "primary",
1658                HealthProbeKind::ReplicationLag,
1659                Duration::from_millis(1),
1660                false,
1661                None,
1662            );
1663        });
1664
1665        let gauges = capture.gauges.lock().unwrap();
1666        let lag = gauges
1667            .iter()
1668            .rfind(|metric| metric.name == "redis_tower.pool.replication_lag")
1669            .unwrap();
1670        let observed = gauges
1671            .iter()
1672            .rfind(|metric| metric.name == "redis_tower.pool.replication_lag_observed")
1673            .unwrap();
1674        assert_eq!(lag.value(), 0.0);
1675        assert_eq!(observed.value(), 0.0);
1676    }
1677
1678    #[cfg(feature = "metrics")]
1679    #[test]
1680    fn queue_depth_uses_a_named_gauge() {
1681        let capture = CapturingFacadeRecorder::default();
1682        metrics::with_local_recorder(&capture, || record_queue_depth("commands", 7));
1683
1684        let gauges = capture.gauges.lock().unwrap();
1685        let queue = gauges
1686            .iter()
1687            .find(|metric| metric.name == "redis_tower.pipeline.queue_depth")
1688            .unwrap();
1689        assert_eq!(queue.label("redis_tower.pipeline.name"), Some("commands"));
1690        assert_eq!(queue.value(), 7.0);
1691    }
1692
1693    #[cfg(feature = "metrics")]
1694    #[tokio::test]
1695    async fn pool_stats_exporter_shuts_down_without_redis() {
1696        let pool =
1697            ConnectionPool::from_connections(vec![(), ()], crate::pool::PoolConfig::default())
1698                .unwrap();
1699        let exporter = spawn_pool_stats_exporter(pool, Duration::from_millis(1));
1700        tokio::task::yield_now().await;
1701        exporter.shutdown().await;
1702    }
1703
1704    #[cfg(feature = "metrics")]
1705    #[tokio::test]
1706    async fn exporter_shutdown_emits_a_final_snapshot() {
1707        let snapshots = Arc::new(std::sync::atomic::AtomicUsize::new(0));
1708        let task_snapshots = Arc::clone(&snapshots);
1709        let exporter = MetricsExporterHandle::spawn(Duration::from_secs(60), move || {
1710            task_snapshots.fetch_add(1, Ordering::SeqCst);
1711        });
1712
1713        while snapshots.load(Ordering::SeqCst) == 0 {
1714            tokio::task::yield_now().await;
1715        }
1716        let before_shutdown = snapshots.load(Ordering::SeqCst);
1717
1718        exporter.shutdown().await;
1719
1720        assert_eq!(snapshots.load(Ordering::SeqCst), before_shutdown + 1);
1721    }
1722
1723    /// A mock service that always returns SimpleString "OK".
1724    #[derive(Clone)]
1725    struct OkService;
1726
1727    impl Service<Frame> for OkService {
1728        type Response = Frame;
1729        type Error = RedisError;
1730        type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
1731
1732        fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
1733            Poll::Ready(Ok(()))
1734        }
1735
1736        fn call(&mut self, _request: Frame) -> Self::Future {
1737            Box::pin(async { Ok(Frame::SimpleString("OK".into())) })
1738        }
1739    }
1740
1741    /// A mock service that always returns a transport error.
1742    struct ErrService;
1743
1744    impl Service<Frame> for ErrService {
1745        type Response = Frame;
1746        type Error = RedisError;
1747        type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
1748
1749        fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
1750            Poll::Ready(Ok(()))
1751        }
1752
1753        fn call(&mut self, _request: Frame) -> Self::Future {
1754            Box::pin(async { Err(RedisError::ConnectionClosed) })
1755        }
1756    }
1757
1758    /// A mock service that returns a Frame::Error (Redis-level error).
1759    struct FrameErrService {
1760        msg: &'static str,
1761    }
1762
1763    impl Service<Frame> for FrameErrService {
1764        type Response = Frame;
1765        type Error = RedisError;
1766        type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
1767
1768        fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
1769            Poll::Ready(Ok(()))
1770        }
1771
1772        fn call(&mut self, _request: Frame) -> Self::Future {
1773            let msg = self.msg;
1774            Box::pin(async move { Ok(Frame::Error(Bytes::from(msg))) })
1775        }
1776    }
1777
1778    #[tokio::test]
1779    async fn records_success() {
1780        let recorder = Arc::new(TestRecorder::new());
1781        let mut svc = MetricsService::new(OkService, Arc::clone(&recorder));
1782
1783        let frame = Frame::Array(Some(vec![
1784            Frame::BulkString(Some(Bytes::from("GET"))),
1785            Frame::BulkString(Some(Bytes::from("key"))),
1786        ]));
1787
1788        let result = svc.call(frame).await;
1789        assert!(result.is_ok());
1790        assert!(recorder.was_called());
1791        assert_eq!(recorder.recorded_error(), None);
1792        assert!(recorder.duration_ns.load(Ordering::SeqCst) > 0);
1793    }
1794
1795    #[tokio::test]
1796    async fn cloned_services_share_a_non_clone_recorder() {
1797        let recorder = Arc::new(TestRecorder::new());
1798        let svc = MetricsService::new(OkService, Arc::clone(&recorder));
1799        let mut cloned = svc.clone();
1800
1801        let frame = Frame::Array(Some(vec![Frame::BulkString(Some(Bytes::from("PING")))]));
1802        assert!(cloned.call(frame).await.is_ok());
1803        assert!(recorder.was_called());
1804    }
1805
1806    #[tokio::test]
1807    async fn records_failure_connection_closed() {
1808        let recorder = Arc::new(TestRecorder::new());
1809        let mut svc = MetricsService::new(ErrService, Arc::clone(&recorder));
1810
1811        let frame = Frame::Array(Some(vec![
1812            Frame::BulkString(Some(Bytes::from("SET"))),
1813            Frame::BulkString(Some(Bytes::from("key"))),
1814            Frame::BulkString(Some(Bytes::from("val"))),
1815        ]));
1816
1817        let result = svc.call(frame).await;
1818        assert!(result.is_err());
1819        assert!(recorder.was_called());
1820        assert_eq!(recorder.recorded_error(), Some(ErrorKind::Connection));
1821    }
1822
1823    #[tokio::test]
1824    async fn frame_error_is_classified_as_failure() {
1825        let recorder = Arc::new(TestRecorder::new());
1826        let mut svc = MetricsService::new(
1827            FrameErrService {
1828                msg: "WRONGTYPE Operation against a key holding the wrong kind of value",
1829            },
1830            Arc::clone(&recorder),
1831        );
1832
1833        let frame = Frame::Array(Some(vec![
1834            Frame::BulkString(Some(Bytes::from("SADD"))),
1835            Frame::BulkString(Some(Bytes::from("key"))),
1836            Frame::BulkString(Some(Bytes::from("member"))),
1837        ]));
1838
1839        // The service returns Ok(Frame::Error(...)) -- a Redis-level error,
1840        // not a transport error. The Result is Ok.
1841        let result = svc.call(frame).await;
1842        assert!(result.is_ok());
1843        assert!(recorder.was_called());
1844        // But the recorder should classify it as a WrongType failure.
1845        assert_eq!(recorder.recorded_error(), Some(ErrorKind::WrongType));
1846    }
1847
1848    #[tokio::test]
1849    async fn frame_error_generic_classified_as_other() {
1850        let recorder = Arc::new(TestRecorder::new());
1851        let mut svc = MetricsService::new(
1852            FrameErrService {
1853                msg: "ERR syntax error",
1854            },
1855            Arc::clone(&recorder),
1856        );
1857
1858        let frame = Frame::Array(Some(vec![
1859            Frame::BulkString(Some(Bytes::from("GET"))),
1860            Frame::BulkString(Some(Bytes::from("key"))),
1861        ]));
1862
1863        let result = svc.call(frame).await;
1864        assert!(result.is_ok());
1865        assert_eq!(recorder.recorded_error(), Some(ErrorKind::Other));
1866    }
1867}