1use std::future::Future;
28use std::pin::Pin;
29use std::sync::Arc;
30use std::task::{Context, Poll};
31use std::time::{Duration, Instant};
32
33use redis_tower_core::{Frame, RedisError};
34use tower_service::Service;
35
36#[cfg(feature = "metrics")]
37use crate::multiplexed::MultiplexedClient;
38use crate::pool::HealthProbeKind;
39#[cfg(feature = "metrics")]
40use crate::pool::{ConnectionPool, PoolStats};
41
42#[derive(Debug, Clone, Copy, PartialEq, Eq)]
60pub enum ErrorKind {
61 Connection,
63 Timeout,
65 WrongType,
67 CircuitOpen,
69 QueueFull,
71 Auth,
73 Other,
75}
76
77impl ErrorKind {
78 pub fn from_error(e: &RedisError) -> Self {
80 match e {
81 RedisError::Connection { .. } | RedisError::ConnectionClosed => ErrorKind::Connection,
82 RedisError::PoolAcquisitionTimeout { .. } => ErrorKind::Timeout,
83 RedisError::CircuitOpen => ErrorKind::CircuitOpen,
84 RedisError::QueueFull => ErrorKind::QueueFull,
85 RedisError::Redis(msg) => {
86 if msg.starts_with("WRONGTYPE") {
87 ErrorKind::WrongType
88 } else if msg.starts_with("AUTH_PROVIDER")
89 || msg.starts_with("NOAUTH")
90 || msg.starts_with("WRONGPASS")
91 {
92 ErrorKind::Auth
93 } else {
94 ErrorKind::Other
95 }
96 }
97 _ => ErrorKind::Other,
98 }
99 }
100
101 fn from_frame_error(bytes: &bytes::Bytes) -> Self {
104 let msg = std::str::from_utf8(bytes).unwrap_or("");
105 if msg.starts_with("WRONGTYPE") {
106 ErrorKind::WrongType
107 } else if msg.starts_with("NOAUTH") || msg.starts_with("WRONGPASS") {
108 ErrorKind::Auth
109 } else {
110 ErrorKind::Other
111 }
112 }
113
114 #[cfg(feature = "metrics")]
115 fn as_label(self) -> &'static str {
116 match self {
117 Self::Connection => "connection",
118 Self::Timeout => "timeout",
119 Self::WrongType => "wrong_type",
120 Self::CircuitOpen => "circuit_open",
121 Self::QueueFull => "queue_full",
122 Self::Auth => "auth",
123 Self::Other => "_OTHER",
124 }
125 }
126}
127
128#[derive(Debug, Clone, Copy, PartialEq, Eq)]
135pub enum ClusterRedirectKind {
136 Moved,
138 Ask,
140}
141
142impl ClusterRedirectKind {
143 #[cfg(feature = "metrics")]
144 fn as_label(self) -> &'static str {
145 match self {
146 Self::Moved => "moved",
147 Self::Ask => "ask",
148 }
149 }
150}
151
152#[derive(Debug, Clone, Copy, PartialEq, Eq)]
158pub enum ClusterTopologyRefreshOutcome {
159 Success,
161 Partial,
164 Error,
166}
167
168#[derive(Debug, Clone, Copy, PartialEq, Eq)]
175pub enum CacheEvent {
176 Hit,
178 Miss,
180 Invalidation,
182 Eviction,
184}
185
186impl CacheEvent {
187 #[cfg(feature = "metrics")]
188 fn as_label(self) -> &'static str {
189 match self {
190 Self::Hit => "hit",
191 Self::Miss => "miss",
192 Self::Invalidation => "invalidation",
193 Self::Eviction => "eviction",
194 }
195 }
196}
197
198impl ClusterTopologyRefreshOutcome {
199 #[cfg(feature = "metrics")]
200 fn as_label(self) -> &'static str {
201 match self {
202 Self::Success => "success",
203 Self::Partial => "partial",
204 Self::Error => "error",
205 }
206 }
207}
208
209pub trait MetricsRecorder: Send + Sync + 'static {
213 fn command_completed(&self, command: &str, duration: Duration, error: Option<ErrorKind>);
225
226 fn command_completed_on_node(
238 &self,
239 command: &str,
240 duration: Duration,
241 error: Option<ErrorKind>,
242 node: Option<&str>,
243 ) {
244 let _ = node;
245 self.command_completed(command, duration, error);
246 }
247
248 fn cluster_redirected(&self, kind: ClusterRedirectKind) {
256 let _ = kind;
257 }
258
259 fn cluster_topology_refresh_completed(
266 &self,
267 duration: Duration,
268 outcome: ClusterTopologyRefreshOutcome,
269 ) {
270 let _ = (duration, outcome);
271 }
272
273 fn pipeline_flushed(&self, batch_size: usize) {
283 let _ = batch_size;
284 }
285
286 fn cache_event(&self, event: CacheEvent, count: u64) {
294 let _ = (event, count);
295 }
296
297 fn pool_acquisition_completed(&self, pool_name: &str, duration: Duration, timed_out: bool) {
307 let _ = (pool_name, duration, timed_out);
308 }
309
310 fn pool_health_check_failed(&self, pool_name: &str) {
314 let _ = pool_name;
315 }
316
317 fn pool_connection_replaced(&self, pool_name: &str) {
321 let _ = pool_name;
322 }
323
324 fn pool_health_probe_completed(
329 &self,
330 pool_name: &str,
331 kind: HealthProbeKind,
332 duration: Duration,
333 healthy: bool,
334 replication_lag_bytes: Option<u64>,
335 ) {
336 let _ = (pool_name, kind, duration, healthy, replication_lag_bytes);
337 }
338
339 fn pool_connections_reaped(&self, pool_name: &str, count: usize) {
341 let _ = (pool_name, count);
342 }
343}
344
345impl<R> MetricsRecorder for Arc<R>
346where
347 R: MetricsRecorder + ?Sized,
348{
349 fn command_completed(&self, command: &str, duration: Duration, error: Option<ErrorKind>) {
350 (**self).command_completed(command, duration, error);
351 }
352
353 fn command_completed_on_node(
354 &self,
355 command: &str,
356 duration: Duration,
357 error: Option<ErrorKind>,
358 node: Option<&str>,
359 ) {
360 (**self).command_completed_on_node(command, duration, error, node);
361 }
362
363 fn cluster_redirected(&self, kind: ClusterRedirectKind) {
364 (**self).cluster_redirected(kind);
365 }
366
367 fn cluster_topology_refresh_completed(
368 &self,
369 duration: Duration,
370 outcome: ClusterTopologyRefreshOutcome,
371 ) {
372 (**self).cluster_topology_refresh_completed(duration, outcome);
373 }
374
375 fn pipeline_flushed(&self, batch_size: usize) {
376 (**self).pipeline_flushed(batch_size);
377 }
378
379 fn cache_event(&self, event: CacheEvent, count: u64) {
380 (**self).cache_event(event, count);
381 }
382
383 fn pool_acquisition_completed(&self, pool_name: &str, duration: Duration, timed_out: bool) {
384 (**self).pool_acquisition_completed(pool_name, duration, timed_out);
385 }
386
387 fn pool_health_check_failed(&self, pool_name: &str) {
388 (**self).pool_health_check_failed(pool_name);
389 }
390
391 fn pool_connection_replaced(&self, pool_name: &str) {
392 (**self).pool_connection_replaced(pool_name);
393 }
394
395 fn pool_health_probe_completed(
396 &self,
397 pool_name: &str,
398 kind: HealthProbeKind,
399 duration: Duration,
400 healthy: bool,
401 replication_lag_bytes: Option<u64>,
402 ) {
403 (**self).pool_health_probe_completed(
404 pool_name,
405 kind,
406 duration,
407 healthy,
408 replication_lag_bytes,
409 );
410 }
411
412 fn pool_connections_reaped(&self, pool_name: &str, count: usize) {
413 (**self).pool_connections_reaped(pool_name, count);
414 }
415}
416
417#[cfg(feature = "metrics")]
432#[cfg_attr(docsrs, doc(cfg(feature = "metrics")))]
433#[derive(Debug, Default, Clone, Copy)]
434pub struct MetricsFacadeRecorder;
435
436#[cfg(feature = "metrics")]
437impl MetricsFacadeRecorder {
438 pub const fn new() -> Self {
440 Self
441 }
442}
443
444#[cfg(feature = "metrics")]
445fn record_command_metrics(
446 command: &str,
447 duration: Duration,
448 error: Option<ErrorKind>,
449 node: Option<&str>,
450) {
451 let error = error.map(ErrorKind::as_label);
452 let outcome = if error.is_some() { "error" } else { "success" };
453
454 let mut duration_labels = vec![
455 metrics::Label::new("db.system.name", "redis"),
456 metrics::Label::new("db.operation.name", command.to_owned()),
457 ];
458 let mut counter_labels = vec![
459 metrics::Label::new("db.operation.name", command.to_owned()),
460 metrics::Label::new("outcome", outcome),
461 ];
462
463 if let Some(error) = error {
464 duration_labels.push(metrics::Label::new("error.type", error));
465 counter_labels.push(metrics::Label::new("error.type", error));
466 }
467 if let Some(node) = node {
468 let node = node.to_owned();
469 duration_labels.push(metrics::Label::new(
470 "redis_tower.cluster.node",
471 node.clone(),
472 ));
473 counter_labels.push(metrics::Label::new("redis_tower.cluster.node", node));
474 }
475
476 metrics::histogram!(
477 description: "Duration of Redis client operations",
478 unit: metrics::Unit::Seconds,
479 "db.client.operation.duration",
480 duration_labels,
481 )
482 .record(duration.as_secs_f64());
483 metrics::counter!(
484 description: "Redis commands completed",
485 unit: metrics::Unit::Count,
486 "redis_tower.commands",
487 counter_labels,
488 )
489 .increment(1);
490}
491
492#[cfg(feature = "metrics")]
493impl MetricsRecorder for MetricsFacadeRecorder {
494 fn command_completed(&self, command: &str, duration: Duration, error: Option<ErrorKind>) {
495 record_command_metrics(command, duration, error, None);
496 }
497
498 fn command_completed_on_node(
499 &self,
500 command: &str,
501 duration: Duration,
502 error: Option<ErrorKind>,
503 node: Option<&str>,
504 ) {
505 record_command_metrics(command, duration, error, node);
506 }
507
508 fn cluster_redirected(&self, kind: ClusterRedirectKind) {
509 metrics::counter!(
510 description: "Redis Cluster redirect responses",
511 unit: metrics::Unit::Count,
512 "redis_tower.cluster.redirects",
513 "kind" => kind.as_label(),
514 )
515 .increment(1);
516 }
517
518 fn cluster_topology_refresh_completed(
519 &self,
520 duration: Duration,
521 outcome: ClusterTopologyRefreshOutcome,
522 ) {
523 let outcome = outcome.as_label();
524 metrics::histogram!(
525 description: "Duration of Redis Cluster topology refresh attempts",
526 unit: metrics::Unit::Seconds,
527 "redis_tower.cluster.topology_refresh.duration",
528 "outcome" => outcome,
529 )
530 .record(duration.as_secs_f64());
531 metrics::counter!(
532 description: "Redis Cluster topology refresh attempts",
533 unit: metrics::Unit::Count,
534 "redis_tower.cluster.topology_refreshes",
535 "outcome" => outcome,
536 )
537 .increment(1);
538 }
539
540 fn pipeline_flushed(&self, batch_size: usize) {
541 metrics::histogram!(
542 description: "Redis commands sent in each automatic pipeline flush",
543 unit: metrics::Unit::Count,
544 "redis_tower.pipeline.batch_size",
545 )
546 .record(batch_size as f64);
547 }
548
549 fn cache_event(&self, event: CacheEvent, count: u64) {
550 metrics::counter!(
551 description: "Redis client-side cache events",
552 unit: metrics::Unit::Count,
553 "redis_tower.cache.events",
554 "event" => event.as_label(),
555 )
556 .increment(count);
557 }
558
559 fn pool_acquisition_completed(&self, pool_name: &str, duration: Duration, timed_out: bool) {
560 metrics::histogram!(
561 description: "Time spent waiting to acquire a Redis pool connection",
562 unit: metrics::Unit::Seconds,
563 "db.client.connection.wait_time",
564 "db.client.connection.pool.name" => pool_name.to_owned(),
565 )
566 .record(duration.as_secs_f64());
567
568 if timed_out {
569 metrics::counter!(
570 description: "Redis pool connection acquisition timeouts",
571 unit: metrics::Unit::Count,
572 "db.client.connection.timeouts",
573 "db.client.connection.pool.name" => pool_name.to_owned(),
574 )
575 .increment(1);
576 }
577 }
578
579 fn pool_health_check_failed(&self, pool_name: &str) {
580 metrics::counter!(
581 description: "Redis pool connection health-check failures",
582 unit: metrics::Unit::Count,
583 "redis_tower.pool.health_check_failures",
584 "db.client.connection.pool.name" => pool_name.to_owned(),
585 )
586 .increment(1);
587 }
588
589 fn pool_connection_replaced(&self, pool_name: &str) {
590 metrics::counter!(
591 description: "Redis pool connections replaced after failed health checks",
592 unit: metrics::Unit::Count,
593 "redis_tower.pool.connection_replacements",
594 "db.client.connection.pool.name" => pool_name.to_owned(),
595 )
596 .increment(1);
597 }
598
599 fn pool_health_probe_completed(
600 &self,
601 pool_name: &str,
602 kind: HealthProbeKind,
603 duration: Duration,
604 healthy: bool,
605 replication_lag_bytes: Option<u64>,
606 ) {
607 let outcome = if healthy { "healthy" } else { "unhealthy" };
608 metrics::histogram!(
609 description: "Duration of active Redis pool health probes",
610 unit: metrics::Unit::Seconds,
611 "redis_tower.pool.health_probe.duration",
612 "db.client.connection.pool.name" => pool_name.to_owned(),
613 "probe" => kind.as_str(),
614 "outcome" => outcome,
615 )
616 .record(duration.as_secs_f64());
617 metrics::counter!(
618 description: "Active Redis pool health probe outcomes",
619 unit: metrics::Unit::Count,
620 "redis_tower.pool.health_probes",
621 "db.client.connection.pool.name" => pool_name.to_owned(),
622 "probe" => kind.as_str(),
623 "outcome" => outcome,
624 )
625 .increment(1);
626 metrics::gauge!(
627 description: "Latest Redis replication lag observed by a pool probe",
628 unit: metrics::Unit::Bytes,
629 "redis_tower.pool.replication_lag",
630 "db.client.connection.pool.name" => pool_name.to_owned(),
631 )
632 .set(replication_lag_bytes.unwrap_or(0) as f64);
633 metrics::gauge!(
634 description: "Whether the Redis pool replication-lag gauge has a current observation",
635 unit: metrics::Unit::Count,
636 "redis_tower.pool.replication_lag_observed",
637 "db.client.connection.pool.name" => pool_name.to_owned(),
638 )
639 .set(if replication_lag_bytes.is_some() {
640 1.0
641 } else {
642 0.0
643 });
644 }
645
646 fn pool_connections_reaped(&self, pool_name: &str, count: usize) {
647 metrics::counter!(
648 description: "Redis pool connections removed after becoming idle",
649 unit: metrics::Unit::Count,
650 "redis_tower.pool.connections_reaped",
651 "db.client.connection.pool.name" => pool_name.to_owned(),
652 )
653 .increment(count as u64);
654 }
655}
656
657#[cfg(feature = "metrics")]
663#[cfg_attr(docsrs, doc(cfg(feature = "metrics")))]
664#[must_use = "dropping the handle immediately stops metrics export"]
665pub struct MetricsExporterHandle {
666 shutdown: Option<tokio::sync::oneshot::Sender<()>>,
667 task: Option<tokio::task::JoinHandle<()>>,
668}
669
670#[cfg(feature = "metrics")]
671impl MetricsExporterHandle {
672 fn spawn(interval: Duration, mut record: impl FnMut() + Send + 'static) -> Self {
673 assert!(
674 !interval.is_zero(),
675 "stats export interval must be non-zero"
676 );
677
678 let (shutdown, mut shutdown_rx) = tokio::sync::oneshot::channel();
679 let task = tokio::spawn(async move {
680 let mut ticker = tokio::time::interval(interval);
681 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
682
683 loop {
684 tokio::select! {
685 biased;
686 _ = &mut shutdown_rx => {
687 record();
688 break;
689 },
690 _ = ticker.tick() => record(),
691 }
692 }
693 });
694
695 Self {
696 shutdown: Some(shutdown),
697 task: Some(task),
698 }
699 }
700
701 fn cancel(&mut self) {
702 if let Some(shutdown) = self.shutdown.take() {
703 let _ = shutdown.send(());
704 }
705 }
706
707 pub async fn shutdown(mut self) {
709 self.cancel();
710 if let Some(task) = self.task.take() {
711 let _ = task.await;
712 }
713 }
714}
715
716#[cfg(feature = "metrics")]
717impl Drop for MetricsExporterHandle {
718 fn drop(&mut self) {
719 self.cancel();
720 if let Some(task) = &self.task {
721 task.abort();
722 }
723 }
724}
725
726#[cfg(feature = "metrics")]
727fn pool_gauge_values(stats: &PoolStats) -> (usize, usize) {
728 let used_connections = stats.size.saturating_sub(stats.idle_count);
729 let pending_requests = stats.total_inflight.saturating_sub(used_connections);
730 (used_connections, pending_requests)
731}
732
733#[cfg(feature = "metrics")]
734fn record_pool_stats(pool_name: &str, stats: &PoolStats) {
735 let (used_connections, pending_requests) = pool_gauge_values(stats);
736 let pool_name = pool_name.to_owned();
737
738 metrics::gauge!(
739 description: "Redis pool connections by state",
740 unit: metrics::Unit::Count,
741 "db.client.connection.count",
742 "db.client.connection.pool.name" => pool_name.clone(),
743 "db.client.connection.state" => "idle",
744 )
745 .set(stats.idle_count as f64);
746 metrics::gauge!(
747 description: "Redis pool connections by state",
748 unit: metrics::Unit::Count,
749 "db.client.connection.count",
750 "db.client.connection.pool.name" => pool_name.clone(),
751 "db.client.connection.state" => "used",
752 )
753 .set(used_connections as f64);
754 metrics::gauge!(
755 description: "Maximum Redis pool connection count",
756 unit: metrics::Unit::Count,
757 "db.client.connection.max",
758 "db.client.connection.pool.name" => pool_name.clone(),
759 )
760 .set(stats.max_size as f64);
761 metrics::gauge!(
762 description: "Minimum Redis pool connection count",
763 unit: metrics::Unit::Count,
764 "db.client.connection.min",
765 "db.client.connection.pool.name" => pool_name.clone(),
766 )
767 .set(stats.min_size as f64);
768 metrics::gauge!(
769 description: "Redis requests waiting for a pool connection",
770 unit: metrics::Unit::Count,
771 "db.client.connection.pending_requests",
772 "db.client.connection.pool.name" => pool_name.clone(),
773 )
774 .set(pending_requests as f64);
775 metrics::gauge!(
776 description: "Redis commands active or waiting in the pool",
777 unit: metrics::Unit::Count,
778 "redis_tower.pool.inflight_commands",
779 "db.client.connection.pool.name" => pool_name.clone(),
780 )
781 .set(stats.total_inflight as f64);
782 metrics::gauge!(
783 description: "Highest in-flight command count on one Redis pool connection",
784 unit: metrics::Unit::Count,
785 "redis_tower.pool.max_inflight_per_connection",
786 "db.client.connection.pool.name" => pool_name.clone(),
787 )
788 .set(stats.max_inflight as f64);
789 metrics::gauge!(
790 description: "Redis pool connections by active-probe health state",
791 unit: metrics::Unit::Count,
792 "redis_tower.pool.health_count",
793 "db.client.connection.pool.name" => pool_name.clone(),
794 "state" => "healthy",
795 )
796 .set(stats.healthy_count as f64);
797 metrics::gauge!(
798 description: "Redis pool connections by active-probe health state",
799 unit: metrics::Unit::Count,
800 "redis_tower.pool.health_count",
801 "db.client.connection.pool.name" => pool_name.clone(),
802 "state" => "unhealthy",
803 )
804 .set(stats.unhealthy_count as f64);
805 metrics::gauge!(
806 description: "Redis pool connections by active-probe health state",
807 unit: metrics::Unit::Count,
808 "redis_tower.pool.health_count",
809 "db.client.connection.pool.name" => pool_name.clone(),
810 "state" => "unknown",
811 )
812 .set(stats.unknown_health_count as f64);
813 metrics::gauge!(
814 description: "Cumulative Redis pool connections removed by idle reaping",
815 unit: metrics::Unit::Count,
816 "redis_tower.pool.connections_reaped_total",
817 "db.client.connection.pool.name" => pool_name.clone(),
818 )
819 .set(stats.reaped_connections as f64);
820 metrics::gauge!(
821 description: "Maximum latest Redis replication lag across pool slots",
822 unit: metrics::Unit::Bytes,
823 "redis_tower.pool.max_replication_lag",
824 "db.client.connection.pool.name" => pool_name.clone(),
825 )
826 .set(stats.max_replication_lag_bytes.unwrap_or(0) as f64);
827 metrics::gauge!(
828 description: "Whether the maximum Redis pool replication-lag gauge has a current observation",
829 unit: metrics::Unit::Count,
830 "redis_tower.pool.max_replication_lag_observed",
831 "db.client.connection.pool.name" => pool_name,
832 )
833 .set(if stats.max_replication_lag_bytes.is_some() {
834 1.0
835 } else {
836 0.0
837 });
838}
839
840#[cfg(feature = "metrics")]
841fn record_queue_depth(pipeline_name: &str, queue_depth: usize) {
842 metrics::gauge!(
843 description: "Redis auto-pipeline requests waiting in the internal queue",
844 unit: metrics::Unit::Count,
845 "redis_tower.pipeline.queue_depth",
846 "redis_tower.pipeline.name" => pipeline_name.to_owned(),
847 )
848 .set(queue_depth as f64);
849}
850
851#[cfg(feature = "metrics")]
863#[cfg_attr(docsrs, doc(cfg(feature = "metrics")))]
864pub fn spawn_pool_stats_exporter<S>(
865 pool: ConnectionPool<S>,
866 interval: Duration,
867) -> MetricsExporterHandle
868where
869 S: Send + 'static,
870{
871 let pool_name = pool.name().to_owned();
872 MetricsExporterHandle::spawn(interval, move || {
873 record_pool_stats(&pool_name, &pool.stats())
874 })
875}
876
877#[cfg(feature = "metrics")]
888#[cfg_attr(docsrs, doc(cfg(feature = "metrics")))]
889pub fn spawn_queue_depth_exporter(
890 client: MultiplexedClient,
891 pipeline_name: impl Into<String>,
892 interval: Duration,
893) -> MetricsExporterHandle {
894 let pipeline_name = pipeline_name.into();
895 MetricsExporterHandle::spawn(interval, move || {
896 record_queue_depth(&pipeline_name, client.queue_depth())
897 })
898}
899
900pub struct MetricsLayer<R> {
933 recorder: Arc<R>,
934}
935
936impl<R: MetricsRecorder> MetricsLayer<R> {
937 pub fn new(recorder: R) -> Self {
939 Self {
940 recorder: Arc::new(recorder),
941 }
942 }
943}
944
945impl<R: MetricsRecorder, S> tower_layer::Layer<S> for MetricsLayer<R> {
946 type Service = MetricsService<S, R>;
947
948 fn layer(&self, inner: S) -> Self::Service {
949 MetricsService {
950 inner,
951 recorder: Arc::clone(&self.recorder),
952 }
953 }
954}
955
956pub struct MetricsService<S, R> {
963 inner: S,
964 recorder: Arc<R>,
965}
966
967impl<S, R> Clone for MetricsService<S, R>
968where
969 S: Clone,
970{
971 fn clone(&self) -> Self {
972 Self {
973 inner: self.inner.clone(),
974 recorder: Arc::clone(&self.recorder),
975 }
976 }
977}
978
979impl<S, R> MetricsService<S, R> {
980 pub fn new(inner: S, recorder: Arc<R>) -> Self {
982 Self { inner, recorder }
983 }
984}
985
986fn extract_command_name(frame: &Frame) -> Option<&str> {
991 let items = match frame {
992 Frame::Array(Some(items)) if !items.is_empty() => items,
993 _ => return None,
994 };
995 match &items[0] {
996 Frame::BulkString(Some(b)) => std::str::from_utf8(b).ok(),
997 _ => None,
998 }
999}
1000
1001impl<S, R> Service<Frame> for MetricsService<S, R>
1002where
1003 S: Service<Frame, Response = Frame, Error = RedisError>,
1004 S::Future: Send + 'static,
1005 R: MetricsRecorder,
1006{
1007 type Response = Frame;
1008 type Error = RedisError;
1009 type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
1010
1011 fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
1012 self.inner.poll_ready(cx)
1013 }
1014
1015 fn call(&mut self, request: Frame) -> Self::Future {
1016 let command_name = extract_command_name(&request)
1017 .unwrap_or("UNKNOWN")
1018 .to_ascii_uppercase();
1019 let start = Instant::now();
1020 let recorder = Arc::clone(&self.recorder);
1021 let future = self.inner.call(request);
1022
1023 Box::pin(async move {
1024 let result = future.await;
1025 let elapsed = start.elapsed();
1026 let error_kind = match &result {
1027 Ok(Frame::Error(bytes)) => Some(ErrorKind::from_frame_error(bytes)),
1028 Ok(_) => None,
1029 Err(e) => Some(ErrorKind::from_error(e)),
1030 };
1031 recorder.command_completed(&command_name, elapsed, error_kind);
1032 result
1033 })
1034 }
1035}
1036
1037#[cfg(test)]
1038mod tests {
1039 use super::*;
1040 use bytes::Bytes;
1041 use std::sync::Mutex;
1042 use std::sync::atomic::{AtomicU64, Ordering};
1043
1044 #[cfg(feature = "metrics")]
1045 #[derive(Debug, Clone, PartialEq, Eq)]
1046 struct CapturedMetric {
1047 name: String,
1048 labels: Vec<(String, String)>,
1049 }
1050
1051 #[cfg(feature = "metrics")]
1052 impl CapturedMetric {
1053 fn from_key(key: &metrics::Key) -> Self {
1054 Self {
1055 name: key.name().to_owned(),
1056 labels: key
1057 .labels()
1058 .map(|label| (label.key().to_owned(), label.value().to_owned()))
1059 .collect(),
1060 }
1061 }
1062
1063 fn label(&self, name: &str) -> Option<&str> {
1064 self.labels
1065 .iter()
1066 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
1067 }
1068 }
1069
1070 #[cfg(feature = "metrics")]
1071 struct CapturedGauge {
1072 metric: CapturedMetric,
1073 value: Arc<Mutex<f64>>,
1074 }
1075
1076 #[cfg(feature = "metrics")]
1077 impl std::ops::Deref for CapturedGauge {
1078 type Target = CapturedMetric;
1079
1080 fn deref(&self) -> &Self::Target {
1081 &self.metric
1082 }
1083 }
1084
1085 #[cfg(feature = "metrics")]
1086 impl CapturedGauge {
1087 fn value(&self) -> f64 {
1088 *self.value.lock().unwrap()
1089 }
1090 }
1091
1092 #[cfg(feature = "metrics")]
1093 struct CapturingGauge {
1094 value: Arc<Mutex<f64>>,
1095 }
1096
1097 #[cfg(feature = "metrics")]
1098 impl metrics::GaugeFn for CapturingGauge {
1099 fn increment(&self, value: f64) {
1100 *self.value.lock().unwrap() += value;
1101 }
1102
1103 fn decrement(&self, value: f64) {
1104 *self.value.lock().unwrap() -= value;
1105 }
1106
1107 fn set(&self, value: f64) {
1108 *self.value.lock().unwrap() = value;
1109 }
1110 }
1111
1112 #[cfg(feature = "metrics")]
1113 #[derive(Default)]
1114 struct CapturingFacadeRecorder {
1115 counters: Mutex<Vec<CapturedMetric>>,
1116 gauges: Mutex<Vec<CapturedGauge>>,
1117 histograms: Mutex<Vec<CapturedMetric>>,
1118 }
1119
1120 #[cfg(feature = "metrics")]
1121 impl metrics::Recorder for CapturingFacadeRecorder {
1122 fn describe_counter(
1123 &self,
1124 _key: metrics::KeyName,
1125 _unit: Option<metrics::Unit>,
1126 _description: metrics::SharedString,
1127 ) {
1128 }
1129
1130 fn describe_gauge(
1131 &self,
1132 _key: metrics::KeyName,
1133 _unit: Option<metrics::Unit>,
1134 _description: metrics::SharedString,
1135 ) {
1136 }
1137
1138 fn describe_histogram(
1139 &self,
1140 _key: metrics::KeyName,
1141 _unit: Option<metrics::Unit>,
1142 _description: metrics::SharedString,
1143 ) {
1144 }
1145
1146 fn register_counter(
1147 &self,
1148 key: &metrics::Key,
1149 _metadata: &metrics::Metadata<'_>,
1150 ) -> metrics::Counter {
1151 self.counters
1152 .lock()
1153 .unwrap()
1154 .push(CapturedMetric::from_key(key));
1155 metrics::Counter::noop()
1156 }
1157
1158 fn register_gauge(
1159 &self,
1160 key: &metrics::Key,
1161 _metadata: &metrics::Metadata<'_>,
1162 ) -> metrics::Gauge {
1163 let value = Arc::new(Mutex::new(0.0));
1164 self.gauges.lock().unwrap().push(CapturedGauge {
1165 metric: CapturedMetric::from_key(key),
1166 value: Arc::clone(&value),
1167 });
1168 metrics::Gauge::from_arc(Arc::new(CapturingGauge { value }))
1169 }
1170
1171 fn register_histogram(
1172 &self,
1173 key: &metrics::Key,
1174 _metadata: &metrics::Metadata<'_>,
1175 ) -> metrics::Histogram {
1176 self.histograms
1177 .lock()
1178 .unwrap()
1179 .push(CapturedMetric::from_key(key));
1180 metrics::Histogram::noop()
1181 }
1182 }
1183
1184 #[test]
1185 fn extract_name_from_get() {
1186 let frame = Frame::Array(Some(vec![
1187 Frame::BulkString(Some(Bytes::from("GET"))),
1188 Frame::BulkString(Some(Bytes::from("key"))),
1189 ]));
1190 assert_eq!(extract_command_name(&frame), Some("GET"));
1191 }
1192
1193 #[test]
1194 fn extract_name_from_set() {
1195 let frame = Frame::Array(Some(vec![
1196 Frame::BulkString(Some(Bytes::from("SET"))),
1197 Frame::BulkString(Some(Bytes::from("key"))),
1198 Frame::BulkString(Some(Bytes::from("value"))),
1199 ]));
1200 assert_eq!(extract_command_name(&frame), Some("SET"));
1201 }
1202
1203 #[test]
1204 fn extract_name_empty_array() {
1205 let frame = Frame::Array(Some(vec![]));
1206 assert_eq!(extract_command_name(&frame), None);
1207 }
1208
1209 #[test]
1210 fn extract_name_null_frame() {
1211 assert_eq!(extract_command_name(&Frame::Null), None);
1212 }
1213
1214 #[test]
1215 fn extract_name_none_array() {
1216 let frame = Frame::Array(None);
1217 assert_eq!(extract_command_name(&frame), None);
1218 }
1219
1220 #[test]
1221 fn error_kind_from_error_connection() {
1222 assert_eq!(
1223 ErrorKind::from_error(&RedisError::ConnectionClosed),
1224 ErrorKind::Connection
1225 );
1226 }
1227
1228 #[test]
1229 fn error_kind_from_error_timeout() {
1230 assert_eq!(
1231 ErrorKind::from_error(&RedisError::PoolAcquisitionTimeout {
1232 waited: std::time::Duration::from_millis(50),
1233 pool_size: 4,
1234 }),
1235 ErrorKind::Timeout
1236 );
1237 }
1238
1239 #[test]
1240 fn error_kind_from_error_wrongtype() {
1241 assert_eq!(
1242 ErrorKind::from_error(&RedisError::Redis(
1243 "WRONGTYPE Operation against a key holding the wrong kind of value".into()
1244 )),
1245 ErrorKind::WrongType
1246 );
1247 }
1248
1249 #[test]
1250 fn error_kind_from_error_circuit_open() {
1251 assert_eq!(
1252 ErrorKind::from_error(&RedisError::CircuitOpen),
1253 ErrorKind::CircuitOpen
1254 );
1255 }
1256
1257 #[test]
1258 fn error_kind_from_error_queue_full() {
1259 assert_eq!(
1260 ErrorKind::from_error(&RedisError::QueueFull),
1261 ErrorKind::QueueFull
1262 );
1263 }
1264
1265 #[test]
1266 fn error_kind_from_error_auth_noauth() {
1267 assert_eq!(
1268 ErrorKind::from_error(&RedisError::Redis("NOAUTH Authentication required".into())),
1269 ErrorKind::Auth
1270 );
1271 }
1272
1273 #[test]
1274 fn error_kind_from_error_auth_wrongpass() {
1275 assert_eq!(
1276 ErrorKind::from_error(&RedisError::Redis(
1277 "WRONGPASS invalid username-password pair".into()
1278 )),
1279 ErrorKind::Auth
1280 );
1281 }
1282
1283 #[test]
1284 fn error_kind_from_error_auth_provider() {
1285 assert_eq!(
1286 ErrorKind::from_error(&RedisError::Redis(
1287 "AUTH_PROVIDER current credential lookup failed".into()
1288 )),
1289 ErrorKind::Auth
1290 );
1291 }
1292
1293 #[test]
1294 fn error_kind_from_error_other() {
1295 assert_eq!(
1296 ErrorKind::from_error(&RedisError::Redis("ERR unknown command".into())),
1297 ErrorKind::Other
1298 );
1299 }
1300
1301 #[test]
1302 fn error_kind_from_frame_error_wrongtype() {
1303 assert_eq!(
1304 ErrorKind::from_frame_error(&Bytes::from("WRONGTYPE value is not a set")),
1305 ErrorKind::WrongType
1306 );
1307 }
1308
1309 #[test]
1310 fn error_kind_from_frame_error_other() {
1311 assert_eq!(
1312 ErrorKind::from_frame_error(&Bytes::from("ERR syntax error")),
1313 ErrorKind::Other
1314 );
1315 }
1316
1317 struct TestRecorder {
1318 error_kind: Mutex<Option<Option<ErrorKind>>>,
1319 duration_ns: AtomicU64,
1320 }
1321
1322 impl TestRecorder {
1323 fn new() -> Self {
1324 Self {
1325 error_kind: Mutex::new(None),
1326 duration_ns: AtomicU64::new(0),
1327 }
1328 }
1329
1330 fn was_called(&self) -> bool {
1331 self.error_kind.lock().unwrap().is_some()
1332 }
1333
1334 fn recorded_error(&self) -> Option<ErrorKind> {
1335 self.error_kind.lock().unwrap().flatten()
1336 }
1337 }
1338
1339 impl MetricsRecorder for TestRecorder {
1340 fn command_completed(&self, _command: &str, duration: Duration, error: Option<ErrorKind>) {
1341 *self.error_kind.lock().unwrap() = Some(error);
1342 self.duration_ns
1343 .store(duration.as_nanos() as u64, Ordering::SeqCst);
1344 }
1345 }
1346
1347 #[test]
1348 fn custom_recorders_inherit_noop_pool_hooks() {
1349 let recorder = TestRecorder::new();
1350 recorder.pool_acquisition_completed("default", Duration::from_millis(2), false);
1351 recorder.pool_health_check_failed("default");
1352 recorder.pool_connection_replaced("default");
1353 assert!(!recorder.was_called());
1354 }
1355
1356 #[test]
1357 fn existing_recorders_receive_node_completions_through_the_original_hook() {
1358 let recorder = TestRecorder::new();
1359 recorder.command_completed_on_node(
1360 "GET",
1361 Duration::from_millis(2),
1362 Some(ErrorKind::Connection),
1363 Some("127.0.0.1:7000"),
1364 );
1365 recorder.cluster_redirected(ClusterRedirectKind::Moved);
1366 recorder.cluster_topology_refresh_completed(
1367 Duration::from_millis(3),
1368 ClusterTopologyRefreshOutcome::Success,
1369 );
1370
1371 assert!(recorder.was_called());
1372 assert_eq!(recorder.recorded_error(), Some(ErrorKind::Connection));
1373 }
1374
1375 #[derive(Default)]
1376 struct ClusterHookRecorder {
1377 node: Mutex<Option<String>>,
1378 redirect: Mutex<Option<ClusterRedirectKind>>,
1379 refresh: Mutex<Option<(Duration, ClusterTopologyRefreshOutcome)>>,
1380 }
1381
1382 impl MetricsRecorder for ClusterHookRecorder {
1383 fn command_completed(
1384 &self,
1385 _command: &str,
1386 _duration: Duration,
1387 _error: Option<ErrorKind>,
1388 ) {
1389 }
1390
1391 fn command_completed_on_node(
1392 &self,
1393 _command: &str,
1394 _duration: Duration,
1395 _error: Option<ErrorKind>,
1396 node: Option<&str>,
1397 ) {
1398 *self.node.lock().unwrap() = node.map(str::to_owned);
1399 }
1400
1401 fn cluster_redirected(&self, kind: ClusterRedirectKind) {
1402 *self.redirect.lock().unwrap() = Some(kind);
1403 }
1404
1405 fn cluster_topology_refresh_completed(
1406 &self,
1407 duration: Duration,
1408 outcome: ClusterTopologyRefreshOutcome,
1409 ) {
1410 *self.refresh.lock().unwrap() = Some((duration, outcome));
1411 }
1412 }
1413
1414 #[test]
1415 fn arc_delegates_cluster_hooks_to_its_metrics_recorder() {
1416 let recorder = Arc::new(ClusterHookRecorder::default());
1417
1418 MetricsRecorder::command_completed_on_node(
1419 &recorder,
1420 "SET",
1421 Duration::from_millis(1),
1422 None,
1423 Some("127.0.0.1:7001"),
1424 );
1425 MetricsRecorder::cluster_redirected(&recorder, ClusterRedirectKind::Ask);
1426 MetricsRecorder::cluster_topology_refresh_completed(
1427 &recorder,
1428 Duration::from_millis(4),
1429 ClusterTopologyRefreshOutcome::Partial,
1430 );
1431
1432 assert_eq!(
1433 recorder.node.lock().unwrap().as_deref(),
1434 Some("127.0.0.1:7001")
1435 );
1436 assert_eq!(
1437 *recorder.redirect.lock().unwrap(),
1438 Some(ClusterRedirectKind::Ask)
1439 );
1440 assert_eq!(
1441 *recorder.refresh.lock().unwrap(),
1442 Some((
1443 Duration::from_millis(4),
1444 ClusterTopologyRefreshOutcome::Partial
1445 ))
1446 );
1447 }
1448
1449 #[test]
1450 fn arc_delegates_to_its_metrics_recorder() {
1451 let recorder = Arc::new(TestRecorder::new());
1452 MetricsRecorder::command_completed(
1453 &recorder,
1454 "GET",
1455 Duration::from_millis(2),
1456 Some(ErrorKind::Connection),
1457 );
1458 assert!(recorder.was_called());
1459 assert_eq!(recorder.recorded_error(), Some(ErrorKind::Connection));
1460 }
1461
1462 #[cfg(feature = "metrics")]
1463 #[test]
1464 fn facade_recorder_uses_semantic_convention_labels() {
1465 let capture = CapturingFacadeRecorder::default();
1466 let recorder = MetricsFacadeRecorder::new();
1467 metrics::with_local_recorder(&capture, || {
1468 recorder.command_completed("GET", Duration::from_millis(2), None);
1469 recorder.command_completed(
1470 "SET",
1471 Duration::from_millis(3),
1472 Some(ErrorKind::Connection),
1473 );
1474 recorder.command_completed("CUSTOM", Duration::from_millis(1), Some(ErrorKind::Other));
1475 });
1476
1477 let histograms = capture.histograms.lock().unwrap();
1478 let success = &histograms[0];
1479 assert_eq!(success.name, "db.client.operation.duration");
1480 assert_eq!(success.label("db.system.name"), Some("redis"));
1481 assert_eq!(success.label("db.operation.name"), Some("GET"));
1482 assert_eq!(success.label("error.type"), None);
1483
1484 let failure = &histograms[1];
1485 assert_eq!(failure.label("db.operation.name"), Some("SET"));
1486 assert_eq!(failure.label("error.type"), Some("connection"));
1487
1488 let fallback = &histograms[2];
1489 assert_eq!(fallback.label("db.operation.name"), Some("CUSTOM"));
1490 assert_eq!(fallback.label("error.type"), Some("_OTHER"));
1491
1492 let counters = capture.counters.lock().unwrap();
1493 assert_eq!(counters[0].label("outcome"), Some("success"));
1494 assert_eq!(counters[1].label("outcome"), Some("error"));
1495 assert_eq!(counters[1].label("error.type"), Some("connection"));
1496 assert_eq!(counters[2].label("error.type"), Some("_OTHER"));
1497 }
1498
1499 #[cfg(feature = "metrics")]
1500 #[test]
1501 fn facade_recorder_adds_the_optional_cluster_node_label() {
1502 let capture = CapturingFacadeRecorder::default();
1503 let recorder = MetricsFacadeRecorder::new();
1504 metrics::with_local_recorder(&capture, || {
1505 recorder.command_completed_on_node(
1506 "GET",
1507 Duration::from_millis(2),
1508 None,
1509 Some("127.0.0.1:7000"),
1510 );
1511 recorder.command_completed_on_node(
1512 "SET",
1513 Duration::from_millis(3),
1514 Some(ErrorKind::Connection),
1515 None,
1516 );
1517 });
1518
1519 let histograms = capture.histograms.lock().unwrap();
1520 assert_eq!(
1521 histograms[0].label("redis_tower.cluster.node"),
1522 Some("127.0.0.1:7000")
1523 );
1524 assert_eq!(histograms[1].label("redis_tower.cluster.node"), None);
1525
1526 let counters = capture.counters.lock().unwrap();
1527 assert_eq!(
1528 counters[0].label("redis_tower.cluster.node"),
1529 Some("127.0.0.1:7000")
1530 );
1531 assert_eq!(counters[1].label("redis_tower.cluster.node"), None);
1532 }
1533
1534 #[cfg(feature = "metrics")]
1535 #[test]
1536 fn facade_recorder_emits_bounded_cluster_event_labels() {
1537 let capture = CapturingFacadeRecorder::default();
1538 let recorder = MetricsFacadeRecorder::new();
1539 metrics::with_local_recorder(&capture, || {
1540 recorder.cluster_redirected(ClusterRedirectKind::Moved);
1541 recorder.cluster_redirected(ClusterRedirectKind::Ask);
1542 recorder.cluster_topology_refresh_completed(
1543 Duration::from_millis(3),
1544 ClusterTopologyRefreshOutcome::Success,
1545 );
1546 recorder.cluster_topology_refresh_completed(
1547 Duration::from_millis(4),
1548 ClusterTopologyRefreshOutcome::Partial,
1549 );
1550 recorder.cluster_topology_refresh_completed(
1551 Duration::from_millis(5),
1552 ClusterTopologyRefreshOutcome::Error,
1553 );
1554 });
1555
1556 let counters = capture.counters.lock().unwrap();
1557 let redirect_kinds: Vec<_> = counters
1558 .iter()
1559 .filter(|metric| metric.name == "redis_tower.cluster.redirects")
1560 .map(|metric| metric.label("kind").unwrap())
1561 .collect();
1562 assert_eq!(redirect_kinds, ["moved", "ask"]);
1563
1564 let refresh_outcomes: Vec<_> = counters
1565 .iter()
1566 .filter(|metric| metric.name == "redis_tower.cluster.topology_refreshes")
1567 .map(|metric| metric.label("outcome").unwrap())
1568 .collect();
1569 assert_eq!(refresh_outcomes, ["success", "partial", "error"]);
1570
1571 let histograms = capture.histograms.lock().unwrap();
1572 let duration_outcomes: Vec<_> = histograms
1573 .iter()
1574 .filter(|metric| metric.name == "redis_tower.cluster.topology_refresh.duration")
1575 .map(|metric| metric.label("outcome").unwrap())
1576 .collect();
1577 assert_eq!(duration_outcomes, ["success", "partial", "error"]);
1578 }
1579
1580 #[cfg(feature = "metrics")]
1581 #[test]
1582 fn pending_requests_exclude_connections_doing_work() {
1583 let stats = PoolStats {
1584 size: 4,
1585 idle_count: 1,
1586 total_inflight: 7,
1587 max_inflight: 3,
1588 min_size: 2,
1589 max_size: 8,
1590 healthy_count: 2,
1591 unhealthy_count: 1,
1592 unknown_health_count: 1,
1593 max_replication_lag_bytes: Some(42),
1594 reaped_connections: 5,
1595 };
1596 assert_eq!(pool_gauge_values(&stats), (3, 4));
1597 }
1598
1599 #[cfg(feature = "metrics")]
1600 #[test]
1601 fn pool_stats_use_canonical_names_and_labels() {
1602 let capture = CapturingFacadeRecorder::default();
1603 let stats = PoolStats {
1604 size: 4,
1605 idle_count: 1,
1606 total_inflight: 7,
1607 max_inflight: 3,
1608 min_size: 2,
1609 max_size: 8,
1610 healthy_count: 2,
1611 unhealthy_count: 1,
1612 unknown_health_count: 1,
1613 max_replication_lag_bytes: Some(42),
1614 reaped_connections: 5,
1615 };
1616 metrics::with_local_recorder(&capture, || record_pool_stats("primary", &stats));
1617
1618 let gauges = capture.gauges.lock().unwrap();
1619 let idle = gauges
1620 .iter()
1621 .find(|metric| {
1622 metric.name == "db.client.connection.count"
1623 && metric.label("db.client.connection.state") == Some("idle")
1624 })
1625 .unwrap();
1626 assert_eq!(
1627 idle.label("db.client.connection.pool.name"),
1628 Some("primary")
1629 );
1630 assert_eq!(idle.value(), 1.0);
1631 let max = gauges
1632 .iter()
1633 .find(|metric| metric.name == "db.client.connection.max")
1634 .unwrap();
1635 assert_eq!(max.value(), 8.0);
1636 let pending = gauges
1637 .iter()
1638 .find(|metric| metric.name == "db.client.connection.pending_requests")
1639 .unwrap();
1640 assert_eq!(pending.value(), 4.0);
1641 }
1642
1643 #[cfg(feature = "metrics")]
1644 #[test]
1645 fn unknown_replication_lag_resets_value_and_freshness_gauges() {
1646 let capture = CapturingFacadeRecorder::default();
1647 let recorder = MetricsFacadeRecorder::new();
1648 metrics::with_local_recorder(&capture, || {
1649 recorder.pool_health_probe_completed(
1650 "primary",
1651 HealthProbeKind::ReplicationLag,
1652 Duration::from_millis(1),
1653 true,
1654 Some(42),
1655 );
1656 recorder.pool_health_probe_completed(
1657 "primary",
1658 HealthProbeKind::ReplicationLag,
1659 Duration::from_millis(1),
1660 false,
1661 None,
1662 );
1663 });
1664
1665 let gauges = capture.gauges.lock().unwrap();
1666 let lag = gauges
1667 .iter()
1668 .rfind(|metric| metric.name == "redis_tower.pool.replication_lag")
1669 .unwrap();
1670 let observed = gauges
1671 .iter()
1672 .rfind(|metric| metric.name == "redis_tower.pool.replication_lag_observed")
1673 .unwrap();
1674 assert_eq!(lag.value(), 0.0);
1675 assert_eq!(observed.value(), 0.0);
1676 }
1677
1678 #[cfg(feature = "metrics")]
1679 #[test]
1680 fn queue_depth_uses_a_named_gauge() {
1681 let capture = CapturingFacadeRecorder::default();
1682 metrics::with_local_recorder(&capture, || record_queue_depth("commands", 7));
1683
1684 let gauges = capture.gauges.lock().unwrap();
1685 let queue = gauges
1686 .iter()
1687 .find(|metric| metric.name == "redis_tower.pipeline.queue_depth")
1688 .unwrap();
1689 assert_eq!(queue.label("redis_tower.pipeline.name"), Some("commands"));
1690 assert_eq!(queue.value(), 7.0);
1691 }
1692
1693 #[cfg(feature = "metrics")]
1694 #[tokio::test]
1695 async fn pool_stats_exporter_shuts_down_without_redis() {
1696 let pool =
1697 ConnectionPool::from_connections(vec![(), ()], crate::pool::PoolConfig::default())
1698 .unwrap();
1699 let exporter = spawn_pool_stats_exporter(pool, Duration::from_millis(1));
1700 tokio::task::yield_now().await;
1701 exporter.shutdown().await;
1702 }
1703
1704 #[cfg(feature = "metrics")]
1705 #[tokio::test]
1706 async fn exporter_shutdown_emits_a_final_snapshot() {
1707 let snapshots = Arc::new(std::sync::atomic::AtomicUsize::new(0));
1708 let task_snapshots = Arc::clone(&snapshots);
1709 let exporter = MetricsExporterHandle::spawn(Duration::from_secs(60), move || {
1710 task_snapshots.fetch_add(1, Ordering::SeqCst);
1711 });
1712
1713 while snapshots.load(Ordering::SeqCst) == 0 {
1714 tokio::task::yield_now().await;
1715 }
1716 let before_shutdown = snapshots.load(Ordering::SeqCst);
1717
1718 exporter.shutdown().await;
1719
1720 assert_eq!(snapshots.load(Ordering::SeqCst), before_shutdown + 1);
1721 }
1722
1723 #[derive(Clone)]
1725 struct OkService;
1726
1727 impl Service<Frame> for OkService {
1728 type Response = Frame;
1729 type Error = RedisError;
1730 type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
1731
1732 fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
1733 Poll::Ready(Ok(()))
1734 }
1735
1736 fn call(&mut self, _request: Frame) -> Self::Future {
1737 Box::pin(async { Ok(Frame::SimpleString("OK".into())) })
1738 }
1739 }
1740
1741 struct ErrService;
1743
1744 impl Service<Frame> for ErrService {
1745 type Response = Frame;
1746 type Error = RedisError;
1747 type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
1748
1749 fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
1750 Poll::Ready(Ok(()))
1751 }
1752
1753 fn call(&mut self, _request: Frame) -> Self::Future {
1754 Box::pin(async { Err(RedisError::ConnectionClosed) })
1755 }
1756 }
1757
1758 struct FrameErrService {
1760 msg: &'static str,
1761 }
1762
1763 impl Service<Frame> for FrameErrService {
1764 type Response = Frame;
1765 type Error = RedisError;
1766 type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
1767
1768 fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
1769 Poll::Ready(Ok(()))
1770 }
1771
1772 fn call(&mut self, _request: Frame) -> Self::Future {
1773 let msg = self.msg;
1774 Box::pin(async move { Ok(Frame::Error(Bytes::from(msg))) })
1775 }
1776 }
1777
1778 #[tokio::test]
1779 async fn records_success() {
1780 let recorder = Arc::new(TestRecorder::new());
1781 let mut svc = MetricsService::new(OkService, Arc::clone(&recorder));
1782
1783 let frame = Frame::Array(Some(vec![
1784 Frame::BulkString(Some(Bytes::from("GET"))),
1785 Frame::BulkString(Some(Bytes::from("key"))),
1786 ]));
1787
1788 let result = svc.call(frame).await;
1789 assert!(result.is_ok());
1790 assert!(recorder.was_called());
1791 assert_eq!(recorder.recorded_error(), None);
1792 assert!(recorder.duration_ns.load(Ordering::SeqCst) > 0);
1793 }
1794
1795 #[tokio::test]
1796 async fn cloned_services_share_a_non_clone_recorder() {
1797 let recorder = Arc::new(TestRecorder::new());
1798 let svc = MetricsService::new(OkService, Arc::clone(&recorder));
1799 let mut cloned = svc.clone();
1800
1801 let frame = Frame::Array(Some(vec![Frame::BulkString(Some(Bytes::from("PING")))]));
1802 assert!(cloned.call(frame).await.is_ok());
1803 assert!(recorder.was_called());
1804 }
1805
1806 #[tokio::test]
1807 async fn records_failure_connection_closed() {
1808 let recorder = Arc::new(TestRecorder::new());
1809 let mut svc = MetricsService::new(ErrService, Arc::clone(&recorder));
1810
1811 let frame = Frame::Array(Some(vec![
1812 Frame::BulkString(Some(Bytes::from("SET"))),
1813 Frame::BulkString(Some(Bytes::from("key"))),
1814 Frame::BulkString(Some(Bytes::from("val"))),
1815 ]));
1816
1817 let result = svc.call(frame).await;
1818 assert!(result.is_err());
1819 assert!(recorder.was_called());
1820 assert_eq!(recorder.recorded_error(), Some(ErrorKind::Connection));
1821 }
1822
1823 #[tokio::test]
1824 async fn frame_error_is_classified_as_failure() {
1825 let recorder = Arc::new(TestRecorder::new());
1826 let mut svc = MetricsService::new(
1827 FrameErrService {
1828 msg: "WRONGTYPE Operation against a key holding the wrong kind of value",
1829 },
1830 Arc::clone(&recorder),
1831 );
1832
1833 let frame = Frame::Array(Some(vec![
1834 Frame::BulkString(Some(Bytes::from("SADD"))),
1835 Frame::BulkString(Some(Bytes::from("key"))),
1836 Frame::BulkString(Some(Bytes::from("member"))),
1837 ]));
1838
1839 let result = svc.call(frame).await;
1842 assert!(result.is_ok());
1843 assert!(recorder.was_called());
1844 assert_eq!(recorder.recorded_error(), Some(ErrorKind::WrongType));
1846 }
1847
1848 #[tokio::test]
1849 async fn frame_error_generic_classified_as_other() {
1850 let recorder = Arc::new(TestRecorder::new());
1851 let mut svc = MetricsService::new(
1852 FrameErrService {
1853 msg: "ERR syntax error",
1854 },
1855 Arc::clone(&recorder),
1856 );
1857
1858 let frame = Frame::Array(Some(vec![
1859 Frame::BulkString(Some(Bytes::from("GET"))),
1860 Frame::BulkString(Some(Bytes::from("key"))),
1861 ]));
1862
1863 let result = svc.call(frame).await;
1864 assert!(result.is_ok());
1865 assert_eq!(recorder.recorded_error(), Some(ErrorKind::Other));
1866 }
1867}