Skip to main content

redis_tower/
tracing_layer.rs

1//! Tower layer for tracing Redis commands at the Frame level.
2//!
3//! Wraps a `Service<Frame, Response=Frame>` and creates a tracing span
4//! for each Redis command, recording the command name, status, and
5//! OpenTelemetry DB semantic convention attributes.
6//!
7//! # Example
8//!
9//! ```no_run
10//! # async fn example() -> Result<(), Box<dyn std::error::Error>> {
11//! use std::time::Duration;
12//! use tower::ServiceBuilder;
13//! use redis_tower::FrameService;
14//! use redis_tower::tracing_layer::TracingLayer;
15//!
16//! // Basic -- no peer address:
17//! let basic = ServiceBuilder::new()
18//!     .layer(TracingLayer::new())
19//!     .service(FrameService::connect("127.0.0.1:6379").await?);
20//!
21//! // With server address for OTel per-node flame graphs:
22//! let with_peer = ServiceBuilder::new()
23//!     .layer(TracingLayer::with_peer("127.0.0.1:6379"))
24//!     .service(FrameService::connect("127.0.0.1:6379").await?);
25//!
26//! // Warn on commands slower than 50ms (a differentiator neither
27//! // redis-rs nor fred ships):
28//! let slow_command_warnings = ServiceBuilder::new()
29//!     .layer(TracingLayer::new().with_slow_command_threshold(Duration::from_millis(50)))
30//!     .service(FrameService::connect("127.0.0.1:6379").await?);
31//! # let _ = (basic, with_peer, slow_command_warnings);
32//! # Ok(())
33//! # }
34//! ```
35
36use std::future::Future;
37use std::pin::Pin;
38use std::task::{Context, Poll};
39use std::time::{Duration, Instant};
40
41use redis_tower_core::{Frame, RedisError};
42use tower_layer::Layer;
43use tower_service::Service;
44use tracing::Instrument;
45
46/// Tower `Layer` that adds tracing spans for Redis commands.
47///
48/// Each call creates an `info`-level span carrying the **stable OpenTelemetry
49/// database semantic conventions** so it lights up Redis dashboards in Grafana,
50/// Datadog, and Dynatrace out of the box:
51///
52/// - `otel.name` set to the command (the exported span name is dynamic, e.g.
53///   `GET`, not a static `redis.command`),
54/// - `db.system.name = "redis"` and `db.operation.name` = the command,
55/// - `server.address` / `server.port` (split from the peer address),
56/// - `otel.status_code` / `otel.status_message`, and `error.type` on failure.
57///
58/// The deprecated `db.system` / `db.statement` attributes are emitted only when
59/// [`with_legacy_attributes`](Self::with_legacy_attributes) is set, for callers
60/// mid-migration.
61///
62/// Use [`TracingLayer::with_peer`] to set the peer address for per-node flame
63/// graphs in cluster or sentinel topologies, or
64/// [`with_slow_command_threshold`](Self::with_slow_command_threshold) to log a
65/// `WARN` event for commands that exceed a latency budget.
66#[derive(Clone, Debug, Default)]
67pub struct TracingLayer {
68    server_address: Option<String>,
69    legacy_attributes: bool,
70    slow_command_threshold: Option<Duration>,
71}
72
73impl TracingLayer {
74    /// Create a new tracing layer without a peer address.
75    pub fn new() -> Self {
76        Self::default()
77    }
78
79    /// Create a new tracing layer with a peer address.
80    ///
81    /// The address is split into the `server.address` and `server.port` OTel
82    /// attributes on every span, enabling per-node flame graphs in backends
83    /// like Grafana Tempo.
84    pub fn with_peer(addr: impl Into<String>) -> Self {
85        Self {
86            server_address: Some(addr.into()),
87            legacy_attributes: false,
88            slow_command_threshold: None,
89        }
90    }
91
92    /// Also emit the deprecated `db.system` / `db.statement` attributes
93    /// alongside the stable conventions, for dashboards not yet migrated.
94    pub fn with_legacy_attributes(mut self) -> Self {
95        self.legacy_attributes = true;
96        self
97    }
98
99    /// Log a `WARN`-level event for any command whose round-trip latency meets
100    /// or exceeds `threshold`.
101    ///
102    /// The event is emitted inside the command's span and carries the command
103    /// name, the measured `elapsed_ms`, and the configured `threshold_ms`. Slow
104    /// commands are still recorded as normal spans; this only adds the warning.
105    /// Disabled by default -- a differentiator neither `redis-rs` nor `fred`
106    /// ships out of the box.
107    pub fn with_slow_command_threshold(mut self, threshold: Duration) -> Self {
108        self.slow_command_threshold = Some(threshold);
109        self
110    }
111}
112
113impl<S> Layer<S> for TracingLayer {
114    type Service = TracingService<S>;
115
116    fn layer(&self, inner: S) -> Self::Service {
117        TracingService {
118            inner,
119            server_address: self.server_address.clone(),
120            legacy_attributes: self.legacy_attributes,
121            slow_command_threshold: self.slow_command_threshold,
122        }
123    }
124}
125
126/// Tower `Service` that instruments Redis commands with tracing spans.
127///
128/// Created by [`TracingLayer`]. Extracts the command name from the request
129/// `Frame::Array` and records OTel DB semantic convention attributes on
130/// each span.
131#[derive(Clone, Debug)]
132pub struct TracingService<S> {
133    inner: S,
134    server_address: Option<String>,
135    legacy_attributes: bool,
136    slow_command_threshold: Option<Duration>,
137}
138
139impl<S> TracingService<S> {
140    /// Create a new tracing service wrapping an inner service.
141    pub fn new(inner: S) -> Self {
142        Self {
143            inner,
144            server_address: None,
145            legacy_attributes: false,
146            slow_command_threshold: None,
147        }
148    }
149
150    /// Create a new tracing service with an explicit peer address.
151    pub fn with_peer(inner: S, addr: impl Into<String>) -> Self {
152        Self {
153            inner,
154            server_address: Some(addr.into()),
155            legacy_attributes: false,
156            slow_command_threshold: None,
157        }
158    }
159
160    /// Also emit the deprecated `db.system` / `db.statement` attributes.
161    pub fn with_legacy_attributes(mut self) -> Self {
162        self.legacy_attributes = true;
163        self
164    }
165
166    /// Log a `WARN`-level event for any command whose round-trip latency meets
167    /// or exceeds `threshold`. See [`TracingLayer::with_slow_command_threshold`].
168    pub fn with_slow_command_threshold(mut self, threshold: Duration) -> Self {
169        self.slow_command_threshold = Some(threshold);
170        self
171    }
172}
173
174/// Extract the command name from a request frame.
175///
176/// Expects `Frame::Array(Some(vec![Frame::BulkString(Some(name)), ...]))`.
177/// Returns `"UNKNOWN"` if the frame does not match this pattern.
178fn extract_command_name(frame: &Frame) -> String {
179    if let Frame::Array(Some(items)) = frame
180        && let Some(Frame::BulkString(Some(bytes))) = items.first()
181        && let Ok(name) = std::str::from_utf8(bytes)
182    {
183        return name.to_ascii_uppercase();
184    }
185    "UNKNOWN".to_string()
186}
187
188impl<S> Service<Frame> for TracingService<S>
189where
190    S: Service<Frame, Response = Frame, Error = RedisError>,
191    S::Future: Send + 'static,
192{
193    type Response = Frame;
194    type Error = RedisError;
195    type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
196
197    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
198        self.inner.poll_ready(cx)
199    }
200
201    fn call(&mut self, request: Frame) -> Self::Future {
202        let cmd_name = extract_command_name(&request);
203        // `otel.name` drives the exported OTel span name (dynamic, per command);
204        // the static macro name is what non-OTel tracing subscribers see.
205        let span = tracing::info_span!(
206            "redis.command",
207            otel.name = %cmd_name,
208            otel.kind = "client",
209            db.system.name = "redis",
210            db.operation.name = %cmd_name,
211            server.address = tracing::field::Empty,
212            server.port = tracing::field::Empty,
213            otel.status_code = tracing::field::Empty,
214            otel.status_message = tracing::field::Empty,
215            error.type = tracing::field::Empty,
216            // Convenience field kept for existing filters.
217            redis.command = %cmd_name,
218            // Deprecated; recorded only with legacy compat enabled.
219            db.system = tracing::field::Empty,
220            db.statement = tracing::field::Empty,
221        );
222
223        if let Some(ref addr) = self.server_address {
224            let (host, port) = split_server_addr(addr);
225            span.record("server.address", host);
226            if let Some(port) = port {
227                span.record("server.port", i64::from(port));
228            }
229        }
230
231        if self.legacy_attributes {
232            span.record("db.system", "redis");
233            span.record("db.statement", cmd_name.as_str());
234        }
235
236        let slow_threshold = self.slow_command_threshold;
237        let started = Instant::now();
238        let cmd_for_log = cmd_name.clone();
239        let future = self.inner.call(request);
240
241        Box::pin(
242            async move {
243                let result = future.await;
244                let span = tracing::Span::current();
245                let elapsed = started.elapsed();
246                if is_slow(slow_threshold, elapsed) {
247                    let threshold_ms = slow_threshold.map_or(0, |t| t.as_millis() as u64);
248                    tracing::warn!(
249                        command = %cmd_for_log,
250                        elapsed_ms = elapsed.as_millis() as u64,
251                        threshold_ms,
252                        "Redis command exceeded slow-command threshold"
253                    );
254                }
255                match &result {
256                    Ok(Frame::Error(bytes)) => {
257                        let msg = String::from_utf8_lossy(bytes);
258                        span.record("otel.status_code", "ERROR");
259                        span.record("otel.status_message", msg.as_ref());
260                        // The Redis error prefix (WRONGTYPE, MOVED, ...).
261                        span.record("error.type", msg.split_whitespace().next().unwrap_or("ERR"));
262                    }
263                    Ok(_) => {
264                        span.record("otel.status_code", "OK");
265                    }
266                    Err(e) => {
267                        span.record("otel.status_code", "ERROR");
268                        span.record("otel.status_message", e.to_string().as_str());
269                        span.record("error.type", error_type(e));
270                    }
271                }
272                result
273            }
274            .instrument(span),
275        )
276    }
277}
278
279/// Split a `host:port` peer address into `(host, port)` for the OTel
280/// `server.address` / `server.port` attributes. IPv6 addresses (which contain
281/// colons) split on the last colon; an address with no port yields `None`.
282fn split_server_addr(addr: &str) -> (&str, Option<u16>) {
283    match addr.rsplit_once(':') {
284        Some((host, port)) => (host, port.parse::<u16>().ok()),
285        None => (addr, None),
286    }
287}
288
289/// The OTel `error.type` for a [`RedisError`]: the server error prefix for a
290/// Redis error (`WRONGTYPE`, `MOVED`, ...), otherwise a stable category.
291fn error_type(err: &RedisError) -> &str {
292    match err {
293        RedisError::Redis(msg) => msg.split_whitespace().next().unwrap_or("ERR"),
294        RedisError::Connection { .. } | RedisError::ConnectionClosed => "CONNECTION",
295        RedisError::Protocol(_) => "PROTOCOL",
296        RedisError::CommandTimeout => "COMMAND_TIMEOUT",
297        RedisError::ConnectTimeout => "CONNECT_TIMEOUT",
298        RedisError::CircuitOpen => "CIRCUIT_OPEN",
299        RedisError::PoolAcquisitionTimeout { .. } => "POOL_TIMEOUT",
300        RedisError::QueueFull => "QUEUE_FULL",
301        _ => "CLIENT_ERROR",
302    }
303}
304
305/// Whether a command's `elapsed` round-trip duration crosses the configured
306/// slow-command `threshold`. Returns `false` when no threshold is set, so a
307/// disabled threshold never flags a command as slow.
308fn is_slow(threshold: Option<Duration>, elapsed: Duration) -> bool {
309    threshold.is_some_and(|t| elapsed >= t)
310}
311
312#[cfg(test)]
313mod tests {
314    use super::*;
315    use bytes::Bytes;
316
317    #[test]
318    fn extracts_get_command() {
319        let frame = Frame::Array(Some(vec![
320            Frame::BulkString(Some(Bytes::from("get"))),
321            Frame::BulkString(Some(Bytes::from("mykey"))),
322        ]));
323        assert_eq!(extract_command_name(&frame), "GET");
324    }
325
326    #[test]
327    fn extracts_set_command() {
328        let frame = Frame::Array(Some(vec![
329            Frame::BulkString(Some(Bytes::from("SET"))),
330            Frame::BulkString(Some(Bytes::from("key"))),
331            Frame::BulkString(Some(Bytes::from("value"))),
332        ]));
333        assert_eq!(extract_command_name(&frame), "SET");
334    }
335
336    #[test]
337    fn unknown_for_non_array() {
338        let frame = Frame::SimpleString(Bytes::from("PING"));
339        assert_eq!(extract_command_name(&frame), "UNKNOWN");
340    }
341
342    #[test]
343    fn unknown_for_empty_array() {
344        let frame = Frame::Array(Some(vec![]));
345        assert_eq!(extract_command_name(&frame), "UNKNOWN");
346    }
347
348    #[test]
349    fn unknown_for_null_array() {
350        let frame = Frame::Array(None);
351        assert_eq!(extract_command_name(&frame), "UNKNOWN");
352    }
353
354    #[test]
355    fn split_server_addr_ipv4() {
356        assert_eq!(
357            split_server_addr("127.0.0.1:6379"),
358            ("127.0.0.1", Some(6379))
359        );
360    }
361
362    #[test]
363    fn split_server_addr_ipv6_splits_on_last_colon() {
364        assert_eq!(split_server_addr("::1:6380"), ("::1", Some(6380)));
365    }
366
367    #[test]
368    fn split_server_addr_no_port() {
369        assert_eq!(split_server_addr("localhost"), ("localhost", None));
370    }
371
372    #[test]
373    fn split_server_addr_bad_port() {
374        // Host kept, port dropped when it doesn't parse.
375        assert_eq!(split_server_addr("host:notaport"), ("host", None));
376    }
377
378    #[test]
379    fn error_type_uses_redis_prefix() {
380        let err = RedisError::Redis("WRONGTYPE Operation against a key".into());
381        assert_eq!(error_type(&err), "WRONGTYPE");
382        let moved = RedisError::Redis("MOVED 3999 127.0.0.1:6381".into());
383        assert_eq!(error_type(&moved), "MOVED");
384    }
385
386    #[test]
387    fn error_type_categorizes_client_errors() {
388        assert_eq!(error_type(&RedisError::ConnectionClosed), "CONNECTION");
389        assert_eq!(error_type(&RedisError::CommandTimeout), "COMMAND_TIMEOUT");
390        assert_eq!(error_type(&RedisError::CircuitOpen), "CIRCUIT_OPEN");
391    }
392
393    #[test]
394    fn tracing_layer_new_has_no_peer() {
395        let layer = TracingLayer::new();
396        assert!(layer.server_address.is_none());
397    }
398
399    #[test]
400    fn tracing_layer_with_peer_sets_address() {
401        let layer = TracingLayer::with_peer("127.0.0.1:6379");
402        assert_eq!(layer.server_address.as_deref(), Some("127.0.0.1:6379"));
403    }
404
405    #[test]
406    fn tracing_layer_default_has_no_peer() {
407        let layer = TracingLayer::default();
408        assert!(layer.server_address.is_none());
409    }
410
411    #[test]
412    fn is_slow_false_without_threshold() {
413        assert!(!is_slow(None, Duration::from_secs(10)));
414    }
415
416    #[test]
417    fn is_slow_true_when_elapsed_exceeds_threshold() {
418        assert!(is_slow(
419            Some(Duration::from_millis(50)),
420            Duration::from_millis(75)
421        ));
422    }
423
424    #[test]
425    fn is_slow_true_when_elapsed_equals_threshold() {
426        assert!(is_slow(
427            Some(Duration::from_millis(50)),
428            Duration::from_millis(50)
429        ));
430    }
431
432    #[test]
433    fn is_slow_false_when_elapsed_below_threshold() {
434        assert!(!is_slow(
435            Some(Duration::from_millis(50)),
436            Duration::from_millis(49)
437        ));
438    }
439
440    #[test]
441    fn layer_with_slow_command_threshold_sets_field() {
442        let layer = TracingLayer::new().with_slow_command_threshold(Duration::from_millis(100));
443        assert_eq!(
444            layer.slow_command_threshold,
445            Some(Duration::from_millis(100))
446        );
447    }
448
449    #[test]
450    fn layer_without_slow_command_threshold_is_none() {
451        assert!(TracingLayer::new().slow_command_threshold.is_none());
452    }
453
454    #[test]
455    fn service_with_slow_command_threshold_sets_field() {
456        let svc = TracingService::new(()).with_slow_command_threshold(Duration::from_millis(100));
457        assert_eq!(svc.slow_command_threshold, Some(Duration::from_millis(100)));
458    }
459
460    #[test]
461    fn layer_propagates_slow_command_threshold_to_service() {
462        let layer = TracingLayer::new().with_slow_command_threshold(Duration::from_millis(25));
463        let svc = layer.layer(());
464        assert_eq!(svc.slow_command_threshold, Some(Duration::from_millis(25)));
465    }
466}