Skip to main content

mcp_server_core/
observability.rs

1//! Logs, metrics and traces.
2//!
3//! [`init`] installs a `tracing` subscriber writing JSON (or pretty) logs to stdout, a Prometheus
4//! exporter when `METRICS_ADDR` is set (feature `prometheus`) and an OTLP trace exporter when
5//! `OTEL_EXPORTER_OTLP_ENDPOINT` is set (feature `otel`).
6//!
7//! Secrets and personal content stay out of logs by default. Services should log content (message
8//! bodies, search terms) only when [`content_logging_enabled`] says the operator asked for it with
9//! `DEBUG_UNSAFE_LOG_CONTENT`.
10
11#[cfg(feature = "otel")]
12use opentelemetry::trace::TracerProvider as _;
13#[cfg(feature = "otel")]
14use opentelemetry_sdk::Resource;
15#[cfg(feature = "otel")]
16use opentelemetry_sdk::trace::SdkTracerProvider;
17use std::net::SocketAddr;
18use std::str::FromStr;
19use std::sync::atomic::{AtomicBool, Ordering};
20use tracing_subscriber::layer::SubscriberExt;
21use tracing_subscriber::util::SubscriberInitExt;
22use tracing_subscriber::{EnvFilter, Layer};
23
24/// `LOG_FORMAT`.
25#[derive(Debug, Clone, Copy, PartialEq, Eq)]
26pub enum LogFormat {
27    /// One JSON object per line, for log collectors.
28    Json,
29    /// Multi-line, coloured output for people.
30    Pretty,
31}
32
33impl FromStr for LogFormat {
34    type Err = ();
35    fn from_str(s: &str) -> Result<Self, ()> {
36        match s.to_ascii_lowercase().as_str() {
37            "json" => Ok(LogFormat::Json),
38            "pretty" => Ok(LogFormat::Pretty),
39            _ => Err(()),
40        }
41    }
42}
43
44/// What [`init`] sets up.
45#[derive(Debug, Clone)]
46pub struct ObservabilityConfig {
47    /// The OpenTelemetry service name.
48    pub service_name: String,
49    /// The tracing target prefix of your crates, raised to debug by `debug`.
50    pub crate_target: String,
51    /// A `tracing` filter, `LOG_LEVEL`.
52    pub log_level: String,
53    /// `LOG_FORMAT`.
54    pub log_format: LogFormat,
55    /// `DEBUG`.
56    pub debug: bool,
57    /// `DEBUG_UNSAFE_LOG_CONTENT`.
58    pub unsafe_content: bool,
59    /// Whether `OTEL_EXPORTER_OTLP_ENDPOINT` is set.
60    pub otlp_enabled: bool,
61    /// `METRICS_ADDR`.
62    pub metrics_addr: Option<SocketAddr>,
63}
64
65static DEBUG: AtomicBool = AtomicBool::new(false);
66static CONTENT: AtomicBool = AtomicBool::new(false);
67
68/// Whether `DEBUG` (or `DEBUG_UNSAFE_LOG_CONTENT`) is on.
69pub fn debug_enabled() -> bool {
70    DEBUG.load(Ordering::Relaxed)
71}
72
73/// Whether the operator allowed content in logs with `DEBUG_UNSAFE_LOG_CONTENT`. Check it before
74/// logging anything a person wrote or read.
75pub fn content_logging_enabled() -> bool {
76    CONTENT.load(Ordering::Relaxed)
77}
78
79/// The filter directives for `level`, with `crate_target` and this crate at debug when `debug`.
80pub fn build_filter(level: &str, debug: bool, crate_target: &str) -> String {
81    if !debug {
82        return level.to_string();
83    }
84    format!("{level},{crate_target}=debug,mcp_server_core=debug")
85}
86
87/// Log levels from least to most verbose.
88const LEVELS: &[&str] = &["off", "error", "warn", "info", "debug", "trace"];
89
90/// Rewrites a `LOG_LEVEL` filter so it cannot reveal secrets: every target in `caps` (and its
91/// children) is capped at info and the rmcp targets in
92/// [`QUIET_LOG_TARGETS`](crate::service::QUIET_LOG_TARGETS), which log MCP session ids, at warn.
93/// A capped target is set to info, or to the level `LOG_LEVEL` sets for everything when that is
94/// lower, replacing any directive `LOG_LEVEL` gave the target itself. Every target
95/// in `always_on` is set to info whatever `LOG_LEVEL` says, which is how the audit log stays on.
96/// Span-scoped directives (containing `[`) are dropped, as they could re-enable a capped target.
97///
98/// ```
99/// use mcp_server_core::observability::log_filter;
100///
101/// let filter = log_filter("trace,noisy_dep=trace", &["noisy_dep"], &["audit"]);
102/// assert!(filter.starts_with("trace,noisy_dep=info,"));
103/// assert!(log_filter("error", &["noisy_dep"], &[]).contains("noisy_dep=error"));
104/// assert!(log_filter("off", &[], &["audit"]).contains("audit=info"));
105/// ```
106pub fn log_filter(level: &str, caps: &[&str], always_on: &[&str]) -> String {
107    use crate::service::QUIET_LOG_TARGETS;
108    let mut parts: Vec<String> = level
109        .split(',')
110        .map(str::trim)
111        .filter(|d| {
112            !d.is_empty()
113                && !d.contains('[')
114                && !caps
115                    .iter()
116                    .chain(always_on)
117                    .chain(QUIET_LOG_TARGETS)
118                    .any(|c| d.starts_with(c))
119        })
120        .map(String::from)
121        .collect();
122    let global = level
123        .split(',')
124        .map(str::trim)
125        .rfind(|d| LEVELS.contains(&d.to_ascii_lowercase().as_str()))
126        .map(str::to_ascii_lowercase)
127        .unwrap_or_else(|| "off".into());
128    let cap = |max: &str| -> String {
129        let rank = |l: &str| LEVELS.iter().position(|x| *x == l).unwrap_or(0);
130        if rank(&global) < rank(max) {
131            global.clone()
132        } else {
133            max.to_string()
134        }
135    };
136    parts.extend(caps.iter().map(|c| format!("{c}={}", cap("info"))));
137    parts.extend(always_on.iter().map(|c| format!("{c}=info")));
138    parts.extend(
139        QUIET_LOG_TARGETS
140            .iter()
141            .map(|t| format!("{t}={}", cap("warn"))),
142    );
143    parts.join(",")
144}
145
146/// Keeps exporters running; flushes traces when dropped. Hold it for the life of the process.
147pub struct ObservabilityGuard {
148    #[cfg(feature = "otel")]
149    provider: Option<SdkTracerProvider>,
150}
151
152impl Drop for ObservabilityGuard {
153    fn drop(&mut self) {
154        #[cfg(feature = "otel")]
155        if let Some(p) = self.provider.take() {
156            let _ = p.shutdown();
157        }
158    }
159}
160
161#[cfg(feature = "prometheus")]
162const DURATION_BUCKETS: &[f64] = &[
163    0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0,
164];
165
166/// Installs the global subscriber and exporters. Call once, early in `main`.
167pub fn init(cfg: &ObservabilityConfig) -> anyhow::Result<ObservabilityGuard> {
168    let debug = cfg.debug || cfg.unsafe_content;
169    DEBUG.store(debug, Ordering::Relaxed);
170    CONTENT.store(cfg.unsafe_content, Ordering::Relaxed);
171
172    let filter = EnvFilter::try_new(build_filter(&cfg.log_level, debug, &cfg.crate_target))?;
173    let fmt_layer = match cfg.log_format {
174        LogFormat::Json => tracing_subscriber::fmt::layer()
175            .json()
176            .with_current_span(true)
177            .flatten_event(true)
178            .boxed(),
179        LogFormat::Pretty => tracing_subscriber::fmt::layer().pretty().boxed(),
180    };
181
182    #[cfg(feature = "otel")]
183    let provider = if cfg.otlp_enabled {
184        let exporter = opentelemetry_otlp::SpanExporter::builder()
185            .with_http()
186            .build()?;
187        Some(
188            SdkTracerProvider::builder()
189                .with_batch_exporter(exporter)
190                .with_resource(
191                    Resource::builder()
192                        .with_service_name(cfg.service_name.clone())
193                        .build(),
194                )
195                .build(),
196        )
197    } else {
198        None
199    };
200    #[cfg(feature = "otel")]
201    let otel_layer = provider
202        .as_ref()
203        .map(|p| tracing_opentelemetry::layer().with_tracer(p.tracer(cfg.service_name.clone())));
204    #[cfg(not(feature = "otel"))]
205    let otel_layer: Option<tracing_subscriber::layer::Identity> = None;
206
207    tracing_subscriber::registry()
208        .with(filter)
209        .with(fmt_layer)
210        .with(otel_layer)
211        .try_init()?;
212
213    #[cfg(feature = "otel")]
214    if let Some(p) = &provider {
215        opentelemetry::global::set_tracer_provider(p.clone());
216    }
217    #[cfg(not(feature = "otel"))]
218    if cfg.otlp_enabled {
219        tracing::warn!(
220            "OTEL_EXPORTER_OTLP_ENDPOINT is set but this build has no OpenTelemetry support (feature `otel`): traces are not exported"
221        );
222    }
223
224    if let Some(addr) = cfg.metrics_addr {
225        #[cfg(feature = "prometheus")]
226        {
227            metrics_exporter_prometheus::PrometheusBuilder::new()
228                .with_http_listener(addr)
229                .set_buckets(DURATION_BUCKETS)?
230                .install()?;
231            tracing::info!(%addr, "prometheus metrics listener started");
232        }
233        #[cfg(not(feature = "prometheus"))]
234        tracing::warn!(
235            %addr,
236            "METRICS_ADDR is set but this build has no Prometheus support (feature `prometheus`): metrics are not served"
237        );
238    }
239    if cfg.unsafe_content {
240        tracing::warn!(
241            "DEBUG_UNSAFE_LOG_CONTENT is enabled: message content WILL be written to logs"
242        );
243    }
244    Ok(ObservabilityGuard {
245        #[cfg(feature = "otel")]
246        provider,
247    })
248}
249
250#[cfg(test)]
251mod tests {
252    use super::*;
253
254    #[test]
255    fn log_format_parses() {
256        assert!(matches!("json".parse::<LogFormat>(), Ok(LogFormat::Json)));
257        assert!(matches!(
258            "Pretty".parse::<LogFormat>(),
259            Ok(LogFormat::Pretty)
260        ));
261        assert!("xml".parse::<LogFormat>().is_err());
262    }
263
264    #[test]
265    fn filter_without_debug_is_level_as_given() {
266        assert_eq!(build_filter("info", false, "mcp_example"), "info");
267        assert_eq!(
268            build_filter("warn,mcp_example::api=trace", false, "mcp_example"),
269            "warn,mcp_example::api=trace"
270        );
271    }
272
273    #[test]
274    fn debug_raises_own_crates() {
275        assert_eq!(
276            build_filter("info", true, "mcp_example"),
277            "info,mcp_example=debug,mcp_server_core=debug"
278        );
279    }
280
281    const Q: &str = "rmcp::transport::streamable_http_server=warn,rmcp::transport::worker=warn";
282
283    #[test]
284    fn log_filter_caps_every_listed_target() {
285        assert_eq!(
286            log_filter("trace,a=trace,b::x=debug", &["a", "b"], &[]),
287            format!("trace,a=info,b=info,{Q}")
288        );
289        assert_eq!(
290            log_filter("warn", &[], &[]),
291            "warn,rmcp::transport::streamable_http_server=warn,rmcp::transport::worker=warn"
292        );
293        assert_eq!(
294            log_filter("info,[{x}]=trace,[x]=trace,debug", &["a"], &[]),
295            format!("info,debug,a=info,{Q}")
296        );
297        assert_eq!(
298            log_filter("debug,a=trace,a::client=trace", &["a"], &[]),
299            format!("debug,a=info,{Q}")
300        );
301    }
302
303    #[test]
304    fn caps_never_raise_a_target_above_log_level() {
305        assert_eq!(
306            log_filter("error", &["a"], &[]),
307            "error,a=error,rmcp::transport::streamable_http_server=error,rmcp::transport::worker=error"
308        );
309        assert_eq!(
310            log_filter("off", &[], &[]),
311            "off,rmcp::transport::streamable_http_server=off,rmcp::transport::worker=off"
312        );
313        assert!(log_filter("b=debug", &["a"], &[]).contains("a=off"));
314    }
315
316    #[test]
317    fn session_ids_stay_out_of_logs_whatever_the_level() {
318        use tracing::Level;
319        let filter = log_filter(
320            "trace,rmcp=trace,rmcp::transport::streamable_http_server::session::local=trace",
321            &[],
322            &[],
323        );
324        let subscriber = tracing_subscriber::registry().with(EnvFilter::new(filter));
325        tracing::subscriber::with_default(subscriber, || {
326            assert!(!tracing::enabled!(
327                target: "rmcp::transport::streamable_http_server::session::local",
328                Level::INFO
329            ));
330            assert!(tracing::enabled!(
331                target: "rmcp::transport::streamable_http_server::session::local",
332                Level::WARN
333            ));
334            assert!(!tracing::enabled!(
335                target: "rmcp::transport::streamable_http_server::tower",
336                Level::INFO
337            ));
338            // The transport worker's trace span carries the session id in its name.
339            assert!(!tracing::span_enabled!(
340                target: "rmcp::transport::worker",
341                Level::TRACE
342            ));
343            assert!(tracing::enabled!(target: "rmcp::handler", Level::TRACE));
344        });
345    }
346
347    #[test]
348    fn an_always_on_target_stays_on_at_info_at_any_level() {
349        use tracing::Level;
350        for level in [
351            "warn",
352            "error",
353            "off",
354            "info,audit=off",
355            "trace,audit=error",
356        ] {
357            let directives =
358                build_filter(&log_filter(level, &[], &["audit"]), false, "mcp_example");
359            let subscriber = tracing_subscriber::registry().with(EnvFilter::new(directives));
360            tracing::subscriber::with_default(subscriber, || {
361                assert!(tracing::enabled!(target: "audit", Level::INFO), "{level}");
362            });
363        }
364    }
365
366    #[test]
367    fn debug_covers_own_crates_but_not_capped_ones() {
368        use tracing::Level;
369        let level = log_filter("trace", &["noisy_dep"], &[]);
370        let directives = build_filter(&level, true, "mcp_example");
371        let subscriber = tracing_subscriber::registry().with(EnvFilter::new(directives));
372        tracing::subscriber::with_default(subscriber, || {
373            assert!(tracing::enabled!(target: "mcp_example_tools::api", Level::DEBUG));
374            assert!(tracing::enabled!(target: "mcp_example", Level::DEBUG));
375            assert!(!tracing::enabled!(target: "noisy_dep::client", Level::DEBUG));
376            assert!(tracing::enabled!(target: "noisy_dep::client", Level::INFO));
377        });
378        let directives = build_filter("info", true, "mcp_example");
379        let subscriber = tracing_subscriber::registry().with(EnvFilter::new(directives));
380        tracing::subscriber::with_default(subscriber, || {
381            assert!(tracing::enabled!(target: "mcp_example_tools", Level::DEBUG));
382            assert!(!tracing::enabled!(target: "hyper", Level::DEBUG));
383        });
384    }
385}