forge_ops_tracker/

lib.rs

//! ForgeOps error tracking client for ForgeOps:
//!
//! ```no_run
//! forge_ops_tracker::init(|c| {
//!     c.dsn = Some("https://<api_key>@getforgeops.net/api/v1/events".to_string());
//! });
//! ```
//!
//! See the README for what gets captured automatically vs. what needs an explicit
//! [`capture_error`] call. A from-scratch port of `gems/forge_ops_tracker` (the Rails client):
//! see that gem's README for the shared design rationale behind the pieces this crate is built
//! from (Configuration, EventBuilder, DeliveryQueue, Reporter, Client).

mod breadcrumb_buffer;
mod client;
mod configuration;
mod delivery_queue;
mod event_builder;
mod histogram_bucketer;
mod metric_buffer;
mod performance_flusher;
mod pii_scrubber;
mod reporter;
mod span_buffer;
mod span_queue;
mod sql_statement;
#[cfg(test)]
mod test_support;

pub use breadcrumb_buffer::Breadcrumb;
pub use configuration::Configuration;
pub use event_builder::{Event, Frame};
pub use pii_scrubber::Value;
pub use sql_statement::SqlObjects;

use std::cell::RefCell;
use std::collections::HashMap;
use std::sync::{Arc, OnceLock, RwLock};

use client::Client;
use delivery_queue::DeliveryQueue;
use metric_buffer::{Kind, MetricBuffer};
use performance_flusher::PerformanceFlusher;
use reporter::Reporter;
use span_queue::SpanQueue;

struct State {
    configuration: Arc<RwLock<Configuration>>,
    reporter: Reporter,
    performance_flusher: Arc<PerformanceFlusher>,
    span_queue: SpanQueue,
    metrics: Arc<MetricBuffer>,
    infrastructure_metrics: Arc<MetricBuffer>,
}

static STATE: OnceLock<State> = OnceLock::new();

thread_local! {
    // The user set via `set_user`, if any. A plain thread-local, not a process-wide global: the
    // right choice for the thread-per-request model this crate's own synchronous, non-async
    // design naturally pairs with (see Cargo.toml's own comment on why `ureq`, not an async HTTP
    // client, was chosen), the same reasoning `gems/forge_ops_tracker` documents for its own
    // `Thread.current` use. **Does not propagate across an `.await` in an async runtime**: unlike
    // Ruby's green/native threads, a single OS thread in an async executor (tokio, async-std)
    // interleaves multiple unrelated tasks, so a value set on one task can leak into, or simply
    // never reach, another. A host app built on an async runtime should pass `user` explicitly to
    // `capture_error`/`capture_error_with_class` instead of relying on `set_user`, the same way
    // `sdks/node` needs `AsyncLocalStorage` rather than a bare thread-local for the identical
    // reason.
    static CURRENT_USER: RefCell<Option<HashMap<String, Value>>> = const { RefCell::new(None) };
}

fn current_user() -> Option<HashMap<String, Value>> {
    CURRENT_USER.with(|u| u.borrow().clone())
}

fn state() -> &'static State {
    STATE.get_or_init(|| {
        let configuration = Arc::new(RwLock::new(Configuration::new()));
        let client = Arc::new(Client::new(Arc::clone(&configuration)));
        let queue_size = configuration.read().unwrap().queue_size;
        let delivery_queue = DeliveryQueue::new(queue_size, Arc::clone(&client));
        let span_queue = SpanQueue::new(queue_size, Arc::clone(&client));
        let metrics = MetricBuffer::new(
            Kind::Custom,
            Arc::clone(&configuration),
            Arc::clone(&client),
        );
        let infrastructure_metrics = MetricBuffer::new(
            Kind::Infrastructure,
            Arc::clone(&configuration),
            Arc::clone(&client),
        );
        let reporter = Reporter::new(Arc::clone(&configuration), delivery_queue);
        let performance_flusher = PerformanceFlusher::new(Arc::clone(&configuration), client);
        State {
            configuration,
            reporter,
            performance_flusher,
            span_queue,
            metrics,
            infrastructure_metrics,
        }
    })
}

/// Configures the client. Call once at startup, before your server starts accepting requests.
/// Pass a closure to set any [`Configuration`] field:
///
/// ```no_run
/// forge_ops_tracker::init(|c| {
///     c.dsn = Some("https://<api_key>@getforgeops.net/api/v1/events".to_string());
///     c.release = Some("a1b2c3d".to_string());
/// });
/// ```
///
/// Installs the global panic hook (see [`install_panic_hook`]) unless
/// `Configuration.install_panic_hook` is set to `false` inside the closure.
pub fn init(configure: impl FnOnce(&mut Configuration)) {
    let s = state();
    let install_hook = {
        let mut config = s.configuration.write().unwrap();
        configure(&mut config);
        config.install_panic_hook
    };
    if install_hook {
        install_panic_hook();
    }
}

/// Reports an error you've already handled. Call it right at the point you'd otherwise just log
/// it:
///
/// ```no_run
/// # use std::collections::HashMap;
/// # fn charge_card() -> Result<(), std::io::Error> { Ok(()) }
/// if let Err(err) = charge_card() {
///     forge_ops_tracker::capture_error(&err, HashMap::new(), None);
/// }
/// ```
///
/// The backtrace is captured right here, at the call site: unlike Python/Java/PHP, a plain Rust
/// `std::error::Error` carries no stack of its own, so `capture_error` has to be the one call that
/// knows where the trace starts. `exception_class` is inferred via [`std::any::type_name`], which
/// needs `E` to be a concrete, statically-known type: for a `Box<dyn Error>` or other trait
/// object, where that isn't possible, use [`capture_error_with_class`] instead and supply the
/// class yourself.
///
/// `user` defaults to whatever [`set_user`] last established on this thread, if anything (`None`
/// here means "use that", not "no user"); pass `Some(..)` to override it for this one report.
pub fn capture_error<E: std::error::Error>(
    err: &E,
    context: HashMap<String, Value>,
    user: Option<HashMap<String, Value>>,
) {
    capture_error_with_class(std::any::type_name::<E>(), err, context, user);
}

/// The same as [`capture_error`], but for a `&dyn std::error::Error` (a `Box<dyn Error>`, a trait
/// object) whose concrete type isn't known at the call site, so `exception_class` has to be
/// supplied explicitly rather than inferred.
pub fn capture_error_with_class(
    exception_class: &str,
    err: &dyn std::error::Error,
    context: HashMap<String, Value>,
    user: Option<HashMap<String, Value>>,
) {
    let s = state();
    s.reporter.report_with_captured_backtrace(
        exception_class,
        &err.to_string(),
        context,
        user.or_else(current_user),
        breadcrumb_buffer::current_breadcrumbs(),
    );
}

/// The same as [`capture_error`], for an error caused by a database call: pass the SQL that ran.
/// A Rust error carries no statement of its own and no Rust database crate puts one on its error
/// types, so the code that ran the query has to hand it over.
///
/// With `Configuration::capture_sql_objects` on (the default), the names of the stored procedure,
/// table and view the statement touched are sent, so an issue says where to start looking. With
/// `Configuration::capture_sql_statement` on too (off by default), the statement itself is sent as
/// well, with every string and number replaced by `?` first. The raw statement never leaves this
/// process either way.
///
/// ```ignore
/// if let Err(err) = sqlx::query(QUERY).bind(id).execute(&pool).await {
///     forge_ops_tracker::capture_error_with_sql(&err, QUERY, HashMap::new(), None);
/// }
/// ```
pub fn capture_error_with_sql<E: std::error::Error>(
    err: &E,
    statement: &str,
    context: HashMap<String, Value>,
    user: Option<HashMap<String, Value>>,
) {
    capture_error_with_class_and_sql(std::any::type_name::<E>(), err, statement, context, user);
}

/// The same as [`capture_error_with_sql`], but for a `&dyn std::error::Error` whose concrete type
/// isn't known at the call site, so `exception_class` is supplied explicitly.
pub fn capture_error_with_class_and_sql(
    exception_class: &str,
    err: &dyn std::error::Error,
    statement: &str,
    context: HashMap<String, Value>,
    user: Option<HashMap<String, Value>>,
) {
    let s = state();
    s.reporter.report_with_captured_backtrace_and_sql(
        exception_class,
        &err.to_string(),
        context,
        user.or_else(current_user),
        breadcrumb_buffer::current_breadcrumbs(),
        statement,
    );
}

/// Manually attaches an affected user to whatever gets reported from here on, *on this thread*
/// (an explicit [`capture_error`]/[`capture_error_with_class`] call with no `user` argument, or a
/// panic the installed hook catches): there's no way to automatically detect "the current user"
/// the way a server-side web framework with its own session/auth middleware can, so call this
/// yourself, e.g. right after sign-in. `id`/`email`/`username` are all independently optional;
/// call with an empty map to clear whatever was set, e.g. on sign-out. See this crate's own
/// `CURRENT_USER` thread-local (in the source) for why this is thread-local, and the real caveat
/// that comes with that choice under an async runtime.
pub fn set_user(user: HashMap<String, Value>) {
    let user = if user.is_empty() { None } else { Some(user) };
    CURRENT_USER.with(|u| *u.borrow_mut() = user);
}

/// Records one entry into the current thread's breadcrumb trail: a query, an outbound call, or
/// anything worth remembering right up to the moment something actually goes wrong. `category`
/// defaults to `"custom"` and `level` to `"info"` when passed an empty string. A no-op, not an
/// error, when `Configuration.track_breadcrumbs` is `false`.
///
/// This crate has no web framework integration of its own (unlike `sdks/go`'s net/http/Gin
/// middleware), so there's no automatic breadcrumb source and no middleware to start a fresh trail
/// per request on its own: call [`clear_breadcrumbs`] yourself at the start of each request, the
/// same place you'd already be calling [`set_user`] from, or entries from an earlier request
/// handled on a reused thread will bleed into this one's own report.
pub fn add_breadcrumb(message: &str, category: &str, level: &str, data: HashMap<String, Value>) {
    let config = state().configuration.read().unwrap();
    breadcrumb_buffer::add_breadcrumb(
        config.track_breadcrumbs,
        config.max_breadcrumbs,
        message,
        category,
        level,
        data,
    );
}

/// Clears the current thread's breadcrumb trail. See [`add_breadcrumb`]'s own doc for why calling
/// this yourself, at the start of each request, is this crate's responsibility to ask of you
/// rather than something it can do on its own.
pub fn clear_breadcrumbs() {
    breadcrumb_buffer::clear_breadcrumbs();
}

/// Records one timed call's duration, in milliseconds, under `transaction_name`: tallied
/// in-process (count, total, max) and flushed periodically as one small aggregate report, for the
/// Performance page's per-transaction table, not one network call per call. A no-op when
/// `Configuration.track_performance` is `false` or reporting isn't enabled for this environment.
///
/// This crate has no web framework integration of its own (unlike `sdks/go`'s net/http/Gin
/// middleware), so nothing is timed automatically: wrap whatever you want on the Performance page
/// yourself, e.g. a request handler, with [`time_transaction`], or call this directly with a
/// duration you measured. Keep `transaction_name` low-cardinality (`"GET /users/:id"`, not
/// `"GET /users/42"`): every distinct name is its own row.
pub fn record_performance(transaction_name: &str, duration_ms: f64) {
    state()
        .performance_flusher
        .record(transaction_name, duration_ms);
}

/// Runs `f`, records how long it took under `transaction_name` (see [`record_performance`]), and
/// returns whatever `f` returned. Recorded even if `f` panics: the duration up to the panic is
/// still a real duration, and a handler that panics is exactly one worth seeing on the
/// Performance page.
///
/// ```no_run
/// # fn handle_request() -> u32 { 200 }
/// let status = forge_ops_tracker::time_transaction("GET /users/:id", || handle_request());
/// ```
pub fn time_transaction<T>(transaction_name: &str, f: impl FnOnce() -> T) -> T {
    struct Timing<'a> {
        name: &'a str,
        started_at: std::time::Instant,
    }
    impl Drop for Timing<'_> {
        fn drop(&mut self) {
            record_performance(self.name, self.started_at.elapsed().as_secs_f64() * 1000.0);
        }
    }

    let _timing = Timing {
        name: transaction_name,
        started_at: std::time::Instant::now(),
    };
    f()
}

/// Delivers whatever has been tallied so far right now, instead of waiting for the next
/// `performance_flush_interval` tick. The background flush thread is a daemon: it does not run on
/// a normal process exit the way the Ruby gem's `at_exit` hook does (Rust has no equivalent), so a
/// short-lived program, or one about to shut down, should call this itself to avoid losing the
/// last partial window.
pub fn flush_performance() {
    state().performance_flusher.flush();
}

/// Records a named business metric (a signup, a payment, anything you want to name), buffered and
/// flushed periodically as one batch rather than one network call per capture. Pass `1.0` for a bare
/// counter-style call ("a signup happened") or a real magnitude ("a $49 payment"); it may be negative
/// (a refund). A no-op when the client isn't enabled (no DSN, or this environment isn't in
/// `enabled_environments`), and a NaN or infinite value is dropped.
///
/// Rust has no exit hook to flush from, so a short-lived program should call [`flush_metrics`]
/// before it returns from `main`.
///
/// ```no_run
/// forge_ops_tracker::capture_metric("signup", 1.0);
/// forge_ops_tracker::capture_metric("payment", 49.0);
/// ```
pub fn capture_metric(name: &str, value: f64) {
    let s = state();
    let (enabled, environment, release) = {
        let config = s.configuration.read().unwrap();
        (
            config.is_enabled(),
            config.environment.clone(),
            config.release.clone(),
        )
    };
    if !enabled {
        return;
    }
    let release = release
        .as_deref()
        .map(pii_scrubber::json_string)
        .unwrap_or_else(|| "null".to_string());
    s.metrics.record(
        value,
        &format!(
            "\"metric_name\":{},\"value\":{value},\"environment\":{},\"release\":{release}",
            pii_scrubber::json_string(name),
            pii_scrubber::json_string(&environment),
        ),
    );
}

/// Records one infrastructure reading (CPU, memory, disk, anything else a program of yours reads)
/// from one of your own hosts. `hostname` defaults to `Configuration.server_name` when `None`, so a
/// script running on the box it reports about needs no argument. Same buffered-batch delivery and
/// no-op-when-disabled contract as [`capture_metric`].
///
/// ```no_run
/// forge_ops_tracker::capture_infrastructure_metric("cpu", 0.42, None);
/// forge_ops_tracker::capture_infrastructure_metric("disk", 0.81, Some("db-1"));
/// forge_ops_tracker::flush_metrics();
/// ```
pub fn capture_infrastructure_metric(name: &str, value: f64, hostname: Option<&str>) {
    let s = state();
    let (enabled, server_name) = {
        let config = s.configuration.read().unwrap();
        (config.is_enabled(), config.server_name.clone())
    };
    if !enabled {
        return;
    }
    let hostname = hostname
        .map(str::to_string)
        .or(server_name)
        .unwrap_or_default();
    s.infrastructure_metrics.record(
        value,
        &format!(
            "\"metric_name\":{},\"value\":{value},\"hostname\":{}",
            pii_scrubber::json_string(name),
            pii_scrubber::json_string(&hostname),
        ),
    );
}

/// Delivers every buffered metric and infrastructure reading right now, instead of waiting for the
/// next flush interval. The background flush thread is a daemon and Rust has no exit hook, so call
/// this before a short-lived program ends.
pub fn flush_metrics() {
    let s = state();
    s.metrics.flush();
    s.infrastructure_metrics.flush();
}

/// Runs `f` as a trace named `root_name` (for example `"GET /checkout"` or `"job:reindex"`): every
/// [`span`] and [`record_span`] inside it, on this thread, nests beneath this root. When `f` took
/// at least `Configuration.trace_capture_threshold` (1 second by default) the whole trace is sent
/// to ForgeOps, so fast calls cost nothing on the wire. Sent even if `f` panics. Called inside an
/// already-open trace it just records a span instead.
///
/// This crate has no web framework integration, so nothing starts a trace automatically: wrap
/// whatever you want traced, typically a request handler or a background job.
///
/// ```no_run
/// # fn handle_request() -> u32 { 200 }
/// let status = forge_ops_tracker::trace("GET /checkout", || handle_request());
/// ```
pub fn trace<T>(root_name: &str, f: impl FnOnce() -> T) -> T {
    let (owns_trace, threshold) = {
        let config = state().configuration.read().unwrap();
        (span_buffer::begin(&config), config.trace_capture_threshold)
    };
    if !owns_trace {
        return span(root_name, "service", HashMap::new(), f);
    }

    struct Root<'a> {
        name: &'a str,
        started_at: std::time::SystemTime,
        timer: std::time::Instant,
        threshold: std::time::Duration,
    }
    impl Drop for Root<'_> {
        fn drop(&mut self) {
            let duration_ms = self.timer.elapsed().as_secs_f64() * 1000.0;
            if let Some(body) = span_buffer::end(
                self.threshold,
                self.name,
                "controller",
                self.started_at,
                duration_ms,
            ) {
                state().span_queue.push(body);
            }
        }
    }

    let _root = Root {
        name: root_name,
        started_at: std::time::SystemTime::now(),
        timer: std::time::Instant::now(),
        threshold,
    };
    f()
}

/// Times `f` as a child span of whatever span is open on this thread (or of the trace's root),
/// returning what `f` returned. Outside a [`trace`] it just runs `f`. Recorded even if `f` panics.
/// `kind` is one of `"controller"`, `"service"`, `"database"`, `"redis"`, `"http"`, `"job"`,
/// `"other"` (anything else is sent as `"other"`).
///
/// ```no_run
/// # use std::collections::HashMap;
/// # fn charge(id: u32) -> u32 { id }
/// let charged = forge_ops_tracker::span("charge card", "service", HashMap::new(), || charge(7));
/// ```
pub fn span<T>(name: &str, kind: &str, data: HashMap<String, Value>, f: impl FnOnce() -> T) -> T {
    let Some((id, parent)) = span_buffer::open_span() else {
        return f();
    };

    struct Open<'a> {
        id: String,
        parent: String,
        name: &'a str,
        kind: &'a str,
        data: HashMap<String, Value>,
        started_at: std::time::SystemTime,
        timer: std::time::Instant,
    }
    impl Drop for Open<'_> {
        fn drop(&mut self) {
            span_buffer::close_span(
                std::mem::take(&mut self.id),
                std::mem::take(&mut self.parent),
                self.name,
                self.kind,
                self.started_at,
                self.timer.elapsed().as_secs_f64() * 1000.0,
                std::mem::take(&mut self.data),
            );
        }
    }

    let _open = Open {
        id,
        parent,
        name,
        kind,
        data,
        started_at: std::time::SystemTime::now(),
        timer: std::time::Instant::now(),
    };
    f()
}

/// Records a span you timed yourself under the current one; a no-op outside a [`trace`].
pub fn record_span(
    name: &str,
    kind: &str,
    started_at: std::time::SystemTime,
    duration_ms: f64,
    data: HashMap<String, Value>,
) {
    span_buffer::record_leaf(name, kind, started_at, duration_ms, data);
}

/// Installs a global panic hook that reports any panic on any thread, then calls whatever hook
/// was previously installed (Rust's own default, which prints to stderr, unless something else
/// already replaced it): never changing panic behavior itself, the same "report, then don't
/// change program behavior" rule the .NET middleware and Python `excepthook` wrapper both follow.
///
/// Unlike Go, where only a `defer Recover()` in the same goroutine can see a panic, Rust's panic
/// hook is genuinely process-wide: it fires for a panic on *any* thread, including a web
/// framework's own worker threads, with no per-framework middleware needed at all. `init()` calls
/// this automatically unless `Configuration.install_panic_hook` is set to `false`; call it
/// directly only if you're managing configuration some other way.
///
/// Always chains onto whatever hook is *currently* installed via `take_hook()`, rather than
/// latching "already installed" after the first call: deliberately, even though that means
/// calling this more than once wraps another reporting layer each time (a real panic would then
/// report once per accumulated layer). A one-shot latch was tried first and rejected: it makes
/// this call a silent no-op the moment anything else calls `std::panic::set_hook` after this one
/// runs (a host app installing its own hook after `init()`, say), discarding this crate's
/// reporting entirely with no error or warning. Duplicate reports from calling this redundantly
/// is a far more visible, far less damaging failure mode than reporting silently going dark, and
/// is easily avoided the same way `init()` already asks to be called: once, at startup.
pub fn install_panic_hook() {
    let previous = std::panic::take_hook();
    std::panic::set_hook(Box::new(move |info| {
        report_panic(info);
        previous(info);
    }));
}

fn report_panic(info: &std::panic::PanicHookInfo) {
    let s = state();
    let (exception_class, message) = panic_message(info);

    let mut context = HashMap::new();
    if let Some(location) = info.location() {
        context.insert(
            "panic_location".to_string(),
            Value::String(format!(
                "{}:{}:{}",
                location.file(),
                location.line(),
                location.column()
            )),
        );
    }

    s.reporter.report_with_captured_backtrace(
        &exception_class,
        &message,
        context,
        current_user(),
        breadcrumb_buffer::current_breadcrumbs(),
    );
}

/// panic!() accepts any value, but the overwhelming majority of real panics carry either a `&str`
/// (`panic!("boom")`) or a `String` (`panic!("boom: {err}")`) payload: these are the only two
/// downcast targets std's own default panic hook special-cases too. Anything else reports as a
/// generic "non-string panic payload" message, since there's no way to `Display` an arbitrary
/// `dyn Any` payload.
fn panic_message(info: &std::panic::PanicHookInfo) -> (String, String) {
    let payload = info.payload();
    if let Some(s) = payload.downcast_ref::<&str>() {
        ("panic".to_string(), s.to_string())
    } else if let Some(s) = payload.downcast_ref::<String>() {
        ("panic".to_string(), s.clone())
    } else {
        ("panic".to_string(), "non-string panic payload".to_string())
    }
}

/// Builds a `HashMap<String, Value>` from `key => value` pairs, the same literal-context ergonomics
/// every other client in this repo gets for free from its own language (a Python dict, a JS object
/// literal, a PHP array):
///
/// ```
/// let ctx = forge_ops_tracker::context!{"order_id" => 42, "customer" => "acme-inc"};
/// ```
#[macro_export]
macro_rules! context {
    ( $( $key:expr => $value:expr ),* $(,)? ) => {{
        #[allow(unused_mut)]
        let mut map = ::std::collections::HashMap::new();
        $( map.insert(::std::string::ToString::to_string($key), $crate::Value::from($value)); )*
        map
    }};
}

/// Extension trait for `Result`, so a fallible call can report its own error and still propagate
/// it in one step:
///
/// ```no_run
/// # use std::collections::HashMap;
/// use forge_ops_tracker::ResultReportExt;
/// # fn charge_card() -> Result<(), std::io::Error> { Ok(()) }
/// # fn run() -> Result<(), std::io::Error> {
/// charge_card().report_err(HashMap::new(), None)?;
/// # Ok(())
/// # }
/// ```
pub trait ResultReportExt<T> {
    fn report_err(
        self,
        context: HashMap<String, Value>,
        user: Option<HashMap<String, Value>>,
    ) -> Self;
}

impl<T, E: std::error::Error> ResultReportExt<T> for Result<T, E> {
    fn report_err(
        self,
        context: HashMap<String, Value>,
        user: Option<HashMap<String, Value>>,
    ) -> Self {
        if let Err(ref err) = self {
            capture_error(err, context, user);
        }
        self
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::io::{Read, Write};
    use std::net::TcpListener;
    use std::sync::atomic::{AtomicBool, AtomicI32, Ordering as AtomicOrdering};
    use std::sync::Mutex;
    use std::time::Duration;

    // A minimal single-request-per-connection HTTP server, the same pattern client.rs's own tests
    // use, kept local to this module rather than shared: these tests specifically drive the
    // *public* init/capture_error/panic-hook API end to end, not the lower-level types directly.
    fn spawn_tracker_server() -> (std::net::SocketAddr, Arc<AtomicI32>) {
        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = listener.local_addr().unwrap();
        let received = Arc::new(AtomicI32::new(0));
        let received_clone = Arc::clone(&received);

        std::thread::spawn(move || {
            for stream in listener.incoming().flatten() {
                let mut stream = stream;
                crate::test_support::read_full_request(&mut stream);
                received_clone.fetch_add(1, AtomicOrdering::SeqCst);
                let _ = stream.write_all(
                    b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}",
                );
            }
        });

        (addr, received)
    }

    // Same shape as spawn_tracker_server, but also reads and records each request's full body
    // (headers *and* the Content-Length-declared body after them, not just the header block): the
    // set_user/explicit-user-override scenarios below need to inspect the delivered JSON itself,
    // not merely count deliveries.
    fn spawn_tracker_server_capturing_body(
    ) -> (std::net::SocketAddr, Arc<AtomicI32>, Arc<Mutex<String>>) {
        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = listener.local_addr().unwrap();
        let received = Arc::new(AtomicI32::new(0));
        let received_clone = Arc::clone(&received);
        let last_body = Arc::new(Mutex::new(String::new()));
        let last_body_clone = Arc::clone(&last_body);

        std::thread::spawn(move || {
            for stream in listener.incoming().flatten() {
                let mut stream = stream;
                let mut buf = [0u8; 8192];
                let mut total = Vec::new();
                let mut header_end = None;
                loop {
                    let n = stream.read(&mut buf).unwrap_or(0);
                    if n == 0 {
                        break;
                    }
                    total.extend_from_slice(&buf[..n]);
                    if let Some(pos) = total.windows(4).position(|w| w == b"\r\n\r\n") {
                        header_end = Some(pos + 4);
                        let header_text = String::from_utf8_lossy(&total[..pos]).into_owned();
                        let content_length: usize = header_text
                            .lines()
                            .find(|l| l.to_lowercase().starts_with("content-length:"))
                            .and_then(|l| l.split(':').nth(1))
                            .and_then(|v| v.trim().parse().ok())
                            .unwrap_or(0);
                        while total.len() < pos + 4 + content_length {
                            let n = stream.read(&mut buf).unwrap_or(0);
                            if n == 0 {
                                break;
                            }
                            total.extend_from_slice(&buf[..n]);
                        }
                        break;
                    }
                }
                if let Some(header_end) = header_end {
                    let body = String::from_utf8_lossy(&total[header_end..]).into_owned();
                    *last_body_clone.lock().unwrap() = body;
                }
                received_clone.fetch_add(1, AtomicOrdering::SeqCst);
                let _ = stream.write_all(
                    b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}",
                );
            }
        });

        (addr, received, last_body)
    }

    fn wait_for(received: &AtomicI32, count: i32) {
        let deadline = std::time::Instant::now() + Duration::from_secs(2);
        while received.load(AtomicOrdering::SeqCst) < count && std::time::Instant::now() < deadline
        {
            std::thread::sleep(Duration::from_millis(5));
        }
        assert_eq!(received.load(AtomicOrdering::SeqCst), count);
    }

    // The public API sits behind one process-wide OnceLock (see `state()` above), so every test
    // touching it has to run against that same singleton: unlike this crate's other modules,
    // which build fresh, independent instances per test. Rather than fight Rust's default
    // parallel test execution (or add a dev-dependency purely to serialize a handful of tests),
    // every scenario that touches the public API lives in this one #[test] function and runs
    // sequentially. Configuration itself is re-read from its RwLock on every delivery attempt
    // (see Client::deliver), so repeatedly calling `init()` to point at a fresh DSN between
    // scenarios below works correctly even though the underlying Reporter/DeliveryQueue/Client
    // are only ever constructed once.
    #[test]
    fn public_api_end_to_end() {
        // init + capture_error delivers through the full stack
        let (addr, received) = spawn_tracker_server();
        init(|c| {
            c.dsn = Some(format!("http://key@{addr}/events"));
            c.environment = "production".to_string();
            c.timeout = Duration::from_secs(2);
            c.install_panic_hook = false; // installed explicitly, in the last scenario below instead
        });

        let err = std::io::Error::other("boom");
        capture_error(&err, context! {"order_id" => 7}, None);
        wait_for(&received, 1);

        // capture_error_with_class works for a trait-object error
        let (addr, received) = spawn_tracker_server();
        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
        let boxed: Box<dyn std::error::Error> = Box::new(std::io::Error::other("boxed boom"));
        capture_error_with_class("std::io::Error", boxed.as_ref(), HashMap::new(), None);
        wait_for(&received, 1);

        // ResultReportExt reports on Err and passes the Result through unchanged
        let (addr, received) = spawn_tracker_server();
        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
        let result: Result<(), std::io::Error> = Err(std::io::Error::other("reported via ext"));
        let passed_through = result.report_err(HashMap::new(), None);
        assert!(passed_through.is_err());
        wait_for(&received, 1);

        let ok: Result<i32, std::io::Error> = Ok(42);
        assert_eq!(ok.report_err(HashMap::new(), None).unwrap(), 42);

        // set_user attaches the user to a later capture_error call with no explicit user, on this
        // thread
        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
        set_user(context! {"id" => 42, "email" => "alice@example.com"});
        capture_error(&std::io::Error::other("boom"), HashMap::new(), None);
        wait_for(&received, 1);
        assert!(last_body.lock().unwrap().contains("alice@example.com"));

        // an explicit user argument overrides whatever set_user last set
        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
        set_user(context! {"id" => 42});
        capture_error(
            &std::io::Error::other("boom"),
            HashMap::new(),
            Some(context! {"id" => 99}),
        );
        wait_for(&received, 1);
        assert!(last_body.lock().unwrap().contains("\"id\":99"));
        set_user(HashMap::new()); // clear, so it doesn't leak into whatever test runs on this thread next

        // add_breadcrumb accumulates on this thread and capture_error attaches the trail
        // automatically; clear_breadcrumbs empties it again
        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
        add_breadcrumb("opened checkout", "custom", "info", HashMap::new());
        add_breadcrumb(
            "charged card",
            "custom",
            "info",
            context! {"order_id" => 42},
        );
        capture_error(&std::io::Error::other("boom"), HashMap::new(), None);
        wait_for(&received, 1);
        {
            let body = last_body.lock().unwrap();
            assert!(body.contains("opened checkout"));
            assert!(body.contains("charged card"));
        }
        clear_breadcrumbs();

        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
        capture_error(&std::io::Error::other("boom"), HashMap::new(), None);
        wait_for(&received, 1);
        assert!(!last_body.lock().unwrap().contains("\"breadcrumbs\""));

        // record_performance tallies in-process and flush_performance delivers one aggregate batch
        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
        record_performance("GET /users/:id", 10.0);
        record_performance("GET /users/:id", 30.0);
        flush_performance();
        wait_for(&received, 1);
        {
            let body = last_body.lock().unwrap();
            assert!(body.starts_with("{\"samples\":["), "body = {body}");
            assert!(body.contains("\"transaction_name\":\"GET /users/:id\""));
            assert!(body.contains("\"request_count\":2"));
        }

        // time_transaction returns the closure's own value and records how long it took, even if
        // the closure panics
        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
        let value = time_transaction("timed", || 42);
        assert_eq!(value, 42);
        let panicked = std::panic::catch_unwind(|| {
            time_transaction("timed panics", || -> u32 {
                panic!("inside a timed transaction")
            })
        });
        assert!(panicked.is_err());
        flush_performance();
        wait_for(&received, 1);
        {
            let body = last_body.lock().unwrap();
            assert!(
                body.contains("\"transaction_name\":\"timed\""),
                "body = {body}"
            );
            assert!(
                body.contains("\"transaction_name\":\"timed panics\""),
                "body = {body}"
            );
        }

        // track_performance = false records and delivers nothing
        let (addr, received, _last_body) = spawn_tracker_server_capturing_body();
        init(|c| {
            c.dsn = Some(format!("http://key@{addr}/events"));
            c.track_performance = false;
        });
        record_performance("never recorded", 10.0);
        flush_performance();
        std::thread::sleep(Duration::from_millis(150));
        assert_eq!(received.load(AtomicOrdering::SeqCst), 0);
        init(|c| c.track_performance = true);

        // init's automatic panic hook reports a panic, then still lets it unwind unchanged
        let (addr, received) = spawn_tracker_server();
        let previous_hook_ran = Arc::new(AtomicBool::new(false));
        let previous_hook_ran_clone = Arc::clone(&previous_hook_ran);
        // Installed *before* init() specifically to prove install_panic_hook chains onto whatever
        // hook already exists (via take_hook()) rather than replacing it outright.
        std::panic::set_hook(Box::new(move |_| {
            previous_hook_ran_clone.store(true, AtomicOrdering::SeqCst);
        }));
        init(|c| {
            c.dsn = Some(format!("http://key@{addr}/events"));
            c.install_panic_hook = true;
        });

        let result = std::panic::catch_unwind(|| {
            panic!("test panic");
        });
        assert!(result.is_err());
        assert!(
            previous_hook_ran.load(AtomicOrdering::SeqCst),
            "the previously-installed hook should still have run"
        );
        wait_for(&received, 1);

        // Restore a silent hook so later tests in this binary don't print this test's own
        // intentional panic to stderr.
        std::panic::set_hook(Box::new(|_| {}));
    }

    #[test]
    fn span_just_runs_the_closure_outside_a_trace() {
        assert_eq!(span("free", "service", HashMap::new(), || 7), 7);
        // record_span outside a trace is a no-op, not a panic.
        record_span(
            "free",
            "database",
            std::time::SystemTime::now(),
            1.0,
            HashMap::new(),
        );
    }

    #[test]
    fn a_panic_inside_trace_or_span_propagates_and_leaves_no_trace_behind() {
        // Only the thread-local matters here, so this holds whether or not another test has
        // already enabled reporting on the shared global configuration.
        let result = std::panic::catch_unwind(|| {
            trace("GET /boom", || {
                span("bad", "service", HashMap::new(), || panic!("boom"));
            })
        });
        assert!(result.is_err());
        assert!(!span_buffer::is_active());
    }

    #[test]
    fn trace_returns_the_closures_value_and_a_nested_trace_is_a_span() {
        let value = trace("outer", || trace("inner", || 5));
        assert_eq!(value, 5);
        assert!(!span_buffer::is_active());
    }

    #[test]
    fn context_macro_builds_expected_map() {
        let ctx = context! {"order_id" => 42, "customer" => "acme-inc"};
        assert_eq!(ctx.get("order_id"), Some(&Value::Number(42.0)));
        assert_eq!(
            ctx.get("customer"),
            Some(&Value::String("acme-inc".to_string()))
        );
    }
}