Skip to main content

forge_ops_tracker/
lib.rs

1//! ForgeOps error tracking client for ForgeOps:
2//!
3//! ```no_run
4//! forge_ops_tracker::init(|c| {
5//!     c.dsn = Some("https://<api_key>@getforgeops.net/api/v1/events".to_string());
6//! });
7//! ```
8//!
9//! See the README for what gets captured automatically vs. what needs an explicit
10//! [`capture_error`] call. A from-scratch port of `gems/forge_ops_tracker` (the Rails client):
11//! see that gem's README for the shared design rationale behind the pieces this crate is built
12//! from (Configuration, EventBuilder, DeliveryQueue, Reporter, Client).
13
14mod breadcrumb_buffer;
15mod change_tracking;
16mod client;
17mod configuration;
18mod delivery_queue;
19mod event_builder;
20mod histogram_bucketer;
21mod metric_buffer;
22mod performance_flusher;
23mod pii_scrubber;
24mod reporter;
25mod span_buffer;
26mod span_queue;
27mod sql_statement;
28#[cfg(test)]
29mod test_support;
30mod trace_parent;
31
32pub use breadcrumb_buffer::Breadcrumb;
33pub use change_tracking::Change;
34pub use configuration::{Configuration, TracePropagationTarget};
35pub use event_builder::{Event, Frame};
36pub use pii_scrubber::Value;
37pub use sql_statement::SqlObjects;
38
39use std::cell::RefCell;
40use std::collections::HashMap;
41use std::sync::atomic::{AtomicBool, Ordering};
42use std::sync::{Arc, OnceLock, RwLock};
43
44use client::Client;
45use delivery_queue::DeliveryQueue;
46use metric_buffer::{Kind, MetricBuffer};
47use performance_flusher::PerformanceFlusher;
48use reporter::Reporter;
49use span_queue::SpanQueue;
50
51struct State {
52    configuration: Arc<RwLock<Configuration>>,
53    reporter: Reporter,
54    performance_flusher: Arc<PerformanceFlusher>,
55    span_queue: SpanQueue,
56    metrics: Arc<MetricBuffer>,
57    infrastructure_metrics: Arc<MetricBuffer>,
58}
59
60static STATE: OnceLock<State> = OnceLock::new();
61
62// Whether this process has queued its startup change snapshot: once per process, however many
63// times `init()` runs.
64static CHANGE_SNAPSHOT_SENT: AtomicBool = AtomicBool::new(false);
65
66thread_local! {
67    // The user set via `set_user`, if any. A plain thread-local, not a process-wide global: the
68    // right choice for the thread-per-request model this crate's own synchronous, non-async
69    // design naturally pairs with (see Cargo.toml's own comment on why `ureq`, not an async HTTP
70    // client, was chosen), the same reasoning `gems/forge_ops_tracker` documents for its own
71    // `Thread.current` use. **Does not propagate across an `.await` in an async runtime**: unlike
72    // Ruby's green/native threads, a single OS thread in an async executor (tokio, async-std)
73    // interleaves multiple unrelated tasks, so a value set on one task can leak into, or simply
74    // never reach, another. A host app built on an async runtime should pass `user` explicitly to
75    // `capture_error`/`capture_error_with_class` instead of relying on `set_user`, the same way
76    // `sdks/node` needs `AsyncLocalStorage` rather than a bare thread-local for the identical
77    // reason.
78    static CURRENT_USER: RefCell<Option<HashMap<String, Value>>> = const { RefCell::new(None) };
79}
80
81fn current_user() -> Option<HashMap<String, Value>> {
82    CURRENT_USER.with(|u| u.borrow().clone())
83}
84
85fn state() -> &'static State {
86    STATE.get_or_init(|| {
87        let configuration = Arc::new(RwLock::new(Configuration::new()));
88        let client = Arc::new(Client::new(Arc::clone(&configuration)));
89        let queue_size = configuration.read().unwrap().queue_size;
90        let delivery_queue = DeliveryQueue::new(queue_size, Arc::clone(&client));
91        let span_queue = SpanQueue::new(queue_size, Arc::clone(&client));
92        let metrics = MetricBuffer::new(
93            Kind::Custom,
94            Arc::clone(&configuration),
95            Arc::clone(&client),
96        );
97        let infrastructure_metrics = MetricBuffer::new(
98            Kind::Infrastructure,
99            Arc::clone(&configuration),
100            Arc::clone(&client),
101        );
102        let reporter = Reporter::new(Arc::clone(&configuration), delivery_queue);
103        let performance_flusher = PerformanceFlusher::new(Arc::clone(&configuration), client);
104        State {
105            configuration,
106            reporter,
107            performance_flusher,
108            span_queue,
109            metrics,
110            infrastructure_metrics,
111        }
112    })
113}
114
115/// Configures the client. Call once at startup, before your server starts accepting requests.
116/// Pass a closure to set any [`Configuration`] field:
117///
118/// ```no_run
119/// forge_ops_tracker::init(|c| {
120///     c.dsn = Some("https://<api_key>@getforgeops.net/api/v1/events".to_string());
121///     c.release = Some("a1b2c3d".to_string());
122/// });
123/// ```
124///
125/// Installs the global panic hook (see [`install_panic_hook`]) unless
126/// `Configuration.install_panic_hook` is set to `false` inside the closure.
127///
128/// The first `init()` that leaves reporting enabled also queues one change snapshot (see
129/// `Configuration.detect_changes`), delivered on the background thread so startup never waits on
130/// it or fails because of it.
131pub fn init(configure: impl FnOnce(&mut Configuration)) {
132    let s = state();
133    let install_hook = {
134        let mut config = s.configuration.write().unwrap();
135        configure(&mut config);
136        config.install_panic_hook
137    };
138    if install_hook {
139        install_panic_hook();
140    }
141    if CHANGE_SNAPSHOT_SENT
142        .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
143        .is_ok()
144        && !s.reporter.send_change_snapshot()
145    {
146        CHANGE_SNAPSHOT_SENT.store(false, Ordering::SeqCst);
147    }
148}
149
150/// Records one change (a feature flag flip, a config edit, a migration, anything worth lining up
151/// against your errors) so ForgeOps can show it next to what happened afterward. Delivered on the
152/// background thread like any error, so this never blocks; a no-op when reporting isn't enabled,
153/// and a failed delivery (including a plan without change tracking) is silently dropped.
154///
155/// ```no_run
156/// use forge_ops_tracker::{context, Change};
157///
158/// forge_ops_tracker::record_change(Change::new("migration", "Add index to orders.created_at"));
159/// forge_ops_tracker::record_change(Change {
160///     details: context! {"flag" => "new_checkout", "enabled" => true},
161///     actor: Some("alice@example.com".to_string()),
162///     ..Change::new("feature_flag", "Enabled new checkout for everyone")
163/// });
164/// ```
165pub fn record_change(change: Change) {
166    state().reporter.record_change(&change);
167}
168
169/// Reports an error you've already handled. Call it right at the point you'd otherwise just log
170/// it:
171///
172/// ```no_run
173/// # use std::collections::HashMap;
174/// # fn charge_card() -> Result<(), std::io::Error> { Ok(()) }
175/// if let Err(err) = charge_card() {
176///     forge_ops_tracker::capture_error(&err, HashMap::new(), None);
177/// }
178/// ```
179///
180/// The backtrace is captured right here, at the call site: unlike Python/Java/PHP, a plain Rust
181/// `std::error::Error` carries no stack of its own, so `capture_error` has to be the one call that
182/// knows where the trace starts. `exception_class` is inferred via [`std::any::type_name`], which
183/// needs `E` to be a concrete, statically-known type: for a `Box<dyn Error>` or other trait
184/// object, where that isn't possible, use [`capture_error_with_class`] instead and supply the
185/// class yourself.
186///
187/// `user` defaults to whatever [`set_user`] last established on this thread, if anything (`None`
188/// here means "use that", not "no user"); pass `Some(..)` to override it for this one report.
189pub fn capture_error<E: std::error::Error>(
190    err: &E,
191    context: HashMap<String, Value>,
192    user: Option<HashMap<String, Value>>,
193) {
194    capture_error_with_class(std::any::type_name::<E>(), err, context, user);
195}
196
197/// The same as [`capture_error`], but for a `&dyn std::error::Error` (a `Box<dyn Error>`, a trait
198/// object) whose concrete type isn't known at the call site, so `exception_class` has to be
199/// supplied explicitly rather than inferred.
200pub fn capture_error_with_class(
201    exception_class: &str,
202    err: &dyn std::error::Error,
203    context: HashMap<String, Value>,
204    user: Option<HashMap<String, Value>>,
205) {
206    let s = state();
207    s.reporter.report_with_captured_backtrace(
208        exception_class,
209        &err.to_string(),
210        context,
211        user.or_else(current_user),
212        breadcrumb_buffer::current_breadcrumbs(),
213    );
214}
215
216/// The same as [`capture_error`], for an error caused by a database call: pass the SQL that ran.
217/// A Rust error carries no statement of its own and no Rust database crate puts one on its error
218/// types, so the code that ran the query has to hand it over.
219///
220/// With `Configuration::capture_sql_objects` on (the default), the names of the stored procedure,
221/// table and view the statement touched are sent, so an issue says where to start looking. With
222/// `Configuration::capture_sql_statement` on too (off by default), the statement itself is sent as
223/// well, with every string and number replaced by `?` first. The raw statement never leaves this
224/// process either way.
225///
226/// ```ignore
227/// if let Err(err) = sqlx::query(QUERY).bind(id).execute(&pool).await {
228///     forge_ops_tracker::capture_error_with_sql(&err, QUERY, HashMap::new(), None);
229/// }
230/// ```
231pub fn capture_error_with_sql<E: std::error::Error>(
232    err: &E,
233    statement: &str,
234    context: HashMap<String, Value>,
235    user: Option<HashMap<String, Value>>,
236) {
237    capture_error_with_class_and_sql(std::any::type_name::<E>(), err, statement, context, user);
238}
239
240/// The same as [`capture_error_with_sql`], but for a `&dyn std::error::Error` whose concrete type
241/// isn't known at the call site, so `exception_class` is supplied explicitly.
242pub fn capture_error_with_class_and_sql(
243    exception_class: &str,
244    err: &dyn std::error::Error,
245    statement: &str,
246    context: HashMap<String, Value>,
247    user: Option<HashMap<String, Value>>,
248) {
249    let s = state();
250    s.reporter.report_with_captured_backtrace_and_sql(
251        exception_class,
252        &err.to_string(),
253        context,
254        user.or_else(current_user),
255        breadcrumb_buffer::current_breadcrumbs(),
256        statement,
257    );
258}
259
260/// Manually attaches an affected user to whatever gets reported from here on, *on this thread*
261/// (an explicit [`capture_error`]/[`capture_error_with_class`] call with no `user` argument, or a
262/// panic the installed hook catches): there's no way to automatically detect "the current user"
263/// the way a server-side web framework with its own session/auth middleware can, so call this
264/// yourself, e.g. right after sign-in. `id`/`email`/`username` are all independently optional;
265/// call with an empty map to clear whatever was set, e.g. on sign-out. See this crate's own
266/// `CURRENT_USER` thread-local (in the source) for why this is thread-local, and the real caveat
267/// that comes with that choice under an async runtime.
268pub fn set_user(user: HashMap<String, Value>) {
269    let user = if user.is_empty() { None } else { Some(user) };
270    CURRENT_USER.with(|u| *u.borrow_mut() = user);
271}
272
273/// Records one entry into the current thread's breadcrumb trail: a query, an outbound call, or
274/// anything worth remembering right up to the moment something actually goes wrong. `category`
275/// defaults to `"custom"` and `level` to `"info"` when passed an empty string. A no-op, not an
276/// error, when `Configuration.track_breadcrumbs` is `false`.
277///
278/// This crate has no web framework integration of its own (unlike `sdks/go`'s net/http/Gin
279/// middleware), so there's no automatic breadcrumb source and no middleware to start a fresh trail
280/// per request on its own: call [`clear_breadcrumbs`] yourself at the start of each request, the
281/// same place you'd already be calling [`set_user`] from, or entries from an earlier request
282/// handled on a reused thread will bleed into this one's own report.
283pub fn add_breadcrumb(message: &str, category: &str, level: &str, data: HashMap<String, Value>) {
284    let config = state().configuration.read().unwrap();
285    breadcrumb_buffer::add_breadcrumb(
286        config.track_breadcrumbs,
287        config.max_breadcrumbs,
288        message,
289        category,
290        level,
291        data,
292    );
293}
294
295/// Clears the current thread's breadcrumb trail. See [`add_breadcrumb`]'s own doc for why calling
296/// this yourself, at the start of each request, is this crate's responsibility to ask of you
297/// rather than something it can do on its own.
298pub fn clear_breadcrumbs() {
299    breadcrumb_buffer::clear_breadcrumbs();
300}
301
302/// Records one timed call's duration, in milliseconds, under `transaction_name`: tallied
303/// in-process (count, total, max) and flushed periodically as one small aggregate report, for the
304/// Performance page's per-transaction table, not one network call per call. A no-op when
305/// `Configuration.track_performance` is `false` or reporting isn't enabled for this environment.
306///
307/// This crate has no web framework integration of its own (unlike `sdks/go`'s net/http/Gin
308/// middleware), so nothing is timed automatically: wrap whatever you want on the Performance page
309/// yourself, e.g. a request handler, with [`time_transaction`], or call this directly with a
310/// duration you measured. Keep `transaction_name` low-cardinality (`"GET /users/:id"`, not
311/// `"GET /users/42"`): every distinct name is its own row.
312pub fn record_performance(transaction_name: &str, duration_ms: f64) {
313    state()
314        .performance_flusher
315        .record(transaction_name, duration_ms);
316}
317
318/// Runs `f`, records how long it took under `transaction_name` (see [`record_performance`]), and
319/// returns whatever `f` returned. Recorded even if `f` panics: the duration up to the panic is
320/// still a real duration, and a handler that panics is exactly one worth seeing on the
321/// Performance page.
322///
323/// ```no_run
324/// # fn handle_request() -> u32 { 200 }
325/// let status = forge_ops_tracker::time_transaction("GET /users/:id", || handle_request());
326/// ```
327pub fn time_transaction<T>(transaction_name: &str, f: impl FnOnce() -> T) -> T {
328    struct Timing<'a> {
329        name: &'a str,
330        started_at: std::time::Instant,
331    }
332    impl Drop for Timing<'_> {
333        fn drop(&mut self) {
334            record_performance(self.name, self.started_at.elapsed().as_secs_f64() * 1000.0);
335        }
336    }
337
338    let _timing = Timing {
339        name: transaction_name,
340        started_at: std::time::Instant::now(),
341    };
342    f()
343}
344
345/// Delivers whatever has been tallied so far right now, instead of waiting for the next
346/// `performance_flush_interval` tick. The background flush thread is a daemon: it does not run on
347/// a normal process exit the way the Ruby gem's `at_exit` hook does (Rust has no equivalent), so a
348/// short-lived program, or one about to shut down, should call this itself to avoid losing the
349/// last partial window.
350pub fn flush_performance() {
351    state().performance_flusher.flush();
352}
353
354/// Records a named business metric (a signup, a payment, anything you want to name), buffered and
355/// flushed periodically as one batch rather than one network call per capture. Pass `1.0` for a bare
356/// counter-style call ("a signup happened") or a real magnitude ("a $49 payment"); it may be negative
357/// (a refund). A no-op when the client isn't enabled (no DSN, or this environment isn't in
358/// `enabled_environments`), and a NaN or infinite value is dropped.
359///
360/// Rust has no exit hook to flush from, so a short-lived program should call [`flush_metrics`]
361/// before it returns from `main`.
362///
363/// ```no_run
364/// forge_ops_tracker::capture_metric("signup", 1.0);
365/// forge_ops_tracker::capture_metric("payment", 49.0);
366/// ```
367pub fn capture_metric(name: &str, value: f64) {
368    let s = state();
369    let (enabled, environment, release) = {
370        let config = s.configuration.read().unwrap();
371        (
372            config.is_enabled(),
373            config.environment.clone(),
374            config.release.clone(),
375        )
376    };
377    if !enabled {
378        return;
379    }
380    let release = release
381        .as_deref()
382        .map(pii_scrubber::json_string)
383        .unwrap_or_else(|| "null".to_string());
384    s.metrics.record(
385        value,
386        &format!(
387            "\"metric_name\":{},\"value\":{value},\"environment\":{},\"release\":{release}",
388            pii_scrubber::json_string(name),
389            pii_scrubber::json_string(&environment),
390        ),
391    );
392}
393
394/// Records one infrastructure reading (CPU, memory, disk, anything else a program of yours reads)
395/// from one of your own hosts. `hostname` defaults to `Configuration.server_name` when `None`, so a
396/// script running on the box it reports about needs no argument. Same buffered-batch delivery and
397/// no-op-when-disabled contract as [`capture_metric`].
398///
399/// ```no_run
400/// forge_ops_tracker::capture_infrastructure_metric("cpu", 0.42, None);
401/// forge_ops_tracker::capture_infrastructure_metric("disk", 0.81, Some("db-1"));
402/// forge_ops_tracker::flush_metrics();
403/// ```
404pub fn capture_infrastructure_metric(name: &str, value: f64, hostname: Option<&str>) {
405    let s = state();
406    let (enabled, server_name) = {
407        let config = s.configuration.read().unwrap();
408        (config.is_enabled(), config.server_name.clone())
409    };
410    if !enabled {
411        return;
412    }
413    let hostname = hostname
414        .map(str::to_string)
415        .or(server_name)
416        .unwrap_or_default();
417    s.infrastructure_metrics.record(
418        value,
419        &format!(
420            "\"metric_name\":{},\"value\":{value},\"hostname\":{}",
421            pii_scrubber::json_string(name),
422            pii_scrubber::json_string(&hostname),
423        ),
424    );
425}
426
427/// Delivers every buffered metric and infrastructure reading right now, instead of waiting for the
428/// next flush interval. The background flush thread is a daemon and Rust has no exit hook, so call
429/// this before a short-lived program ends.
430pub fn flush_metrics() {
431    let s = state();
432    s.metrics.flush();
433    s.infrastructure_metrics.flush();
434}
435
436/// The name of the W3C Trace Context header: what [`continue_trace`] reads from an incoming
437/// request and what [`http_span`]'s value goes out as.
438pub const TRACEPARENT_HEADER: &str = trace_parent::HEADER;
439
440/// Runs `f` as a trace named `root_name` (for example `"GET /checkout"` or `"job:reindex"`): every
441/// [`span`], [`http_span`] and [`record_span`] inside it, on this thread, nests beneath this root.
442/// When `f` took at least `Configuration.trace_capture_threshold` (1 second by default) the whole
443/// trace is sent to ForgeOps, so fast calls cost nothing on the wire. Sent even if `f` panics.
444/// Called inside an already-open trace it just records a span instead. Errors captured inside it
445/// carry its trace id (see [`current_trace_id`]).
446///
447/// This crate has no web framework integration, so nothing starts a trace automatically: wrap
448/// whatever you want traced, typically a request handler or a background job. To continue a trace
449/// another service started, use [`continue_trace`] with its `traceparent` header.
450///
451/// ```no_run
452/// # fn handle_request() -> u32 { 200 }
453/// let status = forge_ops_tracker::trace("GET /checkout", || handle_request());
454/// ```
455pub fn trace<T>(root_name: &str, f: impl FnOnce() -> T) -> T {
456    continue_trace(root_name, None, f)
457}
458
459/// The same as [`trace`], continuing the caller's trace when `traceparent` is a usable W3C
460/// `traceparent` header value, typically the incoming request's own header: the trace keeps the
461/// caller's trace id, and its root span records the caller's span as its parent, so it nests under
462/// that span on ForgeOps. `None`, blank, or malformed starts a fresh trace, exactly like [`trace`].
463/// Ignored inside an already-open trace.
464///
465/// ```no_run
466/// # fn handle_request() -> u32 { 200 }
467/// # let incoming: Option<String> = None;
468/// // `incoming` is the request's own "traceparent" header, however your server exposes it.
469/// let status = forge_ops_tracker::continue_trace("POST /orders", incoming.as_deref(), || {
470///     handle_request()
471/// });
472/// ```
473pub fn continue_trace<T>(root_name: &str, traceparent: Option<&str>, f: impl FnOnce() -> T) -> T {
474    let (owns_trace, threshold) = {
475        let config = state().configuration.read().unwrap();
476        let incoming = traceparent.and_then(trace_parent::parse);
477        (
478            span_buffer::begin(&config, incoming),
479            config.trace_capture_threshold,
480        )
481    };
482    if !owns_trace {
483        return span(root_name, "service", HashMap::new(), f);
484    }
485
486    struct Root<'a> {
487        name: &'a str,
488        started_at: std::time::SystemTime,
489        timer: std::time::Instant,
490        threshold: std::time::Duration,
491    }
492    impl Drop for Root<'_> {
493        fn drop(&mut self) {
494            let duration_ms = self.timer.elapsed().as_secs_f64() * 1000.0;
495            if let Some(body) = span_buffer::end(
496                self.threshold,
497                self.name,
498                "controller",
499                self.started_at,
500                duration_ms,
501            ) {
502                state().span_queue.push(body);
503            }
504        }
505    }
506
507    let _root = Root {
508        name: root_name,
509        started_at: std::time::SystemTime::now(),
510        timer: std::time::Instant::now(),
511        threshold,
512    };
513    f()
514}
515
516/// Times `f` as a child span of whatever span is open on this thread (or of the trace's root),
517/// returning what `f` returned. Outside a [`trace`] it just runs `f`. Recorded even if `f` panics.
518/// `kind` is one of `"controller"`, `"service"`, `"database"`, `"redis"`, `"http"`, `"job"`,
519/// `"other"` (anything else is sent as `"other"`).
520///
521/// ```no_run
522/// # use std::collections::HashMap;
523/// # fn charge(id: u32) -> u32 { id }
524/// let charged = forge_ops_tracker::span("charge card", "service", HashMap::new(), || charge(7));
525/// ```
526pub fn span<T>(name: &str, kind: &str, data: HashMap<String, Value>, f: impl FnOnce() -> T) -> T {
527    let Some((id, parent)) = span_buffer::open_span() else {
528        return f();
529    };
530
531    struct Open<'a> {
532        id: String,
533        parent: String,
534        name: &'a str,
535        kind: &'a str,
536        data: HashMap<String, Value>,
537        started_at: std::time::SystemTime,
538        timer: std::time::Instant,
539    }
540    impl Drop for Open<'_> {
541        fn drop(&mut self) {
542            span_buffer::close_span(
543                std::mem::take(&mut self.id),
544                std::mem::take(&mut self.parent),
545                self.name,
546                self.kind,
547                self.started_at,
548                self.timer.elapsed().as_secs_f64() * 1000.0,
549                std::mem::take(&mut self.data),
550            );
551        }
552    }
553
554    let _open = Open {
555        id,
556        parent,
557        name,
558        kind,
559        data,
560        started_at: std::time::SystemTime::now(),
561        timer: std::time::Instant::now(),
562    };
563    f()
564}
565
566/// Times a database query as a `"database"` span carrying its SQL, so ForgeOps can show which
567/// statement a slow request spent its time in. Returns what `f` returned; outside a [`trace`] it
568/// just runs `f`.
569///
570/// ```no_run
571/// # fn load_orders(sql: &str) -> Vec<u64> { let _ = sql; vec![] }
572/// let sql = "SELECT id FROM orders WHERE customer_id = $1 AND status = 'open'";
573/// let orders = forge_ops_tracker::database_span("load orders", sql, Some("postgresql"), || load_orders(sql));
574/// ```
575///
576/// The statement is masked (every string and number replaced by `?`) and cut to 4000 characters
577/// when the span is recorded, then sent as `db.statement`; `db_system` (`"postgresql"`, `"mysql"`,
578/// `"sqlite"`, `"mssql"`, `"oracle"`, or any other name) goes out lowercased as `db.system`. Bind
579/// values are never taken.
580pub fn database_span<T>(
581    name: &str,
582    statement: &str,
583    db_system: Option<&str>,
584    f: impl FnOnce() -> T,
585) -> T {
586    span(
587        name,
588        "database",
589        database_span_data(statement, db_system),
590        f,
591    )
592}
593
594/// [`record_span`] for a query you timed yourself, carrying its SQL as [`database_span`] does.
595pub fn record_database_span(
596    name: &str,
597    statement: &str,
598    db_system: Option<&str>,
599    started_at: std::time::SystemTime,
600    duration_ms: f64,
601) {
602    record_span(
603        name,
604        "database",
605        started_at,
606        duration_ms,
607        database_span_data(statement, db_system),
608    );
609}
610
611/// The data a database span carries: `db.statement` and `db.system`, each left out when blank.
612/// Add your own keys and pass it to [`span`] or [`record_span`] with kind `"database"` if you
613/// prefer; a `db.statement` on any database span is masked when it's recorded, however the map
614/// was built.
615pub fn database_span_data(statement: &str, db_system: Option<&str>) -> HashMap<String, Value> {
616    let mut data = HashMap::new();
617    if !statement.trim().is_empty() {
618        data.insert(
619            span_buffer::DB_STATEMENT.to_string(),
620            Value::from(statement),
621        );
622    }
623    if let Some(system) = db_system.map(str::trim).filter(|s| !s.is_empty()) {
624        data.insert(
625            span_buffer::DB_SYSTEM.to_string(),
626            Value::String(system.to_lowercase()),
627        );
628    }
629    data
630}
631
632/// Times an outgoing HTTP call as an `http` span named `"<METHOD> <host>"` (never the path or
633/// query, which can carry ids or tokens) and hands `f` the `traceparent` header value to send with
634/// that request: its parent id is this span's own id, so the called service's root span nests under
635/// it when it continues the trace. Set it however your HTTP client does:
636///
637/// ```no_run
638/// # use std::collections::HashMap;
639/// let url = "https://api.example.com/orders";
640/// let response = forge_ops_tracker::http_span("POST", url, HashMap::new(), |traceparent| {
641///     let mut request = ureq::post(url);
642///     if let Some(value) = traceparent {
643///         request = request.set(forge_ops_tracker::TRACEPARENT_HEADER, value);
644///     }
645///     request.send_string("{}")
646/// });
647/// ```
648///
649/// `f` gets `None` outside a [`trace`], when `Configuration.propagate_traces` is off, or when the
650/// URL's host isn't in `Configuration.trace_propagation_targets`; outside a trace no span is
651/// recorded either. Recorded even if `f` panics. This crate doesn't instrument any HTTP client on
652/// its own, so a call made without this helper carries no header.
653pub fn http_span<T>(
654    method: &str,
655    url: &str,
656    data: HashMap<String, Value>,
657    f: impl FnOnce(Option<&str>) -> T,
658) -> T {
659    let Some((id, parent)) = span_buffer::open_span() else {
660        return f(None);
661    };
662    let host = trace_parent::url_host(url);
663    let traceparent = {
664        let config = state().configuration.read().unwrap();
665        config.should_propagate_trace(host.as_deref())
666    }
667    .then(span_buffer::current_trace_id)
668    .flatten()
669    .map(|trace_id| trace_parent::build(&trace_id, &id));
670    let name = format!(
671        "{} {}",
672        method.to_ascii_uppercase(),
673        host.as_deref().unwrap_or("unknown")
674    );
675
676    struct Open {
677        id: String,
678        parent: String,
679        name: String,
680        data: HashMap<String, Value>,
681        started_at: std::time::SystemTime,
682        timer: std::time::Instant,
683    }
684    impl Drop for Open {
685        fn drop(&mut self) {
686            span_buffer::close_span(
687                std::mem::take(&mut self.id),
688                std::mem::take(&mut self.parent),
689                &self.name,
690                "http",
691                self.started_at,
692                self.timer.elapsed().as_secs_f64() * 1000.0,
693                std::mem::take(&mut self.data),
694            );
695        }
696    }
697
698    let _open = Open {
699        id,
700        parent,
701        name,
702        data,
703        started_at: std::time::SystemTime::now(),
704        timer: std::time::Instant::now(),
705    };
706    f(traceparent.as_deref())
707}
708
709/// The id of the trace open on this thread (32 lowercase hex characters), or `None` outside a
710/// [`trace`]. Errors captured on this thread while it's open carry it automatically.
711pub fn current_trace_id() -> Option<String> {
712    span_buffer::current_trace_id()
713}
714
715/// Records a span you timed yourself under the current one; a no-op outside a [`trace`].
716pub fn record_span(
717    name: &str,
718    kind: &str,
719    started_at: std::time::SystemTime,
720    duration_ms: f64,
721    data: HashMap<String, Value>,
722) {
723    span_buffer::record_leaf(name, kind, started_at, duration_ms, data);
724}
725
726/// Installs a global panic hook that reports any panic on any thread, then calls whatever hook
727/// was previously installed (Rust's own default, which prints to stderr, unless something else
728/// already replaced it): never changing panic behavior itself, the same "report, then don't
729/// change program behavior" rule the .NET middleware and Python `excepthook` wrapper both follow.
730///
731/// Unlike Go, where only a `defer Recover()` in the same goroutine can see a panic, Rust's panic
732/// hook is genuinely process-wide: it fires for a panic on *any* thread, including a web
733/// framework's own worker threads, with no per-framework middleware needed at all. `init()` calls
734/// this automatically unless `Configuration.install_panic_hook` is set to `false`; call it
735/// directly only if you're managing configuration some other way.
736///
737/// Always chains onto whatever hook is *currently* installed via `take_hook()`, rather than
738/// latching "already installed" after the first call: deliberately, even though that means
739/// calling this more than once wraps another reporting layer each time (a real panic would then
740/// report once per accumulated layer). A one-shot latch was tried first and rejected: it makes
741/// this call a silent no-op the moment anything else calls `std::panic::set_hook` after this one
742/// runs (a host app installing its own hook after `init()`, say), discarding this crate's
743/// reporting entirely with no error or warning. Duplicate reports from calling this redundantly
744/// is a far more visible, far less damaging failure mode than reporting silently going dark, and
745/// is easily avoided the same way `init()` already asks to be called: once, at startup.
746pub fn install_panic_hook() {
747    let previous = std::panic::take_hook();
748    std::panic::set_hook(Box::new(move |info| {
749        report_panic(info);
750        previous(info);
751    }));
752}
753
754fn report_panic(info: &std::panic::PanicHookInfo) {
755    let s = state();
756    let (exception_class, message) = panic_message(info);
757
758    let mut context = HashMap::new();
759    if let Some(location) = info.location() {
760        context.insert(
761            "panic_location".to_string(),
762            Value::String(format!(
763                "{}:{}:{}",
764                location.file(),
765                location.line(),
766                location.column()
767            )),
768        );
769    }
770
771    s.reporter.report_with_captured_backtrace(
772        &exception_class,
773        &message,
774        context,
775        current_user(),
776        breadcrumb_buffer::current_breadcrumbs(),
777    );
778}
779
780/// panic!() accepts any value, but the overwhelming majority of real panics carry either a `&str`
781/// (`panic!("boom")`) or a `String` (`panic!("boom: {err}")`) payload: these are the only two
782/// downcast targets std's own default panic hook special-cases too. Anything else reports as a
783/// generic "non-string panic payload" message, since there's no way to `Display` an arbitrary
784/// `dyn Any` payload.
785fn panic_message(info: &std::panic::PanicHookInfo) -> (String, String) {
786    let payload = info.payload();
787    if let Some(s) = payload.downcast_ref::<&str>() {
788        ("panic".to_string(), s.to_string())
789    } else if let Some(s) = payload.downcast_ref::<String>() {
790        ("panic".to_string(), s.clone())
791    } else {
792        ("panic".to_string(), "non-string panic payload".to_string())
793    }
794}
795
796/// Builds a `HashMap<String, Value>` from `key => value` pairs, the same literal-context ergonomics
797/// every other client in this repo gets for free from its own language (a Python dict, a JS object
798/// literal, a PHP array):
799///
800/// ```
801/// let ctx = forge_ops_tracker::context!{"order_id" => 42, "customer" => "acme-inc"};
802/// ```
803#[macro_export]
804macro_rules! context {
805    ( $( $key:expr => $value:expr ),* $(,)? ) => {{
806        #[allow(unused_mut)]
807        let mut map = ::std::collections::HashMap::new();
808        $( map.insert(::std::string::ToString::to_string($key), $crate::Value::from($value)); )*
809        map
810    }};
811}
812
813/// Extension trait for `Result`, so a fallible call can report its own error and still propagate
814/// it in one step:
815///
816/// ```no_run
817/// # use std::collections::HashMap;
818/// use forge_ops_tracker::ResultReportExt;
819/// # fn charge_card() -> Result<(), std::io::Error> { Ok(()) }
820/// # fn run() -> Result<(), std::io::Error> {
821/// charge_card().report_err(HashMap::new(), None)?;
822/// # Ok(())
823/// # }
824/// ```
825pub trait ResultReportExt<T> {
826    fn report_err(
827        self,
828        context: HashMap<String, Value>,
829        user: Option<HashMap<String, Value>>,
830    ) -> Self;
831}
832
833impl<T, E: std::error::Error> ResultReportExt<T> for Result<T, E> {
834    fn report_err(
835        self,
836        context: HashMap<String, Value>,
837        user: Option<HashMap<String, Value>>,
838    ) -> Self {
839        if let Err(ref err) = self {
840            capture_error(err, context, user);
841        }
842        self
843    }
844}
845
846#[cfg(test)]
847mod tests {
848    use super::*;
849    use std::io::{Read, Write};
850    use std::net::TcpListener;
851    use std::sync::atomic::{AtomicBool, AtomicI32, Ordering as AtomicOrdering};
852    use std::sync::Mutex;
853    use std::time::Duration;
854
855    // A minimal single-request-per-connection HTTP server, the same pattern client.rs's own tests
856    // use, kept local to this module rather than shared: these tests specifically drive the
857    // *public* init/capture_error/panic-hook API end to end, not the lower-level types directly.
858    fn spawn_tracker_server() -> (std::net::SocketAddr, Arc<AtomicI32>) {
859        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
860        let addr = listener.local_addr().unwrap();
861        let received = Arc::new(AtomicI32::new(0));
862        let received_clone = Arc::clone(&received);
863
864        std::thread::spawn(move || {
865            for stream in listener.incoming().flatten() {
866                let mut stream = stream;
867                crate::test_support::read_full_request(&mut stream);
868                received_clone.fetch_add(1, AtomicOrdering::SeqCst);
869                let _ = stream.write_all(
870                    b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}",
871                );
872            }
873        });
874
875        (addr, received)
876    }
877
878    // Same shape as spawn_tracker_server, but also reads and records each request's full body
879    // (headers *and* the Content-Length-declared body after them, not just the header block): the
880    // set_user/explicit-user-override scenarios below need to inspect the delivered JSON itself,
881    // not merely count deliveries.
882    fn spawn_tracker_server_capturing_body(
883    ) -> (std::net::SocketAddr, Arc<AtomicI32>, Arc<Mutex<String>>) {
884        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
885        let addr = listener.local_addr().unwrap();
886        let received = Arc::new(AtomicI32::new(0));
887        let received_clone = Arc::clone(&received);
888        let last_body = Arc::new(Mutex::new(String::new()));
889        let last_body_clone = Arc::clone(&last_body);
890
891        std::thread::spawn(move || {
892            for stream in listener.incoming().flatten() {
893                let mut stream = stream;
894                let mut buf = [0u8; 8192];
895                let mut total = Vec::new();
896                let mut header_end = None;
897                loop {
898                    let n = stream.read(&mut buf).unwrap_or(0);
899                    if n == 0 {
900                        break;
901                    }
902                    total.extend_from_slice(&buf[..n]);
903                    if let Some(pos) = total.windows(4).position(|w| w == b"\r\n\r\n") {
904                        header_end = Some(pos + 4);
905                        let header_text = String::from_utf8_lossy(&total[..pos]).into_owned();
906                        let content_length: usize = header_text
907                            .lines()
908                            .find(|l| l.to_lowercase().starts_with("content-length:"))
909                            .and_then(|l| l.split(':').nth(1))
910                            .and_then(|v| v.trim().parse().ok())
911                            .unwrap_or(0);
912                        while total.len() < pos + 4 + content_length {
913                            let n = stream.read(&mut buf).unwrap_or(0);
914                            if n == 0 {
915                                break;
916                            }
917                            total.extend_from_slice(&buf[..n]);
918                        }
919                        break;
920                    }
921                }
922                if let Some(header_end) = header_end {
923                    let body = String::from_utf8_lossy(&total[header_end..]).into_owned();
924                    *last_body_clone.lock().unwrap() = body;
925                }
926                received_clone.fetch_add(1, AtomicOrdering::SeqCst);
927                let _ = stream.write_all(
928                    b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}",
929                );
930            }
931        });
932
933        (addr, received, last_body)
934    }
935
936    fn wait_for(received: &AtomicI32, count: i32) {
937        let deadline = std::time::Instant::now() + Duration::from_secs(2);
938        while received.load(AtomicOrdering::SeqCst) < count && std::time::Instant::now() < deadline
939        {
940            std::thread::sleep(Duration::from_millis(5));
941        }
942        assert_eq!(received.load(AtomicOrdering::SeqCst), count);
943    }
944
945    // The public API sits behind one process-wide OnceLock (see `state()` above), so every test
946    // touching it has to run against that same singleton: unlike this crate's other modules,
947    // which build fresh, independent instances per test. Rather than fight Rust's default
948    // parallel test execution (or add a dev-dependency purely to serialize a handful of tests),
949    // every scenario that touches the public API lives in this one #[test] function and runs
950    // sequentially. Configuration itself is re-read from its RwLock on every delivery attempt
951    // (see Client::deliver), so repeatedly calling `init()` to point at a fresh DSN between
952    // scenarios below works correctly even though the underlying Reporter/DeliveryQueue/Client
953    // are only ever constructed once.
954    #[test]
955    fn public_api_end_to_end() {
956        // init + capture_error delivers through the full stack
957        let (addr, received) = spawn_tracker_server();
958        init(|c| {
959            c.dsn = Some(format!("http://key@{addr}/events"));
960            c.environment = "production".to_string();
961            c.timeout = Duration::from_secs(2);
962            c.install_panic_hook = false; // installed explicitly, in the last scenario below instead
963            c.detect_changes = false; // the snapshot has its own scenario below
964        });
965
966        let err = std::io::Error::other("boom");
967        capture_error(&err, context! {"order_id" => 7}, None);
968        wait_for(&received, 1);
969
970        // capture_error_with_class works for a trait-object error
971        let (addr, received) = spawn_tracker_server();
972        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
973        let boxed: Box<dyn std::error::Error> = Box::new(std::io::Error::other("boxed boom"));
974        capture_error_with_class("std::io::Error", boxed.as_ref(), HashMap::new(), None);
975        wait_for(&received, 1);
976
977        // ResultReportExt reports on Err and passes the Result through unchanged
978        let (addr, received) = spawn_tracker_server();
979        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
980        let result: Result<(), std::io::Error> = Err(std::io::Error::other("reported via ext"));
981        let passed_through = result.report_err(HashMap::new(), None);
982        assert!(passed_through.is_err());
983        wait_for(&received, 1);
984
985        let ok: Result<i32, std::io::Error> = Ok(42);
986        assert_eq!(ok.report_err(HashMap::new(), None).unwrap(), 42);
987
988        // set_user attaches the user to a later capture_error call with no explicit user, on this
989        // thread
990        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
991        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
992        set_user(context! {"id" => 42, "email" => "alice@example.com"});
993        capture_error(&std::io::Error::other("boom"), HashMap::new(), None);
994        wait_for(&received, 1);
995        assert!(last_body.lock().unwrap().contains("alice@example.com"));
996
997        // an explicit user argument overrides whatever set_user last set
998        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
999        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
1000        set_user(context! {"id" => 42});
1001        capture_error(
1002            &std::io::Error::other("boom"),
1003            HashMap::new(),
1004            Some(context! {"id" => 99}),
1005        );
1006        wait_for(&received, 1);
1007        assert!(last_body.lock().unwrap().contains("\"id\":99"));
1008        set_user(HashMap::new()); // clear, so it doesn't leak into whatever test runs on this thread next
1009
1010        // add_breadcrumb accumulates on this thread and capture_error attaches the trail
1011        // automatically; clear_breadcrumbs empties it again
1012        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
1013        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
1014        add_breadcrumb("opened checkout", "custom", "info", HashMap::new());
1015        add_breadcrumb(
1016            "charged card",
1017            "custom",
1018            "info",
1019            context! {"order_id" => 42},
1020        );
1021        capture_error(&std::io::Error::other("boom"), HashMap::new(), None);
1022        wait_for(&received, 1);
1023        {
1024            let body = last_body.lock().unwrap();
1025            assert!(body.contains("opened checkout"));
1026            assert!(body.contains("charged card"));
1027        }
1028        clear_breadcrumbs();
1029
1030        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
1031        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
1032        capture_error(&std::io::Error::other("boom"), HashMap::new(), None);
1033        wait_for(&received, 1);
1034        assert!(!last_body.lock().unwrap().contains("\"breadcrumbs\""));
1035
1036        // record_performance tallies in-process and flush_performance delivers one aggregate batch
1037        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
1038        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
1039        record_performance("GET /users/:id", 10.0);
1040        record_performance("GET /users/:id", 30.0);
1041        flush_performance();
1042        wait_for(&received, 1);
1043        {
1044            let body = last_body.lock().unwrap();
1045            assert!(body.starts_with("{\"samples\":["), "body = {body}");
1046            assert!(body.contains("\"transaction_name\":\"GET /users/:id\""));
1047            assert!(body.contains("\"request_count\":2"));
1048        }
1049
1050        // time_transaction returns the closure's own value and records how long it took, even if
1051        // the closure panics
1052        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
1053        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
1054        let value = time_transaction("timed", || 42);
1055        assert_eq!(value, 42);
1056        let panicked = std::panic::catch_unwind(|| {
1057            time_transaction("timed panics", || -> u32 {
1058                panic!("inside a timed transaction")
1059            })
1060        });
1061        assert!(panicked.is_err());
1062        flush_performance();
1063        wait_for(&received, 1);
1064        {
1065            let body = last_body.lock().unwrap();
1066            assert!(
1067                body.contains("\"transaction_name\":\"timed\""),
1068                "body = {body}"
1069            );
1070            assert!(
1071                body.contains("\"transaction_name\":\"timed panics\""),
1072                "body = {body}"
1073            );
1074        }
1075
1076        // track_performance = false records and delivers nothing
1077        let (addr, received, _last_body) = spawn_tracker_server_capturing_body();
1078        init(|c| {
1079            c.dsn = Some(format!("http://key@{addr}/events"));
1080            c.track_performance = false;
1081        });
1082        record_performance("never recorded", 10.0);
1083        flush_performance();
1084        std::thread::sleep(Duration::from_millis(150));
1085        assert_eq!(received.load(AtomicOrdering::SeqCst), 0);
1086        init(|c| c.track_performance = true);
1087
1088        // an error captured inside a continued trace carries the caller's trace id, even with
1089        // track_tracing off; outside a trace it carries none
1090        let incoming = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01";
1091        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
1092        init(|c| {
1093            c.dsn = Some(format!("http://key@{addr}/events"));
1094            c.track_tracing = false;
1095        });
1096        continue_trace("POST /orders", Some(incoming), || {
1097            assert_eq!(
1098                current_trace_id().as_deref(),
1099                Some("4bf92f3577b34da6a3ce929d0e0e4736")
1100            );
1101            capture_error(&std::io::Error::other("boom"), HashMap::new(), None);
1102        });
1103        wait_for(&received, 1);
1104        assert!(last_body
1105            .lock()
1106            .unwrap()
1107            .contains("\"trace_id\":\"4bf92f3577b34da6a3ce929d0e0e4736\""));
1108        assert_eq!(current_trace_id(), None);
1109
1110        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
1111        init(|c| {
1112            c.dsn = Some(format!("http://key@{addr}/events"));
1113            c.track_tracing = true;
1114        });
1115        capture_error(&std::io::Error::other("boom"), HashMap::new(), None);
1116        wait_for(&received, 1);
1117        assert!(!last_body.lock().unwrap().contains("\"trace_id\""));
1118
1119        // a fresh trace gets a W3C trace id of its own, which errors inside it carry
1120        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
1121        init(|c| c.dsn = Some(format!("http://key@{addr}/events")));
1122        let fresh = trace("job:reindex", || {
1123            capture_error(&std::io::Error::other("boom"), HashMap::new(), None);
1124            current_trace_id().unwrap()
1125        });
1126        wait_for(&received, 1);
1127        assert_eq!(fresh.len(), 32);
1128        assert!(last_body
1129            .lock()
1130            .unwrap()
1131            .contains(&format!("\"trace_id\":\"{fresh}\"")));
1132        // a malformed traceparent starts a fresh trace too
1133        let fallback = continue_trace("GET /x", Some("00-nope"), || current_trace_id().unwrap());
1134        assert_eq!(fallback.len(), 32);
1135
1136        // http_span hands out a traceparent naming its own span, records that span, and the root
1137        // of a continued trace is sent under the caller's span
1138        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
1139        init(|c| {
1140            c.dsn = Some(format!("http://key@{addr}/events"));
1141            c.trace_capture_threshold = Duration::ZERO;
1142        });
1143        let header = continue_trace("POST /orders", Some(incoming), || {
1144            http_span(
1145                "post",
1146                "https://Payments.example.com/charges/42?token=secret",
1147                HashMap::new(),
1148                |traceparent| traceparent.map(str::to_string),
1149            )
1150        })
1151        .unwrap();
1152        wait_for(&received, 1);
1153        let parsed = trace_parent::parse(&header).unwrap();
1154        assert_eq!(parsed.trace_id, "4bf92f3577b34da6a3ce929d0e0e4736");
1155        assert!(header.starts_with("00-") && header.ends_with("-01"));
1156        {
1157            let body = last_body.lock().unwrap();
1158            assert!(body.starts_with("{\"trace_id\":\"4bf92f3577b34da6a3ce929d0e0e4736\""));
1159            assert!(body.contains(&format!(
1160                "{{\"span_id\":\"{}\",\"parent_span_id\":",
1161                parsed.parent_span_id
1162            )));
1163            assert!(body.contains("\"name\":\"POST payments.example.com\",\"kind\":\"http\""));
1164            assert!(body.contains("\"parent_span_id\":\"00f067aa0ba902b7\""));
1165            assert!(!body.contains("secret") && !body.contains("/charges"));
1166        }
1167
1168        // no header for a host outside trace_propagation_targets, or with propagate_traces off,
1169        // though the span is still recorded; and outside a trace, no header and no span
1170        init(|c| {
1171            c.trace_propagation_targets = Some(vec!["example.com".into()]);
1172        });
1173        trace("GET /x", || {
1174            http_span("GET", "https://badexample.com/", HashMap::new(), |tp| {
1175                assert_eq!(tp, None)
1176            });
1177            http_span("GET", "https://api.example.com/", HashMap::new(), |tp| {
1178                assert!(tp.is_some())
1179            });
1180        });
1181        init(|c| {
1182            c.trace_propagation_targets = None;
1183            c.propagate_traces = false;
1184        });
1185        trace("GET /x", || {
1186            http_span("GET", "https://api.example.com/", HashMap::new(), |tp| {
1187                assert_eq!(tp, None)
1188            });
1189        });
1190        wait_for(&received, 3);
1191        assert!(last_body.lock().unwrap().contains("\"kind\":\"http\""));
1192        init(|c| {
1193            c.propagate_traces = true;
1194            c.trace_capture_threshold = Duration::from_secs(1);
1195        });
1196        let outside = http_span("GET", "https://api.example.com/", HashMap::new(), |tp| {
1197            assert_eq!(tp, None);
1198            7
1199        });
1200        assert_eq!(outside, 7);
1201        std::thread::sleep(Duration::from_millis(100));
1202        assert_eq!(received.load(AtomicOrdering::SeqCst), 3);
1203
1204        // record_change goes out on the delivery thread to the changes endpoint
1205        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
1206        init(|c| c.dsn = Some(format!("http://key@{addr}/api/v1/events")));
1207        record_change(Change {
1208            id: Some("deploy-7".to_string()),
1209            ..Change::new("infrastructure", "Moved to the new database host")
1210        });
1211        wait_for(&received, 1);
1212        assert!(last_body.lock().unwrap().starts_with(
1213            "{\"kind\":\"infrastructure\",\"title\":\"Moved to the new database host\""
1214        ));
1215
1216        // with detect_changes back on, the next init sends the snapshot, env var names included
1217        // only when opted in; any later init sends nothing more
1218        let (addr, received, last_body) = spawn_tracker_server_capturing_body();
1219        std::env::set_var("FORGE_OPS_TEST_APP_SETTING", "never sent");
1220        init(|c| {
1221            c.dsn = Some(format!("http://key@{addr}/api/v1/events"));
1222            c.detect_changes = true;
1223            c.track_env_var_names = true;
1224        });
1225        wait_for(&received, 1);
1226        {
1227            let body = last_body.lock().unwrap();
1228            assert!(body.contains("\"runtime\":\"rust"), "body = {body}");
1229            assert!(body.contains("\"env_var_names\":["), "body = {body}");
1230            assert!(!body.contains("FORGE_OPS_") && !body.contains("never sent"));
1231        }
1232        init(|c| c.track_env_var_names = false);
1233        std::thread::sleep(Duration::from_millis(150));
1234        assert_eq!(received.load(AtomicOrdering::SeqCst), 1);
1235
1236        // init's automatic panic hook reports a panic, then still lets it unwind unchanged
1237        let (addr, received) = spawn_tracker_server();
1238        let previous_hook_ran = Arc::new(AtomicBool::new(false));
1239        let previous_hook_ran_clone = Arc::clone(&previous_hook_ran);
1240        // Installed *before* init() specifically to prove install_panic_hook chains onto whatever
1241        // hook already exists (via take_hook()) rather than replacing it outright.
1242        std::panic::set_hook(Box::new(move |_| {
1243            previous_hook_ran_clone.store(true, AtomicOrdering::SeqCst);
1244        }));
1245        init(|c| {
1246            c.dsn = Some(format!("http://key@{addr}/events"));
1247            c.install_panic_hook = true;
1248        });
1249
1250        let result = std::panic::catch_unwind(|| {
1251            panic!("test panic");
1252        });
1253        assert!(result.is_err());
1254        assert!(
1255            previous_hook_ran.load(AtomicOrdering::SeqCst),
1256            "the previously-installed hook should still have run"
1257        );
1258        wait_for(&received, 1);
1259
1260        // Restore a silent hook so later tests in this binary don't print this test's own
1261        // intentional panic to stderr.
1262        std::panic::set_hook(Box::new(|_| {}));
1263    }
1264
1265    #[test]
1266    fn span_just_runs_the_closure_outside_a_trace() {
1267        assert_eq!(span("free", "service", HashMap::new(), || 7), 7);
1268        // record_span outside a trace is a no-op, not a panic.
1269        record_span(
1270            "free",
1271            "database",
1272            std::time::SystemTime::now(),
1273            1.0,
1274            HashMap::new(),
1275        );
1276    }
1277
1278    #[test]
1279    fn a_panic_inside_trace_or_span_propagates_and_leaves_no_trace_behind() {
1280        // Only the thread-local matters here, so this holds whether or not another test has
1281        // already enabled reporting on the shared global configuration.
1282        let result = std::panic::catch_unwind(|| {
1283            trace("GET /boom", || {
1284                span("bad", "service", HashMap::new(), || panic!("boom"));
1285            })
1286        });
1287        assert!(result.is_err());
1288        assert!(!span_buffer::is_active());
1289    }
1290
1291    #[test]
1292    fn trace_returns_the_closures_value_and_a_nested_trace_is_a_span() {
1293        let value = trace("outer", || trace("inner", || 5));
1294        assert_eq!(value, 5);
1295        assert!(!span_buffer::is_active());
1296    }
1297
1298    #[test]
1299    fn context_macro_builds_expected_map() {
1300        let ctx = context! {"order_id" => 42, "customer" => "acme-inc"};
1301        assert_eq!(ctx.get("order_id"), Some(&Value::Number(42.0)));
1302        assert_eq!(
1303            ctx.get("customer"),
1304            Some(&Value::String("acme-inc".to_string()))
1305        );
1306    }
1307}