use crate::exn::{Attachment, BuiltinKey, Frame};
use parking_lot::Mutex;
#[cfg(feature = "fastrace")]
pub(crate) const ERROR_EVENT: &str = "error";
const NS_PER_SEC: u64 = 1_000_000_000;
use std::collections::HashMap;
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, LazyLock, OnceLock};
type Hook = Arc<dyn Fn(&Frame) + Send + Sync + 'static>;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct HookId(u64);
const DEFAULT_HOOK_ID: HookId = HookId(0);
static NEXT_HOOK_ID: AtomicU64 = AtomicU64::new(1);
struct HookRegistry<F> {
default: fn() -> Vec<F>,
slots: Mutex<Arc<Vec<F>>>,
}
impl<F: Clone> HookRegistry<F> {
fn new(default: fn() -> Vec<F>) -> Self {
Self {
default,
slots: Mutex::new(Arc::new(default())),
}
}
fn snapshot(&self) -> Arc<Vec<F>> {
Arc::clone(&self.slots.lock())
}
fn push(&self, hook: F) {
let mut guard = self.slots.lock();
let mut new_vec = (**guard).clone();
new_vec.push(hook);
*guard = Arc::new(new_vec);
}
fn remove_where(&self, pred: impl Fn(&F) -> bool) -> bool {
let mut guard = self.slots.lock();
let mut new_vec = (**guard).clone();
let before = new_vec.len();
new_vec.retain(|entry| !pred(entry));
if new_vec.len() == before {
return false;
}
*guard = Arc::new(new_vec);
true
}
fn reset(&self) {
*self.slots.lock() = Arc::new((self.default)());
}
fn len(&self) -> usize {
self.slots.lock().len()
}
}
static HOOKS: OnceLock<HookRegistry<(HookId, Hook)>> = OnceLock::new();
fn default_sink_hooks() -> Vec<(HookId, Hook)> {
vec![(DEFAULT_HOOK_ID, default_hook())]
}
fn hooks() -> &'static HookRegistry<(HookId, Hook)> {
HOOKS.get_or_init(|| HookRegistry::new(default_sink_hooks))
}
struct Envelope {
occurrence: u64,
thread: std::borrow::Cow<'static, str>,
uptime_ms: u64,
}
impl Envelope {
fn capture(type_name: &'static str) -> Self {
let occurrence = crate::exn::error_counts()
.iter()
.find(|(k, _)| *k == type_name)
.map_or(0, |(_, c)| *c);
let thread = std::thread::current();
Self {
occurrence,
thread: thread
.name()
.map_or(std::borrow::Cow::Borrowed("<unnamed>"), |n| {
std::borrow::Cow::Owned(n.to_string())
}),
uptime_ms: crate::profiling::clock::now_ns() / 1_000_000,
}
}
}
fn default_hook() -> Hook {
Arc::new(|frame: &Frame| {
let _span = crate::scope!("error");
let envelope = Envelope::capture(frame.type_name());
match crate::config::report_mode() {
crate::config::ReportMode::Off => {
let message = if matches!(frame.context, crate::exn::Context::None) {
frame.error().to_string()
} else {
format!("{} — {}", frame.error(), frame.context)
};
log::error!(
target: "fast_observe.error",
error_type = frame.type_name(),
error_file = frame.location().file(),
error_line = frame.location().line(),
occurrence = envelope.occurrence,
thread = &*envelope.thread,
uptime_ms = envelope.uptime_ms;
"{message}",
);
}
crate::config::ReportMode::Text => log_report(frame, &envelope, false),
crate::config::ReportMode::Json => log_report(frame, &envelope, true),
}
})
}
fn log_report(frame: &Frame, envelope: &Envelope, json: bool) {
#[cfg(feature = "serde")]
let report = if json {
crate::report::render_frame_report_json(frame)
} else {
crate::report::render_frame_report(frame)
};
#[cfg(not(feature = "serde"))]
let report = {
let _ = json;
crate::report::render_frame_report(frame)
};
log::error!(
target: "fast_observe.error",
error_type = frame.type_name(),
error_file = frame.location().file(),
error_line = frame.location().line(),
occurrence = envelope.occurrence,
thread = &*envelope.thread,
uptime_ms = envelope.uptime_ms;
"{report}"
);
}
pub fn add_error_hook(hook: impl Fn(&Frame) + Send + Sync + 'static) -> HookId {
let id = HookId(NEXT_HOOK_ID.fetch_add(1, Ordering::Relaxed));
hooks().push((id, Arc::new(hook)));
id
}
#[must_use]
pub fn remove_error_hook(id: HookId) -> bool {
if id == DEFAULT_HOOK_ID {
return false;
}
hooks().remove_where(|(entry_id, _)| *entry_id == id)
}
pub fn clear_error_hooks() {
hooks().reset();
}
#[must_use]
pub fn hooks_len() -> usize {
hooks().len()
}
type CaptureHook = Arc<dyn Fn(&mut Frame) + Send + Sync + 'static>;
static CAPTURE_HOOKS: OnceLock<HookRegistry<CaptureHook>> = OnceLock::new();
fn default_capture_hooks() -> Vec<CaptureHook> {
vec![
#[cfg(feature = "fastrace")]
trace_context_capture_hook(),
scope_path_capture_hook(),
#[cfg(feature = "instant")]
span_trail_capture_hook(),
#[cfg(feature = "backtrace")]
backtrace_capture_hook(),
]
}
fn capture_hooks() -> &'static HookRegistry<CaptureHook> {
CAPTURE_HOOKS.get_or_init(|| HookRegistry::new(default_capture_hooks))
}
#[cfg(feature = "fastrace")]
fn trace_context_capture_hook() -> CaptureHook {
Arc::new(|frame: &mut Frame| {
if let Some(ctx) = fastrace::collector::SpanContext::current_local_parent() {
frame.push_attachment(Attachment::with_key(
BuiltinKey::TraceId.as_str(),
ctx.trace_id,
));
}
fastrace::local::LocalSpan::add_event(fastrace::Event::new(ERROR_EVENT).with_properties(
|| {
use std::borrow::Cow;
[
(Cow::Borrowed("type"), Cow::Borrowed(frame.type_name())),
(
Cow::Borrowed("location"),
Cow::Owned(format!(
"{}:{}",
frame.location().file(),
frame.location().line()
)),
),
]
},
));
})
}
fn scope_path_capture_hook() -> CaptureHook {
Arc::new(|frame: &mut Frame| {
let path = crate::profiling::scope_path();
if !path.is_empty() {
frame.push_attachment(Attachment::with_key(
BuiltinKey::ScopePath.as_str(),
path.join(" → "),
));
}
if let Some(ms) = crate::profiling::current_scope_elapsed_ms() {
frame.push_attachment(Attachment::with_key(
BuiltinKey::ScopeElapsedMs.as_str(),
ms,
));
}
})
}
#[cfg(feature = "instant")]
const SPAN_TRAIL_LEN: usize = 8;
#[cfg(feature = "instant")]
struct SpanTrail(Vec<crate::profiling::instant::SpanRecord>);
#[cfg(feature = "instant")]
impl std::fmt::Display for SpanTrail {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let mut first = true;
for span in &self.0 {
if !first {
f.write_str("; ")?;
}
first = false;
write!(
f,
"{}({})",
span.name,
humantime::format_duration(span.duration())
)?;
}
Ok(())
}
}
#[cfg(feature = "instant")]
fn span_trail_capture_hook() -> CaptureHook {
Arc::new(|frame: &mut Frame| {
use crate::config::Backends;
let backends = crate::config::config().backends();
if !backends.contains(Backends::INSTANT) && !backends.contains(Backends::WEB) {
return;
}
let trail = crate::profiling::instant::peek_recent(SPAN_TRAIL_LEN);
if trail.is_empty() {
return;
}
frame.push_attachment(
Attachment::with_key(BuiltinKey::SpanTrail.as_str(), SpanTrail(trail))
.with_placement(crate::exn::Placement::Appendix),
);
})
}
#[cfg(feature = "backtrace")]
fn backtrace_enabled(read: impl Fn(&str) -> Option<String>) -> bool {
if let Some(override_value) = read(crate::env_vars::OBSERVE_BACKTRACE) {
return matches!(
override_value.trim().to_ascii_lowercase().as_str(),
"1" | "true" | "full"
);
}
match read(crate::env_vars::RUST_BACKTRACE) {
Some(value) => matches!(value.trim().to_ascii_lowercase().as_str(), "1" | "full"),
None => false,
}
}
#[cfg(feature = "backtrace")]
static BACKTRACE_ENABLED: LazyLock<bool> =
LazyLock::new(|| backtrace_enabled(|name| std::env::var(name).ok()));
#[cfg(feature = "backtrace")]
struct BacktraceAttachment(std::backtrace::Backtrace);
#[cfg(feature = "backtrace")]
impl std::fmt::Display for BacktraceAttachment {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
#[cfg(feature = "backtrace")]
{
let frames = self.0.frames();
for (i, frame) in frames.iter().enumerate() {
writeln!(f, "frame {i}: {frame:?}")?;
}
Ok(())
}
}
}
#[cfg(feature = "backtrace")]
fn backtrace_capture_hook() -> CaptureHook {
Arc::new(|frame: &mut Frame| {
if !*BACKTRACE_ENABLED {
return;
}
frame.push_attachment(
Attachment::with_key(
BuiltinKey::Backtrace.as_str(),
BacktraceAttachment(std::backtrace::Backtrace::force_capture()),
)
.with_placement(crate::exn::Placement::Appendix),
);
})
}
pub fn add_capture_hook(hook: impl Fn(&mut Frame) + Send + Sync + 'static) {
capture_hooks().push(Arc::new(hook));
}
#[must_use]
pub fn capture_hooks_len() -> usize {
capture_hooks().len()
}
pub(crate) fn run_capture_hooks(frame: &mut Frame) {
let snapshot: Arc<Vec<CaptureHook>> = capture_hooks().snapshot();
for hook in snapshot.iter() {
let _ = catch_unwind(AssertUnwindSafe(|| hook(frame)));
}
}
static DEFAULT_HOOK_ENABLED: AtomicBool = AtomicBool::new(true);
pub fn set_default_hook_enabled(enabled: bool) {
DEFAULT_HOOK_ENABLED.store(enabled, Ordering::Relaxed);
}
struct ThrottleState {
window_start_ns: u64,
count: u32,
}
struct RateLimiter {
state: Mutex<HashMap<&'static str, ThrottleState>>,
}
impl RateLimiter {
fn new() -> Self {
Self {
state: Mutex::new(HashMap::new()),
}
}
fn over_budget(&self, key: &'static str, limit: u32, now_ns: u64) -> bool {
if limit == 0 {
return false; }
let mut map = self.state.lock();
let state = map.entry(key).or_insert(ThrottleState {
window_start_ns: now_ns,
count: 0,
});
if now_ns.saturating_sub(state.window_start_ns) >= NS_PER_SEC {
state.window_start_ns = now_ns;
state.count = 0;
}
if state.count >= limit {
return true;
}
state.count += 1;
false
}
}
impl Default for RateLimiter {
fn default() -> Self {
Self::new()
}
}
static THROTTLE: LazyLock<RateLimiter> = LazyLock::new(RateLimiter::new);
fn throttled(type_name: &'static str) -> bool {
THROTTLE.over_budget(
type_name,
crate::config::config().error_hook_throttle(),
crate::profiling::clock::now_ns(),
)
}
pub(crate) fn invoke(frame: &Frame) {
crate::exn::record_error(frame.type_name);
#[cfg(feature = "metrics-facade")]
crate::exn::record_error_metrics(frame.type_name);
if throttled(frame.type_name) {
return;
}
let snapshot: Arc<Vec<(HookId, Hook)>> = hooks().snapshot();
let skip_default = !DEFAULT_HOOK_ENABLED.load(Ordering::Relaxed);
for (i, (_id, sink)) in snapshot.iter().enumerate() {
if i == 0 && skip_default {
continue;
}
let _ = catch_unwind(AssertUnwindSafe(|| sink(frame)));
}
}
pub fn init() {
drop(crate::deploy::observe().init());
}
#[cfg(test)]
mod rate_limiter_tests {
use super::{NS_PER_SEC, RateLimiter};
#[test]
fn under_limit_passes() {
let limiter = RateLimiter::new();
assert!(!limiter.over_budget("key", 2, 0));
assert!(!limiter.over_budget("key", 2, 1));
}
#[test]
fn over_limit_blocks() {
let limiter = RateLimiter::new();
assert!(!limiter.over_budget("key", 2, 0));
assert!(!limiter.over_budget("key", 2, 1));
assert!(limiter.over_budget("key", 2, 2));
assert!(limiter.over_budget("key", 2, 3));
}
#[test]
fn window_rolls_after_one_second() {
let limiter = RateLimiter::new();
assert!(!limiter.over_budget("key", 1, 0));
assert!(limiter.over_budget("key", 1, 1));
assert!(!limiter.over_budget("key", 1, NS_PER_SEC));
assert!(limiter.over_budget("key", 1, NS_PER_SEC + 1));
}
#[test]
fn limit_zero_never_blocks() {
let limiter = RateLimiter::new();
for now in 0..100u64 {
assert!(!limiter.over_budget("key", 0, now));
}
}
#[test]
fn keys_are_independent() {
let limiter = RateLimiter::new();
assert!(!limiter.over_budget("a", 1, 0));
assert!(limiter.over_budget("a", 1, 0));
assert!(!limiter.over_budget("b", 1, 0));
}
}
#[cfg(all(test, feature = "backtrace"))]
mod tests {
use super::backtrace_enabled;
fn env<'a>(pairs: &'a [(&'a str, &'a str)]) -> impl Fn(&str) -> Option<String> + 'a {
move |name| {
pairs
.iter()
.find(|(key, _)| *key == name)
.map(|(_, value)| (*value).to_owned())
}
}
#[test]
fn backtrace_enabled_matrix() {
assert!(!backtrace_enabled(env(&[])));
assert!(!backtrace_enabled(env(&[(
crate::env_vars::RUST_BACKTRACE,
"0"
)])));
assert!(backtrace_enabled(env(&[(
crate::env_vars::RUST_BACKTRACE,
"1"
)])));
assert!(backtrace_enabled(env(&[(
crate::env_vars::RUST_BACKTRACE,
"full"
)])));
assert!(!backtrace_enabled(env(&[(
crate::env_vars::RUST_BACKTRACE,
"yes"
)])));
assert!(backtrace_enabled(env(&[(
crate::env_vars::OBSERVE_BACKTRACE,
"1"
)])));
assert!(backtrace_enabled(env(&[(
crate::env_vars::OBSERVE_BACKTRACE,
"true"
)])));
assert!(backtrace_enabled(env(&[(
crate::env_vars::OBSERVE_BACKTRACE,
"full"
)])));
assert!(backtrace_enabled(env(&[(
crate::env_vars::OBSERVE_BACKTRACE,
"TRUE"
)])));
assert!(!backtrace_enabled(env(&[(
crate::env_vars::OBSERVE_BACKTRACE,
"0"
)])));
assert!(!backtrace_enabled(env(&[(
crate::env_vars::OBSERVE_BACKTRACE,
"no"
)])));
assert!(!backtrace_enabled(env(&[
(crate::env_vars::RUST_BACKTRACE, "1"),
(crate::env_vars::OBSERVE_BACKTRACE, "0"),
])));
assert!(backtrace_enabled(env(&[
(crate::env_vars::RUST_BACKTRACE, "0"),
(crate::env_vars::OBSERVE_BACKTRACE, "1"),
])));
}
}
#[cfg(feature = "otel")]
pub fn init_otel(reporter: fastrace_opentelemetry::OpenTelemetryReporter) {
fastrace::set_reporter(reporter, fastrace::collector::Config::default());
}