linera_base/
panic_hook.rs

1// Copyright (c) Zefchain Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Process-wide reporting of panics.
5//!
6//! Tokio catches a panic at the task boundary and hands it to whoever joins the task, so a
7//! panicking task neither stops the runtime nor, on its own, produces anything a monitoring
8//! system can act on: the default hook writes to standard error and nothing else. The hook
9//! installed here reports the panic through `tracing` and the metrics registry first, so
10//! that panics are visible wherever the process's other logs and metrics are collected.
11
12use std::{
13    panic::PanicHookInfo,
14    sync::{Mutex, Once},
15};
16
17#[cfg(with_metrics)]
18mod metrics {
19    use std::sync::LazyLock;
20
21    use prometheus::IntCounter;
22
23    use crate::prometheus_util::register_int_counter;
24
25    /// Panics observed by the hook installed by [`super::init`].
26    ///
27    /// A panic does not stop the process, so this counter is often the only durable signal
28    /// that one happened: whatever the panicking task was responsible for has stopped, and
29    /// the effect on the rest of the process depends entirely on who was joining it. Any
30    /// increase deserves investigation.
31    pub(super) static PANICS: LazyLock<IntCounter> =
32        LazyLock::new(|| register_int_counter("linera_panics_total", "Number of panics observed"));
33}
34
35/// A panic hook, in the form [`std::panic::take_hook`] returns it.
36type PanicHook = Box<dyn Fn(&PanicHookInfo<'_>) + Sync + Send>;
37
38/// The hook that was installed before ours, which we delegate to so that the standard
39/// message and the `RUST_BACKTRACE` backtrace are still printed.
40static PREVIOUS_HOOK: Mutex<Option<PanicHook>> = Mutex::new(None);
41
42static INIT: Once = Once::new();
43
44/// Installs a panic hook that reports panics through `tracing` and the metrics registry
45/// before delegating to the hook that was previously installed.
46///
47/// Calling this more than once has no further effect. It does not change what a panic
48/// *does* — the process still unwinds the panicking task and keeps running — only what is
49/// recorded about it.
50pub fn init() {
51    INIT.call_once(|| {
52        *PREVIOUS_HOOK.lock().expect("hook mutex is never poisoned") =
53            Some(std::panic::take_hook());
54        std::panic::set_hook(Box::new(report_panic));
55    });
56}
57
58/// The panic message, for the two payload types `panic!` produces.
59///
60/// `PanicHookInfo::payload_as_str` does the same thing, but is not yet stable in the
61/// toolchain the release branches pin, and this code is backported to them.
62fn payload_message<'a>(info: &'a PanicHookInfo<'_>) -> &'a str {
63    let payload = info.payload();
64    payload
65        .downcast_ref::<&str>()
66        .copied()
67        .or_else(|| payload.downcast_ref::<String>().map(String::as_str))
68        .unwrap_or("<non-string payload>")
69}
70
71fn report_panic(info: &PanicHookInfo<'_>) {
72    #[cfg(with_metrics)]
73    metrics::PANICS.inc();
74
75    let thread = std::thread::current();
76    tracing::error!(
77        thread = thread.name().unwrap_or("<unnamed>"),
78        location = info.location().map(tracing::field::display),
79        message = payload_message(info),
80        "Panic",
81    );
82
83    // The lock is only ever held while installing the hook, and the hook is installed
84    // once, so a panic here would mean a panic inside `init` itself.
85    if let Ok(guard) = PREVIOUS_HOOK.lock() {
86        if let Some(previous) = guard.as_ref() {
87            previous(info);
88        }
89    }
90}
91
92#[cfg(test)]
93mod tests {
94    use std::{
95        panic::AssertUnwindSafe,
96        sync::atomic::{AtomicUsize, Ordering},
97    };
98
99    use super::*;
100
101    /// Installing the hook twice must not chain it to itself, which would make every panic
102    /// report grow by one line per call.
103    #[test]
104    fn test_init_is_idempotent() {
105        static DELEGATIONS: AtomicUsize = AtomicUsize::new(0);
106
107        std::panic::set_hook(Box::new(|_| {
108            DELEGATIONS.fetch_add(1, Ordering::SeqCst);
109        }));
110        init();
111        init();
112
113        let panicked = std::panic::catch_unwind(AssertUnwindSafe(|| panic!("boom"))).is_err();
114
115        assert!(panicked);
116        assert_eq!(
117            DELEGATIONS.load(Ordering::SeqCst),
118            1,
119            "the hook installed before `init` ran exactly once",
120        );
121    }
122}
linera_base/panic_hook.rs

linera_base/
panic_hook.rs
