1use std::any::Any;
75use std::backtrace::{Backtrace, BacktraceStatus};
76use std::cell::RefCell;
77use std::future::Future;
78use std::panic::AssertUnwindSafe;
79use std::pin::Pin;
80use std::sync::{Arc, Once};
81use std::task::{Context, Poll};
82
83use axum::extract::MatchedPath;
84use axum::http::{Request, StatusCode};
85use axum::response::{IntoResponse, Response};
86use futures::FutureExt;
87use pin_project_lite::pin_project;
88use tower::{Layer, Service};
89
90use crate::middleware::RequestId;
91use crate::middleware::exception_filter::AutumnErrorInfo;
92
93pub type ReportFuture<'a> = Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
99
100#[derive(Debug, Clone)]
107#[non_exhaustive]
108pub struct ErrorEvent {
109 pub status: StatusCode,
111 pub message: String,
114 pub problem_type: Option<String>,
116 pub request_id: Option<String>,
118 pub route: Option<String>,
120 pub method: Option<String>,
122 pub panic: Option<PanicInfo>,
125}
126
127#[derive(Debug, Clone)]
129#[non_exhaustive]
130pub struct PanicInfo {
131 pub payload: String,
133 pub backtrace: Option<String>,
135}
136
137pub trait ErrorReporter: Send + Sync + 'static {
148 fn report<'a>(&'a self, event: &'a ErrorEvent) -> ReportFuture<'a>;
153}
154
155#[derive(Debug, Clone, Default)]
160pub struct LogReporter;
161
162impl ErrorReporter for LogReporter {
163 fn report<'a>(&'a self, event: &'a ErrorEvent) -> ReportFuture<'a> {
164 Box::pin(async move {
165 if let Some(panic) = event.panic.as_ref() {
166 tracing::error!(
167 status = %event.status,
168 method = event.method.as_deref().unwrap_or("-"),
169 route = event.route.as_deref().unwrap_or("-"),
170 request_id = event.request_id.as_deref().unwrap_or("-"),
171 backtrace = panic.backtrace.as_deref().unwrap_or("(set RUST_BACKTRACE=1 to capture)"),
172 "handler panic captured: {}",
173 panic.payload
174 );
175 } else {
176 tracing::error!(
177 status = %event.status,
178 method = event.method.as_deref().unwrap_or("-"),
179 route = event.route.as_deref().unwrap_or("-"),
180 request_id = event.request_id.as_deref().unwrap_or("-"),
181 problem_type = event.problem_type.as_deref().unwrap_or("-"),
182 "server error captured: {}",
183 event.message
184 );
185 }
186 })
187 }
188}
189
190#[derive(Clone, Default)]
194pub(crate) struct RegisteredReporters(pub(crate) Vec<Arc<dyn ErrorReporter>>);
195
196struct ReporterChain {
198 reporters: Vec<Arc<dyn ErrorReporter>>,
199 enabled: bool,
200 sample_rate: f64,
201}
202
203impl ReporterChain {
204 fn dispatch(self: &Arc<Self>, event: ErrorEvent) {
207 if !self.enabled || !sampled(self.sample_rate) {
208 return;
209 }
210 if let Ok(handle) = tokio::runtime::Handle::try_current() {
213 let chain = Arc::clone(self);
214 handle.spawn(async move {
215 chain.report_all(&event).await;
216 });
217 }
218 }
219
220 async fn report_all(&self, event: &ErrorEvent) {
221 for reporter in &self.reporters {
222 match std::panic::catch_unwind(AssertUnwindSafe(|| reporter.report(event))) {
225 Ok(future) => {
226 if AssertUnwindSafe(future).catch_unwind().await.is_err() {
227 tracing::warn!("error reporter panicked while reporting; ignoring");
228 }
229 }
230 Err(_panic) => {
231 tracing::warn!("error reporter panicked constructing report future; ignoring");
232 }
233 }
234 }
235 }
236}
237
238thread_local! {
239 static RNG_STATE: std::cell::Cell<u64> = std::cell::Cell::new(seed_rng());
243}
244
245fn seed_rng() -> u64 {
248 let mut buf = [0u8; 8];
249 if getrandom::getrandom(&mut buf).is_ok() {
250 let seed = u64::from_ne_bytes(buf);
251 if seed != 0 {
252 return seed;
253 }
254 }
255 0x5555_5555_5555_5555
256}
257
258fn next_u64() -> u64 {
260 RNG_STATE.with(|cell| {
261 let mut x = cell.get();
262 x ^= x << 13;
263 x ^= x >> 7;
264 x ^= x << 17;
265 cell.set(x);
266 x
267 })
268}
269
270#[allow(clippy::cast_precision_loss)]
275fn sampled(rate: f64) -> bool {
276 if rate >= 1.0 {
277 return true;
278 }
279 if rate <= 0.0 {
280 return false;
281 }
282 let draw = next_u64() >> 11;
284 let value = draw as f64 / (1u64 << 53) as f64;
285 value < rate
286}
287
288thread_local! {
291 static LAST_PANIC: RefCell<Option<CapturedPanic>> = const { RefCell::new(None) };
292}
293
294struct CapturedPanic {
295 backtrace: Option<String>,
296}
297
298static HOOK_INSTALLED: Once = Once::new();
299
300fn ensure_panic_hook() {
305 HOOK_INSTALLED.call_once(|| {
306 let previous = std::panic::take_hook();
307 std::panic::set_hook(Box::new(move |info| {
308 let backtrace = Backtrace::capture();
311 let backtrace =
312 (backtrace.status() == BacktraceStatus::Captured).then(|| backtrace.to_string());
313 LAST_PANIC.with(|cell| {
314 *cell.borrow_mut() = Some(CapturedPanic { backtrace });
315 });
316 previous(info);
317 }));
318 });
319}
320
321fn format_panic_payload(payload: &(dyn Any + Send)) -> String {
324 payload
325 .downcast_ref::<&str>()
326 .map(|s| (*s).to_owned())
327 .or_else(|| payload.downcast_ref::<String>().cloned())
328 .unwrap_or_else(|| "handler panicked".to_owned())
329}
330
331#[derive(Clone)]
336struct RequestContext {
337 method: String,
338 route: Option<String>,
339 request_id: Option<String>,
340}
341
342#[derive(Clone)]
349pub struct ReportingLayer {
350 chain: Arc<ReporterChain>,
351}
352
353impl ReportingLayer {
354 #[must_use]
359 pub(crate) fn new(
360 reporters: Vec<Arc<dyn ErrorReporter>>,
361 enabled: bool,
362 sample_rate: f64,
363 ) -> Self {
364 ensure_panic_hook();
365 let reporters = if reporters.is_empty() {
366 vec![Arc::new(LogReporter) as Arc<dyn ErrorReporter>]
367 } else {
368 reporters
369 };
370 Self {
371 chain: Arc::new(ReporterChain {
372 reporters,
373 enabled,
374 sample_rate,
375 }),
376 }
377 }
378}
379
380impl<S> Layer<S> for ReportingLayer {
381 type Service = ReportingService<S>;
382
383 fn layer(&self, inner: S) -> Self::Service {
384 ReportingService {
385 inner,
386 chain: Arc::clone(&self.chain),
387 }
388 }
389}
390
391#[derive(Clone)]
393pub struct ReportingService<S> {
394 inner: S,
395 chain: Arc<ReporterChain>,
396}
397
398impl<S, ReqBody> Service<Request<ReqBody>> for ReportingService<S>
399where
400 S: Service<Request<ReqBody>, Response = Response>,
401{
402 type Response = Response;
403 type Error = S::Error;
404 type Future = ReportingFuture<S::Future>;
405
406 fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
407 self.inner.poll_ready(cx)
408 }
409
410 fn call(&mut self, req: Request<ReqBody>) -> Self::Future {
411 let method = req.method().as_str().to_owned();
412 let route = req
413 .extensions()
414 .get::<MatchedPath>()
415 .map(|m| m.as_str().to_owned());
416 let request_id = req
417 .extensions()
418 .get::<RequestId>()
419 .map(std::string::ToString::to_string);
420 let context = Some(RequestContext {
421 method,
422 route,
423 request_id,
424 });
425
426 let inner = &mut self.inner;
431 match std::panic::catch_unwind(AssertUnwindSafe(|| inner.call(req))) {
432 Ok(future) => ReportingFuture {
433 inner: Some(future),
434 pending_panic: None,
435 context,
436 chain: Arc::clone(&self.chain),
437 },
438 Err(panic) => ReportingFuture {
439 inner: None,
440 pending_panic: Some(panic),
441 context,
442 chain: Arc::clone(&self.chain),
443 },
444 }
445 }
446}
447
448pin_project! {
449 pub struct ReportingFuture<F> {
452 #[pin]
453 inner: Option<F>,
454 pending_panic: Option<Box<dyn Any + Send>>,
457 context: Option<RequestContext>,
458 chain: Arc<ReporterChain>,
459 }
460}
461
462impl<F, E> Future for ReportingFuture<F>
463where
464 F: Future<Output = Result<Response, E>>,
465{
466 type Output = Result<Response, E>;
467
468 fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
469 let this = self.project();
470
471 if let Some(panic) = this.pending_panic.take() {
473 let context = this.context.take();
474 return Poll::Ready(Ok(handle_panic(&*panic, context, this.chain)));
475 }
476
477 let Some(inner) = this.inner.as_pin_mut() else {
478 return Poll::Pending;
480 };
481
482 match std::panic::catch_unwind(AssertUnwindSafe(move || inner.poll(cx))) {
485 Ok(Poll::Pending) => Poll::Pending,
486 Ok(Poll::Ready(Ok(response))) => {
487 if let Some(context) = this.context.take() {
488 report_response(&response, context, this.chain);
489 }
490 Poll::Ready(Ok(response))
491 }
492 Ok(Poll::Ready(Err(error))) => Poll::Ready(Err(error)),
493 Err(panic) => {
494 let context = this.context.take();
495 let response = handle_panic(&*panic, context, this.chain);
496 Poll::Ready(Ok(response))
497 }
498 }
499 }
500}
501
502fn report_response(response: &Response, context: RequestContext, chain: &Arc<ReporterChain>) {
504 if !response.status().is_server_error() {
505 return;
506 }
507 let info = response.extensions().get::<AutumnErrorInfo>();
508 let (message, problem_type) = info.map_or_else(
509 || {
510 (
511 response
512 .status()
513 .canonical_reason()
514 .unwrap_or("server error")
515 .to_owned(),
516 None,
517 )
518 },
519 |info| (info.message.clone(), info.problem_type.map(str::to_owned)),
520 );
521
522 chain.dispatch(ErrorEvent {
523 status: response.status(),
524 message,
525 problem_type,
526 request_id: context.request_id,
527 route: context.route,
528 method: Some(context.method),
529 panic: None,
530 });
531}
532
533fn handle_panic(
535 payload: &(dyn Any + Send),
536 context: Option<RequestContext>,
537 chain: &Arc<ReporterChain>,
538) -> Response {
539 let message = format_panic_payload(payload);
540 let backtrace = LAST_PANIC
541 .with(|cell| cell.borrow_mut().take())
542 .and_then(|captured| captured.backtrace);
543
544 if let Some(context) = context {
545 chain.dispatch(ErrorEvent {
546 status: StatusCode::INTERNAL_SERVER_ERROR,
547 message: message.clone(),
548 problem_type: None,
549 request_id: context.request_id,
550 route: context.route,
551 method: Some(context.method),
552 panic: Some(PanicInfo {
553 payload: message,
554 backtrace,
555 }),
556 });
557 }
558
559 crate::error::AutumnError::internal_server_error_msg("Internal server error").into_response()
564}
565
566#[cfg(test)]
567mod tests {
568 use super::*;
569 use std::sync::Mutex;
570
571 #[test]
572 fn sampled_extremes_are_deterministic() {
573 assert!(sampled(1.0));
574 assert!(sampled(2.0));
575 assert!(!sampled(0.0));
576 assert!(!sampled(-1.0));
577 }
578
579 #[test]
580 fn sampled_full_rate_always_true_over_many_draws() {
581 for _ in 0..1000 {
582 assert!(sampled(1.0));
583 }
584 }
585
586 #[test]
587 fn format_panic_payload_handles_str_and_string() {
588 let s: &str = "boom";
589 assert_eq!(format_panic_payload(&s), "boom");
590 let owned: String = "kaboom".to_owned();
591 assert_eq!(format_panic_payload(&owned), "kaboom");
592 let other: u32 = 7;
593 assert_eq!(format_panic_payload(&other), "handler panicked");
594 }
595
596 #[test]
597 fn log_reporter_is_the_default_when_empty() {
598 let layer = ReportingLayer::new(Vec::new(), true, 1.0);
599 assert_eq!(layer.chain.reporters.len(), 1);
600 }
601
602 #[tokio::test]
603 async fn panic_in_inner_call_is_caught_as_500() {
604 use axum::body::Body;
605 use std::convert::Infallible;
606 use tower::ServiceExt;
607
608 #[derive(Clone)]
611 struct PanicInCall;
612 impl Service<Request<Body>> for PanicInCall {
613 type Response = Response;
614 type Error = Infallible;
615 type Future = std::future::Ready<Result<Response, Infallible>>;
616
617 fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
618 Poll::Ready(Ok(()))
619 }
620
621 fn call(&mut self, _req: Request<Body>) -> Self::Future {
622 panic!("boom in call");
623 }
624 }
625
626 let service = ReportingLayer::new(Vec::new(), true, 1.0).layer(PanicInCall);
627 let response = service
628 .oneshot(Request::new(Body::empty()))
629 .await
630 .expect("panic in call must be converted to a response, not propagated");
631 assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
632 }
633
634 #[tokio::test]
635 async fn disabled_chain_does_not_dispatch() {
636 #[derive(Clone)]
637 struct Counter(Arc<Mutex<u32>>);
638 impl ErrorReporter for Counter {
639 fn report<'a>(&'a self, _event: &'a ErrorEvent) -> ReportFuture<'a> {
640 let count = self.0.clone();
641 Box::pin(async move {
642 *count.lock().unwrap() += 1;
643 })
644 }
645 }
646
647 let count = Arc::new(Mutex::new(0));
648 let chain = Arc::new(ReporterChain {
649 reporters: vec![Arc::new(Counter(count.clone()))],
650 enabled: false,
651 sample_rate: 1.0,
652 });
653 chain.dispatch(ErrorEvent {
654 status: StatusCode::INTERNAL_SERVER_ERROR,
655 message: "x".into(),
656 problem_type: None,
657 request_id: None,
658 route: None,
659 method: None,
660 panic: None,
661 });
662 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
663 assert_eq!(*count.lock().unwrap(), 0);
664 }
665
666 fn server_error_event() -> ErrorEvent {
667 ErrorEvent {
668 status: StatusCode::INTERNAL_SERVER_ERROR,
669 message: "boom".into(),
670 problem_type: Some("https://autumn.dev/problems/x".into()),
671 request_id: Some("req-1".into()),
672 route: Some("/x".into()),
673 method: Some("GET".into()),
674 panic: None,
675 }
676 }
677
678 fn panic_event() -> ErrorEvent {
679 ErrorEvent {
680 status: StatusCode::INTERNAL_SERVER_ERROR,
681 message: "kaboom".into(),
682 problem_type: None,
683 request_id: None,
684 route: None,
685 method: None,
686 panic: Some(PanicInfo {
687 payload: "kaboom".into(),
688 backtrace: Some("<backtrace>".into()),
689 }),
690 }
691 }
692
693 #[tokio::test]
694 async fn log_reporter_reports_both_event_kinds() {
695 let reporter = LogReporter;
698 reporter.report(&server_error_event()).await;
699 reporter.report(&panic_event()).await;
700 }
701
702 #[test]
703 fn sampled_fractional_uses_prng_and_varies() {
704 let mut trues = 0;
707 for _ in 0..10_000 {
708 if sampled(0.5) {
709 trues += 1;
710 }
711 }
712 assert!(
713 trues > 0 && trues < 10_000,
714 "fractional sampling should produce a mix of decisions, got {trues}"
715 );
716 }
717
718 #[tokio::test]
719 async fn reporter_panicking_while_constructing_future_is_swallowed() {
720 struct PanicOnConstruct;
724 impl ErrorReporter for PanicOnConstruct {
725 fn report<'a>(&'a self, _event: &'a ErrorEvent) -> ReportFuture<'a> {
726 panic!("panic before returning the future");
727 }
728 }
729
730 let chain = ReporterChain {
731 reporters: vec![Arc::new(PanicOnConstruct)],
732 enabled: true,
733 sample_rate: 1.0,
734 };
735 chain.report_all(&server_error_event()).await;
737 }
738
739 #[test]
740 fn dispatch_without_a_runtime_is_a_noop() {
741 let chain = Arc::new(ReporterChain {
744 reporters: vec![Arc::new(LogReporter)],
745 enabled: true,
746 sample_rate: 1.0,
747 });
748 chain.dispatch(server_error_event());
749 }
750}