Skip to main content

ash_log/
async_backend.rs

1//! Moving writes off the calling thread.
2//!
3//! # Why this exists
4//!
5//! Every built-in backend writes synchronously: [`log_audit`] returns only once
6//! the event has reached the OS. That is the right default for an audit trail,
7//! but it means a slow disk stalls whichever thread is serving a request, and
8//! inside an async runtime it blocks a worker that could be running other tasks.
9//!
10//! [`AsyncBackend`] hands events to a dedicated writer thread through a bounded
11//! queue. Callers pay an enqueue and return; the writer does the I/O.
12//!
13//! # When this actually helps
14//!
15//! Measured, not assumed (`cargo bench --features hmac-chain`):
16//!
17//! - **A burst that fits the queue**: 20 events to a 200us-per-write
18//!   destination took **6.15ms synchronously, 24us enqueued** — roughly 250x.
19//!   This is the case the type exists for: a request handler emitting a handful
20//!   of events should not wait on a slow destination.
21//! - **Sustained load beyond the drain rate**: no benefit at all. Once the queue
22//!   saturates, enqueueing blocks and the caller waits anyway. The writer
23//!   cannot outpace the destination it wraps; it can only defer the wait.
24//! - **A fast local file**: a net *loss*. Enqueueing costs ~1.9us against
25//!   ~1.1us for a page-cache-backed write, because the event is cloned and
26//!   boxed on the way into the queue. Wrap a fast destination and you pay for
27//!   the queue without getting anything for it.
28//!
29//! Use it when the destination is genuinely slow — a network target, a
30//! congested disk, `fsync` per event — and traffic is bursty. Wrapping a local
31//! file to "make logging async" makes it slower.
32//!
33//! # What is traded away
34//!
35//! An event that is queued is not yet written. A crash loses the queue, so this
36//! is not appropriate for a record that must survive process death — the same
37//! caveat as [`BufferedAuditBackend`](crate::BufferedAuditBackend), and for the
38//! same reason.
39//!
40//! [`AsyncBackend::flush`] blocks until the queue has drained, and [`Drop`]
41//! flushes, so a normal shutdown loses nothing.
42//!
43//! # Backpressure
44//!
45//! The queue is bounded. [`OverflowPolicy`] decides what a full queue means:
46//! block the caller (never lose an event, but reintroduce the stall),
47//! or drop and report (stay responsive, and know exactly how much was lost).
48//! Neither is right for every deployment, which is why it must be chosen.
49//!
50//! [`log_audit`]: crate::AuditBackend::log_audit
51
52use super::{AuditBackend, AuditEvent, ErrorSink, IgnoreErrors, WriteError};
53use std::sync::atomic::{AtomicU64, Ordering};
54use std::sync::mpsc::{Receiver, SyncSender, TrySendError};
55use std::sync::{Arc, Mutex};
56
57/// What to do when the queue is full.
58#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
59pub enum OverflowPolicy {
60    /// Block the caller until space frees up. No event is ever lost, but a
61    /// stalled writer stalls producers — the behaviour of a synchronous
62    /// backend, arrived at only under sustained overload.
63    #[default]
64    Block,
65    /// Drop the event and report it to the error sink. Keeps producers moving
66    /// at the cost of a gap in the record, which the sink makes visible.
67    DropAndReport,
68}
69
70/// One unit of work for the writer thread.
71enum Message {
72    Audit(Box<AuditEvent>),
73    Security(Box<serde_json::Value>),
74    /// Drain marker: the writer replies once everything before it is written.
75    Flush(SyncSender<()>),
76}
77
78/// Writes events on a dedicated thread.
79///
80/// # Examples
81///
82/// ```rust
83/// use ash_log::*;
84/// use std::sync::Arc;
85///
86/// let backend = AsyncBackend::builder(Arc::new(StdoutAuditBackend))
87///     .capacity(4096)
88///     .overflow(OverflowPolicy::DropAndReport)
89///     .errors(Arc::new(StderrErrorSink))
90///     .build();
91///
92/// let logger = Logger::builder(Arc::new(backend)).build();
93/// logger.info("does not block on the write");
94/// logger.flush(); // waits for the queue to drain
95/// ```
96pub struct AsyncBackend {
97    sender: Mutex<Option<SyncSender<Message>>>,
98    worker: Mutex<Option<std::thread::JoinHandle<()>>>,
99    policy: OverflowPolicy,
100    dropped: Arc<AtomicU64>,
101    errors: Arc<dyn ErrorSink>,
102}
103
104impl AsyncBackend {
105    /// Start building an async backend wrapping `inner`.
106    #[must_use]
107    pub fn builder(inner: Arc<dyn AuditBackend>) -> AsyncBackendBuilder {
108        AsyncBackendBuilder {
109            inner,
110            capacity: 1024,
111            policy: OverflowPolicy::Block,
112            errors: Arc::new(IgnoreErrors),
113        }
114    }
115
116    /// Wrap `inner` with default settings: a 1024-event queue that blocks when
117    /// full.
118    #[must_use]
119    pub fn new(inner: Arc<dyn AuditBackend>) -> Self {
120        Self::builder(inner).build()
121    }
122
123    /// Events dropped because the queue was full.
124    ///
125    /// Always 0 under [`OverflowPolicy::Block`].
126    #[must_use]
127    pub fn dropped(&self) -> u64 {
128        self.dropped.load(Ordering::Relaxed)
129    }
130
131    /// Send one message, applying the overflow policy.
132    fn send(&self, message: Message) {
133        let guard = self
134            .sender
135            .lock()
136            .unwrap_or_else(std::sync::PoisonError::into_inner);
137        let Some(sender) = guard.as_ref() else {
138            return; // Already shut down.
139        };
140
141        match self.policy {
142            OverflowPolicy::Block => {
143                // A closed channel means the writer thread is gone; there is
144                // nowhere to put the event and nothing useful to do about it.
145                let _ = sender.send(message);
146            }
147            // The `Ok` and `Disconnected` arms are deliberately separate
148            // despite both doing nothing: one is the success path, the other is
149            // a dead writer with nowhere to put the event. Merging them would
150            // hide that distinction from the next reader.
151            #[allow(clippy::match_same_arms)]
152            OverflowPolicy::DropAndReport => match sender.try_send(message) {
153                Ok(()) => {}
154                Err(TrySendError::Full(_)) => {
155                    self.dropped.fetch_add(1, Ordering::Relaxed);
156                    self.errors.on_error(&WriteError {
157                        backend: "AsyncBackend",
158                        source: std::io::Error::new(
159                            std::io::ErrorKind::WouldBlock,
160                            "audit queue full",
161                        ),
162                        events_lost: 1,
163                    });
164                }
165                Err(TrySendError::Disconnected(_)) => {}
166            },
167        }
168    }
169
170    /// Block until everything queued so far has been written.
171    ///
172    /// The flush marker is always sent with a blocking `send`, never through
173    /// the overflow policy. Under `DropAndReport` a full queue would otherwise
174    /// discard the marker itself, leaving nothing to acknowledge and making
175    /// `flush` return while events were still queued.
176    fn drain(&self) {
177        let (ack, done) = std::sync::mpsc::sync_channel(1);
178        {
179            let guard = self
180                .sender
181                .lock()
182                .unwrap_or_else(std::sync::PoisonError::into_inner);
183            let Some(sender) = guard.as_ref() else {
184                return; // Already shut down; nothing is queued.
185            };
186            if sender.send(Message::Flush(ack)).is_err() {
187                return; // The writer is gone; there is nothing to wait for.
188            }
189        }
190        // A failed receive means the writer exited after accepting the marker.
191        let _ = done.recv();
192    }
193
194    /// Stop the writer thread, draining first.
195    fn shutdown(&self) {
196        self.drain();
197        // Dropping the sender closes the channel, which ends the writer loop.
198        {
199            let mut guard = self
200                .sender
201                .lock()
202                .unwrap_or_else(std::sync::PoisonError::into_inner);
203            *guard = None;
204        }
205        let handle = self
206            .worker
207            .lock()
208            .unwrap_or_else(std::sync::PoisonError::into_inner)
209            .take();
210        if let Some(handle) = handle {
211            drop(handle.join());
212        }
213    }
214}
215
216impl AuditBackend for AsyncBackend {
217    fn log_audit(&self, event: &AuditEvent) {
218        self.send(Message::Audit(Box::new(event.clone())));
219    }
220
221    fn security_log(&self, event: &serde_json::Value) {
222        self.send(Message::Security(Box::new(event.clone())));
223    }
224
225    fn flush(&self) {
226        self.drain();
227    }
228}
229
230impl Drop for AsyncBackend {
231    fn drop(&mut self) {
232        self.shutdown();
233    }
234}
235
236impl std::fmt::Debug for AsyncBackend {
237    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
238        f.debug_struct("AsyncBackend")
239            .field("policy", &self.policy)
240            .field("dropped", &self.dropped())
241            .finish_non_exhaustive()
242    }
243}
244
245/// The writer thread's loop.
246fn run_writer(inner: &Arc<dyn AuditBackend>, receiver: &Receiver<Message>) {
247    for message in receiver {
248        match message {
249            Message::Audit(event) => inner.log_audit(&event),
250            Message::Security(event) => inner.security_log(&event),
251            Message::Flush(ack) => {
252                inner.flush();
253                // A dropped receiver means the waiter gave up; carry on.
254                let _ = ack.send(());
255            }
256        }
257    }
258    inner.flush();
259}
260
261/// Builder for [`AsyncBackend`].
262pub struct AsyncBackendBuilder {
263    inner: Arc<dyn AuditBackend>,
264    capacity: usize,
265    policy: OverflowPolicy,
266    errors: Arc<dyn ErrorSink>,
267}
268
269impl AsyncBackendBuilder {
270    /// Queue capacity in events. Defaults to 1024.
271    ///
272    /// A capacity of 0 is treated as 1: a zero-capacity channel would make
273    /// every send wait for the writer, which defeats the purpose.
274    #[must_use]
275    pub const fn capacity(mut self, capacity: usize) -> Self {
276        self.capacity = capacity;
277        self
278    }
279
280    /// What a full queue means. Defaults to [`OverflowPolicy::Block`].
281    #[must_use]
282    pub const fn overflow(mut self, policy: OverflowPolicy) -> Self {
283        self.policy = policy;
284        self
285    }
286
287    /// Where to report dropped events. Defaults to discarding the reports.
288    #[must_use]
289    pub fn errors(mut self, errors: Arc<dyn ErrorSink>) -> Self {
290        self.errors = errors;
291        self
292    }
293
294    /// Start the writer thread.
295    ///
296    /// # Panics
297    ///
298    /// Panics if the writer thread cannot be spawned. A logger that silently
299    /// failed to start its writer would accept events and discard every one,
300    /// which is worse than failing at construction.
301    #[must_use]
302    pub fn build(self) -> AsyncBackend {
303        let (sender, receiver) = std::sync::mpsc::sync_channel(self.capacity.max(1));
304        let inner = self.inner;
305        let worker = std::thread::Builder::new()
306            .name("ash-log-writer".to_string())
307            .spawn(move || run_writer(&inner, &receiver))
308            .expect("spawn the audit writer thread");
309
310        AsyncBackend {
311            sender: Mutex::new(Some(sender)),
312            worker: Mutex::new(Some(worker)),
313            policy: self.policy,
314            dropped: Arc::new(AtomicU64::new(0)),
315            errors: self.errors,
316        }
317    }
318}
319
320#[cfg(test)]
321mod tests {
322    use super::*;
323    use crate::{AuditEventType, AuditResult, CountingErrorSink};
324    use std::sync::PoisonError;
325    use std::time::Duration;
326
327    #[derive(Default)]
328    struct Collector(Mutex<Vec<AuditEvent>>);
329
330    impl Collector {
331        fn count(&self) -> usize {
332            self.0.lock().unwrap_or_else(PoisonError::into_inner).len()
333        }
334    }
335
336    impl AuditBackend for Collector {
337        fn log_audit(&self, event: &AuditEvent) {
338            self.0
339                .lock()
340                .unwrap_or_else(PoisonError::into_inner)
341                .push(event.clone());
342        }
343    }
344
345    /// A backend that takes its time, standing in for a slow disk.
346    struct SlowBackend {
347        inner: Arc<Collector>,
348        delay: Duration,
349    }
350
351    impl AuditBackend for SlowBackend {
352        fn log_audit(&self, event: &AuditEvent) {
353            std::thread::sleep(self.delay);
354            self.inner.log_audit(event);
355        }
356    }
357
358    fn event() -> AuditEvent {
359        AuditEvent::builder()
360            .event_type(AuditEventType::MethodInvocation)
361            .result(AuditResult::Success)
362            .build()
363    }
364
365    #[test]
366    fn test_events_reach_the_inner_backend() {
367        let collector = Arc::new(Collector::default());
368        let backend = AsyncBackend::new(collector.clone());
369
370        for _ in 0..50 {
371            backend.log_audit(&event());
372        }
373        backend.flush();
374
375        assert_eq!(collector.count(), 50);
376    }
377
378    #[test]
379    fn test_flush_waits_for_the_queue_to_drain() {
380        let collector = Arc::new(Collector::default());
381        let slow = Arc::new(SlowBackend {
382            inner: collector.clone(),
383            delay: Duration::from_millis(2),
384        });
385        let backend = AsyncBackend::new(slow);
386
387        for _ in 0..10 {
388            backend.log_audit(&event());
389        }
390        backend.flush();
391
392        assert_eq!(
393            collector.count(),
394            10,
395            "flush returns only once every queued event is written"
396        );
397    }
398
399    #[test]
400    fn test_dropping_the_backend_drains_the_queue() {
401        let collector = Arc::new(Collector::default());
402        {
403            let backend = AsyncBackend::new(collector.clone());
404            for _ in 0..20 {
405                backend.log_audit(&event());
406            }
407        }
408        assert_eq!(
409            collector.count(),
410            20,
411            "shutdown flushes rather than discards"
412        );
413    }
414
415    #[test]
416    fn test_blocking_policy_loses_nothing_under_overload() {
417        // A queue far smaller than the burst: with Block, every event still
418        // arrives, because producers wait rather than discard.
419        let collector = Arc::new(Collector::default());
420        let slow = Arc::new(SlowBackend {
421            inner: collector.clone(),
422            delay: Duration::from_micros(200),
423        });
424        let backend = AsyncBackend::builder(slow)
425            .capacity(4)
426            .overflow(OverflowPolicy::Block)
427            .build();
428
429        for _ in 0..100 {
430            backend.log_audit(&event());
431        }
432        backend.flush();
433
434        assert_eq!(collector.count(), 100);
435        assert_eq!(backend.dropped(), 0, "Block never drops");
436    }
437
438    #[test]
439    fn test_drop_policy_reports_what_it_discards() {
440        let collector = Arc::new(Collector::default());
441        let slow = Arc::new(SlowBackend {
442            inner: collector.clone(),
443            delay: Duration::from_millis(5),
444        });
445        let sink = Arc::new(CountingErrorSink::new());
446        let backend = AsyncBackend::builder(slow)
447            .capacity(2)
448            .overflow(OverflowPolicy::DropAndReport)
449            .errors(sink.clone())
450            .build();
451
452        for _ in 0..200 {
453            backend.log_audit(&event());
454        }
455        backend.flush();
456
457        assert!(
458            backend.dropped() > 0,
459            "a tiny queue under a burst overflows"
460        );
461        assert_eq!(
462            sink.events_lost(),
463            backend.dropped(),
464            "every dropped event is reported, so the gap is knowable"
465        );
466        assert_eq!(
467            collector.count() as u64 + backend.dropped(),
468            200,
469            "written plus dropped accounts for everything submitted"
470        );
471    }
472
473    #[test]
474    fn test_producers_are_not_blocked_by_a_slow_writer() {
475        // The point of the whole type: enqueueing must be much faster than the
476        // underlying write.
477        let collector = Arc::new(Collector::default());
478        let slow = Arc::new(SlowBackend {
479            inner: collector.clone(),
480            delay: Duration::from_millis(10),
481        });
482        let backend = AsyncBackend::builder(slow).capacity(64).build();
483
484        let start = std::time::Instant::now();
485        for _ in 0..20 {
486            backend.log_audit(&event());
487        }
488        let enqueue = start.elapsed();
489
490        assert!(
491            enqueue < Duration::from_millis(100),
492            "20 enqueues took {enqueue:?}, but 20 synchronous writes would take 200ms"
493        );
494        backend.flush();
495        assert_eq!(collector.count(), 20);
496    }
497
498    #[test]
499    fn test_concurrent_producers_all_arrive() {
500        let collector = Arc::new(Collector::default());
501        let backend = Arc::new(AsyncBackend::new(collector.clone()));
502
503        let mut handles = Vec::new();
504        for _ in 0..8 {
505            let backend = backend.clone();
506            handles.push(std::thread::spawn(move || {
507                for _ in 0..50 {
508                    backend.log_audit(&event());
509                }
510            }));
511        }
512        for handle in handles {
513            handle.join().expect("worker panicked");
514        }
515        backend.flush();
516
517        assert_eq!(collector.count(), 400);
518    }
519}