Skip to main content

grow_better_stack_logs/logging/
batch.rs

1// Generated by sdk-kit 0.3.0 — do not edit; change sdk-kit and run "sdk-kit sync"
2//! Background batching of log records (SPEC §4 `BatchProcessor`).
3
4use std::collections::VecDeque;
5use std::fmt;
6use std::panic::{catch_unwind, AssertUnwindSafe};
7use std::sync::atomic::{AtomicU64, Ordering};
8use std::sync::{Arc, Condvar, Mutex, MutexGuard};
9use std::time::{Duration, Instant};
10
11use serde::Serialize;
12use tokio::runtime::{Handle, RuntimeFlavor};
13use tokio::sync::{watch, Notify};
14use tokio::task::JoinHandle;
15
16use super::{ExportOptions, LogExporter, LogRecord};
17use crate::core::{Error, Result};
18
19/// What happens when the queue is full.
20#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
21pub enum DropPolicy {
22    /// Drop the record being added (default).
23    #[default]
24    DropNewest,
25    /// Drop the oldest queued record to make room.
26    DropOldest,
27    /// Wait for room, up to `export_timeout`.
28    ///
29    /// The async [`BatchProcessor::add_async`] always waits. The sync [`BatchProcessor::add`]
30    /// blocks the calling thread (bounded by `export_timeout`); on a current-thread runtime it
31    /// cannot block without stalling the worker, so it drops the record (`queue_full`)
32    /// instead. In async code prefer `add_async` / `BatchingLogClient::send_async`.
33    Block,
34}
35
36/// Why records were dropped (passed to `on_drop`).
37#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
38pub enum DropReason {
39    /// The queue was full.
40    QueueFull,
41    /// The processor was shut down.
42    Shutdown,
43    /// The exporter failed and retries were exhausted.
44    ExportFailed,
45}
46
47impl DropReason {
48    /// Wire value: `queue_full`, `shutdown` or `export_failed`.
49    pub fn as_str(self) -> &'static str {
50        match self {
51            DropReason::QueueFull => "queue_full",
52            DropReason::Shutdown => "shutdown",
53            DropReason::ExportFailed => "export_failed",
54        }
55    }
56}
57
58impl fmt::Display for DropReason {
59    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
60        f.write_str(self.as_str())
61    }
62}
63
64/// Callback for dropped records.
65pub type DropCallback = Arc<dyn Fn(&[LogRecord], DropReason) + Send + Sync>;
66/// Callback for export errors.
67pub type ErrorCallback = Arc<dyn Fn(&Error, &[LogRecord]) + Send + Sync>;
68
69/// Batch processor options (SPEC §4 `BatchOptions`).
70#[derive(Clone)]
71pub struct BatchOptions {
72    /// Maximum queued records (default 2048).
73    pub max_queue_size: usize,
74    /// Maximum records per export (default 512).
75    pub max_batch_size: usize,
76    /// Maximum UTF-8 JSON bytes per export (default 1 MiB). A single larger record is sent alone.
77    pub max_batch_bytes: usize,
78    /// Export at least this often (default 1 s).
79    pub flush_interval: Duration,
80    /// Deadline per export attempt (default 30 s); also the `Block` wait limit.
81    pub export_timeout: Duration,
82    /// Queue-full behaviour (default [`DropPolicy::DropNewest`]).
83    pub drop_policy: DropPolicy,
84    /// Ask the exporter to gzip payloads (default `true`).
85    pub gzip: bool,
86    /// Extra attempts for a batch whose export failed with a retryable error (default 2).
87    pub max_export_retries: u32,
88    /// Wait before retry *n* (1-based) is `retry_backoff * 2^(n-1)` (default 200 ms).
89    pub retry_backoff: Duration,
90    /// Called with records that were dropped.
91    pub on_drop: Option<DropCallback>,
92    /// Called when an export fails or partially fails.
93    pub on_error: Option<ErrorCallback>,
94}
95
96impl Default for BatchOptions {
97    fn default() -> Self {
98        BatchOptions {
99            max_queue_size: 2048,
100            max_batch_size: 512,
101            max_batch_bytes: 1_048_576,
102            flush_interval: Duration::from_millis(1000),
103            export_timeout: Duration::from_millis(30_000),
104            drop_policy: DropPolicy::DropNewest,
105            gzip: true,
106            max_export_retries: 2,
107            retry_backoff: Duration::from_millis(200),
108            on_drop: None,
109            on_error: None,
110        }
111    }
112}
113
114impl fmt::Debug for BatchOptions {
115    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
116        f.debug_struct("BatchOptions")
117            .field("max_queue_size", &self.max_queue_size)
118            .field("max_batch_size", &self.max_batch_size)
119            .field("max_batch_bytes", &self.max_batch_bytes)
120            .field("flush_interval", &self.flush_interval)
121            .field("export_timeout", &self.export_timeout)
122            .field("drop_policy", &self.drop_policy)
123            .field("gzip", &self.gzip)
124            .field("max_export_retries", &self.max_export_retries)
125            .field("retry_backoff", &self.retry_backoff)
126            .field("on_drop", &self.on_drop.is_some())
127            .field("on_error", &self.on_error.is_some())
128            .finish()
129    }
130}
131
132impl BatchOptions {
133    /// Sets `max_queue_size`.
134    pub fn max_queue_size(mut self, n: usize) -> Self {
135        self.max_queue_size = n;
136        self
137    }
138    /// Sets `max_batch_size`.
139    pub fn max_batch_size(mut self, n: usize) -> Self {
140        self.max_batch_size = n;
141        self
142    }
143    /// Sets `max_batch_bytes`.
144    pub fn max_batch_bytes(mut self, n: usize) -> Self {
145        self.max_batch_bytes = n;
146        self
147    }
148    /// Sets `flush_interval`.
149    pub fn flush_interval(mut self, d: Duration) -> Self {
150        self.flush_interval = d;
151        self
152    }
153    /// Sets `export_timeout`.
154    pub fn export_timeout(mut self, d: Duration) -> Self {
155        self.export_timeout = d;
156        self
157    }
158    /// Sets `drop_policy`.
159    pub fn drop_policy(mut self, p: DropPolicy) -> Self {
160        self.drop_policy = p;
161        self
162    }
163    /// Sets `gzip`.
164    pub fn gzip(mut self, on: bool) -> Self {
165        self.gzip = on;
166        self
167    }
168    /// Sets `max_export_retries`.
169    pub fn max_export_retries(mut self, n: u32) -> Self {
170        self.max_export_retries = n;
171        self
172    }
173    /// Sets `retry_backoff`.
174    pub fn retry_backoff(mut self, d: Duration) -> Self {
175        self.retry_backoff = d;
176        self
177    }
178    /// Sets the drop callback.
179    pub fn on_drop(mut self, f: impl Fn(&[LogRecord], DropReason) + Send + Sync + 'static) -> Self {
180        self.on_drop = Some(Arc::new(f));
181        self
182    }
183    /// Sets the error callback.
184    pub fn on_error(mut self, f: impl Fn(&Error, &[LogRecord]) + Send + Sync + 'static) -> Self {
185        self.on_error = Some(Arc::new(f));
186        self
187    }
188
189    /// Checks that every limit and duration is positive.
190    pub fn validate(&self) -> Result<()> {
191        let checks = [
192            (self.max_queue_size == 0, "maxQueueSize"),
193            (self.max_batch_size == 0, "maxBatchSize"),
194            (self.max_batch_bytes == 0, "maxBatchBytes"),
195            (self.flush_interval.is_zero(), "flushInterval"),
196            (self.export_timeout.is_zero(), "exportTimeout"),
197        ];
198        match checks.iter().find(|(bad, _)| *bad) {
199            Some((_, name)) => Err(Error::config(format!(
200                "batch option {name} must be positive"
201            ))),
202            None => Ok(()),
203        }
204    }
205}
206
207/// Processor counters (SPEC §4 `BatchStats`).
208#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize)]
209#[serde(rename_all = "camelCase")]
210pub struct BatchStats {
211    /// Records waiting in the queue or being exported now.
212    pub queued: u64,
213    /// Records the exporter accepted.
214    pub sent: u64,
215    /// Records dropped before export (queue full, shutdown).
216    pub dropped: u64,
217    /// Records whose export failed or was rejected.
218    pub failed: u64,
219    /// Export retry attempts.
220    pub retries: u64,
221}
222
223struct Item {
224    record: LogRecord,
225    bytes: usize,
226}
227
228#[derive(Default)]
229struct State {
230    queue: VecDeque<Item>,
231    queued_bytes: usize,
232    in_flight: Option<Arc<Vec<LogRecord>>>,
233    accepted_total: u64,
234    flush_target: u64,
235    closed: bool,
236}
237
238struct Shared {
239    opts: BatchOptions,
240    exporter: Arc<dyn LogExporter>,
241    state: Mutex<State>,
242    space: Condvar,
243    space_async: Notify,
244    wake: Notify,
245    processed: watch::Sender<u64>,
246    sent: AtomicU64,
247    dropped: AtomicU64,
248    failed: AtomicU64,
249    retries: AtomicU64,
250}
251
252/// Generic background batcher: queues records and exports them from a tokio task.
253pub struct BatchProcessor {
254    shared: Arc<Shared>,
255    worker: Mutex<Option<JoinHandle<()>>>,
256    shutdown_lock: tokio::sync::Mutex<bool>,
257}
258
259impl fmt::Debug for BatchProcessor {
260    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
261        f.debug_struct("BatchProcessor")
262            .field("options", &self.shared.opts)
263            .field("stats", &self.stats())
264            .finish()
265    }
266}
267
268enum Admit<'a> {
269    Room(MutexGuard<'a, State>),
270    Full,
271    Closed,
272}
273
274impl BatchProcessor {
275    /// Starts a processor; must be called inside a tokio runtime.
276    pub fn new(exporter: Arc<dyn LogExporter>, opts: BatchOptions) -> Result<Self> {
277        opts.validate()?;
278        let handle = Handle::try_current().map_err(|_| {
279            Error::config("BatchProcessor::new must be called inside a tokio runtime")
280        })?;
281        let shared = Arc::new(Shared {
282            opts,
283            exporter,
284            state: Mutex::new(State::default()),
285            space: Condvar::new(),
286            space_async: Notify::new(),
287            wake: Notify::new(),
288            processed: watch::Sender::new(0),
289            sent: AtomicU64::new(0),
290            dropped: AtomicU64::new(0),
291            failed: AtomicU64::new(0),
292            retries: AtomicU64::new(0),
293        });
294        let worker = handle.spawn(run_worker(Arc::clone(&shared)));
295        Ok(BatchProcessor {
296            shared,
297            worker: Mutex::new(Some(worker)),
298            shutdown_lock: tokio::sync::Mutex::new(false),
299        })
300    }
301
302    /// Options in use.
303    pub fn options(&self) -> &BatchOptions {
304        &self.shared.opts
305    }
306
307    /// Enqueues a record; returns `false` when it was dropped. Never blocks unless the policy
308    /// is [`DropPolicy::Block`].
309    pub fn add(&self, record: LogRecord) -> bool {
310        let bytes = record.json_size();
311        let shared = &self.shared;
312        let admit = match shared.lock() {
313            st if st.closed => Admit::Closed,
314            st if st.queue.len() < shared.opts.max_queue_size => Admit::Room(st),
315            st => match shared.opts.drop_policy {
316                DropPolicy::DropNewest => Admit::Full,
317                DropPolicy::DropOldest => Admit::Room(st),
318                DropPolicy::Block => shared.block_for_space(st),
319            },
320        };
321        shared.admit(admit, record, bytes)
322    }
323
324    /// Like [`add`](Self::add), but waits asynchronously for room under [`DropPolicy::Block`].
325    pub async fn add_async(&self, record: LogRecord) -> bool {
326        let shared = &self.shared;
327        if shared.opts.drop_policy != DropPolicy::Block {
328            return self.add(record);
329        }
330        let bytes = record.json_size();
331        let deadline = tokio::time::Instant::now() + shared.opts.export_timeout;
332        loop {
333            let notified = shared.space_async.notified();
334            tokio::pin!(notified);
335            notified.as_mut().enable();
336            {
337                let st = shared.lock();
338                if st.closed {
339                    drop(st);
340                    return shared.admit(Admit::Closed, record, bytes);
341                }
342                if st.queue.len() < shared.opts.max_queue_size {
343                    return shared.admit(Admit::Room(st), record, bytes);
344                }
345            }
346            if tokio::time::timeout_at(deadline, notified).await.is_err() {
347                return shared.admit(Admit::Full, record, bytes);
348            }
349        }
350    }
351
352    /// Exports everything queued at the time of the call. `None` waits without a limit (every
353    /// export attempt is still bounded by `export_timeout`); on expiry returns `TIMEOUT`.
354    pub async fn flush(&self, timeout: Option<Duration>) -> Result<()> {
355        let shared = &self.shared;
356        let target = {
357            let mut st = shared.lock();
358            st.flush_target = st.flush_target.max(st.accepted_total);
359            st.accepted_total
360        };
361        shared.wake.notify_one();
362        let mut rx = shared.processed.subscribe();
363        let wait = async move {
364            // The sender lives in `shared`, which we hold, so this cannot fail.
365            let _ = rx.wait_for(|p| *p >= target).await;
366        };
367        match timeout {
368            None => {
369                wait.await;
370                Ok(())
371            }
372            Some(t) => tokio::time::timeout(t, wait)
373                .await
374                .map_err(|_| Error::timeout(format!("log flush did not finish within {t:?}"))),
375        }
376    }
377
378    /// Drains the queue, stops the worker and shuts the exporter down. Idempotent; later
379    /// [`add`](Self::add) calls drop records with reason `shutdown`. When `timeout` expires the
380    /// remaining records are dropped and `TIMEOUT` is returned.
381    pub async fn shutdown(&self, timeout: Option<Duration>) -> Result<()> {
382        let mut done = self.shutdown_lock.lock().await;
383        if *done {
384            return Ok(());
385        }
386        *done = true;
387        let shared = &self.shared;
388        shared.lock().closed = true;
389        shared.wake.notify_one();
390        shared.notify_space();
391        let worker = self.worker.lock().unwrap_or_else(|e| e.into_inner()).take();
392        let mut result = Ok(());
393        if let Some(mut worker) = worker {
394            let finished = match timeout {
395                None => {
396                    let _ = (&mut worker).await;
397                    true
398                }
399                Some(t) => tokio::time::timeout(t, &mut worker).await.is_ok(),
400            };
401            if !finished {
402                worker.abort();
403                let _ = worker.await;
404                shared.drop_remaining();
405                result = Err(Error::timeout(format!(
406                    "log shutdown did not drain within {:?}",
407                    timeout.unwrap_or_default()
408                )));
409            }
410        }
411        let exporter = shared.exporter.shutdown().await;
412        result.and(exporter)
413    }
414
415    /// Current counters.
416    pub fn stats(&self) -> BatchStats {
417        let s = &self.shared;
418        BatchStats {
419            queued: {
420                let st = s.lock();
421                (st.queue.len() + st.in_flight.as_ref().map_or(0, |b| b.len())) as u64
422            },
423            sent: s.sent.load(Ordering::Relaxed),
424            dropped: s.dropped.load(Ordering::Relaxed),
425            failed: s.failed.load(Ordering::Relaxed),
426            retries: s.retries.load(Ordering::Relaxed),
427        }
428    }
429}
430
431impl Drop for BatchProcessor {
432    /// Best effort: marks the processor closed so the worker exports what is queued and exits.
433    fn drop(&mut self) {
434        self.shared.lock().closed = true;
435        self.shared.wake.notify_one();
436        self.shared.notify_space();
437    }
438}
439
440impl Shared {
441    /// Locks the state, recovering from poisoning (callbacks never run under the lock).
442    fn lock(&self) -> MutexGuard<'_, State> {
443        self.state.lock().unwrap_or_else(|e| e.into_inner())
444    }
445
446    fn notify_space(&self) {
447        self.space.notify_all();
448        self.space_async.notify_waiters();
449    }
450
451    fn block_for_space<'a>(&'a self, st: MutexGuard<'a, State>) -> Admit<'a> {
452        let wait = |mut st: MutexGuard<'a, State>| {
453            let deadline = Instant::now() + self.opts.export_timeout;
454            loop {
455                if st.closed {
456                    return Admit::Closed;
457                }
458                if st.queue.len() < self.opts.max_queue_size {
459                    return Admit::Room(st);
460                }
461                let now = Instant::now();
462                if now >= deadline {
463                    return Admit::Full;
464                }
465                st = self
466                    .space
467                    .wait_timeout(st, deadline - now)
468                    .unwrap_or_else(|e| e.into_inner())
469                    .0;
470            }
471        };
472        match Handle::try_current().map(|h| h.runtime_flavor()) {
473            // Blocking here would stall the only thread, including the worker.
474            Ok(RuntimeFlavor::CurrentThread) => Admit::Full,
475            // Other runtime threads keep running the worker; the wait is bounded.
476            _ => wait(st),
477        }
478    }
479
480    fn admit(&self, admit: Admit<'_>, record: LogRecord, bytes: usize) -> bool {
481        let mut st = match admit {
482            Admit::Room(st) => st,
483            Admit::Full => {
484                self.dropped_records(&[record], DropReason::QueueFull);
485                return false;
486            }
487            Admit::Closed => {
488                self.dropped_records(&[record], DropReason::Shutdown);
489                return false;
490            }
491        };
492        let evicted = if st.queue.len() >= self.opts.max_queue_size {
493            st.queue
494                .pop_front()
495                .inspect(|old| st.queued_bytes -= old.bytes)
496        } else {
497            None
498        };
499        st.queue.push_back(Item { record, bytes });
500        st.queued_bytes += bytes;
501        st.accepted_total += 1;
502        let wake = st.queue.len() >= self.opts.max_batch_size
503            || st.queued_bytes >= self.opts.max_batch_bytes;
504        drop(st);
505        if let Some(old) = evicted {
506            self.processed.send_modify(|p| *p += 1);
507            self.dropped_records(&[old.record], DropReason::QueueFull);
508        }
509        if wake {
510            self.wake.notify_one();
511        }
512        true
513    }
514
515    fn dropped_records(&self, records: &[LogRecord], reason: DropReason) {
516        self.dropped
517            .fetch_add(records.len() as u64, Ordering::Relaxed);
518        if let Some(cb) = &self.opts.on_drop {
519            let _ = catch_unwind(AssertUnwindSafe(|| cb(records, reason)));
520        }
521    }
522
523    fn report_error(&self, error: &Error, records: &[LogRecord]) {
524        if let Some(cb) = &self.opts.on_error {
525            let _ = catch_unwind(AssertUnwindSafe(|| cb(error, records)));
526        }
527    }
528
529    /// Drops the in-flight batch of an aborted worker and everything queued.
530    fn drop_remaining(&self) {
531        let records: Vec<LogRecord> = {
532            let mut st = self.lock();
533            st.queued_bytes = 0;
534            let in_flight = st.in_flight.take();
535            let mut records: Vec<LogRecord> =
536                in_flight.map(|b| b.as_ref().clone()).unwrap_or_default();
537            records.extend(st.queue.drain(..).map(|i| i.record));
538            records
539        };
540        if records.is_empty() {
541            return;
542        }
543        self.processed.send_modify(|p| *p += records.len() as u64);
544        self.dropped_records(&records, DropReason::Shutdown);
545    }
546
547    /// Takes the next batch when one is due; `None` when the worker should wait.
548    fn next_batch(&self, due: bool) -> Option<Arc<Vec<LogRecord>>> {
549        let mut st = self.lock();
550        let flushing = *self.processed.borrow() < st.flush_target;
551        let full = st.queue.len() >= self.opts.max_batch_size
552            || st.queued_bytes >= self.opts.max_batch_bytes;
553        if st.queue.is_empty() || !(due || flushing || full || st.closed) {
554            return None;
555        }
556        let mut batch = Vec::new();
557        let mut bytes = 0usize;
558        while let Some(item) = st.queue.front() {
559            let over = bytes + item.bytes > self.opts.max_batch_bytes;
560            if batch.len() >= self.opts.max_batch_size || (!batch.is_empty() && over) {
561                break;
562            }
563            bytes += item.bytes;
564            let item = st.queue.pop_front().expect("front exists");
565            batch.push(item.record);
566        }
567        st.queued_bytes -= bytes;
568        let batch = Arc::new(batch);
569        st.in_flight = Some(Arc::clone(&batch));
570        drop(st);
571        self.notify_space();
572        Some(batch)
573    }
574
575    async fn export_batch(&self, records: Arc<Vec<LogRecord>>) {
576        let n = records.len() as u64;
577        let opts = ExportOptions::new(self.opts.gzip);
578        let mut attempt = 0u32;
579        loop {
580            let res = tokio::time::timeout(
581                self.opts.export_timeout,
582                self.exporter.export(&records, opts),
583            )
584            .await
585            .unwrap_or_else(|_| {
586                Err(Error::timeout(format!(
587                    "log export timed out after {:?}",
588                    self.opts.export_timeout
589                )))
590            });
591            match res {
592                Ok(r) => {
593                    let accepted = (r.accepted as u64).min(n);
594                    self.sent.fetch_add(accepted, Ordering::Relaxed);
595                    let failed = n - accepted;
596                    self.failed.fetch_add(failed, Ordering::Relaxed);
597                    match r.error {
598                        Some(e) => self.report_error(&e, &records),
599                        None if failed > 0 => self.report_error(
600                            &Error::validation(format!(
601                                "{failed} of {n} log records were rejected"
602                            )),
603                            &records,
604                        ),
605                        None => {}
606                    }
607                }
608                Err(e) if e.is_retryable() && attempt < self.opts.max_export_retries => {
609                    attempt += 1;
610                    self.retries.fetch_add(1, Ordering::Relaxed);
611                    let backoff = self.opts.retry_backoff * (1u32 << (attempt - 1).min(16));
612                    tokio::time::sleep(backoff).await;
613                    continue;
614                }
615                Err(e) => {
616                    self.failed.fetch_add(n, Ordering::Relaxed);
617                    self.report_error(&e, &records);
618                    if let Some(cb) = &self.opts.on_drop {
619                        let _ = catch_unwind(AssertUnwindSafe(|| {
620                            cb(&records, DropReason::ExportFailed)
621                        }));
622                    }
623                }
624            }
625            break;
626        }
627        self.lock().in_flight = None;
628        self.processed.send_modify(|p| *p += n);
629    }
630}
631
632async fn run_worker(shared: Arc<Shared>) {
633    let interval = shared.opts.flush_interval;
634    let mut deadline = tokio::time::Instant::now() + interval;
635    loop {
636        let due = tokio::time::Instant::now() >= deadline;
637        if let Some(batch) = shared.next_batch(due) {
638            shared.export_batch(batch).await;
639            continue;
640        }
641        if due {
642            deadline = tokio::time::Instant::now() + interval;
643        }
644        {
645            let st = shared.lock();
646            if st.closed && st.queue.is_empty() {
647                return;
648            }
649        }
650        tokio::select! {
651            _ = shared.wake.notified() => {}
652            _ = tokio::time::sleep_until(deadline) => {}
653        }
654    }
655}