1use std::collections::VecDeque;
5use std::fmt;
6use std::panic::{catch_unwind, AssertUnwindSafe};
7use std::sync::atomic::{AtomicU64, Ordering};
8use std::sync::{Arc, Condvar, Mutex, MutexGuard};
9use std::time::{Duration, Instant};
10
11use serde::Serialize;
12use tokio::runtime::{Handle, RuntimeFlavor};
13use tokio::sync::{watch, Notify};
14use tokio::task::JoinHandle;
15
16use super::{ExportOptions, LogExporter, LogRecord};
17use crate::core::{Error, Result};
18
19#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
21pub enum DropPolicy {
22 #[default]
24 DropNewest,
25 DropOldest,
27 Block,
34}
35
36#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
38pub enum DropReason {
39 QueueFull,
41 Shutdown,
43 ExportFailed,
45}
46
47impl DropReason {
48 pub fn as_str(self) -> &'static str {
50 match self {
51 DropReason::QueueFull => "queue_full",
52 DropReason::Shutdown => "shutdown",
53 DropReason::ExportFailed => "export_failed",
54 }
55 }
56}
57
58impl fmt::Display for DropReason {
59 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
60 f.write_str(self.as_str())
61 }
62}
63
64pub type DropCallback = Arc<dyn Fn(&[LogRecord], DropReason) + Send + Sync>;
66pub type ErrorCallback = Arc<dyn Fn(&Error, &[LogRecord]) + Send + Sync>;
68
69#[derive(Clone)]
71pub struct BatchOptions {
72 pub max_queue_size: usize,
74 pub max_batch_size: usize,
76 pub max_batch_bytes: usize,
78 pub flush_interval: Duration,
80 pub export_timeout: Duration,
82 pub drop_policy: DropPolicy,
84 pub gzip: bool,
86 pub max_export_retries: u32,
88 pub retry_backoff: Duration,
90 pub on_drop: Option<DropCallback>,
92 pub on_error: Option<ErrorCallback>,
94}
95
96impl Default for BatchOptions {
97 fn default() -> Self {
98 BatchOptions {
99 max_queue_size: 2048,
100 max_batch_size: 512,
101 max_batch_bytes: 1_048_576,
102 flush_interval: Duration::from_millis(1000),
103 export_timeout: Duration::from_millis(30_000),
104 drop_policy: DropPolicy::DropNewest,
105 gzip: true,
106 max_export_retries: 2,
107 retry_backoff: Duration::from_millis(200),
108 on_drop: None,
109 on_error: None,
110 }
111 }
112}
113
114impl fmt::Debug for BatchOptions {
115 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
116 f.debug_struct("BatchOptions")
117 .field("max_queue_size", &self.max_queue_size)
118 .field("max_batch_size", &self.max_batch_size)
119 .field("max_batch_bytes", &self.max_batch_bytes)
120 .field("flush_interval", &self.flush_interval)
121 .field("export_timeout", &self.export_timeout)
122 .field("drop_policy", &self.drop_policy)
123 .field("gzip", &self.gzip)
124 .field("max_export_retries", &self.max_export_retries)
125 .field("retry_backoff", &self.retry_backoff)
126 .field("on_drop", &self.on_drop.is_some())
127 .field("on_error", &self.on_error.is_some())
128 .finish()
129 }
130}
131
132impl BatchOptions {
133 pub fn max_queue_size(mut self, n: usize) -> Self {
135 self.max_queue_size = n;
136 self
137 }
138 pub fn max_batch_size(mut self, n: usize) -> Self {
140 self.max_batch_size = n;
141 self
142 }
143 pub fn max_batch_bytes(mut self, n: usize) -> Self {
145 self.max_batch_bytes = n;
146 self
147 }
148 pub fn flush_interval(mut self, d: Duration) -> Self {
150 self.flush_interval = d;
151 self
152 }
153 pub fn export_timeout(mut self, d: Duration) -> Self {
155 self.export_timeout = d;
156 self
157 }
158 pub fn drop_policy(mut self, p: DropPolicy) -> Self {
160 self.drop_policy = p;
161 self
162 }
163 pub fn gzip(mut self, on: bool) -> Self {
165 self.gzip = on;
166 self
167 }
168 pub fn max_export_retries(mut self, n: u32) -> Self {
170 self.max_export_retries = n;
171 self
172 }
173 pub fn retry_backoff(mut self, d: Duration) -> Self {
175 self.retry_backoff = d;
176 self
177 }
178 pub fn on_drop(mut self, f: impl Fn(&[LogRecord], DropReason) + Send + Sync + 'static) -> Self {
180 self.on_drop = Some(Arc::new(f));
181 self
182 }
183 pub fn on_error(mut self, f: impl Fn(&Error, &[LogRecord]) + Send + Sync + 'static) -> Self {
185 self.on_error = Some(Arc::new(f));
186 self
187 }
188
189 pub fn validate(&self) -> Result<()> {
191 let checks = [
192 (self.max_queue_size == 0, "maxQueueSize"),
193 (self.max_batch_size == 0, "maxBatchSize"),
194 (self.max_batch_bytes == 0, "maxBatchBytes"),
195 (self.flush_interval.is_zero(), "flushInterval"),
196 (self.export_timeout.is_zero(), "exportTimeout"),
197 ];
198 match checks.iter().find(|(bad, _)| *bad) {
199 Some((_, name)) => Err(Error::config(format!(
200 "batch option {name} must be positive"
201 ))),
202 None => Ok(()),
203 }
204 }
205}
206
207#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize)]
209#[serde(rename_all = "camelCase")]
210pub struct BatchStats {
211 pub queued: u64,
213 pub sent: u64,
215 pub dropped: u64,
217 pub failed: u64,
219 pub retries: u64,
221}
222
223struct Item {
224 record: LogRecord,
225 bytes: usize,
226}
227
228#[derive(Default)]
229struct State {
230 queue: VecDeque<Item>,
231 queued_bytes: usize,
232 in_flight: Option<Arc<Vec<LogRecord>>>,
233 accepted_total: u64,
234 flush_target: u64,
235 closed: bool,
236}
237
238struct Shared {
239 opts: BatchOptions,
240 exporter: Arc<dyn LogExporter>,
241 state: Mutex<State>,
242 space: Condvar,
243 space_async: Notify,
244 wake: Notify,
245 processed: watch::Sender<u64>,
246 sent: AtomicU64,
247 dropped: AtomicU64,
248 failed: AtomicU64,
249 retries: AtomicU64,
250}
251
252pub struct BatchProcessor {
254 shared: Arc<Shared>,
255 worker: Mutex<Option<JoinHandle<()>>>,
256 shutdown_lock: tokio::sync::Mutex<bool>,
257}
258
259impl fmt::Debug for BatchProcessor {
260 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
261 f.debug_struct("BatchProcessor")
262 .field("options", &self.shared.opts)
263 .field("stats", &self.stats())
264 .finish()
265 }
266}
267
268enum Admit<'a> {
269 Room(MutexGuard<'a, State>),
270 Full,
271 Closed,
272}
273
274impl BatchProcessor {
275 pub fn new(exporter: Arc<dyn LogExporter>, opts: BatchOptions) -> Result<Self> {
277 opts.validate()?;
278 let handle = Handle::try_current().map_err(|_| {
279 Error::config("BatchProcessor::new must be called inside a tokio runtime")
280 })?;
281 let shared = Arc::new(Shared {
282 opts,
283 exporter,
284 state: Mutex::new(State::default()),
285 space: Condvar::new(),
286 space_async: Notify::new(),
287 wake: Notify::new(),
288 processed: watch::Sender::new(0),
289 sent: AtomicU64::new(0),
290 dropped: AtomicU64::new(0),
291 failed: AtomicU64::new(0),
292 retries: AtomicU64::new(0),
293 });
294 let worker = handle.spawn(run_worker(Arc::clone(&shared)));
295 Ok(BatchProcessor {
296 shared,
297 worker: Mutex::new(Some(worker)),
298 shutdown_lock: tokio::sync::Mutex::new(false),
299 })
300 }
301
302 pub fn options(&self) -> &BatchOptions {
304 &self.shared.opts
305 }
306
307 pub fn add(&self, record: LogRecord) -> bool {
310 let bytes = record.json_size();
311 let shared = &self.shared;
312 let admit = match shared.lock() {
313 st if st.closed => Admit::Closed,
314 st if st.queue.len() < shared.opts.max_queue_size => Admit::Room(st),
315 st => match shared.opts.drop_policy {
316 DropPolicy::DropNewest => Admit::Full,
317 DropPolicy::DropOldest => Admit::Room(st),
318 DropPolicy::Block => shared.block_for_space(st),
319 },
320 };
321 shared.admit(admit, record, bytes)
322 }
323
324 pub async fn add_async(&self, record: LogRecord) -> bool {
326 let shared = &self.shared;
327 if shared.opts.drop_policy != DropPolicy::Block {
328 return self.add(record);
329 }
330 let bytes = record.json_size();
331 let deadline = tokio::time::Instant::now() + shared.opts.export_timeout;
332 loop {
333 let notified = shared.space_async.notified();
334 tokio::pin!(notified);
335 notified.as_mut().enable();
336 {
337 let st = shared.lock();
338 if st.closed {
339 drop(st);
340 return shared.admit(Admit::Closed, record, bytes);
341 }
342 if st.queue.len() < shared.opts.max_queue_size {
343 return shared.admit(Admit::Room(st), record, bytes);
344 }
345 }
346 if tokio::time::timeout_at(deadline, notified).await.is_err() {
347 return shared.admit(Admit::Full, record, bytes);
348 }
349 }
350 }
351
352 pub async fn flush(&self, timeout: Option<Duration>) -> Result<()> {
355 let shared = &self.shared;
356 let target = {
357 let mut st = shared.lock();
358 st.flush_target = st.flush_target.max(st.accepted_total);
359 st.accepted_total
360 };
361 shared.wake.notify_one();
362 let mut rx = shared.processed.subscribe();
363 let wait = async move {
364 let _ = rx.wait_for(|p| *p >= target).await;
366 };
367 match timeout {
368 None => {
369 wait.await;
370 Ok(())
371 }
372 Some(t) => tokio::time::timeout(t, wait)
373 .await
374 .map_err(|_| Error::timeout(format!("log flush did not finish within {t:?}"))),
375 }
376 }
377
378 pub async fn shutdown(&self, timeout: Option<Duration>) -> Result<()> {
382 let mut done = self.shutdown_lock.lock().await;
383 if *done {
384 return Ok(());
385 }
386 *done = true;
387 let shared = &self.shared;
388 shared.lock().closed = true;
389 shared.wake.notify_one();
390 shared.notify_space();
391 let worker = self.worker.lock().unwrap_or_else(|e| e.into_inner()).take();
392 let mut result = Ok(());
393 if let Some(mut worker) = worker {
394 let finished = match timeout {
395 None => {
396 let _ = (&mut worker).await;
397 true
398 }
399 Some(t) => tokio::time::timeout(t, &mut worker).await.is_ok(),
400 };
401 if !finished {
402 worker.abort();
403 let _ = worker.await;
404 shared.drop_remaining();
405 result = Err(Error::timeout(format!(
406 "log shutdown did not drain within {:?}",
407 timeout.unwrap_or_default()
408 )));
409 }
410 }
411 let exporter = shared.exporter.shutdown().await;
412 result.and(exporter)
413 }
414
415 pub fn stats(&self) -> BatchStats {
417 let s = &self.shared;
418 BatchStats {
419 queued: {
420 let st = s.lock();
421 (st.queue.len() + st.in_flight.as_ref().map_or(0, |b| b.len())) as u64
422 },
423 sent: s.sent.load(Ordering::Relaxed),
424 dropped: s.dropped.load(Ordering::Relaxed),
425 failed: s.failed.load(Ordering::Relaxed),
426 retries: s.retries.load(Ordering::Relaxed),
427 }
428 }
429}
430
431impl Drop for BatchProcessor {
432 fn drop(&mut self) {
434 self.shared.lock().closed = true;
435 self.shared.wake.notify_one();
436 self.shared.notify_space();
437 }
438}
439
440impl Shared {
441 fn lock(&self) -> MutexGuard<'_, State> {
443 self.state.lock().unwrap_or_else(|e| e.into_inner())
444 }
445
446 fn notify_space(&self) {
447 self.space.notify_all();
448 self.space_async.notify_waiters();
449 }
450
451 fn block_for_space<'a>(&'a self, st: MutexGuard<'a, State>) -> Admit<'a> {
452 let wait = |mut st: MutexGuard<'a, State>| {
453 let deadline = Instant::now() + self.opts.export_timeout;
454 loop {
455 if st.closed {
456 return Admit::Closed;
457 }
458 if st.queue.len() < self.opts.max_queue_size {
459 return Admit::Room(st);
460 }
461 let now = Instant::now();
462 if now >= deadline {
463 return Admit::Full;
464 }
465 st = self
466 .space
467 .wait_timeout(st, deadline - now)
468 .unwrap_or_else(|e| e.into_inner())
469 .0;
470 }
471 };
472 match Handle::try_current().map(|h| h.runtime_flavor()) {
473 Ok(RuntimeFlavor::CurrentThread) => Admit::Full,
475 _ => wait(st),
477 }
478 }
479
480 fn admit(&self, admit: Admit<'_>, record: LogRecord, bytes: usize) -> bool {
481 let mut st = match admit {
482 Admit::Room(st) => st,
483 Admit::Full => {
484 self.dropped_records(&[record], DropReason::QueueFull);
485 return false;
486 }
487 Admit::Closed => {
488 self.dropped_records(&[record], DropReason::Shutdown);
489 return false;
490 }
491 };
492 let evicted = if st.queue.len() >= self.opts.max_queue_size {
493 st.queue
494 .pop_front()
495 .inspect(|old| st.queued_bytes -= old.bytes)
496 } else {
497 None
498 };
499 st.queue.push_back(Item { record, bytes });
500 st.queued_bytes += bytes;
501 st.accepted_total += 1;
502 let wake = st.queue.len() >= self.opts.max_batch_size
503 || st.queued_bytes >= self.opts.max_batch_bytes;
504 drop(st);
505 if let Some(old) = evicted {
506 self.processed.send_modify(|p| *p += 1);
507 self.dropped_records(&[old.record], DropReason::QueueFull);
508 }
509 if wake {
510 self.wake.notify_one();
511 }
512 true
513 }
514
515 fn dropped_records(&self, records: &[LogRecord], reason: DropReason) {
516 self.dropped
517 .fetch_add(records.len() as u64, Ordering::Relaxed);
518 if let Some(cb) = &self.opts.on_drop {
519 let _ = catch_unwind(AssertUnwindSafe(|| cb(records, reason)));
520 }
521 }
522
523 fn report_error(&self, error: &Error, records: &[LogRecord]) {
524 if let Some(cb) = &self.opts.on_error {
525 let _ = catch_unwind(AssertUnwindSafe(|| cb(error, records)));
526 }
527 }
528
529 fn drop_remaining(&self) {
531 let records: Vec<LogRecord> = {
532 let mut st = self.lock();
533 st.queued_bytes = 0;
534 let in_flight = st.in_flight.take();
535 let mut records: Vec<LogRecord> =
536 in_flight.map(|b| b.as_ref().clone()).unwrap_or_default();
537 records.extend(st.queue.drain(..).map(|i| i.record));
538 records
539 };
540 if records.is_empty() {
541 return;
542 }
543 self.processed.send_modify(|p| *p += records.len() as u64);
544 self.dropped_records(&records, DropReason::Shutdown);
545 }
546
547 fn next_batch(&self, due: bool) -> Option<Arc<Vec<LogRecord>>> {
549 let mut st = self.lock();
550 let flushing = *self.processed.borrow() < st.flush_target;
551 let full = st.queue.len() >= self.opts.max_batch_size
552 || st.queued_bytes >= self.opts.max_batch_bytes;
553 if st.queue.is_empty() || !(due || flushing || full || st.closed) {
554 return None;
555 }
556 let mut batch = Vec::new();
557 let mut bytes = 0usize;
558 while let Some(item) = st.queue.front() {
559 let over = bytes + item.bytes > self.opts.max_batch_bytes;
560 if batch.len() >= self.opts.max_batch_size || (!batch.is_empty() && over) {
561 break;
562 }
563 bytes += item.bytes;
564 let item = st.queue.pop_front().expect("front exists");
565 batch.push(item.record);
566 }
567 st.queued_bytes -= bytes;
568 let batch = Arc::new(batch);
569 st.in_flight = Some(Arc::clone(&batch));
570 drop(st);
571 self.notify_space();
572 Some(batch)
573 }
574
575 async fn export_batch(&self, records: Arc<Vec<LogRecord>>) {
576 let n = records.len() as u64;
577 let opts = ExportOptions::new(self.opts.gzip);
578 let mut attempt = 0u32;
579 loop {
580 let res = tokio::time::timeout(
581 self.opts.export_timeout,
582 self.exporter.export(&records, opts),
583 )
584 .await
585 .unwrap_or_else(|_| {
586 Err(Error::timeout(format!(
587 "log export timed out after {:?}",
588 self.opts.export_timeout
589 )))
590 });
591 match res {
592 Ok(r) => {
593 let accepted = (r.accepted as u64).min(n);
594 self.sent.fetch_add(accepted, Ordering::Relaxed);
595 let failed = n - accepted;
596 self.failed.fetch_add(failed, Ordering::Relaxed);
597 match r.error {
598 Some(e) => self.report_error(&e, &records),
599 None if failed > 0 => self.report_error(
600 &Error::validation(format!(
601 "{failed} of {n} log records were rejected"
602 )),
603 &records,
604 ),
605 None => {}
606 }
607 }
608 Err(e) if e.is_retryable() && attempt < self.opts.max_export_retries => {
609 attempt += 1;
610 self.retries.fetch_add(1, Ordering::Relaxed);
611 let backoff = self.opts.retry_backoff * (1u32 << (attempt - 1).min(16));
612 tokio::time::sleep(backoff).await;
613 continue;
614 }
615 Err(e) => {
616 self.failed.fetch_add(n, Ordering::Relaxed);
617 self.report_error(&e, &records);
618 if let Some(cb) = &self.opts.on_drop {
619 let _ = catch_unwind(AssertUnwindSafe(|| {
620 cb(&records, DropReason::ExportFailed)
621 }));
622 }
623 }
624 }
625 break;
626 }
627 self.lock().in_flight = None;
628 self.processed.send_modify(|p| *p += n);
629 }
630}
631
632async fn run_worker(shared: Arc<Shared>) {
633 let interval = shared.opts.flush_interval;
634 let mut deadline = tokio::time::Instant::now() + interval;
635 loop {
636 let due = tokio::time::Instant::now() >= deadline;
637 if let Some(batch) = shared.next_batch(due) {
638 shared.export_batch(batch).await;
639 continue;
640 }
641 if due {
642 deadline = tokio::time::Instant::now() + interval;
643 }
644 {
645 let st = shared.lock();
646 if st.closed && st.queue.is_empty() {
647 return;
648 }
649 }
650 tokio::select! {
651 _ = shared.wake.notified() => {}
652 _ = tokio::time::sleep_until(deadline) => {}
653 }
654 }
655}