use std::collections::VecDeque;
use std::fmt;
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Condvar, Mutex, MutexGuard};
use std::time::{Duration, Instant};
use serde::Serialize;
use tokio::runtime::{Handle, RuntimeFlavor};
use tokio::sync::{watch, Notify};
use tokio::task::JoinHandle;
use super::{ExportOptions, LogExporter, LogRecord};
use crate::core::{Error, Result};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum DropPolicy {
#[default]
DropNewest,
DropOldest,
Block,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum DropReason {
QueueFull,
Shutdown,
ExportFailed,
}
impl DropReason {
pub fn as_str(self) -> &'static str {
match self {
DropReason::QueueFull => "queue_full",
DropReason::Shutdown => "shutdown",
DropReason::ExportFailed => "export_failed",
}
}
}
impl fmt::Display for DropReason {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
pub type DropCallback = Arc<dyn Fn(&[LogRecord], DropReason) + Send + Sync>;
pub type ErrorCallback = Arc<dyn Fn(&Error, &[LogRecord]) + Send + Sync>;
#[derive(Clone)]
pub struct BatchOptions {
pub max_queue_size: usize,
pub max_batch_size: usize,
pub max_batch_bytes: usize,
pub flush_interval: Duration,
pub export_timeout: Duration,
pub drop_policy: DropPolicy,
pub gzip: bool,
pub max_export_retries: u32,
pub retry_backoff: Duration,
pub on_drop: Option<DropCallback>,
pub on_error: Option<ErrorCallback>,
}
impl Default for BatchOptions {
fn default() -> Self {
BatchOptions {
max_queue_size: 2048,
max_batch_size: 512,
max_batch_bytes: 1_048_576,
flush_interval: Duration::from_millis(1000),
export_timeout: Duration::from_millis(30_000),
drop_policy: DropPolicy::DropNewest,
gzip: true,
max_export_retries: 2,
retry_backoff: Duration::from_millis(200),
on_drop: None,
on_error: None,
}
}
}
impl fmt::Debug for BatchOptions {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("BatchOptions")
.field("max_queue_size", &self.max_queue_size)
.field("max_batch_size", &self.max_batch_size)
.field("max_batch_bytes", &self.max_batch_bytes)
.field("flush_interval", &self.flush_interval)
.field("export_timeout", &self.export_timeout)
.field("drop_policy", &self.drop_policy)
.field("gzip", &self.gzip)
.field("max_export_retries", &self.max_export_retries)
.field("retry_backoff", &self.retry_backoff)
.field("on_drop", &self.on_drop.is_some())
.field("on_error", &self.on_error.is_some())
.finish()
}
}
impl BatchOptions {
pub fn max_queue_size(mut self, n: usize) -> Self {
self.max_queue_size = n;
self
}
pub fn max_batch_size(mut self, n: usize) -> Self {
self.max_batch_size = n;
self
}
pub fn max_batch_bytes(mut self, n: usize) -> Self {
self.max_batch_bytes = n;
self
}
pub fn flush_interval(mut self, d: Duration) -> Self {
self.flush_interval = d;
self
}
pub fn export_timeout(mut self, d: Duration) -> Self {
self.export_timeout = d;
self
}
pub fn drop_policy(mut self, p: DropPolicy) -> Self {
self.drop_policy = p;
self
}
pub fn gzip(mut self, on: bool) -> Self {
self.gzip = on;
self
}
pub fn max_export_retries(mut self, n: u32) -> Self {
self.max_export_retries = n;
self
}
pub fn retry_backoff(mut self, d: Duration) -> Self {
self.retry_backoff = d;
self
}
pub fn on_drop(mut self, f: impl Fn(&[LogRecord], DropReason) + Send + Sync + 'static) -> Self {
self.on_drop = Some(Arc::new(f));
self
}
pub fn on_error(mut self, f: impl Fn(&Error, &[LogRecord]) + Send + Sync + 'static) -> Self {
self.on_error = Some(Arc::new(f));
self
}
pub fn validate(&self) -> Result<()> {
let checks = [
(self.max_queue_size == 0, "maxQueueSize"),
(self.max_batch_size == 0, "maxBatchSize"),
(self.max_batch_bytes == 0, "maxBatchBytes"),
(self.flush_interval.is_zero(), "flushInterval"),
(self.export_timeout.is_zero(), "exportTimeout"),
];
match checks.iter().find(|(bad, _)| *bad) {
Some((_, name)) => Err(Error::config(format!(
"batch option {name} must be positive"
))),
None => Ok(()),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct BatchStats {
pub queued: u64,
pub sent: u64,
pub dropped: u64,
pub failed: u64,
pub retries: u64,
}
struct Item {
record: LogRecord,
bytes: usize,
}
#[derive(Default)]
struct State {
queue: VecDeque<Item>,
queued_bytes: usize,
in_flight: Option<Arc<Vec<LogRecord>>>,
accepted_total: u64,
flush_target: u64,
closed: bool,
}
struct Shared {
opts: BatchOptions,
exporter: Arc<dyn LogExporter>,
state: Mutex<State>,
space: Condvar,
space_async: Notify,
wake: Notify,
processed: watch::Sender<u64>,
sent: AtomicU64,
dropped: AtomicU64,
failed: AtomicU64,
retries: AtomicU64,
}
pub struct BatchProcessor {
shared: Arc<Shared>,
worker: Mutex<Option<JoinHandle<()>>>,
shutdown_lock: tokio::sync::Mutex<bool>,
}
impl fmt::Debug for BatchProcessor {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("BatchProcessor")
.field("options", &self.shared.opts)
.field("stats", &self.stats())
.finish()
}
}
enum Admit<'a> {
Room(MutexGuard<'a, State>),
Full,
Closed,
}
impl BatchProcessor {
pub fn new(exporter: Arc<dyn LogExporter>, opts: BatchOptions) -> Result<Self> {
opts.validate()?;
let handle = Handle::try_current().map_err(|_| {
Error::config("BatchProcessor::new must be called inside a tokio runtime")
})?;
let shared = Arc::new(Shared {
opts,
exporter,
state: Mutex::new(State::default()),
space: Condvar::new(),
space_async: Notify::new(),
wake: Notify::new(),
processed: watch::Sender::new(0),
sent: AtomicU64::new(0),
dropped: AtomicU64::new(0),
failed: AtomicU64::new(0),
retries: AtomicU64::new(0),
});
let worker = handle.spawn(run_worker(Arc::clone(&shared)));
Ok(BatchProcessor {
shared,
worker: Mutex::new(Some(worker)),
shutdown_lock: tokio::sync::Mutex::new(false),
})
}
pub fn options(&self) -> &BatchOptions {
&self.shared.opts
}
pub fn add(&self, record: LogRecord) -> bool {
let bytes = record.json_size();
let shared = &self.shared;
let admit = match shared.lock() {
st if st.closed => Admit::Closed,
st if st.queue.len() < shared.opts.max_queue_size => Admit::Room(st),
st => match shared.opts.drop_policy {
DropPolicy::DropNewest => Admit::Full,
DropPolicy::DropOldest => Admit::Room(st),
DropPolicy::Block => shared.block_for_space(st),
},
};
shared.admit(admit, record, bytes)
}
pub async fn add_async(&self, record: LogRecord) -> bool {
let shared = &self.shared;
if shared.opts.drop_policy != DropPolicy::Block {
return self.add(record);
}
let bytes = record.json_size();
let deadline = tokio::time::Instant::now() + shared.opts.export_timeout;
loop {
let notified = shared.space_async.notified();
tokio::pin!(notified);
notified.as_mut().enable();
{
let st = shared.lock();
if st.closed {
drop(st);
return shared.admit(Admit::Closed, record, bytes);
}
if st.queue.len() < shared.opts.max_queue_size {
return shared.admit(Admit::Room(st), record, bytes);
}
}
if tokio::time::timeout_at(deadline, notified).await.is_err() {
return shared.admit(Admit::Full, record, bytes);
}
}
}
pub async fn flush(&self, timeout: Option<Duration>) -> Result<()> {
let shared = &self.shared;
let target = {
let mut st = shared.lock();
st.flush_target = st.flush_target.max(st.accepted_total);
st.accepted_total
};
shared.wake.notify_one();
let mut rx = shared.processed.subscribe();
let wait = async move {
let _ = rx.wait_for(|p| *p >= target).await;
};
match timeout {
None => {
wait.await;
Ok(())
}
Some(t) => tokio::time::timeout(t, wait)
.await
.map_err(|_| Error::timeout(format!("log flush did not finish within {t:?}"))),
}
}
pub async fn shutdown(&self, timeout: Option<Duration>) -> Result<()> {
let mut done = self.shutdown_lock.lock().await;
if *done {
return Ok(());
}
*done = true;
let shared = &self.shared;
shared.lock().closed = true;
shared.wake.notify_one();
shared.notify_space();
let worker = self.worker.lock().unwrap_or_else(|e| e.into_inner()).take();
let mut result = Ok(());
if let Some(mut worker) = worker {
let finished = match timeout {
None => {
let _ = (&mut worker).await;
true
}
Some(t) => tokio::time::timeout(t, &mut worker).await.is_ok(),
};
if !finished {
worker.abort();
let _ = worker.await;
shared.drop_remaining();
result = Err(Error::timeout(format!(
"log shutdown did not drain within {:?}",
timeout.unwrap_or_default()
)));
}
}
let exporter = shared.exporter.shutdown().await;
result.and(exporter)
}
pub fn stats(&self) -> BatchStats {
let s = &self.shared;
BatchStats {
queued: {
let st = s.lock();
(st.queue.len() + st.in_flight.as_ref().map_or(0, |b| b.len())) as u64
},
sent: s.sent.load(Ordering::Relaxed),
dropped: s.dropped.load(Ordering::Relaxed),
failed: s.failed.load(Ordering::Relaxed),
retries: s.retries.load(Ordering::Relaxed),
}
}
}
impl Drop for BatchProcessor {
fn drop(&mut self) {
self.shared.lock().closed = true;
self.shared.wake.notify_one();
self.shared.notify_space();
}
}
impl Shared {
fn lock(&self) -> MutexGuard<'_, State> {
self.state.lock().unwrap_or_else(|e| e.into_inner())
}
fn notify_space(&self) {
self.space.notify_all();
self.space_async.notify_waiters();
}
fn block_for_space<'a>(&'a self, st: MutexGuard<'a, State>) -> Admit<'a> {
let wait = |mut st: MutexGuard<'a, State>| {
let deadline = Instant::now() + self.opts.export_timeout;
loop {
if st.closed {
return Admit::Closed;
}
if st.queue.len() < self.opts.max_queue_size {
return Admit::Room(st);
}
let now = Instant::now();
if now >= deadline {
return Admit::Full;
}
st = self
.space
.wait_timeout(st, deadline - now)
.unwrap_or_else(|e| e.into_inner())
.0;
}
};
match Handle::try_current().map(|h| h.runtime_flavor()) {
Ok(RuntimeFlavor::CurrentThread) => Admit::Full,
_ => wait(st),
}
}
fn admit(&self, admit: Admit<'_>, record: LogRecord, bytes: usize) -> bool {
let mut st = match admit {
Admit::Room(st) => st,
Admit::Full => {
self.dropped_records(&[record], DropReason::QueueFull);
return false;
}
Admit::Closed => {
self.dropped_records(&[record], DropReason::Shutdown);
return false;
}
};
let evicted = if st.queue.len() >= self.opts.max_queue_size {
st.queue
.pop_front()
.inspect(|old| st.queued_bytes -= old.bytes)
} else {
None
};
st.queue.push_back(Item { record, bytes });
st.queued_bytes += bytes;
st.accepted_total += 1;
let wake = st.queue.len() >= self.opts.max_batch_size
|| st.queued_bytes >= self.opts.max_batch_bytes;
drop(st);
if let Some(old) = evicted {
self.processed.send_modify(|p| *p += 1);
self.dropped_records(&[old.record], DropReason::QueueFull);
}
if wake {
self.wake.notify_one();
}
true
}
fn dropped_records(&self, records: &[LogRecord], reason: DropReason) {
self.dropped
.fetch_add(records.len() as u64, Ordering::Relaxed);
if let Some(cb) = &self.opts.on_drop {
let _ = catch_unwind(AssertUnwindSafe(|| cb(records, reason)));
}
}
fn report_error(&self, error: &Error, records: &[LogRecord]) {
if let Some(cb) = &self.opts.on_error {
let _ = catch_unwind(AssertUnwindSafe(|| cb(error, records)));
}
}
fn drop_remaining(&self) {
let records: Vec<LogRecord> = {
let mut st = self.lock();
st.queued_bytes = 0;
let in_flight = st.in_flight.take();
let mut records: Vec<LogRecord> =
in_flight.map(|b| b.as_ref().clone()).unwrap_or_default();
records.extend(st.queue.drain(..).map(|i| i.record));
records
};
if records.is_empty() {
return;
}
self.processed.send_modify(|p| *p += records.len() as u64);
self.dropped_records(&records, DropReason::Shutdown);
}
fn next_batch(&self, due: bool) -> Option<Arc<Vec<LogRecord>>> {
let mut st = self.lock();
let flushing = *self.processed.borrow() < st.flush_target;
let full = st.queue.len() >= self.opts.max_batch_size
|| st.queued_bytes >= self.opts.max_batch_bytes;
if st.queue.is_empty() || !(due || flushing || full || st.closed) {
return None;
}
let mut batch = Vec::new();
let mut bytes = 0usize;
while let Some(item) = st.queue.front() {
let over = bytes + item.bytes > self.opts.max_batch_bytes;
if batch.len() >= self.opts.max_batch_size || (!batch.is_empty() && over) {
break;
}
bytes += item.bytes;
let item = st.queue.pop_front().expect("front exists");
batch.push(item.record);
}
st.queued_bytes -= bytes;
let batch = Arc::new(batch);
st.in_flight = Some(Arc::clone(&batch));
drop(st);
self.notify_space();
Some(batch)
}
async fn export_batch(&self, records: Arc<Vec<LogRecord>>) {
let n = records.len() as u64;
let opts = ExportOptions::new(self.opts.gzip);
let mut attempt = 0u32;
loop {
let res = tokio::time::timeout(
self.opts.export_timeout,
self.exporter.export(&records, opts),
)
.await
.unwrap_or_else(|_| {
Err(Error::timeout(format!(
"log export timed out after {:?}",
self.opts.export_timeout
)))
});
match res {
Ok(r) => {
let accepted = (r.accepted as u64).min(n);
self.sent.fetch_add(accepted, Ordering::Relaxed);
let failed = n - accepted;
self.failed.fetch_add(failed, Ordering::Relaxed);
match r.error {
Some(e) => self.report_error(&e, &records),
None if failed > 0 => self.report_error(
&Error::validation(format!(
"{failed} of {n} log records were rejected"
)),
&records,
),
None => {}
}
}
Err(e) if e.is_retryable() && attempt < self.opts.max_export_retries => {
attempt += 1;
self.retries.fetch_add(1, Ordering::Relaxed);
let backoff = self.opts.retry_backoff * (1u32 << (attempt - 1).min(16));
tokio::time::sleep(backoff).await;
continue;
}
Err(e) => {
self.failed.fetch_add(n, Ordering::Relaxed);
self.report_error(&e, &records);
if let Some(cb) = &self.opts.on_drop {
let _ = catch_unwind(AssertUnwindSafe(|| {
cb(&records, DropReason::ExportFailed)
}));
}
}
}
break;
}
self.lock().in_flight = None;
self.processed.send_modify(|p| *p += n);
}
}
async fn run_worker(shared: Arc<Shared>) {
let interval = shared.opts.flush_interval;
let mut deadline = tokio::time::Instant::now() + interval;
loop {
let due = tokio::time::Instant::now() >= deadline;
if let Some(batch) = shared.next_batch(due) {
shared.export_batch(batch).await;
continue;
}
if due {
deadline = tokio::time::Instant::now() + interval;
}
{
let st = shared.lock();
if st.closed && st.queue.is_empty() {
return;
}
}
tokio::select! {
_ = shared.wake.notified() => {}
_ = tokio::time::sleep_until(deadline) => {}
}
}
}