use std::{
sync::{Arc, atomic::Ordering},
time::Duration,
};
use http::StatusCode;
use tokio::{
sync::{mpsc, watch},
task::JoinSet,
};
use crate::outbound::{HttpClient, OutboundError, Response};
use crate::hooks::{
CONTENT_TYPE, Dispatcher, Dispatchers, Envelope, HookObserver, Loss, Occurrence, Queue, Shared,
};
const INITIAL_BACKOFF: Duration = Duration::from_millis(250);
const MAXIMUM_BACKOFF: Duration = Duration::from_secs(30);
type Item<E> = Envelope<<E as Occurrence>::Subject, <E as Occurrence>::Kind>;
type Pending<E> = Queue<<E as Occurrence>::Subject, <E as Occurrence>::Kind>;
impl<E: Occurrence> Dispatchers<E> {
pub async fn run(self, mut stop: watch::Receiver<bool>) {
let mut hooks = JoinSet::new();
for dispatcher in self.hooks {
let client = dispatcher
.shared
.config
.client
.clone()
.unwrap_or_else(|| self.client.clone());
hooks.spawn(run_hook(
dispatcher,
client,
self.drain_timeout,
Arc::clone(&self.observer),
stop.clone(),
));
}
if hooks.is_empty() {
let _ = stop.wait_for(|stopped| *stopped).await;
return;
}
while hooks.join_next().await.is_some() {}
}
}
async fn run_hook<E: Occurrence>(
dispatcher: Dispatcher<E>,
client: HttpClient,
drain_timeout: Duration,
observer: Arc<dyn HookObserver<E::Kind>>,
mut stop: watch::Receiver<bool>,
) {
let Dispatcher {
shared,
mut ingress,
} = dispatcher;
let mut queue = Pending::<E>::new(shared.config.queue_capacity);
let mut sending = JoinSet::new();
loop {
drain_ingress(&shared, &mut ingress, &mut queue);
start_available(&shared, &client, &observer, &mut queue, &mut sending);
tokio::select! {
biased;
changed = stop.changed() => {
if changed.is_err() || *stop.borrow() {
break;
}
}
Some(finished) = sending.join_next(), if !sending.is_empty() => {
release(&shared, &mut queue, finished.ok());
}
Some(envelope) = ingress.recv() => {
accept(&shared, &mut queue, envelope);
}
}
}
drain(
shared,
ingress,
queue,
client,
drain_timeout,
observer,
sending,
)
.await;
}
fn drain_ingress<E: Occurrence>(
shared: &Shared<E>,
ingress: &mut mpsc::Receiver<Item<E>>,
queue: &mut Pending<E>,
) {
while let Ok(envelope) = ingress.try_recv() {
accept(shared, queue, envelope);
}
}
fn accept<E: Occurrence>(shared: &Shared<E>, queue: &mut Pending<E>, envelope: Item<E>) {
shared.meters.ingress_depth.fetch_sub(1, Ordering::Relaxed);
if queue.push(envelope) {
shared.meters.record_loss(Loss::Overflow);
}
shared
.meters
.queue_depth
.store(queue.queued(), Ordering::Relaxed);
}
fn start_available<E: Occurrence>(
shared: &Arc<Shared<E>>,
client: &HttpClient,
observer: &Arc<dyn HookObserver<E::Kind>>,
queue: &mut Pending<E>,
sending: &mut JoinSet<E::Subject>,
) {
while sending.len() < shared.config.maximum_in_flight {
let Some(envelope) = queue.take_ready() else {
break;
};
sending.spawn(send(
Arc::clone(shared),
client.clone(),
Arc::clone(observer),
envelope,
));
}
shared
.meters
.queue_depth
.store(queue.queued(), Ordering::Relaxed);
shared
.meters
.in_flight
.store(sending.len(), Ordering::Relaxed);
}
fn release<E: Occurrence>(
shared: &Arc<Shared<E>>,
queue: &mut Pending<E>,
subject: Option<E::Subject>,
) {
if let Some(subject) = subject {
queue.finish(&subject);
}
shared.meters.in_flight.fetch_sub(1, Ordering::Relaxed);
}
async fn send<E: Occurrence>(
shared: Arc<Shared<E>>,
client: HttpClient,
observer: Arc<dyn HookObserver<E::Kind>>,
envelope: Item<E>,
) -> E::Subject {
let mut backoff = INITIAL_BACKOFF;
for attempt in 1..=shared.config.maximum_attempts {
let headers =
shared
.config
.signing_secret
.as_ref()
.map_or_else(http::HeaderMap::new, |secret| {
secret.headers(
&envelope.id,
time::OffsetDateTime::now_utc().unix_timestamp(),
&envelope.body,
)
});
let result = client
.post_with_headers(
&shared.config.endpoint,
CONTENT_TYPE,
shared.config.bearer.as_ref(),
envelope.body.clone(),
&headers,
)
.await;
match verdict(&result) {
Verdict::Delivered => {
shared.meters.delivered.fetch_add(1, Ordering::Relaxed);
return envelope.subject;
}
Verdict::Rejected => {
report(&shared, &observer, &envelope, Loss::Rejected, &result);
return envelope.subject;
}
Verdict::Retry if attempt == shared.config.maximum_attempts => {
report(&shared, &observer, &envelope, Loss::Exhausted, &result);
return envelope.subject;
}
Verdict::Retry => {
shared.meters.retried.fetch_add(1, Ordering::Relaxed);
let wait = result
.as_ref()
.ok()
.and_then(|response| response.retry_after(MAXIMUM_BACKOFF))
.unwrap_or(backoff);
tokio::time::sleep(wait).await;
backoff = (backoff * 2).min(MAXIMUM_BACKOFF);
}
}
}
envelope.subject
}
enum Verdict {
Delivered,
Retry,
Rejected,
}
fn verdict(result: &Result<Response, OutboundError>) -> Verdict {
match result {
Ok(response) if response.status.is_success() => Verdict::Delivered,
Ok(response) => match response.status {
StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_EARLY | StatusCode::TOO_MANY_REQUESTS => {
Verdict::Retry
}
status if status.is_server_error() => Verdict::Retry,
_ => Verdict::Rejected,
},
Err(
OutboundError::Timeout | OutboundError::Unreachable(_) | OutboundError::Response(_),
) => Verdict::Retry,
Err(OutboundError::ResponseTooLarge { .. }) => Verdict::Rejected,
}
}
fn report<E: Occurrence>(
shared: &Arc<Shared<E>>,
observer: &Arc<dyn HookObserver<E::Kind>>,
envelope: &Item<E>,
loss: Loss,
result: &Result<Response, OutboundError>,
) {
shared.meters.record_loss(loss);
observer.event_dropped(
&shared.config.name,
&envelope.id,
envelope.kind,
loss,
&match result {
Ok(response) => format!("HTTP {}", response.status),
Err(error) => error.to_string(),
},
);
}
async fn drain<E: Occurrence>(
shared: Arc<Shared<E>>,
mut ingress: mpsc::Receiver<Item<E>>,
mut queue: Pending<E>,
client: HttpClient,
timeout: Duration,
observer: Arc<dyn HookObserver<E::Kind>>,
mut sending: JoinSet<E::Subject>,
) {
let deadline = tokio::time::Instant::now() + timeout;
let drained = tokio::time::timeout_at(deadline, async {
loop {
drain_ingress(&shared, &mut ingress, &mut queue);
start_available(&shared, &client, &observer, &mut queue, &mut sending);
let Some(finished) = sending.join_next().await else {
break;
};
release(&shared, &mut queue, finished.ok());
}
})
.await;
if drained.is_err() {
sending.abort_all();
let mut unknown = 0;
while let Some(finished) = sending.join_next().await {
if finished.is_err() {
unknown += 1;
}
}
shared
.meters
.outcome_unknown_shutdown
.fetch_add(unknown as u64, Ordering::Relaxed);
if unknown > 0 {
observer.delivery_outcomes_unknown(&shared.config.name, unknown);
}
}
let mut lost = queue.clear();
while ingress.try_recv().is_ok() {
shared.meters.ingress_depth.fetch_sub(1, Ordering::Relaxed);
lost += 1;
}
shared.meters.queue_depth.store(0, Ordering::Relaxed);
shared.meters.in_flight.store(0, Ordering::Relaxed);
for _ in 0..lost {
shared.meters.record_loss(Loss::Shutdown);
}
if lost > 0 {
observer.events_abandoned(&shared.config.name, lost);
}
}