use super::{
config::TransactionBatch,
instruments::{Instruments, Stage},
};
use crate::{
error::{BoxContext, BoxError, Error, chain},
message::SourceMessage,
retry::RetryPolicy,
shutdown::CancellationToken,
sink::{self, PublishRejected},
transaction::{TransactionEntry, TransactionalSink},
};
use std::{marker::PhantomData, sync::Arc, time::Instant};
use tokio::{
sync::{mpsc, oneshot},
task::JoinHandle,
time::sleep_until,
};
use tracing::{Instrument, Span, debug, info_span};
struct Entry<M, P> {
delivery: M,
outputs: Vec<P>,
revocation: Option<CancellationToken>,
enlisted: Instant,
done: oneshot::Sender<Result<(), BoxError>>,
}
impl<M, P> Entry<M, P> {
fn abandoned(&self) -> bool {
self.done.is_closed()
|| self
.revocation
.as_ref()
.is_some_and(CancellationToken::is_cancelled)
}
}
pub(super) struct Batcher<M, P> {
entries: Enlister<M, P>,
task: JoinHandle<()>,
}
pub(super) struct Enlister<M, P>(mpsc::Sender<Entry<M, P>>);
impl<M, P> Clone for Enlister<M, P> {
fn clone(&self) -> Self {
Self(self.0.clone())
}
}
impl<M, P> Batcher<M, P>
where
M: SourceMessage + Sync,
P: Send + Sync + 'static,
{
pub(super) fn spawn<O, K>(
sink: Arc<K>,
batch: TransactionBatch,
policy: RetryPolicy,
instruments: Instruments,
) -> Self
where
O: Send + 'static,
K: TransactionalSink<M, O, Prepared = P>,
{
let (entries, receiver) = mpsc::channel(batch.max_deliveries);
let committer = Committer {
sink,
batch,
policy,
instruments,
marker: PhantomData::<fn(O)>,
};
let task = tokio::spawn(committer.run(receiver).instrument(Span::current()));
Self {
entries: Enlister(entries),
task,
}
}
pub(super) fn enlister(&self) -> Enlister<M, P> {
self.entries.clone()
}
pub(super) async fn close(self) -> Result<(), BoxError> {
drop(self.entries);
Ok(self.task.await?)
}
}
impl<M: SourceMessage, P> Enlister<M, P> {
pub(super) async fn enlist(
&self,
delivery: M,
outputs: Vec<P>,
) -> Result<sink::Completion, BoxError> {
let (done, committed) = oneshot::channel();
let entry = Entry {
revocation: delivery.revocation(),
delivery,
outputs,
enlisted: Instant::now(),
done,
};
self.0
.send(entry)
.await
.map_err(|_| Error::closed("transaction batching has stopped"))?;
Ok(sink::Completion::pending(async move {
committed
.await
.unwrap_or_else(|_| Err(Error::msg("transaction batch was abandoned").into()))
}))
}
}
struct Committer<K, O> {
sink: Arc<K>,
batch: TransactionBatch,
policy: RetryPolicy,
instruments: Instruments,
marker: PhantomData<fn(O)>,
}
impl<K, O> Committer<K, O>
where
O: Send + 'static,
{
async fn run<M>(self, mut receiver: mpsc::Receiver<Entry<M, K::Prepared>>)
where
M: SourceMessage + Sync,
K: TransactionalSink<M, O>,
{
let mut failure: Option<String> = None;
while let Some(first) = receiver.recv().await {
let deadline = (first.enlisted + self.batch.max_linger).into();
let mut members = vec![first];
while members.len() < self.batch.max_deliveries {
tokio::select! {
biased;
entry = receiver.recv() => match entry {
Some(entry) => members.push(entry),
None => break,
},
_ = sleep_until(deadline) => break,
}
}
if failure.is_none()
&& let Err(error) = self.commit(&mut members).await
{
failure = Some(chain(error.as_ref()));
}
for member in members {
let result = match &failure {
Some(error) => Err(Error::msg(error.clone()).into()),
None => Ok(()),
};
let _ = member.done.send(result);
}
}
}
async fn commit<M>(&self, members: &mut Vec<Entry<M, K::Prepared>>) -> Result<(), BoxError>
where
M: SourceMessage + Sync,
K: TransactionalSink<M, O>,
{
let mut attempt = 1;
let mut retrying = None;
loop {
members.retain(|member| !member.abandoned());
if members.is_empty() {
return Ok(());
}
let started = Instant::now();
let result = {
let batch: Vec<_> = members
.iter()
.map(|member| TransactionEntry {
delivery: &member.delivery,
outputs: &member.outputs,
})
.collect();
let outputs: usize = members.iter().map(|member| member.outputs.len()).sum();
self.sink
.commit(&batch)
.instrument(info_span!("commit", deliveries = batch.len(), outputs))
.await
};
let error = match result {
Ok(()) => {
self.instruments.record(Stage::Commit, started);
self.instruments
.transaction_deliveries
.record(members.len() as f64);
return Ok(());
}
Err(error) => error,
};
self.instruments.publish_failures.increment(1);
if PublishRejected::is(error.as_ref()) {
return Err(error.context("a transaction batch cannot route a rejected output"));
}
if members.iter().any(Entry::abandoned) {
debug!(
error = chain(error.as_ref()),
"retrying commit without revoked deliveries"
);
continue;
}
if attempt >= self.policy.max_attempts {
return Err(error.context("publish retry exhausted"));
}
debug!(attempt, error = chain(error.as_ref()), "retrying commit");
if retrying.is_none() {
retrying = Some(self.instruments.health.publish_retry());
}
tokio::time::sleep(self.policy.delay(attempt)).await;
attempt += 1;
}
}
}