use crate::error::ObixFault;
use async_trait::async_trait;
use es_entity::ResultExt;
use futures::{FutureExt, StreamExt};
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use std::sync::Arc;
use job::{
CurrentJob, Job, JobType, ResidentJobCompletion, ResidentJobInitializer, ResidentJobRunner,
RetrySettings,
};
use crate::out::ctx::*;
use crate::out::lane::{InsertOrder, Lane};
use crate::out::subscription::StreamPosition;
use crate::out::{Outbox, event::*};
use crate::sequence::{CommitSequence, EventSequence};
use crate::tables::MailboxTables;
pub trait SingletonSubscriber<P, L = InsertOrder>: Send + Sync + 'static
where
P: Serialize + DeserializeOwned + Send + Sync + 'static + Unpin,
L: Lane,
{
type Batch: Default + Send + 'static;
fn handle_persistent<'inv>(
&self,
ctx: EventCtx<'inv, Self::Batch>,
event: &EventDelivery<P, L>,
) -> impl std::future::Future<
Output = Result<Handled<'inv>, Box<dyn std::error::Error + Send + Sync>>,
> + Send {
let _ = event;
async move { Ok(ctx.skip()) }
}
fn handle_undecodable(
&self,
error: &UndecodableDelivery<L>,
) -> impl std::future::Future<Output = Result<(), Box<dyn std::error::Error + Send + Sync>>> + Send
{
let error = error.inner().clone();
async move { Err(error.into()) }
}
fn flush(
&self,
op: &mut FlushOp<'_, L>,
items: Self::Batch,
) -> impl std::future::Future<Output = Result<(), Box<dyn std::error::Error + Send + Sync>>> + Send
{
let _ = (op, items);
async { Ok(()) }
}
}
struct SubscriberFlusher<H, P, L> {
handler: Arc<H>,
_payload: std::marker::PhantomData<fn() -> (P, L)>,
}
impl<H, P, L> ItemFlush<H::Batch> for SubscriberFlusher<H, P, L>
where
H: SingletonSubscriber<P, L>,
P: Serialize + DeserializeOwned + Send + Sync + 'static + Unpin,
L: Lane,
{
fn flush_items<'a>(
&'a self,
op: &'a mut es_entity::DbOp<'static>,
items: H::Batch,
state: &'a OutboxEventJobState,
) -> BoxFuture<'a, Result<(), HandlerError>> {
Box::pin(async move {
let mut op =
FlushOp::<L>::new(op, L::checkpoint(state.sequence, state.commit_sequence));
self.handler.flush(&mut op, items).await
})
}
fn position_of(&self, state: &OutboxEventJobState) -> StreamPosition {
L::checkpoint(state.sequence, state.commit_sequence).into()
}
}
fn checkpoint_of<L: Lane>(state: &OutboxEventJobState) -> StreamPosition {
L::checkpoint(state.sequence, state.commit_sequence).into()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum LaneChoice {
Insert(EventSequence),
Commit(CommitSequence),
}
pub(crate) fn decide_lane(
stored_commit: Option<CommitSequence>,
stored_insert: EventSequence,
configured: Ordering,
) -> Result<LaneChoice, crate::out::error::LaneMismatch> {
match (stored_commit, configured) {
(None, Ordering::Insert) => Ok(LaneChoice::Insert(stored_insert)),
(Some(commit_sequence), Ordering::Commit) => Ok(LaneChoice::Commit(commit_sequence)),
(None, Ordering::Commit) => {
if stored_insert == EventSequence::BEGIN {
Ok(LaneChoice::Commit(CommitSequence::BEGIN))
} else {
Err(crate::out::error::LaneMismatch {
stored: Ordering::Insert,
configured: Ordering::Commit,
})
}
}
(Some(_), Ordering::Insert) => Err(crate::out::error::LaneMismatch {
stored: Ordering::Commit,
configured: Ordering::Insert,
}),
}
}
const DEFAULT_MAX_BATCH_SIZE: usize = 100;
const DEFAULT_CHECKPOINT_INTERVAL: std::time::Duration = std::time::Duration::from_secs(5);
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Ordering {
#[default]
Insert,
Commit,
}
impl std::fmt::Display for Ordering {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Insert => f.write_str("insert"),
Self::Commit => f.write_str("commit"),
}
}
}
#[derive(Clone)]
pub struct OutboxEventJobConfig {
pub job_type: JobType,
pub retry_settings: RetrySettings,
pub max_batch_size: usize,
pub checkpoint_interval: std::time::Duration,
}
impl OutboxEventJobConfig {
pub fn new(job_type: JobType) -> Self {
Self {
job_type,
retry_settings: RetrySettings::repeat_indefinitely(),
max_batch_size: DEFAULT_MAX_BATCH_SIZE,
checkpoint_interval: DEFAULT_CHECKPOINT_INTERVAL,
}
}
pub fn with_retry_settings(mut self, settings: RetrySettings) -> Self {
self.retry_settings = settings;
self
}
pub fn with_max_batch_size(mut self, max_batch_size: usize) -> Self {
self.max_batch_size = max_batch_size.max(1);
self
}
pub fn with_checkpoint_interval(mut self, interval: std::time::Duration) -> Self {
self.checkpoint_interval = interval;
self
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub(in crate::out) struct OutboxEventJobData {}
pub(in crate::out) struct OutboxEventJobInitializer<H, P, Tables, L>
where
H: SingletonSubscriber<P, L>,
P: Serialize + DeserializeOwned + Send + Sync + 'static + Unpin,
Tables: MailboxTables,
L: Lane,
{
outbox: Outbox<P, Tables>,
handler: Arc<H>,
job_type: JobType,
retry_settings: RetrySettings,
max_batch_size: usize,
checkpoint_interval: std::time::Duration,
_lane: std::marker::PhantomData<fn() -> L>,
}
impl<H, P, Tables, L> OutboxEventJobInitializer<H, P, Tables, L>
where
H: SingletonSubscriber<P, L>,
P: Serialize + DeserializeOwned + Send + Sync + 'static + Unpin,
Tables: MailboxTables,
L: Lane,
{
pub fn new(outbox: Outbox<P, Tables>, handler: H, config: &OutboxEventJobConfig) -> Self {
Self {
outbox,
handler: Arc::new(handler),
job_type: config.job_type.clone(),
retry_settings: config.retry_settings.clone(),
max_batch_size: config.max_batch_size,
checkpoint_interval: config.checkpoint_interval,
_lane: std::marker::PhantomData,
}
}
}
impl<H, P, Tables, L> ResidentJobInitializer for OutboxEventJobInitializer<H, P, Tables, L>
where
H: SingletonSubscriber<P, L>,
P: Serialize + DeserializeOwned + Send + Sync + 'static + Unpin,
Tables: MailboxTables,
L: Lane,
{
type Config = OutboxEventJobData;
fn job_type(&self) -> JobType {
self.job_type.clone()
}
fn retry_on_error_settings(&self) -> RetrySettings {
self.retry_settings.clone()
}
fn init(
&self,
_job: &Job,
) -> Result<Box<dyn ResidentJobRunner>, Box<dyn std::error::Error + Send + Sync>> {
Ok(Box::new(OutboxEventJobRunner::<H, P, Tables, L> {
outbox: self.outbox.clone(),
handler: self.handler.clone(),
max_batch_size: self.max_batch_size,
checkpoint_interval: self.checkpoint_interval,
_lane: std::marker::PhantomData,
}))
}
}
struct OutboxEventJobRunner<H, P, Tables, L>
where
H: SingletonSubscriber<P, L>,
P: Serialize + DeserializeOwned + Send + Sync + 'static + Unpin,
Tables: MailboxTables,
L: Lane,
{
outbox: Outbox<P, Tables>,
handler: Arc<H>,
max_batch_size: usize,
checkpoint_interval: std::time::Duration,
_lane: std::marker::PhantomData<fn() -> L>,
}
#[async_trait]
impl<H, P, Tables, L> ResidentJobRunner for OutboxEventJobRunner<H, P, Tables, L>
where
H: SingletonSubscriber<P, L>,
P: Serialize + DeserializeOwned + Send + Sync + 'static + Unpin,
Tables: MailboxTables,
L: Lane,
{
async fn run(
&self,
current_job: CurrentJob,
) -> Result<ResidentJobCompletion, Box<dyn std::error::Error + Send + Sync>> {
self.run_persistent(current_job).await
}
}
impl<H, P, Tables, L> OutboxEventJobRunner<H, P, Tables, L>
where
H: SingletonSubscriber<P, L>,
P: Serialize + DeserializeOwned + Send + Sync + 'static + Unpin,
Tables: MailboxTables,
L: Lane,
{
fn select_lane(
&self,
state: &OutboxEventJobState,
) -> Result<crate::out::LaneListener<L, P>, ObixFault> {
let start_after = L::resume_from(state.sequence, state.commit_sequence)?;
Ok(self.outbox.listen::<L>(start_after).narrow_rejected()?)
}
async fn run_persistent(
&self,
mut current_job: CurrentJob,
) -> Result<ResidentJobCompletion, Box<dyn std::error::Error + Send + Sync>> {
let mut state = decode_execution_state(¤t_job)?.unwrap_or_default();
let mut persistent = self.select_lane(&state)?;
let mut op_slot: Option<es_entity::DbOp<'static>> = None;
let mut tracker = BatchTracker {
collected: 0,
persisted: checkpoint_of::<L>(&state),
last_persist: tokio::time::Instant::now(),
};
let mut in_group = false;
let mut batch = H::Batch::default();
let flusher = SubscriberFlusher::<H, P, L> {
handler: self.handler.clone(),
_payload: std::marker::PhantomData,
};
loop {
let item = if tracker.collected > 0 {
match persistent.next().now_or_never() {
Some(Some(item)) => item,
Some(None) => {
let mut parts = CtxParts {
op_slot: &mut op_slot,
current_job: &mut current_job,
state: &mut state,
tracker: &mut tracker,
mirror: None,
};
flush_batch(&mut parts, &mut batch, &flusher, "stream_closed").await?;
return Ok(ResidentJobCompletion::RescheduleNow);
}
None if in_group => match persistent.next().await {
Some(item) => item,
None => {
let mut parts = CtxParts {
op_slot: &mut op_slot,
current_job: &mut current_job,
state: &mut state,
tracker: &mut tracker,
mirror: None,
};
flush_batch(&mut parts, &mut batch, &flusher, "stream_closed").await?;
return Ok(ResidentJobCompletion::RescheduleNow);
}
},
None => {
let mut parts = CtxParts {
op_slot: &mut op_slot,
current_job: &mut current_job,
state: &mut state,
tracker: &mut tracker,
mirror: None,
};
flush_batch(&mut parts, &mut batch, &flusher, "backlog_drained").await?;
continue;
}
}
} else {
tokio::select! {
biased;
_ = current_job.shutdown_requested() => {
if tracker.persisted < checkpoint_of::<L>(&state) {
persist_checkpoint(&mut current_job, &state, None)
.await?;
}
return Ok(ResidentJobCompletion::RescheduleNow);
}
_ = tokio::time::sleep_until(tracker.last_persist + self.checkpoint_interval),
if tracker.persisted < checkpoint_of::<L>(&state) => {
persist_checkpoint(&mut current_job, &state, None)
.await?;
tracker.persisted = checkpoint_of::<L>(&state);
tracker.last_persist = tokio::time::Instant::now();
continue;
}
next = persistent.next() => match next {
Some(item) => item,
None => {
if tracker.persisted < checkpoint_of::<L>(&state) {
persist_checkpoint(&mut current_job, &state, None).await?;
}
return Ok(ResidentJobCompletion::RescheduleNow);
}
}
}
};
let event = match item {
Ok(event) => event,
Err(undecodable) => {
let mut parts = CtxParts {
op_slot: &mut op_slot,
current_job: &mut current_job,
state: &mut state,
tracker: &mut tracker,
mirror: None,
};
flush_batch(&mut parts, &mut batch, &flusher, "undecodable_event").await?;
match self.handler.handle_undecodable(&undecodable).await {
Ok(()) => {
state.sequence = undecodable.sequence;
L::record(&mut state.commit_sequence, undecodable.position());
continue;
}
Err(error) => {
if tracker.persisted < checkpoint_of::<L>(&state) {
persist_checkpoint(&mut current_job, &state, None).await?;
}
return Err(error);
}
}
}
};
let ctx = EventCtx {
parts: CtxParts {
op_slot: &mut op_slot,
current_job: &mut current_job,
state: &mut state,
tracker: &mut tracker,
mirror: None,
},
batch: &mut batch,
flusher: &flusher,
};
let outcome = self.handler.handle_persistent(ctx, &event).await?.outcome;
state.sequence = event.sequence;
L::record(&mut state.commit_sequence, event.position());
in_group = !event.is_boundary();
match outcome {
Outcome::Skip => {}
Outcome::Commit => {
let mut parts = CtxParts {
op_slot: &mut op_slot,
current_job: &mut current_job,
state: &mut state,
tracker: &mut tracker,
mirror: None,
};
flush_batch(&mut parts, &mut batch, &flusher, "commit").await?;
}
Outcome::Collect => {
if tracker.collected >= self.max_batch_size && !in_group {
let mut parts = CtxParts {
op_slot: &mut op_slot,
current_job: &mut current_job,
state: &mut state,
tracker: &mut tracker,
mirror: None,
};
flush_batch(&mut parts, &mut batch, &flusher, "batch_full").await?;
}
}
Outcome::Pause(_) | Outcome::CommitAndPause(_) => {
unreachable!(
"Outcome::Pause/CommitAndPause cannot be minted from a singleton \
subscriber's EventCtx"
)
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn insert_lane_stays_on_the_insert_cursor() {
let choice = decide_lane(None, EventSequence::from(7u64), Ordering::Insert);
assert_eq!(choice, Ok(LaneChoice::Insert(EventSequence::from(7u64))));
}
#[test]
fn commit_lane_continues_from_the_commit_cursor() {
let choice = decide_lane(
Some(CommitSequence::from(4u64)),
EventSequence::from(9u64),
Ordering::Commit,
);
assert_eq!(choice, Ok(LaneChoice::Commit(CommitSequence::from(4u64))));
}
#[test]
fn a_fresh_subscription_starts_at_the_beginning_of_the_commit_lane() {
let choice = decide_lane(None, EventSequence::BEGIN, Ordering::Commit);
assert_eq!(choice, Ok(LaneChoice::Commit(CommitSequence::BEGIN)));
}
#[test]
fn switching_an_established_subscription_to_commit_is_refused() {
let error =
decide_lane(None, EventSequence::from(12u64), Ordering::Commit).expect_err("refuses");
assert_eq!(
error,
crate::out::error::LaneMismatch {
stored: Ordering::Insert,
configured: Ordering::Commit,
}
);
assert!(error.to_string().contains("register a new job type"));
}
#[test]
fn switching_back_to_the_insert_lane_is_refused() {
let error = decide_lane(
Some(CommitSequence::from(3u64)),
EventSequence::from(9u64),
Ordering::Insert,
)
.expect_err("refuses");
assert_eq!(
error,
crate::out::error::LaneMismatch {
stored: Ordering::Commit,
configured: Ordering::Insert,
}
);
assert!(error.to_string().contains("register a new job type"));
}
}