pub(crate) mod keyed;
pub(crate) mod singleton;
use serde::{Serialize, de::DeserializeOwned};
use std::{marker::PhantomData, time::Duration};
use crate::out::ctx::OutboxEventJobState;
use crate::out::lane::{CommitOrder, InsertOrder, Lane};
use crate::out::persistent::SequencerPositions;
use crate::{
sequence::{CommitSequence, EventSequence},
tables::{DefaultMailboxTables, MailboxTables},
};
use self::singleton::Ordering;
const INITIAL_POLL_INTERVAL: Duration = Duration::from_millis(100);
const MAX_POLL_INTERVAL: Duration = Duration::from_millis(250);
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum StreamPosition {
Insert(EventSequence),
Commit(CommitSequence),
}
impl StreamPosition {
pub fn ordering(&self) -> Ordering {
match self {
Self::Insert(_) => Ordering::Insert,
Self::Commit(_) => Ordering::Commit,
}
}
pub fn value(&self) -> u64 {
match self {
Self::Insert(sequence) => u64::from(*sequence),
Self::Commit(commit_sequence) => u64::from(*commit_sequence),
}
}
}
impl From<EventSequence> for StreamPosition {
fn from(sequence: EventSequence) -> Self {
Self::Insert(sequence)
}
}
impl From<CommitSequence> for StreamPosition {
fn from(commit_sequence: CommitSequence) -> Self {
Self::Commit(commit_sequence)
}
}
impl std::fmt::Display for StreamPosition {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Insert(sequence) => write!(f, "insert:{sequence}"),
Self::Commit(commit_sequence) => write!(f, "commit:{commit_sequence}"),
}
}
}
#[derive(Debug, thiserror::Error)]
pub enum SubscriptionError {
#[error("SubscriptionError - Sqlx: {0}")]
Sqlx(#[from] sqlx::Error),
#[error("SubscriptionError - LaneMismatch: {0}")]
LaneMismatch(String),
#[error("SubscriptionError - Job: {0}")]
Job(#[from] ::job::JobError),
#[error("SubscriptionError - StateDecode: {0}")]
StateDecode(#[from] serde_json::Error),
#[error("SubscriptionError - NoSuchJob: no job for ({subscriber_type}, {key})")]
NoSuchJob {
subscriber_type: String,
key: String,
},
#[error(
"SubscriptionError - CaughtUpTimeout: checkpoint {checkpoint} behind target {target} after {waited:?}"
)]
CaughtUpTimeout {
checkpoint: StreamPosition,
target: StreamPosition,
waited: Duration,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SubscriptionStreamStatus {
pub checkpoint: StreamPosition,
pub frontier: StreamPosition,
}
impl SubscriptionStreamStatus {
pub fn lag(&self) -> u64 {
self.frontier
.value()
.saturating_sub(self.checkpoint.value())
}
pub fn is_caught_up(&self) -> bool {
self.checkpoint >= self.frontier
}
}
pub struct SubscriptionSnapshot<L = InsertOrder>
where
L: Lane,
{
job: ::job::JobSnapshot,
checkpoint: L::Position,
frontier: L::Position,
}
impl<L> SubscriptionSnapshot<L>
where
L: Lane,
{
pub fn ordering(&self) -> Ordering {
L::ORDERING
}
pub fn checkpoint(&self) -> L::Position {
self.checkpoint
}
pub fn frontier(&self) -> L::Position {
self.frontier
}
pub fn stream_status(&self) -> SubscriptionStreamStatus {
SubscriptionStreamStatus {
checkpoint: self.checkpoint.into(),
frontier: self.frontier.into(),
}
}
pub fn lag(&self) -> u64 {
self.stream_status().lag()
}
pub fn is_caught_up(&self) -> bool {
self.stream_status().is_caught_up()
}
pub fn job_status(&self) -> ::job::JobStatus {
self.job.state()
}
pub fn last_error(&self) -> Option<&str> {
self.job.last_error()
}
pub fn attempt(&self) -> Option<u32> {
self.job.attempt()
}
pub fn job(&self) -> &::job::JobSnapshot {
&self.job
}
}
pub struct Subscription<P, L = InsertOrder, Tables = DefaultMailboxTables>
where
P: Serialize + DeserializeOwned + Send + Sync + 'static,
L: Lane,
{
anchor: JobAnchor,
pool: sqlx::PgPool,
positions: Option<SequencerPositions>,
_phantom: PhantomData<(P, L, Tables)>,
}
#[derive(Clone)]
struct KeyedAnchor {
jobs: ::job::Jobs,
job_type: ::job::JobType,
key: String,
}
#[derive(Clone)]
enum JobAnchor {
Resident(::job::JobHandle),
Keyed(Box<KeyedAnchor>),
}
impl<P, L, Tables> Clone for Subscription<P, L, Tables>
where
P: Serialize + DeserializeOwned + Send + Sync + 'static,
L: Lane,
{
fn clone(&self) -> Self {
Self {
anchor: self.anchor.clone(),
pool: self.pool.clone(),
positions: self.positions.clone(),
_phantom: PhantomData,
}
}
}
impl<P, L, Tables> std::fmt::Debug for Subscription<P, L, Tables>
where
P: Serialize + DeserializeOwned + Send + Sync + 'static,
L: Lane,
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let mut out = f.debug_struct("Subscription");
match &self.anchor {
JobAnchor::Resident(job) => out.field("job_id", &job.id()),
JobAnchor::Keyed(anchor) => out
.field("subscriber_type", &anchor.job_type)
.field("key", &anchor.key),
}
.finish_non_exhaustive()
}
}
impl<P, L, Tables> Subscription<P, L, Tables>
where
P: Serialize + DeserializeOwned + Send + Sync + 'static + Unpin,
Tables: MailboxTables,
L: Lane,
{
pub(super) fn new(
job: ::job::JobHandle,
pool: sqlx::PgPool,
positions: Option<SequencerPositions>,
) -> Self {
Self {
anchor: JobAnchor::Resident(job),
pool,
positions,
_phantom: PhantomData,
}
}
pub(super) fn new_keyed(
jobs: ::job::Jobs,
job_type: ::job::JobType,
key: String,
pool: sqlx::PgPool,
) -> Self {
Self {
anchor: JobAnchor::Keyed(Box::new(KeyedAnchor {
jobs,
job_type,
key,
})),
pool,
positions: None,
_phantom: PhantomData,
}
}
pub fn job_id(&self) -> Option<::job::JobId> {
match &self.anchor {
JobAnchor::Resident(job) => Some(job.id()),
JobAnchor::Keyed { .. } => None,
}
}
async fn handle(&self) -> Result<::job::JobHandle, SubscriptionError> {
match &self.anchor {
JobAnchor::Resident(job) => Ok(job.clone()),
JobAnchor::Keyed(anchor) => anchor
.jobs
.keyed_handle(anchor.job_type.clone(), anchor.key.clone())
.await?
.ok_or_else(|| SubscriptionError::NoSuchJob {
subscriber_type: anchor.job_type.to_string(),
key: anchor.key.clone(),
}),
}
}
#[tracing::instrument(name = "obix.registered_handler.load", skip_all, err)]
pub async fn load(&self) -> Result<SubscriptionSnapshot<L>, SubscriptionError> {
let job = self.handle().await?.load().await?;
let state = decode_state(&job)?;
L::resume_from(state.sequence, state.commit_sequence)
.map_err(SubscriptionError::LaneMismatch)?;
let checkpoint = L::checkpoint(state.sequence, state.commit_sequence);
let frontier = self.frontier().await?;
Ok(SubscriptionSnapshot {
job,
checkpoint,
frontier,
})
}
#[tracing::instrument(
name = "obix.registered_handler.await_position",
skip_all,
// Not `target`: that name collides with `instrument`'s own span-target
// argument.
fields(target_position = %target, timeout_ms = timeout.as_millis()),
err
)]
pub async fn await_position(
&self,
target: L::Position,
timeout: Duration,
) -> Result<(), SubscriptionError> {
let start = tokio::time::Instant::now();
self.poll_checkpoint_until(target, start, start + timeout)
.await
}
async fn poll_checkpoint_until(
&self,
target: L::Position,
start: tokio::time::Instant,
deadline: tokio::time::Instant,
) -> Result<(), SubscriptionError> {
let mut interval = INITIAL_POLL_INTERVAL;
loop {
let checkpoint = self.checkpoint().await?;
if checkpoint >= target {
return Ok(());
}
let now = tokio::time::Instant::now();
if now >= deadline {
return Err(SubscriptionError::CaughtUpTimeout {
checkpoint: checkpoint.into(),
target: target.into(),
waited: now.duration_since(start),
});
}
tokio::time::sleep(interval.min(deadline - now)).await;
interval = (interval * 2).min(MAX_POLL_INTERVAL);
}
}
#[tracing::instrument(
name = "obix.registered_handler.await_caught_up",
skip_all,
fields(timeout_ms = timeout.as_millis()),
err
)]
pub async fn await_caught_up(&self, timeout: Duration) -> Result<(), SubscriptionError> {
L::await_caught_up(self, timeout).await
}
async fn checkpoint(&self) -> Result<L::Position, SubscriptionError> {
let state = self
.handle()
.await?
.execution_state::<OutboxEventJobState>()
.await?
.unwrap_or_default();
Ok(L::checkpoint(state.sequence, state.commit_sequence))
}
async fn frontier(&self) -> Result<L::Position, SubscriptionError> {
L::frontier(self).await
}
pub(crate) fn pool(&self) -> &sqlx::PgPool {
&self.pool
}
pub(crate) fn sequencer_positions(&self) -> Result<&SequencerPositions, SubscriptionError> {
self.positions.as_ref().ok_or_else(|| {
SubscriptionError::LaneMismatch(
"a commit-lane subscription without sequencer positions is unreachable: the lane \
cannot be registered on an outbox that runs no sequencer, and keyed \
subscriptions are insert-lane by construction"
.to_string(),
)
})
}
}
pub(crate) async fn await_caught_up_insert_lane<P, Tables>(
subscription: &Subscription<P, InsertOrder, Tables>,
timeout: Duration,
) -> Result<(), SubscriptionError>
where
P: Serialize + DeserializeOwned + Send + Sync + 'static + Unpin,
Tables: MailboxTables,
{
let frontier = subscription.frontier().await?;
subscription.await_position(frontier, timeout).await
}
pub(crate) async fn await_caught_up_commit_lane<P, Tables>(
subscription: &Subscription<P, CommitOrder, Tables>,
timeout: Duration,
) -> Result<(), SubscriptionError>
where
P: Serialize + DeserializeOwned + Send + Sync + 'static + Unpin,
Tables: MailboxTables,
{
let insert_frontier = read_frontier::<Tables>(&subscription.pool).await?;
let start = tokio::time::Instant::now();
let deadline = start + timeout;
let positions = subscription.sequencer_positions()?;
let mut interval = INITIAL_POLL_INTERVAL;
loop {
let folded = positions.fold_position();
if folded >= insert_frontier {
break;
}
let now = tokio::time::Instant::now();
if now >= deadline {
return Err(SubscriptionError::CaughtUpTimeout {
checkpoint: folded.into(),
target: insert_frontier.into(),
waited: now.duration_since(start),
});
}
tokio::time::sleep(interval.min(deadline - now)).await;
interval = (interval * 2).min(MAX_POLL_INTERVAL);
}
let commit_frontier = positions.commit_head();
subscription
.poll_checkpoint_until(commit_frontier, start, deadline)
.await
}
pub(super) async fn read_frontier<Tables: MailboxTables>(
pool: &sqlx::PgPool,
) -> Result<EventSequence, sqlx::Error> {
let pool = pool.clone();
let fut: std::pin::Pin<
Box<dyn std::future::Future<Output = Result<EventSequence, sqlx::Error>> + Send>,
> = Box::pin(async move { Tables::highest_known_persistent_sequence(&pool).await });
fut.await
}
fn decode_state(job: &::job::JobSnapshot) -> Result<OutboxEventJobState, SubscriptionError> {
Ok(job
.execution_state::<OutboxEventJobState>()?
.unwrap_or_default())
}