use std::{
any::Any,
collections::HashSet,
panic::{AssertUnwindSafe, catch_unwind},
sync::{
Arc, LazyLock, Mutex,
atomic::{AtomicBool, AtomicU64, Ordering},
},
time::{Duration, Instant},
};
use datum::actor::{Actor, ActorProcessingErr, ActorRef};
use futures_util::{FutureExt, StreamExt};
use tokio::{
sync::{Mutex as AsyncMutex, Semaphore, mpsc, oneshot, watch},
task::JoinHandle,
};
use crate::query::QueryMode;
#[cfg(feature = "postgres")]
use super::{
ExactlyOncePostgresHandler, GroupedExactlyOncePostgresHandler, PostgresProjectionOffsetStore,
};
use super::{
GroupedHandler, Handler, Projection, ProjectionCursor, ProjectionError, ProjectionGeneration,
ProjectionHandlerFactory, ProjectionId, ProjectionOffsetStore, ProjectionResult,
RecoveryStrategy, RestartBackoff,
source::{SourceProvider, advance_cursor, resume_allows},
};
static RUNNING_PROJECTIONS: LazyLock<Mutex<HashSet<ProjectionId>>> =
LazyLock::new(|| Mutex::new(HashSet::new()));
static NEXT_INCARNATION: AtomicU64 = AtomicU64::new(1);
static RNG_SEED: AtomicU64 = AtomicU64::new(0x6a09_e667_f3bc_c909);
pub trait BackoffRng: Send + Sync + 'static {
fn next_fraction(&self) -> f64;
}
#[derive(Debug, Default)]
struct XorShiftBackoffRng;
impl BackoffRng for XorShiftBackoffRng {
fn next_fraction(&self) -> f64 {
let mut current = RNG_SEED.load(Ordering::Relaxed);
loop {
let mut next = current;
next ^= next << 13;
next ^= next >> 7;
next ^= next << 17;
match RNG_SEED.compare_exchange_weak(
current,
next,
Ordering::Relaxed,
Ordering::Relaxed,
) {
Ok(_) => return ((next >> 11) as f64) / ((1_u64 << 53) as f64),
Err(observed) => current = observed,
}
}
}
}
#[derive(Clone)]
pub struct ProjectionRuntime {
pub(crate) offset_store: Arc<dyn ProjectionOffsetStore>,
pub(crate) backoff_rng: Arc<dyn BackoffRng>,
worker_synchronization: Option<WorkerSynchronizationProbe>,
}
impl ProjectionRuntime {
#[must_use]
pub fn new(offset_store: Arc<dyn ProjectionOffsetStore>) -> Self {
Self {
offset_store,
backoff_rng: Arc::new(XorShiftBackoffRng),
worker_synchronization: None,
}
}
#[must_use]
pub fn with_backoff_rng<R>(mut self, rng: R) -> Self
where
R: BackoffRng,
{
self.backoff_rng = Arc::new(rng);
self
}
#[cfg(test)]
fn with_worker_synchronization(mut self, probe: WorkerSynchronizationProbe) -> Self {
self.worker_synchronization = Some(probe);
self
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum ProjectionStatus {
Starting,
Running {
generation: ProjectionGeneration,
},
Paused,
Restarting {
attempt: usize,
delay: Duration,
error: ProjectionError,
},
Completed,
Stopped,
Failed(ProjectionError),
}
impl ProjectionStatus {
#[must_use]
pub const fn is_running(&self) -> bool {
matches!(self, Self::Running { .. })
}
#[must_use]
pub const fn is_terminal(&self) -> bool {
matches!(self, Self::Completed | Self::Stopped | Self::Failed(_))
}
}
pub trait StatusObserver<Envelope>: Send + Sync + 'static {
fn status_changed(&self, _projection_id: &ProjectionId, _status: &ProjectionStatus) {}
fn before_process(&self, _projection_id: &ProjectionId, _envelope: &Envelope, _attempt: usize) {
}
fn after_process(&self, _projection_id: &ProjectionId, _envelope: &Envelope) {}
fn handler_error(
&self,
_projection_id: &ProjectionId,
_envelope: &Envelope,
_attempt: usize,
_error: &ProjectionError,
) {
}
fn before_process_group(
&self,
projection_id: &ProjectionId,
envelopes: &[Envelope],
attempt: usize,
) {
for envelope in envelopes {
self.before_process(projection_id, envelope, attempt);
}
}
fn after_process_group(&self, projection_id: &ProjectionId, envelopes: &[Envelope]) {
for envelope in envelopes {
self.after_process(projection_id, envelope);
}
}
fn group_handler_error(
&self,
projection_id: &ProjectionId,
envelopes: &[Envelope],
attempt: usize,
error: &ProjectionError,
) {
for envelope in envelopes {
self.handler_error(projection_id, envelope, attempt, error);
}
}
fn offset_progress(&self, _projection_id: &ProjectionId, _cursor: &ProjectionCursor) {}
}
pub(crate) struct NoopStatusObserver;
impl<Envelope> StatusObserver<Envelope> for NoopStatusObserver {}
#[must_use]
pub fn restart_delay(
settings: &RestartBackoff,
consecutive_failures: usize,
random_sample: f64,
) -> Duration {
let exponent = u32::try_from(consecutive_failures).unwrap_or(u32::MAX);
let base_nanos = settings
.min_backoff
.as_nanos()
.checked_shl(exponent)
.unwrap_or(u128::MAX)
.min(settings.max_backoff.as_nanos());
if settings.random_factor == 0.0 || base_nanos == 0 {
return duration_from_nanos(base_nanos);
}
let sample = if random_sample.is_finite() {
random_sample.clamp(0.0, 1.0)
} else {
0.0
};
let jitter = (base_nanos as f64 * sample * settings.random_factor) as u128;
duration_from_nanos(base_nanos.saturating_add(jitter))
}
fn duration_from_nanos(nanos: u128) -> Duration {
const NANOS_PER_SECOND: u128 = 1_000_000_000;
let max_nanos = u128::from(u64::MAX)
.saturating_mul(NANOS_PER_SECOND)
.saturating_add(NANOS_PER_SECOND - 1);
let nanos = nanos.min(max_nanos);
let seconds = u64::try_from(nanos / NANOS_PER_SECOND).unwrap_or(u64::MAX);
let subsecond = u32::try_from(nanos % NANOS_PER_SECOND).unwrap_or(999_999_999);
Duration::new(seconds, subsecond)
}
enum RuntimeMessage {
Start,
SourceReady(u64),
FlushDue {
incarnation: u64,
epoch: u64,
},
GroupDue {
incarnation: u64,
epoch: u64,
},
Restart {
epoch: u64,
},
GetOffset(oneshot::Sender<ProjectionResult<Option<ProjectionCursor>>>),
ClearOffset(oneshot::Sender<ProjectionResult<()>>),
UpdateOffset {
cursor: ProjectionCursor,
reply: oneshot::Sender<ProjectionResult<()>>,
},
Pause(oneshot::Sender<ProjectionResult<()>>),
Resume(oneshot::Sender<ProjectionResult<()>>),
Stop(oneshot::Sender<ProjectionResult<()>>),
BestEffortStop,
}
enum WorkerEvent<Envelope> {
Envelope(Envelope),
Completed,
Failed(ProjectionError),
}
struct SourceWorker<Envelope> {
demand: mpsc::Sender<Option<ProjectionCursor>>,
events: mpsc::Receiver<WorkerEvent<Envelope>>,
tracked: TrackedSourceWorker,
synchronization: Option<WorkerSynchronizationProbe>,
}
#[derive(Clone)]
struct TrackedSourceWorker {
cancellation: watch::Sender<bool>,
join: Arc<AsyncMutex<Option<JoinHandle<()>>>>,
}
#[derive(Clone)]
struct WorkerSynchronizationProbe {
inner: Arc<WorkerSynchronizationState>,
}
struct WorkerSynchronizationState {
deliveries: watch::Sender<usize>,
exit_started: Semaphore,
allow_exit: Semaphore,
exited: AtomicBool,
}
impl WorkerSynchronizationProbe {
#[cfg(test)]
fn new() -> Self {
let (deliveries, _receiver) = watch::channel(0);
Self {
inner: Arc::new(WorkerSynchronizationState {
deliveries,
exit_started: Semaphore::new(0),
allow_exit: Semaphore::new(0),
exited: AtomicBool::new(false),
}),
}
}
fn delivery_count(&self) -> usize {
*self.inner.deliveries.borrow()
}
fn delivered(&self) {
let next = self.delivery_count().saturating_add(1);
self.inner.deliveries.send_replace(next);
}
async fn wait_for_delivery_after(&self, observed: usize) -> ProjectionResult<()> {
let mut deliveries = self.inner.deliveries.subscribe();
while *deliveries.borrow_and_update() <= observed {
deliveries.changed().await.map_err(|_| {
ProjectionError::Runtime("source worker delivery probe closed".into())
})?;
}
Ok(())
}
#[cfg(test)]
async fn wait_for_delivery(&self, expected: usize) {
let mut deliveries = self.inner.deliveries.subscribe();
while *deliveries.borrow_and_update() < expected {
deliveries
.changed()
.await
.expect("test worker delivery probe remains open");
}
}
#[cfg(test)]
async fn wait_for_exit_start(&self) {
self.inner
.exit_started
.acquire()
.await
.expect("test worker exit probe remains open")
.forget();
}
#[cfg(test)]
fn allow_exit(&self) {
self.inner.allow_exit.add_permits(1);
}
#[cfg(test)]
fn exited(&self) -> bool {
self.inner.exited.load(Ordering::Acquire)
}
}
struct Incarnation<Envelope> {
token: u64,
generation: ProjectionGeneration,
handler: IncarnationHandler<Envelope>,
worker: SourceWorker<Envelope>,
durable_cursor: Option<ProjectionCursor>,
logical_cursor: Option<ProjectionCursor>,
pending_envelopes: usize,
last_save: Instant,
flush_epoch: u64,
group_buffer: Vec<Envelope>,
group_cursor: Option<ProjectionCursor>,
group_epoch: u64,
}
enum IncarnationHandler<Envelope> {
Single(Box<dyn Handler<Envelope>>),
Grouped(Box<dyn GroupedHandler<Envelope>>),
#[cfg(feature = "postgres")]
ExactlyOncePostgres(Box<dyn ExactlyOncePostgresHandler<Envelope>>),
#[cfg(feature = "postgres")]
GroupedExactlyOncePostgres(Box<dyn GroupedExactlyOncePostgresHandler<Envelope>>),
}
enum ProcessEnvelopeError {
Handler(ProjectionError),
Infrastructure(ProjectionError),
}
#[cfg(feature = "postgres")]
fn recovery_settings(strategy: RecoveryStrategy) -> (usize, Duration, bool) {
match strategy {
RecoveryStrategy::Fail => (0, Duration::ZERO, false),
RecoveryStrategy::Skip => (0, Duration::ZERO, true),
RecoveryStrategy::RetryAndFail { retries, delay } => (retries, delay, false),
RecoveryStrategy::RetryAndSkip { retries, delay } => (retries, delay, true),
}
}
enum ActorPhase {
Starting,
Running,
Paused,
Restarting,
Terminal,
}
struct ProjectionBackoff {
consecutive_failures: usize,
restarts_in_window: usize,
window_started: Instant,
}
impl ProjectionBackoff {
fn new() -> Self {
Self {
consecutive_failures: 0,
restarts_in_window: 0,
window_started: Instant::now(),
}
}
fn successful_invocation(&mut self) {
self.consecutive_failures = 0;
}
fn reserve_restart(
&mut self,
settings: &RestartBackoff,
now: Instant,
) -> Option<(usize, usize)> {
if now.duration_since(self.window_started) > settings.max_restarts_within {
self.consecutive_failures = 0;
self.restarts_in_window = 0;
self.window_started = now;
}
if settings
.max_restarts
.is_some_and(|max| self.restarts_in_window >= max)
{
return None;
}
let failure_index = self.consecutive_failures;
self.consecutive_failures = self.consecutive_failures.saturating_add(1);
self.restarts_in_window = self.restarts_in_window.saturating_add(1);
Some((failure_index, self.restarts_in_window))
}
}
struct RuntimeState<Envelope> {
phase: ActorPhase,
incarnation: Option<Incarnation<Envelope>>,
backoff: ProjectionBackoff,
restart_epoch: u64,
terminal_status_set: bool,
}
struct ProjectionActor<Envelope> {
projection: Projection<Envelope>,
runtime: ProjectionRuntime,
status: watch::Sender<ProjectionStatus>,
worker_tracker: Arc<AsyncMutex<Option<TrackedSourceWorker>>>,
_registry_lease: ProjectionRegistryLease,
}
pub(crate) async fn spawn<Envelope>(
projection: Projection<Envelope>,
runtime: ProjectionRuntime,
) -> ProjectionResult<ProjectionHandle>
where
Envelope: Send + Sync + 'static,
{
validate_spawn(&projection, &runtime)?;
let registry_lease = reserve_projection(projection.projection_id())?;
spawn_reserved(projection, runtime, registry_lease).await
}
pub(crate) fn validate_spawn<Envelope>(
projection: &Projection<Envelope>,
runtime: &ProjectionRuntime,
) -> ProjectionResult<()>
where
Envelope: Send + Sync + 'static,
{
projection.validate()?;
#[cfg(not(feature = "postgres"))]
let _ = runtime;
#[cfg(feature = "postgres")]
if projection.is_exactly_once_postgres() {
runtime.offset_store.validate_exactly_once()?;
if runtime.offset_store.postgres_store().is_none() {
return Err(ProjectionError::invalid_config(
"projection.runtime.offset_store",
"PostgreSQL exactly-once delivery requires PostgresProjectionOffsetStore",
));
}
}
Ok(())
}
pub(crate) async fn spawn_reserved<Envelope>(
projection: Projection<Envelope>,
runtime: ProjectionRuntime,
registry_lease: ProjectionRegistryLease,
) -> ProjectionResult<ProjectionHandle>
where
Envelope: Send + Sync + 'static,
{
if registry_lease.projection_id != *projection.projection_id() {
return Err(ProjectionError::Runtime(format!(
"projection registry reservation {} does not match spawned projection {}",
registry_lease.projection_id,
projection.projection_id()
)));
}
let projection_id = projection.projection_id().clone();
let (status_sender, status_receiver) = watch::channel(ProjectionStatus::Starting);
let handle_status_sender = status_sender.clone();
let worker_tracker = Arc::new(AsyncMutex::new(None));
let actor = ProjectionActor {
projection,
runtime,
status: status_sender,
worker_tracker: Arc::clone(&worker_tracker),
_registry_lease: registry_lease,
};
let initial_state = RuntimeState {
phase: ActorPhase::Starting,
incarnation: None,
backoff: ProjectionBackoff::new(),
restart_epoch: 0,
terminal_status_set: false,
};
let spawned = Actor::spawn(None, actor, initial_state).await;
let (control, _join) = match spawned {
Ok(spawned) => spawned,
Err(error) => {
return Err(ProjectionError::Runtime(format!(
"projection actor failed to spawn: {error}"
)));
}
};
Ok(ProjectionHandle {
projection_id,
control,
status: status_receiver,
status_sender: handle_status_sender,
stop_requested: AtomicBool::new(false),
worker_tracker,
})
}
impl<Envelope> Actor for ProjectionActor<Envelope>
where
Envelope: Send + Sync + 'static,
{
type Msg = RuntimeMessage;
type State = RuntimeState<Envelope>;
type Arguments = RuntimeState<Envelope>;
async fn pre_start(
&self,
myself: ActorRef<Self::Msg>,
state: Self::Arguments,
) -> Result<Self::State, ActorProcessingErr> {
myself.cast(RuntimeMessage::Start).map_err(|error| {
boxed_runtime(format!("failed to enqueue projection start: {error}"))
})?;
Ok(state)
}
async fn handle(
&self,
myself: ActorRef<Self::Msg>,
message: Self::Msg,
state: &mut Self::State,
) -> Result<(), ActorProcessingErr> {
self.handle_message(&myself, message, state).await;
Ok(())
}
async fn post_stop(
&self,
_myself: ActorRef<Self::Msg>,
state: &mut Self::State,
) -> Result<(), ActorProcessingErr> {
if let Some(mut incarnation) = state.incarnation.take() {
let _ = stop_source_worker(&mut incarnation.worker).await;
}
*self.worker_tracker.lock().await = None;
if !state.terminal_status_set {
self.publish_status(ProjectionStatus::Stopped);
}
Ok(())
}
}
impl<Envelope> ProjectionActor<Envelope>
where
Envelope: Send + Sync + 'static,
{
async fn handle_message(
&self,
myself: &ActorRef<RuntimeMessage>,
message: RuntimeMessage,
state: &mut RuntimeState<Envelope>,
) {
match message {
RuntimeMessage::Start => {
if matches!(state.phase, ActorPhase::Terminal) {
return;
}
if let Err(error) = self.start_incarnation(myself, state, None).await {
self.infrastructure_failure(myself, state, error).await;
}
}
RuntimeMessage::Restart { epoch } => {
if !matches!(state.phase, ActorPhase::Restarting) || state.restart_epoch != epoch {
return;
}
if let Err(error) = self.start_incarnation(myself, state, None).await {
self.infrastructure_failure(myself, state, error).await;
}
}
RuntimeMessage::SourceReady(token) => {
self.source_ready(myself, state, token).await;
}
RuntimeMessage::FlushDue { incarnation, epoch } => {
self.flush_due(myself, state, incarnation, epoch).await;
}
RuntimeMessage::GroupDue { incarnation, epoch } => {
self.group_due(myself, state, incarnation, epoch).await;
}
RuntimeMessage::GetOffset(reply) => {
let result = self
.runtime
.offset_store
.load(self.projection.projection_id())
.await;
let _ = reply.send(result);
}
RuntimeMessage::ClearOffset(reply) => {
let result = self.clear_offset(myself, state).await;
let failed = result.as_ref().err().cloned();
let _ = reply.send(result);
if let Some(error) = failed {
self.infrastructure_failure(myself, state, error).await;
}
}
RuntimeMessage::UpdateOffset { cursor, reply } => {
let result = self.update_offset(myself, state, cursor).await;
let failed = result.as_ref().err().cloned();
let _ = reply.send(result);
if let Some(error) = failed {
self.infrastructure_failure(myself, state, error).await;
}
}
RuntimeMessage::Pause(reply) => {
let result = self.pause(state).await;
let failed = result.as_ref().err().cloned();
let _ = reply.send(result);
if let Some(error) = failed {
self.infrastructure_failure(myself, state, error).await;
}
}
RuntimeMessage::Resume(reply) => {
let result = self.resume(myself, state).await;
let failed = result.as_ref().err().cloned();
let _ = reply.send(result);
if let Some(error) = failed {
self.infrastructure_failure(myself, state, error).await;
}
}
RuntimeMessage::Stop(reply) => {
let result = self.stop_acknowledged(state).await;
let status = result.as_ref().err().cloned();
let _ = reply.send(result);
match status {
Some(error) => self.set_terminal(state, ProjectionStatus::Failed(error)),
None => self.set_terminal(state, ProjectionStatus::Stopped),
}
myself.stop(None);
}
RuntimeMessage::BestEffortStop => {
let result = self.stop_acknowledged(state).await;
match result {
Ok(()) => self.set_terminal(state, ProjectionStatus::Stopped),
Err(error) => self.set_terminal(state, ProjectionStatus::Failed(error)),
}
myself.stop(None);
}
}
}
async fn start_incarnation(
&self,
myself: &ActorRef<RuntimeMessage>,
state: &mut RuntimeState<Envelope>,
claimed_generation: Option<ProjectionGeneration>,
) -> ProjectionResult<()> {
if self
.runtime
.offset_store
.load_paused(self.projection.projection_id())
.await?
{
state.phase = ActorPhase::Paused;
self.publish_status(ProjectionStatus::Paused);
return Ok(());
}
let generation = match claimed_generation {
Some(generation) => generation,
None => {
self.runtime
.offset_store
.claim(self.projection.projection_id())
.await?
}
};
let cursor = self
.runtime
.offset_store
.load(self.projection.projection_id())
.await?;
if let Some(cursor) = &cursor {
cursor.ensure_manifest(self.projection.source_provider.manifest())?;
}
let handler = match &self.projection.handler_factory {
ProjectionHandlerFactory::Single(factory) => {
let handler = catch_unwind(AssertUnwindSafe(|| factory())).map_err(|panic| {
ProjectionError::Handler(panic_message("handler factory", panic))
})?;
IncarnationHandler::Single(handler)
}
ProjectionHandlerFactory::Grouped(factory) => {
let handler = catch_unwind(AssertUnwindSafe(|| factory())).map_err(|panic| {
ProjectionError::Handler(panic_message("grouped handler factory", panic))
})?;
IncarnationHandler::Grouped(handler)
}
#[cfg(feature = "postgres")]
ProjectionHandlerFactory::ExactlyOncePostgres(factory) => {
let handler = catch_unwind(AssertUnwindSafe(|| factory())).map_err(|panic| {
ProjectionError::Handler(panic_message(
"PostgreSQL exactly-once handler factory",
panic,
))
})?;
IncarnationHandler::ExactlyOncePostgres(handler)
}
#[cfg(feature = "postgres")]
ProjectionHandlerFactory::GroupedExactlyOncePostgres(factory) => {
let handler = catch_unwind(AssertUnwindSafe(|| factory())).map_err(|panic| {
ProjectionError::Handler(panic_message(
"grouped PostgreSQL exactly-once handler factory",
panic,
))
})?;
IncarnationHandler::GroupedExactlyOncePostgres(handler)
}
};
let token = NEXT_INCARNATION.fetch_add(1, Ordering::Relaxed);
let worker = spawn_source_worker(
token,
Arc::clone(&self.projection.source_provider),
cursor.clone(),
self.projection.query_mode,
myself.clone(),
self.runtime.worker_synchronization.clone(),
);
*self.worker_tracker.lock().await = Some(worker.tracked.clone());
let mut incarnation = Incarnation {
token,
generation,
handler,
worker,
durable_cursor: cursor.clone(),
logical_cursor: cursor,
pending_envelopes: 0,
last_save: Instant::now(),
flush_epoch: 0,
group_buffer: Vec::new(),
group_cursor: None,
group_epoch: 0,
};
send_demand(&mut incarnation).await?;
state.incarnation = Some(incarnation);
state.phase = ActorPhase::Running;
self.publish_status(ProjectionStatus::Running { generation });
Ok(())
}
async fn source_ready(
&self,
myself: &ActorRef<RuntimeMessage>,
state: &mut RuntimeState<Envelope>,
token: u64,
) {
let Some(mut incarnation) = state.incarnation.take() else {
return;
};
if incarnation.token != token || !matches!(state.phase, ActorPhase::Running) {
state.incarnation = Some(incarnation);
return;
}
let event = match incarnation.worker.events.try_recv() {
Ok(event) => event,
Err(mpsc::error::TryRecvError::Empty) => {
state.incarnation = Some(incarnation);
return;
}
Err(mpsc::error::TryRecvError::Disconnected) => {
WorkerEvent::Failed(ProjectionError::Source(
"source worker disconnected without a terminal event".into(),
))
}
};
match event {
WorkerEvent::Envelope(envelope) => {
let result = if self.projection.is_grouped() {
self.buffer_grouped_envelope(myself, &mut incarnation, envelope)
.await
} else {
self.process_envelope(myself, &mut incarnation, &envelope)
.await
.map(|()| true)
};
match result {
Ok(invoked) => {
if invoked {
state.backoff.successful_invocation();
}
if let Err(error) = send_demand(&mut incarnation).await {
state.incarnation = Some(incarnation);
self.infrastructure_failure(myself, state, error).await;
} else {
state.incarnation = Some(incarnation);
}
}
Err(ProcessEnvelopeError::Handler(error)) => {
state.incarnation = Some(incarnation);
self.handler_failure(myself, state, error).await;
}
Err(ProcessEnvelopeError::Infrastructure(error)) => {
state.incarnation = Some(incarnation);
self.infrastructure_failure(myself, state, error).await;
}
}
}
WorkerEvent::Completed => {
if self.projection.is_grouped() && !incarnation.group_buffer.is_empty() {
match self.process_group(&mut incarnation).await {
Ok(()) => state.backoff.successful_invocation(),
Err(ProcessEnvelopeError::Handler(error)) => {
state.incarnation = Some(incarnation);
self.handler_failure(myself, state, error).await;
return;
}
Err(ProcessEnvelopeError::Infrastructure(error)) => {
state.incarnation = Some(incarnation);
self.infrastructure_failure(myself, state, error).await;
return;
}
}
}
if let Err(error) = self.quiesce_incarnation(&mut incarnation).await {
state.incarnation = Some(incarnation);
self.infrastructure_failure(myself, state, error).await;
return;
}
state.incarnation = None;
let terminal = match self.projection.query_mode {
QueryMode::Current => ProjectionStatus::Completed,
QueryMode::Live => ProjectionStatus::Stopped,
};
self.set_terminal(state, terminal);
myself.stop(None);
}
WorkerEvent::Failed(error) => {
state.incarnation = Some(incarnation);
self.infrastructure_failure(myself, state, error).await;
}
}
}
async fn buffer_grouped_envelope(
&self,
myself: &ActorRef<RuntimeMessage>,
incarnation: &mut Incarnation<Envelope>,
envelope: Envelope,
) -> Result<bool, ProcessEnvelopeError> {
let cursor = advance_cursor(
self.projection.source_provider.as_ref(),
incarnation
.group_cursor
.as_ref()
.or(incarnation.logical_cursor.as_ref()),
&envelope,
)
.map_err(ProcessEnvelopeError::Infrastructure)?;
incarnation.group_cursor = Some(cursor);
incarnation.group_buffer.push(envelope);
if incarnation.group_buffer.len() == 1 {
let token = incarnation.token;
let epoch = incarnation.group_epoch;
let duration = self.projection.group.group_after_duration;
std::mem::drop(
myself.send_after(duration, move || RuntimeMessage::GroupDue {
incarnation: token,
epoch,
}),
);
}
if incarnation.group_buffer.len() >= self.projection.group.group_after_envelopes {
self.process_group(incarnation).await?;
Ok(true)
} else {
Ok(false)
}
}
async fn process_group(
&self,
incarnation: &mut Incarnation<Envelope>,
) -> Result<(), ProcessEnvelopeError> {
if incarnation.group_buffer.is_empty() {
return Err(ProcessEnvelopeError::Infrastructure(
ProjectionError::Runtime("group invocation has no buffered envelopes".into()),
));
}
#[cfg(feature = "postgres")]
if self.projection.is_exactly_once_postgres() {
let cursor = incarnation.group_cursor.clone().ok_or_else(|| {
ProcessEnvelopeError::Infrastructure(ProjectionError::Runtime(
"group buffer exists without a logical cursor".into(),
))
})?;
self.process_exactly_once_group(incarnation, cursor).await?;
incarnation.group_buffer.clear();
incarnation.group_cursor = None;
incarnation.group_epoch = incarnation.group_epoch.saturating_add(1);
return Ok(());
}
self.invoke_grouped_handler(incarnation)
.await
.map_err(ProcessEnvelopeError::Handler)?;
let cursor = incarnation.group_cursor.clone().ok_or_else(|| {
ProcessEnvelopeError::Infrastructure(ProjectionError::Runtime(
"group buffer exists without a logical cursor".into(),
))
})?;
self.save(incarnation, cursor)
.await
.map_err(ProcessEnvelopeError::Infrastructure)?;
incarnation.group_buffer.clear();
incarnation.group_cursor = None;
incarnation.group_epoch = incarnation.group_epoch.saturating_add(1);
Ok(())
}
async fn process_envelope(
&self,
myself: &ActorRef<RuntimeMessage>,
incarnation: &mut Incarnation<Envelope>,
envelope: &Envelope,
) -> Result<(), ProcessEnvelopeError> {
#[cfg(feature = "postgres")]
if self.projection.is_exactly_once_postgres() {
let cursor = advance_cursor(
self.projection.source_provider.as_ref(),
incarnation.logical_cursor.as_ref(),
envelope,
)
.map_err(ProcessEnvelopeError::Infrastructure)?;
return self
.process_exactly_once_envelope(incarnation, envelope, cursor)
.await;
}
if self.projection.delivery == super::DeliverySemantics::AtMostOnce {
let cursor = advance_cursor(
self.projection.source_provider.as_ref(),
incarnation.logical_cursor.as_ref(),
envelope,
)
.map_err(ProcessEnvelopeError::Infrastructure)?;
self.save(incarnation, cursor)
.await
.map_err(ProcessEnvelopeError::Infrastructure)?;
}
let skipped = self
.invoke_handler(incarnation, envelope)
.await
.map_err(ProcessEnvelopeError::Handler)?;
if self.projection.delivery == super::DeliverySemantics::AtLeastOnce {
let cursor = advance_cursor(
self.projection.source_provider.as_ref(),
incarnation.logical_cursor.as_ref(),
envelope,
)
.map_err(ProcessEnvelopeError::Infrastructure)?;
incarnation.logical_cursor = Some(cursor);
incarnation.pending_envelopes = incarnation.pending_envelopes.saturating_add(1);
self.maybe_flush_or_schedule(myself, incarnation)
.await
.map_err(ProcessEnvelopeError::Infrastructure)?;
} else if skipped {
}
Ok(())
}
#[cfg(feature = "postgres")]
fn postgres_offset_store(
&self,
) -> Result<&PostgresProjectionOffsetStore, ProcessEnvelopeError> {
self.runtime.offset_store.postgres_store().ok_or_else(|| {
ProcessEnvelopeError::Infrastructure(ProjectionError::invalid_config(
"projection.runtime.offset_store",
"PostgreSQL exactly-once delivery requires PostgresProjectionOffsetStore",
))
})
}
#[cfg(feature = "postgres")]
async fn process_exactly_once_envelope(
&self,
incarnation: &mut Incarnation<Envelope>,
envelope: &Envelope,
cursor: ProjectionCursor,
) -> Result<(), ProcessEnvelopeError> {
let store = self.postgres_offset_store()?;
let (retries, delay, skip_after_exhaustion) =
recovery_settings(self.projection.recovery_strategy);
let mut retry_index = 0_usize;
loop {
let attempt = retry_index.saturating_add(1);
self.observe(|observer| {
observer.before_process(self.projection.projection_id(), envelope, attempt);
});
let mut transaction = store
.begin_exactly_once(
self.projection.projection_id(),
incarnation.generation,
incarnation
.durable_cursor
.as_ref()
.map(ProjectionCursor::position),
)
.await
.map_err(ProcessEnvelopeError::Infrastructure)?;
let IncarnationHandler::ExactlyOncePostgres(handler) = &mut incarnation.handler else {
return Err(ProcessEnvelopeError::Infrastructure(
ProjectionError::Runtime(
"PostgreSQL exactly-once delivery has an incompatible handler".into(),
),
));
};
let invocation = AssertUnwindSafe(handler.process(&mut transaction, envelope))
.catch_unwind()
.await;
let result = match invocation {
Ok(result) => result,
Err(panic) => Err(ProjectionError::Handler(panic_message(
"PostgreSQL exactly-once handler",
panic,
))),
};
match result {
Ok(()) => {
store
.commit_exactly_once(
transaction,
self.projection.projection_id(),
incarnation.generation,
&cursor,
)
.await
.map_err(ProcessEnvelopeError::Infrastructure)?;
self.record_saved_cursor(incarnation, cursor.clone());
self.observe(|observer| {
observer.after_process(self.projection.projection_id(), envelope);
});
return Ok(());
}
Err(error) => {
transaction
.rollback()
.await
.map_err(ProcessEnvelopeError::Infrastructure)?;
self.observe(|observer| {
observer.handler_error(
self.projection.projection_id(),
envelope,
attempt,
&error,
);
});
if retry_index < retries {
tokio::time::sleep(delay).await;
retry_index = retry_index.saturating_add(1);
} else if skip_after_exhaustion {
self.save(incarnation, cursor.clone())
.await
.map_err(ProcessEnvelopeError::Infrastructure)?;
return Ok(());
} else {
return Err(ProcessEnvelopeError::Handler(error));
}
}
}
}
}
#[cfg(feature = "postgres")]
async fn process_exactly_once_group(
&self,
incarnation: &mut Incarnation<Envelope>,
cursor: ProjectionCursor,
) -> Result<(), ProcessEnvelopeError> {
let store = self.postgres_offset_store()?;
let (retries, delay, skip_after_exhaustion) =
recovery_settings(self.projection.recovery_strategy);
let mut retry_index = 0_usize;
loop {
let attempt = retry_index.saturating_add(1);
self.observe(|observer| {
observer.before_process_group(
self.projection.projection_id(),
&incarnation.group_buffer,
attempt,
);
});
let mut transaction = store
.begin_exactly_once(
self.projection.projection_id(),
incarnation.generation,
incarnation
.durable_cursor
.as_ref()
.map(ProjectionCursor::position),
)
.await
.map_err(ProcessEnvelopeError::Infrastructure)?;
let IncarnationHandler::GroupedExactlyOncePostgres(handler) = &mut incarnation.handler
else {
return Err(ProcessEnvelopeError::Infrastructure(
ProjectionError::Runtime(
"grouped PostgreSQL exactly-once delivery has an incompatible handler"
.into(),
),
));
};
let invocation =
AssertUnwindSafe(handler.process(&mut transaction, &incarnation.group_buffer))
.catch_unwind()
.await;
let result = match invocation {
Ok(result) => result,
Err(panic) => Err(ProjectionError::Handler(panic_message(
"grouped PostgreSQL exactly-once handler",
panic,
))),
};
match result {
Ok(()) => {
store
.commit_exactly_once(
transaction,
self.projection.projection_id(),
incarnation.generation,
&cursor,
)
.await
.map_err(ProcessEnvelopeError::Infrastructure)?;
self.record_saved_cursor(incarnation, cursor.clone());
self.observe(|observer| {
observer.after_process_group(
self.projection.projection_id(),
&incarnation.group_buffer,
);
});
return Ok(());
}
Err(error) => {
transaction
.rollback()
.await
.map_err(ProcessEnvelopeError::Infrastructure)?;
self.observe(|observer| {
observer.group_handler_error(
self.projection.projection_id(),
&incarnation.group_buffer,
attempt,
&error,
);
});
if retry_index < retries {
tokio::time::sleep(delay).await;
retry_index = retry_index.saturating_add(1);
} else if skip_after_exhaustion {
self.save(incarnation, cursor.clone())
.await
.map_err(ProcessEnvelopeError::Infrastructure)?;
return Ok(());
} else {
return Err(ProcessEnvelopeError::Handler(error));
}
}
}
}
}
async fn invoke_handler(
&self,
incarnation: &mut Incarnation<Envelope>,
envelope: &Envelope,
) -> ProjectionResult<bool> {
let (retries, delay, skip_after_exhaustion) = match self.projection.recovery_strategy {
RecoveryStrategy::Fail => (0, Duration::ZERO, false),
RecoveryStrategy::Skip => (0, Duration::ZERO, true),
RecoveryStrategy::RetryAndFail { retries, delay } => (retries, delay, false),
RecoveryStrategy::RetryAndSkip { retries, delay } => (retries, delay, true),
};
let mut retry_index = 0_usize;
loop {
let attempt = retry_index.saturating_add(1);
self.observe(|observer| {
observer.before_process(self.projection.projection_id(), envelope, attempt);
});
let IncarnationHandler::Single(handler) = &mut incarnation.handler else {
return Err(ProjectionError::Runtime(
"single-envelope delivery has a grouped handler".into(),
));
};
let invocation = AssertUnwindSafe(handler.process(envelope))
.catch_unwind()
.await;
let result = match invocation {
Ok(result) => result,
Err(panic) => Err(ProjectionError::Handler(panic_message("handler", panic))),
};
match result {
Ok(()) => {
self.observe(|observer| {
observer.after_process(self.projection.projection_id(), envelope);
});
return Ok(false);
}
Err(error) => {
self.observe(|observer| {
observer.handler_error(
self.projection.projection_id(),
envelope,
attempt,
&error,
);
});
if retry_index < retries {
tokio::time::sleep(delay).await;
retry_index = retry_index.saturating_add(1);
} else if skip_after_exhaustion {
return Ok(true);
} else {
return Err(error);
}
}
}
}
}
async fn invoke_grouped_handler(
&self,
incarnation: &mut Incarnation<Envelope>,
) -> ProjectionResult<bool> {
let (retries, delay, skip_after_exhaustion) = match self.projection.recovery_strategy {
RecoveryStrategy::Fail => (0, Duration::ZERO, false),
RecoveryStrategy::Skip => (0, Duration::ZERO, true),
RecoveryStrategy::RetryAndFail { retries, delay } => (retries, delay, false),
RecoveryStrategy::RetryAndSkip { retries, delay } => (retries, delay, true),
};
let mut retry_index = 0_usize;
loop {
let attempt = retry_index.saturating_add(1);
self.observe(|observer| {
observer.before_process_group(
self.projection.projection_id(),
&incarnation.group_buffer,
attempt,
);
});
let IncarnationHandler::Grouped(handler) = &mut incarnation.handler else {
return Err(ProjectionError::Runtime(
"grouped delivery has a single-envelope handler".into(),
));
};
let invocation = AssertUnwindSafe(handler.process(&incarnation.group_buffer))
.catch_unwind()
.await;
let result = match invocation {
Ok(result) => result,
Err(panic) => Err(ProjectionError::Handler(panic_message(
"grouped handler",
panic,
))),
};
match result {
Ok(()) => {
self.observe(|observer| {
observer.after_process_group(
self.projection.projection_id(),
&incarnation.group_buffer,
);
});
return Ok(false);
}
Err(error) => {
self.observe(|observer| {
observer.group_handler_error(
self.projection.projection_id(),
&incarnation.group_buffer,
attempt,
&error,
);
});
if retry_index < retries {
tokio::time::sleep(delay).await;
retry_index = retry_index.saturating_add(1);
} else if skip_after_exhaustion {
return Ok(true);
} else {
return Err(error);
}
}
}
}
}
async fn maybe_flush_or_schedule(
&self,
myself: &ActorRef<RuntimeMessage>,
incarnation: &mut Incarnation<Envelope>,
) -> ProjectionResult<()> {
let save_config = self.projection.save_offset;
let elapsed = incarnation.last_save.elapsed();
if incarnation.pending_envelopes >= save_config.after_envelopes
|| elapsed >= save_config.after_duration
{
return self.flush(incarnation).await;
}
if incarnation.pending_envelopes == 1 {
let epoch = incarnation.flush_epoch;
let token = incarnation.token;
let remaining = save_config.after_duration.saturating_sub(elapsed);
std::mem::drop(
myself.send_after(remaining, move || RuntimeMessage::FlushDue {
incarnation: token,
epoch,
}),
);
}
Ok(())
}
async fn flush_due(
&self,
myself: &ActorRef<RuntimeMessage>,
state: &mut RuntimeState<Envelope>,
token: u64,
epoch: u64,
) {
let Some(mut incarnation) = state.incarnation.take() else {
return;
};
if incarnation.token != token
|| incarnation.flush_epoch != epoch
|| incarnation.pending_envelopes == 0
{
state.incarnation = Some(incarnation);
return;
}
match self.flush(&mut incarnation).await {
Ok(()) => state.incarnation = Some(incarnation),
Err(error) => {
state.incarnation = Some(incarnation);
self.infrastructure_failure(myself, state, error).await;
}
}
}
async fn group_due(
&self,
myself: &ActorRef<RuntimeMessage>,
state: &mut RuntimeState<Envelope>,
token: u64,
epoch: u64,
) {
let Some(mut incarnation) = state.incarnation.take() else {
return;
};
if incarnation.token != token
|| incarnation.group_epoch != epoch
|| incarnation.group_buffer.is_empty()
|| !matches!(state.phase, ActorPhase::Running)
{
state.incarnation = Some(incarnation);
return;
}
match self.process_group(&mut incarnation).await {
Ok(()) => {
state.backoff.successful_invocation();
state.incarnation = Some(incarnation);
}
Err(ProcessEnvelopeError::Handler(error)) => {
state.incarnation = Some(incarnation);
self.handler_failure(myself, state, error).await;
}
Err(ProcessEnvelopeError::Infrastructure(error)) => {
state.incarnation = Some(incarnation);
self.infrastructure_failure(myself, state, error).await;
}
}
}
async fn flush(&self, incarnation: &mut Incarnation<Envelope>) -> ProjectionResult<()> {
if incarnation.pending_envelopes == 0 {
return Ok(());
}
let cursor = incarnation.logical_cursor.clone().ok_or_else(|| {
ProjectionError::Runtime("pending envelopes exist without a logical cursor".into())
})?;
self.save(incarnation, cursor).await
}
async fn save(
&self,
incarnation: &mut Incarnation<Envelope>,
cursor: ProjectionCursor,
) -> ProjectionResult<()> {
self.runtime
.offset_store
.save(
self.projection.projection_id(),
incarnation.generation,
incarnation
.durable_cursor
.as_ref()
.map(ProjectionCursor::position),
cursor.clone(),
)
.await?;
self.record_saved_cursor(incarnation, cursor);
Ok(())
}
fn record_saved_cursor(
&self,
incarnation: &mut Incarnation<Envelope>,
cursor: ProjectionCursor,
) {
incarnation.durable_cursor = Some(cursor.clone());
incarnation.logical_cursor = Some(cursor.clone());
incarnation.pending_envelopes = 0;
incarnation.last_save = Instant::now();
incarnation.flush_epoch = incarnation.flush_epoch.saturating_add(1);
self.observe(|observer| {
observer.offset_progress(self.projection.projection_id(), &cursor);
});
}
async fn handler_failure(
&self,
myself: &ActorRef<RuntimeMessage>,
state: &mut RuntimeState<Envelope>,
error: ProjectionError,
) {
self.schedule_restart(myself, state, error).await;
}
async fn infrastructure_failure(
&self,
myself: &ActorRef<RuntimeMessage>,
state: &mut RuntimeState<Envelope>,
error: ProjectionError,
) {
if error.is_fatal() {
self.stop_worker_without_flush(state).await;
self.set_terminal(state, ProjectionStatus::Failed(error));
myself.stop(None);
} else {
self.schedule_restart(myself, state, error).await;
}
}
async fn schedule_restart(
&self,
myself: &ActorRef<RuntimeMessage>,
state: &mut RuntimeState<Envelope>,
error: ProjectionError,
) {
self.stop_worker_without_flush(state).await;
let Some((failure_index, attempt)) = state
.backoff
.reserve_restart(&self.projection.restart_backoff, Instant::now())
else {
self.set_terminal(state, ProjectionStatus::Failed(error));
myself.stop(None);
return;
};
let sample = catch_unwind(AssertUnwindSafe(|| {
self.runtime.backoff_rng.next_fraction()
}))
.unwrap_or(0.0);
let delay = restart_delay(&self.projection.restart_backoff, failure_index, sample);
state.restart_epoch = state.restart_epoch.saturating_add(1);
let restart_epoch = state.restart_epoch;
state.phase = ActorPhase::Restarting;
self.publish_status(ProjectionStatus::Restarting {
attempt,
delay,
error,
});
if delay.is_zero() {
if myself
.cast(RuntimeMessage::Restart {
epoch: restart_epoch,
})
.is_err()
{
self.set_terminal(
state,
ProjectionStatus::Failed(ProjectionError::Runtime(
"failed to enqueue immediate projection restart".into(),
)),
);
myself.stop(None);
}
} else {
std::mem::drop(myself.send_after(delay, move || RuntimeMessage::Restart {
epoch: restart_epoch,
}));
}
}
async fn stop_worker_without_flush(&self, state: &mut RuntimeState<Envelope>) {
if let Some(mut incarnation) = state.incarnation.take() {
let _ = stop_source_worker(&mut incarnation.worker).await;
}
}
async fn quiesce(&self, state: &mut RuntimeState<Envelope>) -> ProjectionResult<()> {
let Some(mut incarnation) = state.incarnation.take() else {
return Ok(());
};
self.quiesce_incarnation(&mut incarnation).await
}
async fn quiesce_incarnation(
&self,
incarnation: &mut Incarnation<Envelope>,
) -> ProjectionResult<()> {
cancel_source_worker(&incarnation.worker);
incarnation.group_buffer.clear();
incarnation.group_cursor = None;
incarnation.group_epoch = incarnation.group_epoch.saturating_add(1);
let flush_result = self.flush(incarnation).await;
let join_result = join_source_worker(&mut incarnation.worker).await;
match (flush_result, join_result) {
(Ok(()), join_result) => join_result,
(Err(flush_error), Ok(())) => Err(flush_error),
(Err(flush_error), Err(join_error)) => {
if join_error.is_fatal() && !flush_error.is_fatal() {
Err(join_error)
} else {
Err(flush_error)
}
}
}
}
async fn clear_offset(
&self,
myself: &ActorRef<RuntimeMessage>,
state: &mut RuntimeState<Envelope>,
) -> ProjectionResult<()> {
state.restart_epoch = state.restart_epoch.saturating_add(1);
self.quiesce(state).await?;
let generation = self
.runtime
.offset_store
.claim(self.projection.projection_id())
.await?;
self.runtime
.offset_store
.clear(self.projection.projection_id(), generation)
.await?;
if !self
.runtime
.offset_store
.load_paused(self.projection.projection_id())
.await?
{
self.start_incarnation(myself, state, Some(generation))
.await?;
} else {
state.phase = ActorPhase::Paused;
self.publish_status(ProjectionStatus::Paused);
}
Ok(())
}
async fn update_offset(
&self,
myself: &ActorRef<RuntimeMessage>,
state: &mut RuntimeState<Envelope>,
cursor: ProjectionCursor,
) -> ProjectionResult<()> {
state.restart_epoch = state.restart_epoch.saturating_add(1);
cursor.ensure_manifest(self.projection.source_provider.manifest())?;
self.quiesce(state).await?;
let generation = self
.runtime
.offset_store
.claim(self.projection.projection_id())
.await?;
let previous = self
.runtime
.offset_store
.load(self.projection.projection_id())
.await?;
self.runtime
.offset_store
.save(
self.projection.projection_id(),
generation,
previous.as_ref().map(ProjectionCursor::position),
cursor,
)
.await?;
if !self
.runtime
.offset_store
.load_paused(self.projection.projection_id())
.await?
{
self.start_incarnation(myself, state, Some(generation))
.await?;
} else {
state.phase = ActorPhase::Paused;
self.publish_status(ProjectionStatus::Paused);
}
Ok(())
}
async fn pause(&self, state: &mut RuntimeState<Envelope>) -> ProjectionResult<()> {
if matches!(state.phase, ActorPhase::Paused) {
return Ok(());
}
state.restart_epoch = state.restart_epoch.saturating_add(1);
self.quiesce(state).await?;
self.runtime
.offset_store
.set_paused(self.projection.projection_id(), true)
.await?;
state.phase = ActorPhase::Paused;
self.publish_status(ProjectionStatus::Paused);
Ok(())
}
async fn resume(
&self,
myself: &ActorRef<RuntimeMessage>,
state: &mut RuntimeState<Envelope>,
) -> ProjectionResult<()> {
state.restart_epoch = state.restart_epoch.saturating_add(1);
self.runtime
.offset_store
.set_paused(self.projection.projection_id(), false)
.await?;
if matches!(state.phase, ActorPhase::Paused) {
self.start_incarnation(myself, state, None).await?;
}
Ok(())
}
async fn stop_acknowledged(&self, state: &mut RuntimeState<Envelope>) -> ProjectionResult<()> {
self.quiesce(state).await
}
fn set_terminal(&self, state: &mut RuntimeState<Envelope>, status: ProjectionStatus) {
state.phase = ActorPhase::Terminal;
state.terminal_status_set = true;
self.publish_status(status);
}
fn publish_status(&self, status: ProjectionStatus) {
self.status.send_replace(status.clone());
self.observe(|observer| {
observer.status_changed(self.projection.projection_id(), &status);
});
}
fn observe(&self, callback: impl FnOnce(&dyn StatusObserver<Envelope>)) {
if catch_unwind(AssertUnwindSafe(|| {
callback(self.projection.status_observer.as_ref());
}))
.is_err()
{
tracing::warn!(
projection_id = %self.projection.projection_id(),
"projection status observer panicked; callback was ignored"
);
}
}
}
fn spawn_source_worker<Envelope>(
incarnation: u64,
provider: Arc<dyn SourceProvider<Envelope>>,
stored_cursor: Option<ProjectionCursor>,
mode: QueryMode,
actor: ActorRef<RuntimeMessage>,
synchronization: Option<WorkerSynchronizationProbe>,
) -> SourceWorker<Envelope>
where
Envelope: Send + Sync + 'static,
{
let (cancel_sender, cancel_receiver) = watch::channel(false);
let (demand_sender, demand_receiver) = mpsc::channel(1);
let (event_sender, event_receiver) = mpsc::channel(1);
let join = tokio::spawn(source_worker(
incarnation,
provider,
stored_cursor,
mode,
actor,
cancel_receiver,
demand_receiver,
event_sender,
synchronization.clone(),
));
let tracked = TrackedSourceWorker {
cancellation: cancel_sender,
join: Arc::new(AsyncMutex::new(Some(join))),
};
SourceWorker {
demand: demand_sender,
events: event_receiver,
tracked,
synchronization,
}
}
#[allow(clippy::too_many_arguments)]
async fn source_worker<Envelope>(
incarnation: u64,
provider: Arc<dyn SourceProvider<Envelope>>,
stored_cursor: Option<ProjectionCursor>,
mode: QueryMode,
actor: ActorRef<RuntimeMessage>,
mut cancellation: watch::Receiver<bool>,
mut demand: mpsc::Receiver<Option<ProjectionCursor>>,
events: mpsc::Sender<WorkerEvent<Envelope>>,
synchronization: Option<WorkerSynchronizationProbe>,
) where
Envelope: Send + Sync + 'static,
{
source_worker_inner(
incarnation,
provider,
stored_cursor,
mode,
actor,
&mut cancellation,
&mut demand,
&events,
synchronization.as_ref(),
)
.await;
if let Some(synchronization) = synchronization {
synchronization.inner.exit_started.add_permits(1);
if let Ok(permit) = synchronization.inner.allow_exit.acquire().await {
permit.forget();
}
synchronization.inner.exited.store(true, Ordering::Release);
}
}
#[allow(clippy::too_many_arguments)]
async fn source_worker_inner<Envelope>(
incarnation: u64,
provider: Arc<dyn SourceProvider<Envelope>>,
stored_cursor: Option<ProjectionCursor>,
mode: QueryMode,
actor: ActorRef<RuntimeMessage>,
cancellation: &mut watch::Receiver<bool>,
demand: &mut mpsc::Receiver<Option<ProjectionCursor>>,
events: &mpsc::Sender<WorkerEvent<Envelope>>,
synchronization: Option<&WorkerSynchronizationProbe>,
) where
Envelope: Send + Sync + 'static,
{
let source_future =
AssertUnwindSafe(provider.source_from(stored_cursor.as_ref(), mode)).catch_unwind();
let source_result = tokio::select! {
_ = cancelled(cancellation) => return,
result = source_future => match result {
Ok(result) => result,
Err(panic) => Err(ProjectionError::Source(panic_message("source factory", panic))),
},
};
let mut source = match source_result {
Ok(source) => source,
Err(error) => {
notify_actor(
incarnation,
&actor,
cancellation,
events,
synchronization,
WorkerEvent::Failed(error),
)
.await;
return;
}
};
loop {
let logical_cursor = tokio::select! {
_ = cancelled(cancellation) => return,
next = demand.recv() => match next {
Some(cursor) => cursor,
None => return,
},
};
loop {
let next = tokio::select! {
_ = cancelled(cancellation) => return,
next = AssertUnwindSafe(source.next()).catch_unwind() => match next {
Ok(next) => next,
Err(panic) => {
notify_actor(
incarnation,
&actor,
cancellation,
events,
synchronization,
WorkerEvent::Failed(ProjectionError::Source(
panic_message("source stream", panic),
)),
)
.await;
return;
}
},
};
match next {
Some(Ok(envelope)) => {
let allowed = catch_unwind(AssertUnwindSafe(|| {
resume_allows(provider.as_ref(), logical_cursor.as_ref(), &envelope)
}));
match allowed {
Ok(Ok(true)) => {
notify_actor(
incarnation,
&actor,
cancellation,
events,
synchronization,
WorkerEvent::Envelope(envelope),
)
.await;
break;
}
Ok(Ok(false)) => {}
Ok(Err(error)) => {
notify_actor(
incarnation,
&actor,
cancellation,
events,
synchronization,
WorkerEvent::Failed(error),
)
.await;
return;
}
Err(panic) => {
notify_actor(
incarnation,
&actor,
cancellation,
events,
synchronization,
WorkerEvent::Failed(ProjectionError::Source(panic_message(
"source cursor extraction",
panic,
))),
)
.await;
return;
}
}
}
Some(Err(error)) => {
notify_actor(
incarnation,
&actor,
cancellation,
events,
synchronization,
WorkerEvent::Failed(error),
)
.await;
return;
}
None => {
notify_actor(
incarnation,
&actor,
cancellation,
events,
synchronization,
WorkerEvent::Completed,
)
.await;
return;
}
}
}
}
}
async fn notify_actor<Envelope>(
incarnation: u64,
actor: &ActorRef<RuntimeMessage>,
cancellation: &mut watch::Receiver<bool>,
events: &mpsc::Sender<WorkerEvent<Envelope>>,
synchronization: Option<&WorkerSynchronizationProbe>,
event: WorkerEvent<Envelope>,
) {
let sent = tokio::select! {
_ = cancelled(cancellation) => false,
result = events.send(event) => result.is_ok(),
};
if sent {
if let Some(synchronization) = synchronization {
synchronization.delivered();
}
let _ = actor.cast(RuntimeMessage::SourceReady(incarnation));
}
}
async fn cancelled(cancellation: &mut watch::Receiver<bool>) {
if *cancellation.borrow() {
return;
}
let _ = cancellation.changed().await;
}
async fn send_demand<Envelope>(incarnation: &mut Incarnation<Envelope>) -> ProjectionResult<()> {
let observed_deliveries = incarnation
.worker
.synchronization
.as_ref()
.map(WorkerSynchronizationProbe::delivery_count);
incarnation
.worker
.demand
.send(
incarnation
.group_cursor
.clone()
.or_else(|| incarnation.logical_cursor.clone()),
)
.await
.map_err(|_| {
ProjectionError::Source("source worker stopped before accepting demand".into())
})?;
if let (Some(synchronization), Some(observed)) =
(&incarnation.worker.synchronization, observed_deliveries)
{
synchronization.wait_for_delivery_after(observed).await?;
}
Ok(())
}
async fn stop_source_worker<Envelope>(worker: &mut SourceWorker<Envelope>) -> ProjectionResult<()> {
cancel_source_worker(worker);
join_source_worker(worker).await
}
async fn stop_tracked_source_worker(worker: &TrackedSourceWorker) -> ProjectionResult<()> {
let _ = worker.cancellation.send(true);
if let Some(join) = worker.join.lock().await.take() {
join.await.map_err(|error| {
ProjectionError::Runtime(format!("source worker join failed: {error}"))
})?;
}
Ok(())
}
fn cancel_source_worker<Envelope>(worker: &SourceWorker<Envelope>) {
let _ = worker.tracked.cancellation.send(true);
}
async fn join_source_worker<Envelope>(worker: &mut SourceWorker<Envelope>) -> ProjectionResult<()> {
if let Some(join) = worker.tracked.join.lock().await.take() {
join.await.map_err(|error| {
ProjectionError::Runtime(format!("source worker join failed: {error}"))
})?;
}
Ok(())
}
pub(crate) struct ProjectionRegistryLease {
projection_id: ProjectionId,
}
impl Drop for ProjectionRegistryLease {
fn drop(&mut self) {
release_projection(&self.projection_id);
}
}
fn reserve_projection(projection_id: &ProjectionId) -> ProjectionResult<ProjectionRegistryLease> {
let mut leases = reserve_projections(std::slice::from_ref(projection_id))?;
Ok(leases
.pop()
.expect("one projection id produces one registry lease"))
}
pub(crate) fn reserve_projections(
projection_ids: &[ProjectionId],
) -> ProjectionResult<Vec<ProjectionRegistryLease>> {
let mut registry = RUNNING_PROJECTIONS
.lock()
.map_err(|_| ProjectionError::Runtime("projection registry lock poisoned".into()))?;
let mut reservation = HashSet::with_capacity(projection_ids.len());
for projection_id in projection_ids {
if registry.contains(projection_id) || !reservation.insert(projection_id.clone()) {
return Err(ProjectionError::AlreadyRunning {
projection_id: projection_id.clone(),
});
}
}
registry.extend(reservation);
Ok(projection_ids
.iter()
.cloned()
.map(|projection_id| ProjectionRegistryLease { projection_id })
.collect())
}
fn release_projection(projection_id: &ProjectionId) {
match RUNNING_PROJECTIONS.lock() {
Ok(mut registry) => {
registry.remove(projection_id);
}
Err(poisoned) => {
poisoned.into_inner().remove(projection_id);
}
}
}
fn boxed_runtime(message: String) -> ActorProcessingErr {
Box::new(ProjectionError::Runtime(message))
}
fn panic_message(context: &str, panic: Box<dyn Any + Send + 'static>) -> String {
let message = if let Some(message) = panic.downcast_ref::<&str>() {
(*message).to_owned()
} else if let Some(message) = panic.downcast_ref::<String>() {
message.clone()
} else {
"non-string panic".to_owned()
};
format!("{context} panicked: {message}")
}
pub struct ProjectionHandle {
projection_id: ProjectionId,
control: ActorRef<RuntimeMessage>,
status: watch::Receiver<ProjectionStatus>,
status_sender: watch::Sender<ProjectionStatus>,
stop_requested: AtomicBool,
worker_tracker: Arc<AsyncMutex<Option<TrackedSourceWorker>>>,
}
impl ProjectionHandle {
#[must_use]
pub fn projection_id(&self) -> &ProjectionId {
&self.projection_id
}
#[must_use]
pub fn status(&self) -> ProjectionStatus {
self.status.borrow().clone()
}
#[must_use]
pub fn status_receiver(&self) -> watch::Receiver<ProjectionStatus> {
self.status.clone()
}
pub async fn wait(&self) -> ProjectionResult<ProjectionStatus> {
let _ = self.control.wait(None).await;
terminal_result(self.status())
}
pub async fn get_offset(&self) -> ProjectionResult<Option<ProjectionCursor>> {
self.request(RuntimeMessage::GetOffset, "get offset")
.await?
}
pub async fn clear_offset(&self) -> ProjectionResult<()> {
self.request(RuntimeMessage::ClearOffset, "clear offset")
.await?
}
pub async fn update_offset(&self, cursor: ProjectionCursor) -> ProjectionResult<()> {
self.request(
|reply| RuntimeMessage::UpdateOffset { cursor, reply },
"update offset",
)
.await?
}
pub async fn pause(&self) -> ProjectionResult<()> {
self.request(RuntimeMessage::Pause, "pause").await?
}
pub async fn resume(&self) -> ProjectionResult<()> {
self.request(RuntimeMessage::Resume, "resume").await?
}
pub async fn stop_and_wait(&self) -> ProjectionResult<()> {
let response = match self.signal_stop() {
Ok(receiver) => receiver.await.map_err(|_| {
ProjectionError::Runtime("projection stopped before answering stop".into())
}),
Err(error) => Err(error),
};
let _ = self.control.wait(None).await;
match response {
Ok(result) => result,
Err(send_error) => match self.status() {
ProjectionStatus::Failed(error) => Err(error),
status if status.is_terminal() => Ok(()),
_ => Err(send_error),
},
}
}
fn signal_stop(&self) -> ProjectionResult<oneshot::Receiver<ProjectionResult<()>>> {
self.stop_requested.store(true, Ordering::Release);
let (sender, receiver) = oneshot::channel();
self.control
.cast(RuntimeMessage::Stop(sender))
.map_err(|error| {
ProjectionError::Runtime(format!("projection stop send failed: {error}"))
})?;
Ok(receiver)
}
pub async fn abort_and_wait(&self) -> ProjectionResult<()> {
self.stop_requested.store(true, Ordering::Release);
self.control.kill_and_wait(None).await.map_err(|error| {
ProjectionError::Runtime(format!("projection abort failed: {error}"))
})?;
if let Some(worker) = self.worker_tracker.lock().await.take() {
stop_tracked_source_worker(&worker).await?;
}
self.status_sender.send_replace(ProjectionStatus::Stopped);
Ok(())
}
async fn request<T>(
&self,
build: impl FnOnce(oneshot::Sender<T>) -> RuntimeMessage,
operation: &str,
) -> ProjectionResult<T> {
let (sender, receiver) = oneshot::channel();
self.control.cast(build(sender)).map_err(|error| {
ProjectionError::Runtime(format!("projection {operation} send failed: {error}"))
})?;
receiver.await.map_err(|_| {
ProjectionError::Runtime(format!("projection stopped before answering {operation}"))
})
}
}
impl Drop for ProjectionHandle {
fn drop(&mut self) {
if !self.stop_requested.swap(true, Ordering::AcqRel) {
let _ = self.control.cast(RuntimeMessage::BestEffortStop);
}
}
}
fn terminal_result(status: ProjectionStatus) -> ProjectionResult<ProjectionStatus> {
match status {
ProjectionStatus::Failed(error) => Err(error),
status => Ok(status),
}
}
#[cfg(test)]
mod tests {
use std::{
collections::BTreeMap,
pin::Pin,
sync::{
Mutex,
atomic::{AtomicBool, AtomicUsize, Ordering},
},
task::{Context, Poll},
};
use futures_core::Stream;
use tokio::sync::Semaphore;
use super::*;
use crate::{
PersistenceId, SequenceNr,
projection::{
BoundaryEvent, CursorPosition, DeduplicationBoundary, GroupConfig, GroupedHandler,
MemoryProjectionOffsetStore, Projection, ProjectionSource, SaveOffsetConfig,
TestEnvelope, TestSourceProvider,
},
};
static NEXT_TEST_ID: AtomicUsize = AtomicUsize::new(1);
#[derive(Clone)]
struct HandlerGate {
payload: i32,
started: Arc<Semaphore>,
release: Arc<Semaphore>,
}
impl HandlerGate {
fn new(payload: i32) -> Self {
Self {
payload,
started: Arc::new(Semaphore::new(0)),
release: Arc::new(Semaphore::new(0)),
}
}
async fn wait_started(&self) {
if let Ok(permit) = Arc::clone(&self.started).acquire_owned().await {
permit.forget();
}
}
fn release(&self) {
self.release.add_permits(1);
}
}
struct ScriptedHandler {
invocations: Arc<Mutex<Vec<i32>>>,
completed: Arc<Mutex<Vec<i32>>>,
failures: Arc<Mutex<BTreeMap<i32, usize>>>,
gate: Option<HandlerGate>,
}
#[async_trait::async_trait]
impl Handler<TestEnvelope<i32>> for ScriptedHandler {
async fn process(&mut self, envelope: &TestEnvelope<i32>) -> ProjectionResult<()> {
lock_test(&self.invocations)?.push(envelope.payload);
if let Some(gate) = &self.gate
&& gate.payload == envelope.payload
{
gate.started.add_permits(1);
Arc::clone(&gate.release)
.acquire_owned()
.await
.map_err(|_| ProjectionError::Handler("test handler gate closed".into()))?
.forget();
}
let mut failures = lock_test(&self.failures)?;
let remaining = failures.entry(envelope.payload).or_default();
if *remaining > 0 {
*remaining -= 1;
return Err(ProjectionError::Handler(format!(
"scripted failure for {}",
envelope.payload
)));
}
drop(failures);
lock_test(&self.completed)?.push(envelope.payload);
Ok(())
}
}
struct ScriptedGroupedHandler {
invocations: Arc<Mutex<Vec<Vec<i32>>>>,
completed: Arc<Mutex<Vec<Vec<i32>>>>,
failures_remaining: Arc<AtomicUsize>,
invocation_signal: Arc<Semaphore>,
}
#[async_trait::async_trait]
impl GroupedHandler<TestEnvelope<i32>> for ScriptedGroupedHandler {
async fn process(&mut self, envelopes: &[TestEnvelope<i32>]) -> ProjectionResult<()> {
let group = envelopes
.iter()
.map(|envelope| envelope.payload)
.collect::<Vec<_>>();
lock_test(&self.invocations)?.push(group.clone());
self.invocation_signal.add_permits(1);
let failed = self
.failures_remaining
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
(remaining > 0).then(|| remaining - 1)
})
.is_ok();
if failed {
return Err(ProjectionError::Handler(format!(
"scripted grouped failure for {group:?}"
)));
}
lock_test(&self.completed)?.push(group);
Ok(())
}
}
fn grouped_factory(
invocations: Arc<Mutex<Vec<Vec<i32>>>>,
completed: Arc<Mutex<Vec<Vec<i32>>>>,
failures_remaining: Arc<AtomicUsize>,
invocation_signal: Arc<Semaphore>,
factories: Arc<AtomicUsize>,
) -> impl Fn() -> ScriptedGroupedHandler + Send + Sync + 'static {
move || {
factories.fetch_add(1, Ordering::Relaxed);
ScriptedGroupedHandler {
invocations: Arc::clone(&invocations),
completed: Arc::clone(&completed),
failures_remaining: Arc::clone(&failures_remaining),
invocation_signal: Arc::clone(&invocation_signal),
}
}
}
async fn wait_for_group_invocations(signal: &Arc<Semaphore>, count: u32) {
tokio::time::timeout(
Duration::from_secs(30),
Arc::clone(signal).acquire_many_owned(count),
)
.await
.expect("grouped handler invocation timed out")
.expect("grouped handler invocation signal remains open")
.forget();
}
#[derive(Clone)]
struct RecordingOffsetStore {
inner: MemoryProjectionOffsetStore,
saves: Arc<Mutex<Vec<ProjectionCursor>>>,
save_signal: Arc<Semaphore>,
}
impl Default for RecordingOffsetStore {
fn default() -> Self {
Self {
inner: MemoryProjectionOffsetStore::new(),
saves: Arc::new(Mutex::new(Vec::new())),
save_signal: Arc::new(Semaphore::new(0)),
}
}
}
impl RecordingOffsetStore {
async fn wait_for_saves(&self, count: u32) {
tokio::time::timeout(
Duration::from_secs(30),
Arc::clone(&self.save_signal).acquire_many_owned(count),
)
.await
.expect("projection cursor save timed out")
.expect("projection cursor save signal remains open")
.forget();
}
}
#[async_trait::async_trait]
impl ProjectionOffsetStore for RecordingOffsetStore {
async fn claim(
&self,
projection_id: &ProjectionId,
) -> ProjectionResult<ProjectionGeneration> {
self.inner.claim(projection_id).await
}
async fn load(
&self,
projection_id: &ProjectionId,
) -> ProjectionResult<Option<ProjectionCursor>> {
self.inner.load(projection_id).await
}
async fn save(
&self,
projection_id: &ProjectionId,
generation: ProjectionGeneration,
expected_position: Option<&CursorPosition>,
cursor: ProjectionCursor,
) -> ProjectionResult<()> {
self.inner
.save(projection_id, generation, expected_position, cursor.clone())
.await?;
lock_test(&self.saves)?.push(cursor);
self.save_signal.add_permits(1);
Ok(())
}
async fn clear(
&self,
projection_id: &ProjectionId,
generation: ProjectionGeneration,
) -> ProjectionResult<()> {
self.inner.clear(projection_id, generation).await
}
async fn load_paused(&self, projection_id: &ProjectionId) -> ProjectionResult<bool> {
self.inner.load_paused(projection_id).await
}
async fn set_paused(
&self,
projection_id: &ProjectionId,
paused: bool,
) -> ProjectionResult<()> {
self.inner.set_paused(projection_id, paused).await
}
}
#[derive(Clone)]
struct CountingCursorProvider {
inner: TestSourceProvider<i32>,
extractions: Arc<AtomicUsize>,
extraction_signal: watch::Sender<usize>,
}
impl CountingCursorProvider {
fn new(inner: TestSourceProvider<i32>) -> Self {
let (extraction_signal, _receiver) = watch::channel(0);
Self {
inner,
extractions: Arc::new(AtomicUsize::new(0)),
extraction_signal,
}
}
async fn wait_for_extractions(&self, expected: usize) {
let mut receiver = self.extraction_signal.subscribe();
tokio::time::timeout(Duration::from_secs(30), async {
while *receiver.borrow_and_update() < expected {
receiver
.changed()
.await
.expect("cursor extraction signal remains open");
}
})
.await
.expect("cursor extraction timed out");
}
fn extraction_count(&self) -> usize {
self.extractions.load(Ordering::Acquire)
}
}
#[async_trait::async_trait]
impl SourceProvider<TestEnvelope<i32>> for CountingCursorProvider {
fn manifest(&self) -> &super::super::ProviderManifest {
self.inner.manifest()
}
async fn source_from(
&self,
cursor: Option<&ProjectionCursor>,
mode: QueryMode,
) -> ProjectionResult<ProjectionSource<TestEnvelope<i32>>> {
self.inner.source_from(cursor, mode).await
}
fn extract_cursor(
&self,
envelope: &TestEnvelope<i32>,
) -> ProjectionResult<ProjectionCursor> {
let cursor = self.inner.extract_cursor(envelope)?;
let count = self.extractions.fetch_add(1, Ordering::AcqRel) + 1;
self.extraction_signal.send_replace(count);
Ok(cursor)
}
fn compare_positions(
&self,
left: &CursorPosition,
right: &CursorPosition,
) -> ProjectionResult<std::cmp::Ordering> {
self.inner.compare_positions(left, right)
}
}
fn lock_test<T>(mutex: &Mutex<T>) -> ProjectionResult<std::sync::MutexGuard<'_, T>> {
mutex
.lock()
.map_err(|_| ProjectionError::Runtime("test lock poisoned".into()))
}
fn test_id(label: &str) -> ProjectionId {
ProjectionId::new(
"runtime-fidelity",
format!("{label}-{}", NEXT_TEST_ID.fetch_add(1, Ordering::Relaxed)),
)
}
fn envelope(position: u64, entity: &str, sequence_nr: u64, payload: i32) -> TestEnvelope<i32> {
TestEnvelope::new(
position,
PersistenceId::from_unique(format!("entity|{entity}"))
.expect("test persistence id is valid"),
SequenceNr::new(sequence_nr),
payload,
)
}
fn envelopes(count: u64) -> Vec<TestEnvelope<i32>> {
(1..=count)
.map(|position| {
envelope(
position,
"one",
position,
i32::try_from(position).expect("small test position fits i32"),
)
})
.collect()
}
fn runtime(
store: &Arc<MemoryProjectionOffsetStore>,
rng: impl BackoffRng,
) -> ProjectionRuntime {
ProjectionRuntime::new(Arc::clone(store) as Arc<dyn ProjectionOffsetStore>)
.with_backoff_rng(rng)
}
#[derive(Clone)]
struct FixedRng {
sample: f64,
calls: Arc<AtomicUsize>,
}
impl FixedRng {
fn new(sample: f64) -> Self {
Self {
sample,
calls: Arc::new(AtomicUsize::new(0)),
}
}
}
impl BackoffRng for FixedRng {
fn next_fraction(&self) -> f64 {
self.calls.fetch_add(1, Ordering::Relaxed);
self.sample
}
}
#[derive(Clone, Default)]
struct RecordingObserver {
restart_delays: Arc<Mutex<Vec<Duration>>>,
}
impl StatusObserver<TestEnvelope<i32>> for RecordingObserver {
fn status_changed(&self, _projection_id: &ProjectionId, status: &ProjectionStatus) {
if let ProjectionStatus::Restarting { delay, .. } = status
&& let Ok(mut delays) = self.restart_delays.lock()
{
delays.push(*delay);
}
}
}
#[derive(Clone, Default)]
struct CountingOffsetStore {
inner: MemoryProjectionOffsetStore,
loads: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl ProjectionOffsetStore for CountingOffsetStore {
async fn claim(
&self,
projection_id: &ProjectionId,
) -> ProjectionResult<ProjectionGeneration> {
self.inner.claim(projection_id).await
}
async fn load(
&self,
projection_id: &ProjectionId,
) -> ProjectionResult<Option<ProjectionCursor>> {
self.loads.fetch_add(1, Ordering::Relaxed);
self.inner.load(projection_id).await
}
async fn save(
&self,
projection_id: &ProjectionId,
generation: ProjectionGeneration,
expected_position: Option<&super::super::CursorPosition>,
cursor: ProjectionCursor,
) -> ProjectionResult<()> {
self.inner
.save(projection_id, generation, expected_position, cursor)
.await
}
async fn clear(
&self,
projection_id: &ProjectionId,
generation: ProjectionGeneration,
) -> ProjectionResult<()> {
self.inner.clear(projection_id, generation).await
}
async fn load_paused(&self, projection_id: &ProjectionId) -> ProjectionResult<bool> {
self.inner.load_paused(projection_id).await
}
async fn set_paused(
&self,
projection_id: &ProjectionId,
paused: bool,
) -> ProjectionResult<()> {
self.inner.set_paused(projection_id, paused).await
}
}
#[derive(Clone)]
struct AliasingOffsetStore {
inner: Arc<MemoryProjectionOffsetStore>,
durable_id: ProjectionId,
}
#[async_trait::async_trait]
impl ProjectionOffsetStore for AliasingOffsetStore {
async fn claim(
&self,
_projection_id: &ProjectionId,
) -> ProjectionResult<ProjectionGeneration> {
self.inner.claim(&self.durable_id).await
}
async fn load(
&self,
_projection_id: &ProjectionId,
) -> ProjectionResult<Option<ProjectionCursor>> {
self.inner.load(&self.durable_id).await
}
async fn save(
&self,
_projection_id: &ProjectionId,
generation: ProjectionGeneration,
expected_position: Option<&CursorPosition>,
cursor: ProjectionCursor,
) -> ProjectionResult<()> {
self.inner
.save(&self.durable_id, generation, expected_position, cursor)
.await
}
async fn clear(
&self,
_projection_id: &ProjectionId,
generation: ProjectionGeneration,
) -> ProjectionResult<()> {
self.inner.clear(&self.durable_id, generation).await
}
async fn load_paused(&self, _projection_id: &ProjectionId) -> ProjectionResult<bool> {
self.inner.load_paused(&self.durable_id).await
}
async fn set_paused(
&self,
_projection_id: &ProjectionId,
paused: bool,
) -> ProjectionResult<()> {
self.inner.set_paused(&self.durable_id, paused).await
}
}
#[derive(Clone)]
struct ControlledSaveOffsetStore {
inner: Arc<MemoryProjectionOffsetStore>,
block_next_save: Arc<AtomicBool>,
save_entered: Arc<Semaphore>,
release_save: Arc<Semaphore>,
}
impl ControlledSaveOffsetStore {
fn new(inner: Arc<MemoryProjectionOffsetStore>) -> Self {
Self {
inner,
block_next_save: Arc::new(AtomicBool::new(true)),
save_entered: Arc::new(Semaphore::new(0)),
release_save: Arc::new(Semaphore::new(0)),
}
}
async fn wait_for_save(&self) {
Arc::clone(&self.save_entered)
.acquire_owned()
.await
.expect("controlled save barrier remains open")
.forget();
}
fn release_save(&self) {
self.release_save.add_permits(1);
}
}
#[async_trait::async_trait]
impl ProjectionOffsetStore for ControlledSaveOffsetStore {
async fn claim(
&self,
projection_id: &ProjectionId,
) -> ProjectionResult<ProjectionGeneration> {
self.inner.claim(projection_id).await
}
async fn load(
&self,
projection_id: &ProjectionId,
) -> ProjectionResult<Option<ProjectionCursor>> {
self.inner.load(projection_id).await
}
async fn save(
&self,
projection_id: &ProjectionId,
generation: ProjectionGeneration,
expected_position: Option<&CursorPosition>,
cursor: ProjectionCursor,
) -> ProjectionResult<()> {
if self.block_next_save.swap(false, Ordering::AcqRel) {
self.save_entered.add_permits(1);
Arc::clone(&self.release_save)
.acquire_owned()
.await
.map_err(|_| ProjectionError::OffsetStore("save barrier closed".into()))?
.forget();
}
self.inner
.save(projection_id, generation, expected_position, cursor)
.await
}
async fn clear(
&self,
projection_id: &ProjectionId,
generation: ProjectionGeneration,
) -> ProjectionResult<()> {
self.inner.clear(projection_id, generation).await
}
async fn load_paused(&self, projection_id: &ProjectionId) -> ProjectionResult<bool> {
self.inner.load_paused(projection_id).await
}
async fn set_paused(
&self,
projection_id: &ProjectionId,
paused: bool,
) -> ProjectionResult<()> {
self.inner.set_paused(projection_id, paused).await
}
}
#[derive(Clone)]
struct FatalCursorProvider {
inner: TestSourceProvider<i32>,
}
#[async_trait::async_trait]
impl SourceProvider<TestEnvelope<i32>> for FatalCursorProvider {
fn manifest(&self) -> &super::super::ProviderManifest {
self.inner.manifest()
}
async fn source_from(
&self,
cursor: Option<&ProjectionCursor>,
mode: QueryMode,
) -> ProjectionResult<ProjectionSource<TestEnvelope<i32>>> {
self.inner.source_from(cursor, mode).await
}
fn extract_cursor(
&self,
_envelope: &TestEnvelope<i32>,
) -> ProjectionResult<ProjectionCursor> {
Err(ProjectionError::CursorCodec(
"scripted cursor extraction failure".into(),
))
}
fn compare_positions(
&self,
left: &CursorPosition,
right: &CursorPosition,
) -> ProjectionResult<std::cmp::Ordering> {
self.inner.compare_positions(left, right)
}
}
#[derive(Clone)]
struct PanicOnDropProvider {
inner: TestSourceProvider<i32>,
envelope: TestEnvelope<i32>,
}
struct PanicOnDropSource {
envelope: Option<TestEnvelope<i32>>,
}
impl Stream for PanicOnDropSource {
type Item = ProjectionResult<TestEnvelope<i32>>;
fn poll_next(
mut self: Pin<&mut Self>,
_context: &mut Context<'_>,
) -> Poll<Option<Self::Item>> {
self.envelope
.take()
.map_or(Poll::Pending, |envelope| Poll::Ready(Some(Ok(envelope))))
}
}
impl Drop for PanicOnDropSource {
fn drop(&mut self) {
panic!("scripted source destructor panic");
}
}
#[async_trait::async_trait]
impl SourceProvider<TestEnvelope<i32>> for PanicOnDropProvider {
fn manifest(&self) -> &super::super::ProviderManifest {
self.inner.manifest()
}
async fn source_from(
&self,
cursor: Option<&ProjectionCursor>,
_mode: QueryMode,
) -> ProjectionResult<ProjectionSource<TestEnvelope<i32>>> {
if let Some(cursor) = cursor {
cursor.ensure_manifest(self.manifest())?;
}
Ok(Box::pin(PanicOnDropSource {
envelope: Some(self.envelope.clone()),
}))
}
fn extract_cursor(
&self,
envelope: &TestEnvelope<i32>,
) -> ProjectionResult<ProjectionCursor> {
self.inner.extract_cursor(envelope)
}
fn compare_positions(
&self,
left: &CursorPosition,
right: &CursorPosition,
) -> ProjectionResult<std::cmp::Ordering> {
self.inner.compare_positions(left, right)
}
}
#[derive(Clone)]
struct AmbiguousBoundaryProvider {
inner: TestSourceProvider<i32>,
other: TestEnvelope<i32>,
}
#[async_trait::async_trait]
impl SourceProvider<TestEnvelope<i32>> for AmbiguousBoundaryProvider {
fn manifest(&self) -> &super::super::ProviderManifest {
self.inner.manifest()
}
async fn source_from(
&self,
cursor: Option<&ProjectionCursor>,
mode: QueryMode,
) -> ProjectionResult<ProjectionSource<TestEnvelope<i32>>> {
self.inner.source_from(cursor, mode).await
}
fn extract_cursor(
&self,
envelope: &TestEnvelope<i32>,
) -> ProjectionResult<ProjectionCursor> {
Ok(ProjectionCursor::new(
self.manifest().clone(),
DeduplicationBoundary::with_consumed(
envelope.position.to_string(),
[
BoundaryEvent::new(envelope.persistence_id.clone(), envelope.sequence_nr),
BoundaryEvent::new(
self.other.persistence_id.clone(),
self.other.sequence_nr,
),
],
),
))
}
fn compare_positions(
&self,
left: &CursorPosition,
right: &CursorPosition,
) -> ProjectionResult<std::cmp::Ordering> {
self.inner.compare_positions(left, right)
}
}
fn scripted_factory(
invocations: Arc<Mutex<Vec<i32>>>,
completed: Arc<Mutex<Vec<i32>>>,
failures: Arc<Mutex<BTreeMap<i32, usize>>>,
gate: Option<HandlerGate>,
factories: Arc<AtomicUsize>,
) -> impl Fn() -> ScriptedHandler + Send + Sync + 'static {
move || {
factories.fetch_add(1, Ordering::Relaxed);
ScriptedHandler {
invocations: Arc::clone(&invocations),
completed: Arc::clone(&completed),
failures: Arc::clone(&failures),
gate: gate.clone(),
}
}
}
async fn wait_until(mut predicate: impl FnMut() -> bool) {
tokio::time::timeout(Duration::from_secs(2), async {
while !predicate() {
tokio::task::yield_now().await;
tokio::time::sleep(Duration::from_millis(1)).await;
}
})
.await
.expect("bounded test condition timed out");
}
#[test]
fn fidelity_7_backoff_formula_caps_before_jitter_and_saturates() {
let settings = RestartBackoff {
min_backoff: Duration::from_millis(100),
max_backoff: Duration::from_millis(250),
random_factor: 0.2,
max_restarts: None,
max_restarts_within: Duration::from_millis(100),
};
assert_eq!(restart_delay(&settings, 0, 1.0), Duration::from_millis(120));
assert_eq!(restart_delay(&settings, 1, 1.0), Duration::from_millis(240));
assert_eq!(
restart_delay(&settings, 2, 1.0),
Duration::from_millis(300),
"jitter must be applied after the 250 ms cap"
);
assert_eq!(
restart_delay(&settings, usize::MAX, 1.0),
Duration::from_millis(300),
"large exponents must saturate instead of overflowing"
);
assert_eq!(
restart_delay(
&RestartBackoff {
random_factor: 0.0,
..settings
},
usize::MAX,
1.0,
),
Duration::from_millis(250)
);
}
#[tokio::test]
async fn fidelity_1_blueprint_is_pure_and_stop_is_acknowledged() {
let provider = TestSourceProvider::inclusive(envelopes(2));
let (provider, probe) = provider.with_manual_pace();
let materialization_probe = provider.clone();
let store = Arc::new(MemoryProjectionOffsetStore::new());
let invocations = Arc::new(Mutex::new(Vec::new()));
let completed = Arc::new(Mutex::new(Vec::new()));
let failures = Arc::new(Mutex::new(BTreeMap::new()));
let factories = Arc::new(AtomicUsize::new(0));
let gate = HandlerGate::new(1);
let projection_id = test_id("ack-stop");
let projection = Projection::at_least_once(
projection_id.clone(),
provider,
scripted_factory(
Arc::clone(&invocations),
Arc::clone(&completed),
failures,
Some(gate.clone()),
Arc::clone(&factories),
),
)
.with_query_mode(QueryMode::Current)
.with_save_offset(SaveOffsetConfig {
after_envelopes: 100,
after_duration: Duration::from_secs(60),
});
assert_eq!(materialization_probe.materializations().unwrap(), 0);
assert_eq!(factories.load(Ordering::Relaxed), 0);
let worker_probe = WorkerSynchronizationProbe::new();
let handle = projection
.spawn(
runtime(&store, FixedRng::new(0.0))
.with_worker_synchronization(worker_probe.clone()),
)
.await
.unwrap();
probe.request(2).unwrap();
gate.wait_started().await;
assert_eq!(
store.load(&projection_id).await.unwrap(),
None,
"ALO must not save before the in-flight handler completes"
);
let mut stop_reply = handle.signal_stop().unwrap();
gate.release();
worker_probe.wait_for_delivery(2).await;
worker_probe.wait_for_exit_start().await;
assert!(
matches!(
stop_reply.try_recv(),
Err(tokio::sync::oneshot::error::TryRecvError::Empty)
),
"stop acknowledgement must wait until the source worker exits"
);
assert_eq!(*lock_test(&completed).unwrap(), vec![1]);
assert_eq!(*lock_test(&invocations).unwrap(), vec![1]);
let cursor = store
.load(&projection_id)
.await
.unwrap()
.expect("acknowledged stop flushes the completed invocation");
assert_eq!(cursor.position().as_str(), "1");
worker_probe.allow_exit();
stop_reply.await.unwrap().unwrap();
let _ = handle.control.wait(None).await;
assert!(worker_probe.exited());
assert_eq!(handle.status(), ProjectionStatus::Stopped);
}
#[tokio::test]
async fn fidelity_1_flushes_before_surfacing_worker_join_failure() {
let envelope = envelope(1, "panic-drop", 1, 1);
let provider = PanicOnDropProvider {
inner: TestSourceProvider::inclusive(Vec::new()),
envelope,
};
let store = Arc::new(MemoryProjectionOffsetStore::new());
let completed = Arc::new(Mutex::new(Vec::new()));
let projection_id = test_id("flush-before-join");
let handle = Projection::at_least_once(
projection_id.clone(),
provider,
scripted_factory(
Arc::new(Mutex::new(Vec::new())),
Arc::clone(&completed),
Arc::new(Mutex::new(BTreeMap::new())),
None,
Arc::new(AtomicUsize::new(0)),
),
)
.with_save_offset(SaveOffsetConfig {
after_envelopes: 100,
after_duration: Duration::from_secs(60),
})
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
wait_until(|| lock_test(&completed).is_ok_and(|log| log.as_slice() == [1])).await;
let error = handle
.stop_and_wait()
.await
.expect_err("the scripted source destructor must fail worker join");
assert!(matches!(error, ProjectionError::Runtime(ref message) if message.contains("join")));
assert_eq!(
store
.load(&projection_id)
.await
.unwrap()
.expect("completed progress must flush even when worker join fails")
.position()
.as_str(),
"1"
);
}
#[tokio::test]
async fn fidelity_1_fatal_join_failure_is_not_masked_by_restartable_flush_failure() {
let envelope = envelope(1, "panic-drop-flush-fail", 1, 1);
let provider = PanicOnDropProvider {
inner: TestSourceProvider::inclusive(Vec::new()),
envelope,
};
let saves_fail = Arc::new(AtomicBool::new(false));
let failing_store = Arc::new(FailingSaveOffsetStore {
inner: MemoryProjectionOffsetStore::new(),
fail_saves: Arc::clone(&saves_fail),
});
let completed = Arc::new(Mutex::new(Vec::new()));
let handle = Projection::at_least_once(
test_id("fatal-join-not-masked"),
provider,
scripted_factory(
Arc::new(Mutex::new(Vec::new())),
Arc::clone(&completed),
Arc::new(Mutex::new(BTreeMap::new())),
None,
Arc::new(AtomicUsize::new(0)),
),
)
.with_save_offset(SaveOffsetConfig {
after_envelopes: 100,
after_duration: Duration::from_secs(60),
})
.spawn(
ProjectionRuntime::new(Arc::clone(&failing_store) as Arc<dyn ProjectionOffsetStore>)
.with_backoff_rng(FixedRng::new(0.0)),
)
.await
.unwrap();
wait_until(|| lock_test(&completed).is_ok_and(|log| log.as_slice() == [1])).await;
saves_fail.store(true, Ordering::SeqCst);
let error = handle
.stop_and_wait()
.await
.expect_err("both the flush and the worker join must fail");
assert!(
matches!(error, ProjectionError::Runtime(ref message) if message.contains("join")),
"expected the fatal join error to surface, got: {error:?}"
);
}
struct FailingSaveOffsetStore {
inner: MemoryProjectionOffsetStore,
fail_saves: Arc<AtomicBool>,
}
#[async_trait::async_trait]
impl ProjectionOffsetStore for FailingSaveOffsetStore {
async fn claim(
&self,
projection_id: &ProjectionId,
) -> ProjectionResult<ProjectionGeneration> {
self.inner.claim(projection_id).await
}
async fn load(
&self,
projection_id: &ProjectionId,
) -> ProjectionResult<Option<ProjectionCursor>> {
self.inner.load(projection_id).await
}
async fn save(
&self,
projection_id: &ProjectionId,
generation: ProjectionGeneration,
expected_position: Option<&super::super::CursorPosition>,
cursor: ProjectionCursor,
) -> ProjectionResult<()> {
if self.fail_saves.load(Ordering::SeqCst) {
return Err(ProjectionError::OffsetStore(
"injected restartable save failure".into(),
));
}
self.inner
.save(projection_id, generation, expected_position, cursor)
.await
}
async fn clear(
&self,
projection_id: &ProjectionId,
generation: ProjectionGeneration,
) -> ProjectionResult<()> {
self.inner.clear(projection_id, generation).await
}
async fn load_paused(&self, projection_id: &ProjectionId) -> ProjectionResult<bool> {
self.inner.load_paused(projection_id).await
}
async fn set_paused(
&self,
projection_id: &ProjectionId,
paused: bool,
) -> ProjectionResult<()> {
self.inner.set_paused(projection_id, paused).await
}
}
#[tokio::test]
async fn fidelity_1_handle_drop_is_signal_only() {
let provider = TestSourceProvider::new(envelopes(1));
let store = Arc::new(MemoryProjectionOffsetStore::new());
let gate = HandlerGate::new(1);
let invocations = Arc::new(Mutex::new(Vec::new()));
let handle = Projection::at_least_once(
test_id("drop-signal"),
provider,
scripted_factory(
Arc::clone(&invocations),
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(BTreeMap::new())),
Some(gate.clone()),
Arc::new(AtomicUsize::new(0)),
),
)
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
gate.wait_started().await;
let mut status = handle.status_receiver();
drop(handle);
assert!(
status.borrow().is_running(),
"dropping the handle must not synchronously await the in-flight handler"
);
assert_eq!(*lock_test(&invocations).unwrap(), vec![1]);
gate.release();
tokio::time::timeout(Duration::from_secs(2), async {
while !status.borrow_and_update().is_terminal() {
status
.changed()
.await
.expect("the actor retains its status sender through best-effort stop");
}
})
.await
.expect("best-effort drop signal did not eventually stop the projection");
assert_eq!(*status.borrow(), ProjectionStatus::Stopped);
}
#[tokio::test]
async fn fidelity_2_alo_crash_replays_only_unsaved_tail() {
let provider = TestSourceProvider::inclusive(envelopes(5));
let (provider, probe) = provider.with_manual_pace();
let provider_probe = provider.clone();
let store = Arc::new(MemoryProjectionOffsetStore::new());
let invocations = Arc::new(Mutex::new(Vec::new()));
let completed = Arc::new(Mutex::new(Vec::new()));
let failures = Arc::new(Mutex::new(BTreeMap::new()));
let factories = Arc::new(AtomicUsize::new(0));
let projection_id = test_id("alo-crash");
let build = |provider: TestSourceProvider<i32>| {
Projection::at_least_once(
projection_id.clone(),
provider,
scripted_factory(
Arc::clone(&invocations),
Arc::clone(&completed),
Arc::clone(&failures),
None,
Arc::clone(&factories),
),
)
.with_query_mode(QueryMode::Current)
.with_save_offset(SaveOffsetConfig {
after_envelopes: 3,
after_duration: Duration::from_secs(60),
})
};
let handle = build(provider)
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
probe.request(4).unwrap();
wait_until(|| lock_test(&completed).is_ok_and(|log| log.len() == 4)).await;
handle.abort_and_wait().await.unwrap();
let saved = store
.load(&projection_id)
.await
.unwrap()
.expect("three-envelope cadence must have saved");
assert_eq!(saved.position().as_str(), "3");
let (resumed, resumed_probe) = provider_probe.with_manual_pace();
let resumed_handle = build(resumed)
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
resumed_probe.request(3).unwrap();
let status = resumed_handle.wait().await.unwrap();
assert_eq!(status, ProjectionStatus::Completed);
assert_eq!(
*lock_test(&invocations).unwrap(),
vec![1, 2, 3, 4, 4, 5],
"only the unsaved cadence tail is replayed; the inclusive boundary is deduplicated"
);
}
#[tokio::test]
async fn fidelity_2_alo_duration_flushes_below_count_threshold() {
tokio::time::pause();
let store = Arc::new(MemoryProjectionOffsetStore::new());
let completed = Arc::new(Mutex::new(Vec::new()));
let projection_id = test_id("duration-cadence");
let handle = Projection::at_least_once(
projection_id.clone(),
TestSourceProvider::new(envelopes(1)),
scripted_factory(
Arc::new(Mutex::new(Vec::new())),
Arc::clone(&completed),
Arc::new(Mutex::new(BTreeMap::new())),
None,
Arc::new(AtomicUsize::new(0)),
),
)
.with_save_offset(SaveOffsetConfig {
after_envelopes: 100,
after_duration: Duration::from_secs(10),
})
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
while lock_test(&completed).unwrap().is_empty() {
tokio::task::yield_now().await;
}
assert_eq!(
handle.get_offset().await.unwrap(),
None,
"mailbox synchronization proves cadence was scheduled below the count threshold"
);
tokio::time::advance(Duration::from_secs(11)).await;
tokio::time::timeout(Duration::from_secs(1), async {
while store.load(&projection_id).await.unwrap().is_none() {
tokio::time::sleep(Duration::from_millis(1)).await;
}
})
.await
.expect("duration cadence did not flush after its deadline");
assert_eq!(
store
.load(&projection_id)
.await
.unwrap()
.expect("duration cadence must flush")
.position()
.as_str(),
"1"
);
handle.stop_and_wait().await.unwrap();
}
#[tokio::test]
async fn fidelity_2_alo_duration_expiring_during_handler_flushes_on_completion() {
let store = Arc::new(MemoryProjectionOffsetStore::new());
let completed = Arc::new(Mutex::new(Vec::new()));
let gate = HandlerGate::new(1);
let projection_id = test_id("duration-during-handler");
let handle = Projection::at_least_once(
projection_id.clone(),
TestSourceProvider::new(envelopes(1)),
scripted_factory(
Arc::new(Mutex::new(Vec::new())),
Arc::clone(&completed),
Arc::new(Mutex::new(BTreeMap::new())),
Some(gate.clone()),
Arc::new(AtomicUsize::new(0)),
),
)
.with_save_offset(SaveOffsetConfig {
after_envelopes: 100,
after_duration: Duration::from_millis(20),
})
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
gate.wait_started().await;
tokio::time::sleep(Duration::from_millis(30)).await;
assert_eq!(store.load(&projection_id).await.unwrap(), None);
gate.release();
tokio::time::timeout(Duration::from_secs(2), async {
while store.load(&projection_id).await.unwrap().is_none() {
tokio::task::yield_now().await;
}
})
.await
.expect("expired duration did not flush after handler completion");
assert_eq!(*lock_test(&completed).unwrap(), vec![1]);
assert_eq!(
store
.load(&projection_id)
.await
.unwrap()
.expect("expired duration must flush after handler completion")
.position()
.as_str(),
"1"
);
handle.stop_and_wait().await.unwrap();
}
#[tokio::test]
async fn fidelity_3_amo_dual_id_boundary_never_reprocesses() {
let shared_position = vec![envelope(7, "left", 1, 1), envelope(7, "right", 1, 2)];
let provider = TestSourceProvider::inclusive(shared_position);
let (provider, probe) = provider.with_manual_pace();
let provider_probe = provider.clone();
let store = Arc::new(MemoryProjectionOffsetStore::new());
let invocations = Arc::new(Mutex::new(Vec::new()));
let completed = Arc::new(Mutex::new(Vec::new()));
let failures = Arc::new(Mutex::new(BTreeMap::new()));
let factories = Arc::new(AtomicUsize::new(0));
let gate = HandlerGate::new(2);
let projection_id = test_id("amo-boundary");
let build = |provider: TestSourceProvider<i32>, gate: Option<HandlerGate>| {
Projection::at_most_once(
projection_id.clone(),
provider,
scripted_factory(
Arc::clone(&invocations),
Arc::clone(&completed),
Arc::clone(&failures),
gate,
Arc::clone(&factories),
),
)
.with_query_mode(QueryMode::Current)
};
let handle = build(provider, Some(gate.clone()))
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
probe.request(2).unwrap();
gate.wait_started().await;
let saved = store
.load(&projection_id)
.await
.unwrap()
.expect("AMO saves before invoking the second boundary event");
assert_eq!(saved.boundary().consumed().len(), 2);
handle.abort_and_wait().await.unwrap();
assert_eq!(*lock_test(&completed).unwrap(), vec![1]);
let resumed = build(provider_probe.without_manual_pace(), None)
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
assert_eq!(resumed.wait().await.unwrap(), ProjectionStatus::Completed);
assert_eq!(
*lock_test(&invocations).unwrap(),
vec![1, 2],
"both persistence ids at the inclusive boundary must be filtered on resume"
);
assert_eq!(*lock_test(&completed).unwrap(), vec![1]);
}
#[test]
fn fidelity_3_resume_filter_uses_the_emitted_envelope_identity() {
let seen = envelope(7, "seen", 1, 1);
let unseen = envelope(7, "unseen", 1, 2);
let provider = TestSourceProvider::inclusive(vec![seen.clone(), unseen.clone()]);
let stored = provider.extract_cursor(&seen).unwrap();
assert!(!provider.resume_allows(&stored, &seen).unwrap());
assert!(provider.resume_allows(&stored, &unseen).unwrap());
let ambiguous = AmbiguousBoundaryProvider {
inner: provider,
other: unseen,
};
let error = ambiguous
.resume_allows(&stored, &seen)
.expect_err("a cumulative candidate boundary is not an envelope identity");
assert!(
matches!(
error,
ProjectionError::InvalidConfig {
setting: "source_provider.resume_boundary",
..
}
),
"the default provider-owned filter must reject ambiguous equal-position identities"
);
}
#[tokio::test]
async fn fidelity_6_recovery_orders_retries_skip_and_escalation() {
let store = Arc::new(MemoryProjectionOffsetStore::new());
let invocations = Arc::new(Mutex::new(Vec::new()));
let completed = Arc::new(Mutex::new(Vec::new()));
let failures = Arc::new(Mutex::new(BTreeMap::from([(1, 3)])));
let factories = Arc::new(AtomicUsize::new(0));
let projection_id = test_id("retry-skip");
let provider = TestSourceProvider::inclusive(envelopes(2));
let projection = Projection::at_least_once(
projection_id.clone(),
provider.clone(),
scripted_factory(
Arc::clone(&invocations),
Arc::clone(&completed),
Arc::clone(&failures),
None,
Arc::clone(&factories),
),
)
.with_query_mode(QueryMode::Current)
.with_recovery_strategy(RecoveryStrategy::RetryAndSkip {
retries: 2,
delay: Duration::ZERO,
});
let handle = projection
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
assert_eq!(handle.wait().await.unwrap(), ProjectionStatus::Completed);
assert_eq!(*lock_test(&invocations).unwrap(), vec![1, 1, 1, 2]);
assert_eq!(*lock_test(&completed).unwrap(), vec![2]);
assert_eq!(
store
.load(&projection_id)
.await
.unwrap()
.expect("current completion flushes skipped progress")
.position()
.as_str(),
"2"
);
let replay_log = Arc::new(Mutex::new(Vec::new()));
let replay_completed = Arc::new(Mutex::new(Vec::new()));
let replay = Projection::at_least_once(
projection_id,
provider,
scripted_factory(
Arc::clone(&replay_log),
replay_completed,
Arc::new(Mutex::new(BTreeMap::new())),
None,
Arc::new(AtomicUsize::new(0)),
),
)
.with_query_mode(QueryMode::Current);
let replay_handle = replay
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
assert_eq!(
replay_handle.wait().await.unwrap(),
ProjectionStatus::Completed
);
assert!(lock_test(&replay_log).unwrap().is_empty());
let escalation_log = Arc::new(Mutex::new(Vec::new()));
let escalation_factories = Arc::new(AtomicUsize::new(0));
let escalation = Projection::at_least_once(
test_id("retry-fail"),
TestSourceProvider::new(envelopes(1)),
scripted_factory(
Arc::clone(&escalation_log),
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(BTreeMap::from([(1, usize::MAX)]))),
None,
Arc::clone(&escalation_factories),
),
)
.with_query_mode(QueryMode::Current)
.with_recovery_strategy(RecoveryStrategy::RetryAndFail {
retries: 1,
delay: Duration::ZERO,
})
.with_restart_backoff(RestartBackoff {
min_backoff: Duration::ZERO,
max_backoff: Duration::ZERO,
random_factor: 0.0,
max_restarts: Some(1),
max_restarts_within: Duration::from_secs(60),
});
let escalation_handle = escalation
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
assert!(escalation_handle.wait().await.is_err());
assert_eq!(*lock_test(&escalation_log).unwrap(), vec![1, 1, 1, 1]);
assert_eq!(escalation_factories.load(Ordering::Relaxed), 2);
}
#[tokio::test]
async fn fidelity_6_skip_and_successful_retry_use_ordinary_alo_cadence() {
let store = Arc::new(MemoryProjectionOffsetStore::new());
let skip_id = test_id("direct-skip-cadence");
let skip_invocations = Arc::new(Mutex::new(Vec::new()));
let skip = Projection::at_least_once(
skip_id.clone(),
TestSourceProvider::new(envelopes(1)),
scripted_factory(
Arc::clone(&skip_invocations),
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(BTreeMap::from([(1, usize::MAX)]))),
None,
Arc::new(AtomicUsize::new(0)),
),
)
.with_recovery_strategy(RecoveryStrategy::Skip)
.with_save_offset(SaveOffsetConfig {
after_envelopes: 1,
after_duration: Duration::from_secs(60),
})
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
tokio::time::timeout(Duration::from_secs(2), async {
while store.load(&skip_id).await.unwrap().is_none() {
tokio::task::yield_now().await;
}
})
.await
.expect("Skip did not participate in the one-envelope save cadence");
assert_eq!(*lock_test(&skip_invocations).unwrap(), vec![1]);
skip.stop_and_wait().await.unwrap();
let retry_id = test_id("successful-retry-cadence");
let retry_invocations = Arc::new(Mutex::new(Vec::new()));
let retry_completed = Arc::new(Mutex::new(Vec::new()));
let retry = Projection::at_least_once(
retry_id.clone(),
TestSourceProvider::new(envelopes(1)),
scripted_factory(
Arc::clone(&retry_invocations),
Arc::clone(&retry_completed),
Arc::new(Mutex::new(BTreeMap::from([(1, 1)]))),
None,
Arc::new(AtomicUsize::new(0)),
),
)
.with_recovery_strategy(RecoveryStrategy::RetryAndFail {
retries: 1,
delay: Duration::ZERO,
})
.with_save_offset(SaveOffsetConfig {
after_envelopes: 1,
after_duration: Duration::from_secs(60),
})
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
tokio::time::timeout(Duration::from_secs(2), async {
while store.load(&retry_id).await.unwrap().is_none() {
tokio::task::yield_now().await;
}
})
.await
.expect("a successful retry did not advance through ordinary cadence");
assert_eq!(*lock_test(&retry_invocations).unwrap(), vec![1, 1]);
assert_eq!(*lock_test(&retry_completed).unwrap(), vec![1]);
assert_eq!(
store
.load(&retry_id)
.await
.unwrap()
.expect("successful retry cursor is durable")
.position()
.as_str(),
"1"
);
retry.stop_and_wait().await.unwrap();
}
#[tokio::test]
async fn fidelity_6_amo_retry_and_skip_advances_without_reprocessing() {
let store = Arc::new(MemoryProjectionOffsetStore::new());
let projection_id = test_id("amo-recovery");
let invocations = Arc::new(Mutex::new(Vec::new()));
let completed = Arc::new(Mutex::new(Vec::new()));
let provider = TestSourceProvider::inclusive(envelopes(2));
let handle = Projection::at_most_once(
projection_id.clone(),
provider.clone(),
scripted_factory(
Arc::clone(&invocations),
Arc::clone(&completed),
Arc::new(Mutex::new(BTreeMap::from([(1, 2)]))),
None,
Arc::new(AtomicUsize::new(0)),
),
)
.with_query_mode(QueryMode::Current)
.with_recovery_strategy(RecoveryStrategy::RetryAndSkip {
retries: 1,
delay: Duration::ZERO,
})
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
assert_eq!(handle.wait().await.unwrap(), ProjectionStatus::Completed);
assert_eq!(*lock_test(&invocations).unwrap(), vec![1, 1, 2]);
assert_eq!(*lock_test(&completed).unwrap(), vec![2]);
assert_eq!(
store
.load(&projection_id)
.await
.unwrap()
.expect("AMO saves skipped and successful progress before invocation")
.position()
.as_str(),
"2"
);
let replayed = Arc::new(Mutex::new(Vec::new()));
let replay = Projection::at_most_once(
projection_id,
provider,
scripted_factory(
Arc::clone(&replayed),
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(BTreeMap::new())),
None,
Arc::new(AtomicUsize::new(0)),
),
)
.with_query_mode(QueryMode::Current)
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
assert_eq!(replay.wait().await.unwrap(), ProjectionStatus::Completed);
assert!(lock_test(&replayed).unwrap().is_empty());
}
#[test]
fn fidelity_7_max_restarts_is_bounded_within_tracking_window() {
let settings = RestartBackoff {
min_backoff: Duration::ZERO,
max_backoff: Duration::ZERO,
random_factor: 0.0,
max_restarts: Some(1),
max_restarts_within: Duration::from_millis(10),
};
let mut within = ProjectionBackoff::new();
let within_start = within.window_started;
assert_eq!(
within.reserve_restart(&settings, within_start),
Some((0, 1))
);
assert_eq!(
within.reserve_restart(&settings, within_start + Duration::from_millis(10)),
None,
"restarts at the window boundary still count toward the limit"
);
let mut separated = ProjectionBackoff::new();
let separated_start = separated.window_started;
assert_eq!(
separated.reserve_restart(&settings, separated_start),
Some((0, 1))
);
assert_eq!(
separated.reserve_restart(&settings, separated_start + Duration::from_millis(11)),
Some((0, 1)),
"restarts after the tracking window must not accumulate for life"
);
}
#[tokio::test]
async fn fidelity_7_actor_restart_limit_resets_only_after_tracking_window() {
let store = Arc::new(MemoryProjectionOffsetStore::new());
let within_provider = TestSourceProvider::new(envelopes(1));
within_provider
.fail_next_materialization(ProjectionError::Source(
"first within-window failure".into(),
))
.unwrap();
let within_factories = Arc::new(AtomicUsize::new(0));
let within = Projection::at_least_once(
test_id("restarts-within-window"),
within_provider,
scripted_factory(
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(BTreeMap::from([(1, 1)]))),
None,
Arc::clone(&within_factories),
),
)
.with_query_mode(QueryMode::Current)
.with_restart_backoff(RestartBackoff {
min_backoff: Duration::ZERO,
max_backoff: Duration::ZERO,
random_factor: 0.0,
max_restarts: Some(1),
max_restarts_within: Duration::from_secs(60),
})
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
assert!(matches!(
within.wait().await,
Err(ProjectionError::Handler(_))
));
assert_eq!(
within_factories.load(Ordering::Relaxed),
2,
"the second failure within the window must exhaust one allowed restart"
);
let separated_provider = TestSourceProvider::new(envelopes(1));
separated_provider
.fail_next_materialization(ProjectionError::Source("first separated failure".into()))
.unwrap();
let separated_factories = Arc::new(AtomicUsize::new(0));
let separated_gate = HandlerGate::new(1);
let separated = Projection::at_least_once(
test_id("restarts-after-window"),
separated_provider,
scripted_factory(
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(BTreeMap::from([(1, 1)]))),
Some(separated_gate.clone()),
Arc::clone(&separated_factories),
),
)
.with_query_mode(QueryMode::Current)
.with_restart_backoff(RestartBackoff {
min_backoff: Duration::ZERO,
max_backoff: Duration::ZERO,
random_factor: 0.0,
max_restarts: Some(1),
max_restarts_within: Duration::from_millis(10),
})
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
separated_gate.wait_started().await;
tokio::time::sleep(Duration::from_millis(20)).await;
separated_gate.release.add_permits(2);
assert_eq!(separated.wait().await.unwrap(), ProjectionStatus::Completed);
assert_eq!(
separated_factories.load(Ordering::Relaxed),
3,
"the later failure must receive a fresh restart allowance"
);
}
#[tokio::test]
async fn fidelity_7_restarts_are_fresh_and_fatal_errors_bypass_backoff() {
let store = Arc::new(MemoryProjectionOffsetStore::new());
let counting_store = Arc::new(CountingOffsetStore::default());
let load_probe = Arc::clone(&counting_store.loads);
let provider = TestSourceProvider::new(envelopes(1));
provider
.fail_next_materialization(ProjectionError::Source("first".into()))
.unwrap();
provider
.fail_next_materialization(ProjectionError::Source("second".into()))
.unwrap();
let provider_probe = provider.clone();
let factories = Arc::new(AtomicUsize::new(0));
let rng = FixedRng::new(1.0);
let rng_probe = rng.clone();
let observer = RecordingObserver::default();
let observer_probe = observer.clone();
let projection = Projection::at_least_once(
test_id("fresh-restart"),
provider,
scripted_factory(
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(BTreeMap::new())),
None,
Arc::clone(&factories),
),
)
.with_query_mode(QueryMode::Current)
.with_status_observer(observer)
.with_restart_backoff(RestartBackoff {
min_backoff: Duration::from_millis(1),
max_backoff: Duration::from_millis(2),
random_factor: 0.5,
max_restarts: Some(3),
max_restarts_within: Duration::from_secs(60),
});
let handle = projection
.spawn(
ProjectionRuntime::new(
Arc::clone(&counting_store) as Arc<dyn ProjectionOffsetStore>
)
.with_backoff_rng(rng),
)
.await
.unwrap();
assert_eq!(handle.wait().await.unwrap(), ProjectionStatus::Completed);
assert_eq!(provider_probe.materializations().unwrap(), 3);
assert_eq!(factories.load(Ordering::Relaxed), 3);
assert_eq!(
load_probe.load(Ordering::Relaxed),
3,
"every restarted incarnation must re-read the durable cursor"
);
assert_eq!(rng_probe.calls.load(Ordering::Relaxed), 2);
assert_eq!(
*lock_test(&observer_probe.restart_delays).unwrap(),
vec![Duration::from_micros(1_500), Duration::from_millis(3)],
"the injected RNG must deterministically drive cap-then-jitter delays"
);
let fatal_provider = TestSourceProvider::new(envelopes(1));
fatal_provider
.fail_next_materialization(ProjectionError::CursorCodec("corrupt".into()))
.unwrap();
let fatal_probe = fatal_provider.clone();
let fatal_rng = FixedRng::new(0.0);
let fatal_rng_probe = fatal_rng.clone();
let fatal = Projection::at_least_once(
test_id("fatal"),
fatal_provider,
scripted_factory(
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(BTreeMap::new())),
None,
Arc::new(AtomicUsize::new(0)),
),
)
.with_query_mode(QueryMode::Current)
.with_restart_backoff(RestartBackoff {
min_backoff: Duration::ZERO,
max_backoff: Duration::ZERO,
random_factor: 0.0,
max_restarts: None,
max_restarts_within: Duration::ZERO,
});
let fatal_handle = fatal.spawn(runtime(&store, fatal_rng)).await.unwrap();
assert!(matches!(
fatal_handle.wait().await,
Err(ProjectionError::CursorCodec(_))
));
assert_eq!(fatal_probe.materializations().unwrap(), 1);
assert_eq!(fatal_rng_probe.calls.load(Ordering::Relaxed), 0);
}
#[tokio::test]
async fn fidelity_7_fatal_cursor_extraction_bypasses_handler_recovery_and_backoff() {
let store = Arc::new(MemoryProjectionOffsetStore::new());
let inner = TestSourceProvider::new(envelopes(1));
let provider_probe = inner.clone();
let factories = Arc::new(AtomicUsize::new(0));
let rng = FixedRng::new(0.0);
let rng_probe = rng.clone();
let handle = Projection::at_least_once(
test_id("fatal-cursor-extraction"),
FatalCursorProvider { inner },
scripted_factory(
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(BTreeMap::new())),
None,
Arc::clone(&factories),
),
)
.with_query_mode(QueryMode::Current)
.with_recovery_strategy(RecoveryStrategy::RetryAndSkip {
retries: 10,
delay: Duration::ZERO,
})
.with_restart_backoff(RestartBackoff {
min_backoff: Duration::ZERO,
max_backoff: Duration::ZERO,
random_factor: 0.0,
max_restarts: None,
max_restarts_within: Duration::ZERO,
})
.spawn(runtime(&store, rng))
.await
.unwrap();
assert!(matches!(
tokio::time::timeout(Duration::from_secs(30), handle.wait())
.await
.expect("fatal cursor error must terminate promptly, not restart"),
Err(ProjectionError::CursorCodec(_))
));
assert!(matches!(
handle.status(),
ProjectionStatus::Failed(ProjectionError::CursorCodec(_))
));
assert_eq!(provider_probe.materializations().unwrap(), 1);
assert_eq!(factories.load(Ordering::Relaxed), 1);
assert_eq!(rng_probe.calls.load(Ordering::Relaxed), 0);
}
#[tokio::test]
async fn fidelity_7_fenced_during_save_terminates_while_new_owner_keeps_running() {
let durable_id = test_id("shared-durable-owner");
let memory = Arc::new(MemoryProjectionOffsetStore::new());
let aliasing = Arc::new(AliasingOffsetStore {
inner: memory,
durable_id,
});
let runtime =
ProjectionRuntime::new(Arc::clone(&aliasing) as Arc<dyn ProjectionOffsetStore>)
.with_backoff_rng(FixedRng::new(0.0));
let first_provider = TestSourceProvider::new(envelopes(1));
let (first_provider, first_pace) = first_provider.with_manual_pace();
let first_invocations = Arc::new(Mutex::new(Vec::new()));
let first = Projection::at_most_once(
test_id("local-owner-one"),
first_provider,
scripted_factory(
Arc::clone(&first_invocations),
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(BTreeMap::new())),
None,
Arc::new(AtomicUsize::new(0)),
),
)
.spawn(runtime.clone())
.await
.unwrap();
wait_until(|| first.status().is_running()).await;
let second = Projection::at_least_once(
test_id("local-owner-two"),
TestSourceProvider::new(Vec::<TestEnvelope<i32>>::new()),
scripted_factory(
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(BTreeMap::new())),
None,
Arc::new(AtomicUsize::new(0)),
),
)
.spawn(runtime)
.await
.unwrap();
wait_until(|| second.status().is_running()).await;
first_pace.request_one().unwrap();
assert!(matches!(
tokio::time::timeout(Duration::from_secs(30), first.wait())
.await
.expect("fenced save must terminate promptly, not restart"),
Err(ProjectionError::Fenced { .. })
));
assert!(matches!(
first.status(),
ProjectionStatus::Failed(ProjectionError::Fenced { .. })
));
assert!(
first_invocations.lock().unwrap().is_empty(),
"AMO fencing must fail before invoking the stale owner's handler"
);
assert!(
second.status().is_running(),
"the claimant with the newer generation must keep running"
);
second.stop_and_wait().await.unwrap();
}
#[tokio::test]
async fn fidelity_8_management_update_wins_pause_persists_and_duplicates_fail() {
let store = Arc::new(MemoryProjectionOffsetStore::new());
let provider = TestSourceProvider::inclusive(envelopes(1));
let (provider, probe) = provider.with_manual_pace();
let provider_probe = provider.clone();
let gate = HandlerGate::new(1);
let projection_id = test_id("management");
let projection = Projection::at_least_once(
projection_id.clone(),
provider,
scripted_factory(
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(BTreeMap::new())),
Some(gate.clone()),
Arc::new(AtomicUsize::new(0)),
),
)
.with_save_offset(SaveOffsetConfig {
after_envelopes: 100,
after_duration: Duration::from_secs(60),
});
let handle = projection
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
let duplicate = Projection::at_least_once(
projection_id.clone(),
TestSourceProvider::new(Vec::<TestEnvelope<i32>>::new()),
scripted_factory(
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(BTreeMap::new())),
None,
Arc::new(AtomicUsize::new(0)),
),
)
.spawn(runtime(&store, FixedRng::new(0.0)))
.await;
assert!(matches!(
duplicate,
Err(ProjectionError::AlreadyRunning { .. })
));
probe.request_one().unwrap();
gate.wait_started().await;
let update_envelope = envelope(100, "management", 1, 100);
let update_cursor = provider_probe.extract_cursor(&update_envelope).unwrap();
let update = handle.update_offset(update_cursor);
let release = async {
tokio::task::yield_now().await;
gate.release();
};
let (update_result, ()) = tokio::join!(update, release);
update_result.unwrap();
assert_eq!(
store
.load(&projection_id)
.await
.unwrap()
.expect("management update is durable")
.position()
.as_str(),
"100",
"the completing old incarnation must not overwrite the newer management generation"
);
assert_eq!(
handle
.get_offset()
.await
.unwrap()
.expect("get_offset observes the management update")
.position()
.as_str(),
"100"
);
handle.pause().await.unwrap();
assert!(store.load_paused(&projection_id).await.unwrap());
handle.stop_and_wait().await.unwrap();
let paused_provider = TestSourceProvider::new(envelopes(1));
let paused_probe = paused_provider.clone();
let paused = Projection::at_least_once(
projection_id,
paused_provider,
scripted_factory(
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(BTreeMap::new())),
None,
Arc::new(AtomicUsize::new(0)),
),
)
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
wait_until(|| paused.status() == ProjectionStatus::Paused).await;
assert_eq!(paused_probe.materializations().unwrap(), 0);
assert_eq!(
paused
.get_offset()
.await
.unwrap()
.expect("paused projection retains its cursor")
.position()
.as_str(),
"100"
);
paused.clear_offset().await.unwrap();
assert_eq!(paused.get_offset().await.unwrap(), None);
assert_eq!(
paused.status(),
ProjectionStatus::Paused,
"clear_offset must preserve the durable pause flag"
);
paused.resume().await.unwrap();
wait_until(|| paused.status().is_running()).await;
assert_eq!(
paused_probe.materializations().unwrap(),
1,
"resume must construct a fresh source after clearing the pause flag"
);
paused.stop_and_wait().await.unwrap();
}
#[tokio::test]
async fn fidelity_8_stale_save_linearized_after_management_claim_is_fenced() {
let memory = Arc::new(MemoryProjectionOffsetStore::new());
let controlled = Arc::new(ControlledSaveOffsetStore::new(Arc::clone(&memory)));
let provider = TestSourceProvider::new(envelopes(1));
let provider_probe = provider.clone();
let projection_id = test_id("stale-save-management");
let handle = Projection::at_least_once(
projection_id.clone(),
provider,
scripted_factory(
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(BTreeMap::new())),
None,
Arc::new(AtomicUsize::new(0)),
),
)
.with_save_offset(SaveOffsetConfig {
after_envelopes: 1,
after_duration: Duration::from_secs(60),
})
.spawn(
ProjectionRuntime::new(Arc::clone(&controlled) as Arc<dyn ProjectionOffsetStore>)
.with_backoff_rng(FixedRng::new(0.0)),
)
.await
.unwrap();
controlled.wait_for_save().await;
let management_generation = memory.claim(&projection_id).await.unwrap();
let management_cursor = provider_probe
.extract_cursor(&envelope(100, "management", 1, 100))
.unwrap();
memory
.save(
&projection_id,
management_generation,
None,
management_cursor,
)
.await
.unwrap();
controlled.release_save();
assert!(matches!(
handle.wait().await,
Err(ProjectionError::Fenced { .. })
));
assert!(matches!(
handle.status(),
ProjectionStatus::Failed(ProjectionError::Fenced { .. })
));
assert_eq!(
memory
.load(&projection_id)
.await
.unwrap()
.expect("the management claimant's cursor must win")
.position()
.as_str(),
"100",
"the released old-generation save must be rejected after the management claim"
);
}
#[tokio::test]
async fn fidelity_10_completion_modes_and_source_failure_are_distinct() {
let store = Arc::new(MemoryProjectionOffsetStore::new());
let factory = || {
scripted_factory(
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(BTreeMap::new())),
None,
Arc::new(AtomicUsize::new(0)),
)
};
let current = Projection::at_least_once(
test_id("current-complete"),
TestSourceProvider::new(Vec::<TestEnvelope<i32>>::new()),
factory(),
)
.with_query_mode(QueryMode::Current)
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
assert_eq!(current.wait().await.unwrap(), ProjectionStatus::Completed);
let live_provider = TestSourceProvider::new(Vec::<TestEnvelope<i32>>::new());
live_provider.complete().unwrap();
let live = Projection::at_least_once(test_id("live-complete"), live_provider, factory())
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
assert_eq!(live.wait().await.unwrap(), ProjectionStatus::Stopped);
let failed_provider = TestSourceProvider::new(Vec::<TestEnvelope<i32>>::new());
failed_provider
.fail_next_source_after(0, ProjectionError::Source("boom".into()))
.unwrap();
let failed =
Projection::at_least_once(test_id("source-failure"), failed_provider, factory())
.with_restart_backoff(RestartBackoff {
min_backoff: Duration::ZERO,
max_backoff: Duration::ZERO,
random_factor: 0.0,
max_restarts: Some(0),
max_restarts_within: Duration::ZERO,
})
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
assert!(matches!(
failed.wait().await,
Err(ProjectionError::Source(_))
));
}
#[tokio::test]
async fn fidelity_5_groups_form_by_count_or_duration_whichever_is_first() {
tokio::time::pause();
let (provider, pace) = TestSourceProvider::new(envelopes(4)).with_manual_pace();
let provider = CountingCursorProvider::new(provider);
let store = Arc::new(RecordingOffsetStore::default());
let invocations = Arc::new(Mutex::new(Vec::new()));
let completed = Arc::new(Mutex::new(Vec::new()));
let invocation_signal = Arc::new(Semaphore::new(0));
let started_at = tokio::time::Instant::now();
let handle = Projection::grouped_at_least_once(
test_id("group-count-duration"),
provider.clone(),
grouped_factory(
Arc::clone(&invocations),
Arc::clone(&completed),
Arc::new(AtomicUsize::new(0)),
Arc::clone(&invocation_signal),
Arc::new(AtomicUsize::new(0)),
),
)
.with_group(GroupConfig {
group_after_envelopes: 3,
group_after_duration: Duration::from_secs(10),
})
.spawn(ProjectionRuntime::new(
Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>
))
.await
.unwrap();
pace.request(3).unwrap();
wait_for_group_invocations(&invocation_signal, 1).await;
assert_eq!(
tokio::time::Instant::now(),
started_at,
"the count threshold must close the first group before virtual time advances"
);
assert_eq!(*lock_test(&invocations).unwrap(), vec![vec![1, 2, 3]]);
let before_fourth = provider.extraction_count();
pace.request_one().unwrap();
provider.wait_for_extractions(before_fourth + 2).await;
tokio::time::advance(Duration::from_secs(9)).await;
tokio::task::yield_now().await;
assert_eq!(invocation_signal.available_permits(), 0);
tokio::time::advance(Duration::from_secs(1)).await;
wait_for_group_invocations(&invocation_signal, 1).await;
assert_eq!(
*lock_test(&invocations).unwrap(),
vec![vec![1, 2, 3], vec![4]]
);
assert_eq!(
*lock_test(&completed).unwrap(),
*lock_test(&invocations).unwrap()
);
handle.stop_and_wait().await.unwrap();
}
#[tokio::test]
async fn fidelity_1_grouped_blueprint_is_pure_and_validates_before_materialization() {
let provider = TestSourceProvider::new(envelopes(1));
let provider_probe = provider.clone();
let factories = Arc::new(AtomicUsize::new(0));
let projection = Projection::grouped_at_least_once(
test_id("group-blueprint"),
provider,
grouped_factory(
Arc::new(Mutex::new(Vec::new())),
Arc::new(Mutex::new(Vec::new())),
Arc::new(AtomicUsize::new(0)),
Arc::new(Semaphore::new(0)),
Arc::clone(&factories),
),
)
.with_group(GroupConfig {
group_after_envelopes: 0,
group_after_duration: Duration::from_secs(1),
});
assert_eq!(provider_probe.materializations().unwrap(), 0);
assert_eq!(factories.load(Ordering::Relaxed), 0);
let store = Arc::new(MemoryProjectionOffsetStore::new());
assert!(matches!(
projection.spawn(runtime(&store, FixedRng::new(0.0))).await,
Err(ProjectionError::InvalidConfig {
setting: "group.group_after_envelopes",
..
})
));
assert_eq!(provider_probe.materializations().unwrap(), 0);
assert_eq!(factories.load(Ordering::Relaxed), 0);
}
#[tokio::test]
async fn fidelity_5_grouped_alo_saves_once_per_group_and_ignores_single_cadence() {
let store = Arc::new(RecordingOffsetStore::default());
let projection_id = test_id("group-cadence");
let invocations = Arc::new(Mutex::new(Vec::new()));
let handle = Projection::grouped_at_least_once(
projection_id.clone(),
TestSourceProvider::new(envelopes(4)),
grouped_factory(
Arc::clone(&invocations),
Arc::new(Mutex::new(Vec::new())),
Arc::new(AtomicUsize::new(0)),
Arc::new(Semaphore::new(0)),
Arc::new(AtomicUsize::new(0)),
),
)
.with_query_mode(QueryMode::Current)
.with_group(GroupConfig {
group_after_envelopes: 2,
group_after_duration: Duration::from_secs(60),
})
.with_save_offset(SaveOffsetConfig {
after_envelopes: 1,
after_duration: Duration::ZERO,
})
.spawn(ProjectionRuntime::new(
Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>
))
.await
.unwrap();
assert_eq!(handle.wait().await.unwrap(), ProjectionStatus::Completed);
assert_eq!(
*lock_test(&invocations).unwrap(),
vec![vec![1, 2], vec![3, 4]]
);
assert_eq!(
lock_test(&store.saves)
.unwrap()
.iter()
.map(|cursor| cursor.position().as_str())
.collect::<Vec<_>>(),
vec!["2", "4"],
"grouped ALO must perform exactly one save at each whole-group tail"
);
assert_eq!(
store
.load(&projection_id)
.await
.unwrap()
.expect("the second group cursor is durable")
.position()
.as_str(),
"4"
);
}
#[tokio::test]
async fn fidelity_1_grouped_count_partial_is_dropped_on_stop_and_reread() {
let (inner, pace) = TestSourceProvider::inclusive(envelopes(3)).with_manual_pace();
let provider = CountingCursorProvider::new(inner);
let provider_probe = provider.clone();
let store = Arc::new(MemoryProjectionOffsetStore::new());
let projection_id = test_id("group-partial-stop");
let invocations = Arc::new(Mutex::new(Vec::new()));
let build = |source| {
Projection::grouped_at_least_once(
projection_id.clone(),
source,
grouped_factory(
Arc::clone(&invocations),
Arc::new(Mutex::new(Vec::new())),
Arc::new(AtomicUsize::new(0)),
Arc::new(Semaphore::new(0)),
Arc::new(AtomicUsize::new(0)),
),
)
.with_group(GroupConfig {
group_after_envelopes: 3,
group_after_duration: Duration::from_secs(60),
})
};
let handle = build(provider)
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
pace.request(2).unwrap();
provider_probe.wait_for_extractions(3).await;
handle.stop_and_wait().await.unwrap();
assert!(lock_test(&invocations).unwrap().is_empty());
assert_eq!(store.load(&projection_id).await.unwrap(), None);
let resumed = build(CountingCursorProvider::new(
provider_probe.inner.clone().without_manual_pace(),
))
.with_query_mode(QueryMode::Current)
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
assert_eq!(resumed.wait().await.unwrap(), ProjectionStatus::Completed);
assert_eq!(*lock_test(&invocations).unwrap(), vec![vec![1, 2, 3]]);
assert_eq!(
store
.load(&projection_id)
.await
.unwrap()
.expect("the reread whole group is durable")
.position()
.as_str(),
"3"
);
}
#[tokio::test]
async fn fidelity_1_grouped_duration_partial_is_dropped_on_pause_and_reread() {
tokio::time::pause();
let (inner, pace) = TestSourceProvider::inclusive(envelopes(1)).with_manual_pace();
let provider = CountingCursorProvider::new(inner);
let store = Arc::new(MemoryProjectionOffsetStore::new());
let projection_id = test_id("group-partial-pause");
let invocations = Arc::new(Mutex::new(Vec::new()));
let invocation_signal = Arc::new(Semaphore::new(0));
let handle = Projection::grouped_at_least_once(
projection_id.clone(),
provider.clone(),
grouped_factory(
Arc::clone(&invocations),
Arc::new(Mutex::new(Vec::new())),
Arc::new(AtomicUsize::new(0)),
Arc::clone(&invocation_signal),
Arc::new(AtomicUsize::new(0)),
),
)
.with_group(GroupConfig {
group_after_envelopes: 100,
group_after_duration: Duration::from_secs(10),
})
.spawn(runtime(&store, FixedRng::new(0.0)))
.await
.unwrap();
pace.request_one().unwrap();
provider.wait_for_extractions(1).await;
handle.pause().await.unwrap();
assert_eq!(handle.status(), ProjectionStatus::Paused);
assert!(lock_test(&invocations).unwrap().is_empty());
assert_eq!(store.load(&projection_id).await.unwrap(), None);
tokio::time::advance(Duration::from_secs(20)).await;
tokio::task::yield_now().await;
assert!(
lock_test(&invocations).unwrap().is_empty(),
"the old partial-group deadline must be inert while paused"
);
handle.resume().await.unwrap();
pace.request_one().unwrap();
provider.wait_for_extractions(2).await;
tokio::time::advance(Duration::from_secs(10)).await;
wait_for_group_invocations(&invocation_signal, 1).await;
assert_eq!(*lock_test(&invocations).unwrap(), vec![vec![1]]);
assert_eq!(
store
.load(&projection_id)
.await
.unwrap()
.expect("the reread duration group is durable")
.position()
.as_str(),
"1"
);
handle.stop_and_wait().await.unwrap();
}
#[tokio::test]
async fn fidelity_5_group_retry_reuses_the_same_ordered_group_then_saves_once() {
let store = Arc::new(RecordingOffsetStore::default());
let projection_id = test_id("group-retry");
let invocations = Arc::new(Mutex::new(Vec::new()));
let completed = Arc::new(Mutex::new(Vec::new()));
let handle = Projection::grouped_at_least_once(
projection_id.clone(),
TestSourceProvider::new(envelopes(3)),
grouped_factory(
Arc::clone(&invocations),
Arc::clone(&completed),
Arc::new(AtomicUsize::new(1)),
Arc::new(Semaphore::new(0)),
Arc::new(AtomicUsize::new(0)),
),
)
.with_query_mode(QueryMode::Current)
.with_group(GroupConfig {
group_after_envelopes: 3,
group_after_duration: Duration::from_secs(60),
})
.with_recovery_strategy(RecoveryStrategy::RetryAndFail {
retries: 1,
delay: Duration::ZERO,
})
.spawn(ProjectionRuntime::new(
Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>
))
.await
.unwrap();
assert_eq!(handle.wait().await.unwrap(), ProjectionStatus::Completed);
assert_eq!(
*lock_test(&invocations).unwrap(),
vec![vec![1, 2, 3], vec![1, 2, 3]]
);
assert_eq!(*lock_test(&completed).unwrap(), vec![vec![1, 2, 3]]);
assert_eq!(lock_test(&store.saves).unwrap().len(), 1);
assert_eq!(
store
.load(&projection_id)
.await
.unwrap()
.expect("successful group retry advances the cursor")
.position()
.as_str(),
"3"
);
}
#[tokio::test]
async fn fidelity_6_group_skip_advances_whole_groups_and_never_redelivers() {
let store = Arc::new(RecordingOffsetStore::default());
let projection_id = test_id("group-skip");
let provider = TestSourceProvider::inclusive(envelopes(4));
let invocations = Arc::new(Mutex::new(Vec::new()));
let handle = Projection::grouped_at_least_once(
projection_id.clone(),
provider.clone(),
grouped_factory(
Arc::clone(&invocations),
Arc::new(Mutex::new(Vec::new())),
Arc::new(AtomicUsize::new(usize::MAX)),
Arc::new(Semaphore::new(0)),
Arc::new(AtomicUsize::new(0)),
),
)
.with_query_mode(QueryMode::Current)
.with_group(GroupConfig {
group_after_envelopes: 2,
group_after_duration: Duration::from_secs(60),
})
.with_recovery_strategy(RecoveryStrategy::Skip)
.spawn(ProjectionRuntime::new(
Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>
))
.await
.unwrap();
assert_eq!(handle.wait().await.unwrap(), ProjectionStatus::Completed);
assert_eq!(
*lock_test(&invocations).unwrap(),
vec![vec![1, 2], vec![3, 4]]
);
assert_eq!(
lock_test(&store.saves)
.unwrap()
.iter()
.map(|cursor| cursor.position().as_str())
.collect::<Vec<_>>(),
vec!["2", "4"]
);
let replayed = Arc::new(Mutex::new(Vec::new()));
let replay = Projection::grouped_at_least_once(
projection_id,
provider,
grouped_factory(
Arc::clone(&replayed),
Arc::new(Mutex::new(Vec::new())),
Arc::new(AtomicUsize::new(0)),
Arc::new(Semaphore::new(0)),
Arc::new(AtomicUsize::new(0)),
),
)
.with_query_mode(QueryMode::Current)
.with_group(GroupConfig {
group_after_envelopes: 2,
group_after_duration: Duration::from_secs(60),
})
.spawn(ProjectionRuntime::new(
Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>
))
.await
.unwrap();
assert_eq!(replay.wait().await.unwrap(), ProjectionStatus::Completed);
assert!(lock_test(&replayed).unwrap().is_empty());
}
#[tokio::test]
async fn fidelity_6_group_retry_exhaustion_restarts_and_rebuffers() {
let store = Arc::new(RecordingOffsetStore::default());
let provider = TestSourceProvider::inclusive(envelopes(2));
let provider_probe = provider.clone();
let invocations = Arc::new(Mutex::new(Vec::new()));
let factories = Arc::new(AtomicUsize::new(0));
let handle = Projection::grouped_at_least_once(
test_id("group-restart"),
provider,
grouped_factory(
Arc::clone(&invocations),
Arc::new(Mutex::new(Vec::new())),
Arc::new(AtomicUsize::new(2)),
Arc::new(Semaphore::new(0)),
Arc::clone(&factories),
),
)
.with_query_mode(QueryMode::Current)
.with_group(GroupConfig {
group_after_envelopes: 2,
group_after_duration: Duration::from_secs(60),
})
.with_recovery_strategy(RecoveryStrategy::RetryAndFail {
retries: 1,
delay: Duration::ZERO,
})
.with_restart_backoff(RestartBackoff {
min_backoff: Duration::ZERO,
max_backoff: Duration::ZERO,
random_factor: 0.0,
max_restarts: Some(1),
max_restarts_within: Duration::from_secs(60),
})
.spawn(ProjectionRuntime::new(
Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>
))
.await
.unwrap();
assert_eq!(handle.wait().await.unwrap(), ProjectionStatus::Completed);
assert_eq!(
*lock_test(&invocations).unwrap(),
vec![vec![1, 2], vec![1, 2], vec![1, 2]]
);
assert_eq!(factories.load(Ordering::Relaxed), 2);
assert_eq!(provider_probe.materializations().unwrap(), 2);
assert_eq!(lock_test(&store.saves).unwrap().len(), 1);
}
#[tokio::test]
async fn fidelity_2_grouped_crash_replays_only_the_unadvanced_partial_group() {
let (inner, pace) = TestSourceProvider::inclusive(envelopes(5)).with_manual_pace();
let provider = CountingCursorProvider::new(inner);
let provider_probe = provider.clone();
let store = Arc::new(RecordingOffsetStore::default());
let projection_id = test_id("group-crash");
let invocations = Arc::new(Mutex::new(Vec::new()));
let invocation_signal = Arc::new(Semaphore::new(0));
let build = |source, log: Arc<Mutex<Vec<Vec<i32>>>>, signal: Arc<Semaphore>| {
Projection::grouped_at_least_once(
projection_id.clone(),
source,
grouped_factory(
log,
Arc::new(Mutex::new(Vec::new())),
Arc::new(AtomicUsize::new(0)),
signal,
Arc::new(AtomicUsize::new(0)),
),
)
.with_group(GroupConfig {
group_after_envelopes: 2,
group_after_duration: Duration::from_secs(60),
})
};
let handle = build(
provider,
Arc::clone(&invocations),
Arc::clone(&invocation_signal),
)
.spawn(ProjectionRuntime::new(
Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>
))
.await
.unwrap();
pace.request(4).unwrap();
wait_for_group_invocations(&invocation_signal, 2).await;
store.wait_for_saves(2).await;
let before_partial = provider_probe.extraction_count();
pace.request_one().unwrap();
provider_probe
.wait_for_extractions(before_partial + 2)
.await;
handle.abort_and_wait().await.unwrap();
assert_eq!(
*lock_test(&invocations).unwrap(),
vec![vec![1, 2], vec![3, 4]]
);
assert_eq!(
lock_test(&store.saves)
.unwrap()
.iter()
.map(|cursor| cursor.position().as_str())
.collect::<Vec<_>>(),
vec!["2", "4"]
);
let replayed = Arc::new(Mutex::new(Vec::new()));
let replay = build(
CountingCursorProvider::new(provider_probe.inner.clone().without_manual_pace()),
Arc::clone(&replayed),
Arc::new(Semaphore::new(0)),
)
.with_query_mode(QueryMode::Current)
.spawn(ProjectionRuntime::new(
Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>
))
.await
.unwrap();
assert_eq!(replay.wait().await.unwrap(), ProjectionStatus::Completed);
assert_eq!(*lock_test(&replayed).unwrap(), vec![vec![5]]);
}
#[tokio::test]
async fn fidelity_5_group_resume_deduplicates_every_committed_boundary_member() {
let boundary_envelopes = vec![
envelope(7, "left", 1, 1),
envelope(7, "right", 1, 2),
envelope(8, "later", 1, 3),
];
let (provider, pace) = TestSourceProvider::inclusive(boundary_envelopes).with_manual_pace();
let provider_probe = provider.clone();
let store = Arc::new(RecordingOffsetStore::default());
let projection_id = test_id("group-boundary");
let first_log = Arc::new(Mutex::new(Vec::new()));
let first_signal = Arc::new(Semaphore::new(0));
let first = Projection::grouped_at_least_once(
projection_id.clone(),
provider,
grouped_factory(
Arc::clone(&first_log),
Arc::new(Mutex::new(Vec::new())),
Arc::new(AtomicUsize::new(0)),
Arc::clone(&first_signal),
Arc::new(AtomicUsize::new(0)),
),
)
.with_group(GroupConfig {
group_after_envelopes: 2,
group_after_duration: Duration::from_secs(60),
})
.spawn(ProjectionRuntime::new(
Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>
))
.await
.unwrap();
pace.request(2).unwrap();
wait_for_group_invocations(&first_signal, 1).await;
store.wait_for_saves(1).await;
first.abort_and_wait().await.unwrap();
let boundary_cursor = store
.load(&projection_id)
.await
.unwrap()
.expect("the shared-position group cursor is durable");
assert_eq!(boundary_cursor.position().as_str(), "7");
assert_eq!(boundary_cursor.boundary().consumed().len(), 2);
assert_eq!(*lock_test(&first_log).unwrap(), vec![vec![1, 2]]);
let replayed = Arc::new(Mutex::new(Vec::new()));
let replay = Projection::grouped_at_least_once(
projection_id,
provider_probe.without_manual_pace(),
grouped_factory(
Arc::clone(&replayed),
Arc::new(Mutex::new(Vec::new())),
Arc::new(AtomicUsize::new(0)),
Arc::new(Semaphore::new(0)),
Arc::new(AtomicUsize::new(0)),
),
)
.with_query_mode(QueryMode::Current)
.with_group(GroupConfig {
group_after_envelopes: 2,
group_after_duration: Duration::from_secs(60),
})
.spawn(ProjectionRuntime::new(
Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>
))
.await
.unwrap();
assert_eq!(replay.wait().await.unwrap(), ProjectionStatus::Completed);
assert_eq!(
*lock_test(&replayed).unwrap(),
vec![vec![3]],
"both committed identities at the inclusive boundary must be filtered"
);
}
#[test]
fn same_position_cursor_merge_keeps_both_boundary_identities() {
let provider = TestSourceProvider::inclusive(Vec::<TestEnvelope<i32>>::new());
let left = envelope(9, "left", 1, 1);
let right = envelope(9, "right", 1, 2);
let first = advance_cursor(&provider, None, &left).unwrap();
let merged = advance_cursor(&provider, Some(&first), &right).unwrap();
assert_eq!(merged.boundary().consumed().len(), 2);
assert_eq!(
merged,
ProjectionCursor::new(
provider.manifest().clone(),
DeduplicationBoundary::with_consumed(
"9",
[
BoundaryEvent::new(left.persistence_id, left.sequence_nr),
BoundaryEvent::new(right.persistence_id, right.sequence_nr),
],
),
)
);
}
}