use std::{
cmp::Ordering as CmpOrdering,
collections::VecDeque,
sync::{
Arc, Mutex,
atomic::{AtomicU64, Ordering},
},
time::{Duration, Instant},
};
use futures_util::{future::join, stream};
use tokio::sync::{Semaphore, watch};
use crate::query::QueryMode;
use crate::{PersistenceId, SequenceNr};
use super::{
BoundaryEvent, CursorPosition, DeduplicationBoundary, ProjectionCursor, ProjectionError,
ProjectionGeneration, ProjectionId, ProjectionOffsetStore, ProjectionResult, ProjectionRuntime,
ProviderManifest, SourceProvider,
};
static TEST_NAMESPACE: AtomicU64 = AtomicU64::new(1);
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TestEnvelope<Payload> {
pub position: u64,
pub persistence_id: PersistenceId,
pub sequence_nr: SequenceNr,
pub payload: Payload,
}
impl<Payload> TestEnvelope<Payload> {
#[must_use]
pub fn new(
position: u64,
persistence_id: PersistenceId,
sequence_nr: SequenceNr,
payload: Payload,
) -> Self {
Self {
position,
persistence_id,
sequence_nr,
payload,
}
}
}
struct TestSourceFailure {
after_envelopes: usize,
error: ProjectionError,
}
struct TestSourceState<Payload> {
envelopes: Vec<TestEnvelope<Payload>>,
closed: bool,
materializations: usize,
materialization_failures: VecDeque<ProjectionError>,
stream_failures: VecDeque<TestSourceFailure>,
}
#[derive(Clone)]
pub struct TestSourceProvider<Payload> {
manifest: ProviderManifest,
state: Arc<Mutex<TestSourceState<Payload>>>,
notifications: watch::Sender<u64>,
inclusive_resume: bool,
permits: Option<Arc<Semaphore>>,
}
impl<Payload> TestSourceProvider<Payload> {
#[must_use]
pub fn new(envelopes: Vec<TestEnvelope<Payload>>) -> Self {
Self::with_resume_mode(envelopes, false)
}
#[must_use]
pub fn inclusive(envelopes: Vec<TestEnvelope<Payload>>) -> Self {
Self::with_resume_mode(envelopes, true)
}
fn with_resume_mode(envelopes: Vec<TestEnvelope<Payload>>, inclusive_resume: bool) -> Self {
let (notifications, _receiver) = watch::channel(0);
Self {
manifest: ProviderManifest::new("datum-test-source/position-v1")
.expect("the fixed test provider manifest is valid"),
state: Arc::new(Mutex::new(TestSourceState {
envelopes,
closed: false,
materializations: 0,
materialization_failures: VecDeque::new(),
stream_failures: VecDeque::new(),
})),
notifications,
inclusive_resume,
permits: None,
}
}
#[must_use]
pub fn with_manual_pace(mut self) -> (Self, ManualPaceProbe) {
let permits = Arc::new(Semaphore::new(0));
self.permits = Some(Arc::clone(&permits));
(self, ManualPaceProbe { permits })
}
#[must_use]
pub fn without_manual_pace(mut self) -> Self {
self.permits = None;
self
}
pub fn push(&self, envelope: TestEnvelope<Payload>) -> ProjectionResult<()> {
let version = {
let mut state = self.lock_state()?;
state.envelopes.push(envelope);
u64::try_from(state.envelopes.len()).unwrap_or(u64::MAX)
};
self.notifications.send_replace(version);
Ok(())
}
pub fn complete(&self) -> ProjectionResult<()> {
let version = {
let mut state = self.lock_state()?;
state.closed = true;
u64::try_from(state.envelopes.len())
.unwrap_or(u64::MAX)
.saturating_add(1)
};
self.notifications.send_replace(version);
Ok(())
}
pub fn fail_next_materialization(&self, error: ProjectionError) -> ProjectionResult<()> {
self.lock_state()?.materialization_failures.push_back(error);
Ok(())
}
pub fn fail_next_source_after(
&self,
after_envelopes: usize,
error: ProjectionError,
) -> ProjectionResult<()> {
self.lock_state()?
.stream_failures
.push_back(TestSourceFailure {
after_envelopes,
error,
});
Ok(())
}
pub fn materializations(&self) -> ProjectionResult<usize> {
Ok(self.lock_state()?.materializations)
}
fn lock_state(&self) -> ProjectionResult<std::sync::MutexGuard<'_, TestSourceState<Payload>>> {
self.state
.lock()
.map_err(|_| ProjectionError::Runtime("test source lock poisoned".into()))
}
}
#[derive(Clone)]
pub struct ManualPaceProbe {
permits: Arc<Semaphore>,
}
impl ManualPaceProbe {
pub fn request_one(&self) -> ProjectionResult<()> {
self.request(1)
}
pub fn request(&self, count: usize) -> ProjectionResult<()> {
if count > Semaphore::MAX_PERMITS.saturating_sub(self.permits.available_permits()) {
return Err(ProjectionError::invalid_config(
"testkit.manual_pace.request",
"requested permits exceed the Tokio semaphore limit",
));
}
self.permits.add_permits(count);
Ok(())
}
}
struct TestStreamCursor<Payload> {
state: Arc<Mutex<TestSourceState<Payload>>>,
notifications: watch::Receiver<u64>,
next_index: usize,
current_limit: Option<usize>,
permits: Option<Arc<Semaphore>>,
emitted: usize,
failure: Option<TestSourceFailure>,
}
#[async_trait::async_trait]
impl<Payload> SourceProvider<TestEnvelope<Payload>> for TestSourceProvider<Payload>
where
Payload: Clone + Send + Sync + 'static,
{
fn manifest(&self) -> &ProviderManifest {
&self.manifest
}
async fn source_from(
&self,
cursor: Option<&ProjectionCursor>,
mode: QueryMode,
) -> ProjectionResult<super::ProjectionSource<TestEnvelope<Payload>>> {
if let Some(cursor) = cursor {
cursor.ensure_manifest(&self.manifest)?;
}
let resume_position = cursor
.map(|cursor| parse_test_position(cursor.position()))
.transpose()?;
let (next_index, current_limit, failure) = {
let mut state = self.lock_state()?;
state.materializations = state.materializations.saturating_add(1);
if let Some(error) = state.materialization_failures.pop_front() {
return Err(error);
}
let next_index = state
.envelopes
.iter()
.position(|envelope| {
resume_position.is_none_or(|position| {
if self.inclusive_resume {
envelope.position >= position
} else {
envelope.position > position
}
})
})
.unwrap_or(state.envelopes.len());
let current_limit = (mode == QueryMode::Current).then_some(state.envelopes.len());
(next_index, current_limit, state.stream_failures.pop_front())
};
let stream_state = TestStreamCursor {
state: Arc::clone(&self.state),
notifications: self.notifications.subscribe(),
next_index,
current_limit,
permits: self.permits.clone(),
emitted: 0,
failure,
};
Ok(Box::pin(stream::unfold(
stream_state,
|mut cursor| async move {
loop {
if cursor
.failure
.as_ref()
.is_some_and(|failure| cursor.emitted >= failure.after_envelopes)
&& let Some(failure) = cursor.failure.take()
{
return Some((Err(failure.error), cursor));
}
let observation = match cursor.state.lock() {
Ok(state) => {
if cursor
.current_limit
.is_some_and(|limit| cursor.next_index >= limit)
{
TestObservation::Completed
} else if let Some(envelope) =
state.envelopes.get(cursor.next_index).cloned()
{
TestObservation::Envelope(envelope)
} else if state.closed {
TestObservation::Completed
} else {
TestObservation::Waiting
}
}
Err(_) => TestObservation::Failed(ProjectionError::Runtime(
"test source lock poisoned".into(),
)),
};
match observation {
TestObservation::Envelope(envelope) => {
if let Some(permits) = &cursor.permits {
match Arc::clone(permits).acquire_owned().await {
Ok(permit) => permit.forget(),
Err(_) => return None,
}
}
cursor.next_index = cursor.next_index.saturating_add(1);
cursor.emitted = cursor.emitted.saturating_add(1);
return Some((Ok(envelope), cursor));
}
TestObservation::Completed => return None,
TestObservation::Failed(error) => {
return Some((Err(error), cursor));
}
TestObservation::Waiting => {
if cursor.notifications.changed().await.is_err() {
return None;
}
}
}
}
},
)))
}
fn extract_cursor(
&self,
envelope: &TestEnvelope<Payload>,
) -> ProjectionResult<ProjectionCursor> {
Ok(ProjectionCursor::new(
self.manifest.clone(),
DeduplicationBoundary::with_consumed(
envelope.position.to_string(),
[BoundaryEvent::new(
envelope.persistence_id.clone(),
envelope.sequence_nr,
)],
),
))
}
fn compare_positions(
&self,
left: &CursorPosition,
right: &CursorPosition,
) -> ProjectionResult<CmpOrdering> {
Ok(parse_test_position(left)?.cmp(&parse_test_position(right)?))
}
}
enum TestObservation<Payload> {
Envelope(TestEnvelope<Payload>),
Completed,
Failed(ProjectionError),
Waiting,
}
fn parse_test_position(position: &CursorPosition) -> ProjectionResult<u64> {
position.as_str().parse::<u64>().map_err(|_| {
ProjectionError::CursorCodec(format!(
"test source position {:?} is not an unsigned integer",
position.as_str()
))
})
}
#[derive(Clone)]
pub struct ProjectionTestKit {
runtime: ProjectionRuntime,
max: Duration,
interval: Duration,
}
impl ProjectionTestKit {
#[must_use]
pub fn new(runtime: ProjectionRuntime) -> Self {
Self {
runtime,
max: Duration::from_secs(3),
interval: Duration::from_millis(100),
}
}
#[must_use]
pub fn with_polling(mut self, max: Duration, interval: Duration) -> Self {
self.max = max;
self.interval = interval;
self
}
pub async fn run<Envelope, F>(
&self,
projection: super::Projection<Envelope>,
mut assertion: F,
) -> ProjectionResult<()>
where
Envelope: Send + Sync + 'static,
F: FnMut() -> bool,
{
let handle = projection.spawn(self.runtime.clone()).await?;
let started = Instant::now();
loop {
if assertion() {
if !handle.status().is_terminal() {
handle.stop_and_wait().await?;
}
return Ok(());
}
if let super::ProjectionStatus::Failed(error) = handle.status() {
let _ = handle.wait().await;
return Err(error);
}
if started.elapsed() >= self.max {
if !handle.status().is_terminal() {
let _ = handle.stop_and_wait().await;
}
return Err(ProjectionError::TestKitTimeout {
timeout_ms: u64::try_from(self.max.as_millis()).unwrap_or(u64::MAX),
});
}
tokio::time::sleep(self.interval).await;
}
}
}
pub async fn assert_projection_offset_store_semantic_conformance(
store: &dyn ProjectionOffsetStore,
) -> ProjectionResult<()> {
assert_cursor_codec_conformance()?;
let namespace = TEST_NAMESPACE.fetch_add(1, Ordering::Relaxed);
let first_id = test_id("isolation-first", namespace);
let second_id = test_id("isolation-second", namespace);
let first_generation = store.claim(&first_id).await?;
let second_generation = store.claim(&second_id).await?;
let first_cursor = test_cursor("sequence", 11, 2);
let second_cursor = test_cursor("timestamp", 29, 3);
store
.save(&first_id, first_generation, None, first_cursor.clone())
.await?;
store
.save(&second_id, second_generation, None, second_cursor.clone())
.await?;
assert_eq!(
store.load(&first_id).await?,
Some(first_cursor),
"saving one projection id must not affect another"
);
assert_eq!(
store.load(&second_id).await?,
Some(second_cursor),
"each projection id must retain its own cursor"
);
let fenced_id = test_id("generation-fencing", namespace);
let superseded = store.claim(&fenced_id).await?;
let current = store.claim(&fenced_id).await?;
assert!(
current > superseded,
"a later claim must return a greater generation"
);
let stale_save = store
.save(&fenced_id, superseded, None, test_cursor("sequence", 31, 1))
.await
.expect_err("a superseded claimant must not save");
assert_fenced(stale_save, &fenced_id, superseded, current);
let stale_clear = store
.clear(&fenced_id, superseded)
.await
.expect_err("a superseded claimant must not clear");
assert_fenced(stale_clear, &fenced_id, superseded, current);
store
.save(&fenced_id, current, None, test_cursor("sequence", 32, 1))
.await?;
let cas_id = test_id("expected-position", namespace);
let cas_generation = store.claim(&cas_id).await?;
let before = test_cursor("sequence", 41, 1);
let after = test_cursor("sequence", 42, 1);
store
.save(&cas_id, cas_generation, None, before.clone())
.await?;
let wrong_position = CursorPosition::new("position-that-was-never-loaded");
let conflict = store
.save(
&cas_id,
cas_generation,
Some(&wrong_position),
after.clone(),
)
.await
.expect_err("an unexpected current position must reject the save");
assert!(matches!(
conflict,
ProjectionError::UnexpectedPosition {
projection_id,
expected: Some(_),
actual: Some(_),
} if projection_id == cas_id
));
assert_eq!(
store.load(&cas_id).await?,
Some(before.clone()),
"a rejected CAS must not change the cursor"
);
store
.save(
&cas_id,
cas_generation,
Some(before.position()),
after.clone(),
)
.await?;
assert_eq!(store.load(&cas_id).await?, Some(after));
let management_id = test_id("clear-management", namespace);
let management_generation = store.claim(&management_id).await?;
let cursor = test_cursor("sequence", 51, 1);
store
.save(&management_id, management_generation, None, cursor)
.await?;
store.set_paused(&management_id, true).await?;
store.clear(&management_id, management_generation).await?;
assert_eq!(
store.load(&management_id).await?,
None,
"clear must remove the durable cursor"
);
assert!(
store.load_paused(&management_id).await?,
"clear must not remove or reset the management row"
);
store.set_paused(&management_id, false).await?;
assert!(
!store.load_paused(&management_id).await?,
"paused must be independently mutable after cursor clear"
);
assert_eq!(
store.load(&management_id).await?,
None,
"changing paused must not recreate a cursor"
);
let paused_without_cursor_id = test_id("paused-without-cursor", namespace);
assert!(!store.load_paused(&paused_without_cursor_id).await?);
store.set_paused(&paused_without_cursor_id, true).await?;
assert!(store.load_paused(&paused_without_cursor_id).await?);
assert_eq!(
store.load(&paused_without_cursor_id).await?,
None,
"management state must exist independently of an offset row"
);
let concurrent_id = test_id("same-key-concurrency", namespace);
let (left_claim, right_claim) =
join(store.claim(&concurrent_id), store.claim(&concurrent_id)).await;
let left_generation = left_claim?;
let right_generation = right_claim?;
assert_ne!(
left_generation, right_generation,
"atomic same-key claims must return distinct generations"
);
let left_cursor = test_cursor("sequence", 61, 1);
let right_cursor = test_cursor("sequence", 62, 1);
let (left_save, right_save) = join(
store.save(&concurrent_id, left_generation, None, left_cursor.clone()),
store.save(&concurrent_id, right_generation, None, right_cursor.clone()),
)
.await;
assert_eq!(
usize::from(left_save.is_ok()) + usize::from(right_save.is_ok()),
1,
"two claimants for one key must not both save"
);
let (winning_generation, winning_cursor, losing_result) = if left_generation > right_generation
{
(left_generation, left_cursor, right_save)
} else {
(right_generation, right_cursor, left_save)
};
assert_eq!(
store.load(&concurrent_id).await?,
Some(winning_cursor),
"only the highest claimed generation may save"
);
assert_fenced(
losing_result.expect_err("the lower generation must lose fencing"),
&concurrent_id,
if winning_generation == left_generation {
right_generation
} else {
left_generation
},
winning_generation,
);
Ok(())
}
pub fn assert_cursor_codec_conformance() -> ProjectionResult<()> {
for provider in [
"datum-test-provider/sequence-v1",
"datum-test-provider/per-id-v1",
"datum-test-provider/timestamp-v1",
] {
let manifest = ProviderManifest::new(provider)?;
for marker in 0_u64..64 {
let event_count = usize::try_from(marker % 5).expect("test count fits usize");
let consumed = (0..event_count)
.map(|index| {
BoundaryEvent::new(
PersistenceId::from_unique(format!(
"test-entity|{provider}|{marker}|{index}"
))
.expect("test persistence id is valid"),
SequenceNr::new(
marker
.saturating_mul(10)
.saturating_add(u64::try_from(index).expect("index fits u64")),
),
)
})
.collect::<Vec<_>>();
let cursor = ProjectionCursor::new(
manifest.clone(),
DeduplicationBoundary::with_consumed(
format!("position|{marker}:λ,{}", marker.saturating_mul(17)),
consumed,
),
);
let encoded = cursor.encode();
let decoded = ProjectionCursor::decode(&encoded)?;
assert_eq!(
decoded, cursor,
"cursor codec must round-trip for provider manifest {provider}"
);
assert_eq!(
decoded.encode(),
encoded,
"cursor codec output must be stable for provider manifest {provider}"
);
decoded.ensure_manifest(&manifest)?;
}
}
Ok(())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum SaveCrashPoint {
BeforeCommit,
AfterCommitBeforeAcknowledgement,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum SaveCrashOutcome {
NotCommitted,
CommitOutcomeUncertain,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CrashSaveRequest {
pub projection_id: ProjectionId,
pub generation: ProjectionGeneration,
pub expected_position: Option<CursorPosition>,
pub cursor: ProjectionCursor,
}
#[async_trait::async_trait]
pub trait DurableProjectionOffsetStoreHarness: Send + Sync {
async fn create_new(&self) -> ProjectionResult<Box<dyn ProjectionOffsetStore>>;
async fn reopen_existing(&self) -> ProjectionResult<Box<dyn ProjectionOffsetStore>>;
async fn exercise_reopen_migration(&self) -> ProjectionResult<()>;
async fn save_with_crash(
&self,
request: CrashSaveRequest,
crash_point: SaveCrashPoint,
) -> ProjectionResult<SaveCrashOutcome>;
}
pub async fn assert_projection_offset_store_durable_conformance(
harness: &dyn DurableProjectionOffsetStoreHarness,
) -> ProjectionResult<()> {
let namespace = TEST_NAMESPACE.fetch_add(1, Ordering::Relaxed);
let projection_id = test_id("durable-reopen", namespace);
let initial_cursor = test_cursor("durable", 71, 2);
let store = harness.create_new().await?;
let initial_generation = store.claim(&projection_id).await?;
store
.save(
&projection_id,
initial_generation,
None,
initial_cursor.clone(),
)
.await?;
store.set_paused(&projection_id, true).await?;
drop(store);
let reopened = harness.reopen_existing().await?;
assert_eq!(
reopened.load(&projection_id).await?,
Some(initial_cursor.clone()),
"cursor must survive complete store close/reopen"
);
assert!(
reopened.load_paused(&projection_id).await?,
"paused management state must survive complete store close/reopen"
);
let reopened_generation = reopened.claim(&projection_id).await?;
assert_eq!(
reopened_generation.as_u64(),
initial_generation
.as_u64()
.checked_add(1)
.expect("test generation cannot exhaust"),
"generation must survive reopen and advance monotonically"
);
let stale = reopened
.save(
&projection_id,
initial_generation,
Some(initial_cursor.position()),
test_cursor("durable", 72, 1),
)
.await
.expect_err("a pre-reopen generation must remain fenced");
assert_fenced(
stale,
&projection_id,
initial_generation,
reopened_generation,
);
let second_cursor = test_cursor("durable", 73, 3);
reopened
.save(
&projection_id,
reopened_generation,
Some(initial_cursor.position()),
second_cursor.clone(),
)
.await?;
drop(reopened);
let reopened = harness.reopen_existing().await?;
assert_eq!(
reopened.load(&projection_id).await?,
Some(second_cursor.clone())
);
assert!(reopened.load_paused(&projection_id).await?);
let before_crash_generation = reopened.claim(&projection_id).await?;
drop(reopened);
let not_committed_cursor = test_cursor("durable", 74, 1);
let outcome = harness
.save_with_crash(
CrashSaveRequest {
projection_id: projection_id.clone(),
generation: before_crash_generation,
expected_position: Some(second_cursor.position().clone()),
cursor: not_committed_cursor,
},
SaveCrashPoint::BeforeCommit,
)
.await?;
assert_eq!(
outcome,
SaveCrashOutcome::NotCommitted,
"the pre-commit crash point must have a deterministic not-committed result"
);
let reopened = harness.reopen_existing().await?;
assert_eq!(
reopened.load(&projection_id).await?,
Some(second_cursor.clone()),
"a pre-commit crash must preserve the previous cursor"
);
let ambiguous_generation = reopened.claim(&projection_id).await?;
drop(reopened);
let committed_cursor = test_cursor("durable", 75, 4);
let outcome = harness
.save_with_crash(
CrashSaveRequest {
projection_id: projection_id.clone(),
generation: ambiguous_generation,
expected_position: Some(second_cursor.position().clone()),
cursor: committed_cursor.clone(),
},
SaveCrashPoint::AfterCommitBeforeAcknowledgement,
)
.await?;
assert_eq!(
outcome,
SaveCrashOutcome::CommitOutcomeUncertain,
"losing the post-commit acknowledgement must remain distinguishable from success"
);
let reopened = harness.reopen_existing().await?;
assert_eq!(
reopened.load(&projection_id).await?,
Some(committed_cursor),
"reloading must resolve the injected ambiguous outcome as committed"
);
assert!(
reopened.load_paused(&projection_id).await?,
"crash injection must not disturb independent management state"
);
drop(reopened);
harness.exercise_reopen_migration().await
}
fn test_id(kind: &str, namespace: u64) -> ProjectionId {
ProjectionId::new("datum-projection-tck", format!("{kind}-{namespace}"))
}
fn test_cursor(provider: &str, position: u64, consumed_count: usize) -> ProjectionCursor {
let manifest = ProviderManifest::new(format!("datum-tck-provider/{provider}-v1"))
.expect("test provider manifest is valid");
let consumed = (0..consumed_count).map(|index| {
BoundaryEvent::new(
PersistenceId::from_unique(format!(
"datum-projection-tck|{provider}-{position}-{index}"
))
.expect("test persistence id is valid"),
SequenceNr::new(u64::try_from(index + 1).expect("test index fits u64")),
)
});
ProjectionCursor::new(
manifest,
DeduplicationBoundary::with_consumed(format!("{provider}-position-{position}"), consumed),
)
}
fn assert_fenced(
error: ProjectionError,
projection_id: &ProjectionId,
attempted: ProjectionGeneration,
current: ProjectionGeneration,
) {
assert!(
matches!(
error,
ProjectionError::Fenced {
projection_id: ref error_id,
attempted_generation,
current_generation,
} if error_id == projection_id
&& attempted_generation == attempted.as_u64()
&& current_generation == current.as_u64()
),
"stale generation must produce a precise fencing error"
);
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use crate::projection::{Handler, MemoryProjectionOffsetStore, Projection, ProjectionStatus};
struct CountingHandler(Arc<AtomicUsize>);
#[async_trait::async_trait]
impl Handler<TestEnvelope<usize>> for CountingHandler {
async fn process(&mut self, _envelope: &TestEnvelope<usize>) -> ProjectionResult<()> {
self.0.fetch_add(1, Ordering::Relaxed);
Ok(())
}
}
#[tokio::test]
async fn memory_store_passes_projection_semantic_conformance() -> ProjectionResult<()> {
assert_projection_offset_store_semantic_conformance(&MemoryProjectionOffsetStore::new())
.await
}
#[tokio::test]
async fn fidelity_12_assert_poll_times_out_on_an_empty_source() {
let store = Arc::new(MemoryProjectionOffsetStore::new());
let runtime = ProjectionRuntime::new(Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>);
let projection = Projection::at_least_once(
ProjectionId::new(
"testkit",
format!("empty-{}", TEST_NAMESPACE.fetch_add(1, Ordering::Relaxed)),
),
TestSourceProvider::new(Vec::<TestEnvelope<usize>>::new()),
|| CountingHandler(Arc::new(AtomicUsize::new(0))),
);
let result = ProjectionTestKit::new(runtime)
.with_polling(Duration::from_millis(25), Duration::from_millis(2))
.run(projection, || false)
.await;
assert!(matches!(
result,
Err(ProjectionError::TestKitTimeout { .. })
));
}
#[tokio::test]
async fn testkit_surfaces_terminal_projection_failure_without_timeout_masking() {
let store = Arc::new(MemoryProjectionOffsetStore::new());
let runtime = ProjectionRuntime::new(Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>);
let provider = TestSourceProvider::new(Vec::<TestEnvelope<usize>>::new());
provider
.fail_next_materialization(ProjectionError::CursorCodec("corrupt".into()))
.unwrap();
let projection = Projection::at_least_once(
ProjectionId::new(
"testkit",
format!(
"terminal-failure-{}",
TEST_NAMESPACE.fetch_add(1, Ordering::Relaxed)
),
),
provider,
|| CountingHandler(Arc::new(AtomicUsize::new(0))),
);
let result = ProjectionTestKit::new(runtime)
.with_polling(Duration::from_secs(1), Duration::from_millis(2))
.run(projection, || false)
.await;
assert!(matches!(result, Err(ProjectionError::CursorCodec(_))));
}
#[tokio::test]
async fn fidelity_12_manual_pace_emits_only_after_probe_demand() {
let test_envelope = TestEnvelope::new(
1,
PersistenceId::from_unique("testkit|manual").expect("test id is valid"),
SequenceNr::new(1),
1_usize,
);
let (provider, probe) = TestSourceProvider::new(vec![test_envelope]).with_manual_pace();
let count = Arc::new(AtomicUsize::new(0));
let handler_count = Arc::clone(&count);
let store = Arc::new(MemoryProjectionOffsetStore::new());
let projection = Projection::at_least_once(
ProjectionId::new(
"testkit",
format!("manual-{}", TEST_NAMESPACE.fetch_add(1, Ordering::Relaxed)),
),
provider,
move || CountingHandler(Arc::clone(&handler_count)),
)
.with_query_mode(QueryMode::Current);
let handle = projection
.spawn(ProjectionRuntime::new(
Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>
))
.await
.unwrap();
tokio::task::yield_now().await;
assert_eq!(count.load(Ordering::Relaxed), 0);
probe.request_one().unwrap();
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), handle.wait())
.await
.expect("manual-paced current source must complete")
.unwrap(),
ProjectionStatus::Completed
);
assert_eq!(count.load(Ordering::Relaxed), 1);
}
}