1use std::{
9 cmp::Ordering as CmpOrdering,
10 collections::VecDeque,
11 sync::{
12 Arc, Mutex,
13 atomic::{AtomicU64, Ordering},
14 },
15 time::{Duration, Instant},
16};
17
18use futures_util::{future::join, stream};
19use tokio::sync::{Semaphore, watch};
20
21use crate::query::QueryMode;
22use crate::{PersistenceId, SequenceNr};
23
24use super::{
25 BoundaryEvent, CursorPosition, DeduplicationBoundary, ProjectionCursor, ProjectionError,
26 ProjectionGeneration, ProjectionId, ProjectionOffsetStore, ProjectionResult, ProjectionRuntime,
27 ProviderManifest, SourceProvider,
28};
29
30static TEST_NAMESPACE: AtomicU64 = AtomicU64::new(1);
31
32#[derive(Debug, Clone, PartialEq, Eq)]
34pub struct TestEnvelope<Payload> {
35 pub position: u64,
37 pub persistence_id: PersistenceId,
39 pub sequence_nr: SequenceNr,
41 pub payload: Payload,
43}
44
45impl<Payload> TestEnvelope<Payload> {
46 #[must_use]
48 pub fn new(
49 position: u64,
50 persistence_id: PersistenceId,
51 sequence_nr: SequenceNr,
52 payload: Payload,
53 ) -> Self {
54 Self {
55 position,
56 persistence_id,
57 sequence_nr,
58 payload,
59 }
60 }
61}
62
63struct TestSourceFailure {
64 after_envelopes: usize,
65 error: ProjectionError,
66}
67
68struct TestSourceState<Payload> {
69 envelopes: Vec<TestEnvelope<Payload>>,
70 closed: bool,
71 materializations: usize,
72 materialization_failures: VecDeque<ProjectionError>,
73 stream_failures: VecDeque<TestSourceFailure>,
74}
75
76#[derive(Clone)]
83pub struct TestSourceProvider<Payload> {
84 manifest: ProviderManifest,
85 state: Arc<Mutex<TestSourceState<Payload>>>,
86 notifications: watch::Sender<u64>,
87 inclusive_resume: bool,
88 permits: Option<Arc<Semaphore>>,
89}
90
91impl<Payload> TestSourceProvider<Payload> {
92 #[must_use]
94 pub fn new(envelopes: Vec<TestEnvelope<Payload>>) -> Self {
95 Self::with_resume_mode(envelopes, false)
96 }
97
98 #[must_use]
100 pub fn inclusive(envelopes: Vec<TestEnvelope<Payload>>) -> Self {
101 Self::with_resume_mode(envelopes, true)
102 }
103
104 fn with_resume_mode(envelopes: Vec<TestEnvelope<Payload>>, inclusive_resume: bool) -> Self {
105 let (notifications, _receiver) = watch::channel(0);
106 Self {
107 manifest: ProviderManifest::new("datum-test-source/position-v1")
108 .expect("the fixed test provider manifest is valid"),
109 state: Arc::new(Mutex::new(TestSourceState {
110 envelopes,
111 closed: false,
112 materializations: 0,
113 materialization_failures: VecDeque::new(),
114 stream_failures: VecDeque::new(),
115 })),
116 notifications,
117 inclusive_resume,
118 permits: None,
119 }
120 }
121
122 #[must_use]
124 pub fn with_manual_pace(mut self) -> (Self, ManualPaceProbe) {
125 let permits = Arc::new(Semaphore::new(0));
126 self.permits = Some(Arc::clone(&permits));
127 (self, ManualPaceProbe { permits })
128 }
129
130 #[must_use]
132 pub fn without_manual_pace(mut self) -> Self {
133 self.permits = None;
134 self
135 }
136
137 pub fn push(&self, envelope: TestEnvelope<Payload>) -> ProjectionResult<()> {
139 let version = {
140 let mut state = self.lock_state()?;
141 state.envelopes.push(envelope);
142 u64::try_from(state.envelopes.len()).unwrap_or(u64::MAX)
143 };
144 self.notifications.send_replace(version);
145 Ok(())
146 }
147
148 pub fn complete(&self) -> ProjectionResult<()> {
150 let version = {
151 let mut state = self.lock_state()?;
152 state.closed = true;
153 u64::try_from(state.envelopes.len())
154 .unwrap_or(u64::MAX)
155 .saturating_add(1)
156 };
157 self.notifications.send_replace(version);
158 Ok(())
159 }
160
161 pub fn fail_next_materialization(&self, error: ProjectionError) -> ProjectionResult<()> {
163 self.lock_state()?.materialization_failures.push_back(error);
164 Ok(())
165 }
166
167 pub fn fail_next_source_after(
169 &self,
170 after_envelopes: usize,
171 error: ProjectionError,
172 ) -> ProjectionResult<()> {
173 self.lock_state()?
174 .stream_failures
175 .push_back(TestSourceFailure {
176 after_envelopes,
177 error,
178 });
179 Ok(())
180 }
181
182 pub fn materializations(&self) -> ProjectionResult<usize> {
184 Ok(self.lock_state()?.materializations)
185 }
186
187 fn lock_state(&self) -> ProjectionResult<std::sync::MutexGuard<'_, TestSourceState<Payload>>> {
188 self.state
189 .lock()
190 .map_err(|_| ProjectionError::Runtime("test source lock poisoned".into()))
191 }
192}
193
194#[derive(Clone)]
196pub struct ManualPaceProbe {
197 permits: Arc<Semaphore>,
198}
199
200impl ManualPaceProbe {
201 pub fn request_one(&self) -> ProjectionResult<()> {
203 self.request(1)
204 }
205
206 pub fn request(&self, count: usize) -> ProjectionResult<()> {
208 if count > Semaphore::MAX_PERMITS.saturating_sub(self.permits.available_permits()) {
209 return Err(ProjectionError::invalid_config(
210 "testkit.manual_pace.request",
211 "requested permits exceed the Tokio semaphore limit",
212 ));
213 }
214 self.permits.add_permits(count);
215 Ok(())
216 }
217}
218
219struct TestStreamCursor<Payload> {
220 state: Arc<Mutex<TestSourceState<Payload>>>,
221 notifications: watch::Receiver<u64>,
222 next_index: usize,
223 current_limit: Option<usize>,
224 permits: Option<Arc<Semaphore>>,
225 emitted: usize,
226 failure: Option<TestSourceFailure>,
227}
228
229#[async_trait::async_trait]
230impl<Payload> SourceProvider<TestEnvelope<Payload>> for TestSourceProvider<Payload>
231where
232 Payload: Clone + Send + Sync + 'static,
233{
234 fn manifest(&self) -> &ProviderManifest {
235 &self.manifest
236 }
237
238 async fn source_from(
239 &self,
240 cursor: Option<&ProjectionCursor>,
241 mode: QueryMode,
242 ) -> ProjectionResult<super::ProjectionSource<TestEnvelope<Payload>>> {
243 if let Some(cursor) = cursor {
244 cursor.ensure_manifest(&self.manifest)?;
245 }
246 let resume_position = cursor
247 .map(|cursor| parse_test_position(cursor.position()))
248 .transpose()?;
249 let (next_index, current_limit, failure) = {
250 let mut state = self.lock_state()?;
251 state.materializations = state.materializations.saturating_add(1);
252 if let Some(error) = state.materialization_failures.pop_front() {
253 return Err(error);
254 }
255 let next_index = state
256 .envelopes
257 .iter()
258 .position(|envelope| {
259 resume_position.is_none_or(|position| {
260 if self.inclusive_resume {
261 envelope.position >= position
262 } else {
263 envelope.position > position
264 }
265 })
266 })
267 .unwrap_or(state.envelopes.len());
268 let current_limit = (mode == QueryMode::Current).then_some(state.envelopes.len());
269 (next_index, current_limit, state.stream_failures.pop_front())
270 };
271 let stream_state = TestStreamCursor {
272 state: Arc::clone(&self.state),
273 notifications: self.notifications.subscribe(),
274 next_index,
275 current_limit,
276 permits: self.permits.clone(),
277 emitted: 0,
278 failure,
279 };
280 Ok(Box::pin(stream::unfold(
281 stream_state,
282 |mut cursor| async move {
283 loop {
284 if cursor
285 .failure
286 .as_ref()
287 .is_some_and(|failure| cursor.emitted >= failure.after_envelopes)
288 && let Some(failure) = cursor.failure.take()
289 {
290 return Some((Err(failure.error), cursor));
291 }
292
293 let observation = match cursor.state.lock() {
294 Ok(state) => {
295 if cursor
296 .current_limit
297 .is_some_and(|limit| cursor.next_index >= limit)
298 {
299 TestObservation::Completed
300 } else if let Some(envelope) =
301 state.envelopes.get(cursor.next_index).cloned()
302 {
303 TestObservation::Envelope(envelope)
304 } else if state.closed {
305 TestObservation::Completed
306 } else {
307 TestObservation::Waiting
308 }
309 }
310 Err(_) => TestObservation::Failed(ProjectionError::Runtime(
311 "test source lock poisoned".into(),
312 )),
313 };
314 match observation {
315 TestObservation::Envelope(envelope) => {
316 if let Some(permits) = &cursor.permits {
317 match Arc::clone(permits).acquire_owned().await {
318 Ok(permit) => permit.forget(),
319 Err(_) => return None,
320 }
321 }
322 cursor.next_index = cursor.next_index.saturating_add(1);
323 cursor.emitted = cursor.emitted.saturating_add(1);
324 return Some((Ok(envelope), cursor));
325 }
326 TestObservation::Completed => return None,
327 TestObservation::Failed(error) => {
328 return Some((Err(error), cursor));
329 }
330 TestObservation::Waiting => {
331 if cursor.notifications.changed().await.is_err() {
332 return None;
333 }
334 }
335 }
336 }
337 },
338 )))
339 }
340
341 fn extract_cursor(
342 &self,
343 envelope: &TestEnvelope<Payload>,
344 ) -> ProjectionResult<ProjectionCursor> {
345 Ok(ProjectionCursor::new(
346 self.manifest.clone(),
347 DeduplicationBoundary::with_consumed(
348 envelope.position.to_string(),
349 [BoundaryEvent::new(
350 envelope.persistence_id.clone(),
351 envelope.sequence_nr,
352 )],
353 ),
354 ))
355 }
356
357 fn compare_positions(
358 &self,
359 left: &CursorPosition,
360 right: &CursorPosition,
361 ) -> ProjectionResult<CmpOrdering> {
362 Ok(parse_test_position(left)?.cmp(&parse_test_position(right)?))
363 }
364}
365
366enum TestObservation<Payload> {
367 Envelope(TestEnvelope<Payload>),
368 Completed,
369 Failed(ProjectionError),
370 Waiting,
371}
372
373fn parse_test_position(position: &CursorPosition) -> ProjectionResult<u64> {
374 position.as_str().parse::<u64>().map_err(|_| {
375 ProjectionError::CursorCodec(format!(
376 "test source position {:?} is not an unsigned integer",
377 position.as_str()
378 ))
379 })
380}
381
382#[derive(Clone)]
384pub struct ProjectionTestKit {
385 runtime: ProjectionRuntime,
386 max: Duration,
387 interval: Duration,
388}
389
390impl ProjectionTestKit {
391 #[must_use]
393 pub fn new(runtime: ProjectionRuntime) -> Self {
394 Self {
395 runtime,
396 max: Duration::from_secs(3),
397 interval: Duration::from_millis(100),
398 }
399 }
400
401 #[must_use]
403 pub fn with_polling(mut self, max: Duration, interval: Duration) -> Self {
404 self.max = max;
405 self.interval = interval;
406 self
407 }
408
409 pub async fn run<Envelope, F>(
416 &self,
417 projection: super::Projection<Envelope>,
418 mut assertion: F,
419 ) -> ProjectionResult<()>
420 where
421 Envelope: Send + Sync + 'static,
422 F: FnMut() -> bool,
423 {
424 let handle = projection.spawn(self.runtime.clone()).await?;
425 let started = Instant::now();
426 loop {
427 if assertion() {
428 if !handle.status().is_terminal() {
429 handle.stop_and_wait().await?;
430 }
431 return Ok(());
432 }
433 if let super::ProjectionStatus::Failed(error) = handle.status() {
434 let _ = handle.wait().await;
435 return Err(error);
436 }
437 if started.elapsed() >= self.max {
438 if !handle.status().is_terminal() {
439 let _ = handle.stop_and_wait().await;
440 }
441 return Err(ProjectionError::TestKitTimeout {
442 timeout_ms: u64::try_from(self.max.as_millis()).unwrap_or(u64::MAX),
443 });
444 }
445 tokio::time::sleep(self.interval).await;
446 }
447 }
448}
449
450pub async fn assert_projection_offset_store_semantic_conformance(
459 store: &dyn ProjectionOffsetStore,
460) -> ProjectionResult<()> {
461 assert_cursor_codec_conformance()?;
462 let namespace = TEST_NAMESPACE.fetch_add(1, Ordering::Relaxed);
463
464 let first_id = test_id("isolation-first", namespace);
465 let second_id = test_id("isolation-second", namespace);
466 let first_generation = store.claim(&first_id).await?;
467 let second_generation = store.claim(&second_id).await?;
468 let first_cursor = test_cursor("sequence", 11, 2);
469 let second_cursor = test_cursor("timestamp", 29, 3);
470 store
471 .save(&first_id, first_generation, None, first_cursor.clone())
472 .await?;
473 store
474 .save(&second_id, second_generation, None, second_cursor.clone())
475 .await?;
476 assert_eq!(
477 store.load(&first_id).await?,
478 Some(first_cursor),
479 "saving one projection id must not affect another"
480 );
481 assert_eq!(
482 store.load(&second_id).await?,
483 Some(second_cursor),
484 "each projection id must retain its own cursor"
485 );
486
487 let fenced_id = test_id("generation-fencing", namespace);
488 let superseded = store.claim(&fenced_id).await?;
489 let current = store.claim(&fenced_id).await?;
490 assert!(
491 current > superseded,
492 "a later claim must return a greater generation"
493 );
494 let stale_save = store
495 .save(&fenced_id, superseded, None, test_cursor("sequence", 31, 1))
496 .await
497 .expect_err("a superseded claimant must not save");
498 assert_fenced(stale_save, &fenced_id, superseded, current);
499 let stale_clear = store
500 .clear(&fenced_id, superseded)
501 .await
502 .expect_err("a superseded claimant must not clear");
503 assert_fenced(stale_clear, &fenced_id, superseded, current);
504 store
505 .save(&fenced_id, current, None, test_cursor("sequence", 32, 1))
506 .await?;
507
508 let cas_id = test_id("expected-position", namespace);
509 let cas_generation = store.claim(&cas_id).await?;
510 let before = test_cursor("sequence", 41, 1);
511 let after = test_cursor("sequence", 42, 1);
512 store
513 .save(&cas_id, cas_generation, None, before.clone())
514 .await?;
515 let wrong_position = CursorPosition::new("position-that-was-never-loaded");
516 let conflict = store
517 .save(
518 &cas_id,
519 cas_generation,
520 Some(&wrong_position),
521 after.clone(),
522 )
523 .await
524 .expect_err("an unexpected current position must reject the save");
525 assert!(matches!(
526 conflict,
527 ProjectionError::UnexpectedPosition {
528 projection_id,
529 expected: Some(_),
530 actual: Some(_),
531 } if projection_id == cas_id
532 ));
533 assert_eq!(
534 store.load(&cas_id).await?,
535 Some(before.clone()),
536 "a rejected CAS must not change the cursor"
537 );
538 store
539 .save(
540 &cas_id,
541 cas_generation,
542 Some(before.position()),
543 after.clone(),
544 )
545 .await?;
546 assert_eq!(store.load(&cas_id).await?, Some(after));
547
548 let management_id = test_id("clear-management", namespace);
549 let management_generation = store.claim(&management_id).await?;
550 let cursor = test_cursor("sequence", 51, 1);
551 store
552 .save(&management_id, management_generation, None, cursor)
553 .await?;
554 store.set_paused(&management_id, true).await?;
555 store.clear(&management_id, management_generation).await?;
556 assert_eq!(
557 store.load(&management_id).await?,
558 None,
559 "clear must remove the durable cursor"
560 );
561 assert!(
562 store.load_paused(&management_id).await?,
563 "clear must not remove or reset the management row"
564 );
565 store.set_paused(&management_id, false).await?;
566 assert!(
567 !store.load_paused(&management_id).await?,
568 "paused must be independently mutable after cursor clear"
569 );
570 assert_eq!(
571 store.load(&management_id).await?,
572 None,
573 "changing paused must not recreate a cursor"
574 );
575
576 let paused_without_cursor_id = test_id("paused-without-cursor", namespace);
577 assert!(!store.load_paused(&paused_without_cursor_id).await?);
578 store.set_paused(&paused_without_cursor_id, true).await?;
579 assert!(store.load_paused(&paused_without_cursor_id).await?);
580 assert_eq!(
581 store.load(&paused_without_cursor_id).await?,
582 None,
583 "management state must exist independently of an offset row"
584 );
585
586 let concurrent_id = test_id("same-key-concurrency", namespace);
587 let (left_claim, right_claim) =
588 join(store.claim(&concurrent_id), store.claim(&concurrent_id)).await;
589 let left_generation = left_claim?;
590 let right_generation = right_claim?;
591 assert_ne!(
592 left_generation, right_generation,
593 "atomic same-key claims must return distinct generations"
594 );
595 let left_cursor = test_cursor("sequence", 61, 1);
596 let right_cursor = test_cursor("sequence", 62, 1);
597 let (left_save, right_save) = join(
598 store.save(&concurrent_id, left_generation, None, left_cursor.clone()),
599 store.save(&concurrent_id, right_generation, None, right_cursor.clone()),
600 )
601 .await;
602 assert_eq!(
603 usize::from(left_save.is_ok()) + usize::from(right_save.is_ok()),
604 1,
605 "two claimants for one key must not both save"
606 );
607 let (winning_generation, winning_cursor, losing_result) = if left_generation > right_generation
608 {
609 (left_generation, left_cursor, right_save)
610 } else {
611 (right_generation, right_cursor, left_save)
612 };
613 assert_eq!(
614 store.load(&concurrent_id).await?,
615 Some(winning_cursor),
616 "only the highest claimed generation may save"
617 );
618 assert_fenced(
619 losing_result.expect_err("the lower generation must lose fencing"),
620 &concurrent_id,
621 if winning_generation == left_generation {
622 right_generation
623 } else {
624 left_generation
625 },
626 winning_generation,
627 );
628
629 Ok(())
630}
631
632pub fn assert_cursor_codec_conformance() -> ProjectionResult<()> {
638 for provider in [
639 "datum-test-provider/sequence-v1",
640 "datum-test-provider/per-id-v1",
641 "datum-test-provider/timestamp-v1",
642 ] {
643 let manifest = ProviderManifest::new(provider)?;
644 for marker in 0_u64..64 {
645 let event_count = usize::try_from(marker % 5).expect("test count fits usize");
646 let consumed = (0..event_count)
647 .map(|index| {
648 BoundaryEvent::new(
649 PersistenceId::from_unique(format!(
650 "test-entity|{provider}|{marker}|{index}"
651 ))
652 .expect("test persistence id is valid"),
653 SequenceNr::new(
654 marker
655 .saturating_mul(10)
656 .saturating_add(u64::try_from(index).expect("index fits u64")),
657 ),
658 )
659 })
660 .collect::<Vec<_>>();
661 let cursor = ProjectionCursor::new(
662 manifest.clone(),
663 DeduplicationBoundary::with_consumed(
664 format!("position|{marker}:λ,{}", marker.saturating_mul(17)),
665 consumed,
666 ),
667 );
668 let encoded = cursor.encode();
669 let decoded = ProjectionCursor::decode(&encoded)?;
670 assert_eq!(
671 decoded, cursor,
672 "cursor codec must round-trip for provider manifest {provider}"
673 );
674 assert_eq!(
675 decoded.encode(),
676 encoded,
677 "cursor codec output must be stable for provider manifest {provider}"
678 );
679 decoded.ensure_manifest(&manifest)?;
680 }
681 }
682 Ok(())
683}
684
685#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
687pub enum SaveCrashPoint {
688 BeforeCommit,
690 AfterCommitBeforeAcknowledgement,
692}
693
694#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
696pub enum SaveCrashOutcome {
697 NotCommitted,
699 CommitOutcomeUncertain,
703}
704
705#[derive(Debug, Clone, PartialEq, Eq)]
707pub struct CrashSaveRequest {
708 pub projection_id: ProjectionId,
710 pub generation: ProjectionGeneration,
712 pub expected_position: Option<CursorPosition>,
714 pub cursor: ProjectionCursor,
716}
717
718#[async_trait::async_trait]
731pub trait DurableProjectionOffsetStoreHarness: Send + Sync {
732 async fn create_new(&self) -> ProjectionResult<Box<dyn ProjectionOffsetStore>>;
734
735 async fn reopen_existing(&self) -> ProjectionResult<Box<dyn ProjectionOffsetStore>>;
737
738 async fn exercise_reopen_migration(&self) -> ProjectionResult<()>;
740
741 async fn save_with_crash(
743 &self,
744 request: CrashSaveRequest,
745 crash_point: SaveCrashPoint,
746 ) -> ProjectionResult<SaveCrashOutcome>;
747}
748
749pub async fn assert_projection_offset_store_durable_conformance(
757 harness: &dyn DurableProjectionOffsetStoreHarness,
758) -> ProjectionResult<()> {
759 let namespace = TEST_NAMESPACE.fetch_add(1, Ordering::Relaxed);
760 let projection_id = test_id("durable-reopen", namespace);
761 let initial_cursor = test_cursor("durable", 71, 2);
762
763 let store = harness.create_new().await?;
764 let initial_generation = store.claim(&projection_id).await?;
765 store
766 .save(
767 &projection_id,
768 initial_generation,
769 None,
770 initial_cursor.clone(),
771 )
772 .await?;
773 store.set_paused(&projection_id, true).await?;
774 drop(store);
775
776 let reopened = harness.reopen_existing().await?;
777 assert_eq!(
778 reopened.load(&projection_id).await?,
779 Some(initial_cursor.clone()),
780 "cursor must survive complete store close/reopen"
781 );
782 assert!(
783 reopened.load_paused(&projection_id).await?,
784 "paused management state must survive complete store close/reopen"
785 );
786 let reopened_generation = reopened.claim(&projection_id).await?;
787 assert_eq!(
788 reopened_generation.as_u64(),
789 initial_generation
790 .as_u64()
791 .checked_add(1)
792 .expect("test generation cannot exhaust"),
793 "generation must survive reopen and advance monotonically"
794 );
795 let stale = reopened
796 .save(
797 &projection_id,
798 initial_generation,
799 Some(initial_cursor.position()),
800 test_cursor("durable", 72, 1),
801 )
802 .await
803 .expect_err("a pre-reopen generation must remain fenced");
804 assert_fenced(
805 stale,
806 &projection_id,
807 initial_generation,
808 reopened_generation,
809 );
810
811 let second_cursor = test_cursor("durable", 73, 3);
812 reopened
813 .save(
814 &projection_id,
815 reopened_generation,
816 Some(initial_cursor.position()),
817 second_cursor.clone(),
818 )
819 .await?;
820 drop(reopened);
821
822 let reopened = harness.reopen_existing().await?;
823 assert_eq!(
824 reopened.load(&projection_id).await?,
825 Some(second_cursor.clone())
826 );
827 assert!(reopened.load_paused(&projection_id).await?);
828 let before_crash_generation = reopened.claim(&projection_id).await?;
829 drop(reopened);
830
831 let not_committed_cursor = test_cursor("durable", 74, 1);
832 let outcome = harness
833 .save_with_crash(
834 CrashSaveRequest {
835 projection_id: projection_id.clone(),
836 generation: before_crash_generation,
837 expected_position: Some(second_cursor.position().clone()),
838 cursor: not_committed_cursor,
839 },
840 SaveCrashPoint::BeforeCommit,
841 )
842 .await?;
843 assert_eq!(
844 outcome,
845 SaveCrashOutcome::NotCommitted,
846 "the pre-commit crash point must have a deterministic not-committed result"
847 );
848
849 let reopened = harness.reopen_existing().await?;
850 assert_eq!(
851 reopened.load(&projection_id).await?,
852 Some(second_cursor.clone()),
853 "a pre-commit crash must preserve the previous cursor"
854 );
855 let ambiguous_generation = reopened.claim(&projection_id).await?;
856 drop(reopened);
857
858 let committed_cursor = test_cursor("durable", 75, 4);
859 let outcome = harness
860 .save_with_crash(
861 CrashSaveRequest {
862 projection_id: projection_id.clone(),
863 generation: ambiguous_generation,
864 expected_position: Some(second_cursor.position().clone()),
865 cursor: committed_cursor.clone(),
866 },
867 SaveCrashPoint::AfterCommitBeforeAcknowledgement,
868 )
869 .await?;
870 assert_eq!(
871 outcome,
872 SaveCrashOutcome::CommitOutcomeUncertain,
873 "losing the post-commit acknowledgement must remain distinguishable from success"
874 );
875
876 let reopened = harness.reopen_existing().await?;
877 assert_eq!(
878 reopened.load(&projection_id).await?,
879 Some(committed_cursor),
880 "reloading must resolve the injected ambiguous outcome as committed"
881 );
882 assert!(
883 reopened.load_paused(&projection_id).await?,
884 "crash injection must not disturb independent management state"
885 );
886 drop(reopened);
887
888 harness.exercise_reopen_migration().await
889}
890
891fn test_id(kind: &str, namespace: u64) -> ProjectionId {
892 ProjectionId::new("datum-projection-tck", format!("{kind}-{namespace}"))
893}
894
895fn test_cursor(provider: &str, position: u64, consumed_count: usize) -> ProjectionCursor {
896 let manifest = ProviderManifest::new(format!("datum-tck-provider/{provider}-v1"))
897 .expect("test provider manifest is valid");
898 let consumed = (0..consumed_count).map(|index| {
899 BoundaryEvent::new(
900 PersistenceId::from_unique(format!(
901 "datum-projection-tck|{provider}-{position}-{index}"
902 ))
903 .expect("test persistence id is valid"),
904 SequenceNr::new(u64::try_from(index + 1).expect("test index fits u64")),
905 )
906 });
907 ProjectionCursor::new(
908 manifest,
909 DeduplicationBoundary::with_consumed(format!("{provider}-position-{position}"), consumed),
910 )
911}
912
913fn assert_fenced(
914 error: ProjectionError,
915 projection_id: &ProjectionId,
916 attempted: ProjectionGeneration,
917 current: ProjectionGeneration,
918) {
919 assert!(
920 matches!(
921 error,
922 ProjectionError::Fenced {
923 projection_id: ref error_id,
924 attempted_generation,
925 current_generation,
926 } if error_id == projection_id
927 && attempted_generation == attempted.as_u64()
928 && current_generation == current.as_u64()
929 ),
930 "stale generation must produce a precise fencing error"
931 );
932}
933
934#[cfg(test)]
935mod tests {
936 use super::*;
937 use std::sync::{
938 Arc,
939 atomic::{AtomicUsize, Ordering},
940 };
941
942 use crate::projection::{Handler, MemoryProjectionOffsetStore, Projection, ProjectionStatus};
943
944 struct CountingHandler(Arc<AtomicUsize>);
945
946 #[async_trait::async_trait]
947 impl Handler<TestEnvelope<usize>> for CountingHandler {
948 async fn process(&mut self, _envelope: &TestEnvelope<usize>) -> ProjectionResult<()> {
949 self.0.fetch_add(1, Ordering::Relaxed);
950 Ok(())
951 }
952 }
953
954 #[tokio::test]
955 async fn memory_store_passes_projection_semantic_conformance() -> ProjectionResult<()> {
956 assert_projection_offset_store_semantic_conformance(&MemoryProjectionOffsetStore::new())
957 .await
958 }
959
960 #[tokio::test]
961 async fn fidelity_12_assert_poll_times_out_on_an_empty_source() {
962 let store = Arc::new(MemoryProjectionOffsetStore::new());
963 let runtime = ProjectionRuntime::new(Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>);
964 let projection = Projection::at_least_once(
965 ProjectionId::new(
966 "testkit",
967 format!("empty-{}", TEST_NAMESPACE.fetch_add(1, Ordering::Relaxed)),
968 ),
969 TestSourceProvider::new(Vec::<TestEnvelope<usize>>::new()),
970 || CountingHandler(Arc::new(AtomicUsize::new(0))),
971 );
972 let result = ProjectionTestKit::new(runtime)
973 .with_polling(Duration::from_millis(25), Duration::from_millis(2))
974 .run(projection, || false)
975 .await;
976 assert!(matches!(
977 result,
978 Err(ProjectionError::TestKitTimeout { .. })
979 ));
980 }
981
982 #[tokio::test]
983 async fn testkit_surfaces_terminal_projection_failure_without_timeout_masking() {
984 let store = Arc::new(MemoryProjectionOffsetStore::new());
985 let runtime = ProjectionRuntime::new(Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>);
986 let provider = TestSourceProvider::new(Vec::<TestEnvelope<usize>>::new());
987 provider
988 .fail_next_materialization(ProjectionError::CursorCodec("corrupt".into()))
989 .unwrap();
990 let projection = Projection::at_least_once(
991 ProjectionId::new(
992 "testkit",
993 format!(
994 "terminal-failure-{}",
995 TEST_NAMESPACE.fetch_add(1, Ordering::Relaxed)
996 ),
997 ),
998 provider,
999 || CountingHandler(Arc::new(AtomicUsize::new(0))),
1000 );
1001 let result = ProjectionTestKit::new(runtime)
1002 .with_polling(Duration::from_secs(1), Duration::from_millis(2))
1003 .run(projection, || false)
1004 .await;
1005 assert!(matches!(result, Err(ProjectionError::CursorCodec(_))));
1006 }
1007
1008 #[tokio::test]
1009 async fn fidelity_12_manual_pace_emits_only_after_probe_demand() {
1010 let test_envelope = TestEnvelope::new(
1011 1,
1012 PersistenceId::from_unique("testkit|manual").expect("test id is valid"),
1013 SequenceNr::new(1),
1014 1_usize,
1015 );
1016 let (provider, probe) = TestSourceProvider::new(vec![test_envelope]).with_manual_pace();
1017 let count = Arc::new(AtomicUsize::new(0));
1018 let handler_count = Arc::clone(&count);
1019 let store = Arc::new(MemoryProjectionOffsetStore::new());
1020 let projection = Projection::at_least_once(
1021 ProjectionId::new(
1022 "testkit",
1023 format!("manual-{}", TEST_NAMESPACE.fetch_add(1, Ordering::Relaxed)),
1024 ),
1025 provider,
1026 move || CountingHandler(Arc::clone(&handler_count)),
1027 )
1028 .with_query_mode(QueryMode::Current);
1029 let handle = projection
1030 .spawn(ProjectionRuntime::new(
1031 Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>
1032 ))
1033 .await
1034 .unwrap();
1035
1036 tokio::task::yield_now().await;
1037 assert_eq!(count.load(Ordering::Relaxed), 0);
1038 probe.request_one().unwrap();
1039 assert_eq!(
1040 tokio::time::timeout(Duration::from_secs(1), handle.wait())
1041 .await
1042 .expect("manual-paced current source must complete")
1043 .unwrap(),
1044 ProjectionStatus::Completed
1045 );
1046 assert_eq!(count.load(Ordering::Relaxed), 1);
1047 }
1048}