Skip to main content

datum_persistence/projection/
testkit.rs

1//! Public conformance suites for projection offset stores.
2//!
3//! Every backend must run the semantic suite. Durable backends additionally
4//! implement [`DurableProjectionOffsetStoreHarness`] and run the durable suite
5//! against isolated storage. Invariant violations panic with an assertion
6//! message; backend errors are returned to the ordinary Rust test runner.
7
8use std::{
9    cmp::Ordering as CmpOrdering,
10    collections::VecDeque,
11    sync::{
12        Arc, Mutex,
13        atomic::{AtomicU64, Ordering},
14    },
15    time::{Duration, Instant},
16};
17
18use futures_util::{future::join, stream};
19use tokio::sync::{Semaphore, watch};
20
21use crate::query::QueryMode;
22use crate::{PersistenceId, SequenceNr};
23
24use super::{
25    BoundaryEvent, CursorPosition, DeduplicationBoundary, ProjectionCursor, ProjectionError,
26    ProjectionGeneration, ProjectionId, ProjectionOffsetStore, ProjectionResult, ProjectionRuntime,
27    ProviderManifest, SourceProvider,
28};
29
30static TEST_NAMESPACE: AtomicU64 = AtomicU64::new(1);
31
32/// In-process envelope used by [`TestSourceProvider`].
33#[derive(Debug, Clone, PartialEq, Eq)]
34pub struct TestEnvelope<Payload> {
35    /// Provider-global ordered position. Multiple persistence ids may share it.
36    pub position: u64,
37    /// Persistent entity identity used by boundary deduplication.
38    pub persistence_id: PersistenceId,
39    /// Entity-local sequence number used by boundary deduplication.
40    pub sequence_nr: SequenceNr,
41    /// Test-defined handler payload.
42    pub payload: Payload,
43}
44
45impl<Payload> TestEnvelope<Payload> {
46    /// Creates a test envelope at a provider position.
47    #[must_use]
48    pub fn new(
49        position: u64,
50        persistence_id: PersistenceId,
51        sequence_nr: SequenceNr,
52        payload: Payload,
53    ) -> Self {
54        Self {
55            position,
56            persistence_id,
57            sequence_nr,
58            payload,
59        }
60    }
61}
62
63struct TestSourceFailure {
64    after_envelopes: usize,
65    error: ProjectionError,
66}
67
68struct TestSourceState<Payload> {
69    envelopes: Vec<TestEnvelope<Payload>>,
70    closed: bool,
71    materializations: usize,
72    materialization_failures: VecDeque<ProjectionError>,
73    stream_failures: VecDeque<TestSourceFailure>,
74}
75
76/// Fresh-query test provider with optional inclusive resume and manual pacing.
77///
78/// Inclusive resume starts each materialization at the stored position, so all
79/// events sharing that exact position are re-emitted. The runtime must use the
80/// cursor's `(persistence_id, sequence_nr)` boundary set to remove only the
81/// identities already consumed.
82#[derive(Clone)]
83pub struct TestSourceProvider<Payload> {
84    manifest: ProviderManifest,
85    state: Arc<Mutex<TestSourceState<Payload>>>,
86    notifications: watch::Sender<u64>,
87    inclusive_resume: bool,
88    permits: Option<Arc<Semaphore>>,
89}
90
91impl<Payload> TestSourceProvider<Payload> {
92    /// Creates a strictly-after-resume test provider.
93    #[must_use]
94    pub fn new(envelopes: Vec<TestEnvelope<Payload>>) -> Self {
95        Self::with_resume_mode(envelopes, false)
96    }
97
98    /// Creates an inclusive-resume provider that re-emits boundary duplicates.
99    #[must_use]
100    pub fn inclusive(envelopes: Vec<TestEnvelope<Payload>>) -> Self {
101        Self::with_resume_mode(envelopes, true)
102    }
103
104    fn with_resume_mode(envelopes: Vec<TestEnvelope<Payload>>, inclusive_resume: bool) -> Self {
105        let (notifications, _receiver) = watch::channel(0);
106        Self {
107            manifest: ProviderManifest::new("datum-test-source/position-v1")
108                .expect("the fixed test provider manifest is valid"),
109            state: Arc::new(Mutex::new(TestSourceState {
110                envelopes,
111                closed: false,
112                materializations: 0,
113                materialization_failures: VecDeque::new(),
114                stream_failures: VecDeque::new(),
115            })),
116            notifications,
117            inclusive_resume,
118            permits: None,
119        }
120    }
121
122    /// Adds manual demand pacing and returns its controlling probe.
123    #[must_use]
124    pub fn with_manual_pace(mut self) -> (Self, ManualPaceProbe) {
125        let permits = Arc::new(Semaphore::new(0));
126        self.permits = Some(Arc::clone(&permits));
127        (self, ManualPaceProbe { permits })
128    }
129
130    /// Removes manual pacing from this provider clone.
131    #[must_use]
132    pub fn without_manual_pace(mut self) -> Self {
133        self.permits = None;
134        self
135    }
136
137    /// Appends an envelope and wakes live materializations.
138    pub fn push(&self, envelope: TestEnvelope<Payload>) -> ProjectionResult<()> {
139        let version = {
140            let mut state = self.lock_state()?;
141            state.envelopes.push(envelope);
142            u64::try_from(state.envelopes.len()).unwrap_or(u64::MAX)
143        };
144        self.notifications.send_replace(version);
145        Ok(())
146    }
147
148    /// Completes live materializations once all currently appended data drains.
149    pub fn complete(&self) -> ProjectionResult<()> {
150        let version = {
151            let mut state = self.lock_state()?;
152            state.closed = true;
153            u64::try_from(state.envelopes.len())
154                .unwrap_or(u64::MAX)
155                .saturating_add(1)
156        };
157        self.notifications.send_replace(version);
158        Ok(())
159    }
160
161    /// Fails the next fresh-query factory invocation.
162    pub fn fail_next_materialization(&self, error: ProjectionError) -> ProjectionResult<()> {
163        self.lock_state()?.materialization_failures.push_back(error);
164        Ok(())
165    }
166
167    /// Fails the next materialized stream after `after_envelopes` emissions.
168    pub fn fail_next_source_after(
169        &self,
170        after_envelopes: usize,
171        error: ProjectionError,
172    ) -> ProjectionResult<()> {
173        self.lock_state()?
174            .stream_failures
175            .push_back(TestSourceFailure {
176                after_envelopes,
177                error,
178            });
179        Ok(())
180    }
181
182    /// Returns the number of fresh sources materialized so far.
183    pub fn materializations(&self) -> ProjectionResult<usize> {
184        Ok(self.lock_state()?.materializations)
185    }
186
187    fn lock_state(&self) -> ProjectionResult<std::sync::MutexGuard<'_, TestSourceState<Payload>>> {
188        self.state
189            .lock()
190            .map_err(|_| ProjectionError::Runtime("test source lock poisoned".into()))
191    }
192}
193
194/// Manual source-demand controller used by projection tests.
195#[derive(Clone)]
196pub struct ManualPaceProbe {
197    permits: Arc<Semaphore>,
198}
199
200impl ManualPaceProbe {
201    /// Allows one more test envelope to be emitted.
202    pub fn request_one(&self) -> ProjectionResult<()> {
203        self.request(1)
204    }
205
206    /// Allows up to `count` more test envelopes to be emitted.
207    pub fn request(&self, count: usize) -> ProjectionResult<()> {
208        if count > Semaphore::MAX_PERMITS.saturating_sub(self.permits.available_permits()) {
209            return Err(ProjectionError::invalid_config(
210                "testkit.manual_pace.request",
211                "requested permits exceed the Tokio semaphore limit",
212            ));
213        }
214        self.permits.add_permits(count);
215        Ok(())
216    }
217}
218
219struct TestStreamCursor<Payload> {
220    state: Arc<Mutex<TestSourceState<Payload>>>,
221    notifications: watch::Receiver<u64>,
222    next_index: usize,
223    current_limit: Option<usize>,
224    permits: Option<Arc<Semaphore>>,
225    emitted: usize,
226    failure: Option<TestSourceFailure>,
227}
228
229#[async_trait::async_trait]
230impl<Payload> SourceProvider<TestEnvelope<Payload>> for TestSourceProvider<Payload>
231where
232    Payload: Clone + Send + Sync + 'static,
233{
234    fn manifest(&self) -> &ProviderManifest {
235        &self.manifest
236    }
237
238    async fn source_from(
239        &self,
240        cursor: Option<&ProjectionCursor>,
241        mode: QueryMode,
242    ) -> ProjectionResult<super::ProjectionSource<TestEnvelope<Payload>>> {
243        if let Some(cursor) = cursor {
244            cursor.ensure_manifest(&self.manifest)?;
245        }
246        let resume_position = cursor
247            .map(|cursor| parse_test_position(cursor.position()))
248            .transpose()?;
249        let (next_index, current_limit, failure) = {
250            let mut state = self.lock_state()?;
251            state.materializations = state.materializations.saturating_add(1);
252            if let Some(error) = state.materialization_failures.pop_front() {
253                return Err(error);
254            }
255            let next_index = state
256                .envelopes
257                .iter()
258                .position(|envelope| {
259                    resume_position.is_none_or(|position| {
260                        if self.inclusive_resume {
261                            envelope.position >= position
262                        } else {
263                            envelope.position > position
264                        }
265                    })
266                })
267                .unwrap_or(state.envelopes.len());
268            let current_limit = (mode == QueryMode::Current).then_some(state.envelopes.len());
269            (next_index, current_limit, state.stream_failures.pop_front())
270        };
271        let stream_state = TestStreamCursor {
272            state: Arc::clone(&self.state),
273            notifications: self.notifications.subscribe(),
274            next_index,
275            current_limit,
276            permits: self.permits.clone(),
277            emitted: 0,
278            failure,
279        };
280        Ok(Box::pin(stream::unfold(
281            stream_state,
282            |mut cursor| async move {
283                loop {
284                    if cursor
285                        .failure
286                        .as_ref()
287                        .is_some_and(|failure| cursor.emitted >= failure.after_envelopes)
288                        && let Some(failure) = cursor.failure.take()
289                    {
290                        return Some((Err(failure.error), cursor));
291                    }
292
293                    let observation = match cursor.state.lock() {
294                        Ok(state) => {
295                            if cursor
296                                .current_limit
297                                .is_some_and(|limit| cursor.next_index >= limit)
298                            {
299                                TestObservation::Completed
300                            } else if let Some(envelope) =
301                                state.envelopes.get(cursor.next_index).cloned()
302                            {
303                                TestObservation::Envelope(envelope)
304                            } else if state.closed {
305                                TestObservation::Completed
306                            } else {
307                                TestObservation::Waiting
308                            }
309                        }
310                        Err(_) => TestObservation::Failed(ProjectionError::Runtime(
311                            "test source lock poisoned".into(),
312                        )),
313                    };
314                    match observation {
315                        TestObservation::Envelope(envelope) => {
316                            if let Some(permits) = &cursor.permits {
317                                match Arc::clone(permits).acquire_owned().await {
318                                    Ok(permit) => permit.forget(),
319                                    Err(_) => return None,
320                                }
321                            }
322                            cursor.next_index = cursor.next_index.saturating_add(1);
323                            cursor.emitted = cursor.emitted.saturating_add(1);
324                            return Some((Ok(envelope), cursor));
325                        }
326                        TestObservation::Completed => return None,
327                        TestObservation::Failed(error) => {
328                            return Some((Err(error), cursor));
329                        }
330                        TestObservation::Waiting => {
331                            if cursor.notifications.changed().await.is_err() {
332                                return None;
333                            }
334                        }
335                    }
336                }
337            },
338        )))
339    }
340
341    fn extract_cursor(
342        &self,
343        envelope: &TestEnvelope<Payload>,
344    ) -> ProjectionResult<ProjectionCursor> {
345        Ok(ProjectionCursor::new(
346            self.manifest.clone(),
347            DeduplicationBoundary::with_consumed(
348                envelope.position.to_string(),
349                [BoundaryEvent::new(
350                    envelope.persistence_id.clone(),
351                    envelope.sequence_nr,
352                )],
353            ),
354        ))
355    }
356
357    fn compare_positions(
358        &self,
359        left: &CursorPosition,
360        right: &CursorPosition,
361    ) -> ProjectionResult<CmpOrdering> {
362        Ok(parse_test_position(left)?.cmp(&parse_test_position(right)?))
363    }
364}
365
366enum TestObservation<Payload> {
367    Envelope(TestEnvelope<Payload>),
368    Completed,
369    Failed(ProjectionError),
370    Waiting,
371}
372
373fn parse_test_position(position: &CursorPosition) -> ProjectionResult<u64> {
374    position.as_str().parse::<u64>().map_err(|_| {
375        ProjectionError::CursorCodec(format!(
376            "test source position {:?} is not an unsigned integer",
377            position.as_str()
378        ))
379    })
380}
381
382/// Bounded in-process assertion runner for projection blueprints.
383#[derive(Clone)]
384pub struct ProjectionTestKit {
385    runtime: ProjectionRuntime,
386    max: Duration,
387    interval: Duration,
388}
389
390impl ProjectionTestKit {
391    /// Creates a testkit using the supplied in-process runtime resources.
392    #[must_use]
393    pub fn new(runtime: ProjectionRuntime) -> Self {
394        Self {
395            runtime,
396            max: Duration::from_secs(3),
397            interval: Duration::from_millis(100),
398        }
399    }
400
401    /// Overrides the maximum assertion window and poll interval.
402    #[must_use]
403    pub fn with_polling(mut self, max: Duration, interval: Duration) -> Self {
404        self.max = max;
405        self.interval = interval;
406        self
407    }
408
409    /// Runs a projection while polling `assertion` until it returns true.
410    ///
411    /// The window is always bounded. An empty current or live test source
412    /// therefore fails with [`ProjectionError::TestKitTimeout`] instead of
413    /// waiting forever. A terminal [`super::ProjectionStatus::Failed`] is
414    /// returned immediately rather than being masked as an assertion timeout.
415    pub async fn run<Envelope, F>(
416        &self,
417        projection: super::Projection<Envelope>,
418        mut assertion: F,
419    ) -> ProjectionResult<()>
420    where
421        Envelope: Send + Sync + 'static,
422        F: FnMut() -> bool,
423    {
424        let handle = projection.spawn(self.runtime.clone()).await?;
425        let started = Instant::now();
426        loop {
427            if assertion() {
428                if !handle.status().is_terminal() {
429                    handle.stop_and_wait().await?;
430                }
431                return Ok(());
432            }
433            if let super::ProjectionStatus::Failed(error) = handle.status() {
434                let _ = handle.wait().await;
435                return Err(error);
436            }
437            if started.elapsed() >= self.max {
438                if !handle.status().is_terminal() {
439                    let _ = handle.stop_and_wait().await;
440                }
441                return Err(ProjectionError::TestKitTimeout {
442                    timeout_ms: u64::try_from(self.max.as_millis()).unwrap_or(u64::MAX),
443                });
444            }
445            tokio::time::sleep(self.interval).await;
446        }
447    }
448}
449
450/// Asserts the complete semantic projection offset-store contract.
451///
452/// The suite verifies the stable cursor codec for multiple provider manifests,
453/// per-id isolation, atomic generation claiming and fencing, expected-position
454/// CAS, cursor-clear versus management-row independence, a paused flag without
455/// a cursor, and same-key concurrent claimants. It is the normative executable
456/// contract for M16 fidelity item 9 and must run for memory and every durable
457/// backend.
458pub async fn assert_projection_offset_store_semantic_conformance(
459    store: &dyn ProjectionOffsetStore,
460) -> ProjectionResult<()> {
461    assert_cursor_codec_conformance()?;
462    let namespace = TEST_NAMESPACE.fetch_add(1, Ordering::Relaxed);
463
464    let first_id = test_id("isolation-first", namespace);
465    let second_id = test_id("isolation-second", namespace);
466    let first_generation = store.claim(&first_id).await?;
467    let second_generation = store.claim(&second_id).await?;
468    let first_cursor = test_cursor("sequence", 11, 2);
469    let second_cursor = test_cursor("timestamp", 29, 3);
470    store
471        .save(&first_id, first_generation, None, first_cursor.clone())
472        .await?;
473    store
474        .save(&second_id, second_generation, None, second_cursor.clone())
475        .await?;
476    assert_eq!(
477        store.load(&first_id).await?,
478        Some(first_cursor),
479        "saving one projection id must not affect another"
480    );
481    assert_eq!(
482        store.load(&second_id).await?,
483        Some(second_cursor),
484        "each projection id must retain its own cursor"
485    );
486
487    let fenced_id = test_id("generation-fencing", namespace);
488    let superseded = store.claim(&fenced_id).await?;
489    let current = store.claim(&fenced_id).await?;
490    assert!(
491        current > superseded,
492        "a later claim must return a greater generation"
493    );
494    let stale_save = store
495        .save(&fenced_id, superseded, None, test_cursor("sequence", 31, 1))
496        .await
497        .expect_err("a superseded claimant must not save");
498    assert_fenced(stale_save, &fenced_id, superseded, current);
499    let stale_clear = store
500        .clear(&fenced_id, superseded)
501        .await
502        .expect_err("a superseded claimant must not clear");
503    assert_fenced(stale_clear, &fenced_id, superseded, current);
504    store
505        .save(&fenced_id, current, None, test_cursor("sequence", 32, 1))
506        .await?;
507
508    let cas_id = test_id("expected-position", namespace);
509    let cas_generation = store.claim(&cas_id).await?;
510    let before = test_cursor("sequence", 41, 1);
511    let after = test_cursor("sequence", 42, 1);
512    store
513        .save(&cas_id, cas_generation, None, before.clone())
514        .await?;
515    let wrong_position = CursorPosition::new("position-that-was-never-loaded");
516    let conflict = store
517        .save(
518            &cas_id,
519            cas_generation,
520            Some(&wrong_position),
521            after.clone(),
522        )
523        .await
524        .expect_err("an unexpected current position must reject the save");
525    assert!(matches!(
526        conflict,
527        ProjectionError::UnexpectedPosition {
528            projection_id,
529            expected: Some(_),
530            actual: Some(_),
531        } if projection_id == cas_id
532    ));
533    assert_eq!(
534        store.load(&cas_id).await?,
535        Some(before.clone()),
536        "a rejected CAS must not change the cursor"
537    );
538    store
539        .save(
540            &cas_id,
541            cas_generation,
542            Some(before.position()),
543            after.clone(),
544        )
545        .await?;
546    assert_eq!(store.load(&cas_id).await?, Some(after));
547
548    let management_id = test_id("clear-management", namespace);
549    let management_generation = store.claim(&management_id).await?;
550    let cursor = test_cursor("sequence", 51, 1);
551    store
552        .save(&management_id, management_generation, None, cursor)
553        .await?;
554    store.set_paused(&management_id, true).await?;
555    store.clear(&management_id, management_generation).await?;
556    assert_eq!(
557        store.load(&management_id).await?,
558        None,
559        "clear must remove the durable cursor"
560    );
561    assert!(
562        store.load_paused(&management_id).await?,
563        "clear must not remove or reset the management row"
564    );
565    store.set_paused(&management_id, false).await?;
566    assert!(
567        !store.load_paused(&management_id).await?,
568        "paused must be independently mutable after cursor clear"
569    );
570    assert_eq!(
571        store.load(&management_id).await?,
572        None,
573        "changing paused must not recreate a cursor"
574    );
575
576    let paused_without_cursor_id = test_id("paused-without-cursor", namespace);
577    assert!(!store.load_paused(&paused_without_cursor_id).await?);
578    store.set_paused(&paused_without_cursor_id, true).await?;
579    assert!(store.load_paused(&paused_without_cursor_id).await?);
580    assert_eq!(
581        store.load(&paused_without_cursor_id).await?,
582        None,
583        "management state must exist independently of an offset row"
584    );
585
586    let concurrent_id = test_id("same-key-concurrency", namespace);
587    let (left_claim, right_claim) =
588        join(store.claim(&concurrent_id), store.claim(&concurrent_id)).await;
589    let left_generation = left_claim?;
590    let right_generation = right_claim?;
591    assert_ne!(
592        left_generation, right_generation,
593        "atomic same-key claims must return distinct generations"
594    );
595    let left_cursor = test_cursor("sequence", 61, 1);
596    let right_cursor = test_cursor("sequence", 62, 1);
597    let (left_save, right_save) = join(
598        store.save(&concurrent_id, left_generation, None, left_cursor.clone()),
599        store.save(&concurrent_id, right_generation, None, right_cursor.clone()),
600    )
601    .await;
602    assert_eq!(
603        usize::from(left_save.is_ok()) + usize::from(right_save.is_ok()),
604        1,
605        "two claimants for one key must not both save"
606    );
607    let (winning_generation, winning_cursor, losing_result) = if left_generation > right_generation
608    {
609        (left_generation, left_cursor, right_save)
610    } else {
611        (right_generation, right_cursor, left_save)
612    };
613    assert_eq!(
614        store.load(&concurrent_id).await?,
615        Some(winning_cursor),
616        "only the highest claimed generation may save"
617    );
618    assert_fenced(
619        losing_result.expect_err("the lower generation must lose fencing"),
620        &concurrent_id,
621        if winning_generation == left_generation {
622            right_generation
623        } else {
624            left_generation
625        },
626        winning_generation,
627    );
628
629    Ok(())
630}
631
632/// Asserts stable cursor-codec round trips for test-local provider manifests.
633///
634/// The fixtures intentionally model provider identities without constructing a
635/// live query adapter. Providers added in later work packages should add their
636/// own manifest fixtures while retaining these store-independent properties.
637pub fn assert_cursor_codec_conformance() -> ProjectionResult<()> {
638    for provider in [
639        "datum-test-provider/sequence-v1",
640        "datum-test-provider/per-id-v1",
641        "datum-test-provider/timestamp-v1",
642    ] {
643        let manifest = ProviderManifest::new(provider)?;
644        for marker in 0_u64..64 {
645            let event_count = usize::try_from(marker % 5).expect("test count fits usize");
646            let consumed = (0..event_count)
647                .map(|index| {
648                    BoundaryEvent::new(
649                        PersistenceId::from_unique(format!(
650                            "test-entity|{provider}|{marker}|{index}"
651                        ))
652                        .expect("test persistence id is valid"),
653                        SequenceNr::new(
654                            marker
655                                .saturating_mul(10)
656                                .saturating_add(u64::try_from(index).expect("index fits u64")),
657                        ),
658                    )
659                })
660                .collect::<Vec<_>>();
661            let cursor = ProjectionCursor::new(
662                manifest.clone(),
663                DeduplicationBoundary::with_consumed(
664                    format!("position|{marker}:λ,{}", marker.saturating_mul(17)),
665                    consumed,
666                ),
667            );
668            let encoded = cursor.encode();
669            let decoded = ProjectionCursor::decode(&encoded)?;
670            assert_eq!(
671                decoded, cursor,
672                "cursor codec must round-trip for provider manifest {provider}"
673            );
674            assert_eq!(
675                decoded.encode(),
676                encoded,
677                "cursor codec output must be stable for provider manifest {provider}"
678            );
679            decoded.ensure_manifest(&manifest)?;
680        }
681    }
682    Ok(())
683}
684
685/// Point at which a durable conformance harness terminates its save process.
686#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
687pub enum SaveCrashPoint {
688    /// Terminate before the cursor commit can become durable.
689    BeforeCommit,
690    /// Terminate after durable commit but before success can be acknowledged.
691    AfterCommitBeforeAcknowledgement,
692}
693
694/// Observable result reported by a controlled save-crash injection.
695#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
696pub enum SaveCrashOutcome {
697    /// The harness guarantees the cursor was not committed.
698    NotCommitted,
699    /// Commit acknowledgment was lost, so the immediate result is uncertain.
700    ///
701    /// Reloading the cursor resolves whether the durable commit happened.
702    CommitOutcomeUncertain,
703}
704
705/// Owned parameters for a save performed by a crash-injection process.
706#[derive(Debug, Clone, PartialEq, Eq)]
707pub struct CrashSaveRequest {
708    /// Projection whose cursor is being saved.
709    pub projection_id: ProjectionId,
710    /// Claimed generation carried by the save.
711    pub generation: ProjectionGeneration,
712    /// Previously loaded position, or `None` when no cursor is expected.
713    pub expected_position: Option<CursorPosition>,
714    /// Replacement provider-complete cursor.
715    pub cursor: ProjectionCursor,
716}
717
718/// Backend lifecycle and crash controls consumed by the durable suite.
719///
720/// Implementations must use isolated storage. [`Self::create_new`] starts with
721/// schema absent and exercises create-if-missing. [`Self::reopen_existing`]
722/// must close/reopen all process-local resources and run the backend's normal
723/// additive migration path. [`Self::exercise_reopen_migration`] prepares a
724/// backend-specific older-schema fixture in separate isolated storage, opens
725/// it with the current backend, and verifies that existing data survives the
726/// migration.
727///
728/// Crash saves must execute in a disposable process or equivalent isolation:
729/// dropping an in-process handle is not process-restart equivalence.
730#[async_trait::async_trait]
731pub trait DurableProjectionOffsetStoreHarness: Send + Sync {
732    /// Creates an empty durable store with schema initially absent.
733    async fn create_new(&self) -> ProjectionResult<Box<dyn ProjectionOffsetStore>>;
734
735    /// Reopens the same durable store through its existing-schema path.
736    async fn reopen_existing(&self) -> ProjectionResult<Box<dyn ProjectionOffsetStore>>;
737
738    /// Exercises and verifies the backend-specific additive migration fixture.
739    async fn exercise_reopen_migration(&self) -> ProjectionResult<()>;
740
741    /// Performs one save in crash-isolated execution and reports its certainty.
742    async fn save_with_crash(
743        &self,
744        request: CrashSaveRequest,
745        crash_point: SaveCrashPoint,
746    ) -> ProjectionResult<SaveCrashOutcome>;
747}
748
749/// Asserts process-restart, schema lifecycle, and crash-save durability.
750///
751/// PostgreSQL, Cassandra, and RocksDB offset stores must run this suite in
752/// their env-gated or hermetic live-backend tests. The memory backend
753/// deliberately does **not** run it: an in-memory value can satisfy semantic
754/// fencing while all state disappears with the process, so it cannot prove
755/// process-restart equivalence.
756pub async fn assert_projection_offset_store_durable_conformance(
757    harness: &dyn DurableProjectionOffsetStoreHarness,
758) -> ProjectionResult<()> {
759    let namespace = TEST_NAMESPACE.fetch_add(1, Ordering::Relaxed);
760    let projection_id = test_id("durable-reopen", namespace);
761    let initial_cursor = test_cursor("durable", 71, 2);
762
763    let store = harness.create_new().await?;
764    let initial_generation = store.claim(&projection_id).await?;
765    store
766        .save(
767            &projection_id,
768            initial_generation,
769            None,
770            initial_cursor.clone(),
771        )
772        .await?;
773    store.set_paused(&projection_id, true).await?;
774    drop(store);
775
776    let reopened = harness.reopen_existing().await?;
777    assert_eq!(
778        reopened.load(&projection_id).await?,
779        Some(initial_cursor.clone()),
780        "cursor must survive complete store close/reopen"
781    );
782    assert!(
783        reopened.load_paused(&projection_id).await?,
784        "paused management state must survive complete store close/reopen"
785    );
786    let reopened_generation = reopened.claim(&projection_id).await?;
787    assert_eq!(
788        reopened_generation.as_u64(),
789        initial_generation
790            .as_u64()
791            .checked_add(1)
792            .expect("test generation cannot exhaust"),
793        "generation must survive reopen and advance monotonically"
794    );
795    let stale = reopened
796        .save(
797            &projection_id,
798            initial_generation,
799            Some(initial_cursor.position()),
800            test_cursor("durable", 72, 1),
801        )
802        .await
803        .expect_err("a pre-reopen generation must remain fenced");
804    assert_fenced(
805        stale,
806        &projection_id,
807        initial_generation,
808        reopened_generation,
809    );
810
811    let second_cursor = test_cursor("durable", 73, 3);
812    reopened
813        .save(
814            &projection_id,
815            reopened_generation,
816            Some(initial_cursor.position()),
817            second_cursor.clone(),
818        )
819        .await?;
820    drop(reopened);
821
822    let reopened = harness.reopen_existing().await?;
823    assert_eq!(
824        reopened.load(&projection_id).await?,
825        Some(second_cursor.clone())
826    );
827    assert!(reopened.load_paused(&projection_id).await?);
828    let before_crash_generation = reopened.claim(&projection_id).await?;
829    drop(reopened);
830
831    let not_committed_cursor = test_cursor("durable", 74, 1);
832    let outcome = harness
833        .save_with_crash(
834            CrashSaveRequest {
835                projection_id: projection_id.clone(),
836                generation: before_crash_generation,
837                expected_position: Some(second_cursor.position().clone()),
838                cursor: not_committed_cursor,
839            },
840            SaveCrashPoint::BeforeCommit,
841        )
842        .await?;
843    assert_eq!(
844        outcome,
845        SaveCrashOutcome::NotCommitted,
846        "the pre-commit crash point must have a deterministic not-committed result"
847    );
848
849    let reopened = harness.reopen_existing().await?;
850    assert_eq!(
851        reopened.load(&projection_id).await?,
852        Some(second_cursor.clone()),
853        "a pre-commit crash must preserve the previous cursor"
854    );
855    let ambiguous_generation = reopened.claim(&projection_id).await?;
856    drop(reopened);
857
858    let committed_cursor = test_cursor("durable", 75, 4);
859    let outcome = harness
860        .save_with_crash(
861            CrashSaveRequest {
862                projection_id: projection_id.clone(),
863                generation: ambiguous_generation,
864                expected_position: Some(second_cursor.position().clone()),
865                cursor: committed_cursor.clone(),
866            },
867            SaveCrashPoint::AfterCommitBeforeAcknowledgement,
868        )
869        .await?;
870    assert_eq!(
871        outcome,
872        SaveCrashOutcome::CommitOutcomeUncertain,
873        "losing the post-commit acknowledgement must remain distinguishable from success"
874    );
875
876    let reopened = harness.reopen_existing().await?;
877    assert_eq!(
878        reopened.load(&projection_id).await?,
879        Some(committed_cursor),
880        "reloading must resolve the injected ambiguous outcome as committed"
881    );
882    assert!(
883        reopened.load_paused(&projection_id).await?,
884        "crash injection must not disturb independent management state"
885    );
886    drop(reopened);
887
888    harness.exercise_reopen_migration().await
889}
890
891fn test_id(kind: &str, namespace: u64) -> ProjectionId {
892    ProjectionId::new("datum-projection-tck", format!("{kind}-{namespace}"))
893}
894
895fn test_cursor(provider: &str, position: u64, consumed_count: usize) -> ProjectionCursor {
896    let manifest = ProviderManifest::new(format!("datum-tck-provider/{provider}-v1"))
897        .expect("test provider manifest is valid");
898    let consumed = (0..consumed_count).map(|index| {
899        BoundaryEvent::new(
900            PersistenceId::from_unique(format!(
901                "datum-projection-tck|{provider}-{position}-{index}"
902            ))
903            .expect("test persistence id is valid"),
904            SequenceNr::new(u64::try_from(index + 1).expect("test index fits u64")),
905        )
906    });
907    ProjectionCursor::new(
908        manifest,
909        DeduplicationBoundary::with_consumed(format!("{provider}-position-{position}"), consumed),
910    )
911}
912
913fn assert_fenced(
914    error: ProjectionError,
915    projection_id: &ProjectionId,
916    attempted: ProjectionGeneration,
917    current: ProjectionGeneration,
918) {
919    assert!(
920        matches!(
921            error,
922            ProjectionError::Fenced {
923                projection_id: ref error_id,
924                attempted_generation,
925                current_generation,
926            } if error_id == projection_id
927                && attempted_generation == attempted.as_u64()
928                && current_generation == current.as_u64()
929        ),
930        "stale generation must produce a precise fencing error"
931    );
932}
933
934#[cfg(test)]
935mod tests {
936    use super::*;
937    use std::sync::{
938        Arc,
939        atomic::{AtomicUsize, Ordering},
940    };
941
942    use crate::projection::{Handler, MemoryProjectionOffsetStore, Projection, ProjectionStatus};
943
944    struct CountingHandler(Arc<AtomicUsize>);
945
946    #[async_trait::async_trait]
947    impl Handler<TestEnvelope<usize>> for CountingHandler {
948        async fn process(&mut self, _envelope: &TestEnvelope<usize>) -> ProjectionResult<()> {
949            self.0.fetch_add(1, Ordering::Relaxed);
950            Ok(())
951        }
952    }
953
954    #[tokio::test]
955    async fn memory_store_passes_projection_semantic_conformance() -> ProjectionResult<()> {
956        assert_projection_offset_store_semantic_conformance(&MemoryProjectionOffsetStore::new())
957            .await
958    }
959
960    #[tokio::test]
961    async fn fidelity_12_assert_poll_times_out_on_an_empty_source() {
962        let store = Arc::new(MemoryProjectionOffsetStore::new());
963        let runtime = ProjectionRuntime::new(Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>);
964        let projection = Projection::at_least_once(
965            ProjectionId::new(
966                "testkit",
967                format!("empty-{}", TEST_NAMESPACE.fetch_add(1, Ordering::Relaxed)),
968            ),
969            TestSourceProvider::new(Vec::<TestEnvelope<usize>>::new()),
970            || CountingHandler(Arc::new(AtomicUsize::new(0))),
971        );
972        let result = ProjectionTestKit::new(runtime)
973            .with_polling(Duration::from_millis(25), Duration::from_millis(2))
974            .run(projection, || false)
975            .await;
976        assert!(matches!(
977            result,
978            Err(ProjectionError::TestKitTimeout { .. })
979        ));
980    }
981
982    #[tokio::test]
983    async fn testkit_surfaces_terminal_projection_failure_without_timeout_masking() {
984        let store = Arc::new(MemoryProjectionOffsetStore::new());
985        let runtime = ProjectionRuntime::new(Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>);
986        let provider = TestSourceProvider::new(Vec::<TestEnvelope<usize>>::new());
987        provider
988            .fail_next_materialization(ProjectionError::CursorCodec("corrupt".into()))
989            .unwrap();
990        let projection = Projection::at_least_once(
991            ProjectionId::new(
992                "testkit",
993                format!(
994                    "terminal-failure-{}",
995                    TEST_NAMESPACE.fetch_add(1, Ordering::Relaxed)
996                ),
997            ),
998            provider,
999            || CountingHandler(Arc::new(AtomicUsize::new(0))),
1000        );
1001        let result = ProjectionTestKit::new(runtime)
1002            .with_polling(Duration::from_secs(1), Duration::from_millis(2))
1003            .run(projection, || false)
1004            .await;
1005        assert!(matches!(result, Err(ProjectionError::CursorCodec(_))));
1006    }
1007
1008    #[tokio::test]
1009    async fn fidelity_12_manual_pace_emits_only_after_probe_demand() {
1010        let test_envelope = TestEnvelope::new(
1011            1,
1012            PersistenceId::from_unique("testkit|manual").expect("test id is valid"),
1013            SequenceNr::new(1),
1014            1_usize,
1015        );
1016        let (provider, probe) = TestSourceProvider::new(vec![test_envelope]).with_manual_pace();
1017        let count = Arc::new(AtomicUsize::new(0));
1018        let handler_count = Arc::clone(&count);
1019        let store = Arc::new(MemoryProjectionOffsetStore::new());
1020        let projection = Projection::at_least_once(
1021            ProjectionId::new(
1022                "testkit",
1023                format!("manual-{}", TEST_NAMESPACE.fetch_add(1, Ordering::Relaxed)),
1024            ),
1025            provider,
1026            move || CountingHandler(Arc::clone(&handler_count)),
1027        )
1028        .with_query_mode(QueryMode::Current);
1029        let handle = projection
1030            .spawn(ProjectionRuntime::new(
1031                Arc::clone(&store) as Arc<dyn ProjectionOffsetStore>
1032            ))
1033            .await
1034            .unwrap();
1035
1036        tokio::task::yield_now().await;
1037        assert_eq!(count.load(Ordering::Relaxed), 0);
1038        probe.request_one().unwrap();
1039        assert_eq!(
1040            tokio::time::timeout(Duration::from_secs(1), handle.wait())
1041                .await
1042                .expect("manual-paced current source must complete")
1043                .unwrap(),
1044            ProjectionStatus::Completed
1045        );
1046        assert_eq!(count.load(Ordering::Relaxed), 1);
1047    }
1048}