Skip to main content

liminal/durability/
store.rs

1use haematite::{ApiError, Database, DatabaseConfig, Event, EventStore};
2
3use std::path::Path;
4use std::sync::Arc;
5
6use super::DurabilityError;
7
8use tempfile::TempDir;
9
10/// Entry read from a durable haematite stream.
11#[derive(Clone, Debug, PartialEq, Eq)]
12pub struct StoredEntry {
13    /// Opaque stored payload bytes.
14    pub payload: Vec<u8>,
15    /// Sequence number assigned by the stream.
16    pub sequence: u64,
17    /// Store timestamp associated with the entry.
18    pub timestamp: u64,
19}
20
21/// Direct durability surface matching haematite's append/read/cas/scan API.
22#[async_trait::async_trait]
23pub trait DurableStore: std::fmt::Debug + Send + Sync {
24    /// Appends `payload` to `stream_key` if `expected_seq` matches the stream head.
25    async fn append(
26        &self,
27        stream_key: &str,
28        payload: Vec<u8>,
29        expected_seq: u64,
30    ) -> Result<u64, DurabilityError>;
31
32    /// Reads entries from `stream_key` beginning at `offset`, up to `limit` entries.
33    async fn read_from(
34        &self,
35        stream_key: &str,
36        offset: u64,
37        limit: usize,
38    ) -> Result<Vec<StoredEntry>, DurabilityError>;
39
40    /// Reads exactly the event at `sequence` without traversing its suffix.
41    async fn read_at(
42        &self,
43        stream_key: &str,
44        sequence: u64,
45    ) -> Result<Option<StoredEntry>, DurabilityError> {
46        Ok(self
47            .read_from(stream_key, sequence, 1)
48            .await?
49            .into_iter()
50            .next())
51    }
52
53    /// Atomically replaces a stored numeric value if it equals `old_value`.
54    ///
55    /// An `old_value` of `0` matches a key that is currently *absent* as well as
56    /// one explicitly stored as `0`: a fresh cursor is created on its first
57    /// checkpoint without a prior write. See [`HaematiteStore::cas`] for how this
58    /// "absent == 0" contract is preserved atomically over the real engine.
59    async fn cas(&self, key: &str, old_value: u64, new_value: u64) -> Result<(), DurabilityError>;
60
61    /// Reads a numeric value previously updated through compare-and-swap.
62    async fn read_value(&self, key: &str) -> Result<Option<u64>, DurabilityError>;
63
64    /// Scans entries by store prefix.
65    async fn scan(&self, prefix: &str) -> Result<Vec<StoredEntry>, DurabilityError>;
66
67    /// Flushes buffered writes so completed durable operations are persisted.
68    ///
69    /// # Errors
70    /// Returns [`DurabilityError`] when the underlying store cannot complete the flush.
71    async fn flush(&self) -> Result<(), DurabilityError>;
72}
73
74/// `DurableStore` implementation that delegates directly to haematite's `EventStore`.
75///
76/// The real [`EventStore`] is synchronous (every call blocks on the owning
77/// shard actor's reply), so each `async` method below completes on its first
78/// poll. The synchronous bridge in [`super::bridge`] relies on exactly that.
79#[derive(Clone, Debug)]
80pub struct HaematiteStore {
81    event_store: Arc<EventStore>,
82}
83
84impl HaematiteStore {
85    /// Wraps a haematite `EventStore` handle.
86    #[must_use]
87    pub const fn new(event_store: Arc<EventStore>) -> Self {
88        Self { event_store }
89    }
90}
91
92#[async_trait::async_trait]
93impl DurableStore for HaematiteStore {
94    async fn append(
95        &self,
96        stream_key: &str,
97        payload: Vec<u8>,
98        expected_seq: u64,
99    ) -> Result<u64, DurabilityError> {
100        // Contract bridge: liminal's `DurableStore::append` returns the *assigned
101        // event sequence* (0-based position of the just-appended event), which is
102        // exactly `expected_seq` for a single append. The real `EventStore::append`
103        // instead returns the stream's new next-sequence (`expected_seq + 1`), so
104        // subtract one to recover the assigned seq. A `0` next-seq is impossible
105        // after a successful single append, so the `checked_sub` cannot saturate
106        // silently; if it ever did the engine returned a contract-violating value.
107        let next_seq = self
108            .event_store
109            .append(stream_key.as_bytes(), &payload, expected_seq)
110            .map_err(DurabilityError::from)?;
111        next_seq.checked_sub(1).ok_or_else(|| {
112            DurabilityError::StoreError(ApiError::CorruptEvent(format!(
113                "append returned next-seq 0 for stream {stream_key}"
114            )))
115        })
116    }
117
118    async fn read_from(
119        &self,
120        stream_key: &str,
121        offset: u64,
122        limit: usize,
123    ) -> Result<Vec<StoredEntry>, DurabilityError> {
124        // The real `read_from` returns every event with seq >= offset and applies
125        // no limit; truncate to `limit` entries to honour the trait contract.
126        let mut events = self
127            .event_store
128            .read_from(stream_key.as_bytes(), offset)
129            .map_err(DurabilityError::from)?;
130        events.truncate(limit);
131        Ok(events.into_iter().map(StoredEntry::from).collect())
132    }
133
134    async fn read_at(
135        &self,
136        stream_key: &str,
137        sequence: u64,
138    ) -> Result<Option<StoredEntry>, DurabilityError> {
139        const TIMESTAMP_WIDTH: usize = std::mem::size_of::<u64>();
140
141        let engine_sequence = sequence.checked_add(1).ok_or_else(|| {
142            DurabilityError::StoreError(ApiError::CorruptEvent(format!(
143                "point read sequence overflow for stream {stream_key}"
144            )))
145        })?;
146        let event_key = haematite::encode_stream_key(stream_key.as_bytes(), engine_sequence);
147        let Some(value) = self
148            .event_store
149            .database()
150            .get_routed(stream_key.as_bytes(), &event_key)
151            .map_err(ApiError::from)
152            .map_err(DurabilityError::from)?
153        else {
154            return Ok(None);
155        };
156        let Some(timestamp_bytes) = value.get(..TIMESTAMP_WIDTH) else {
157            return Err(DurabilityError::StoreError(ApiError::CorruptEvent(
158                format!(
159                    "point-read event value is shorter than its timestamp for stream {stream_key}"
160                ),
161            )));
162        };
163        let timestamp = u64::from_be_bytes(timestamp_bytes.try_into().map_err(|_| {
164            DurabilityError::StoreError(ApiError::CorruptEvent(format!(
165                "point-read event timestamp has the wrong width for stream {stream_key}"
166            )))
167        })?);
168        let Some(payload) = value.get(TIMESTAMP_WIDTH..) else {
169            return Err(DurabilityError::StoreError(ApiError::CorruptEvent(
170                format!("point-read event has no payload boundary for stream {stream_key}"),
171            )));
172        };
173        Ok(Some(StoredEntry {
174            payload: payload.to_vec(),
175            sequence,
176            timestamp,
177        }))
178    }
179
180    async fn cas(&self, key: &str, old_value: u64, new_value: u64) -> Result<(), DurabilityError> {
181        // Preserve liminal's "absent == 0" cursor contract faithfully over an
182        // engine that distinguishes `None` (absent) from `Some(0)` (a stored
183        // zero). The invariant that makes the mapping below correct: we NEVER
184        // persist a physical zero, so a logical value of 0 and physical absence
185        // always coincide.
186        //
187        // A `cas` whose target `new_value` is 0 must therefore write nothing — it
188        // only asserts the precondition. This is reachable as `cas(0, 0)` (a
189        // cursor checkpoint at offset 0; offsets are monotonic so they never CAS
190        // down to 0 from a higher value). Were we instead to let it store a
191        // physical zero, the *next* `cas(0, n)` — mapped to expect-absent `None`
192        // — would wrongly fail against the now-present key and permanently stall
193        // the cursor. Asserting via a read is race-free here precisely because no
194        // value is written, so there is no lost-update window.
195        if new_value == 0 {
196            return self
197                .event_store
198                .read_value(key.as_bytes())
199                .map_err(DurabilityError::from)?
200                .map_or(Ok(()), |stored| {
201                    Err(DurabilityError::CursorRegression {
202                        stored,
203                        attempted: old_value,
204                    })
205                });
206        }
207        // With a physical zero never stored, `old_value == 0` is exactly the
208        // expect-absent expectation. Any other `old_value` maps to `Some(_)`.
209        // This is a single CAS routed to the owning shard actor, where read,
210        // compare, and write run with no interleaving point (haematite's
211        // `ShardActor::cas`) — the engine's atomicity is preserved end to end.
212        let expected = if old_value == 0 {
213            None
214        } else {
215            Some(old_value)
216        };
217        self.event_store
218            .cas(key.as_bytes(), expected, new_value)
219            .map_err(DurabilityError::from)
220    }
221
222    async fn read_value(&self, key: &str) -> Result<Option<u64>, DurabilityError> {
223        self.event_store
224            .read_value(key.as_bytes())
225            .map_err(DurabilityError::from)
226    }
227
228    async fn scan(&self, prefix: &str) -> Result<Vec<StoredEntry>, DurabilityError> {
229        // The real `scan` predicate yields stream *metadata* (key + next_seq),
230        // not events. Liminal's contract is to return the events of every stream
231        // whose key matches `prefix`, so collect the matching stream keys, then
232        // read each stream's full event list and flatten the results.
233        let prefix_bytes = prefix.as_bytes().to_vec();
234        let matches = self
235            .event_store
236            .scan(|meta| meta.stream_key.starts_with(&prefix_bytes))
237            .map_err(DurabilityError::from)?;
238        let mut entries = Vec::new();
239        for stream in matches {
240            let events = self
241                .event_store
242                .read(&stream.stream_key)
243                .map_err(DurabilityError::from)?;
244            entries.extend(events.into_iter().map(StoredEntry::from));
245        }
246        Ok(entries)
247    }
248
249    async fn flush(&self) -> Result<(), DurabilityError> {
250        self.event_store.flush().map_err(DurabilityError::from)
251    }
252}
253
254/// Drop shell enforcing "close the store, then remove its directory" as
255/// explicit code rather than field declaration order.
256///
257/// Declaration order alone cannot express the unwind case: if dropping the
258/// store panics (a haematite worker failing to join), Rust would still drop
259/// the remaining fields during the unwind and remove the directory under
260/// possibly-live workers. This `Drop` drops the store inside `catch_unwind`;
261/// on unwind it DISARMS the directory guard — the directory is deliberately
262/// leaked, because visible residue is diagnosable while removal under live
263/// workers is filesystem corruption — logs the leaked path, and re-raises the
264/// panic. On the clean path the directory is removed after the store, by the
265/// ordinary field drop that follows this `Drop`.
266///
267/// Both fields are `Option` only so `drop` can move them out; they are `Some`
268/// for the shell's entire life outside `drop`.
269#[derive(Debug)]
270struct EphemeralGuard<S> {
271    store: Option<S>,
272    dir: Option<TempDir>,
273}
274
275impl<S> Drop for EphemeralGuard<S> {
276    fn drop(&mut self) {
277        let store = self.store.take();
278        // AssertUnwindSafe: the closure owns everything it touches (the moved
279        // store), and the unwind path below observes no state the panicking
280        // drop could have left broken — it only disarms the guard and re-raises.
281        let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || drop(store)));
282        if let Err(panic) = outcome {
283            if let Some(dir) = self.dir.take() {
284                let leaked = dir.keep();
285                tracing::error!(
286                    path = %leaked.display(),
287                    "ephemeral store drop panicked; leaking its directory rather than \
288                     removing it under possibly-live database workers"
289                );
290            }
291            std::panic::resume_unwind(panic);
292        }
293    }
294}
295
296/// Exclusive-ownership ephemeral durable store: the sole owner of both the
297/// haematite database and the temporary directory that backs it.
298///
299/// [`HaematiteStore::new`] takes a *caller-supplied* `Arc<EventStore>`, so a
300/// clone of that inner handle can outlive any guard placed merely beside it —
301/// field declaration order proves nothing across that `Arc` boundary. This
302/// wrapper instead owns the database outright: [`open_ephemeral`] constructs the
303/// inner `Arc` itself, this type never exposes it (no getter) and is deliberately
304/// **not `Clone`**, so the only handle a caller can hold is an
305/// `Arc<dyn DurableStore>` over the whole wrapper. When the last such clone
306/// drops, the [`EphemeralGuard`] drops the store FIRST — the database closes,
307/// its shard actors join and the data-dir writer lock releases on fd close —
308/// and only then removes the directory; if closing the database panics, the
309/// directory is deliberately leaked instead (see [`EphemeralGuard`]).
310#[derive(Debug)]
311pub struct EphemeralHaematiteStore {
312    guard: EphemeralGuard<HaematiteStore>,
313}
314
315impl EphemeralHaematiteStore {
316    /// Takes an already-open ephemeral `Database` and the temporary directory it
317    /// was opened under, becoming their single exclusive owner.
318    ///
319    /// The inner `Arc<EventStore>` is created here and never leaves this type, so
320    /// no caller-supplied clone of it can exist to defeat the drop ordering.
321    /// `ephemeral_dir` must be the directory `database` lives in and must have
322    /// been created before the database was opened (so a failed open removed it
323    /// via the guard's `Drop`, before this constructor was ever reached).
324    fn new(database: Database, ephemeral_dir: TempDir) -> Self {
325        Self {
326            guard: EphemeralGuard {
327                store: Some(HaematiteStore::new(Arc::new(EventStore::new(database)))),
328                dir: Some(ephemeral_dir),
329            },
330        }
331    }
332
333    /// Store handle behind the guard's teardown-only `Option`.
334    ///
335    /// `None` exists only inside [`EphemeralGuard::drop`], which cannot overlap
336    /// a `&self` call, so this error is unreachable by construction — it is a
337    /// typed refusal in place of a panic the workspace forbids, not a state a
338    /// caller can produce.
339    fn store(&self) -> Result<&HaematiteStore, DurabilityError> {
340        self.guard
341            .store
342            .as_ref()
343            .ok_or(DurabilityError::EphemeralStoreDetached)
344    }
345
346    /// Path of the guarding temporary directory, for lifecycle assertions only.
347    #[cfg(test)]
348    pub(crate) fn ephemeral_dir_path(&self) -> Option<&Path> {
349        self.guard.dir.as_ref().map(TempDir::path)
350    }
351}
352
353#[async_trait::async_trait]
354impl DurableStore for EphemeralHaematiteStore {
355    async fn append(
356        &self,
357        stream_key: &str,
358        payload: Vec<u8>,
359        expected_seq: u64,
360    ) -> Result<u64, DurabilityError> {
361        self.store()?
362            .append(stream_key, payload, expected_seq)
363            .await
364    }
365
366    async fn read_from(
367        &self,
368        stream_key: &str,
369        offset: u64,
370        limit: usize,
371    ) -> Result<Vec<StoredEntry>, DurabilityError> {
372        self.store()?.read_from(stream_key, offset, limit).await
373    }
374
375    async fn cas(&self, key: &str, old_value: u64, new_value: u64) -> Result<(), DurabilityError> {
376        self.store()?.cas(key, old_value, new_value).await
377    }
378
379    async fn read_value(&self, key: &str) -> Result<Option<u64>, DurabilityError> {
380        self.store()?.read_value(key).await
381    }
382
383    async fn scan(&self, prefix: &str) -> Result<Vec<StoredEntry>, DurabilityError> {
384        self.store()?.scan(prefix).await
385    }
386
387    async fn flush(&self) -> Result<(), DurabilityError> {
388        self.store()?.flush().await
389    }
390}
391
392/// Opens a self-owning ephemeral haematite store under a fresh temporary
393/// directory below the system temp dir.
394///
395/// The directory is created BEFORE [`Database::create`], so every failure path —
396/// including a haematite open/create error — removes it when the guard drops on
397/// the error return; the returned store owns the guard on success. The database
398/// is created directly in the (empty) temporary directory: haematite's `create`
399/// accepts an existing empty dir and, on failure, removes only a directory *it*
400/// created, never this pre-existing guard dir (haematite 0.4.1
401/// `db/startup.rs`), so the `TempDir` is the sole owner of directory lifetime on
402/// every path.
403///
404/// # Errors
405/// Returns [`DurabilityError::EphemeralStoreOpen`] if haematite cannot create the
406/// database; the temporary directory is already removed when this returns.
407pub fn open_ephemeral(shard_count: usize) -> Result<EphemeralHaematiteStore, DurabilityError> {
408    open_ephemeral_in(ephemeral_tempdir(None)?, shard_count)
409}
410
411/// TEST SEAM: [`open_ephemeral`] with the temporary directory placed under
412/// `root` instead of the system temp dir.
413///
414/// Rooting lets construction gates assert on an isolated directory instead of
415/// scanning the shared temp dir. Same lifecycle contract as
416/// [`open_ephemeral`] — the store owns and removes its directory; `root` must
417/// already exist and must outlive the store.
418///
419/// That last requirement is why this is NOT a production API: the store's
420/// exclusive ownership of its directory (the D3 invariant) says nothing about
421/// the PARENT — a caller rooting the store inside a directory they own via
422/// their own guard can drop that guard while the store is live, deleting the
423/// database out from under its running workers. A general rooted API would
424/// need a root-ownership token so parent cleanup cannot outrun the store;
425/// that is deferred until a real embedder need arrives. Until then the
426/// function is gated to tests (`cfg(test)` in this crate, the default-off
427/// `test-support` feature for downstream test harnesses).
428///
429/// # Errors
430/// Returns [`DurabilityError::EphemeralStoreOpen`] if the directory cannot be
431/// created under `root` or haematite cannot create the database; no residue
432/// remains under `root` when this returns an error.
433#[cfg(any(test, feature = "test-support"))]
434pub fn open_ephemeral_rooted(
435    root: &Path,
436    shard_count: usize,
437) -> Result<EphemeralHaematiteStore, DurabilityError> {
438    open_ephemeral_in(ephemeral_tempdir(Some(root))?, shard_count)
439}
440
441/// Creates the guard directory for an ephemeral store, under `root` when given
442/// and under the system temp dir otherwise.
443fn ephemeral_tempdir(root: Option<&Path>) -> Result<TempDir, DurabilityError> {
444    let mut builder = tempfile::Builder::new();
445    builder.prefix("liminal-durability-");
446    root.map_or_else(|| builder.tempdir(), |root| builder.tempdir_in(root))
447        .map_err(|error| {
448            DurabilityError::EphemeralStoreOpen(format!(
449                "could not create temporary directory: {error}"
450            ))
451        })
452}
453
454/// Opens an ephemeral store inside an already-created guard directory.
455///
456/// Split out so the guard exists before `Database::create` and so lifecycle
457/// tests can inject an open failure into a directory they pre-populated.
458fn open_ephemeral_in(
459    ephemeral_dir: TempDir,
460    shard_count: usize,
461) -> Result<EphemeralHaematiteStore, DurabilityError> {
462    let database = Database::create(DatabaseConfig {
463        data_dir: ephemeral_dir.path().to_path_buf(),
464        shard_count,
465        distributed: None,
466    })
467    .map_err(|error| DurabilityError::EphemeralStoreOpen(error.to_string()))?;
468    Ok(EphemeralHaematiteStore::new(database, ephemeral_dir))
469}
470
471impl From<Event> for StoredEntry {
472    fn from(event: Event) -> Self {
473        Self {
474            payload: event.payload,
475            sequence: event.seq,
476            timestamp: event.timestamp,
477        }
478    }
479}
480
481/// Maps a real-engine [`ApiError`] onto liminal's [`DurabilityError`].
482///
483/// The optimistic-concurrency variants route to their dedicated `DurabilityError`
484/// cases (`SequenceConflict`, `CursorRegression`); everything else is a
485/// store-level failure carried verbatim.
486impl From<ApiError> for DurabilityError {
487    fn from(error: ApiError) -> Self {
488        match error {
489            ApiError::SequenceConflict(conflict) => conflict.into(),
490            ApiError::CasMismatch(mismatch) => mismatch.into(),
491            other @ (ApiError::CorruptEvent(_)
492            | ApiError::Storage(_)
493            | ApiError::HistoryCompacted(_)) => Self::StoreError(other),
494        }
495    }
496}
497
498#[cfg(test)]
499#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
500mod ephemeral_lifecycle_tests {
501    //! D3 §9 lifecycle gate. Each test names the gate it pins; all are permanent
502    //! rule-1 assertions that the ephemeral store's directory has an enforced
503    //! owner across every teardown path.
504
505    use std::path::PathBuf;
506    use std::sync::Arc;
507
508    use super::super::bridge::block_on;
509    use super::{
510        DurableStore, EphemeralGuard, open_ephemeral, open_ephemeral_in, open_ephemeral_rooted,
511    };
512
513    const TEST_SHARD_COUNT: usize = 2;
514
515    /// Store stand-in whose `Drop` pins the guard's internal ordering: the
516    /// directory must still exist at store-drop time, so this drop FAILS the
517    /// test if the guard ever removes the directory first.
518    struct OrderProbeStore {
519        dir: PathBuf,
520    }
521
522    impl Drop for OrderProbeStore {
523        fn drop(&mut self) {
524            assert!(
525                self.dir.exists(),
526                "the guard must drop the store BEFORE removing the directory"
527            );
528        }
529    }
530
531    /// Store stand-in whose `Drop` panics, modelling a haematite worker failing
532    /// to join while the database closes.
533    struct PanickingProbeStore;
534
535    impl Drop for PanickingProbeStore {
536        fn drop(&mut self) {
537            panic!("injected store-drop panic");
538        }
539    }
540
541    /// Materialises shard directories and fds so the drop path actually has a
542    /// live database to close before the guard removes the directory.
543    fn write_one_event(store: &dyn DurableStore) {
544        block_on(store.append("lifecycle/probe", b"payload".to_vec(), 0))
545            .expect("bridge completes synchronously")
546            .expect("append to a fresh ephemeral stream succeeds");
547        block_on(store.flush())
548            .expect("bridge completes synchronously")
549            .expect("flush of a live ephemeral store succeeds");
550    }
551
552    /// §9 gate — normal drop: the directory is removed once the last (here, only)
553    /// handle drops.
554    #[test]
555    fn ephemeral_dir_removed_after_last_handle_drops() {
556        let store = open_ephemeral(TEST_SHARD_COUNT).expect("ephemeral open succeeds");
557        let dir = store
558            .ephemeral_dir_path()
559            .expect("ephemeral store carries a guard dir")
560            .to_path_buf();
561        assert!(
562            dir.exists(),
563            "the guard directory exists while the store is live"
564        );
565
566        write_one_event(&store);
567        drop(store);
568
569        assert!(
570            !dir.exists(),
571            "the guard directory is removed on normal drop"
572        );
573    }
574
575    /// §9 gate — teardown with store-handle clones alive: the directory survives
576    /// until the LAST `Arc<dyn DurableStore>` clone drops, then is removed. This
577    /// is the `Arc`-shared-into-channel-handles case: clones share one wrapper,
578    /// so none can close the database early.
579    #[test]
580    fn ephemeral_dir_survives_until_last_store_clone_drops() {
581        let store = open_ephemeral(TEST_SHARD_COUNT).expect("ephemeral open succeeds");
582        let dir = store
583            .ephemeral_dir_path()
584            .expect("ephemeral store carries a guard dir")
585            .to_path_buf();
586        write_one_event(&store);
587
588        let erased: Arc<dyn DurableStore> = Arc::new(store);
589        let clone_a = Arc::clone(&erased);
590        let clone_b = Arc::clone(&erased);
591
592        drop(erased);
593        assert!(
594            dir.exists(),
595            "directory survives while store clones remain alive"
596        );
597        drop(clone_a);
598        assert!(
599            dir.exists(),
600            "directory survives while one store clone remains alive"
601        );
602
603        drop(clone_b);
604        assert!(
605            !dir.exists(),
606            "the last store clone dropping removes the directory"
607        );
608    }
609
610    /// §9 gate — startup rollback: an injected haematite open failure (a
611    /// conflicting `config.json` pre-seeded into the guard dir) makes the
612    /// constructor return `Err` AND leaves zero residue — the guard removes the
613    /// directory independently of haematite's own cleanup.
614    #[test]
615    fn ephemeral_open_failure_rolls_back_directory() {
616        let seeded = tempfile::Builder::new()
617            .prefix("liminal-durability-test-")
618            .tempdir()
619            .expect("test can create a temp dir");
620        let dir = seeded.path().to_path_buf();
621        // A pre-existing `config.json` makes haematite refuse the create with
622        // `DataDirAlreadyInitialised`; because the dir pre-existed the create,
623        // haematite never removes it — only the guard does.
624        std::fs::write(dir.join("config.json"), b"not-a-valid-config")
625            .expect("test can seed a conflicting config");
626
627        let result = open_ephemeral_in(seeded, TEST_SHARD_COUNT);
628
629        assert!(result.is_err(), "an injected open failure returns Err");
630        assert!(
631            !dir.exists(),
632            "the guard removes the directory on open failure — zero residue"
633        );
634    }
635
636    /// §9 gate — repeated start/stop: each cycle owns a distinct directory and
637    /// leaves zero residue after it drops.
638    #[test]
639    fn repeated_ephemeral_cycles_each_own_distinct_dir_zero_residue() {
640        let mut seen: Vec<PathBuf> = Vec::new();
641        for _ in 0..5 {
642            let store = open_ephemeral(TEST_SHARD_COUNT).expect("ephemeral open succeeds");
643            let dir = store
644                .ephemeral_dir_path()
645                .expect("ephemeral store carries a guard dir")
646                .to_path_buf();
647            assert!(
648                dir.exists(),
649                "the cycle's directory exists while its store is live"
650            );
651            assert!(!seen.contains(&dir), "each cycle owns a distinct directory");
652            seen.push(dir.clone());
653
654            write_one_event(&store);
655            drop(store);
656            assert!(
657                !dir.exists(),
658                "the cycle's directory is removed after its store drops"
659            );
660        }
661    }
662
663    /// §9 gate (drop-order pin): the guard drops the store strictly before it
664    /// removes the directory. `OrderProbeStore::drop` asserts the directory
665    /// still exists, so reversing the order inside [`EphemeralGuard`] fails this
666    /// test rather than silently passing.
667    #[test]
668    fn guard_drops_store_before_removing_directory() {
669        let dir = tempfile::tempdir().expect("test can create a temp dir");
670        let path = dir.path().to_path_buf();
671        let guard = EphemeralGuard {
672            store: Some(OrderProbeStore { dir: path.clone() }),
673            dir: Some(dir),
674        };
675
676        drop(guard);
677
678        assert!(!path.exists(), "a clean drop still removes the directory");
679    }
680
681    /// §9 gate (unwind pin): a panic while the store drops leaves the directory
682    /// LEAKED, never removed under possibly-live workers, and the panic still
683    /// propagates.
684    #[test]
685    fn guard_leaks_directory_when_store_drop_panics() {
686        let dir = tempfile::tempdir().expect("test can create a temp dir");
687        let path = dir.path().to_path_buf();
688        let guard = EphemeralGuard {
689            store: Some(PanickingProbeStore),
690            dir: Some(dir),
691        };
692
693        let unwound = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || drop(guard)));
694
695        assert!(unwound.is_err(), "the injected store-drop panic propagates");
696        assert!(
697            path.exists(),
698            "a panicking store drop leaks the directory instead of removing it"
699        );
700        std::fs::remove_dir_all(&path).expect("test cleans up the deliberately leaked directory");
701    }
702
703    /// The rooted factory places (and removes) the guard directory under the
704    /// caller-supplied root, which is what lets construction gates assert on an
705    /// isolated root instead of scanning the system temp dir.
706    #[test]
707    fn rooted_ephemeral_store_lives_and_dies_under_the_given_root() {
708        let root = tempfile::tempdir().expect("test can create a temp root");
709        let store =
710            open_ephemeral_rooted(root.path(), TEST_SHARD_COUNT).expect("rooted open succeeds");
711        let dir = store
712            .ephemeral_dir_path()
713            .expect("ephemeral store carries a guard dir")
714            .to_path_buf();
715        assert!(
716            dir.starts_with(root.path()),
717            "the guard directory is created under the supplied root"
718        );
719
720        write_one_event(&store);
721        drop(store);
722
723        assert!(!dir.exists(), "the rooted directory is removed on drop");
724    }
725}