Skip to main content

corium_store/
lib.rs

1//! Content-addressed blob and fenced root stores for immutable index segments.
2//!
3//! Enable the `postgres` or `turso` Cargo feature to use
4//! [`PostgresBlobStore`] or [`TursoBlobStore`], respectively.
5
6use std::{
7    collections::{BTreeMap, HashMap, HashSet},
8    fmt,
9    fs::{self, File, OpenOptions},
10    io,
11    path::{Path, PathBuf},
12    pin::Pin,
13    sync::{Arc, RwLock},
14    time::{Duration, SystemTime},
15};
16
17use async_trait::async_trait;
18use fs2::FileExt;
19use thiserror::Error;
20use tokio_stream::{Stream, StreamExt, wrappers::ReceiverStream};
21
22#[cfg(feature = "postgres")]
23mod postgres_store;
24#[cfg(feature = "postgres")]
25pub use postgres_store::PostgresBlobStore;
26#[cfg(feature = "turso")]
27mod turso_store;
28#[cfg(feature = "turso")]
29pub use turso_store::TursoBlobStore;
30
31/// A content identifier for immutable blobs.
32#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
33pub struct BlobId(String);
34
35impl BlobId {
36    /// Returns the hexadecimal digest string.
37    #[must_use]
38    pub fn as_str(&self) -> &str {
39        &self.0
40    }
41
42    /// Parses a stored 64-character hexadecimal digest.
43    #[must_use]
44    pub fn from_hex(text: &str) -> Option<Self> {
45        (text.len() == 64 && text.bytes().all(|byte| byte.is_ascii_hexdigit()))
46            .then(|| Self(text.to_owned()))
47    }
48}
49
50impl fmt::Display for BlobId {
51    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
52        f.write_str(&self.0)
53    }
54}
55
56/// Errors raised by store implementations.
57#[derive(Debug, Error)]
58pub enum StoreError {
59    /// I/O failure.
60    #[error("store I/O failed: {0}")]
61    Io(#[from] io::Error),
62    /// Root compare-and-swap failed because the current fence differed.
63    #[error("root CAS failed: expected {expected:?}, actual {actual:?}")]
64    CasFailed {
65        /// Expected root bytes supplied by the caller.
66        expected: Option<Vec<u8>>,
67        /// Actual root bytes currently stored.
68        actual: Option<Vec<u8>>,
69    },
70    /// Blob digest did not match its content.
71    #[error("blob content did not match digest {0}")]
72    CorruptBlob(BlobId),
73    /// A live graph references a blob that is not present.
74    #[error("reachable blob is missing: {0}")]
75    MissingBlob(BlobId),
76    /// Root name cannot be safely represented on the filesystem.
77    #[error("invalid root name {0:?}")]
78    InvalidRootName(String),
79    /// A blocking store worker failed before returning its result.
80    #[error("store blocking task failed: {0}")]
81    BlockingTask(String),
82    /// `PostgreSQL` database failure.
83    #[cfg(feature = "postgres")]
84    #[error("PostgreSQL store failed: {0}")]
85    Postgres(#[from] deadpool_postgres::tokio_postgres::Error),
86    /// `PostgreSQL` connection-pool checkout failure.
87    #[cfg(feature = "postgres")]
88    #[error("PostgreSQL connection pool failed: {0}")]
89    PostgresPool(#[from] deadpool_postgres::PoolError),
90    /// `PostgreSQL` connection-pool configuration failure.
91    #[cfg(feature = "postgres")]
92    #[error("PostgreSQL connection pool configuration failed: {0}")]
93    PostgresPoolCreate(#[from] deadpool_postgres::CreatePoolError),
94    /// No native certificate roots were available for `PostgreSQL` TLS.
95    #[cfg(feature = "postgres")]
96    #[error("cannot load native certificate roots for PostgreSQL TLS: {0}")]
97    PostgresTlsRoots(String),
98    /// `PostgreSQL` returned invalid store data.
99    #[cfg(feature = "postgres")]
100    #[error("PostgreSQL store contains invalid data: {0}")]
101    InvalidPostgresData(String),
102    /// Turso database failure.
103    #[cfg(feature = "turso")]
104    #[error("Turso blob store failed: {0}")]
105    Turso(#[from] turso::Error),
106    /// A filesystem path cannot be passed to Turso.
107    #[cfg(feature = "turso")]
108    #[error("Turso database path is not valid UTF-8: {0:?}")]
109    InvalidTursoPath(PathBuf),
110    /// Turso returned invalid blob-store data.
111    #[cfg(feature = "turso")]
112    #[error("Turso blob store contains invalid data: {0}")]
113    InvalidTursoData(String),
114}
115
116/// Asynchronous stream of blob identifiers produced by [`BlobStore::list`].
117pub type BlobIdStream = Pin<Box<dyn Stream<Item = Result<BlobId, StoreError>> + Send + 'static>>;
118
119async fn run_blocking<T>(
120    operation: impl FnOnce() -> Result<T, StoreError> + Send + 'static,
121) -> Result<T, StoreError>
122where
123    T: Send + 'static,
124{
125    tokio::task::spawn_blocking(operation)
126        .await
127        .map_err(|error| StoreError::BlockingTask(error.to_string()))?
128}
129
130/// Immutable content-addressed blob storage.
131#[async_trait]
132pub trait BlobStore: Send + Sync {
133    /// Stores bytes and returns their content id.
134    ///
135    /// # Errors
136    ///
137    /// Returns an error if the backend cannot persist the blob.
138    async fn put(&self, bytes: &[u8]) -> Result<BlobId, StoreError>;
139    /// Loads bytes by id, returning `None` when missing.
140    ///
141    /// # Errors
142    ///
143    /// Returns an error if the backend cannot read or verify the blob.
144    async fn get(&self, id: &BlobId) -> Result<Option<Vec<u8>>, StoreError>;
145    /// Reports whether a blob is present.
146    ///
147    /// # Errors
148    ///
149    /// Returns an error if the backend cannot inspect the blob.
150    async fn contains(&self, id: &BlobId) -> Result<bool, StoreError> {
151        Ok(self.get(id).await?.is_some())
152    }
153    /// Deletes a blob during garbage collection. Missing blobs are ignored.
154    ///
155    /// # Errors
156    ///
157    /// Returns an error if the backend cannot delete the blob.
158    async fn delete(&self, id: &BlobId) -> Result<(), StoreError>;
159    /// Lists all blob identifiers known to this backend.
160    ///
161    /// # Errors
162    ///
163    /// Returns an error if the backend cannot enumerate blobs.
164    async fn list(&self) -> Result<BlobIdStream, StoreError>;
165    /// Returns the blob's creation/last-modification time when available.
166    /// Backends without timestamps return `None`, which conservatively keeps
167    /// the blob whenever a non-zero retention window is active.
168    ///
169    /// # Errors
170    /// Returns an error if the backend cannot inspect blob metadata.
171    async fn modified_at(&self, _id: &BlobId) -> Result<Option<SystemTime>, StoreError> {
172        Ok(None)
173    }
174}
175
176/// Named root pointer storage with compare-and-swap fencing.
177#[async_trait]
178pub trait RootStore: Send + Sync {
179    /// Reads a root pointer.
180    ///
181    /// # Errors
182    ///
183    /// Returns an error if the backend cannot read the root.
184    async fn get_root(&self, name: &str) -> Result<Option<Vec<u8>>, StoreError>;
185    /// Publishes a root only if the stored pointer equals `expected`.
186    ///
187    /// # Errors
188    ///
189    /// Returns an error if the fence does not match or the backend cannot publish.
190    async fn cas_root(
191        &self,
192        name: &str,
193        expected: Option<&[u8]>,
194        new: &[u8],
195    ) -> Result<(), StoreError>;
196    /// Removes a root pointer. Missing roots are ignored.
197    ///
198    /// # Errors
199    ///
200    /// Returns an error if the backend cannot delete the root.
201    async fn delete_root(&self, name: &str) -> Result<(), StoreError>;
202    /// Lists root names beginning with `prefix`, in sorted order.
203    ///
204    /// # Errors
205    ///
206    /// Returns an error if the backend cannot enumerate roots.
207    async fn list_roots(&self, prefix: &str) -> Result<Vec<String>, StoreError>;
208}
209
210/// In-memory blob and root store for tests and embedded use.
211#[derive(Clone, Default)]
212pub struct MemoryStore {
213    inner: Arc<RwLock<MemoryInner>>,
214}
215#[derive(Default)]
216struct MemoryInner {
217    blobs: HashMap<BlobId, Vec<u8>>,
218    roots: BTreeMap<String, Vec<u8>>,
219}
220
221#[async_trait]
222impl BlobStore for MemoryStore {
223    async fn put(&self, bytes: &[u8]) -> Result<BlobId, StoreError> {
224        let inner = Arc::clone(&self.inner);
225        let bytes = bytes.to_vec();
226        run_blocking(move || {
227            let id = digest(&bytes);
228            inner
229                .write()
230                .expect("poisoned store lock")
231                .blobs
232                .insert(id.clone(), bytes);
233            Ok(id)
234        })
235        .await
236    }
237    async fn get(&self, id: &BlobId) -> Result<Option<Vec<u8>>, StoreError> {
238        let inner = Arc::clone(&self.inner);
239        let id = id.clone();
240        run_blocking(move || {
241            Ok(inner
242                .read()
243                .expect("poisoned store lock")
244                .blobs
245                .get(&id)
246                .cloned())
247        })
248        .await
249    }
250    async fn delete(&self, id: &BlobId) -> Result<(), StoreError> {
251        let inner = Arc::clone(&self.inner);
252        let id = id.clone();
253        run_blocking(move || {
254            inner
255                .write()
256                .expect("poisoned store lock")
257                .blobs
258                .remove(&id);
259            Ok(())
260        })
261        .await
262    }
263    async fn list(&self) -> Result<BlobIdStream, StoreError> {
264        let inner = Arc::clone(&self.inner);
265        let ids = run_blocking(move || {
266            Ok(inner
267                .read()
268                .expect("poisoned store lock")
269                .blobs
270                .keys()
271                .cloned()
272                .collect::<Vec<_>>())
273        })
274        .await?;
275        Ok(Box::pin(tokio_stream::iter(ids.into_iter().map(Ok))))
276    }
277}
278#[async_trait]
279impl RootStore for MemoryStore {
280    async fn get_root(&self, name: &str) -> Result<Option<Vec<u8>>, StoreError> {
281        let inner = Arc::clone(&self.inner);
282        let name = name.to_owned();
283        run_blocking(move || {
284            Ok(inner
285                .read()
286                .expect("poisoned store lock")
287                .roots
288                .get(&name)
289                .cloned())
290        })
291        .await
292    }
293    async fn cas_root(
294        &self,
295        name: &str,
296        expected: Option<&[u8]>,
297        new: &[u8],
298    ) -> Result<(), StoreError> {
299        let inner = Arc::clone(&self.inner);
300        let name = name.to_owned();
301        let expected = expected.map(<[u8]>::to_vec);
302        let new = new.to_vec();
303        run_blocking(move || {
304            let mut inner = inner.write().expect("poisoned store lock");
305            let actual = inner.roots.get(&name).cloned();
306            if actual != expected {
307                return Err(StoreError::CasFailed { expected, actual });
308            }
309            inner.roots.insert(name, new);
310            Ok(())
311        })
312        .await
313    }
314    async fn delete_root(&self, name: &str) -> Result<(), StoreError> {
315        let inner = Arc::clone(&self.inner);
316        let name = name.to_owned();
317        run_blocking(move || {
318            inner
319                .write()
320                .expect("poisoned store lock")
321                .roots
322                .remove(&name);
323            Ok(())
324        })
325        .await
326    }
327    async fn list_roots(&self, prefix: &str) -> Result<Vec<String>, StoreError> {
328        let inner = Arc::clone(&self.inner);
329        let prefix = prefix.to_owned();
330        run_blocking(move || {
331            Ok(inner
332                .read()
333                .expect("poisoned store lock")
334                .roots
335                .keys()
336                .filter(|name| name.starts_with(&prefix))
337                .cloned()
338                .collect())
339        })
340        .await
341    }
342}
343
344/// Filesystem-backed content-addressed blob and fenced root store.
345#[derive(Clone)]
346pub struct FsStore {
347    root: PathBuf,
348}
349impl FsStore {
350    /// Opens or creates a store below `root`.
351    ///
352    /// # Errors
353    ///
354    /// Returns an error if the directory layout cannot be created.
355    pub fn open(root: impl AsRef<Path>) -> Result<Self, StoreError> {
356        let root = root.as_ref().to_path_buf();
357        fs::create_dir_all(root.join("blobs"))?;
358        fs::create_dir_all(root.join("roots"))?;
359        Ok(Self { root })
360    }
361    fn blob_path(&self, id: &BlobId) -> PathBuf {
362        self.root.join("blobs").join(id.as_str())
363    }
364    fn root_path(&self, name: &str) -> Result<PathBuf, StoreError> {
365        if name.is_empty()
366            || name == "."
367            || name == ".."
368            || name.contains('/')
369            || name.contains('\\')
370        {
371            return Err(StoreError::InvalidRootName(name.to_owned()));
372        }
373        Ok(self.root.join("roots").join(name))
374    }
375
376    fn root_lock(&self, name: &str) -> Result<RootLock, StoreError> {
377        let root_path = self.root_path(name)?;
378        RootLock::acquire(&root_path.with_extension("lock"))
379    }
380}
381#[async_trait]
382impl BlobStore for FsStore {
383    async fn put(&self, bytes: &[u8]) -> Result<BlobId, StoreError> {
384        let store = self.clone();
385        let bytes = bytes.to_vec();
386        run_blocking(move || {
387            let id = digest(&bytes);
388            let path = store.blob_path(&id);
389            if !path.exists() {
390                let tmp = path.with_extension("tmp");
391                fs::write(&tmp, &bytes)?;
392                fs::rename(tmp, path)?;
393            }
394            Ok(id)
395        })
396        .await
397    }
398    async fn get(&self, id: &BlobId) -> Result<Option<Vec<u8>>, StoreError> {
399        let store = self.clone();
400        let id = id.clone();
401        run_blocking(move || {
402            let path = store.blob_path(&id);
403            if !path.exists() {
404                return Ok(None);
405            }
406            let bytes = fs::read(path)?;
407            if digest(&bytes) != id {
408                return Err(StoreError::CorruptBlob(id));
409            }
410            Ok(Some(bytes))
411        })
412        .await
413    }
414    async fn contains(&self, id: &BlobId) -> Result<bool, StoreError> {
415        let store = self.clone();
416        let id = id.clone();
417        run_blocking(move || Ok(store.blob_path(&id).is_file())).await
418    }
419    async fn delete(&self, id: &BlobId) -> Result<(), StoreError> {
420        let store = self.clone();
421        let id = id.clone();
422        run_blocking(move || match fs::remove_file(store.blob_path(&id)) {
423            Ok(()) => Ok(()),
424            Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
425            Err(error) => Err(error.into()),
426        })
427        .await
428    }
429    async fn list(&self) -> Result<BlobIdStream, StoreError> {
430        let path = self.root.join("blobs");
431        let entries = run_blocking(move || Ok(fs::read_dir(path)?)).await?;
432        let (tx, rx) = tokio::sync::mpsc::channel(64);
433        let failure_tx = tx.clone();
434        tokio::spawn(async move {
435            let result = tokio::task::spawn_blocking(move || {
436                for entry in entries {
437                    let id = (|| {
438                        let entry = entry?;
439                        if !entry.file_type()?.is_file() {
440                            return Ok(None);
441                        }
442                        Ok(entry.file_name().to_str().and_then(BlobId::from_hex))
443                    })();
444                    match id {
445                        Ok(Some(id)) => {
446                            if tx.blocking_send(Ok(id)).is_err() {
447                                return;
448                            }
449                        }
450                        Ok(None) => {}
451                        Err(error) => {
452                            let _ = tx.blocking_send(Err(StoreError::Io(error)));
453                            return;
454                        }
455                    }
456                }
457            })
458            .await;
459            if let Err(error) = result {
460                let _ = failure_tx
461                    .send(Err(StoreError::BlockingTask(error.to_string())))
462                    .await;
463            }
464        });
465        Ok(Box::pin(ReceiverStream::new(rx)))
466    }
467    async fn modified_at(&self, id: &BlobId) -> Result<Option<SystemTime>, StoreError> {
468        let store = self.clone();
469        let id = id.clone();
470        run_blocking(move || match fs::metadata(store.blob_path(&id)) {
471            Ok(metadata) => Ok(Some(metadata.modified()?)),
472            Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
473            Err(error) => Err(error.into()),
474        })
475        .await
476    }
477}
478#[async_trait]
479impl RootStore for FsStore {
480    async fn get_root(&self, name: &str) -> Result<Option<Vec<u8>>, StoreError> {
481        let store = self.clone();
482        let name = name.to_owned();
483        run_blocking(move || match fs::read(store.root_path(&name)?) {
484            Ok(value) => Ok(Some(value)),
485            Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
486            Err(error) => Err(error.into()),
487        })
488        .await
489    }
490    async fn cas_root(
491        &self,
492        name: &str,
493        expected: Option<&[u8]>,
494        new: &[u8],
495    ) -> Result<(), StoreError> {
496        let store = self.clone();
497        let name = name.to_owned();
498        let expected = expected.map(<[u8]>::to_vec);
499        let new = new.to_vec();
500        run_blocking(move || {
501            let _lock = store.root_lock(&name)?;
502            let path = store.root_path(&name)?;
503            let actual = match fs::read(&path) {
504                Ok(value) => Some(value),
505                Err(error) if error.kind() == io::ErrorKind::NotFound => None,
506                Err(error) => return Err(error.into()),
507            };
508            if actual != expected {
509                return Err(StoreError::CasFailed { expected, actual });
510            }
511            let tmp = path.with_extension("tmp");
512            fs::write(&tmp, new)?;
513            fs::rename(tmp, path)?;
514            Ok(())
515        })
516        .await
517    }
518    async fn delete_root(&self, name: &str) -> Result<(), StoreError> {
519        let store = self.clone();
520        let name = name.to_owned();
521        run_blocking(move || {
522            let _lock = store.root_lock(&name)?;
523            match fs::remove_file(store.root_path(&name)?) {
524                Ok(()) => Ok(()),
525                Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
526                Err(error) => Err(error.into()),
527            }
528        })
529        .await
530    }
531    async fn list_roots(&self, prefix: &str) -> Result<Vec<String>, StoreError> {
532        let root = self.root.clone();
533        let prefix = prefix.to_owned();
534        run_blocking(move || {
535            let mut names = Vec::new();
536            for entry in fs::read_dir(root.join("roots"))? {
537                let entry = entry?;
538                if !entry.file_type()?.is_file() {
539                    continue;
540                }
541                if let Some(name) = entry.file_name().to_str() {
542                    let auxiliary = Path::new(name).extension().is_some_and(|ext| {
543                        ext.eq_ignore_ascii_case("lock") || ext.eq_ignore_ascii_case("tmp")
544                    });
545                    if name.starts_with(&prefix) && !auxiliary {
546                        names.push(name.to_owned());
547                    }
548                }
549            }
550            names.sort();
551            Ok(names)
552        })
553        .await
554    }
555}
556
557fn digest(bytes: &[u8]) -> BlobId {
558    BlobId(blake3::hash(bytes).to_hex().to_string())
559}
560
561/// Root-store key for a database's published index root.
562#[must_use]
563pub fn db_root_name(db: &str) -> String {
564    format!("db:{db}")
565}
566
567/// Storage format written by this release.
568///
569/// Format 2 (M7) folds the write lease into the root record so lease
570/// ownership and index publication are fenced by one atomic CAS; format 1
571/// roots (separate `lease:` record) decode with an unowned lease.
572pub const FORMAT_VERSION: u32 = 2;
573
574/// Published durable index-root metadata carrying the write lease
575/// (see `docs/design/log-and-transactor.md`).
576///
577/// The lease fields and the index fields live in one record on purpose:
578/// every mutation — lease acquisition, renewal, release, index publication —
579/// is a CAS on these bytes, so a writer whose ownership has changed hands
580/// always fails its next CAS and can never install a root. No cross-record
581/// atomicity is required of the store.
582#[derive(Clone, Debug, Eq, PartialEq)]
583pub struct DbRoot {
584    /// On-disk format version. Readers reject roots from newer formats.
585    pub format_version: u32,
586    /// Fencing version; increments on every change of lease ownership.
587    pub lease_version: u64,
588    /// Owning transactor id; empty when the lease has never been acquired
589    /// under format 2.
590    pub owner: String,
591    /// Lease expiry as Unix milliseconds; `0` when released/never held.
592    pub lease_expires_unix_ms: i64,
593    /// Client endpoint advertised by the owner (for peer lease-holder
594    /// rediscovery); empty when the owner does not advertise one.
595    pub owner_endpoint: String,
596    /// Highest indexed transaction.
597    pub index_basis_t: u64,
598    /// EAVT, AEVT, AVET, and VAET blob ids; `None` before the first index
599    /// publication (a bare fence bump).
600    pub roots: Option<[BlobId; 4]>,
601}
602
603/// Encodes a possibly empty single-line field.
604fn field_line(out: &mut String, value: &str) {
605    if value.is_empty() {
606        out.push('-');
607    } else {
608        out.push_str(value);
609    }
610    out.push('\n');
611}
612
613fn parse_field(line: &str) -> String {
614    if line == "-" {
615        String::new()
616    } else {
617        line.to_owned()
618    }
619}
620
621impl DbRoot {
622    /// Encodes the root for the root store.
623    #[must_use]
624    pub fn encode(&self) -> Vec<u8> {
625        let mut out = format!(
626            "corium-root-v{}\n{}\n{}\n",
627            self.format_version, self.lease_version, self.index_basis_t
628        );
629        match &self.roots {
630            Some(roots) => {
631                for root in roots {
632                    out.push_str(root.as_str());
633                    out.push('\n');
634                }
635            }
636            None => out.push_str("-\n-\n-\n-\n"),
637        }
638        field_line(&mut out, &self.owner);
639        out.push_str(&self.lease_expires_unix_ms.to_string());
640        out.push('\n');
641        field_line(&mut out, &self.owner_endpoint);
642        out.into_bytes()
643    }
644
645    /// Decodes stored root bytes (any format up to [`FORMAT_VERSION`];
646    /// newer formats still yield their fence fields so old binaries fence
647    /// correctly, and callers reject them via `format_version`).
648    #[must_use]
649    pub fn decode(bytes: &[u8]) -> Option<Self> {
650        let text = std::str::from_utf8(bytes).ok()?;
651        let mut lines = text.lines();
652        let first = lines.next()?;
653        let (format_version, lease_version) =
654            if let Some(version) = first.strip_prefix("corium-root-v") {
655                let format_version = version.parse().ok()?;
656                let lease_version = lines.next()?.parse().ok()?;
657                (format_version, lease_version)
658            } else {
659                // M1-M5 roots had no header. Keep them readable as format v1 so
660                // an existing database can be upgraded in place.
661                (1, first.parse().ok()?)
662            };
663        let index_basis_t = lines.next()?.parse().ok()?;
664        let ids: Vec<&str> = lines.by_ref().take(4).collect();
665        if ids.len() != 4 {
666            return None;
667        }
668        let roots = if ids.iter().all(|id| *id == "-") {
669            None
670        } else {
671            Some([
672                BlobId::from_hex(ids[0])?,
673                BlobId::from_hex(ids[1])?,
674                BlobId::from_hex(ids[2])?,
675                BlobId::from_hex(ids[3])?,
676            ])
677        };
678        // Lease fields; absent in format-1 roots.
679        let owner = lines.next().map(parse_field).unwrap_or_default();
680        let lease_expires_unix_ms = lines.next().and_then(|l| l.parse().ok()).unwrap_or(0);
681        let owner_endpoint = lines.next().map(parse_field).unwrap_or_default();
682        Some(Self {
683            format_version,
684            lease_version,
685            owner,
686            lease_expires_unix_ms,
687            owner_endpoint,
688            index_basis_t,
689            roots,
690        })
691    }
692}
693
694/// Result counters from a mark-and-sweep garbage collection pass.
695#[derive(Clone, Copy, Debug, Eq, PartialEq)]
696pub struct GcReport {
697    /// Number of blobs reachable from the supplied roots.
698    pub marked: usize,
699    /// Number of unreachable blobs deleted.
700    pub swept: usize,
701    /// Number of unreachable blobs kept because they are inside retention.
702    pub retained: usize,
703}
704
705/// Marks blobs reachable from `live_roots` and deletes every unmarked blob.
706///
707/// `children` decodes references from each present blob. Callers are responsible for
708/// supplying every currently live root and for applying any desired retention window.
709///
710/// # Errors
711///
712/// Returns an error if a blob operation or child-reference decode fails.
713pub async fn mark_and_sweep(
714    store: &dyn BlobStore,
715    live_roots: impl IntoIterator<Item = BlobId>,
716    mut children: impl FnMut(&BlobId, &[u8]) -> Result<Vec<BlobId>, StoreError>,
717) -> Result<GcReport, StoreError> {
718    mark_and_sweep_retained(
719        store,
720        live_roots,
721        &mut children,
722        Duration::ZERO,
723        SystemTime::now(),
724    )
725    .await
726}
727
728/// Marks reachable blobs and deletes only unreachable blobs older than
729/// `retention` relative to `now`.
730///
731/// # Errors
732/// Returns an error if a blob operation or child-reference decode fails.
733pub async fn mark_and_sweep_retained(
734    store: &dyn BlobStore,
735    live_roots: impl IntoIterator<Item = BlobId>,
736    mut children: impl FnMut(&BlobId, &[u8]) -> Result<Vec<BlobId>, StoreError>,
737    retention: Duration,
738    now: SystemTime,
739) -> Result<GcReport, StoreError> {
740    let mut marked = HashSet::new();
741    let mut pending = live_roots.into_iter().collect::<Vec<_>>();
742    while let Some(id) = pending.pop() {
743        if !marked.insert(id.clone()) {
744            continue;
745        }
746        let bytes = store
747            .get(&id)
748            .await?
749            .ok_or_else(|| StoreError::MissingBlob(id.clone()))?;
750        pending.extend(children(&id, &bytes)?);
751    }
752
753    let mut swept = 0;
754    let mut retained = 0;
755    let mut ids = store.list().await?;
756    while let Some(id) = ids.next().await {
757        let id = id?;
758        if !marked.contains(&id) {
759            // A zero window is the explicit immediate-sweep escape hatch and
760            // does not require backend timestamp support. Otherwise, unknown
761            // timestamps fail safe by retaining the blob.
762            let old_enough = retention.is_zero()
763                || store.modified_at(&id).await?.is_some_and(|modified| {
764                    now.duration_since(modified).unwrap_or_default() >= retention
765                });
766            if old_enough {
767                store.delete(&id).await?;
768                swept += 1;
769            } else {
770                retained += 1;
771            }
772        }
773    }
774    Ok(GcReport {
775        marked: marked.len(),
776        swept,
777        retained,
778    })
779}
780
781struct RootLock {
782    file: File,
783}
784
785impl RootLock {
786    fn acquire(path: &Path) -> Result<Self, StoreError> {
787        let file = OpenOptions::new()
788            .read(true)
789            .write(true)
790            .create(true)
791            .truncate(false)
792            .open(path)?;
793        file.lock_exclusive()?;
794        Ok(Self { file })
795    }
796}
797
798impl Drop for RootLock {
799    fn drop(&mut self) {
800        let _ = FileExt::unlock(&self.file);
801        // Keep the lock file in place so every contender locks the same inode.
802        // Unlinking it here would let a new opener lock a replacement file while
803        // a waiter still holds a descriptor for the unlinked original.
804    }
805}
806
807/// Small read-through segment cache keyed by blob id.
808#[derive(Default)]
809pub struct SegmentCache {
810    entries: RwLock<HashMap<BlobId, Arc<[u8]>>>,
811}
812impl SegmentCache {
813    /// Returns cached bytes, loading from `store` on miss.
814    ///
815    /// # Errors
816    ///
817    /// Returns an error if the backing store cannot load the blob.
818    ///
819    /// # Panics
820    ///
821    /// Panics if the internal cache lock is poisoned.
822    pub async fn get_or_load(
823        &self,
824        store: &dyn BlobStore,
825        id: &BlobId,
826    ) -> Result<Option<Arc<[u8]>>, StoreError> {
827        if let Some(v) = self.entries.read().expect("poisoned cache lock").get(id) {
828            return Ok(Some(v.clone()));
829        }
830        let Some(bytes) = store.get(id).await? else {
831            return Ok(None);
832        };
833        let bytes: Arc<[u8]> = bytes.into();
834        self.entries
835            .write()
836            .expect("poisoned cache lock")
837            .insert(id.clone(), bytes.clone());
838        Ok(Some(bytes))
839    }
840}