sui_castore/storage/
mod.rs

1//! Storage backend trait and implementations.
2//!
3//! The `StorageBackend` trait abstracts over where narinfo metadata and
4//! compressed NAR blobs are persisted. Implementations provided:
5//!
6//! - [`LocalStorage`] — local filesystem (default)
7//! - [`S3Storage`] — S3-compatible object storage (AWS, MinIO, R2, RustFS)
8//! - [`RedisBackend`] — Redis L1 hot cache (sub-ms, TTL/eviction-aware)
9//! - [`PgStorageBackend`] — Postgres L2 durable cache tier (shared,
10//!   authoritative)
11//! - [`TieredBackend`] — L1→L2→L3 read-through/write-through resolver
12//! - [`StorageIndex`] — redb ephemeral metadata index (accelerates S3 lookups)
13//!
14//! [`build_backend`] is the typed config-select factory: it dispatches a
15//! [`BackendConfig`](crate::config::BackendConfig) to its concrete backend
16//! (recursing for the tiered arm), so a deployment picks `{disk | s3 | redis |
17//! pg | tiered}` by configuration — never a silent hard-coded constructor.
18
19pub mod index;
20pub mod local;
21pub mod pg;
22pub mod redis;
23pub mod s3;
24pub mod tiered;
25
26use std::sync::Arc;
27
28pub use index::StorageIndex;
29pub use local::LocalStorage;
30pub use pg::{PgCacheConn, PgStorageBackend, PgTable};
31pub use redis::{RedisBackend, RedisConn};
32pub use s3::S3Storage;
33pub use tiered::{TieredBackend, TieredTier, WritePolicy, TIERED_BACKEND_TIER};
34
35#[cfg(feature = "redis-client")]
36pub use redis::RedisConnectionManager;
37
38#[cfg(feature = "postgres")]
39pub use pg::SqlxPgCacheConn;
40
41use async_trait::async_trait;
42use futures::future::BoxFuture;
43
44use crate::config::BackendConfig;
45use crate::StoreError;
46
47/// Abstraction over binary cache storage.
48///
49/// Narinfo files are keyed by the 32-character store path hash.
50/// NAR blobs are keyed by their relative URL path (e.g. `nar/<hash>.nar.xz`).
51#[async_trait]
52pub trait StorageBackend: Send + Sync {
53    /// Retrieve narinfo text by store path hash.
54    async fn get_narinfo(&self, hash: &str) -> Result<Option<String>, StoreError>;
55
56    /// Store narinfo text keyed by store path hash.
57    async fn put_narinfo(&self, hash: &str, content: &str) -> Result<(), StoreError>;
58
59    /// Retrieve a NAR blob by its relative path.
60    async fn get_nar(&self, path: &str) -> Result<Option<Vec<u8>>, StoreError>;
61
62    /// Store a NAR blob at the given relative path.
63    async fn put_nar(&self, path: &str, data: &[u8]) -> Result<(), StoreError>;
64
65    /// Delete a store path's narinfo and associated NAR blob.
66    async fn delete(&self, hash: &str) -> Result<(), StoreError>;
67
68    /// List all stored narinfo hashes.
69    async fn list_narinfos(&self) -> Result<Vec<String>, StoreError>;
70
71    /// Clear EVERY narinfo and NAR blob from this backend. Returns the number
72    /// of narinfos removed.
73    ///
74    /// The default lists every narinfo and best-effort `delete`s it (narinfo-only
75    /// clear; NAR blobs keyed by *narhash* are not reached). Concrete durable
76    /// tiers override with a real truncation that reclaims NAR bytes.
77    async fn wipe_all(&self) -> Result<usize, StoreError> {
78        let hashes = self.list_narinfos().await?;
79        let n = hashes.len();
80        for hash in hashes {
81            self.delete(&hash).await?;
82        }
83        Ok(n)
84    }
85}
86
87/// Config-select factory: build the concrete [`StorageBackend`] a
88/// [`BackendConfig`] names.
89///
90/// This is **typed dispatch, not stringly** — a new backend kind is a
91/// non-exhaustive-`match` compile error, and the [`Tiered`](BackendConfig::Tiered)
92/// arm recurses, composing each sub-backend into a [`TieredBackend`]. The result
93/// is an `Arc<dyn StorageBackend>` ready for injection into any consumer.
94///
95/// The `Redis` and `Pg` arms require their production transports; without the
96/// corresponding Cargo feature (`redis-client` / `postgres`) they return a typed
97/// [`StoreError::NotImplemented`] rather than silently falling back to disk.
98///
99/// Returns a boxed future because the `Tiered` arm is recursive.
100///
101/// # Errors
102///
103/// Propagates any backend construction failure, or [`StoreError::NotImplemented`]
104/// when a config selects a backend whose feature is not compiled in.
105pub fn build_backend(
106    config: &BackendConfig,
107) -> BoxFuture<'_, Result<Arc<dyn StorageBackend>, StoreError>> {
108    Box::pin(async move {
109        match config {
110            BackendConfig::Local { path } => {
111                Ok(Arc::new(LocalStorage::new(path.clone())) as Arc<dyn StorageBackend>)
112            }
113            BackendConfig::S3 { bucket, region, endpoint } => {
114                let s3 = S3Storage::new(bucket.clone(), region.clone(), endpoint.clone())?;
115                Ok(Arc::new(s3) as Arc<dyn StorageBackend>)
116            }
117            BackendConfig::Redis { url, ttl_secs } => build_redis(url, *ttl_secs).await,
118            BackendConfig::Pg { url, max_conns } => build_pg(url, *max_conns).await,
119            BackendConfig::Tiered { l1, l2, l3, write_policy } => {
120                let l1 = build_backend(l1).await?;
121                let l2 = build_backend(l2).await?;
122                let l3 = build_backend(l3).await?;
123                Ok(Arc::new(TieredBackend::with_write_policy(l1, l2, l3, *write_policy))
124                    as Arc<dyn StorageBackend>)
125            }
126        }
127    })
128}
129
130#[cfg(feature = "redis-client")]
131async fn build_redis(
132    url: &str,
133    ttl_secs: Option<u64>,
134) -> Result<Arc<dyn StorageBackend>, StoreError> {
135    let backend = match ttl_secs {
136        Some(t) => RedisBackend::connect_with_ttl(url, t).await?,
137        None => RedisBackend::connect(url).await?,
138    };
139    Ok(Arc::new(backend) as Arc<dyn StorageBackend>)
140}
141
142#[cfg(not(feature = "redis-client"))]
143async fn build_redis(
144    _url: &str,
145    _ttl_secs: Option<u64>,
146) -> Result<Arc<dyn StorageBackend>, StoreError> {
147    Err(StoreError::NotImplemented(
148        "redis L1 backend requires building sui-castore with --features redis-client",
149    ))
150}
151
152#[cfg(feature = "postgres")]
153async fn build_pg(url: &str, max_conns: u32) -> Result<Arc<dyn StorageBackend>, StoreError> {
154    let backend = PgStorageBackend::connect(url, max_conns).await?;
155    Ok(Arc::new(backend) as Arc<dyn StorageBackend>)
156}
157
158#[cfg(not(feature = "postgres"))]
159async fn build_pg(_url: &str, _max_conns: u32) -> Result<Arc<dyn StorageBackend>, StoreError> {
160    Err(StoreError::NotImplemented(
161        "postgres L2 backend requires building sui-castore with --features postgres",
162    ))
163}
sui_castore/storage/mod.rs

sui_castore/storage/
mod.rs
