sui_cache/storage/
mod.rs

1//! Storage backend trait and implementations.
2//!
3//! The `StorageBackend` trait abstracts over where narinfo metadata and
4//! compressed NAR blobs are persisted. Three implementations are provided:
5//!
6//! - [`LocalStorage`] — local filesystem (default)
7//! - [`S3Storage`] — S3-compatible object storage (AWS, MinIO, R2, RustFS)
8//! - [`RedisBackend`] — Redis L1 hot cache (sub-ms, TTL/eviction-aware)
9//! - [`PgStorageBackend`] — Postgres L2 durable cache tier (shared, authoritative)
10//! - [`TieredBackend`] — L1→L2→L3 read-through/write-through resolver
11//! - [`StorageIndex`] — redb ephemeral metadata index (accelerates S3 lookups)
12//!
13//! [`build_backend`] is the typed config-select factory: it dispatches a
14//! [`BackendConfig`](crate::config::BackendConfig) to its concrete backend
15//! (recursing for the tiered arm), so a deployment picks `{disk | s3 | redis | pg
16//! | tiered}` by configuration — never a silent hard-coded constructor.
17
18pub mod index;
19pub mod local;
20pub mod pg;
21pub mod redis;
22pub mod s3;
23pub mod tiered;
24
25use std::sync::Arc;
26
27pub use index::StorageIndex;
28pub use local::LocalStorage;
29pub use pg::{PgCacheConn, PgStorageBackend, PgTable};
30pub use redis::{RedisBackend, RedisConn};
31pub use s3::S3Storage;
32pub use tiered::{TieredBackend, TieredTier, WritePolicy, TIERED_BACKEND_TIER};
33
34#[cfg(feature = "redis-client")]
35pub use redis::RedisConnectionManager;
36
37#[cfg(feature = "postgres")]
38pub use pg::SqlxPgCacheConn;
39
40use async_trait::async_trait;
41use futures::future::BoxFuture;
42
43use crate::config::BackendConfig;
44use crate::CacheError;
45
46/// Abstraction over binary cache storage.
47///
48/// Narinfo files are keyed by the 32-character store path hash.
49/// NAR blobs are keyed by their relative URL path (e.g. `nar/<hash>.nar.xz`).
50#[async_trait]
51pub trait StorageBackend: Send + Sync {
52    /// Retrieve narinfo text by store path hash.
53    async fn get_narinfo(&self, hash: &str) -> Result<Option<String>, CacheError>;
54
55    /// Store narinfo text keyed by store path hash.
56    async fn put_narinfo(&self, hash: &str, content: &str) -> Result<(), CacheError>;
57
58    /// Retrieve a NAR blob by its relative path.
59    async fn get_nar(&self, path: &str) -> Result<Option<Vec<u8>>, CacheError>;
60
61    /// Store a NAR blob at the given relative path.
62    async fn put_nar(&self, path: &str, data: &[u8]) -> Result<(), CacheError>;
63
64    /// Delete a store path's narinfo and associated NAR blob.
65    async fn delete(&self, hash: &str) -> Result<(), CacheError>;
66
67    /// List all stored narinfo hashes.
68    async fn list_narinfos(&self) -> Result<Vec<String>, CacheError>;
69
70    /// Clear EVERY narinfo and NAR blob from this backend (all tiers, for a
71    /// [`TieredBackend`]). Returns the number of narinfos removed.
72    ///
73    /// This is the inverse of a warm push: the cache is regenerable by
74    /// construction — a wipe merely forces the next build to run COLD — so it is
75    /// a safe GC-class operation, never a correctness one. Its purpose is a clean
76    /// cold baseline for repeatable cold/warm benchmarking (the operator's
77    /// cache-wipe lever, the enjulho inverse of pre-warm).
78    ///
79    /// The default lists every narinfo and best-effort `delete`s it (mirroring
80    /// [`delete`](StorageBackend::delete)). Because NAR blobs are keyed by
81    /// *narhash* (from the narinfo `URL:`), not the store-path hash, per-hash
82    /// delete cannot reach the NAR bytes — so a **complete** wipe requires a
83    /// whole-store clear. The concrete durable tiers override this with a real
84    /// truncation (Postgres `DELETE FROM`, Redis prefix-scan, local-dir removal)
85    /// that reclaims the NAR bytes as well; the default remains an honest
86    /// narinfo-only clear (which still forces a cold miss, since Nix asks for the
87    /// narinfo first) for backends without one.
88    async fn wipe_all(&self) -> Result<usize, CacheError> {
89        let hashes = self.list_narinfos().await?;
90        let n = hashes.len();
91        for hash in hashes {
92            self.delete(&hash).await?;
93        }
94        Ok(n)
95    }
96}
97
98/// Config-select factory: build the concrete [`StorageBackend`] a
99/// [`BackendConfig`] names.
100///
101/// This is **typed dispatch, not stringly** — a new backend kind is a
102/// non-exhaustive-`match` compile error, and the [`Tiered`](BackendConfig::Tiered)
103/// arm recurses, composing each sub-backend into a [`TieredBackend`]. The result
104/// is an `Arc<dyn StorageBackend>` that drops straight into
105/// `AppState.storage` — the server never changes shape whether the backend is one
106/// tier or three.
107///
108/// The `Redis` and `Pg` arms require their production transports; without the
109/// corresponding Cargo feature (`redis-client` / `postgres`) they return a typed
110/// [`CacheError::NotImplemented`] rather than silently falling back to disk — the
111/// never-silent-hardcode rule.
112///
113/// Returns a boxed future because the `Tiered` arm is recursive.
114///
115/// # Errors
116///
117/// Propagates any backend construction failure, or [`CacheError::NotImplemented`]
118/// when a config selects a backend whose feature is not compiled in.
119pub fn build_backend(config: &BackendConfig) -> BoxFuture<'_, Result<Arc<dyn StorageBackend>, CacheError>> {
120    Box::pin(async move {
121        match config {
122            BackendConfig::Local { path } => {
123                Ok(Arc::new(LocalStorage::new(path.clone())) as Arc<dyn StorageBackend>)
124            }
125            BackendConfig::S3 { bucket, region, endpoint } => {
126                let s3 = S3Storage::new(bucket.clone(), region.clone(), endpoint.clone())?;
127                Ok(Arc::new(s3) as Arc<dyn StorageBackend>)
128            }
129            BackendConfig::Redis { url, ttl_secs } => build_redis(url, *ttl_secs).await,
130            BackendConfig::Pg { url, max_conns } => build_pg(url, *max_conns).await,
131            BackendConfig::Tiered { l1, l2, l3, write_policy } => {
132                let l1 = build_backend(l1).await?;
133                let l2 = build_backend(l2).await?;
134                let l3 = build_backend(l3).await?;
135                Ok(Arc::new(TieredBackend::with_write_policy(l1, l2, l3, *write_policy))
136                    as Arc<dyn StorageBackend>)
137            }
138        }
139    })
140}
141
142#[cfg(feature = "redis-client")]
143async fn build_redis(url: &str, ttl_secs: Option<u64>) -> Result<Arc<dyn StorageBackend>, CacheError> {
144    let backend = match ttl_secs {
145        Some(t) => RedisBackend::connect_with_ttl(url, t).await?,
146        None => RedisBackend::connect(url).await?,
147    };
148    Ok(Arc::new(backend) as Arc<dyn StorageBackend>)
149}
150
151#[cfg(not(feature = "redis-client"))]
152async fn build_redis(_url: &str, _ttl_secs: Option<u64>) -> Result<Arc<dyn StorageBackend>, CacheError> {
153    Err(CacheError::NotImplemented(
154        "redis L1 backend requires building sui-cache with --features redis-client",
155    ))
156}
157
158#[cfg(feature = "postgres")]
159async fn build_pg(url: &str, max_conns: u32) -> Result<Arc<dyn StorageBackend>, CacheError> {
160    let backend = PgStorageBackend::connect(url, max_conns).await?;
161    Ok(Arc::new(backend) as Arc<dyn StorageBackend>)
162}
163
164#[cfg(not(feature = "postgres"))]
165async fn build_pg(_url: &str, _max_conns: u32) -> Result<Arc<dyn StorageBackend>, CacheError> {
166    Err(CacheError::NotImplemented(
167        "postgres L2 backend requires building sui-cache with --features postgres",
168    ))
169}
sui_cache/storage/mod.rs

sui_cache/storage/
mod.rs
