use std::collections::BTreeSet;
use std::sync::Arc;
use async_trait::async_trait;
use tracing::warn;
use super::StorageBackend;
use crate::CacheError;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum WritePolicy {
#[default]
WriteThrough,
WriteBack,
WriteAround,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TieredTier {
MockParityProven,
LiveClusterProven,
}
pub const TIERED_BACKEND_TIER: TieredTier = TieredTier::MockParityProven;
pub struct TieredBackend {
l1: Arc<dyn StorageBackend>,
l2: Arc<dyn StorageBackend>,
l3: Arc<dyn StorageBackend>,
write_policy: WritePolicy,
}
impl TieredBackend {
#[must_use]
pub fn new(
l1: Arc<dyn StorageBackend>,
l2: Arc<dyn StorageBackend>,
l3: Arc<dyn StorageBackend>,
) -> Self {
Self::with_write_policy(l1, l2, l3, WritePolicy::default())
}
#[must_use]
pub fn with_write_policy(
l1: Arc<dyn StorageBackend>,
l2: Arc<dyn StorageBackend>,
l3: Arc<dyn StorageBackend>,
write_policy: WritePolicy,
) -> Self {
Self { l1, l2, l3, write_policy }
}
#[must_use]
pub fn write_policy(&self) -> WritePolicy {
self.write_policy
}
async fn warm_narinfo(tier: &Arc<dyn StorageBackend>, hash: &str, content: &str) {
if let Err(e) = tier.put_narinfo(hash, content).await {
warn!(hash = %hash, error = %e, "tiered: best-effort narinfo warm failed");
}
}
async fn warm_nar(tier: &Arc<dyn StorageBackend>, path: &str, data: &[u8]) {
if let Err(e) = tier.put_nar(path, data).await {
warn!(path = %path, error = %e, "tiered: best-effort NAR warm failed");
}
}
}
#[async_trait]
impl StorageBackend for TieredBackend {
async fn get_narinfo(&self, hash: &str) -> Result<Option<String>, CacheError> {
if let Some(v) = self.l1.get_narinfo(hash).await? {
return Ok(Some(v));
}
if let Some(v) = self.l2.get_narinfo(hash).await? {
Self::warm_narinfo(&self.l1, hash, &v).await;
return Ok(Some(v));
}
if let Some(v) = self.l3.get_narinfo(hash).await? {
Self::warm_narinfo(&self.l2, hash, &v).await;
Self::warm_narinfo(&self.l1, hash, &v).await;
return Ok(Some(v));
}
Ok(None)
}
async fn get_nar(&self, path: &str) -> Result<Option<Vec<u8>>, CacheError> {
if let Some(v) = self.l1.get_nar(path).await? {
return Ok(Some(v));
}
if let Some(v) = self.l2.get_nar(path).await? {
Self::warm_nar(&self.l1, path, &v).await;
return Ok(Some(v));
}
if let Some(v) = self.l3.get_nar(path).await? {
Self::warm_nar(&self.l2, path, &v).await;
Self::warm_nar(&self.l1, path, &v).await;
return Ok(Some(v));
}
Ok(None)
}
async fn put_narinfo(&self, hash: &str, content: &str) -> Result<(), CacheError> {
match self.write_policy {
WritePolicy::WriteThrough => {
self.l2.put_narinfo(hash, content).await?;
self.l3.put_narinfo(hash, content).await?;
Self::warm_narinfo(&self.l1, hash, content).await;
}
WritePolicy::WriteBack => {
Self::warm_narinfo(&self.l1, hash, content).await;
self.l2.put_narinfo(hash, content).await?;
self.l3.put_narinfo(hash, content).await?;
}
WritePolicy::WriteAround => {
self.l2.put_narinfo(hash, content).await?;
self.l3.put_narinfo(hash, content).await?;
}
}
Ok(())
}
async fn put_nar(&self, path: &str, data: &[u8]) -> Result<(), CacheError> {
match self.write_policy {
WritePolicy::WriteThrough => {
self.l2.put_nar(path, data).await?;
self.l3.put_nar(path, data).await?;
Self::warm_nar(&self.l1, path, data).await;
}
WritePolicy::WriteBack => {
Self::warm_nar(&self.l1, path, data).await;
self.l2.put_nar(path, data).await?;
self.l3.put_nar(path, data).await?;
}
WritePolicy::WriteAround => {
self.l2.put_nar(path, data).await?;
self.l3.put_nar(path, data).await?;
}
}
Ok(())
}
async fn delete(&self, hash: &str) -> Result<(), CacheError> {
for (name, tier) in [("l1", &self.l1), ("l2", &self.l2), ("l3", &self.l3)] {
if let Err(e) = tier.delete(hash).await {
warn!(hash = %hash, tier = name, error = %e, "tiered: best-effort delete failed");
}
}
Ok(())
}
async fn list_narinfos(&self) -> Result<Vec<String>, CacheError> {
let mut set = BTreeSet::new();
set.extend(self.l2.list_narinfos().await?);
set.extend(self.l3.list_narinfos().await?);
Ok(set.into_iter().collect())
}
async fn wipe_all(&self) -> Result<usize, CacheError> {
let mut cleared = 0usize;
for (name, tier) in [("l1", &self.l1), ("l2", &self.l2), ("l3", &self.l3)] {
match tier.wipe_all().await {
Ok(n) => cleared = cleared.max(n),
Err(e) => warn!(tier = name, error = %e, "tiered: best-effort wipe failed"),
}
}
Ok(cleared)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::storage::LocalStorage;
use std::collections::HashMap;
use std::sync::Mutex;
#[derive(Default)]
struct MemBackend {
narinfo: Mutex<HashMap<String, String>>,
nar: Mutex<HashMap<String, Vec<u8>>>,
writes_fail: Mutex<bool>,
}
impl MemBackend {
fn has_narinfo(&self, hash: &str) -> bool {
self.narinfo.lock().unwrap().contains_key(hash)
}
fn has_nar(&self, path: &str) -> bool {
self.nar.lock().unwrap().contains_key(path)
}
fn narinfo_count(&self) -> usize {
self.narinfo.lock().unwrap().len()
}
fn clear(&self) {
self.narinfo.lock().unwrap().clear();
self.nar.lock().unwrap().clear();
}
fn set_writes_fail(&self, v: bool) {
*self.writes_fail.lock().unwrap() = v;
}
fn fail_if_configured(&self) -> Result<(), CacheError> {
if *self.writes_fail.lock().unwrap() {
Err(CacheError::NotImplemented("mock writes disabled"))
} else {
Ok(())
}
}
}
#[async_trait]
impl StorageBackend for MemBackend {
async fn get_narinfo(&self, hash: &str) -> Result<Option<String>, CacheError> {
Ok(self.narinfo.lock().unwrap().get(hash).cloned())
}
async fn put_narinfo(&self, hash: &str, content: &str) -> Result<(), CacheError> {
self.fail_if_configured()?;
self.narinfo.lock().unwrap().insert(hash.to_string(), content.to_string());
Ok(())
}
async fn get_nar(&self, path: &str) -> Result<Option<Vec<u8>>, CacheError> {
Ok(self.nar.lock().unwrap().get(path).cloned())
}
async fn put_nar(&self, path: &str, data: &[u8]) -> Result<(), CacheError> {
self.fail_if_configured()?;
self.nar.lock().unwrap().insert(path.to_string(), data.to_vec());
Ok(())
}
async fn delete(&self, hash: &str) -> Result<(), CacheError> {
self.narinfo.lock().unwrap().remove(hash);
for ext in ["nar.xz", "nar.zst", "nar"] {
self.nar.lock().unwrap().remove(&format!("nar/{hash}.{ext}"));
}
Ok(())
}
async fn list_narinfos(&self) -> Result<Vec<String>, CacheError> {
Ok(self.narinfo.lock().unwrap().keys().cloned().collect())
}
}
const NARINFO: &str = "StorePath: /nix/store/abc-hello\nURL: nar/abc.nar.xz\nCompression: xz\nNarHash: sha256:bbb\nNarSize: 200\nReferences: \n";
fn mocks() -> (Arc<MemBackend>, Arc<MemBackend>, Arc<MemBackend>, TieredBackend) {
let l1 = Arc::new(MemBackend::default());
let l2 = Arc::new(MemBackend::default());
let l3 = Arc::new(MemBackend::default());
let tiered = TieredBackend::new(l1.clone(), l2.clone(), l3.clone());
(l1, l2, l3, tiered)
}
#[tokio::test]
async fn l1_hit_returns_without_touching_lower_tiers() {
let (l1, l2, l3, tiered) = mocks();
l1.put_narinfo("h", "hot").await.unwrap();
assert_eq!(tiered.get_narinfo("h").await.unwrap().unwrap(), "hot");
assert!(!l2.has_narinfo("h"));
assert!(!l3.has_narinfo("h"));
}
#[tokio::test]
async fn l2_hit_promotes_into_l1() {
let (l1, l2, _l3, tiered) = mocks();
l2.put_narinfo("h", NARINFO).await.unwrap();
assert!(!l1.has_narinfo("h"));
let got = tiered.get_narinfo("h").await.unwrap().unwrap();
assert_eq!(got, NARINFO);
assert!(l1.has_narinfo("h"), "L2 hit must promote into L1");
}
#[tokio::test]
async fn l3_hit_promotes_into_l2_and_l1() {
let (l1, l2, l3, tiered) = mocks();
l3.put_narinfo("h", NARINFO).await.unwrap();
let got = tiered.get_narinfo("h").await.unwrap().unwrap();
assert_eq!(got, NARINFO);
assert!(l2.has_narinfo("h"), "L3 hit must promote into L2");
assert!(l1.has_narinfo("h"), "L3 hit must promote into L1");
}
#[tokio::test]
async fn nar_l3_hit_promotes_into_l2_and_l1() {
let (l1, l2, l3, tiered) = mocks();
l3.put_nar("nar/x.nar.xz", b"blob").await.unwrap();
let got = tiered.get_nar("nar/x.nar.xz").await.unwrap().unwrap();
assert_eq!(got, b"blob");
assert!(l2.has_nar("nar/x.nar.xz"));
assert!(l1.has_nar("nar/x.nar.xz"));
}
#[tokio::test]
async fn miss_at_all_tiers_is_none() {
let (_l1, _l2, _l3, tiered) = mocks();
assert!(tiered.get_narinfo("ghost").await.unwrap().is_none());
assert!(tiered.get_nar("nar/ghost.nar.xz").await.unwrap().is_none());
}
#[tokio::test]
async fn promotion_failure_does_not_break_a_read() {
let (l1, l2, _l3, tiered) = mocks();
l2.put_narinfo("h", NARINFO).await.unwrap();
l1.set_writes_fail(true); let got = tiered.get_narinfo("h").await.unwrap();
assert_eq!(got.unwrap(), NARINFO);
assert!(!l1.has_narinfo("h"), "warm failed, so L1 stays empty — but the read still succeeded");
}
#[tokio::test]
async fn write_through_populates_all_tiers() {
let (l1, l2, l3, tiered) = mocks();
tiered.put_narinfo("h", NARINFO).await.unwrap();
assert!(l1.has_narinfo("h"), "write-through warms L1");
assert!(l2.has_narinfo("h"), "write-through persists L2");
assert!(l3.has_narinfo("h"), "write-through persists L3");
}
#[tokio::test]
async fn write_around_skips_l1_but_persists_durable() {
let l1 = Arc::new(MemBackend::default());
let l2 = Arc::new(MemBackend::default());
let l3 = Arc::new(MemBackend::default());
let tiered = TieredBackend::with_write_policy(
l1.clone(), l2.clone(), l3.clone(), WritePolicy::WriteAround,
);
tiered.put_narinfo("h", NARINFO).await.unwrap();
assert!(!l1.has_narinfo("h"), "write-around must NOT touch L1");
assert!(l2.has_narinfo("h"));
assert!(l3.has_narinfo("h"));
let _ = tiered.get_narinfo("h").await.unwrap();
assert!(l1.has_narinfo("h"), "read-through fills L1 after a write-around");
}
#[tokio::test]
async fn write_back_populates_all_tiers_and_is_durable() {
let l1 = Arc::new(MemBackend::default());
let l2 = Arc::new(MemBackend::default());
let l3 = Arc::new(MemBackend::default());
let tiered = TieredBackend::with_write_policy(
l1.clone(), l2.clone(), l3.clone(), WritePolicy::WriteBack,
);
assert_eq!(tiered.write_policy(), WritePolicy::WriteBack);
tiered.put_nar("nar/x.nar.xz", b"blob").await.unwrap();
assert!(l1.has_nar("nar/x.nar.xz"));
assert!(l2.has_nar("nar/x.nar.xz"));
assert!(l3.has_nar("nar/x.nar.xz"));
}
#[tokio::test]
async fn durable_write_failure_propagates() {
let l1 = Arc::new(MemBackend::default());
let l2 = Arc::new(MemBackend::default());
let l3 = Arc::new(MemBackend::default());
l2.set_writes_fail(true);
let tiered = TieredBackend::new(l1.clone(), l2, l3);
let err = tiered.put_narinfo("h", NARINFO).await.unwrap_err();
assert!(matches!(err, CacheError::NotImplemented(_)));
}
#[tokio::test]
async fn pod_roll_losing_l1_loses_nothing() {
let (l1, _l2, _l3, tiered) = mocks();
tiered.put_narinfo("h", NARINFO).await.unwrap();
tiered.put_nar("nar/h.nar.xz", b"blob").await.unwrap();
l1.clear();
assert_eq!(tiered.get_narinfo("h").await.unwrap().unwrap(), NARINFO);
assert_eq!(tiered.get_nar("nar/h.nar.xz").await.unwrap().unwrap(), b"blob");
}
#[tokio::test]
async fn delete_fans_out_to_all_tiers() {
let (l1, l2, l3, tiered) = mocks();
tiered.put_narinfo("h", NARINFO).await.unwrap();
tiered.put_nar("nar/h.nar.xz", b"blob").await.unwrap();
tiered.delete("h").await.unwrap();
for t in [&l1, &l2, &l3] {
assert!(!t.has_narinfo("h"));
assert!(!t.has_nar("nar/h.nar.xz"));
}
}
#[tokio::test]
async fn wipe_all_clears_every_tier() {
let (l1, l2, l3, tiered) = mocks();
tiered.put_narinfo("h", NARINFO).await.unwrap();
tiered.put_nar("nar/h.nar.xz", b"blob").await.unwrap();
l2.put_narinfo("only2", "x").await.unwrap();
let removed = tiered.wipe_all().await.unwrap();
assert!(removed >= 1, "wipe reported nothing cleared");
for t in [&l1, &l2, &l3] {
assert!(t.list_narinfos().await.unwrap().is_empty(), "a tier survived the wipe");
assert!(!t.has_narinfo("h"));
assert!(!t.has_nar("nar/h.nar.xz"));
}
assert!(tiered.list_narinfos().await.unwrap().is_empty(), "cache not cold after wipe");
assert!(tiered.get_narinfo("h").await.unwrap().is_none());
}
#[tokio::test]
async fn list_narinfos_unions_durable_tiers_deduped() {
let (l1, l2, l3, tiered) = mocks();
l2.put_narinfo("shared", "x").await.unwrap();
l3.put_narinfo("shared", "x").await.unwrap();
l2.put_narinfo("only2", "y").await.unwrap();
l3.put_narinfo("only3", "z").await.unwrap();
l1.put_narinfo("hot-only", "w").await.unwrap();
let listed = tiered.list_narinfos().await.unwrap();
assert_eq!(listed, vec!["only2".to_string(), "only3".to_string(), "shared".to_string()]);
}
#[tokio::test]
async fn read_through_from_a_real_local_storage_l3() {
let dir = tempfile::tempdir().unwrap();
let l1 = Arc::new(MemBackend::default());
let l2 = Arc::new(MemBackend::default());
let l3_disk = Arc::new(LocalStorage::new(dir.path()));
l3_disk.put_narinfo("h", NARINFO).await.unwrap();
l3_disk.put_nar("nar/h.nar.xz", b"disk-blob").await.unwrap();
let tiered = TieredBackend::new(l1.clone(), l2.clone(), l3_disk);
assert_eq!(tiered.get_narinfo("h").await.unwrap().unwrap(), NARINFO);
assert_eq!(tiered.get_nar("nar/h.nar.xz").await.unwrap().unwrap(), b"disk-blob");
assert!(l1.has_narinfo("h"));
assert!(l2.has_narinfo("h"));
}
#[test]
fn honest_gate_tier_is_mock_parity_proven_not_live_cluster() {
assert_eq!(TIERED_BACKEND_TIER, TieredTier::MockParityProven);
}
}