use std::collections::BTreeMap;
use std::io::{Cursor, Read, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use async_trait::async_trait;
use ::zip::write::SimpleFileOptions;
use ::zip::{CompressionMethod, ZipArchive, ZipWriter};
use crate::backend::{Backend, TransactFn, WriteOp};
use crate::error::StoreError;
fn backend(err: impl std::fmt::Display) -> StoreError {
StoreError::Backend(err.to_string())
}
static TMP_COUNTER: AtomicU64 = AtomicU64::new(0);
struct Inner {
path: PathBuf,
entries: BTreeMap<String, Vec<u8>>,
}
#[derive(Clone)]
pub struct ZipBackend {
inner: Arc<Mutex<Inner>>,
}
impl ZipBackend {
pub fn open(path: impl AsRef<Path>) -> Result<Self, StoreError> {
let path = path.as_ref().to_path_buf();
let entries = if path.exists() {
read_entries(&path)?
} else {
BTreeMap::new()
};
Ok(Self {
inner: Arc::new(Mutex::new(Inner { path, entries })),
})
}
}
fn read_entries(path: &Path) -> Result<BTreeMap<String, Vec<u8>>, StoreError> {
let file = std::fs::File::open(path).map_err(backend)?;
let mut archive = ZipArchive::new(file).map_err(backend)?;
let mut entries = BTreeMap::new();
for index in 0..archive.len() {
let mut entry = archive.by_index(index).map_err(backend)?;
if entry.is_dir() && entry.size() == 0 {
continue;
}
let name = entry.name().to_string();
let mut bytes = Vec::with_capacity(entry.size() as usize);
entry.read_to_end(&mut bytes).map_err(backend)?;
entries.insert(name, bytes);
}
Ok(entries)
}
fn write_entries(path: &Path, entries: &BTreeMap<String, Vec<u8>>) -> Result<(), StoreError> {
let mut writer = ZipWriter::new(Cursor::new(Vec::new()));
for (name, bytes) in entries {
let method = if worth_deflating(bytes) {
CompressionMethod::Deflated
} else {
CompressionMethod::Stored
};
let options = SimpleFileOptions::default().compression_method(method);
writer.start_file(name.as_str(), options).map_err(backend)?;
writer.write_all(bytes).map_err(backend)?;
}
let buffer = writer.finish().map_err(backend)?.into_inner();
let parent = path.parent().unwrap_or_else(|| Path::new("."));
let counter = TMP_COUNTER.fetch_add(1, Ordering::Relaxed);
let tmp = parent.join(format!(
".muniment-zip-{}-{}.tmp",
std::process::id(),
counter
));
if let Err(error) = write_synced(&tmp, &buffer) {
let _ = std::fs::remove_file(&tmp);
return Err(error);
}
if let Err(error) = std::fs::rename(&tmp, path) {
let _ = std::fs::remove_file(&tmp);
return Err(backend(error));
}
sync_parent_dir(path);
Ok(())
}
fn worth_deflating(data: &[u8]) -> bool {
if data.len() < 256 {
return true;
}
let sample = &data[..data.len().min(16384)];
let mut hist = [0u32; 256];
for &b in sample {
hist[b as usize] += 1;
}
let n = sample.len() as f64;
let entropy: f64 = hist
.iter()
.filter(|&&c| c > 0)
.map(|&c| {
let p = c as f64 / n;
-p * p.log2()
})
.sum();
entropy < 7.5
}
fn write_synced(tmp: &Path, buffer: &[u8]) -> Result<(), StoreError> {
let mut file = std::fs::File::create(tmp).map_err(backend)?;
file.write_all(buffer).map_err(backend)?;
file.sync_all().map_err(backend)?;
Ok(())
}
#[cfg(unix)]
fn sync_parent_dir(path: &Path) {
if let Some(parent) = path.parent() {
if let Ok(dir) = std::fs::File::open(parent) {
let _ = dir.sync_all();
}
}
}
#[cfg(not(unix))]
fn sync_parent_dir(_path: &Path) {}
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
impl Backend for ZipBackend {
async fn get(&self, key: &str) -> Result<Option<Vec<u8>>, StoreError> {
Ok(self.inner.lock().unwrap().entries.get(key).cloned())
}
async fn put(&self, key: &str, bytes: &[u8]) -> Result<(), StoreError> {
let mut inner = self.inner.lock().unwrap();
let previous = inner.entries.insert(key.to_string(), bytes.to_vec());
if let Err(error) = write_entries(&inner.path, &inner.entries) {
match previous {
Some(old) => inner.entries.insert(key.to_string(), old),
None => inner.entries.remove(key),
};
return Err(error);
}
Ok(())
}
async fn delete(&self, key: &str) -> Result<(), StoreError> {
let mut inner = self.inner.lock().unwrap();
let Some(previous) = inner.entries.remove(key) else {
return Ok(());
};
if let Err(error) = write_entries(&inner.path, &inner.entries) {
inner.entries.insert(key.to_string(), previous);
return Err(error);
}
Ok(())
}
async fn list(&self, prefix: &str) -> Result<Vec<String>, StoreError> {
let inner = self.inner.lock().unwrap();
Ok(inner
.entries
.range(prefix.to_string()..)
.take_while(|(key, _)| key.starts_with(prefix))
.map(|(key, _)| key.clone())
.collect())
}
async fn scan(&self, start: &str, end: &str) -> Result<Vec<String>, StoreError> {
if start >= end {
return Ok(Vec::new());
}
let inner = self.inner.lock().unwrap();
Ok(inner
.entries
.range(start.to_string()..end.to_string())
.map(|(key, _)| key.clone())
.collect())
}
async fn apply(&self, ops: &[WriteOp]) -> Result<(), StoreError> {
let mut inner = self.inner.lock().unwrap();
let mut next = inner.entries.clone();
for op in ops {
match op {
WriteOp::Put { key, value } => {
next.insert(key.clone(), value.clone());
},
WriteOp::Delete { key } => {
next.remove(key);
},
}
}
write_entries(&inner.path, &next)?;
inner.entries = next;
Ok(())
}
async fn transact(&self, f: TransactFn) -> Result<(), StoreError> {
let _ = f;
Err(StoreError::NotTransactional)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn temp_backend() -> (tempfile::TempDir, ZipBackend) {
let dir = tempfile::tempdir().unwrap();
let backend = ZipBackend::open(dir.path().join("store.zip")).unwrap();
(dir, backend)
}
#[test]
fn put_get_delete_round_trip() {
pollster::block_on(async {
let (_dir, b) = temp_backend();
assert_eq!(b.get("k").await.unwrap(), None);
b.put("k", b"v").await.unwrap();
assert_eq!(b.get("k").await.unwrap(), Some(b"v".to_vec()));
b.put("k", b"v2").await.unwrap();
assert_eq!(b.get("k").await.unwrap(), Some(b"v2".to_vec()));
b.delete("k").await.unwrap();
assert_eq!(b.get("k").await.unwrap(), None);
b.delete("k").await.unwrap();
});
}
#[test]
fn list_returns_only_the_prefix() {
pollster::block_on(async {
let (_dir, b) = temp_backend();
for k in ["media/a", "media/b", "manifest.cbor"] {
b.put(k, b"v").await.unwrap();
}
let mut media = b.list("media/").await.unwrap();
media.sort();
assert_eq!(media, vec!["media/a".to_string(), "media/b".to_string()]);
assert_eq!(
b.list("manifest").await.unwrap(),
vec!["manifest.cbor".to_string()]
);
assert!(b.list("none/").await.unwrap().is_empty());
});
}
#[test]
fn scan_returns_the_range_in_ascending_order() {
pollster::block_on(async {
let (_dir, b) = temp_backend();
for (k, v) in [
("log/a/0/0000000000000002", "two"),
("log/a/0/0000000000000000", "zero"),
("log/a/0/0000000000000001", "one"),
("log/b/0/0000000000000000", "other-log"),
] {
b.put(k, v.as_bytes()).await.unwrap();
}
let keys = b
.scan("log/a/0/0000000000000000", "log/a/0/0000000000000003")
.await
.unwrap();
assert_eq!(
keys,
vec![
"log/a/0/0000000000000000".to_string(),
"log/a/0/0000000000000001".to_string(),
"log/a/0/0000000000000002".to_string(),
]
);
});
}
#[test]
fn apply_lands_the_whole_batch() {
pollster::block_on(async {
let (_dir, b) = temp_backend();
b.put("keep", b"v").await.unwrap();
b.put("drop", b"v").await.unwrap();
b.apply(&[
WriteOp::Put {
key: "manifest.cbor".into(),
value: b"header".to_vec(),
},
WriteOp::Put {
key: "media/x.wav".into(),
value: b"body".to_vec(),
},
WriteOp::Delete { key: "drop".into() },
])
.await
.unwrap();
assert_eq!(
b.get("manifest.cbor").await.unwrap(),
Some(b"header".to_vec())
);
assert_eq!(b.get("media/x.wav").await.unwrap(), Some(b"body".to_vec()));
assert_eq!(b.get("drop").await.unwrap(), None);
assert_eq!(b.get("keep").await.unwrap(), Some(b"v".to_vec()));
});
}
#[test]
fn scan_with_an_inverted_range_is_empty_not_a_panic() {
pollster::block_on(async {
let (_dir, b) = temp_backend();
for k in ["a", "m", "z"] {
b.put(k, b"v").await.unwrap();
}
assert!(b.scan("z", "a").await.unwrap().is_empty());
assert_eq!(b.get("m").await.unwrap(), Some(b"v".to_vec()));
});
}
#[test]
fn a_content_key_ending_in_slash_round_trips() {
pollster::block_on(async {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("store.zip");
{
let b = ZipBackend::open(&path).unwrap();
b.put("campaign/", b"data").await.unwrap();
b.put("media/a.wav", b"audio").await.unwrap();
}
let reopened = ZipBackend::open(&path).unwrap();
assert_eq!(
reopened.get("campaign/").await.unwrap(),
Some(b"data".to_vec())
);
assert_eq!(
reopened.get("media/a.wav").await.unwrap(),
Some(b"audio".to_vec())
);
});
}
#[test]
fn incompressible_entries_are_stored_compressible_ones_deflated() {
pollster::block_on(async {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("mixed.zip");
let mut noise = vec![0u8; 40_000];
let mut x: u32 = 0x1234_5678;
for b in noise.iter_mut() {
x = x.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
*b = (x >> 24) as u8;
}
let structured = vec![b'a'; 40_000];
{
let store = ZipBackend::open(&path).unwrap();
store.put("media/x.wv", &noise).await.unwrap();
store.put("manifest.cbor", &structured).await.unwrap();
}
let store = ZipBackend::open(&path).unwrap();
assert_eq!(store.get("media/x.wv").await.unwrap().unwrap(), noise);
assert_eq!(
store.get("manifest.cbor").await.unwrap().unwrap(),
structured
);
let file = std::fs::File::open(&path).unwrap();
let mut archive = ZipArchive::new(file).unwrap();
let noise_entry = archive.by_name("media/x.wv").unwrap();
assert_eq!(noise_entry.compression(), ::zip::CompressionMethod::Stored);
assert!(noise_entry.compressed_size() >= noise.len() as u64);
drop(noise_entry);
let struct_entry = archive.by_name("manifest.cbor").unwrap();
assert_eq!(
struct_entry.compression(),
::zip::CompressionMethod::Deflated
);
assert!(struct_entry.compressed_size() < structured.len() as u64 / 10);
});
}
#[test]
fn state_survives_reopen_as_a_real_zip() {
pollster::block_on(async {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("store.zip");
{
let b = ZipBackend::open(&path).unwrap();
b.put("manifest.cbor", b"durable").await.unwrap();
}
let file = std::fs::File::open(&path).unwrap();
let mut archive = ZipArchive::new(file).unwrap();
let mut entry = archive.by_name("manifest.cbor").unwrap();
let mut bytes = Vec::new();
entry.read_to_end(&mut bytes).unwrap();
assert_eq!(bytes, b"durable");
drop(entry);
drop(archive);
let reopened = ZipBackend::open(&path).unwrap();
assert_eq!(
reopened.get("manifest.cbor").await.unwrap(),
Some(b"durable".to_vec())
);
});
}
}