use std::collections::HashMap;
use std::io::Write;
use std::path::PathBuf;
use std::sync::{Mutex, MutexGuard};
use crate::compress::{ChunkCompression, ContainerError, TAG_RAW, TAG_SEALED, pack, try_unpack};
use crate::hash::Hash;
use crate::paths::fsync_dir;
pub trait ContentStore: Send + Sync {
fn has(&self, hash: &Hash) -> std::io::Result<bool>;
fn get(&self, hash: &Hash) -> std::io::Result<Option<Vec<u8>>>;
fn put(&self, hash: &Hash, bytes: &[u8]) -> std::io::Result<()>;
fn remove(&self, hash: &Hash) -> std::io::Result<bool>;
fn for_each_hash(&self, f: &mut dyn FnMut(Hash) -> std::io::Result<()>) -> std::io::Result<()>;
fn has_many(&self, hashes: &[Hash]) -> std::io::Result<Vec<bool>> {
hashes.iter().map(|h| self.has(h)).collect()
}
fn get_many(&self, hashes: &[Hash]) -> std::io::Result<Vec<Option<Vec<u8>>>> {
hashes.iter().map(|h| self.get(h)).collect()
}
fn put_many(&self, chunks: &[(Hash, &[u8])]) -> std::io::Result<()> {
for (hash, bytes) in chunks {
self.put(hash, bytes)?;
}
Ok(())
}
fn hashes(&self) -> std::io::Result<Vec<Hash>> {
let mut out = Vec::new();
self.for_each_hash(&mut |h| {
out.push(h);
Ok(())
})?;
Ok(out)
}
}
impl<T: ContentStore + ?Sized> ContentStore for std::sync::Arc<T> {
fn has(&self, hash: &Hash) -> std::io::Result<bool> {
(**self).has(hash)
}
fn get(&self, hash: &Hash) -> std::io::Result<Option<Vec<u8>>> {
(**self).get(hash)
}
fn put(&self, hash: &Hash, bytes: &[u8]) -> std::io::Result<()> {
(**self).put(hash, bytes)
}
fn remove(&self, hash: &Hash) -> std::io::Result<bool> {
(**self).remove(hash)
}
fn for_each_hash(&self, f: &mut dyn FnMut(Hash) -> std::io::Result<()>) -> std::io::Result<()> {
(**self).for_each_hash(f)
}
fn has_many(&self, hashes: &[Hash]) -> std::io::Result<Vec<bool>> {
(**self).has_many(hashes)
}
fn get_many(&self, hashes: &[Hash]) -> std::io::Result<Vec<Option<Vec<u8>>>> {
(**self).get_many(hashes)
}
fn put_many(&self, chunks: &[(Hash, &[u8])]) -> std::io::Result<()> {
(**self).put_many(chunks)
}
fn hashes(&self) -> std::io::Result<Vec<Hash>> {
(**self).hashes()
}
}
#[derive(Default, Debug)]
pub struct MemoryStore(Mutex<HashMap<Hash, Vec<u8>>>);
impl MemoryStore {
pub fn new() -> Self {
MemoryStore(Mutex::new(HashMap::new()))
}
fn map(&self) -> MutexGuard<'_, HashMap<Hash, Vec<u8>>> {
self.0.lock().unwrap_or_else(|e| e.into_inner())
}
pub fn len(&self) -> usize {
self.map().len()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn clear(&self) {
self.map().clear();
}
}
impl ContentStore for MemoryStore {
fn has(&self, hash: &Hash) -> std::io::Result<bool> {
Ok(self.map().contains_key(hash))
}
fn get(&self, hash: &Hash) -> std::io::Result<Option<Vec<u8>>> {
Ok(self.map().get(hash).cloned())
}
fn put(&self, hash: &Hash, bytes: &[u8]) -> std::io::Result<()> {
self.map().insert(*hash, bytes.to_vec());
Ok(())
}
fn remove(&self, hash: &Hash) -> std::io::Result<bool> {
Ok(self.map().remove(hash).is_some())
}
fn for_each_hash(&self, f: &mut dyn FnMut(Hash) -> std::io::Result<()>) -> std::io::Result<()> {
let keys: Vec<Hash> = self.map().keys().copied().collect();
keys.into_iter().try_for_each(f)
}
fn put_many(&self, chunks: &[(Hash, &[u8])]) -> std::io::Result<()> {
let mut map = self.map();
for (hash, bytes) in chunks {
map.insert(*hash, bytes.to_vec());
}
Ok(())
}
}
#[derive(Debug)]
pub struct DirStore {
root: PathBuf,
verify_on_read: bool,
compression: ChunkCompression,
#[cfg(feature = "encryption")]
encryption: Option<crate::crypt::StoreKey>,
}
impl DirStore {
pub fn open(root: impl Into<PathBuf>) -> std::io::Result<Self> {
let root = root.into();
std::fs::create_dir_all(root.join(Hash::ALGO))?;
Ok(DirStore {
root,
verify_on_read: false,
compression: ChunkCompression::default(),
#[cfg(feature = "encryption")]
encryption: None,
})
}
#[cfg(feature = "encryption")]
#[must_use]
pub fn with_encryption(mut self, key: crate::crypt::StoreKey) -> Self {
self.encryption = Some(key);
self
}
#[must_use]
pub fn with_compression(mut self, compression: ChunkCompression) -> Self {
self.compression = compression;
self
}
#[must_use]
pub fn with_verify_on_read(mut self, verify: bool) -> Self {
self.verify_on_read = verify;
self
}
fn algo_dir(&self) -> PathBuf {
self.root.join(Hash::ALGO)
}
fn path(&self, hash: &Hash) -> PathBuf {
let hex = hash.to_string();
self.algo_dir().join(&hex[..2]).join(hex)
}
fn decode_at_rest(
&self,
packed: &[u8],
hash: &Hash,
strict_seal: bool,
) -> std::result::Result<Vec<u8>, ContainerError> {
let _ = (hash, strict_seal);
if packed.first() == Some(&TAG_SEALED) {
#[cfg(feature = "encryption")]
if let Some(key) = &self.encryption {
return match crate::crypt::open(key, hash, packed) {
Some(inner) => try_unpack(&inner),
None if strict_seal => Err(ContainerError::Corrupt),
None => Err(ContainerError::Unsupported),
};
}
return Err(ContainerError::Unsupported);
}
try_unpack(packed)
}
pub fn scrub(&self) -> std::io::Result<Vec<Hash>> {
let mut corrupted = Vec::new();
for hash in self.hashes()? {
let path = self.path(&hash);
let verdict = std::fs::read(&path)
.map_err(|_| ContainerError::Corrupt)
.and_then(|packed| self.decode_at_rest(&packed, &hash, true));
match verdict {
Ok(bytes) if Hash::of(&bytes) == hash => {}
Err(ContainerError::Unsupported) => {}
Ok(_) | Err(ContainerError::Corrupt) => {
let _ = std::fs::remove_file(&path);
corrupted.push(hash);
}
}
}
Ok(corrupted)
}
}
impl ContentStore for DirStore {
fn has(&self, hash: &Hash) -> std::io::Result<bool> {
use std::io::Read;
let path = self.path(hash);
let mut f = match std::fs::File::open(&path) {
Ok(f) => f,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false),
Err(e) => return Err(e),
};
let mut tag = [0u8; 1];
match f.read_exact(&mut tag) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(false),
Err(e) => return Err(e),
}
if tag[0] == TAG_RAW {
return Ok(true);
}
drop(f);
let packed = match std::fs::read(&path) {
Ok(p) => p,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false),
Err(e) => return Err(e),
};
Ok(self.decode_at_rest(&packed, hash, false).is_ok())
}
fn get(&self, hash: &Hash) -> std::io::Result<Option<Vec<u8>>> {
let path = self.path(hash);
let packed = match std::fs::read(&path) {
Ok(p) => p,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(e) => return Err(e),
};
let bytes = match self.decode_at_rest(&packed, hash, false) {
Ok(bytes) => bytes,
Err(ContainerError::Unsupported) => return Ok(None),
Err(ContainerError::Corrupt) => {
let _ = std::fs::remove_file(&path);
return Ok(None);
}
};
if self.verify_on_read && Hash::of(&bytes) != *hash {
let _ = std::fs::remove_file(&path);
return Ok(None);
}
Ok(Some(bytes))
}
fn put(&self, hash: &Hash, bytes: &[u8]) -> std::io::Result<()> {
#[allow(unused_mut)]
let mut packed = pack(bytes, self.compression).map_err(std::io::Error::other)?;
#[cfg(feature = "encryption")]
if let Some(key) = &self.encryption {
packed = crate::crypt::seal(key, hash, &packed)?;
}
let dst = self.path(hash);
let dir = dst.parent().expect("fanout dir");
std::fs::create_dir_all(dir)?;
let mut tmp = tempfile::NamedTempFile::new_in(dir)?;
tmp.write_all(&packed)?;
tmp.as_file().sync_all()?;
if let Err(e) = tmp.persist(&dst) {
if dst.exists() {
return fsync_dir(dir);
}
return Err(e.error);
}
fsync_dir(dir)
}
fn remove(&self, hash: &Hash) -> std::io::Result<bool> {
match std::fs::remove_file(self.path(hash)) {
Ok(()) => Ok(true),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
Err(e) => Err(e),
}
}
fn for_each_hash(&self, f: &mut dyn FnMut(Hash) -> std::io::Result<()>) -> std::io::Result<()> {
let algo_dir = self.algo_dir();
for fan in std::fs::read_dir(&algo_dir)? {
let fan = fan?;
if !fan.file_type()?.is_dir() {
continue;
}
for entry in std::fs::read_dir(fan.path())? {
let entry = entry?;
if let Some(name) = entry.file_name().to_str()
&& let Ok(hash) = name.parse::<Hash>()
{
f(hash)?;
}
}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn h(data: &[u8]) -> Hash {
Hash::of(data)
}
#[test]
fn memory_store_roundtrip() {
let s = MemoryStore::new();
let hash = h(b"hello");
assert!(!s.has(&hash).unwrap());
s.put(&hash, b"hello").unwrap();
assert!(s.has(&hash).unwrap());
assert_eq!(s.get(&hash).unwrap().unwrap(), b"hello");
assert_eq!(s.hashes().unwrap(), vec![hash]);
}
#[test]
fn dir_store_roundtrip_with_fanout() {
let dir = tempfile::tempdir().unwrap();
let s = DirStore::open(dir.path()).unwrap();
let hash = h(b"world");
assert!(!s.has(&hash).unwrap());
s.put(&hash, b"world").unwrap();
assert!(s.has(&hash).unwrap());
assert_eq!(s.get(&hash).unwrap().unwrap(), b"world");
let hex = hash.to_string();
assert!(
dir.path()
.join("blake3")
.join(&hex[..2])
.join(&hex)
.exists()
);
let s2 = DirStore::open(dir.path()).unwrap();
assert!(s2.has(&hash).unwrap());
assert_eq!(s2.hashes().unwrap(), vec![hash]);
}
#[test]
fn verify_on_read_rejects_corruption() {
let dir = tempfile::tempdir().unwrap();
let s = DirStore::open(dir.path())
.unwrap()
.with_verify_on_read(true);
let hash = h(b"payload");
s.put(&hash, b"payload").unwrap();
let hex = hash.to_string();
let path = dir.path().join("blake3").join(&hex[..2]).join(&hex);
std::fs::write(&path, b"garbage").unwrap();
assert!(s.get(&hash).unwrap().is_none());
assert!(!path.exists());
}
#[cfg(feature = "encryption")]
#[test]
fn encrypted_store_roundtrip_and_key_required() {
use crate::crypt::StoreKey;
let dir = tempfile::tempdir().unwrap();
let s = DirStore::open(dir.path())
.unwrap()
.with_encryption(StoreKey::new([7u8; 32]));
let plaintext = b"very secret chunk contents".to_vec();
let hash = h(&plaintext);
s.put(&hash, &plaintext).unwrap();
assert_eq!(s.get(&hash).unwrap().unwrap(), plaintext);
let hex = hash.to_string();
let path = dir.path().join("blake3").join(&hex[..2]).join(&hex);
let on_disk = std::fs::read(&path).unwrap();
assert_eq!(on_disk[0], 0x02);
assert!(!on_disk.windows(plaintext.len()).any(|w| w == plaintext));
let no_key = DirStore::open(dir.path()).unwrap();
assert!(no_key.get(&hash).unwrap().is_none());
assert!(path.exists(), "sealed chunk must not be deleted");
assert!(no_key.scrub().unwrap().is_empty(), "scrub must skip sealed");
let wrong = DirStore::open(dir.path())
.unwrap()
.with_encryption(StoreKey::new([8u8; 32]));
assert!(wrong.get(&hash).unwrap().is_none());
let mut bytes = std::fs::read(&path).unwrap();
let n = bytes.len();
bytes[n - 1] ^= 0xff;
std::fs::write(&path, &bytes).unwrap();
assert_eq!(s.scrub().unwrap(), vec![hash]);
assert!(!path.exists());
}
#[test]
fn scrub_reports_and_removes_corruption() {
let dir = tempfile::tempdir().unwrap();
let s = DirStore::open(dir.path()).unwrap();
let good = h(b"good");
let bad = h(b"will corrupt");
s.put(&good, b"good").unwrap();
s.put(&bad, b"will corrupt").unwrap();
let hex = bad.to_string();
std::fs::write(
dir.path().join("blake3").join(&hex[..2]).join(&hex),
b"flipped",
)
.unwrap();
let corrupted = s.scrub().unwrap();
assert_eq!(corrupted, vec![bad]);
assert!(s.has(&good).unwrap() && !s.has(&bad).unwrap());
}
}