use std::collections::HashMap;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use serde::{Deserialize, Serialize};
use tokio::sync::{Notify, Semaphore};
use zenoh::qos::Priority;
use zenoh::query::ConsolidationMode;
use crate::cancel::CancelToken;
use crate::chunk::CdcParams;
use crate::client::DownloadRequest;
use crate::compress::{ChunkCompression, pack};
use crate::error::{BlobError, Result};
use crate::hash::{Hash, HashAlgo};
use crate::id::BlobId;
use crate::keys::{store_key, tree_key};
use crate::obs::{TransferStats, zdebug};
use crate::paths::{
assert_parent_within, create_dir_confined, sanitize_rel_path, sanitize_symlink_target,
};
use crate::prefix::{QueryPrefix, ServePrefix};
use crate::progress::{Progress, ProgressSink};
use crate::server::{ErrorCallback, FifoQueryable, ServerHandle, report_error};
use crate::store::ContentStore;
use crate::wire::{ENC_CHUNK, ENC_INDEX, WIRE_VERSION, decode, encode};
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ChunkRef {
pub hash: Hash,
pub len: u32,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum Entry {
Dir {
path: String,
mode: u32,
mtime: i64,
},
File {
path: String,
mode: u32,
mtime: i64,
size: u64,
chunks: Vec<ChunkRef>,
},
Symlink {
path: String,
target: String,
},
Hardlink {
path: String,
target: String,
},
}
impl Entry {
pub fn path(&self) -> &str {
match self {
Entry::Dir { path, .. }
| Entry::File { path, .. }
| Entry::Symlink { path, .. }
| Entry::Hardlink { path, .. } => path,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TreeIndex {
pub version: u16,
pub id: BlobId,
pub algo: HashAlgo,
pub cdc: CdcParams,
pub entries: Vec<Entry>,
pub root_hash: Hash,
}
#[derive(Serialize)]
enum IdentityEntry<'a> {
Dir {
path: &'a str,
mode: u32,
},
File {
path: &'a str,
mode: u32,
size: u64,
chunks: &'a Vec<ChunkRef>,
},
Symlink {
path: &'a str,
target: &'a str,
},
Hardlink {
path: &'a str,
target: &'a str,
},
}
fn root_digest(entries: &[Entry]) -> Result<Hash> {
let view: Vec<IdentityEntry> = entries
.iter()
.map(|e| match e {
Entry::Dir { path, mode, .. } => IdentityEntry::Dir { path, mode: *mode },
Entry::File {
path,
mode,
size,
chunks,
..
} => IdentityEntry::File {
path,
mode: *mode,
size: *size,
chunks,
},
Entry::Symlink { path, target } => IdentityEntry::Symlink { path, target },
Entry::Hardlink { path, target } => IdentityEntry::Hardlink { path, target },
})
.collect();
let bytes = encode(&(WIRE_VERSION, Hash::ALGO, view))?;
Ok(Hash::of(&bytes))
}
impl TreeIndex {
pub fn needed_chunks(&self) -> Vec<Hash> {
let mut seen = std::collections::HashSet::new();
let mut out = Vec::new();
for e in &self.entries {
if let Entry::File { chunks, .. } = e {
for c in chunks {
if seen.insert(c.hash) {
out.push(c.hash);
}
}
}
}
out
}
pub fn needed_chunk_refs(&self) -> Vec<ChunkRef> {
let mut index: std::collections::HashMap<Hash, u32> = std::collections::HashMap::new();
let mut order = Vec::new();
for e in &self.entries {
if let Entry::File { chunks, .. } = e {
for c in chunks {
match index.entry(c.hash) {
std::collections::hash_map::Entry::Occupied(mut slot) => {
let len = slot.get_mut();
*len = (*len).max(c.len);
}
std::collections::hash_map::Entry::Vacant(slot) => {
slot.insert(c.len);
order.push(c.hash);
}
}
}
}
}
order
.into_iter()
.map(|hash| ChunkRef {
hash,
len: index[&hash],
})
.collect()
}
#[must_use]
pub fn entry(&self, path: &str) -> Option<&Entry> {
self.entries.iter().find(|e| e.path() == path)
}
#[must_use]
pub fn entries(&self) -> &[Entry] {
&self.entries
}
pub fn files(&self) -> impl Iterator<Item = (&str, u64, &[ChunkRef])> {
self.entries.iter().filter_map(|e| match e {
Entry::File {
path, size, chunks, ..
} => Some((path.as_str(), *size, chunks.as_slice())),
_ => None,
})
}
#[must_use]
pub fn file_chunks(&self, path: &str) -> Option<&[ChunkRef]> {
match self.entry(path)? {
Entry::File { chunks, .. } => Some(chunks),
_ => None,
}
}
pub fn total_size(&self) -> u64 {
self.entries
.iter()
.map(|e| match e {
Entry::File { size, .. } => *size,
_ => 0,
})
.sum()
}
pub fn file_count(&self) -> usize {
self.entries
.iter()
.filter(|e| matches!(e, Entry::File { .. }))
.count()
}
pub fn compute_root(&self) -> Result<Hash> {
root_digest(&self.entries)
}
#[must_use]
pub fn keyed_by_root(mut self) -> Self {
self.id = BlobId::new(self.root_hash.to_string())
.expect("a hex root hash is a valid single key segment");
self
}
pub fn is_content_addressed(&self) -> bool {
self.id.as_str() == self.root_hash.to_string()
}
pub fn validate(&self) -> Result<()> {
if self.version != WIRE_VERSION {
return Err(BlobError::UnsupportedVersion(self.version));
}
self.cdc.validate()?;
let mut seen: std::collections::HashSet<PathBuf> = std::collections::HashSet::new();
let mut links: std::collections::BTreeMap<Vec<String>, String> = Default::default();
for e in &self.entries {
let rel = sanitize_rel_path(e.path())?;
if !seen.insert(rel.clone()) {
return Err(BlobError::InvalidManifest(format!(
"duplicate entry path {:?}",
e.path()
)));
}
match e {
Entry::File { size, chunks, .. } => {
let sum: u64 = chunks.iter().map(|c| c.len as u64).sum();
if sum != *size {
return Err(BlobError::InvalidManifest(format!(
"file {:?}: size {size} != chunk sum {sum}",
e.path()
)));
}
if let Some(bad) = chunks.iter().find(|c| c.len == 0 || c.len > self.cdc.max) {
return Err(BlobError::InvalidManifest(format!(
"file {:?}: chunk len {} outside 1..={}",
e.path(),
bad.len,
self.cdc.max
)));
}
}
Entry::Symlink { target, .. } => {
sanitize_symlink_target(&rel, target)?;
links.insert(
rel.components()
.map(|c| c.as_os_str().to_string_lossy().into_owned())
.collect(),
target.clone(),
);
}
Entry::Hardlink { target, .. } => {
sanitize_rel_path(target)?;
}
Entry::Dir { .. } => {}
}
}
crate::paths::assert_symlinks_confined(&links)?;
let actual = self.compute_root()?;
if actual != self.root_hash {
return Err(BlobError::RootMismatch {
expected: self.root_hash,
actual,
});
}
Ok(())
}
}
pub fn build_tree(
root: &Path,
id: impl Into<String>,
cdc: &CdcParams,
store: &dyn ContentStore,
) -> Result<TreeIndex> {
build_tree_from(root, id, cdc, store, None)
}
pub fn build_tree_from(
root: &Path,
id: impl Into<String>,
cdc: &CdcParams,
store: &dyn ContentStore,
parent: Option<&TreeIndex>,
) -> Result<TreeIndex> {
cdc.validate()?;
let id = BlobId::new(id.into())?;
let mut reusable: HashMap<&str, &Entry> = HashMap::new();
if let Some(parent) = parent {
if parent.cdc != *cdc {
return Err(BlobError::Usage(format!(
"parent snapshot was cut with different CDC parameters ({:?} vs {:?}); \
its chunk references would not tile under these",
parent.cdc, cdc
)));
}
for e in &parent.entries {
if matches!(e, Entry::File { .. }) {
reusable.insert(e.path(), e);
}
}
}
let mut entries = Vec::new();
walk(root, cdc, store, &reusable, &mut entries)?;
let root_hash = root_digest(&entries)?;
let index = TreeIndex {
version: WIRE_VERSION,
id,
algo: HashAlgo::Blake3,
cdc: *cdc,
entries,
root_hash,
};
index.validate()?;
Ok(index)
}
fn rel_path_of(root: &Path, path: &Path) -> Result<String> {
let rel = path.strip_prefix(root).unwrap_or(path);
let mut out = String::new();
for comp in rel.components() {
let s = comp
.as_os_str()
.to_str()
.ok_or_else(|| BlobError::Usage(format!("non-UTF-8 file name in tree: {rel:?}")))?;
if !out.is_empty() {
out.push('/');
}
out.push_str(s);
}
Ok(out)
}
#[cfg(unix)]
fn mode_of(meta: &std::fs::Metadata) -> u32 {
use std::os::unix::fs::PermissionsExt;
meta.permissions().mode()
}
#[cfg(not(unix))]
fn mode_of(_meta: &std::fs::Metadata) -> u32 {
0
}
fn mtime_of(meta: &std::fs::Metadata) -> i64 {
meta.modified()
.ok()
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.map(|d| d.as_secs() as i64)
.unwrap_or(0)
}
fn walk(
root: &Path,
cdc: &CdcParams,
store: &dyn ContentStore,
reusable: &HashMap<&str, &Entry>,
entries: &mut Vec<Entry>,
) -> Result<()> {
fn read_dir_sorted(dir: &Path) -> std::io::Result<std::vec::IntoIter<std::fs::DirEntry>> {
let mut items: Vec<_> = std::fs::read_dir(dir)?.collect::<std::io::Result<Vec<_>>>()?;
items.sort_by_key(|e| e.file_name());
Ok(items.into_iter())
}
let mut inodes: HashMap<(u64, u64), String> = HashMap::new();
let mut stack = vec![read_dir_sorted(root)?];
while let Some(top) = stack.last_mut() {
let Some(item) = top.next() else {
stack.pop();
continue;
};
let path = item.path();
let meta = std::fs::symlink_metadata(&path)?;
let rel = rel_path_of(root, &path)?;
let ftype = meta.file_type();
if ftype.is_symlink() {
let target_os = std::fs::read_link(&path)?;
let target = target_os
.to_str()
.ok_or_else(|| BlobError::Usage(format!("non-UTF-8 symlink target at {rel:?}")))?
.to_string();
entries.push(Entry::Symlink { path: rel, target });
} else if ftype.is_dir() {
entries.push(Entry::Dir {
path: rel,
mode: mode_of(&meta),
mtime: mtime_of(&meta),
});
stack.push(read_dir_sorted(&path)?);
} else if ftype.is_file() {
if let Some(first) = hardlink_target(&meta, &rel, &mut inodes) {
entries.push(Entry::Hardlink {
path: rel,
target: first,
});
continue;
}
let mtime = mtime_of(&meta);
if let Some(Entry::File {
size: prev_size,
mtime: prev_mtime,
chunks: prev_chunks,
..
}) = reusable.get(rel.as_str()).copied()
&& *prev_size == meta.len()
&& *prev_mtime == mtime
&& prev_chunks
.iter()
.try_fold(true, |acc, c| store.has(&c.hash).map(|p| acc && p))?
{
entries.push(Entry::File {
path: rel,
mode: mode_of(&meta),
mtime,
size: *prev_size,
chunks: prev_chunks.clone(),
});
continue;
}
let file = std::fs::File::open(&path)?;
let mut refs = Vec::new();
let mut size: u64 = 0;
for chunk in cdc.chunk_reader(file) {
let bytes = chunk?;
let hash = Hash::of(&bytes);
size += bytes.len() as u64;
refs.push(ChunkRef {
hash,
len: bytes.len() as u32,
});
if !store.has(&hash)? {
store.put(&hash, &bytes)?;
}
}
entries.push(Entry::File {
path: rel,
mode: mode_of(&meta),
mtime,
size,
chunks: refs,
});
} else {
return Err(BlobError::Usage(format!(
"unsupported file type at {rel:?} (fifo/socket/device)"
)));
}
}
Ok(())
}
#[cfg(unix)]
fn hardlink_target(
meta: &std::fs::Metadata,
rel: &str,
inodes: &mut HashMap<(u64, u64), String>,
) -> Option<String> {
use std::os::unix::fs::MetadataExt;
if meta.nlink() > 1 {
let key = (meta.dev(), meta.ino());
match inodes.get(&key) {
Some(first) => return Some(first.clone()),
None => {
inodes.insert(key, rel.to_string());
}
}
}
None
}
#[cfg(not(unix))]
fn hardlink_target(
_meta: &std::fs::Metadata,
_rel: &str,
_inodes: &mut HashMap<(u64, u64), String>,
) -> Option<String> {
None
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct MaterializePolicy {
replace_directories: bool,
restore_setid: bool,
}
impl MaterializePolicy {
#[must_use]
pub fn replace_directories(mut self, allow: bool) -> Self {
self.replace_directories = allow;
self
}
#[must_use]
pub fn restore_setid(mut self, allow: bool) -> Self {
self.restore_setid = allow;
self
}
}
#[derive(Clone)]
pub struct TreeServer {
inner: Arc<TreeInner>,
}
impl std::fmt::Debug for TreeServer {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TreeServer")
.field("store_prefix", &self.inner.store_prefix)
.field("tree_prefix", &self.inner.tree_prefix)
.field("compression", &self.inner.compression)
.field("max_want_list", &self.inner.max_want_list)
.finish_non_exhaustive()
}
}
struct TreeInner {
session: zenoh::Session,
store_prefix: ServePrefix,
tree_prefix: ServePrefix,
store: Arc<dyn ContentStore>,
index: tokio::sync::RwLock<std::collections::HashMap<BlobId, Registered>>,
index_shard_threshold: usize,
inflight: Arc<Semaphore>,
compression: ChunkCompression,
max_want_list: usize,
on_error: Option<ErrorCallback>,
}
pub struct TreeServerBuilder {
session: zenoh::Session,
store_prefix: ServePrefix,
tree_prefix: ServePrefix,
store: Arc<dyn ContentStore>,
max_inflight: usize,
compression: ChunkCompression,
max_want_list: usize,
index_shard_threshold: usize,
on_error: Option<ErrorCallback>,
}
impl std::fmt::Debug for TreeServerBuilder {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TreeServerBuilder")
.field("store_prefix", &self.store_prefix)
.field("tree_prefix", &self.tree_prefix)
.field("max_inflight", &self.max_inflight)
.field("compression", &self.compression)
.field("max_want_list", &self.max_want_list)
.field("index_shard_threshold", &self.index_shard_threshold)
.finish_non_exhaustive()
}
}
struct Registered {
index: TreeIndex,
sharded: Option<crate::wire::IndexDescriptor>,
}
const INDEX_SHARD: usize = 64 * 1024;
fn shard_index(
encoded: &[u8],
store: &dyn ContentStore,
root: Hash,
) -> Result<crate::wire::IndexDescriptor> {
let mut chunks = Vec::new();
for piece in encoded.chunks(INDEX_SHARD) {
let hash = Hash::of(piece);
if !store.has(&hash)? {
store.put(&hash, piece)?;
}
chunks.push(ChunkRef {
hash,
len: piece.len() as u32,
});
}
Ok(crate::wire::IndexDescriptor {
version: WIRE_VERSION,
root,
algo: HashAlgo::Blake3,
index_chunks: chunks,
index_len: encoded.len() as u64,
ext: crate::wire::Ext::new(),
})
}
impl TreeServerBuilder {
#[must_use]
pub fn max_inflight(mut self, n: usize) -> Self {
self.max_inflight = n.max(1);
self
}
#[must_use]
pub fn compression(mut self, c: ChunkCompression) -> Self {
self.compression = c;
self
}
#[must_use]
pub fn index_shard_threshold(mut self, bytes: usize) -> Self {
self.index_shard_threshold = bytes;
self
}
#[must_use]
pub fn max_want_list(mut self, n: usize) -> Self {
self.max_want_list = n.max(1);
self
}
#[must_use]
pub fn on_error(mut self, cb: ErrorCallback) -> Self {
self.on_error = Some(cb);
self
}
pub fn build(self) -> TreeServer {
TreeServer {
inner: Arc::new(TreeInner {
session: self.session,
store_prefix: self.store_prefix,
tree_prefix: self.tree_prefix,
store: self.store,
index: tokio::sync::RwLock::new(std::collections::HashMap::new()),
index_shard_threshold: self.index_shard_threshold,
inflight: Arc::new(Semaphore::new(self.max_inflight)),
compression: self.compression,
max_want_list: self.max_want_list,
on_error: self.on_error,
}),
}
}
}
impl TreeServer {
pub fn builder(
session: &zenoh::Session,
store_prefix: ServePrefix,
tree_prefix: ServePrefix,
store: Arc<dyn ContentStore>,
) -> TreeServerBuilder {
TreeServerBuilder {
session: session.clone(),
store_prefix,
tree_prefix,
store,
max_inflight: 8,
compression: ChunkCompression::default(),
max_want_list: crate::wire::MAX_WANT_LIST,
index_shard_threshold: 256 * 1024,
on_error: None,
}
}
pub fn new(
session: &zenoh::Session,
store_prefix: ServePrefix,
tree_prefix: ServePrefix,
store: Arc<dyn ContentStore>,
) -> Self {
Self::builder(session, store_prefix, tree_prefix, store).build()
}
pub async fn register(&self, index: TreeIndex) -> Result<()> {
let encoded = encode(&index)?;
let sharded = if encoded.len() > self.inner.index_shard_threshold {
let store = self.inner.store.clone();
let root = index.root_hash;
Some(tokio::task::spawn_blocking(move || shard_index(&encoded, &*store, root)).await??)
} else {
None
};
self.inner
.index
.write()
.await
.insert(index.id.clone(), Registered { index, sharded });
Ok(())
}
pub async fn registered(&self) -> Vec<BlobId> {
self.inner.index.read().await.keys().cloned().collect()
}
pub async fn index(&self, id: &str) -> Option<TreeIndex> {
self.inner
.index
.read()
.await
.get(id)
.map(|r| r.index.clone())
}
pub async fn serves(&self, id: &str) -> bool {
self.inner.index.read().await.contains_key(id)
}
pub async fn unregister(&self, id: &str) {
self.inner.index.write().await.remove(id);
}
async fn declare(&self) -> Result<(FifoQueryable, FifoQueryable)> {
let store_q = self
.inner
.session
.declare_queryable(format!("{}/**", self.inner.store_prefix))
.await
.map_err(BlobError::zenoh)?;
let tree_q = self
.inner
.session
.declare_queryable(format!("{}/**", self.inner.tree_prefix))
.await
.map_err(BlobError::zenoh)?;
Ok((store_q, tree_q))
}
pub async fn spawn(self) -> Result<ServerHandle> {
let (store_q, tree_q) = self.declare().await?;
let stop = Arc::new(Notify::new());
let stop2 = stop.clone();
Ok(ServerHandle::new(
tokio::spawn(self.serve_loop(store_q, tree_q, stop2)),
stop,
))
}
pub async fn run(self) -> Result<()> {
let (store_q, tree_q) = self.declare().await?;
self.serve_loop(store_q, tree_q, Arc::new(Notify::new()))
.await
}
async fn serve_loop(
self,
store_q: FifoQueryable,
tree_q: FifoQueryable,
stop: Arc<Notify>,
) -> Result<()> {
loop {
tokio::select! {
_ = stop.notified() => break,
q = store_q.recv_async() => {
let Ok(query) = q else { break };
let inner = self.inner.clone();
let Ok(permit) = inner.inflight.clone().acquire_owned().await else { break };
tokio::spawn(async move {
let _permit = permit;
if let Err(e) = serve_chunk_query(&inner, query).await {
report_error(&inner.on_error, &e);
}
});
}
q = tree_q.recv_async() => {
let Ok(query) = q else { break };
let inner = self.inner.clone();
let Ok(permit) = inner.inflight.clone().acquire_owned().await else { break };
tokio::spawn(async move {
let _permit = permit;
if let Err(e) = serve_index_query(&inner, query).await {
report_error(&inner.on_error, &e);
}
});
}
}
}
Ok(())
}
}
async fn serve_chunk_query(inner: &TreeInner, query: zenoh::query::Query) -> Result<()> {
let key = query.key_expr().as_str().to_string();
let Some(crate::keys::Tier2Tail::Two(algo, last)) =
crate::keys::parse_tier2_tail(inner.store_prefix.as_str(), &key)
else {
return Ok(());
};
if algo != Hash::ALGO {
return Ok(()); }
match last {
crate::keys::STORE_BATCH => serve_chunk_batch(inner, query).await,
crate::keys::STORE_HAVE => serve_chunk_probe(inner, query).await,
hex => {
let Ok(hash) = hex.parse::<Hash>() else {
return Ok(()); };
let store = inner.store.clone();
let compression = inner.compression;
let packed = tokio::task::spawn_blocking(move || -> Result<Option<Vec<u8>>> {
match store.get(&hash)? {
Some(bytes) => Ok(Some(pack(&bytes, compression)?)),
None => Ok(None),
}
})
.await??;
if let Some(packed) = packed {
query
.reply(
store_key(inner.store_prefix.as_str(), HashAlgo::Blake3, &hash),
packed,
)
.encoding(&ENC_CHUNK)
.await
.map_err(BlobError::zenoh)?;
}
Ok(())
}
}
}
async fn want_list_of(
inner: &TreeInner,
query: &zenoh::query::Query,
) -> Result<Option<crate::wire::WantList>> {
let Some(payload) = query.payload() else {
let _ = query
.reply_err("want-list endpoint requires a payload")
.await;
return Ok(None);
};
let want: crate::wire::WantList = match decode(&payload.to_bytes()) {
Ok(w) => w,
Err(e) => {
let _ = query.reply_err(format!("undecodable want list: {e}")).await;
return Ok(None);
}
};
if let Err(e) = want.validate(inner.max_want_list) {
let _ = query.reply_err(e.to_string()).await;
return Ok(None);
}
Ok(Some(want))
}
async fn serve_chunk_batch(inner: &TreeInner, query: zenoh::query::Query) -> Result<()> {
if query.accepts_replies() != zenoh::query::ReplyKeyExpr::Any {
let _ = query
.reply_err(
"a batch fetch replies on each chunk's own key, which does not \
intersect the batch key: issue the query with \
accept_replies(ReplyKeyExpr::Any)",
)
.await;
return Ok(());
}
let Some(want) = want_list_of(inner, &query).await? else {
return Ok(());
};
let store = inner.store.clone();
let compression = inner.compression;
let prefix = inner.store_prefix.as_str().to_string();
let packed: Vec<(String, Vec<u8>)> =
tokio::task::spawn_blocking(move || -> Result<Vec<(String, Vec<u8>)>> {
want.hashes
.iter()
.filter_map(|h| match store.get(h) {
Ok(Some(bytes)) => match pack(&bytes, compression) {
Ok(framed) => Some(Ok((store_key(&prefix, HashAlgo::Blake3, h), framed))),
Err(e) => Some(Err(e)),
},
Ok(None) => None,
Err(e) => Some(Err(BlobError::from(e))),
})
.collect::<Result<Vec<_>>>()
})
.await??;
for (key, bytes) in packed {
if query
.reply(key, bytes)
.encoding(&ENC_CHUNK)
.await
.map_err(BlobError::zenoh)
.is_err()
{
break; }
}
Ok(())
}
async fn serve_chunk_probe(inner: &TreeInner, query: zenoh::query::Query) -> Result<()> {
let Some(want) = want_list_of(inner, &query).await? else {
return Ok(());
};
let store = inner.store.clone();
let bits = tokio::task::spawn_blocking(move || -> std::io::Result<crate::wire::HaveBits> {
Ok(crate::wire::HaveBits::from_presence(
store.has_many(&want.hashes)?,
))
})
.await??;
query
.reply(
crate::keys::store_have_key(inner.store_prefix.as_str(), HashAlgo::Blake3),
encode(&bits)?,
)
.encoding(&crate::wire::ENC_HAVEBITS)
.await
.map_err(BlobError::zenoh)?;
Ok(())
}
async fn serve_index_query(inner: &TreeInner, query: zenoh::query::Query) -> Result<()> {
let key = query.key_expr().as_str().to_string();
let Some(tail) = crate::keys::parse_tier2_tail(inner.tree_prefix.as_str(), &key) else {
return Ok(());
};
let id = match tail {
crate::keys::Tier2Tail::One(id) => id,
crate::keys::Tier2Tail::Two(id, crate::keys::STORE_HAVE) => {
return serve_tree_probe(inner, &query, id).await;
}
crate::keys::Tier2Tail::Two(..) => return Ok(()),
};
let (payload, encoding) = {
let registry = inner.index.read().await;
let Some(reg) = registry.get(id) else {
return Ok(());
};
match ®.sharded {
Some(desc) => (encode(desc)?, &crate::wire::ENC_INDEX_DESC),
None => (encode(®.index)?, &ENC_INDEX),
}
};
query
.reply(tree_key(inner.tree_prefix.as_str(), id), payload)
.encoding(encoding)
.await
.map_err(BlobError::zenoh)?;
Ok(())
}
async fn serve_tree_probe(inner: &TreeInner, query: &zenoh::query::Query, id: &str) -> Result<()> {
let Some(index) = inner
.index
.read()
.await
.get(id)
.map(|reg| reg.index.clone())
else {
return Ok(());
};
let store = inner.store.clone();
let needed = index.needed_chunks();
let total = needed.len() as u32;
let present = tokio::task::spawn_blocking(move || -> std::io::Result<u32> {
Ok(store.has_many(&needed)?.iter().filter(|p| **p).count() as u32)
})
.await??;
let probe = crate::wire::TreeProbe {
version: WIRE_VERSION,
have_index: true,
chunks_present: present,
chunks_total: total,
};
query
.reply(
crate::keys::tree_have_key(inner.tree_prefix.as_str(), id),
encode(&probe)?,
)
.encoding(&crate::wire::ENC_TREEPROBE)
.await
.map_err(BlobError::zenoh)?;
Ok(())
}
#[derive(Debug)]
pub struct TreeClient {
session: zenoh::Session,
store_prefix: QueryPrefix,
tree_prefix: QueryPrefix,
cfg: TreeClientConfig,
}
#[derive(Debug, Clone)]
struct TreeClientConfig {
query_timeout: Duration,
fetch_concurrency: usize,
max_index_bytes: usize,
max_tree_bytes: u64,
max_tree_chunks: u32,
batch_size: usize,
priority: Priority,
policy: MaterializePolicy,
temps: Option<Arc<crate::gc::TempTags>>,
}
impl Default for TreeClientConfig {
fn default() -> Self {
TreeClientConfig {
query_timeout: Duration::from_secs(30),
fetch_concurrency: 16,
max_index_bytes: 64 * 1024 * 1024,
max_tree_bytes: 1 << 40,
max_tree_chunks: 4_000_000,
batch_size: 256,
priority: Priority::DataLow,
policy: MaterializePolicy::default(),
temps: None,
}
}
}
#[derive(Debug)]
pub struct TreeClientBuilder {
session: zenoh::Session,
store_prefix: QueryPrefix,
tree_prefix: QueryPrefix,
cfg: TreeClientConfig,
}
impl TreeClientBuilder {
#[must_use]
pub fn query_timeout(mut self, t: Duration) -> Self {
self.cfg.query_timeout = t;
self
}
#[must_use]
pub fn fetch_concurrency(mut self, n: usize) -> Self {
self.cfg.fetch_concurrency = n.max(1);
self
}
#[must_use]
pub fn max_index_bytes(mut self, n: usize) -> Self {
self.cfg.max_index_bytes = n;
self
}
#[must_use]
pub fn max_tree_bytes(mut self, n: u64) -> Self {
self.cfg.max_tree_bytes = n;
self
}
#[must_use]
pub fn max_tree_chunks(mut self, n: u32) -> Self {
self.cfg.max_tree_chunks = n;
self
}
#[must_use]
pub fn batch_size(mut self, n: usize) -> Self {
self.cfg.batch_size = n.min(crate::wire::MAX_WANT_LIST);
self
}
#[must_use]
pub fn materialize_policy(mut self, policy: MaterializePolicy) -> Self {
self.cfg.policy = policy;
self
}
#[must_use]
pub fn temp_tags(mut self, temps: Arc<crate::gc::TempTags>) -> Self {
self.cfg.temps = Some(temps);
self
}
#[must_use]
pub fn priority(mut self, priority: Priority) -> Self {
self.cfg.priority = priority;
self
}
pub fn build(self) -> TreeClient {
TreeClient {
session: self.session,
store_prefix: self.store_prefix,
tree_prefix: self.tree_prefix,
cfg: self.cfg,
}
}
}
impl TreeClient {
pub fn builder(
session: &zenoh::Session,
store_prefix: QueryPrefix,
tree_prefix: QueryPrefix,
) -> TreeClientBuilder {
TreeClientBuilder {
session: session.clone(),
store_prefix,
tree_prefix,
cfg: TreeClientConfig::default(),
}
}
pub fn new(
session: &zenoh::Session,
store_prefix: QueryPrefix,
tree_prefix: QueryPrefix,
) -> Self {
Self::builder(session, store_prefix, tree_prefix).build()
}
pub async fn fetch_index(&self, id: &str) -> Result<TreeIndex> {
self.fetch_index_matching(id, None).await
}
async fn assemble_sharded_index(&self, sample: &zenoh::sample::Sample) -> Result<Vec<u8>> {
let desc: crate::wire::IndexDescriptor = decode(&sample.payload().to_bytes())?;
desc.validate(self.cfg.max_index_bytes)?;
let mut got: std::collections::HashMap<Hash, Vec<u8>> = Default::default();
for group in desc.index_chunks.chunks(self.cfg.batch_size.max(1)) {
let (replies, expected) = crate::store_client::batch_query(
&self.session,
self.store_prefix.as_str(),
group,
self.cfg.query_timeout,
self.cfg.priority,
)
.await?;
while let Ok(reply) = replies.recv_async().await {
let Ok(s) = reply.result() else { continue };
if let Some((h, bytes)) = crate::store_client::accept_batch_reply(
self.store_prefix.as_str(),
s,
&expected,
) {
got.entry(h).or_insert(bytes);
}
}
}
let mut out = Vec::with_capacity(desc.index_len as usize);
for c in &desc.index_chunks {
let bytes = match got.remove(&c.hash) {
Some(b) => b,
None => {
let key = store_key(self.store_prefix.as_str(), HashAlgo::Blake3, &c.hash);
crate::store_client::fetch_one_chunk(
&self.session,
&key,
&c.hash,
Some(c.len),
crate::compress::MAX_UNPACKED,
self.cfg.query_timeout,
self.cfg.priority,
)
.await?
.0
}
};
out.extend_from_slice(&bytes);
}
Ok(out)
}
pub async fn probe_snapshot(
&self,
id: &str,
) -> Result<Vec<(QueryPrefix, crate::wire::TreeProbe)>> {
let id = BlobId::new(id)?;
let id = id.as_str();
let replies = self
.session
.get(crate::keys::tree_have_key(self.tree_prefix.as_str(), id))
.consolidation(ConsolidationMode::None)
.priority(self.cfg.priority)
.timeout(self.cfg.query_timeout)
.await
.map_err(BlobError::zenoh)?;
let mut out = Vec::new();
while let Ok(reply) = replies.recv_async().await {
let Ok(sample) = reply.result() else { continue };
if !crate::wire::ENC_TREEPROBE.matches(sample.encoding()) {
continue;
}
let Ok(probe) = decode::<crate::wire::TreeProbe>(&sample.payload().to_bytes()) else {
continue;
};
if probe.validate().is_err() {
continue;
}
let Some(origin) = sample
.key_expr()
.as_str()
.strip_suffix(crate::keys::STORE_HAVE)
.and_then(|k| k.strip_suffix('/'))
.and_then(|k| k.strip_suffix(id))
.and_then(|k| k.strip_suffix('/'))
else {
continue;
};
let Ok(origin) = QueryPrefix::new(origin) else {
continue;
};
if !out.iter().any(|(o, _): &(QueryPrefix, _)| *o == origin) {
out.push((origin, probe));
}
}
Ok(out)
}
pub async fn fetch_index_by_root(&self, root: &Hash) -> Result<TreeIndex> {
self.fetch_index_matching(&root.to_string(), Some(*root))
.await
}
async fn fetch_index_matching(
&self,
id: &str,
expected_root: Option<Hash>,
) -> Result<TreeIndex> {
let id = BlobId::new(id)?;
let id = id.as_str();
let key = tree_key(self.tree_prefix.as_str(), id);
let replies = self
.session
.get(&key)
.consolidation(ConsolidationMode::None)
.priority(self.cfg.priority)
.timeout(self.cfg.query_timeout)
.await
.map_err(BlobError::zenoh)?;
let mut rejected: Option<BlobError> = None;
while let Ok(reply) = replies.recv_async().await {
let Ok(sample) = reply.result() else { continue };
let enc = sample.encoding();
let payload: Vec<u8> = if crate::wire::ENC_INDEX_DESC.matches(enc) {
match self.assemble_sharded_index(sample).await {
Ok(bytes) => bytes,
Err(e) => {
rejected.get_or_insert(e);
continue;
}
}
} else if ENC_INDEX.matches(enc) {
sample.payload().to_bytes().into_owned()
} else {
continue;
};
let verdict = if payload.len() > self.cfg.max_index_bytes {
Err(BlobError::InvalidManifest(format!(
"index payload {} exceeds the {} byte limit",
payload.len(),
self.cfg.max_index_bytes
)))
} else {
decode::<TreeIndex>(&payload)
.and_then(|index| index.validate().map(|()| index))
.and_then(|index| {
if index.id != id {
return Err(BlobError::MalformedMessage(format!(
"index id {:?} does not match requested {id:?}",
index.id
)));
}
if let Some(expected) = expected_root
&& index.root_hash != expected
{
return Err(BlobError::RootMismatch {
expected,
actual: index.root_hash,
});
}
Ok(index)
})
};
match verdict {
Ok(index) => return Ok(index),
Err(e) => {
rejected.get_or_insert(e);
}
}
}
Err(rejected.unwrap_or_else(|| BlobError::NotFound(id.to_string())))
}
pub fn download_tree<'a>(
&'a self,
req: &'a DownloadRequest,
dest_root: &'a Path,
store: &'a Arc<dyn ContentStore>,
) -> TreeDownload<'a> {
TreeDownload {
client: self,
req,
dest_root,
store,
sink: None,
cancel: None,
}
}
pub async fn fetch_file(
&self,
req: &DownloadRequest,
path: &str,
store: &Arc<dyn ContentStore>,
) -> Result<Vec<u8>> {
let index = self
.fetch_index_matching(&req.id, req.expected_root)
.await?;
let Some(chunks) = index.file_chunks(path) else {
return Err(BlobError::NotFound(format!(
"snapshot {:?} has no file at {path:?}",
index.id
)));
};
let size: u64 = chunks.iter().map(|c| u64::from(c.len)).sum();
if size > self.cfg.max_tree_bytes {
return Err(BlobError::InvalidManifest(format!(
"file {path:?} totals {size} bytes, over the configured limit of {}",
self.cfg.max_tree_bytes
)));
}
let wanted: Vec<ChunkRef> = chunks.to_vec();
self.fetch_missing(
&wanted,
store,
crate::client::NO_PROGRESS,
&CancelToken::new(),
wanted.len() as u32,
)
.await?;
let store = store.clone();
tokio::task::spawn_blocking(move || -> Result<Vec<u8>> {
let mut out = Vec::with_capacity(size as usize);
for c in &wanted {
let bytes = store
.get(&c.hash)?
.ok_or(BlobError::CorruptStore { hash: c.hash })?;
if bytes.len() as u32 != c.len {
return Err(BlobError::ChunkLengthMismatch {
expected: c.len,
actual: bytes.len() as u32,
});
}
out.extend_from_slice(&bytes);
}
Ok(out)
})
.await?
}
async fn run_download_tree(
&self,
req: &DownloadRequest,
dest_root: &Path,
store: &Arc<dyn ContentStore>,
sink: &dyn ProgressSink,
cancel: &CancelToken,
) -> Result<TransferStats> {
let started_at = tokio::time::Instant::now();
let index = self
.fetch_index_matching(&req.id, req.expected_root)
.await?;
let total_size = index.total_size();
if total_size > self.cfg.max_tree_bytes {
return Err(BlobError::InvalidManifest(format!(
"snapshot totals {total_size} bytes, over the configured limit of {}",
self.cfg.max_tree_bytes
)));
}
let needed = index.needed_chunks();
let _protect = self
.cfg
.temps
.as_ref()
.map(|t| t.protect(needed.iter().copied()));
if needed.len() as u64 > self.cfg.max_tree_chunks as u64 {
return Err(BlobError::InvalidManifest(format!(
"snapshot references {} chunks, over the configured limit of {}",
needed.len(),
self.cfg.max_tree_chunks
)));
}
let total = needed.len() as u32;
sink.emit(Progress::Started {
total_len: total_size,
chunk_count: total,
});
zdebug!(id = %index.id, chunks = total, "tree download start");
let mut stats = self
.fetch_missing(&index.needed_chunk_refs(), store, sink, cancel, total)
.await?;
sink.emit(Progress::Verifying);
let entries = index.entries.clone();
let dest = dest_root.to_path_buf();
let store2 = store.clone();
let policy = self.cfg.policy;
tokio::task::spawn_blocking(move || reconstruct_tree(&dest, &entries, &*store2, policy))
.await??;
sink.emit(Progress::Completed {
path: dest_root.to_path_buf(),
});
stats.elapsed = started_at.elapsed();
zdebug!(id = %index.id, fetched = stats.chunks_fetched, "tree download complete");
Ok(stats)
}
async fn fetch_missing(
&self,
needed: &[ChunkRef],
store: &Arc<dyn ContentStore>,
sink: &dyn ProgressSink,
cancel: &CancelToken,
total: u32,
) -> Result<TransferStats> {
let mut stats = TransferStats::default();
let mut received: u32 = 0;
let mut bytes_received: u64 = 0;
let presence: Vec<bool> = {
let store = store.clone();
let needed = needed.to_vec();
tokio::task::spawn_blocking(move || -> std::io::Result<Vec<bool>> {
store.has_many(&needed.iter().map(|c| c.hash).collect::<Vec<_>>())
})
.await??
};
let mut missing: Vec<(usize, ChunkRef)> = Vec::new();
for (i, (chunk, present)) in needed.iter().cloned().zip(presence).enumerate() {
if present {
received += 1;
stats.chunks_resumed += 1;
sink.emit(Progress::Chunk {
index: i as u32,
received,
total,
bytes_received,
});
} else {
missing.push((i, chunk));
}
}
if self.cfg.batch_size > 0 && !missing.is_empty() {
let mut unanswered: Vec<(usize, ChunkRef)> = Vec::new();
for group in missing.chunks(self.cfg.batch_size) {
if cancel.is_cancelled() {
sink.emit(Progress::Cancelled { received, total });
return Err(BlobError::Cancelled { received, total });
}
let refs: Vec<ChunkRef> = group.iter().map(|(_, c)| c.clone()).collect();
stats.queries += 1;
let (replies, expected) = crate::store_client::batch_query(
&self.session,
self.store_prefix.as_str(),
&refs,
self.cfg.query_timeout,
self.cfg.priority,
)
.await?;
const FLUSH: usize = 32;
let mut pending: Vec<(Hash, Vec<u8>)> = Vec::with_capacity(FLUSH);
let mut arrived: std::collections::HashSet<Hash> = Default::default();
let mut cancelled = false;
loop {
if cancel.is_cancelled() {
cancelled = true;
}
let next = if cancelled {
None
} else {
replies.recv_async().await.ok()
};
if let Some(reply) = &next
&& let Ok(sample) = reply.result()
&& let Some((hash, bytes)) = crate::store_client::accept_batch_reply(
self.store_prefix.as_str(),
sample,
&expected,
)
&& arrived.insert(hash)
{
pending.push((hash, bytes));
}
if pending.len() >= FLUSH || (next.is_none() && !pending.is_empty()) {
let batch = std::mem::take(&mut pending);
let put_store = store.clone();
let sizes: Vec<(Hash, u64)> =
batch.iter().map(|(h, b)| (*h, b.len() as u64)).collect();
tokio::task::spawn_blocking(move || -> std::io::Result<()> {
for (h, b) in &batch {
put_store.put(h, b)?;
}
Ok(())
})
.await??;
for (hash, len) in sizes {
received += 1;
bytes_received += len;
stats.chunks_fetched += 1;
stats.bytes_fetched += len;
let index = group
.iter()
.find(|(_, c)| c.hash == hash)
.map_or(0, |(i, _)| *i as u32);
sink.emit(Progress::Chunk {
index,
received,
total,
bytes_received,
});
}
}
if next.is_none() {
break;
}
}
for (i, chunk) in group {
if !arrived.contains(&chunk.hash) {
unanswered.push((*i, chunk.clone()));
}
}
if cancelled {
sink.emit(Progress::Cancelled { received, total });
return Err(BlobError::Cancelled { received, total });
}
}
zdebug!(
batched = missing.len() - unanswered.len(),
falling_back = unanswered.len(),
"tier-2 batch rounds complete"
);
missing = unanswered;
if cancel.is_cancelled() {
sink.emit(Progress::Cancelled { received, total });
return Err(BlobError::Cancelled { received, total });
}
}
let mut iter = missing.into_iter();
let mut join: tokio::task::JoinSet<Result<(usize, u64, u32)>> = tokio::task::JoinSet::new();
loop {
while join.len() < self.cfg.fetch_concurrency {
let Some((i, chunk)) = iter.next() else {
break;
};
stats.queries += 1;
let session = self.session.clone();
let hash = chunk.hash;
let key = store_key(self.store_prefix.as_str(), HashAlgo::Blake3, &hash);
let timeout = self.cfg.query_timeout;
let priority = self.cfg.priority;
let store = store.clone();
join.spawn(async move {
let (bytes, rejected) = crate::store_client::fetch_one_chunk(
&session,
&key,
&hash,
Some(chunk.len),
crate::compress::MAX_UNPACKED,
timeout,
priority,
)
.await?;
let len = bytes.len() as u64;
let put_store = store.clone();
tokio::task::spawn_blocking(move || put_store.put(&hash, &bytes)).await??;
Ok((i, len, rejected))
});
}
if join.is_empty() {
break;
}
if cancel.is_cancelled() {
join.abort_all();
sink.emit(Progress::Cancelled { received, total });
return Err(BlobError::Cancelled { received, total });
}
match tokio::time::timeout(Duration::from_millis(200), join.join_next()).await {
Err(_) => continue, Ok(None) => break,
Ok(Some(res)) => {
let (i, len, rejected) = res??;
received += 1;
bytes_received += len;
stats.chunks_fetched += 1;
stats.bytes_fetched += len;
stats.rejected += rejected;
sink.emit(Progress::Chunk {
index: i as u32,
received,
total,
bytes_received,
});
}
}
}
Ok(stats)
}
}
#[must_use = "a download does nothing until it is awaited"]
pub struct TreeDownload<'a> {
client: &'a TreeClient,
req: &'a DownloadRequest,
dest_root: &'a Path,
store: &'a Arc<dyn ContentStore>,
sink: Option<&'a dyn ProgressSink>,
cancel: Option<&'a CancelToken>,
}
impl std::fmt::Debug for TreeDownload<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TreeDownload")
.field("req", &self.req)
.field("dest_root", &self.dest_root)
.finish_non_exhaustive()
}
}
impl<'a> TreeDownload<'a> {
pub fn progress(mut self, sink: &'a dyn ProgressSink) -> Self {
self.sink = Some(sink);
self
}
pub fn cancel(mut self, cancel: &'a CancelToken) -> Self {
self.cancel = Some(cancel);
self
}
}
impl<'a> std::future::IntoFuture for TreeDownload<'a> {
type Output = Result<TransferStats>;
type IntoFuture =
std::pin::Pin<Box<dyn std::future::Future<Output = Self::Output> + Send + 'a>>;
fn into_future(self) -> Self::IntoFuture {
Box::pin(async move {
let sink = self.sink.unwrap_or(crate::client::NO_PROGRESS);
let fresh;
let cancel = match self.cancel {
Some(c) => c,
None => {
fresh = CancelToken::new();
&fresh
}
};
self.client
.run_download_tree(self.req, self.dest_root, self.store, sink, cancel)
.await
})
}
}
fn reconstruct_tree(
dest_root: &Path,
entries: &[Entry],
store: &dyn ContentStore,
policy: MaterializePolicy,
) -> Result<()> {
std::fs::create_dir_all(dest_root)?;
let root = dest_root.canonicalize()?;
for e in entries {
if let Entry::Dir { path, .. } = e {
create_dir_confined(&root, &sanitize_rel_path(path)?)?;
}
}
for e in entries {
match e {
Entry::File {
path,
mode,
mtime,
size,
chunks,
} => {
let rel = sanitize_rel_path(path)?;
if let Some(parent) = rel.parent() {
create_dir_confined(&root, parent)?;
}
let p = root.join(&rel);
assert_parent_within(&root, &p)?;
remove_existing(&p, policy)?;
let mut f = std::fs::File::create(&p)?;
let mut written: u64 = 0;
for c in chunks {
let bytes = store
.get(&c.hash)?
.ok_or_else(|| BlobError::NotFound(c.hash.to_string()))?;
if bytes.len() as u32 != c.len {
return Err(BlobError::ChunkLengthMismatch {
expected: c.len,
actual: bytes.len() as u32,
});
}
if Hash::of(&bytes) != c.hash {
return Err(BlobError::CorruptStore { hash: c.hash });
}
f.write_all(&bytes)?;
written += bytes.len() as u64;
}
if written != *size {
return Err(BlobError::MalformedMessage(format!(
"file {path:?}: wrote {written} of declared {size} bytes"
)));
}
set_mode(&p, *mode, policy);
set_mtime(&f, *mtime);
}
Entry::Hardlink { path, target } => {
let rel = sanitize_rel_path(path)?;
if let Some(parent) = rel.parent() {
create_dir_confined(&root, parent)?;
}
let p = root.join(&rel);
let t = root.join(sanitize_rel_path(target)?);
assert_parent_within(&root, &p)?;
let canon_t = t.canonicalize().map_err(|e| {
BlobError::UnsafePath(format!(
"hardlink target {target:?} is not materialized: {e}"
))
})?;
if !canon_t.starts_with(&root) || !canon_t.is_file() {
return Err(BlobError::UnsafePath(format!(
"hardlink target {target:?} resolves outside the destination root"
)));
}
remove_existing(&p, policy)?;
std::fs::hard_link(&canon_t, &p)?;
}
_ => {}
}
}
for e in entries {
if let Entry::Symlink { path, target } = e {
let rel = sanitize_rel_path(path)?;
sanitize_symlink_target(&rel, target)?;
if let Some(parent) = rel.parent() {
create_dir_confined(&root, parent)?;
}
let p = root.join(&rel);
assert_parent_within(&root, &p)?;
remove_existing(&p, policy)?;
symlink(target, &p)?;
}
}
for e in entries.iter().rev() {
if let Entry::Dir { path, mode, mtime } = e
&& let Ok(rel) = sanitize_rel_path(path)
{
let p = root.join(rel);
if *mtime > 0
&& let Ok(f) = std::fs::File::open(&p)
{
set_mtime(&f, *mtime);
}
set_mode(&p, *mode, policy);
}
}
Ok(())
}
fn remove_existing(p: &Path, policy: MaterializePolicy) -> Result<()> {
let Ok(meta) = std::fs::symlink_metadata(p) else {
return Ok(());
};
if meta.is_dir() {
if !policy.replace_directories {
return Err(BlobError::UnsafePath(format!(
"refusing to replace the existing directory {p:?} (and everything under it) \
with a non-directory entry; enable MaterializePolicy::replace_directories \
if that is intended"
)));
}
let _ = std::fs::remove_dir_all(p);
} else {
let _ = std::fs::remove_file(p);
}
Ok(())
}
#[cfg(unix)]
const MODE_MASK: u32 = 0o0777;
#[cfg(unix)]
fn set_mode(path: &Path, mode: u32, policy: MaterializePolicy) {
use std::os::unix::fs::PermissionsExt;
let mode = if policy.restore_setid {
mode & 0o7777
} else {
mode & MODE_MASK
};
if mode != 0 {
let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode));
}
}
#[cfg(not(unix))]
fn set_mode(_path: &Path, _mode: u32, _policy: MaterializePolicy) {}
fn set_mtime(f: &std::fs::File, mtime: i64) {
if mtime > 0 {
let t = std::time::UNIX_EPOCH + Duration::from_secs(mtime as u64);
let _ = f.set_modified(t);
}
}
#[cfg(unix)]
fn symlink(target: &str, path: &Path) -> std::io::Result<()> {
std::os::unix::fs::symlink(target, path)
}
#[cfg(not(unix))]
fn symlink(_target: &str, _path: &Path) -> std::io::Result<()> {
Err(std::io::Error::other(
"symlink entries are not supported on this platform",
))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::store::MemoryStore;
#[test]
fn build_dedups_identical_chunks() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("a.txt"), b"same").unwrap();
std::fs::write(dir.path().join("b.txt"), b"same").unwrap();
let store = MemoryStore::new();
let index = build_tree(dir.path(), "snap", &CdcParams::default(), &store).unwrap();
assert_eq!(store.len(), 1);
assert_eq!(index.needed_chunks().len(), 1);
index.validate().unwrap();
assert_eq!(index.file_count(), 2);
}
#[test]
fn root_hash_ignores_mtime() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("f.txt"), b"content").unwrap();
let store = MemoryStore::new();
let a = build_tree(dir.path(), "snap", &CdcParams::default(), &store).unwrap();
let file = std::fs::File::options()
.append(true)
.open(dir.path().join("f.txt"))
.unwrap();
file.set_modified(std::time::SystemTime::now() + Duration::from_secs(1000))
.unwrap();
let b = build_tree(dir.path(), "snap", &CdcParams::default(), &store).unwrap();
assert_eq!(a.root_hash, b.root_hash, "mtime must not change identity");
std::fs::write(dir.path().join("f.txt"), b"different").unwrap();
let c = build_tree(dir.path(), "snap", &CdcParams::default(), &store).unwrap();
assert_ne!(a.root_hash, c.root_hash);
}
#[test]
fn traversal_paths_rejected_by_validate() {
let evil_paths = ["../evil", "/etc/passwd", "a/../../b"];
for evil in evil_paths {
let entries = vec![Entry::File {
path: evil.into(),
mode: 0,
mtime: 0,
size: 0,
chunks: vec![],
}];
let root_hash = root_digest(&entries).unwrap();
let index = TreeIndex {
version: WIRE_VERSION,
id: BlobId::new("x").unwrap(),
algo: HashAlgo::Blake3,
cdc: CdcParams::default(),
entries,
root_hash,
};
assert!(index.validate().is_err(), "must reject path {evil:?}");
}
}
#[test]
fn oversized_chunk_len_rejected() {
for bad_len in [0u32, CdcParams::default().max + 1, u32::MAX] {
let entries = vec![Entry::File {
path: "f".into(),
mode: 0,
mtime: 0,
size: bad_len as u64,
chunks: vec![ChunkRef {
hash: Hash::of(b"x"),
len: bad_len,
}],
}];
let root_hash = root_digest(&entries).unwrap();
let index = TreeIndex {
version: WIRE_VERSION,
id: BlobId::new("x").unwrap(),
algo: HashAlgo::Blake3,
cdc: CdcParams::default(),
entries,
root_hash,
};
assert!(index.validate().is_err(), "len {bad_len} must be rejected");
}
}
#[test]
fn size_chunk_mismatch_rejected() {
let entries = vec![Entry::File {
path: "f".into(),
mode: 0,
mtime: 0,
size: 100, chunks: vec![],
}];
let root_hash = root_digest(&entries).unwrap();
let index = TreeIndex {
version: WIRE_VERSION,
id: BlobId::new("x").unwrap(),
algo: HashAlgo::Blake3,
cdc: CdcParams::default(),
entries,
root_hash,
};
assert!(index.validate().is_err());
}
#[cfg(unix)]
#[test]
fn hardlinks_detected() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("orig.bin"), b"shared bytes").unwrap();
std::fs::hard_link(dir.path().join("orig.bin"), dir.path().join("link.bin")).unwrap();
let store = MemoryStore::new();
let index = build_tree(dir.path(), "snap", &CdcParams::default(), &store).unwrap();
let hardlinks: Vec<_> = index
.entries
.iter()
.filter(|e| matches!(e, Entry::Hardlink { .. }))
.collect();
assert_eq!(hardlinks.len(), 1);
assert_eq!(index.file_count(), 1);
assert_eq!(store.len(), 1);
}
#[cfg(unix)]
#[test]
fn fifo_is_an_error_not_a_silent_drop() {
use std::os::unix::ffi::OsStrExt;
let dir = tempfile::tempdir().unwrap();
let fifo = dir.path().join("pipe");
let status = std::process::Command::new("mkfifo")
.arg(fifo.as_os_str())
.status();
if !status.map(|s| s.success()).unwrap_or(false) {
return; }
let store = MemoryStore::new();
let err = build_tree(dir.path(), "snap", &CdcParams::default(), &store)
.expect_err("fifo must error");
assert!(err.to_string().contains("unsupported file type"), "{err}");
let _ = fifo.as_os_str().as_bytes();
}
}