use std::collections::HashMap;
use std::io::Write;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use serde::{Deserialize, Serialize};
use tokio::sync::{Notify, Semaphore};
use zenoh::qos::Priority;
use zenoh::query::ConsolidationMode;
use crate::cancel::CancelToken;
use crate::chunk::CdcParams;
use crate::client::DownloadRequest;
use crate::compress::{ChunkCompression, pack, unpack};
use crate::error::{BlobError, Result};
use crate::hash::Hash;
use crate::manifest::validate_id;
use crate::obs::{TransferStats, zdebug};
use crate::paths::{
assert_parent_within, create_dir_confined, sanitize_rel_path, sanitize_symlink_target,
};
use crate::progress::{Progress, ProgressSink};
use crate::server::{ErrorCallback, FifoQueryable, ServerHandle, report_error};
use crate::store::ContentStore;
use crate::wire::{ENC_CHUNK, ENC_INDEX, WIRE_VERSION, decode, encode};
use crate::{store_key, tree_key};
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ChunkRef {
pub hash: Hash,
pub len: u32,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum Entry {
Dir {
path: String,
mode: u32,
mtime: i64,
},
File {
path: String,
mode: u32,
mtime: i64,
size: u64,
chunks: Vec<ChunkRef>,
},
Symlink {
path: String,
target: String,
},
Hardlink {
path: String,
target: String,
},
}
impl Entry {
pub fn path(&self) -> &str {
match self {
Entry::Dir { path, .. }
| Entry::File { path, .. }
| Entry::Symlink { path, .. }
| Entry::Hardlink { path, .. } => path,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TreeIndex {
pub version: u16,
pub id: String,
pub algo: String,
pub cdc: CdcParams,
pub entries: Vec<Entry>,
pub root_hash: Hash,
}
#[derive(Serialize)]
enum IdentityEntry<'a> {
Dir {
path: &'a str,
mode: u32,
},
File {
path: &'a str,
mode: u32,
size: u64,
chunks: &'a Vec<ChunkRef>,
},
Symlink {
path: &'a str,
target: &'a str,
},
Hardlink {
path: &'a str,
target: &'a str,
},
}
fn root_digest(entries: &[Entry]) -> Result<Hash> {
let view: Vec<IdentityEntry> = entries
.iter()
.map(|e| match e {
Entry::Dir { path, mode, .. } => IdentityEntry::Dir { path, mode: *mode },
Entry::File {
path,
mode,
size,
chunks,
..
} => IdentityEntry::File {
path,
mode: *mode,
size: *size,
chunks,
},
Entry::Symlink { path, target } => IdentityEntry::Symlink { path, target },
Entry::Hardlink { path, target } => IdentityEntry::Hardlink { path, target },
})
.collect();
let bytes = encode(&(WIRE_VERSION, Hash::ALGO, view))?;
Ok(Hash::of(&bytes))
}
impl TreeIndex {
pub fn needed_chunks(&self) -> Vec<Hash> {
let mut seen = std::collections::HashSet::new();
let mut out = Vec::new();
for e in &self.entries {
if let Entry::File { chunks, .. } = e {
for c in chunks {
if seen.insert(c.hash) {
out.push(c.hash);
}
}
}
}
out
}
pub fn total_size(&self) -> u64 {
self.entries
.iter()
.map(|e| match e {
Entry::File { size, .. } => *size,
_ => 0,
})
.sum()
}
pub fn file_count(&self) -> usize {
self.entries
.iter()
.filter(|e| matches!(e, Entry::File { .. }))
.count()
}
pub fn compute_root(&self) -> Result<Hash> {
root_digest(&self.entries)
}
pub fn keyed_by_root(mut self) -> Self {
self.id = self.root_hash.to_string();
self
}
pub fn is_content_addressed(&self) -> bool {
self.id == self.root_hash.to_string()
}
pub fn validate(&self) -> Result<()> {
if self.version != WIRE_VERSION {
return Err(BlobError::UnsupportedVersion(self.version));
}
if self.algo != Hash::ALGO {
return Err(BlobError::Protocol(format!(
"unsupported algo: {}",
self.algo
)));
}
validate_id(&self.id)?;
self.cdc.validate()?;
for e in &self.entries {
let rel = sanitize_rel_path(e.path())?;
match e {
Entry::File { size, chunks, .. } => {
let sum: u64 = chunks.iter().map(|c| c.len as u64).sum();
if sum != *size {
return Err(BlobError::InvalidManifest(format!(
"file {:?}: size {size} != chunk sum {sum}",
e.path()
)));
}
if let Some(bad) = chunks.iter().find(|c| c.len == 0 || c.len > self.cdc.max) {
return Err(BlobError::InvalidManifest(format!(
"file {:?}: chunk len {} outside 1..={}",
e.path(),
bad.len,
self.cdc.max
)));
}
}
Entry::Symlink { target, .. } => {
sanitize_symlink_target(&rel, target)?;
}
Entry::Hardlink { target, .. } => {
sanitize_rel_path(target)?;
}
Entry::Dir { .. } => {}
}
}
let actual = self.compute_root()?;
if actual != self.root_hash {
return Err(BlobError::RootMismatch {
expected: self.root_hash,
actual,
});
}
Ok(())
}
}
pub fn build_tree(
root: &Path,
id: impl Into<String>,
cdc: &CdcParams,
store: &dyn ContentStore,
) -> Result<TreeIndex> {
cdc.validate()?;
let id = id.into();
validate_id(&id)?;
let mut entries = Vec::new();
walk(root, cdc, store, &mut entries)?;
let root_hash = root_digest(&entries)?;
Ok(TreeIndex {
version: WIRE_VERSION,
id,
algo: Hash::ALGO.to_string(),
cdc: *cdc,
entries,
root_hash,
})
}
fn rel_path_of(root: &Path, path: &Path) -> Result<String> {
let rel = path.strip_prefix(root).unwrap_or(path);
let mut out = String::new();
for comp in rel.components() {
let s = comp
.as_os_str()
.to_str()
.ok_or_else(|| BlobError::Protocol(format!("non-UTF-8 file name in tree: {rel:?}")))?;
if !out.is_empty() {
out.push('/');
}
out.push_str(s);
}
Ok(out)
}
#[cfg(unix)]
fn mode_of(meta: &std::fs::Metadata) -> u32 {
use std::os::unix::fs::PermissionsExt;
meta.permissions().mode()
}
#[cfg(not(unix))]
fn mode_of(_meta: &std::fs::Metadata) -> u32 {
0
}
fn mtime_of(meta: &std::fs::Metadata) -> i64 {
meta.modified()
.ok()
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.map(|d| d.as_secs() as i64)
.unwrap_or(0)
}
fn walk(
root: &Path,
cdc: &CdcParams,
store: &dyn ContentStore,
entries: &mut Vec<Entry>,
) -> Result<()> {
fn read_dir_sorted(dir: &Path) -> std::io::Result<std::vec::IntoIter<std::fs::DirEntry>> {
let mut items: Vec<_> = std::fs::read_dir(dir)?.collect::<std::io::Result<Vec<_>>>()?;
items.sort_by_key(|e| e.file_name());
Ok(items.into_iter())
}
let mut inodes: HashMap<(u64, u64), String> = HashMap::new();
let mut stack = vec![read_dir_sorted(root)?];
while let Some(top) = stack.last_mut() {
let Some(item) = top.next() else {
stack.pop();
continue;
};
let path = item.path();
let meta = std::fs::symlink_metadata(&path)?;
let rel = rel_path_of(root, &path)?;
let ftype = meta.file_type();
if ftype.is_symlink() {
let target_os = std::fs::read_link(&path)?;
let target = target_os
.to_str()
.ok_or_else(|| BlobError::Protocol(format!("non-UTF-8 symlink target at {rel:?}")))?
.to_string();
entries.push(Entry::Symlink { path: rel, target });
} else if ftype.is_dir() {
entries.push(Entry::Dir {
path: rel,
mode: mode_of(&meta),
mtime: mtime_of(&meta),
});
stack.push(read_dir_sorted(&path)?);
} else if ftype.is_file() {
if let Some(first) = hardlink_target(&meta, &rel, &mut inodes) {
entries.push(Entry::Hardlink {
path: rel,
target: first,
});
continue;
}
let file = std::fs::File::open(&path)?;
let mut refs = Vec::new();
let mut size: u64 = 0;
for chunk in cdc.chunk_reader(file) {
let bytes = chunk?;
let hash = Hash::of(&bytes);
size += bytes.len() as u64;
refs.push(ChunkRef {
hash,
len: bytes.len() as u32,
});
if !store.has(&hash) {
store.put(&hash, &bytes)?;
}
}
entries.push(Entry::File {
path: rel,
mode: mode_of(&meta),
mtime: mtime_of(&meta),
size,
chunks: refs,
});
} else {
return Err(BlobError::Protocol(format!(
"unsupported file type at {rel:?} (fifo/socket/device)"
)));
}
}
Ok(())
}
#[cfg(unix)]
fn hardlink_target(
meta: &std::fs::Metadata,
rel: &str,
inodes: &mut HashMap<(u64, u64), String>,
) -> Option<String> {
use std::os::unix::fs::MetadataExt;
if meta.nlink() > 1 {
let key = (meta.dev(), meta.ino());
match inodes.get(&key) {
Some(first) => return Some(first.clone()),
None => {
inodes.insert(key, rel.to_string());
}
}
}
None
}
#[cfg(not(unix))]
fn hardlink_target(
_meta: &std::fs::Metadata,
_rel: &str,
_inodes: &mut HashMap<(u64, u64), String>,
) -> Option<String> {
None
}
#[derive(Clone)]
pub struct TreeServer {
inner: Arc<TreeInner>,
}
struct TreeInner {
session: Arc<zenoh::Session>,
store_prefix: String,
tree_prefix: String,
store: Arc<dyn ContentStore>,
index: tokio::sync::RwLock<std::collections::HashMap<String, TreeIndex>>,
inflight: Semaphore,
compression: ChunkCompression,
on_error: Option<ErrorCallback>,
}
pub struct TreeServerBuilder {
session: Arc<zenoh::Session>,
store_prefix: String,
tree_prefix: String,
store: Arc<dyn ContentStore>,
max_inflight: usize,
compression: ChunkCompression,
on_error: Option<ErrorCallback>,
}
impl TreeServerBuilder {
pub fn max_inflight(mut self, n: usize) -> Self {
self.max_inflight = n.max(1);
self
}
pub fn compression(mut self, c: ChunkCompression) -> Self {
self.compression = c;
self
}
pub fn on_error(mut self, cb: ErrorCallback) -> Self {
self.on_error = Some(cb);
self
}
pub fn build(self) -> TreeServer {
TreeServer {
inner: Arc::new(TreeInner {
session: self.session,
store_prefix: self.store_prefix,
tree_prefix: self.tree_prefix,
store: self.store,
index: tokio::sync::RwLock::new(std::collections::HashMap::new()),
inflight: Semaphore::new(self.max_inflight),
compression: self.compression,
on_error: self.on_error,
}),
}
}
}
impl TreeServer {
pub fn builder(
session: Arc<zenoh::Session>,
store_prefix: impl Into<String>,
tree_prefix: impl Into<String>,
store: Arc<dyn ContentStore>,
) -> TreeServerBuilder {
TreeServerBuilder {
session,
store_prefix: store_prefix.into(),
tree_prefix: tree_prefix.into(),
store,
max_inflight: 8,
compression: ChunkCompression::default(),
on_error: None,
}
}
pub fn new(
session: Arc<zenoh::Session>,
store_prefix: impl Into<String>,
tree_prefix: impl Into<String>,
store: Arc<dyn ContentStore>,
) -> Self {
Self::builder(session, store_prefix, tree_prefix, store).build()
}
pub async fn register(&self, index: TreeIndex) {
self.inner
.index
.write()
.await
.insert(index.id.clone(), index);
}
pub async fn unregister(&self, id: &str) {
self.inner.index.write().await.remove(id);
}
async fn declare(&self) -> Result<(FifoQueryable, FifoQueryable)> {
crate::paths::validate_serve_prefix(&self.inner.store_prefix)?;
crate::paths::validate_serve_prefix(&self.inner.tree_prefix)?;
let store_q = self
.inner
.session
.declare_queryable(format!("{}/**", self.inner.store_prefix))
.await
.map_err(BlobError::zenoh)?;
let tree_q = self
.inner
.session
.declare_queryable(format!("{}/**", self.inner.tree_prefix))
.await
.map_err(BlobError::zenoh)?;
Ok((store_q, tree_q))
}
pub async fn spawn(self) -> Result<ServerHandle> {
let (store_q, tree_q) = self.declare().await?;
let stop = Arc::new(Notify::new());
let stop2 = stop.clone();
Ok(ServerHandle::new(
tokio::spawn(self.serve_loop(store_q, tree_q, stop2)),
stop,
))
}
pub async fn run(self) -> Result<()> {
let (store_q, tree_q) = self.declare().await?;
self.serve_loop(store_q, tree_q, Arc::new(Notify::new()))
.await
}
async fn serve_loop(
self,
store_q: FifoQueryable,
tree_q: FifoQueryable,
stop: Arc<Notify>,
) -> Result<()> {
loop {
tokio::select! {
_ = stop.notified() => break,
q = store_q.recv_async() => {
let Ok(query) = q else { break };
let inner = self.inner.clone();
tokio::spawn(async move {
if let Err(e) = serve_chunk_query(&inner, query).await {
report_error(&inner.on_error, &e);
}
});
}
q = tree_q.recv_async() => {
let Ok(query) = q else { break };
let inner = self.inner.clone();
tokio::spawn(async move {
if let Err(e) = serve_index_query(&inner, query).await {
report_error(&inner.on_error, &e);
}
});
}
}
}
Ok(())
}
}
async fn serve_chunk_query(inner: &TreeInner, query: zenoh::query::Query) -> Result<()> {
let _permit = inner
.inflight
.acquire()
.await
.map_err(|e| BlobError::Protocol(e.to_string()))?;
let key = query.key_expr().as_str();
let Some(hash) = parse_store_key(&inner.store_prefix, key) else {
return Ok(()); };
let store = inner.store.clone();
let compression = inner.compression;
let packed = tokio::task::spawn_blocking(move || {
store.get(&hash).map(|bytes| pack(&bytes, compression))
})
.await
.map_err(|e| BlobError::Protocol(format!("store get task: {e}")))?;
if let Some(packed) = packed {
query
.reply(query.key_expr().clone(), packed?)
.encoding(ENC_CHUNK)
.await
.map_err(BlobError::zenoh)?;
}
Ok(())
}
async fn serve_index_query(inner: &TreeInner, query: zenoh::query::Query) -> Result<()> {
let _permit = inner
.inflight
.acquire()
.await
.map_err(|e| BlobError::Protocol(e.to_string()))?;
let key = query.key_expr().as_str().to_string();
let Some(id) = key.strip_prefix(&format!("{}/", inner.tree_prefix)) else {
return Ok(());
};
let Some(index) = inner.index.read().await.get(id).cloned() else {
return Ok(()); };
let payload = encode(&index)?;
query
.reply(query.key_expr().clone(), payload)
.encoding(ENC_INDEX)
.await
.map_err(BlobError::zenoh)?;
Ok(())
}
fn parse_store_key(store_prefix: &str, key: &str) -> Option<Hash> {
let rest = key.strip_prefix(store_prefix)?.strip_prefix('/')?;
let (algo, hex) = rest.split_once('/')?;
if algo != Hash::ALGO {
return None;
}
hex.parse().ok()
}
pub struct TreeClient {
session: Arc<zenoh::Session>,
store_prefix: String,
tree_prefix: String,
cfg: TreeClientConfig,
}
#[derive(Debug, Clone)]
struct TreeClientConfig {
query_timeout: Duration,
fetch_concurrency: usize,
max_index_bytes: usize,
priority: Priority,
}
impl Default for TreeClientConfig {
fn default() -> Self {
TreeClientConfig {
query_timeout: Duration::from_secs(30),
fetch_concurrency: 16,
max_index_bytes: 64 * 1024 * 1024,
priority: Priority::DataLow,
}
}
}
pub struct TreeClientBuilder {
session: Arc<zenoh::Session>,
store_prefix: String,
tree_prefix: String,
cfg: TreeClientConfig,
}
impl TreeClientBuilder {
pub fn query_timeout(mut self, t: Duration) -> Self {
self.cfg.query_timeout = t;
self
}
pub fn fetch_concurrency(mut self, n: usize) -> Self {
self.cfg.fetch_concurrency = n.max(1);
self
}
pub fn max_index_bytes(mut self, n: usize) -> Self {
self.cfg.max_index_bytes = n;
self
}
pub fn priority(mut self, priority: Priority) -> Self {
self.cfg.priority = priority;
self
}
pub fn build(self) -> TreeClient {
TreeClient {
session: self.session,
store_prefix: self.store_prefix,
tree_prefix: self.tree_prefix,
cfg: self.cfg,
}
}
}
impl TreeClient {
pub fn builder(
session: Arc<zenoh::Session>,
store_prefix: impl Into<String>,
tree_prefix: impl Into<String>,
) -> TreeClientBuilder {
TreeClientBuilder {
session,
store_prefix: store_prefix.into(),
tree_prefix: tree_prefix.into(),
cfg: TreeClientConfig::default(),
}
}
pub fn new(
session: Arc<zenoh::Session>,
store_prefix: impl Into<String>,
tree_prefix: impl Into<String>,
) -> Self {
Self::builder(session, store_prefix, tree_prefix).build()
}
pub async fn fetch_index(&self, id: &str) -> Result<TreeIndex> {
self.fetch_index_matching(id, None).await
}
async fn fetch_index_matching(
&self,
id: &str,
expected_root: Option<Hash>,
) -> Result<TreeIndex> {
crate::paths::validate_query_prefix(&self.store_prefix)?;
crate::paths::validate_query_prefix(&self.tree_prefix)?;
validate_id(id)?;
let key = tree_key(&self.tree_prefix, id);
let replies = self
.session
.get(&key)
.consolidation(ConsolidationMode::None)
.priority(self.cfg.priority)
.timeout(self.cfg.query_timeout)
.await
.map_err(BlobError::zenoh)?;
let mut rejected: Option<BlobError> = None;
while let Ok(reply) = replies.recv_async().await {
let Ok(sample) = reply.result() else { continue };
if sample.encoding().to_string() != ENC_INDEX {
continue;
}
let payload = sample.payload().to_bytes();
let verdict = if payload.len() > self.cfg.max_index_bytes {
Err(BlobError::InvalidManifest(format!(
"index payload {} exceeds the {} byte limit",
payload.len(),
self.cfg.max_index_bytes
)))
} else {
decode::<TreeIndex>(&payload)
.and_then(|index| index.validate().map(|()| index))
.and_then(|index| {
if index.id != id {
return Err(BlobError::Protocol(format!(
"index id {:?} does not match requested {id:?}",
index.id
)));
}
if let Some(expected) = expected_root
&& index.root_hash != expected
{
return Err(BlobError::RootMismatch {
expected,
actual: index.root_hash,
});
}
Ok(index)
})
};
match verdict {
Ok(index) => return Ok(index),
Err(e) => {
rejected.get_or_insert(e);
}
}
}
Err(rejected.unwrap_or_else(|| BlobError::NotFound(id.to_string())))
}
pub async fn download_tree(
&self,
req: &DownloadRequest,
dest_root: &Path,
store: &Arc<dyn ContentStore>,
sink: &dyn ProgressSink,
cancel: &CancelToken,
) -> Result<TransferStats> {
let started_at = tokio::time::Instant::now();
let index = self
.fetch_index_matching(&req.id, req.expected_root)
.await?;
let needed = index.needed_chunks();
let total = needed.len() as u32;
sink.emit(Progress::Started {
total_len: index.total_size(),
chunk_count: total,
});
zdebug!(id = %index.id, chunks = total, "tree download start");
let mut stats = self
.fetch_missing(&needed, store, sink, cancel, total)
.await?;
sink.emit(Progress::Verifying);
let entries = index.entries.clone();
let dest = dest_root.to_path_buf();
let store2 = store.clone();
tokio::task::spawn_blocking(move || reconstruct_tree(&dest, &entries, &*store2))
.await
.map_err(|e| BlobError::Protocol(format!("reconstruct task: {e}")))??;
sink.emit(Progress::Completed {
path: dest_root.to_path_buf(),
});
stats.elapsed = started_at.elapsed();
zdebug!(id = %index.id, fetched = stats.chunks_fetched, "tree download complete");
Ok(stats)
}
async fn fetch_missing(
&self,
needed: &[Hash],
store: &Arc<dyn ContentStore>,
sink: &dyn ProgressSink,
cancel: &CancelToken,
total: u32,
) -> Result<TransferStats> {
let mut stats = TransferStats::default();
let mut received: u32 = 0;
let mut bytes_received: u64 = 0;
let presence: Vec<bool> = {
let store = store.clone();
let needed = needed.to_vec();
tokio::task::spawn_blocking(move || {
needed.iter().map(|h| store.has(h)).collect::<Vec<_>>()
})
.await
.map_err(|e| BlobError::Protocol(format!("presence task: {e}")))?
};
let mut iter = needed
.iter()
.copied()
.zip(presence)
.enumerate()
.map(|(i, (h, present))| (i, h, present));
let mut join: tokio::task::JoinSet<Result<(usize, u64, u32)>> = tokio::task::JoinSet::new();
loop {
while join.len() < self.cfg.fetch_concurrency {
let Some((i, hash, present)) = iter.next() else {
break;
};
if present {
received += 1;
stats.chunks_resumed += 1;
sink.emit(Progress::Chunk {
index: i as u32,
received,
total,
bytes_received,
});
continue;
}
let session = self.session.clone();
let key = store_key(&self.store_prefix, Hash::ALGO, &hash);
let timeout = self.cfg.query_timeout;
let priority = self.cfg.priority;
let store = store.clone();
join.spawn(async move {
let (bytes, rejected) =
fetch_one_chunk(&session, &key, &hash, timeout, priority).await?;
let len = bytes.len() as u64;
let put_store = store.clone();
tokio::task::spawn_blocking(move || put_store.put(&hash, &bytes))
.await
.map_err(|e| BlobError::Protocol(format!("store put task: {e}")))??;
Ok((i, len, rejected))
});
}
if join.is_empty() {
break;
}
if cancel.is_cancelled() {
join.abort_all();
sink.emit(Progress::Cancelled { received, total });
return Err(BlobError::Cancelled { received, total });
}
match tokio::time::timeout(Duration::from_millis(200), join.join_next()).await {
Err(_) => continue, Ok(None) => break,
Ok(Some(res)) => {
let (i, len, rejected) =
res.map_err(|e| BlobError::Protocol(format!("fetch task: {e}")))??;
received += 1;
bytes_received += len;
stats.chunks_fetched += 1;
stats.bytes_fetched += len;
stats.rejected += rejected;
sink.emit(Progress::Chunk {
index: i as u32,
received,
total,
bytes_received,
});
}
}
}
Ok(stats)
}
}
async fn fetch_one_chunk(
session: &zenoh::Session,
key: &str,
hash: &Hash,
timeout: Duration,
priority: Priority,
) -> Result<(Vec<u8>, u32)> {
let mut rejected = 0u32;
let replies = session
.get(key)
.consolidation(ConsolidationMode::None)
.priority(priority)
.timeout(timeout)
.await
.map_err(BlobError::zenoh)?;
while let Ok(reply) = replies.recv_async().await {
let Ok(sample) = reply.result() else { continue };
if sample.encoding().to_string() != ENC_CHUNK {
continue;
}
let Ok(bytes) = unpack(&sample.payload().to_bytes()) else {
rejected += 1;
continue; };
if Hash::of(&bytes) != *hash {
rejected += 1;
continue; }
return Ok((bytes, rejected));
}
Err(BlobError::NotFound(hash.to_string()))
}
fn reconstruct_tree(dest_root: &Path, entries: &[Entry], store: &dyn ContentStore) -> Result<()> {
std::fs::create_dir_all(dest_root)?;
let root = dest_root.canonicalize()?;
for e in entries {
if let Entry::Dir { path, .. } = e {
create_dir_confined(&root, &sanitize_rel_path(path)?)?;
}
}
for e in entries {
match e {
Entry::File {
path,
mode,
mtime,
size,
chunks,
} => {
let rel = sanitize_rel_path(path)?;
if let Some(parent) = rel.parent() {
create_dir_confined(&root, parent)?;
}
let p = root.join(&rel);
assert_parent_within(&root, &p)?;
remove_existing(&p);
let mut f = std::fs::File::create(&p)?;
let mut written: u64 = 0;
for c in chunks {
let bytes = store
.get(&c.hash)
.ok_or_else(|| BlobError::NotFound(c.hash.to_string()))?;
if bytes.len() as u32 != c.len {
return Err(BlobError::HashMismatch);
}
f.write_all(&bytes)?;
written += bytes.len() as u64;
}
if written != *size {
return Err(BlobError::Protocol(format!(
"file {path:?}: wrote {written} of declared {size} bytes"
)));
}
set_mode(&p, *mode);
set_mtime(&f, *mtime);
}
Entry::Hardlink { path, target } => {
let rel = sanitize_rel_path(path)?;
if let Some(parent) = rel.parent() {
create_dir_confined(&root, parent)?;
}
let p = root.join(&rel);
let t = root.join(sanitize_rel_path(target)?);
assert_parent_within(&root, &p)?;
let canon_t = t.canonicalize().map_err(|e| {
BlobError::Protocol(format!(
"hardlink target {target:?} is not materialized: {e}"
))
})?;
if !canon_t.starts_with(&root) || !canon_t.is_file() {
return Err(BlobError::Protocol(format!(
"hardlink target {target:?} resolves outside the destination root"
)));
}
remove_existing(&p);
std::fs::hard_link(&canon_t, &p)?;
}
_ => {}
}
}
for e in entries {
if let Entry::Symlink { path, target } = e {
let rel = sanitize_rel_path(path)?;
sanitize_symlink_target(&rel, target)?;
if let Some(parent) = rel.parent() {
create_dir_confined(&root, parent)?;
}
let p = root.join(&rel);
assert_parent_within(&root, &p)?;
remove_existing(&p);
symlink(target, &p)?;
}
}
for e in entries.iter().rev() {
if let Entry::Dir { path, mode, mtime } = e
&& let Ok(rel) = sanitize_rel_path(path)
{
let p = root.join(rel);
if *mtime > 0
&& let Ok(f) = std::fs::File::open(&p)
{
set_mtime(&f, *mtime);
}
set_mode(&p, *mode);
}
}
Ok(())
}
fn remove_existing(p: &Path) {
if let Ok(meta) = std::fs::symlink_metadata(p) {
if meta.is_dir() {
let _ = std::fs::remove_dir_all(p);
} else {
let _ = std::fs::remove_file(p);
}
}
}
#[cfg(unix)]
fn set_mode(path: &Path, mode: u32) {
if mode != 0 {
use std::os::unix::fs::PermissionsExt;
let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode));
}
}
#[cfg(not(unix))]
fn set_mode(_path: &Path, _mode: u32) {}
fn set_mtime(f: &std::fs::File, mtime: i64) {
if mtime > 0 {
let t = std::time::UNIX_EPOCH + Duration::from_secs(mtime as u64);
let _ = f.set_modified(t);
}
}
#[cfg(unix)]
fn symlink(target: &str, path: &Path) -> std::io::Result<()> {
std::os::unix::fs::symlink(target, path)
}
#[cfg(not(unix))]
fn symlink(_target: &str, _path: &Path) -> std::io::Result<()> {
Err(std::io::Error::other(
"symlink entries are not supported on this platform",
))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::store::MemoryStore;
#[test]
fn build_dedups_identical_chunks() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("a.txt"), b"same").unwrap();
std::fs::write(dir.path().join("b.txt"), b"same").unwrap();
let store = MemoryStore::new();
let index = build_tree(dir.path(), "snap", &CdcParams::default(), &store).unwrap();
assert_eq!(store.len(), 1);
assert_eq!(index.needed_chunks().len(), 1);
index.validate().unwrap();
assert_eq!(index.file_count(), 2);
}
#[test]
fn root_hash_ignores_mtime() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("f.txt"), b"content").unwrap();
let store = MemoryStore::new();
let a = build_tree(dir.path(), "snap", &CdcParams::default(), &store).unwrap();
let file = std::fs::File::options()
.append(true)
.open(dir.path().join("f.txt"))
.unwrap();
file.set_modified(std::time::SystemTime::now() + Duration::from_secs(1000))
.unwrap();
let b = build_tree(dir.path(), "snap", &CdcParams::default(), &store).unwrap();
assert_eq!(a.root_hash, b.root_hash, "mtime must not change identity");
std::fs::write(dir.path().join("f.txt"), b"different").unwrap();
let c = build_tree(dir.path(), "snap", &CdcParams::default(), &store).unwrap();
assert_ne!(a.root_hash, c.root_hash);
}
#[test]
fn traversal_paths_rejected_by_validate() {
let evil_paths = ["../evil", "/etc/passwd", "a/../../b"];
for evil in evil_paths {
let entries = vec![Entry::File {
path: evil.into(),
mode: 0,
mtime: 0,
size: 0,
chunks: vec![],
}];
let root_hash = root_digest(&entries).unwrap();
let index = TreeIndex {
version: WIRE_VERSION,
id: "x".into(),
algo: Hash::ALGO.into(),
cdc: CdcParams::default(),
entries,
root_hash,
};
assert!(index.validate().is_err(), "must reject path {evil:?}");
}
}
#[test]
fn oversized_chunk_len_rejected() {
for bad_len in [0u32, CdcParams::default().max + 1, u32::MAX] {
let entries = vec![Entry::File {
path: "f".into(),
mode: 0,
mtime: 0,
size: bad_len as u64,
chunks: vec![ChunkRef {
hash: Hash::of(b"x"),
len: bad_len,
}],
}];
let root_hash = root_digest(&entries).unwrap();
let index = TreeIndex {
version: WIRE_VERSION,
id: "x".into(),
algo: Hash::ALGO.into(),
cdc: CdcParams::default(),
entries,
root_hash,
};
assert!(index.validate().is_err(), "len {bad_len} must be rejected");
}
}
#[test]
fn size_chunk_mismatch_rejected() {
let entries = vec![Entry::File {
path: "f".into(),
mode: 0,
mtime: 0,
size: 100, chunks: vec![],
}];
let root_hash = root_digest(&entries).unwrap();
let index = TreeIndex {
version: WIRE_VERSION,
id: "x".into(),
algo: Hash::ALGO.into(),
cdc: CdcParams::default(),
entries,
root_hash,
};
assert!(index.validate().is_err());
}
#[cfg(unix)]
#[test]
fn hardlinks_detected() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("orig.bin"), b"shared bytes").unwrap();
std::fs::hard_link(dir.path().join("orig.bin"), dir.path().join("link.bin")).unwrap();
let store = MemoryStore::new();
let index = build_tree(dir.path(), "snap", &CdcParams::default(), &store).unwrap();
let hardlinks: Vec<_> = index
.entries
.iter()
.filter(|e| matches!(e, Entry::Hardlink { .. }))
.collect();
assert_eq!(hardlinks.len(), 1);
assert_eq!(index.file_count(), 1);
assert_eq!(store.len(), 1);
}
#[cfg(unix)]
#[test]
fn fifo_is_an_error_not_a_silent_drop() {
use std::os::unix::ffi::OsStrExt;
let dir = tempfile::tempdir().unwrap();
let fifo = dir.path().join("pipe");
let status = std::process::Command::new("mkfifo")
.arg(fifo.as_os_str())
.status();
if !status.map(|s| s.success()).unwrap_or(false) {
return; }
let store = MemoryStore::new();
let err = build_tree(dir.path(), "snap", &CdcParams::default(), &store)
.expect_err("fifo must error");
assert!(err.to_string().contains("unsupported file type"), "{err}");
let _ = fifo.as_os_str().as_bytes();
}
}