use std::collections::{HashMap, HashSet};
use std::io::{Seek, Write};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Instant;
use crate::delta::{Delta, Signature, apply_patch};
use crate::platform::staging::StagedFile;
use crate::protocol::{BatchFile, FileMeta, Frame, LinkKind, LinkSpec, SignatureEntry, SkippedFile, stream};
use crate::security::PathSanitizer;
use crate::sync::executor::{ExecutorOptions, ProgressFn};
use crate::sync::scanner::Manifest;
use crate::sync::stats::SyncStats;
use crate::sync::wire::{file_meta_from_entry, from_peer};
use crate::{Error, Result};
use tokio::io::{AsyncRead, AsyncWrite};
const DEST_CHANGED_MSG: &str = "destination changed after apply; source not removed";
#[expect(clippy::struct_excessive_bools)]
pub(crate) struct Receiver {
root: PathBuf,
sanitizer: PathSanitizer,
fsync: bool,
durable_end: bool,
partial: bool,
backup: bool,
max_delete: Option<u64>,
apply_jobs: usize,
verify: bool,
rollsum: bool,
preserve_times: bool,
archive: bool,
sparse: bool,
xattrs: bool,
preserve_atimes: bool,
progress: Option<ProgressFn>,
}
impl Receiver {
pub(crate) fn new(
root: &Path,
options: &ExecutorOptions,
apply_jobs: usize,
verify: bool,
) -> Result<Self> {
let root = root.canonicalize().map_err(|e| {
Error::Other(format!("failed to canonicalize sync root {}: {e}", root.display()))
})?;
let sanitizer =
PathSanitizer::new(&root).map_err(|e| Error::Other(format!("Path sanitizer: {e}")))?;
Ok(Self {
root,
sanitizer,
fsync: options.fsync,
durable_end: options.remove_source_files || options.fsync,
partial: options.partial,
backup: options.backup,
max_delete: options.max_delete,
apply_jobs: apply_jobs.max(1),
verify,
rollsum: options.rollsum,
preserve_times: options.preserve_times,
archive: options.archive,
sparse: options.sparse,
xattrs: options.xattrs,
preserve_atimes: options.atimes,
progress: options.progress.clone(),
})
}
pub(crate) async fn receive<W: AsyncWrite + Unpin, R: AsyncRead + Unpin>(
&self,
ctrl_send: &mut W,
ctrl_recv: &mut R,
source_files: Vec<FileMeta>,
dest_manifest: &Manifest,
) -> Result<SyncStats> {
let start = Instant::now();
self.send_index_response(ctrl_send, dest_manifest).await?;
let (bytes, files, skipped) = self
.apply_recipes(ctrl_send, ctrl_recv, source_files)
.await?;
#[expect(clippy::cast_possible_truncation)]
let files_received = files as usize;
Ok(SyncStats {
files_sent: 0,
files_received,
bytes_transferred: bytes,
duration: start.elapsed(),
changes: Vec::new(),
skipped,
})
}
async fn send_index_response<W: AsyncWrite + Unpin>(
&self,
ctrl_send: &mut W,
dest_manifest: &Manifest,
) -> Result<()> {
let file_list: Vec<FileMeta> = dest_manifest
.files
.iter()
.map(file_meta_from_entry)
.collect();
stream::send_frame(ctrl_send, &Frame::IndexResponse { file_list }).await?;
Ok(())
}
async fn apply_recipes<W: AsyncWrite + Unpin, R: AsyncRead + Unpin>(
&self,
ctrl_send: &mut W,
ctrl_recv: &mut R,
source_meta: Vec<FileMeta>,
) -> Result<(u64, u64, Vec<SkippedFile>)> {
let files_total = source_meta
.iter()
.filter(|m| m.kind == crate::protocol::FileKind::File)
.count() as u64;
let mut state = ApplyState::new(source_meta, self.apply_jobs);
let outcome: Result<(u64, u64, Vec<SkippedFile>)> = async move {
let loop_result: Result<()> = async {
loop {
let frame = from_peer(stream::receive_frame(ctrl_recv).await?)?;
match frame {
Frame::DeltaRecipe {
file_path,
delta,
source_signature,
basis_path,
..
} => {
self.handle_delta_recipe(
&mut state,
file_path,
delta,
source_signature,
basis_path,
files_total,
)
.await?;
}
Frame::Batch { recipes } => {
self.handle_batch(&mut state, recipes, files_total).await?;
}
Frame::FileStart {
file_id,
file_path,
size,
} => {
self.handle_file_start(&mut state, file_id, &file_path, size)?;
}
Frame::FileChunk { file_id, data, .. } => {
self.handle_file_chunk(&mut state, file_id, data, files_total).await?;
}
Frame::FileEnd { .. } => {
self.handle_file_end(&mut state).await?;
}
Frame::SignatureRequest { paths } => {
self.handle_signature_request(ctrl_send, paths).await?;
}
Frame::MakeDir { paths } => {
self.handle_make_dir(&mut state, paths).await?;
}
Frame::CreateLinks {
links,
hardlinks,
specials,
} => {
self.handle_create_links(&mut state, links, hardlinks, specials)
.await?;
}
Frame::DeleteRequest { paths } => {
self.handle_delete_request(&mut state, paths).await?;
}
Frame::ApplyMeta { paths } => {
self.handle_apply_meta(&mut state, paths).await?;
}
Frame::Done { files, bytes } => {
if self
.handle_done(&mut state, ctrl_send, files, bytes)
.await?
{
break;
}
}
other => {
return Err(Error::Other(format!(
"Unexpected frame in apply loop: {other:?}"
)));
}
}
}
Ok(())
}
.await;
if loop_result.is_err()
&& self.partial
&& let Some(mut f) = state.in_flight.take()
&& let Some(staged) = f.staged
{
if let Some(handle) = f.pending_write.take()
&& let Ok(Ok((file, _hasher))) = handle.await
{
f.file = Some(file);
}
if let Some(file) = &mut f.file {
let _ = file.set_len(f.offset);
}
let _ = std::fs::rename(staged.path(), &f.file_path);
}
while let Some(handle) = state.batch_applies.pop_front() {
let _ = handle.await;
}
for handle in state.applied_this_run.drain().map(|(_, h)| h) {
let _ = handle.await;
}
if !state.dir_metas.is_empty() {
let cfg = self.apply_cfg();
tokio::task::spawn_blocking(move || {
for (path, meta) in state.dir_metas {
let _ = apply_source_meta_sync(&path, &meta, cfg);
}
})
.await
.map_err(|e| Error::Other(format!("Dir meta task panicked: {e}")))?;
}
loop_result.map(|()| (state.bytes_transferred, state.files_received, state.skipped))
}
.await;
outcome
}
async fn handle_delta_recipe(
&self,
state: &mut ApplyState,
file_path: String,
delta: Delta,
source_signature: Option<Signature>,
basis_path: Option<String>,
files_total: u64,
) -> Result<()> {
if let Some(basis) = &basis_path
&& let Some(handle) = state.applied_this_run.remove(basis)
{
handle.await.map_err(|e| {
Error::Other(format!("basis apply task panicked: {e}"))
})??;
}
let path = self.safe_join(&file_path)?;
let basis = basis_path
.map(|p| self.safe_join(&p))
.transpose()?;
let meta = state.meta_map.get(&file_path).cloned();
let permit = state.apply_semaphore.clone().acquire_owned().await;
let cfg = self.apply_cfg();
let progress = self.progress.clone();
let display = file_path;
let map_key = display.clone();
let handle = tokio::spawn(async move {
let _permit = permit;
apply_recipe(
path,
delta,
source_signature,
basis,
meta,
cfg,
display,
progress,
files_total,
)
.await
});
state.applied_this_run.insert(map_key, handle);
Ok(())
}
async fn handle_batch(
&self,
state: &mut ApplyState,
recipes: Vec<BatchFile>,
files_total: u64,
) -> Result<()> {
let mut verified_parents: std::collections::HashSet<PathBuf> =
std::collections::HashSet::new();
let mut items = Vec::with_capacity(recipes.len());
for recipe in recipes {
if let Some(parent) = Path::new(&recipe.file_path)
.parent()
.filter(|p| !p.as_os_str().is_empty())
{
match self.sanitizer.join_preverified(&parent.to_string_lossy()) {
Ok(full_parent) => {
if verified_parents.insert(full_parent.clone())
&& let Err(e) = self.sanitizer.verify_parent(&full_parent)
{
state.skipped.push(SkippedFile::new(
recipe.file_path.clone(),
format!("Unsafe path '{}': {e}", parent.display()),
));
continue;
}
}
Err(e) => {
state.skipped.push(SkippedFile::new(
recipe.file_path.clone(),
format!("Unsafe path '{}': {e}", parent.display()),
));
continue;
}
}
}
let path = match self.sanitizer.join_preverified(&recipe.file_path) {
Ok(path) => path,
Err(e) => {
state.skipped.push(SkippedFile::new(
recipe.file_path.clone(),
format!("Unsafe path '{}': {e}", recipe.file_path),
));
continue;
}
};
let meta = state.meta_map.get(&recipe.file_path).cloned();
items.push((path, recipe.delta, meta, recipe.file_path));
}
let permit = state.apply_semaphore.clone().acquire_owned().await;
let cfg = self.apply_cfg();
let progress = self.progress.clone();
let jobs = self.apply_jobs.clamp(1, 4);
let chunk_size = items.len().div_ceil(jobs).max(1);
let handle = tokio::spawn(async move {
let _permit = permit;
let mut outcomes = Vec::with_capacity(items.len());
let mut tasks = Vec::new();
while !items.is_empty() {
let take = chunk_size.min(items.len());
let chunk: Vec<_> = items.drain(..take).collect();
let progress = progress.clone();
tasks.push(tokio::task::spawn_blocking(move || {
chunk
.into_iter()
.map(|(path, delta, meta, display)| {
apply_recipe_blocking(
path,
delta,
None,
None,
meta,
cfg,
display,
progress.clone(),
files_total,
)
})
.collect::<Vec<_>>()
}));
}
for task in tasks {
outcomes.extend(
task.await
.map_err(|e| Error::Other(format!("Batch apply task panicked: {e}")))?,
);
}
Ok(outcomes)
});
state.batch_applies.push_back(handle);
while state.batch_applies.len() > BATCH_APPLY_SLOTS {
let prev = state.batch_applies.pop_front().expect("len > slots");
for outcome in prev
.await
.map_err(|e| Error::Other(format!("Batch apply task panicked: {e}")))?
?
{
fold_outcome(state, outcome);
}
}
Ok(())
}
fn handle_file_start(
&self,
state: &mut ApplyState,
file_id: u64,
file_path: &str,
size: u64,
) -> Result<()> {
if state.in_flight.is_some() {
return Err(Error::Other(
"FileStart while another file is in flight".to_string(),
));
}
let path = self.safe_join(file_path)?;
let display = file_path.to_string();
let meta = state.meta_map.get(file_path).cloned();
let sparse = self.sparse;
let staged = match StagedFile::new(&path).and_then(|s| s.prepare(size, sparse).map(|()| s)) {
Ok(staged) => Some(staged),
Err(e) => {
tracing::warn!("skipping {}: {e}", path.display());
state.skipped.push(SkippedFile::new(file_path.to_string(), e.to_string()));
None
}
};
let file = staged
.as_ref()
.and_then(|s| s.open().ok())
.map(|f| SparseWriter::new(f, size, sparse));
let failed = staged.is_none();
state.in_flight = Some(ChunkedFile {
file_id,
staged,
file,
pending_write: None,
write_slots: std::sync::Arc::new(tokio::sync::Semaphore::new(WRITE_QUEUE_DEPTH)),
offset: 0,
file_path: path,
display,
hasher: self.verify.then(blake3::Hasher::new),
meta,
size,
failed,
});
Ok(())
}
async fn handle_file_chunk(
&self,
state: &mut ApplyState,
file_id: u64,
data: Vec<u8>,
files_total: u64,
) -> Result<()> {
let Some(f) = &mut state.in_flight else {
return Err(Error::Other("FileChunk without FileStart".to_string()));
};
if f.file_id != file_id {
return Err(Error::Other("FileChunk file_id mismatch".to_string()));
}
if f.failed {
return Ok(()); }
let permit = f.write_slots.clone().acquire_owned().await;
let prev = f.pending_write.take();
let file = f.file.take();
let hasher = f.hasher.take();
let n = data.len() as u64;
f.offset += n;
if let Some(report) = &self.progress {
report(&f.display, f.offset, f.size, files_total);
}
f.pending_write = Some(tokio::task::spawn_blocking(move || {
let _permit = permit;
let (mut file, mut hasher) = match prev {
Some(prev) => {
let (file, hasher) = tokio::runtime::Handle::current()
.block_on(prev)
.map_err(|e| {
std::io::Error::other(format!("chunk write task panicked: {e}"))
})??;
(file, hasher)
}
None => (
file.ok_or_else(|| {
std::io::Error::other("FileChunk for unopenable staged file")
})?,
hasher,
),
};
file.write_all(&data)?;
if let Some(h) = &mut hasher {
h.update(&data);
}
Ok((file, hasher))
}));
Ok(())
}
async fn handle_file_end(&self, state: &mut ApplyState) -> Result<()> {
let Some(mut f) = state.in_flight.take() else {
return Err(Error::Other("FileEnd without FileStart".to_string()));
};
if f.failed {
return Ok(()); }
let Some(staged) = f.staged else {
return Ok(());
};
if let Some(handle) = f.pending_write.take() {
let (file, hasher) = handle
.await
.map_err(|e| Error::Other(format!("Chunk write task panicked: {e}")))??;
f.file = Some(file);
f.hasher = hasher;
}
let sparse_writer = f.file;
let cfg = self.apply_cfg();
let path = f.file_path;
let rel = f.display;
let hash = f.hasher.map(|h| *h.finalize().as_bytes());
let meta = f.meta;
let written = f.offset;
let parent = path
.parent()
.map_or_else(|| path.clone(), Path::to_path_buf);
let durable = tokio::task::spawn_blocking(move || -> Result<bool> {
if let Some(mut writer) = sparse_writer {
writer.finish()?;
}
staged.commit(cfg.fsync || cfg.verify, cfg.backup)?;
if let Some(meta) = &meta {
apply_source_meta_sync(&path, meta, cfg)?;
if cfg.verify {
let stat = std::fs::metadata(&path).map_err(Error::Io)?;
if !dest_still_matches(&stat, meta) {
return Ok(false);
}
}
}
Ok(true)
})
.await
.map_err(|e| Error::Other(format!("Commit task panicked: {e}")))??;
if !durable {
tracing::warn!("destination changed after apply: {rel}; source not removed");
state.skipped.push(SkippedFile::new(rel.clone(), DEST_CHANGED_MSG));
return Ok(());
}
state.renamed_dirs.insert(parent);
state.files_received += 1;
state.bytes_transferred += written;
if let Some(hash) = hash {
state.hashes.insert(rel, hash);
}
Ok(())
}
async fn handle_signature_request<W: AsyncWrite + Unpin>(
&self,
ctrl_send: &mut W,
paths: Vec<String>,
) -> Result<()> {
let jobs = self.apply_jobs.max(1);
let rollsum = self.rollsum;
let semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(jobs));
let mut tasks = Vec::with_capacity(paths.len());
for rel in paths {
let permit = semaphore.clone().acquire_owned().await;
let path = self.safe_join(&rel)?;
tasks.push(tokio::spawn(async move {
let _permit = permit;
(rel, signature_for_path(path, rollsum).await)
}));
}
let mut signatures: Vec<SignatureEntry> = Vec::with_capacity(tasks.len());
for task in tasks {
let (rel, sig) = task
.await
.map_err(|e| Error::Other(format!("Signature task panicked: {e}")))?;
if let Ok(sig) = sig {
signatures.push(SignatureEntry {
file_path: rel,
signature: sig,
});
}
}
stream::send_frame(ctrl_send, &Frame::SignatureResponse { signatures }).await?;
Ok(())
}
async fn handle_make_dir(&self, state: &mut ApplyState, paths: Vec<String>) -> Result<()> {
for rel in paths {
let path = self.safe_join(&rel)?;
let dir = path.clone();
let made =
tokio::task::spawn_blocking(move || crate::platform::staging::make_dir_chain(&dir))
.await
.map_err(|e| Error::Other(format!("MakeDir task panicked: {e}")))?;
match made {
Ok(()) => {
if let Some(meta) = state.meta_map.get(&rel) {
state.dir_metas.push((path.clone(), Arc::clone(meta)));
}
state
.renamed_dirs
.insert(path.parent().map(Path::to_path_buf).unwrap_or(path));
state.files_received += 1;
}
Err(e) => {
tracing::warn!("skipping {}: {e}", path.display());
state.skipped.push(SkippedFile::new(rel.clone(), e.to_string()));
}
}
}
Ok(())
}
async fn handle_create_links(
&self,
state: &mut ApplyState,
links: Vec<LinkSpec>,
hardlinks: Vec<crate::protocol::HardlinkSpec>,
specials: Vec<crate::protocol::SpecialSpec>,
) -> Result<()> {
drain_applies(state).await?;
let cfg = self.apply_cfg();
for spec in links {
let path = self.safe_join(&spec.path)?;
let display_path = path.display().to_string();
let meta = state.meta_map.get(&spec.path).cloned();
let made = tokio::task::spawn_blocking(move || match spec.kind {
LinkKind::Symlink => {
create_symlink(&path, &spec.target)?;
restore_link_meta(&path, meta.as_deref(), cfg, true)
}
LinkKind::Lnk => {
create_lnk(&path, &spec.target)?;
restore_link_meta(&path, meta.as_deref(), cfg, false)
}
})
.await
.map_err(|e| Error::Other(format!("CreateLinks task panicked: {e}")))?;
match made {
Ok(()) => state.files_received += 1,
Err(e) => {
tracing::warn!("skipping link {display_path}: {e}");
state.skipped.push(SkippedFile::new(spec.path, e.to_string()));
}
}
}
for spec in specials {
let rel = spec.path.clone();
let kind = spec.kind;
let rdev = spec.rdev;
let path = self.safe_join(&rel)?;
let display_path = path.display().to_string();
let meta = state.meta_map.get(&rel).cloned();
let made = tokio::task::spawn_blocking(move || -> std::io::Result<()> {
crate::platform::fs::create_special(
&path,
kind,
rdev,
meta.as_ref().map_or(0o644, |m| m.mode),
)?;
if let Some(meta) = &meta {
apply_source_meta_sync(&path, meta, cfg)?;
}
Ok(())
})
.await
.map_err(|e| Error::Other(format!("CreateLinks task panicked: {e}")))?;
match made {
Ok(()) => state.files_received += 1,
Err(e) => {
tracing::warn!("skipping special {display_path}: {e}");
state.skipped.push(SkippedFile::new(rel.clone(), e.to_string()));
}
}
}
for spec in hardlinks {
let rel = spec.path.clone();
let target = spec.target.clone();
let path = self.safe_join(&rel)?;
let target_path = self.safe_join(&target)?;
let display_path = path.display().to_string();
let made = tokio::task::spawn_blocking(move || create_hardlink(&path, &target_path))
.await
.map_err(|e| Error::Other(format!("CreateLinks task panicked: {e}")))?;
match made {
Ok(()) => state.files_received += 1,
Err(e) => {
tracing::warn!("skipping hard link {display_path}: {e}");
state.skipped.push(SkippedFile::new(rel.clone(), e.to_string()));
}
}
}
Ok(())
}
async fn handle_apply_meta(&self, state: &mut ApplyState, paths: Vec<String>) -> Result<()> {
drain_applies(state).await?;
let cfg = self.apply_cfg();
for rel in paths {
let path = match self.safe_join(&rel) {
Ok(p) => p,
Err(e) => {
state
.skipped
.push(SkippedFile::new(rel.clone(), e.to_string()));
continue;
}
};
let Some(meta) = state.meta_map.get(&rel).cloned() else {
state.skipped.push(SkippedFile::new(
rel.clone(),
"no source metadata for metadata-only update".to_string(),
));
continue;
};
let p = path.clone();
match tokio::task::spawn_blocking(move || apply_source_meta_sync(&p, &meta, cfg))
.await
{
Ok(Ok(())) => {
state.files_received += 1;
}
Ok(Err(e)) => {
tracing::warn!("skipping metadata update {}: {e}", path.display());
state
.skipped
.push(SkippedFile::new(rel.clone(), e.to_string()));
}
Err(e) => return Err(Error::Other(format!("Meta task panicked: {e}"))),
}
}
Ok(())
}
async fn handle_delete_request(
&self,
state: &mut ApplyState,
paths: Vec<String>,
) -> Result<()> {
if let Some(max) = self.max_delete
&& paths.len() as u64 > max
{
return Err(Error::Other(format!(
"Refusing to delete {} files: limit is {max} (--max-delete)",
paths.len()
)));
}
drain_applies(state).await?;
for rel in paths {
let path = self.safe_join(&rel)?;
match tokio::fs::symlink_metadata(&path).await {
Ok(meta) if meta.is_dir() => {
let _ = tokio::fs::remove_dir(&path).await;
}
Ok(_) => {
let p = path.clone();
let backup = self.backup;
let removed = tokio::task::spawn_blocking(move || {
if backup {
crate::platform::staging::backup_existing(&p)
} else {
crate::platform::fs::remove_file_any(&p)
}
})
.await
.map_err(|e| Error::Other(format!("Delete task panicked: {e}")))?;
match removed {
Ok(()) => {
prune_empty_dirs(path.parent(), &self.root).await;
}
Err(e) => {
tracing::warn!("skipping delete {}: {e}", path.display());
state.skipped.push(SkippedFile::new(rel.clone(), e.to_string()));
}
}
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(Error::Io(e)),
}
}
Ok(())
}
async fn handle_done<W: AsyncWrite + Unpin>(
&self,
state: &mut ApplyState,
ctrl_send: &mut W,
files: u64,
bytes: u64,
) -> Result<bool> {
tracing::info!("peer done: {files} files, {bytes} bytes");
drain_applies(state).await?;
if self.durable_end {
sync_dirs(&state.renamed_dirs).await;
}
stream::send_frame(
ctrl_send,
&Frame::Ack {
files: state.files_received,
bytes: state.bytes_transferred,
skipped: state.skipped.clone(),
hashes: std::mem::take(&mut state.hashes).into_iter().collect(),
},
)
.await?;
Ok(true)
}
fn safe_join(&self, rel: &str) -> Result<PathBuf> {
self.sanitizer
.join(rel)
.map_err(|e| Error::Other(format!("Unsafe path '{rel}': {e}")))
}
}
async fn signature_for_path(path: PathBuf, rollsum: bool) -> std::io::Result<Signature> {
tokio::task::spawn_blocking(move || {
if let Some(sig) = std::fs::metadata(&path).ok().and_then(|meta| {
crate::sync::sigcache::lookup(
&path,
meta.len(),
crate::platform::fs::mtime_secs(&meta),
crate::platform::fs::mtime_nsecs(&meta),
)
}) {
return Ok(sig);
}
match crate::platform::fs::open_basis(&path) {
Ok(Some(mut file)) => {
let meta = file.metadata().map_err(std::io::Error::other)?;
if rollsum {
let block = crate::delta::rollsum::block_size(meta.len());
Signature::generate_rollsum(&mut file, block)
.map_err(|e| std::io::Error::other(e.to_string()))
} else {
Signature::generate(&mut file)
.map_err(|e| std::io::Error::other(e.to_string()))
}
}
Ok(None) => Ok(Signature::new(0)),
Err(e) => Err(e),
}
})
.await
.map_err(|e| std::io::Error::other(format!("Signature task panicked: {e}")))?
}
struct ApplyState {
bytes_transferred: u64,
files_received: u64,
meta_map: HashMap<String, Arc<FileMeta>>,
apply_semaphore: Arc<tokio::sync::Semaphore>,
dir_metas: Vec<(PathBuf, Arc<FileMeta>)>,
renamed_dirs: HashSet<PathBuf>,
skipped: Vec<SkippedFile>,
hashes: HashMap<String, [u8; 32]>,
applied_this_run: HashMap<String, tokio::task::JoinHandle<Result<ApplyOutcome>>>,
batch_applies: std::collections::VecDeque<tokio::task::JoinHandle<Result<Vec<ApplyOutcome>>>>,
in_flight: Option<ChunkedFile>,
}
impl ApplyState {
fn new(source_meta: Vec<FileMeta>, apply_jobs: usize) -> Self {
let meta_map: HashMap<String, Arc<FileMeta>> = source_meta
.into_iter()
.map(|m| (m.path.clone(), Arc::new(m)))
.collect();
let apply_semaphore = Arc::new(tokio::sync::Semaphore::new(apply_jobs));
Self {
bytes_transferred: 0,
files_received: 0,
meta_map,
apply_semaphore,
dir_metas: Vec::new(),
renamed_dirs: HashSet::new(),
skipped: Vec::new(),
hashes: HashMap::new(),
applied_this_run: HashMap::new(),
batch_applies: std::collections::VecDeque::new(),
in_flight: None,
}
}
}
async fn drain_applies(state: &mut ApplyState) -> Result<()> {
while let Some(handle) = state.batch_applies.pop_front() {
for outcome in handle
.await
.map_err(|e| Error::Other(format!("Batch apply task panicked: {e}")))?
?
{
fold_outcome(state, outcome);
}
}
let handles: Vec<_> = state.applied_this_run.drain().map(|(_, h)| h).collect();
for handle in handles {
fold_outcome(
state,
handle
.await
.map_err(|e| Error::Other(format!("Apply task panicked: {e}")))??,
);
}
Ok(())
}
fn fold_outcome(state: &mut ApplyState, outcome: ApplyOutcome) {
match outcome {
ApplyOutcome::Applied(sent, parent, hash) => {
state.bytes_transferred += sent;
state.files_received += 1;
state.renamed_dirs.insert(parent);
if let Some((rel, hash)) = hash {
state.hashes.insert(rel, hash);
}
}
ApplyOutcome::Skipped(skip_path, reason) => {
tracing::warn!("skipping {skip_path}: {reason}");
state.skipped.push(SkippedFile::new(skip_path, reason));
}
}
}
enum ApplyOutcome {
Applied(u64, PathBuf, Option<(String, [u8; 32])>),
Skipped(String, String),
}
type ChunkWrite = tokio::task::JoinHandle<std::io::Result<(SparseWriter, Option<blake3::Hasher>)>>;
struct ChunkedFile {
file_id: u64,
staged: Option<StagedFile>,
file: Option<SparseWriter>,
pending_write: Option<ChunkWrite>,
write_slots: std::sync::Arc<tokio::sync::Semaphore>,
offset: u64,
file_path: PathBuf,
display: String,
hasher: Option<blake3::Hasher>,
meta: Option<Arc<FileMeta>>,
size: u64,
failed: bool,
}
#[derive(Clone, Copy, Default)]
#[expect(
clippy::struct_excessive_bools,
reason = "the apply flags mirror the CLI one-to-one; bundled once instead of at every site"
)]
struct ApplyCfg {
fsync: bool,
backup: bool,
verify: bool,
preserve_times: bool,
archive: bool,
sparse: bool,
preserve_atimes: bool,
xattrs: bool,
}
impl Receiver {
fn apply_cfg(&self) -> ApplyCfg {
ApplyCfg {
fsync: self.fsync,
backup: self.backup,
verify: self.verify,
preserve_times: self.preserve_times,
archive: self.archive,
sparse: self.sparse,
preserve_atimes: self.preserve_atimes,
xattrs: self.xattrs,
}
}
}
#[expect(clippy::too_many_arguments)]
async fn apply_recipe(
path: PathBuf,
delta: Delta,
source_signature: Option<Signature>,
basis_path: Option<PathBuf>,
meta: Option<Arc<FileMeta>>,
cfg: ApplyCfg,
display_path: String,
progress: Option<ProgressFn>,
files_total: u64,
) -> Result<ApplyOutcome> {
tokio::task::spawn_blocking(move || {
apply_recipe_blocking(
path,
delta,
source_signature,
basis_path,
meta,
cfg,
display_path,
progress,
files_total,
)
})
.await
.map_err(|e| Error::Other(format!("Apply task panicked: {e}")))
}
#[expect(clippy::too_many_arguments)]
#[expect(
clippy::needless_pass_by_value,
reason = "the values arrive owned across the spawn_blocking task boundary"
)]
fn apply_recipe_blocking(
path: PathBuf,
delta: Delta,
source_signature: Option<Signature>,
basis_path: Option<PathBuf>,
meta: Option<Arc<FileMeta>>,
cfg: ApplyCfg,
display_path: String,
progress: Option<ProgressFn>,
files_total: u64,
) -> ApplyOutcome {
let outcome: Result<ApplyOutcome> = (|| {
let basis = if delta.basis_size > 0 {
match crate::platform::fs::open_basis(basis_path.as_deref().unwrap_or(&path)) {
Ok(basis) => basis,
Err(e) => return Err(e.into()),
}
} else {
None
};
let staged = StagedFile::new(&path)?;
staged.prepare(delta.source_size, cfg.sparse)?;
let total = delta.source_size;
let need_hash = delta.checksum.is_some() || cfg.verify;
let mut hasher = need_hash.then(blake3::Hasher::new);
let hash = {
let mut sparse_writer = SparseWriter::new(staged.open()?, total, cfg.sparse);
let hash = if let Some(data) = whole_literal(&delta) {
if let Some(h) = &mut hasher {
h.update(data);
}
match &progress {
Some(report) => {
let mut stream = crate::sync::executor::ProgressStream::new(
&mut sparse_writer,
display_path.clone(),
total,
files_total,
Arc::clone(report),
);
std::io::Write::write_all(&mut stream, data).map_err(Error::Io)?;
}
None => sparse_writer.write_all(data).map_err(Error::Io)?,
}
match hasher {
Some(h) => {
let d = *h.finalize().as_bytes();
if let Some(expected) = delta.checksum
&& d != expected
{
return Err(crate::Error::Delta(
crate::delta::error::DeltaError::ChecksumMismatch {
expected,
actual: d,
},
));
}
d
}
None => [0u8; 32],
}
} else {
match &progress {
Some(report) => {
let mut stream = crate::sync::executor::ProgressStream::new(
&mut sparse_writer,
display_path.clone(),
total,
files_total,
Arc::clone(report),
);
apply_chain(&mut stream, basis, &delta, hasher.as_mut())?
}
None => apply_chain(&mut sparse_writer, basis, &delta, hasher.as_mut())?,
}
};
if let Some(report) = &progress {
report(&display_path, total, total, files_total);
}
sparse_writer.finish()?;
hash
};
staged.commit(cfg.fsync || cfg.verify, cfg.backup)?;
if let Some(meta) = &meta {
apply_source_meta_sync(&path, meta, cfg)?;
if cfg.verify {
let stat = std::fs::metadata(&path).map_err(Error::Io)?;
if !dest_still_matches(&stat, meta) {
return Err(Error::Other(DEST_CHANGED_MSG.to_string()));
}
}
}
if let (Some(sig), Some(meta)) = (source_signature, meta.as_ref())
&& cfg.preserve_times
{
crate::sync::sigcache::store(
&path,
delta.source_size,
meta.mtime,
meta.mtime_nsec,
&sig,
);
}
let parent = path
.parent()
.map_or_else(|| path.clone(), Path::to_path_buf);
let verified = cfg.verify.then_some((display_path.clone(), hash));
Ok(ApplyOutcome::Applied(delta.source_size, parent, verified))
})();
match outcome {
Ok(outcome) => outcome,
Err(e) => {
tracing::warn!("skipping {display_path}: {e}");
ApplyOutcome::Skipped(display_path, e.to_string())
}
}
}
fn whole_literal(delta: &Delta) -> Option<&[u8]> {
if delta.basis_size != 0 || delta.ops.len() != 1 {
return None;
}
match delta.ops.first() {
Some(crate::delta::DeltaOp::Literal(data)) => Some(data),
_ => None,
}
}
fn apply_source_meta_sync(path: &Path, meta: &FileMeta, cfg: ApplyCfg) -> std::io::Result<()> {
let (uid, gid) = (meta.uid, meta.gid);
let pmeta = platform_meta(meta, meta.mode);
crate::platform::fs::apply_meta(path, &pmeta, false, true, cfg.preserve_times, cfg.preserve_atimes)?;
if cfg.archive
&& let Err(e) = crate::platform::fs::apply_owner(path, uid, gid, false)
{
tracing::warn!("chown {}: {e}", path.display());
}
if cfg.xattrs
&& let Some(xattr_list) = &meta.xattrs
&& !xattr_list.is_empty()
&& let Err(e) = crate::platform::fs::apply_xattrs(path, xattr_list)
{
tracing::warn!("xattrs {}: {e}", path.display());
}
Ok(())
}
fn platform_meta(meta: &FileMeta, mode: u32) -> crate::platform::fs::FileMeta {
crate::platform::fs::FileMeta {
size: meta.size,
mode,
mtime_sec: crate::sync::wire::mtime_from_wire(meta.mtime),
mtime_nsec: meta.mtime_nsec,
atime_sec: crate::sync::wire::mtime_from_wire(meta.atime),
atime_nsec: meta.atime_nsec,
}
}
fn restore_link_meta(
path: &Path,
meta: Option<&FileMeta>,
cfg: ApplyCfg,
symlink: bool,
) -> std::io::Result<()> {
#[cfg(not(unix))]
if symlink {
let _ = (path, meta, cfg);
return Ok(());
}
if let Some(meta) = meta {
let (uid, gid) = (meta.uid, meta.gid);
let meta = platform_meta(meta, 0);
crate::platform::fs::apply_meta(path, &meta, symlink, false, cfg.preserve_times, cfg.preserve_atimes)?;
if cfg.archive
&& let Err(e) = crate::platform::fs::apply_owner(path, uid, gid, symlink)
{
tracing::warn!("chown {}: {e}", path.display());
}
}
Ok(())
}
fn create_symlink(path: &Path, target: &str) -> std::io::Result<()> {
#[cfg(unix)]
{
create_then_install(path, |tmp| std::os::unix::fs::symlink(target, tmp))
}
#[cfg(not(unix))]
{
let _ = (path, target);
Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"symlink creation is unsupported on this target; the sender should have sent a .lnk or content",
))
}
}
fn create_lnk(path: &Path, target: &str) -> std::io::Result<()> {
let win_target = target.replace('/', "\\");
create_then_install(path, |tmp| {
let mut link = lnk::ShellLink::default().with_encoding(&lnk::StringEncoding::Unicode);
link.set_relative_path(Some(win_target.clone()));
link.save(tmp)
.map_err(|e| std::io::Error::other(e.to_string()))
})
}
fn create_hardlink(path: &Path, target: &Path) -> std::io::Result<()> {
create_then_install(path, |tmp| std::fs::hard_link(target, tmp))
}
fn create_then_install<F>(path: &Path, make: F) -> std::io::Result<()>
where
F: FnOnce(&Path) -> std::io::Result<()>,
{
let staged = StagedFile::new(path)?;
make(staged.path())?;
staged.commit(false, false)
}
async fn sync_dirs(dirs: &HashSet<PathBuf>) {
if dirs.is_empty() {
return;
}
let dirs: Vec<PathBuf> = dirs.iter().cloned().collect();
let _ = tokio::task::spawn_blocking(move || {
for dir in &dirs {
let _ = crate::platform::fs::sync_dir(dir);
}
})
.await;
}
async fn prune_empty_dirs(dir: Option<&Path>, root: &Path) {
let mut current = dir.map(Path::to_path_buf);
while let Some(path) = current {
if path == root {
break;
}
let Ok(mut entries) = tokio::fs::read_dir(&path).await else {
return;
};
if entries.next_entry().await.ok().flatten().is_some() {
return;
}
let _ = tokio::fs::remove_dir(&path).await;
current = path.parent().map(Path::to_path_buf);
}
}
fn apply_chain<W: std::io::Write>(
out: W,
basis: Option<std::fs::File>,
delta: &Delta,
mut hasher: Option<&mut blake3::Hasher>,
) -> crate::Result<[u8; 32]> {
match basis {
Some(mut basis) => apply_patch(&mut basis, delta, out, hasher.as_deref_mut())?,
None => apply_patch(std::io::Cursor::new(&[][..]), delta, out, hasher.as_deref_mut())?,
}
Ok(match hasher {
Some(h) => *h.finalize().as_bytes(),
None => [0u8; 32],
})
}
const SPARSE_RUN_MIN: usize = 4096;
const WRITE_QUEUE_DEPTH: usize = 32;
const BATCH_APPLY_SLOTS: usize = 2;
struct SparseWriter {
inner: std::fs::File,
enabled: bool,
total: u64,
zeros_pending: usize,
}
impl SparseWriter {
fn new(inner: std::fs::File, total: u64, enabled: bool) -> Self {
Self {
inner,
enabled,
total,
zeros_pending: 0,
}
}
fn finish(&mut self) -> std::io::Result<()> {
self.flush_zeros()?;
self.inner.set_len(self.total)
}
fn set_len(&mut self, size: u64) -> std::io::Result<()> {
self.zeros_pending = 0;
self.inner.set_len(size)
}
fn flush_zeros(&mut self) -> std::io::Result<()> {
if self.zeros_pending > 0 {
self.inner.write_all(&vec![0u8; self.zeros_pending])?;
self.zeros_pending = 0;
}
Ok(())
}
fn buffer_zeros(&mut self, n: usize) -> std::io::Result<()> {
self.zeros_pending += n;
if self.zeros_pending >= SPARSE_RUN_MIN {
self.inner
.seek(std::io::SeekFrom::Current(i64::try_from(self.zeros_pending).map_err(
|_| std::io::Error::other("zero run exceeds file offset range"),
)?))?;
self.zeros_pending = 0;
}
Ok(())
}
}
impl std::io::Write for SparseWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if !self.enabled {
return self.inner.write(buf);
}
let mut i = 0;
while i < buf.len() {
if buf[i] == 0 {
let start = i;
while i < buf.len() && buf[i] == 0 {
i += 1;
}
self.buffer_zeros(i - start)?;
} else {
let start = i;
while i < buf.len() && buf[i] != 0 {
i += 1;
}
self.flush_zeros()?;
self.inner.write_all(&buf[start..i])?;
}
}
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
self.flush_zeros()
}
}
fn dest_still_matches(stat: &std::fs::Metadata, meta: &FileMeta) -> bool {
stat.len() == meta.size
&& crate::platform::fs::mtime_secs(stat) == meta.mtime
&& crate::platform::fs::mtime_nsecs(stat) == meta.mtime_nsec
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn dest_still_matches_compares_size_and_mtime() {
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("img.fits");
let meta = FileMeta {
path: "img.fits".to_string(),
size: 5,
mtime: 1_700_000_000,
mtime_nsec: 0,
mode: 0o644,
hash: None,
kind: crate::protocol::FileKind::File,
link_target: None,
inode: None,
rdev: None,
uid: None,
gid: None,
atime: 0,
atime_nsec: 0,
xattrs: None,
};
std::fs::write(&file, b"12345").unwrap();
let set = filetime_set(&file, meta.mtime);
assert!(set.is_ok(), "{set:?}");
let stat = std::fs::metadata(&file).unwrap();
assert!(dest_still_matches(&stat, &meta), "matching meta must pass");
std::fs::write(&file, b"54321").unwrap();
let stat = std::fs::metadata(&file).unwrap();
assert!(
!dest_still_matches(&stat, &meta),
"a post-apply modification must fail the check"
);
}
#[allow(clippy::cast_possible_wrap)]
fn filetime_set(path: &std::path::Path, mtime: u64) -> std::io::Result<()> {
#[cfg(unix)]
{
use std::os::unix::ffi::OsStrExt;
let c_path = std::ffi::CString::new(path.as_os_str().as_bytes())
.map_err(|_| std::io::Error::other("nul in path"))?;
let times = [
libc::timespec {
tv_sec: mtime as libc::time_t,
tv_nsec: 0,
},
libc::timespec {
tv_sec: mtime as libc::time_t,
tv_nsec: 0,
},
];
let rc = unsafe { libc::utimensat(libc::AT_FDCWD, c_path.as_ptr(), times.as_ptr(), 0) };
if rc == 0 {
Ok(())
} else {
Err(std::io::Error::last_os_error())
}
}
#[cfg(not(unix))]
{
let time = std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(mtime);
std::fs::File::options()
.write(true)
.open(path)
.and_then(|f| f.set_times(std::fs::FileTimes::new().set_modified(time)))
}
}
fn cp2_temps(dir: &Path) -> Vec<String> {
std::fs::read_dir(dir)
.unwrap()
.filter_map(std::result::Result::ok)
.map(|e| e.file_name().to_string_lossy().into_owned())
.filter(|n| n.contains(".cp2."))
.collect()
}
#[cfg(unix)]
#[test]
fn create_symlink_replaces_file_atomically() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("link.txt");
std::fs::write(&path, b"old").unwrap();
create_symlink(&path, "target.txt").unwrap();
assert!(path.is_symlink());
assert_eq!(
std::fs::read_link(&path).unwrap(),
std::path::Path::new("target.txt")
);
assert!(cp2_temps(dir.path()).is_empty(), "no staging temps may remain");
}
#[cfg(unix)]
#[test]
fn create_symlink_replaces_directory() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("thing");
std::fs::create_dir(&path).unwrap();
std::fs::write(path.join("inner.txt"), b"x").unwrap();
create_symlink(&path, "target.txt").unwrap();
assert!(path.is_symlink());
assert!(cp2_temps(dir.path()).is_empty());
}
#[test]
fn create_then_install_failure_keeps_previous_occupant() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("thing");
std::fs::write(&path, b"old").unwrap();
let err = create_then_install(&path, |tmp| {
std::fs::write(tmp, b"partial").unwrap();
Err(std::io::Error::other("boom"))
});
assert!(err.is_err());
assert_eq!(std::fs::read(&path).unwrap(), b"old");
assert!(cp2_temps(dir.path()).is_empty());
}
#[test]
fn restore_lnk_meta_applies_source_mtime() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("shortcut.lnk");
create_lnk(&path, "..\\sub\\target.txt").unwrap();
let meta = FileMeta {
path: "shortcut.lnk".to_string(),
size: 0,
mtime: 1_600_000_000,
mtime_nsec: 0,
mode: 0,
hash: None,
kind: crate::protocol::FileKind::Symlink,
link_target: Some("..\\sub\\target.txt".to_string()),
inode: None,
rdev: None,
uid: None,
gid: None,
atime: 0,
atime_nsec: 0,
xattrs: None,
};
restore_link_meta(
&path,
Some(&meta),
ApplyCfg {
preserve_times: true,
..ApplyCfg::default()
},
false,
)
.unwrap();
let mtime = std::fs::metadata(&path)
.unwrap()
.modified()
.unwrap()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs();
assert_eq!(mtime, 1_600_000_000);
}
#[test]
fn sparse_writer_turns_long_zero_runs_into_holes() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("img");
let file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(&path)
.unwrap();
let mut w = SparseWriter::new(file, 1_048_576, true);
let mut content = vec![0u8; 1_048_576];
content[..4].copy_from_slice(b"head");
content[1_048_572..].copy_from_slice(b"tail");
w.write_all(&content).unwrap();
w.finish().unwrap();
let meta = std::fs::metadata(&path).unwrap();
assert_eq!(meta.len(), 1_048_576, "the logical size is exact");
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let blocks = meta.blocks() * 512;
assert!(
blocks < 1_048_576 / 4,
"a 1 MiB hole must not be allocated ({blocks} bytes allocated)"
);
}
let content = std::fs::read(&path).unwrap();
assert_eq!(&content[..4], b"head");
assert_eq!(&content[content.len() - 4..], b"tail");
assert!(content[4..content.len() - 4].iter().all(|&b| b == 0));
}
#[test]
fn sparse_writer_flushes_pending_zeros_before_data() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("mixed");
let file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(&path)
.unwrap();
let mut w = SparseWriter::new(file, 4096 + 8, true);
w.write_all(&vec![0u8; 4096]).unwrap();
w.write_all(b"payload").unwrap();
w.finish().unwrap();
let content = std::fs::read(&path).unwrap();
assert_eq!(content.len(), 4096 + 8);
assert!(content[..4096].iter().all(|&b| b == 0));
assert_eq!(&content[4096..4096 + 7], b"payload");
assert_eq!(content[4096 + 7], 0, "the last byte is finish() padding");
}
#[test]
fn sparse_writer_finish_pads_trailing_hole() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("tail");
let file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(&path)
.unwrap();
let mut w = SparseWriter::new(file, 8192, true);
w.write_all(b"abc").unwrap();
w.finish().unwrap();
let meta = std::fs::metadata(&path).unwrap();
assert_eq!(meta.len(), 8192);
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
assert!(
meta.blocks() * 512 < 8192,
"the trailing hole must stay unallocated"
);
}
}
#[test]
fn sparse_writer_disabled_is_passthrough() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("plain");
let file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(&path)
.unwrap();
let mut w = SparseWriter::new(file, 8192, false);
w.write_all(&vec![0u8; 8192]).unwrap();
w.finish().unwrap();
let content = std::fs::read(&path).unwrap();
assert_eq!(content.len(), 8192);
assert!(content.iter().all(|&b| b == 0));
}
}