use std::collections::HashSet;
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::path::{Component, Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};
use crate::platform::storage::{StorageClass, StoragePreference};
use crate::protocol::{FileMeta, Frame, TargetOs, stream};
use crate::security::PathSanitizer;
use crate::sync::handshake;
use crate::sync::planner::{Planner, PlannerConfig};
use crate::sync::receiver::Receiver;
use crate::sync::scanner::{Manifest, ScanOptions, Scanner};
use crate::sync::sender::Sender;
use crate::sync::stats::{ItemizeAction, ItemizeEntry, SyncStats};
use crate::sync::strategy::optimal_thread_count;
use crate::sync::wire::{file_meta_from_entry, from_peer, manifest_from_file_meta};
use crate::{Error, Result};
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite};
pub type ProgressFn = Arc<dyn Fn(&str, u64, u64, u64) + Send + Sync>;
pub(crate) struct ProgressStream<T> {
inner: T,
path: String,
total: u64,
files_total: u64,
done: u64,
report: ProgressFn,
}
impl<T> ProgressStream<T> {
#[must_use]
pub(crate) fn new(
inner: T,
path: String,
total: u64,
files_total: u64,
report: ProgressFn,
) -> Self {
Self {
inner,
path,
total,
files_total,
done: 0,
report,
}
}
}
impl<T: std::io::Read> std::io::Read for ProgressStream<T> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let n = self.inner.read(buf)?;
if n > 0 {
self.done += n as u64;
(self.report)(&self.path, self.done, self.total, self.files_total);
}
Ok(n)
}
}
impl<T: std::io::Write> std::io::Write for ProgressStream<T> {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let n = self.inner.write(buf)?;
self.done += n as u64;
(self.report)(&self.path, self.done, self.total, self.files_total);
Ok(n)
}
fn flush(&mut self) -> std::io::Result<()> {
self.inner.flush()
}
}
#[derive(Clone)]
#[expect(clippy::struct_excessive_bools)]
pub struct ExecutorOptions {
pub checksum: bool,
pub delete: bool,
pub update_only: bool,
pub ignore_existing: bool,
pub existing: bool,
pub ignore_times: bool,
pub max_delete: Option<u64>,
pub delete_scope: Vec<String>,
pub backup: bool,
pub jobs: Option<usize>,
pub storage: StoragePreference,
pub compress: bool,
pub bwlimit: Option<u64>,
pub exclude: Vec<String>,
pub include: Vec<String>,
pub itemize: bool,
pub fsync: bool,
pub remote_path: String,
pub remote_paths: Vec<String>,
pub partial: bool,
pub rollsum: bool,
pub quiet: bool,
pub remove_source_files: bool,
pub verify: bool,
pub archive: bool,
pub literal_links: bool,
pub literal_internal_links: bool,
pub literal_external_file_links: bool,
pub literal_external_dir_links: bool,
pub recursive: bool,
pub preserve_links: bool,
pub follow_links: bool,
pub sparse: bool,
pub xattrs: bool,
pub atimes: bool,
pub preserve_perms: bool,
pub preserve_times: bool,
pub target_os: TargetOs,
pub progress: Option<ProgressFn>,
pub include_root_component: bool,
}
fn scanner_options(
options: &ExecutorOptions,
root: &Path,
filter: Option<crate::sync::FilterSet>,
is_source_scan: bool,
no_perms: bool,
source_link_paths: Option<std::collections::HashSet<String>>,
) -> ScanOptions {
ScanOptions {
hash: options.checksum,
hash_workers: resolve_hash_workers(options, root),
filter,
recursive: options.recursive,
preserve_links: options.preserve_links,
follow_links: options.follow_links,
literal_links: options.literal_links,
literal_internal_links: options.literal_internal_links,
literal_external_file_links: options.literal_external_file_links,
literal_external_dir_links: options.literal_external_dir_links,
target_os: options.target_os,
is_source_scan,
no_perms,
source_os: crate::sync::scanner::local_os(),
source_link_paths,
archive: options.archive,
xattrs: options.xattrs,
include_root_component: is_source_scan && options.include_root_component,
}
}
fn pull_request(options: &ExecutorOptions, watch: bool, watch_delay_ms: u32) -> Frame {
let (paths, include_root) = if options.remote_paths.is_empty() {
(vec![options.remote_path.clone()], crate::sync::scanner::include_root_component(&options.remote_path))
} else {
(options.remote_paths.clone(), false)
};
Frame::PullRequest {
paths,
excludes: options.exclude.clone(),
includes: options.include.clone(),
checksum: options.checksum,
delete: options.delete,
delete_scope: options.delete_scope.clone(),
update_only: options.update_only,
ignore_existing: options.ignore_existing,
existing: options.existing,
ignore_times: options.ignore_times,
watch,
watch_delay_ms,
compress: options.compress,
bwlimit: options.bwlimit,
client_os: options.target_os,
include_root,
}
}
impl Default for ExecutorOptions {
fn default() -> Self {
Self {
checksum: false,
delete: false,
update_only: false,
ignore_existing: false,
existing: false,
ignore_times: false,
max_delete: None,
delete_scope: Vec::new(),
backup: false,
jobs: None,
storage: StoragePreference::Auto,
compress: false,
bwlimit: None,
exclude: Vec::new(),
include: Vec::new(),
itemize: false,
fsync: false,
remote_path: String::new(),
remote_paths: Vec::new(),
partial: true,
rollsum: false,
quiet: false,
remove_source_files: false,
verify: false,
archive: false,
literal_links: false,
literal_internal_links: false,
literal_external_file_links: false,
literal_external_dir_links: false,
recursive: true,
preserve_links: true,
follow_links: false,
sparse: false,
xattrs: false,
atimes: false,
preserve_perms: true,
preserve_times: true,
target_os: TargetOs::Unix,
progress: None,
include_root_component: false,
}
}
}
pub struct Executor {
send: Box<dyn AsyncWrite + Unpin + Send>,
recv: Box<dyn AsyncRead + Unpin + Send>,
}
impl Executor {
#[must_use]
pub fn new(
send: Box<dyn AsyncWrite + Unpin + Send>,
recv: Box<dyn AsyncRead + Unpin + Send>,
) -> Self {
Self { send, recv }
}
pub async fn push(&mut self, local: &Path, options: &ExecutorOptions) -> Result<SyncStats> {
let source_manifest = scan_tree(local, options, true).await?;
self.push_scanned(&source_manifest.root, &source_manifest, options)
.await
}
pub async fn push_multi(
&mut self,
base: &Path,
roots: &[PathBuf],
options: &ExecutorOptions,
) -> Result<SyncStats> {
let scanner = Scanner::new(scanner_options(
options,
base,
Some(crate::sync::FilterSet {
includes: options.include.clone(),
excludes: options.exclude.clone(),
}),
true,
!options.preserve_perms,
None,
));
let source_manifest = scanner.scan_multi(base, roots).await?;
self.push_scanned(base, &source_manifest, options).await
}
async fn push_scanned(
&mut self,
source_root: &Path,
source_manifest: &Manifest,
options: &ExecutorOptions,
) -> Result<SyncStats> {
tracing::info!(
"push scan: {} files, {} bytes",
source_manifest.len(),
source_manifest.total_bytes
);
let mut ctrl_send = self.send.as_mut();
let mut ctrl_recv = self.recv.as_mut();
handshake::client(&mut ctrl_send, &mut ctrl_recv).await?;
let sender = Sender::new(
options.bwlimit,
options.progress.clone(),
resolve_apply_jobs(options, source_root),
options.remove_source_files || options.verify,
options.rollsum,
options.itemize,
);
let stats = sender
.send(
&mut ctrl_send,
&mut ctrl_recv,
source_manifest,
source_root,
options,
)
.await?;
Ok(stats)
}
pub async fn pull(&mut self, local: &Path, options: &ExecutorOptions) -> Result<SyncStats> {
let start = Instant::now();
tokio::fs::create_dir_all(local).await.map_err(Error::Io)?;
let mut ctrl_send = self.send.as_mut();
let mut ctrl_recv = self.recv.as_mut();
handshake::client(&mut ctrl_send, &mut ctrl_recv).await?;
stream::send_frame(&mut ctrl_send, &pull_request(options, false, 0)).await?;
let (source_files, verify) = match stream::receive_frame(&mut ctrl_recv).await? {
Frame::IndexRequest {
file_list,
verify,
..
} => (file_list, verify),
Frame::Error { message } => return Err(Error::Other(format!("Peer error: {message}"))),
_ => return Err(Error::Other("Expected IndexRequest from peer".to_string())),
};
let dest_manifest = scan_dest(local, options, &source_files).await?;
let changes = if options.itemize {
let planner = Planner::new(PlannerConfig {
checksum: options.checksum,
delete: options.delete,
delete_scope: (!options.delete_scope.is_empty())
.then(|| options.delete_scope.clone()),
update_only: options.update_only,
ignore_existing: options.ignore_existing,
existing: options.existing,
ignore_times: options.ignore_times,
size_only: !options.preserve_times,
preserve_perms: options.preserve_perms,
preserve_times: options.preserve_times,
});
let source_manifest = manifest_from_file_meta(&source_files);
planner.plan(&source_manifest, &dest_manifest).itemize()
} else {
Vec::new()
};
let receiver = Receiver::new(
local,
options,
resolve_apply_jobs(options, local),
verify,
)?;
let mut stats = receiver
.receive(&mut ctrl_send, &mut ctrl_recv, source_files, &dest_manifest)
.await?;
stats.changes = changes;
tracing::info!(
"pull complete: {} files, {} bytes in {:?}",
stats.files_received,
stats.bytes_transferred,
start.elapsed()
);
Ok(stats)
}
pub async fn list(
&mut self,
remote_path: &str,
options: &ExecutorOptions,
) -> Result<SyncStats> {
let start = Instant::now();
let mut ctrl_send = self.send.as_mut();
let mut ctrl_recv = self.recv.as_mut();
handshake::client(&mut ctrl_send, &mut ctrl_recv).await?;
stream::send_frame(
&mut ctrl_send,
&Frame::ListRequest {
path: remote_path.to_string(),
excludes: options.exclude.clone(),
includes: options.include.clone(),
},
)
.await?;
let file_list = match stream::receive_frame(&mut ctrl_recv).await? {
Frame::ListResponse { file_list } => file_list,
Frame::Error { message } => return Err(Error::Other(format!("Peer error: {message}"))),
other => return Err(Error::Other(format!("Expected ListResponse, got {other:?}"))),
};
let changes = file_list
.iter()
.map(|m| {
let kind = crate::sync::stats::kind_letter(m.kind, m.link_target.is_some());
ItemizeEntry::new(ItemizeAction::Skip, m.path.clone(), kind).with_size(m.size)
})
.collect::<Vec<_>>();
Ok(SyncStats {
files_sent: 0,
files_received: 0,
bytes_transferred: 0,
duration: start.elapsed(),
changes,
skipped: Vec::new(),
})
}
pub async fn pull_watch(
&mut self,
local: &Path,
options: &ExecutorOptions,
delay: Duration,
) -> Result<SyncStats> {
let start = Instant::now();
tokio::fs::create_dir_all(local).await.map_err(Error::Io)?;
let mut ctrl_send = self.send.as_mut();
let mut ctrl_recv = self.recv.as_mut();
handshake::client(&mut ctrl_send, &mut ctrl_recv).await?;
stream::send_frame(
&mut ctrl_send,
&pull_request(
options,
true,
u32::try_from(delay.as_millis()).unwrap_or(u32::MAX),
),
)
.await?;
let mut total = SyncStats::default();
loop {
tokio::select! {
frame = stream::receive_frame(&mut ctrl_recv) => {
let frame = from_peer(frame?)?;
match frame {
Frame::IndexRequest {
file_list,
verify,
..
} => {
let dest_manifest = scan_dest(local, options, &file_list).await?;
let receiver = Receiver::new(
local,
options,
resolve_apply_jobs(options, local),
verify,
)?;
let stats = receiver
.receive(&mut ctrl_send, &mut ctrl_recv, file_list, &dest_manifest)
.await?;
total.files_received += stats.files_received;
total.bytes_transferred += stats.bytes_transferred;
tracing::info!(
"watch pull cycle: {} files, {} bytes",
stats.files_received,
stats.bytes_transferred
);
}
Frame::Error { message } => {
return Err(Error::Other(format!("Peer error: {message}")))
}
other => {
return Err(Error::Other(format!(
"Unexpected frame in pull-watch: {other:?}"
)))
}
}
}
_ = tokio::signal::ctrl_c() => {
break;
}
}
}
total.duration = start.elapsed();
Ok(total)
}
pub async fn serve(&mut self, root: &Path, options: &ExecutorOptions) -> Result<SyncStats> {
let start = Instant::now();
let mut ctrl_send = self.send.as_mut();
let mut ctrl_recv = self.recv.as_mut();
handshake::server(&mut ctrl_send, &mut ctrl_recv).await?;
let stats = match stream::receive_frame(&mut ctrl_recv).await? {
Frame::IndexRequest {
file_list,
path,
verify,
} => {
let target = resolve_serve_path(root, &path)?;
tokio::fs::create_dir_all(&target)
.await
.map_err(Error::Io)?;
let dest_manifest = scan_dest(&target, options, &file_list).await?;
let receiver = Receiver::new(
&target,
options,
resolve_apply_jobs(options, &target),
verify,
)?;
receiver
.receive(&mut ctrl_send, &mut ctrl_recv, file_list, &dest_manifest)
.await?
}
Frame::PullRequest {
paths,
excludes,
includes,
checksum,
delete,
delete_scope,
update_only,
ignore_existing,
existing,
ignore_times,
watch,
watch_delay_ms,
compress,
bwlimit,
client_os,
include_root,
} => {
let pull_options = ExecutorOptions {
checksum,
delete,
delete_scope,
update_only,
ignore_existing,
existing,
ignore_times,
compress,
bwlimit,
exclude: excludes,
include: includes,
target_os: client_os,
include_root_component: include_root,
..options.clone()
};
let classic = paths.len() == 1 && !remote_glob_needs_expansion(root, &paths[0])?;
if watch {
let delay = Duration::from_millis(u64::from(watch_delay_ms));
let source_root = resolve_serve_path(root, &paths[0])?;
return serve_watch_pull(
&mut ctrl_send,
&mut ctrl_recv,
&source_root,
&pull_options,
delay,
)
.await;
}
let (source_manifest, manifest_root) = if classic {
let source_root = resolve_serve_path(root, &paths[0])?;
tracing::info!(
"serve pull: {} from {}",
if source_root.is_dir() { "dir" } else { "path" },
source_root.display()
);
let manifest = scan_tree(&source_root, &pull_options, true).await?;
(manifest, source_root)
} else {
let (base, roots) = resolve_pull_roots(root, &paths)?;
tracing::info!(
"serve pull: {} roots under {}",
roots.len(),
base.display()
);
let filter = Some(crate::sync::FilterSet {
includes: pull_options.include.clone(),
excludes: pull_options.exclude.clone(),
});
let scanner = Scanner::new(scanner_options(
&pull_options,
&base,
filter,
true,
!pull_options.preserve_perms,
None,
));
let manifest = scanner.scan_multi(&base, &roots).await?;
(manifest, base)
};
let sender = Sender::new(
pull_options.bwlimit,
options.progress.clone(),
resolve_apply_jobs(&pull_options, &manifest_root),
pull_options.remove_source_files || pull_options.verify,
pull_options.rollsum,
false,
);
let mut stats = sender
.send(
&mut ctrl_send,
&mut ctrl_recv,
&source_manifest,
source_manifest.root.as_path(),
&pull_options,
)
.await?;
stats.duration = start.elapsed();
stats
}
Frame::ListRequest {
path,
excludes,
includes,
} => {
let source_root = resolve_serve_path(root, &path)?;
let list_options = ExecutorOptions {
exclude: excludes,
include: includes,
..options.clone()
};
let manifest = scan_tree(&source_root, &list_options, true).await?;
let file_list = manifest
.files
.iter()
.map(file_meta_from_entry)
.collect::<Vec<_>>();
stream::send_frame(&mut ctrl_send, &Frame::ListResponse { file_list }).await?;
SyncStats {
duration: start.elapsed(),
..SyncStats::default()
}
}
Frame::Error { message } => return Err(Error::Other(format!("Peer error: {message}"))),
other => {
return Err(Error::Other(format!(
"Expected IndexRequest, PullRequest, or ListRequest, got {other:?}"
)));
}
};
tracing::info!("serve complete: {:?} in {:?}", stats, start.elapsed());
Ok(stats)
}
}
pub(crate) async fn scan_tree(root: &Path, options: &ExecutorOptions, apply_filter: bool) -> Result<Manifest> {
let filter = if apply_filter {
Some(crate::sync::FilterSet {
includes: options.include.clone(),
excludes: options.exclude.clone(),
})
} else {
None
};
let scanner = Scanner::new(scanner_options(
options,
root,
filter,
apply_filter,
!options.preserve_perms,
None,
));
scanner.scan(root).await
}
async fn scan_dest(
root: &Path,
options: &ExecutorOptions,
source_files: &[FileMeta],
) -> Result<Manifest> {
let source_links: HashSet<String> = source_files
.iter()
.filter(|m| m.link_target.is_some())
.map(|m| m.path.clone())
.collect();
let scanner = Scanner::new(scanner_options(
options,
root,
None,
false,
false,
(!source_links.is_empty()).then_some(source_links),
));
if options.delete {
return scanner.scan(root).await;
}
if let Ok(mut it) = std::fs::read_dir(root)
&& it.next().is_none()
{
return Ok(Manifest {
root: root.to_path_buf(),
files: Vec::new(),
total_bytes: 0,
skipped: Vec::new(),
});
}
let source = manifest_from_file_meta(source_files);
scanner.scan_targeted(root, &source).await
}
const APPLY_JOBS_HDD: usize = 1;
const APPLY_JOBS_SSD: usize = 16;
fn effective_storage(options: &ExecutorOptions, root: &Path) -> StorageClass {
match options.storage {
StoragePreference::Hdd => StorageClass::Hdd,
StoragePreference::Ssd => StorageClass::Ssd,
StoragePreference::Auto => crate::platform::storage::detect_storage(root),
}
}
fn resolve_hash_workers(options: &ExecutorOptions, root: &Path) -> usize {
match options.jobs {
Some(n) => n.max(1),
None => match effective_storage(options, root) {
StorageClass::Hdd => 1,
_ => optimal_thread_count(true),
},
}
}
fn resolve_apply_jobs(options: &ExecutorOptions, root: &Path) -> usize {
let storage = effective_storage(options, root);
let jobs = match options.jobs {
Some(n) => n.max(1),
None => match storage {
StorageClass::Hdd => APPLY_JOBS_HDD,
_ => APPLY_JOBS_SSD,
},
};
tracing::info!(
"apply window {jobs} for {} (storage {storage:?}, explicit jobs {:?})",
root.display(),
options.jobs
);
jobs
}
async fn serve_watch_pull<W: AsyncWrite + Unpin, R: AsyncRead + Unpin>(
ctrl_send: &mut W,
ctrl_recv: &mut R,
source_root: &Path,
options: &ExecutorOptions,
delay: Duration,
) -> Result<SyncStats> {
let (tx, mut changes) = tokio::sync::mpsc::unbounded_channel::<()>();
let _watcher = crate::sync::watcher::start_watcher(source_root, tx)
.map_err(|e| Error::Other(format!("Failed to watch source: {e}")))?;
let start = Instant::now();
let mut total = SyncStats::default();
loop {
let source_manifest = loop {
match scan_tree(source_root, options, true).await {
Ok(manifest) => break manifest,
Err(e) => {
tracing::warn!(
"watch pull scan failed: {e}; retrying in {:?}",
crate::sync::watcher::SYNC_ERROR_BACKOFF
);
if wait_for_disconnect(ctrl_recv).await? {
total.duration = start.elapsed();
return Ok(total);
}
}
}
};
let sender = Sender::new(
options.bwlimit,
options.progress.clone(),
resolve_apply_jobs(options, source_root),
options.remove_source_files || options.verify,
options.rollsum,
false,
);
let stats = sender
.send(
ctrl_send,
ctrl_recv,
&source_manifest,
source_manifest.root.as_path(),
options,
)
.await?;
total.files_sent += stats.files_sent;
total.bytes_transferred += stats.bytes_transferred;
tracing::info!(
"watch pull cycle: served {} files, {} bytes",
stats.files_sent,
stats.bytes_transferred
);
match wait_for_changes(ctrl_recv, &mut changes, delay).await? {
WatchWait::Changed => {}
WatchWait::Disconnected => break,
}
}
total.duration = start.elapsed();
Ok(total)
}
async fn wait_for_disconnect<R: AsyncRead + Unpin>(ctrl_recv: &mut R) -> Result<bool> {
let mut probe = [0u8; 64];
tokio::select! {
() = tokio::time::sleep(crate::sync::watcher::SYNC_ERROR_BACKOFF) => Ok(false),
n = ctrl_recv.read(&mut probe) => match n {
Ok(0) => Ok(true), Ok(_) => Err(Error::Other(
"Unexpected data from client during scan retry".to_string(),
)),
Err(e) => Err(Error::Io(e)),
},
}
}
enum WatchWait {
Changed,
Disconnected,
}
async fn wait_for_changes<R: AsyncRead + Unpin>(
ctrl_recv: &mut R,
changes: &mut tokio::sync::mpsc::UnboundedReceiver<()>,
delay: Duration,
) -> Result<WatchWait> {
let mut dirty = false;
while changes.try_recv().is_ok() {
dirty = true;
}
if dirty {
return Ok(WatchWait::Changed);
}
let outcome = crate::sync::watcher::wait_debounce(changes, delay, watch_disconnect(ctrl_recv))
.await?;
Ok(match outcome {
crate::sync::watcher::BurstWait::Ready => WatchWait::Changed,
crate::sync::watcher::BurstWait::ChannelClosed
| crate::sync::watcher::BurstWait::Aborted => WatchWait::Disconnected,
})
}
async fn watch_disconnect<R: AsyncRead + Unpin>(ctrl_recv: &mut R) -> Result<()> {
let mut probe = [0u8; 64];
match ctrl_recv.read(&mut probe).await {
Ok(0) => Ok(()), Ok(_) => Err(Error::Other(
"Unexpected data from client during watch wait".to_string(),
)),
Err(e) => Err(Error::Io(e)),
}
}
fn resolve_serve_path(root: &Path, path: &str) -> Result<std::path::PathBuf> {
if path.is_empty() {
return Ok(root.to_path_buf());
}
let p = Path::new(path);
match p.components().next() {
Some(std::path::Component::RootDir | std::path::Component::Prefix(_)) => {
if p.components()
.any(|c| c == std::path::Component::ParentDir)
{
return Err(Error::Other(format!(
"Unsafe path '{path}': .. is not allowed"
)));
}
Ok(p.to_path_buf())
}
_ => sanitize_join(root, path),
}
}
fn sanitize_join(root: &Path, rel: &str) -> Result<std::path::PathBuf> {
let sanitizer =
PathSanitizer::new(root).map_err(|e| Error::Other(format!("Path sanitizer: {e}")))?;
sanitizer
.join(rel)
.map_err(|e| Error::Other(format!("Unsafe path '{rel}': {e}")))
}
fn has_glob_metachars(s: &str) -> bool {
s.contains(['*', '?', '['])
}
fn remote_glob_needs_expansion(root: &Path, pattern: &str) -> Result<bool> {
if !has_glob_metachars(pattern) {
return Ok(false);
}
let literal = resolve_serve_path(root, pattern)?;
Ok(!literal.exists())
}
fn resolve_pull_roots(root: &Path, paths: &[String]) -> Result<(PathBuf, Vec<PathBuf>)> {
if paths.is_empty() {
return Err(Error::Other("pull request has no paths".to_string()));
}
if paths.len() == 1 {
let (base, matches) = expand_remote_glob(root, &paths[0])?.ok_or_else(|| {
Error::Other(format!("no files match remote source pattern '{}'", paths[0]))
})?;
return Ok((base, matches));
}
let mut roots = Vec::with_capacity(paths.len());
for p in paths {
if !Path::new(p).is_absolute() {
return Err(Error::Other(
"remote --files-from entries must be absolute paths on the server".to_string(),
));
}
roots.push(resolve_serve_path(root, p)?);
}
let base = path_root(&roots[0]);
if roots.iter().any(|r| !r.starts_with(&base)) {
return Err(Error::Other(
"remote --files-from entries must be on the same filesystem root".to_string(),
));
}
Ok((base, roots))
}
fn path_root(path: &Path) -> PathBuf {
let root: PathBuf = path
.components()
.take_while(|c| matches!(c, Component::Prefix(_) | Component::RootDir))
.collect();
if root.as_os_str().is_empty() {
PathBuf::from("/")
} else {
root
}
}
pub(crate) fn glob_expand(pattern: &str) -> std::result::Result<Vec<PathBuf>, String> {
catch_unwind(AssertUnwindSafe(|| {
glob::glob(pattern)
.map_err(|e| format!("invalid glob pattern '{pattern}': {e}"))?
.collect::<std::result::Result<Vec<_>, _>>()
.map_err(|e| format!("glob error for '{pattern}': {e}"))
}))
.map_err(|_| format!("invalid glob pattern '{pattern}'"))?
}
fn expand_remote_glob(root: &Path, pattern: &str) -> Result<Option<(PathBuf, Vec<PathBuf>)>> {
if !has_glob_metachars(pattern) {
return Ok(None);
}
let literal = resolve_serve_path(root, pattern)?;
if literal.exists() {
return Ok(None);
}
let meta_at = pattern.find(['*', '?', '[']).unwrap_or(pattern.len());
let prefix = &pattern[..meta_at];
let base_str = match prefix.rfind(['/', '\\']) {
Some(idx) => &prefix[..idx],
None => "",
};
let base_path = if base_str.is_empty() || base_str == "." {
root.to_path_buf()
} else if base_str.starts_with('/') {
PathBuf::from(base_str)
} else {
sanitize_join(root, base_str)?
};
let joined = base_path.join(&pattern[meta_at..]);
let matches = glob_expand(&joined.to_string_lossy()).map_err(Error::Other)?;
if matches.is_empty() {
return Ok(None);
}
let mut matches = matches;
matches.sort();
Ok(Some((base_path, matches)))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::test_fuzz::FuzzRng;
fn options(jobs: Option<usize>, storage: StoragePreference) -> ExecutorOptions {
ExecutorOptions {
jobs,
storage,
..ExecutorOptions::default()
}
}
#[test]
fn explicit_jobs_wins_over_storage() {
let opts = options(Some(4), StoragePreference::Hdd);
assert_eq!(resolve_apply_jobs(&opts, Path::new("/tmp")), 4);
assert_eq!(resolve_hash_workers(&opts, Path::new("/tmp")), 4);
}
#[test]
fn hdd_tunes_sequential() {
let opts = options(None, StoragePreference::Hdd);
assert_eq!(resolve_apply_jobs(&opts, Path::new("/tmp")), 1);
assert_eq!(resolve_hash_workers(&opts, Path::new("/tmp")), 1);
}
#[test]
fn ssd_tunes_default() {
let opts = options(None, StoragePreference::Ssd);
assert_eq!(resolve_apply_jobs(&opts, Path::new("/tmp")), APPLY_JOBS_SSD);
assert_eq!(
resolve_hash_workers(&opts, Path::new("/tmp")),
optimal_thread_count(true)
);
}
#[test]
fn auto_with_unknown_storage_falls_back_to_ssd_defaults() {
let opts = options(None, StoragePreference::Auto);
let missing = Path::new("/nonexistent/cp2-tune-test");
assert_eq!(resolve_apply_jobs(&opts, missing), APPLY_JOBS_SSD);
assert_eq!(
resolve_hash_workers(&opts, missing),
optimal_thread_count(true)
);
}
#[test]
fn zero_jobs_clamps_to_one() {
let opts = options(Some(0), StoragePreference::Auto);
assert_eq!(resolve_apply_jobs(&opts, Path::new("/tmp")), 1);
assert_eq!(resolve_hash_workers(&opts, Path::new("/tmp")), 1);
}
#[test]
fn resolve_serve_path_relative_and_absolute() {
use soft_canonicalize::soft_canonicalize;
let root = tempfile::tempdir().unwrap();
assert_eq!(
resolve_serve_path(root.path(), "").unwrap(),
root.path().to_path_buf()
);
let root_canon = soft_canonicalize(root.path()).unwrap();
assert_eq!(
resolve_serve_path(root.path(), "backup").unwrap(),
root_canon.join("backup")
);
assert_eq!(
resolve_serve_path(root.path(), "softwares/cp2").unwrap(),
root_canon.join("softwares/cp2")
);
assert_eq!(
resolve_serve_path(root.path(), "/home/user/x").unwrap(),
Path::new("/home/user/x")
);
assert!(resolve_serve_path(root.path(), "..").is_err());
assert!(resolve_serve_path(root.path(), "/home/user/../x").is_err());
}
#[test]
fn fuzz_pull_roots_and_remote_glob_never_panic() {
let root = tempfile::tempdir().unwrap();
std::fs::create_dir_all(root.path().join("d1")).unwrap();
std::fs::write(root.path().join("d1/a.txt"), b"x").unwrap();
std::fs::write(root.path().join("d1/a.bin"), b"x").unwrap();
let mut rng = FuzzRng::new(0x6C08_5EC7_6C0B_5EED);
for _ in 0..1_000 {
let pattern = if rng.below(2) == 0 {
rng.pathish(4)
} else {
rng.string(32)
};
if let Ok(Some((base, matches))) = expand_remote_glob(root.path(), &pattern) {
assert!(
matches.iter().all(|m| m.starts_with(&base)),
"glob {pattern:?} escaped {base:?}: {matches:?}"
);
}
let _ = remote_glob_needs_expansion(root.path(), &pattern);
if let Ok((base, roots)) =
resolve_pull_roots(root.path(), std::slice::from_ref(&pattern))
{
assert!(!roots.is_empty());
assert!(
roots.iter().all(|r| r.starts_with(&base)),
"single-path {pattern:?} escaped {base:?}"
);
}
let n = 1 + rng.below(3);
let paths: Vec<String> = (0..n)
.map(|_| {
if rng.below(2) == 0 {
format!("/{}", rng.pathish(3))
} else {
rng.pathish(3)
}
})
.collect();
if let Ok((base, roots)) = resolve_pull_roots(root.path(), &paths) {
assert!(!roots.is_empty());
assert!(
roots.iter().all(|r| r.starts_with(&base)),
"multi-path {paths:?} escaped {base:?}"
);
}
}
}
}