use super::fs_primitive_error::PrimitiveError;
use super::subvolume_anchor::Anchor;
use super::subvolume_error::{Interrupt, MigrateError};
use super::subvolume_platform::Platform;
use super::subvolume_remove::remove_tree;
use super::subvolume_sync_attrs::{
AttributeFailure, AttributeLosses, carried_attributes_equal, make_fifo, open_fifo, preserve_on_descriptor, preserve_on_link,
};
use std::collections::{HashMap, HashSet};
use std::ffi::{OsStr, OsString};
use std::fs::{File, Metadata};
use std::io::Seek;
use std::os::unix::fs::{DirBuilderExt, FileTypeExt, MetadataExt, OpenOptionsExt};
use std::path::{Path, PathBuf};
const PARTIAL_NAME: &str = ".cli-git-migrate-partial";
const COPY_ATTEMPTS: u32 = 3;
#[derive(Clone, Copy, Debug)]
pub struct SyncOptions<'a> {
pub delete: bool,
pub update_only: bool,
pub excluded: &'a [PathBuf],
pub created_since: Option<i64>,
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct SyncStats {
pub entries: u64,
pub directories: u64,
pub files_copied: u64,
pub cloned: u64,
pub hard_linked: u64,
pub symlinks: u64,
pub fifos: u64,
pub unchanged: u64,
pub deleted: u64,
pub vanished: u64,
pub unreadable: Vec<PathBuf>,
pub special_skipped: Vec<PathBuf>,
pub losses: AttributeLosses,
pub conflicts: Vec<PathBuf>,
pub recopied: u64,
pub left_out: u64,
pub copied_sample: Vec<PathBuf>,
}
const SAMPLE_SIZE: usize = 20;
enum Work {
Visit(PathBuf),
Finish(PathBuf, Metadata),
}
struct Directory {
anchor: Anchor,
temporary: OsString,
}
impl Directory {
fn child(&self, name: &OsStr) -> PathBuf {
return self.anchor.child(name);
}
fn partial(&self) -> PathBuf {
return self.anchor.child(self.temporary.as_os_str());
}
}
struct Pass<'a> {
platform: &'a dyn Platform,
source: &'a Path,
destination: &'a Path,
options: SyncOptions<'a>,
interrupt: &'a Interrupt,
stats: SyncStats,
links: HashMap<(u64, u64), Vec<PathBuf>>,
}
fn file_is_equal(copy: &Metadata, original: &Metadata) -> bool {
return copy.is_file()
&& copy.len() == original.len()
&& copy.mtime() == original.mtime()
&& copy.mtime_nsec() == original.mtime_nsec()
&& copy.mode() & 0o7777 == original.mode() & 0o7777;
}
fn is_newer(left: &Metadata, right: &Metadata) -> bool {
return (left.mtime(), left.mtime_nsec()) > (right.mtime(), right.mtime_nsec());
}
fn is_settled(before: &Metadata, after: &Metadata) -> bool {
return before.len() == after.len()
&& (before.mtime(), before.mtime_nsec()) == (after.mtime(), after.mtime_nsec())
&& (before.ctime(), before.ctime_nsec()) == (after.ctime(), after.ctime_nsec());
}
fn remove_entry(path: &Path, shown: &Path) -> Result<(), MigrateError> {
return remove_tree(path).map_err(|error: std::io::Error| { return MigrateError::io("remove", shown, &error); });
}
fn remove_if_present(path: &Path, shown: &Path) -> Result<(), MigrateError> {
return match std::fs::symlink_metadata(path) {
Ok(_) => remove_entry(path, shown),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(MigrateError::io("examine", shown, &error)),
};
}
fn attribute_error(failure: AttributeFailure, shown: &Path) -> MigrateError {
return MigrateError::io(failure.action, shown, &failure.error);
}
fn is_replaced(error: &std::io::Error) -> bool {
return matches!(error.raw_os_error(), Some(libc::ELOOP | libc::ENOTDIR | libc::ENOENT));
}
impl Pass<'_> {
fn shown(&self, relative: &Path) -> PathBuf {
return self.destination.join(relative);
}
fn open_destination(&mut self, relative: &Path) -> Result<Option<Anchor>, MigrateError> {
return match Anchor::open_below(self.destination, relative) {
Ok(anchor) => Ok(Some(anchor)),
Err(error) if is_replaced(&error) && self.options.update_only && !relative.as_os_str().is_empty() => {
self.stats.conflicts.push(relative.to_path_buf());
Ok(None)
}
Err(error) => Err(MigrateError::io("open the directory", self.shown(relative).as_path(), &error)),
};
}
fn replace_allowed(&mut self, relative: &Path, found: &Metadata, original: &Metadata) -> bool {
if !self.options.update_only {
return true;
}
if found.is_dir() || !is_newer(original, found) {
self.stats.conflicts.push(relative.to_path_buf());
return false;
}
return true;
}
fn visit(&mut self, relative: &Path, stack: &mut Vec<Work>) -> Result<(), MigrateError> {
let source_directory: PathBuf = self.source.join(relative);
let listing: std::fs::ReadDir = match std::fs::read_dir(&source_directory) {
Ok(found) => found,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
self.stats.vanished += 1;
return Ok(());
}
Err(error) => return Err(MigrateError::UnreadableDirectory { path: source_directory, reason: error.to_string() }),
};
let Some(anchor) = self.open_destination(relative)? else { return Ok(()) };
make_writable(&anchor, self.shown(relative).as_path())?;
let mut names: HashSet<OsString> = HashSet::new();
let mut children: Vec<(OsString, Metadata)> = Vec::new();
for item in listing {
let entry: std::fs::DirEntry = item.map_err(|error: std::io::Error| {
return MigrateError::UnreadableDirectory { path: source_directory.clone(), reason: error.to_string() };
})?;
names.insert(entry.file_name());
if self.options.excluded.iter().any(|path: &PathBuf| { return *path == relative.join(entry.file_name()); }) {
continue;
}
match entry.metadata() {
Ok(metadata) => children.push((entry.file_name(), metadata)),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => self.stats.vanished += 1,
Err(error) => return Err(MigrateError::UnreadableDirectory { path: source_directory, reason: error.to_string() }),
}
}
let mut temporary: OsString = OsString::from(PARTIAL_NAME);
while names.contains(&temporary) {
temporary.push("~");
}
let directory: Directory = Directory { anchor, temporary };
remove_if_present(directory.partial().as_path(), self.shown(relative).join(&directory.temporary).as_path())?;
if self.options.delete && !self.options.update_only {
self.delete_extras(&directory, relative, &names)?;
}
for (name, metadata) in children {
self.stats.entries += 1;
self.handle_child(&directory, relative.join(&name), metadata, stack)?;
}
return Ok(());
}
fn delete_extras(&mut self, directory: &Directory, relative: &Path, names: &HashSet<OsString>) -> Result<(), MigrateError> {
let Ok(listing) = std::fs::read_dir(directory.anchor.path()) else { return Ok(()) };
for item in listing.flatten() {
if names.contains(&item.file_name()) {
continue;
}
if self.options.excluded.iter().any(|path: &PathBuf| { return *path == relative.join(item.file_name()); }) {
continue;
}
remove_entry(directory.child(item.file_name().as_os_str()).as_path(), self.shown(relative).join(item.file_name()).as_path())?;
self.stats.deleted += 1;
}
return Ok(());
}
fn handle_child(&mut self, directory: &Directory, relative: PathBuf, metadata: Metadata, stack: &mut Vec<Work>) -> Result<(), MigrateError> {
let name: OsString = relative.file_name().map(OsStr::to_os_string).unwrap_or_default();
let destination: PathBuf = directory.child(name.as_os_str());
let existing: Option<Metadata> = std::fs::symlink_metadata(&destination).ok();
if existing.is_none() && self.options.update_only && self.options.created_since.is_some_and(|since: i64| { return metadata.mtime() < since; }) {
self.stats.left_out += 1;
return Ok(());
}
let file_type: std::fs::FileType = metadata.file_type();
if file_type.is_dir() {
if let Some(found) = &existing && !found.is_dir() {
if !self.replace_allowed(relative.as_path(), found, &metadata) {
return Ok(());
}
remove_entry(destination.as_path(), self.shown(relative.as_path()).as_path())?;
}
if existing.as_ref().is_none_or(|found: &Metadata| { return !found.is_dir(); }) {
std::fs::DirBuilder::new().mode(0o700).create(&destination).map_err(|error: std::io::Error| {
return MigrateError::io("create the directory", self.shown(relative.as_path()).as_path(), &error);
})?;
}
self.stats.directories += 1;
stack.push(Work::Finish(relative.clone(), metadata));
stack.push(Work::Visit(relative));
return Ok(());
}
if file_type.is_symlink() {
return self.copy_symlink(directory, &relative, &metadata, existing);
}
if file_type.is_fifo() {
return self.copy_fifo(directory, &relative, &metadata, existing);
}
if file_type.is_file() {
return self.copy_file(directory, &relative, &metadata, existing);
}
self.stats.special_skipped.push(relative);
return Ok(());
}
fn publish(&self, directory: &Directory, relative: &Path, existing: Option<&Metadata>) -> Result<(), MigrateError> {
let name: &OsStr = relative.file_name().unwrap_or_default();
let destination: PathBuf = directory.child(name);
if let Some(found) = existing && found.is_dir() {
remove_entry(destination.as_path(), self.shown(relative).as_path())?;
}
return std::fs::rename(directory.partial(), &destination).map_err(|error: std::io::Error| {
return MigrateError::io("put the finished copy in place of", self.shown(relative).as_path(), &error);
});
}
fn copy_symlink(&mut self, directory: &Directory, relative: &Path, metadata: &Metadata, existing: Option<Metadata>) -> Result<(), MigrateError> {
let source: PathBuf = self.source.join(relative);
let destination: PathBuf = directory.child(relative.file_name().unwrap_or_default());
let target: PathBuf = std::fs::read_link(&source).map_err(|error: std::io::Error| { return MigrateError::io("read the link", source.as_path(), &error); })?;
if let Some(found) = &existing {
if found.file_type().is_symlink() && std::fs::read_link(&destination).ok().as_ref() == Some(&target) {
if (found.mtime(), found.mtime_nsec()) != (metadata.mtime(), metadata.mtime_nsec()) && !self.options.update_only {
preserve_on_link(source.as_path(), destination.as_path(), metadata, &mut self.stats.losses)
.map_err(|failure: AttributeFailure| { return attribute_error(failure, self.shown(relative).as_path()); })?;
}
self.stats.unchanged += 1;
return Ok(());
}
if !self.replace_allowed(relative, found, metadata) {
return Ok(());
}
}
let partial: PathBuf = directory.partial();
std::os::unix::fs::symlink(&target, &partial).map_err(|error: std::io::Error| { return MigrateError::io("create the link", self.shown(relative).as_path(), &error); })?;
preserve_on_link(source.as_path(), partial.as_path(), metadata, &mut self.stats.losses)
.map_err(|failure: AttributeFailure| { return attribute_error(failure, self.shown(relative).as_path()); })?;
self.publish(directory, relative, existing.as_ref())?;
self.stats.symlinks += 1;
return Ok(());
}
fn copy_fifo(&mut self, directory: &Directory, relative: &Path, metadata: &Metadata, existing: Option<Metadata>) -> Result<(), MigrateError> {
let source: PathBuf = self.source.join(relative);
let destination: PathBuf = directory.child(relative.file_name().unwrap_or_default());
if let Some(found) = &existing {
if found.file_type().is_fifo() {
let same: bool = found.mode() & 0o7777 == metadata.mode() & 0o7777 && (found.mtime(), found.mtime_nsec()) == (metadata.mtime(), metadata.mtime_nsec());
if !same && !self.options.update_only {
let handle: File = open_fifo(destination.as_path()).map_err(|error: std::io::Error| { return MigrateError::io("open the named pipe", self.shown(relative).as_path(), &error); })?;
preserve_on_descriptor(source.as_path(), &handle, metadata, true, &mut self.stats.losses)
.map_err(|failure: AttributeFailure| { return attribute_error(failure, self.shown(relative).as_path()); })?;
}
self.stats.unchanged += 1;
return Ok(());
}
if !self.replace_allowed(relative, found, metadata) {
return Ok(());
}
}
let partial: PathBuf = directory.partial();
make_fifo(partial.as_path(), metadata.mode()).map_err(|error: std::io::Error| { return MigrateError::io("create the named pipe", self.shown(relative).as_path(), &error); })?;
let handle: File = open_fifo(partial.as_path()).map_err(|error: std::io::Error| { return MigrateError::io("open the named pipe", self.shown(relative).as_path(), &error); })?;
preserve_on_descriptor(source.as_path(), &handle, metadata, true, &mut self.stats.losses)
.map_err(|failure: AttributeFailure| { return attribute_error(failure, self.shown(relative).as_path()); })?;
self.publish(directory, relative, existing.as_ref())?;
self.stats.fifos += 1;
return Ok(());
}
fn is_linked_as_recorded(&self, key: (u64, u64), found: &Metadata) -> bool {
let Some(copies) = self.links.get(&key) else { return true };
return copies.iter().any(|copy: &PathBuf| {
return std::fs::symlink_metadata(self.destination.join(copy)).is_ok_and(|copy_meta: Metadata| { return copy_meta.ino() == found.ino(); });
});
}
fn record_link(&mut self, key: (u64, u64), relative: &Path) {
let copies: &mut Vec<PathBuf> = self.links.entry(key).or_default();
if !copies.iter().any(|copy: &PathBuf| { return copy == relative; }) {
copies.push(relative.to_path_buf());
}
}
fn link_to_earlier_copy(&mut self, directory: &Directory, relative: &Path, key: (u64, u64), existing: Option<&Metadata>) -> Result<bool, MigrateError> {
for copy in self.links.get(&key).cloned().unwrap_or_default() {
let Ok(earlier) = Anchor::open_below(self.destination, copy.parent().unwrap_or(Path::new(""))) else { continue };
let first: PathBuf = earlier.child(copy.file_name().unwrap_or_default());
remove_if_present(directory.partial().as_path(), self.shown(relative).as_path())?;
if self.platform.hard_link(first.as_path(), directory.partial().as_path()).is_err() {
continue;
}
self.publish(directory, relative, existing)?;
self.stats.hard_linked += 1;
return Ok(true);
}
return Ok(false);
}
fn copy_file(&mut self, directory: &Directory, relative: &Path, listing: &Metadata, existing: Option<Metadata>) -> Result<(), MigrateError> {
let source: PathBuf = self.source.join(relative);
let destination: PathBuf = directory.child(relative.file_name().unwrap_or_default());
let shown: PathBuf = self.shown(relative);
let listed_key: (u64, u64) = (listing.dev(), listing.ino());
if let Some(found) = &existing {
if !found.is_file() {
if !self.replace_allowed(relative, found, listing) {
return Ok(());
}
} else if self.options.update_only && is_newer(found, listing) {
self.stats.unchanged += 1;
return Ok(());
} else if file_is_equal(found, listing) && (listing.nlink() <= 1 || self.is_linked_as_recorded(listed_key, found)) {
if !self.options.update_only && !carried_attributes_equal(source.as_path(), destination.as_path()) {
self.refresh_attributes(&source, &destination, listing, shown.as_path())?;
}
self.stats.unchanged += 1;
if listing.nlink() > 1 {
self.record_link(listed_key, relative);
}
return Ok(());
}
}
if listing.nlink() > 1 && self.link_to_earlier_copy(directory, relative, listed_key, existing.as_ref())? {
return Ok(());
}
let mut input: File = match File::open(&source) {
Ok(opened) => opened,
Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => {
self.stats.unreadable.push(relative.to_path_buf());
return Ok(());
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
self.stats.vanished += 1;
return Ok(());
}
Err(error) => return Err(MigrateError::io("open", source.as_path(), &error)),
};
let mut attempt: u32 = 0;
loop {
attempt += 1;
let before: Metadata = input.metadata().map_err(|error: std::io::Error| { return MigrateError::io("examine", source.as_path(), &error); })?;
if !before.is_file() {
self.stats.vanished += 1;
return Ok(());
}
remove_if_present(directory.partial().as_path(), shown.as_path())?;
let mut output: File = std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(directory.partial())
.map_err(|error: std::io::Error| { return MigrateError::io("create", shown.as_path(), &error); })?;
self.interrupt.check("copy-partial")?;
input.rewind().map_err(|error: std::io::Error| { return MigrateError::io("rewind", source.as_path(), &error); })?;
let cloned: Result<(), PrimitiveError> = self.platform.clone_file(&input, &output, directory.partial().as_path());
if cloned.is_err() {
std::io::copy(&mut input, &mut output).map_err(|error: std::io::Error| { return MigrateError::io("copy to", shown.as_path(), &error); })?;
}
let after: Metadata = input.metadata().map_err(|error: std::io::Error| { return MigrateError::io("examine", source.as_path(), &error); })?;
if !is_settled(&before, &after) {
drop(output);
remove_if_present(directory.partial().as_path(), shown.as_path())?;
if attempt >= COPY_ATTEMPTS {
return Err(MigrateError::Io {
action: String::from("copy"),
path: source,
reason: format!("it changed while it was being copied, {COPY_ATTEMPTS} times in a row; stop the programs writing to it and run the command again"),
});
}
self.stats.recopied += 1;
continue;
}
preserve_on_descriptor(source.as_path(), &output, &before, !self.options.update_only, &mut self.stats.losses)
.map_err(|failure: AttributeFailure| { return attribute_error(failure, shown.as_path()); })?;
drop(output);
self.publish(directory, relative, existing.as_ref())?;
if cloned.is_ok() {
self.stats.cloned += 1;
}
self.stats.files_copied += 1;
if self.stats.copied_sample.len() < SAMPLE_SIZE {
self.stats.copied_sample.push(relative.to_path_buf());
}
if before.nlink() > 1 {
self.record_link((before.dev(), before.ino()), relative);
}
return self.interrupt.check("copy-entry");
}
}
fn refresh_attributes(&mut self, source: &Path, destination: &Path, metadata: &Metadata, shown: &Path) -> Result<(), MigrateError> {
let handle: File = std::fs::OpenOptions::new().read(true).custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC).open(destination)
.map_err(|error: std::io::Error| { return MigrateError::io("open", shown, &error); })?;
return preserve_on_descriptor(source, &handle, metadata, true, &mut self.stats.losses)
.map_err(|failure: AttributeFailure| { return attribute_error(failure, shown); });
}
}
fn make_writable(anchor: &Anchor, shown: &Path) -> Result<(), MigrateError> {
let metadata: Metadata = anchor.file().metadata().map_err(|error: std::io::Error| { return MigrateError::io("examine", shown, &error); })?;
if metadata.mode() & 0o700 != 0o700 {
anchor.file().set_permissions(std::os::unix::fs::PermissionsExt::from_mode(metadata.mode() | 0o700))
.map_err(|error: std::io::Error| { return MigrateError::io("make writable", shown, &error); })?;
}
return Ok(());
}
pub fn sync_tree(
platform: &dyn Platform,
source: &Path,
destination: &Path,
options: SyncOptions<'_>,
interrupt: &Interrupt,
) -> Result<SyncStats, MigrateError> {
let mut pass: Pass<'_> = Pass { platform, source, destination, options, interrupt, stats: SyncStats::default(), links: HashMap::new() };
let root_metadata: Metadata = std::fs::symlink_metadata(source).map_err(|error: std::io::Error| { return MigrateError::io("examine", source, &error); })?;
let mut stack: Vec<Work> = vec![Work::Finish(PathBuf::new(), root_metadata), Work::Visit(PathBuf::new())];
while let Some(work) = stack.pop() {
match work {
Work::Visit(relative) => pass.visit(relative.as_path(), &mut stack)?,
Work::Finish(relative, metadata) => {
let Some(anchor) = pass.open_destination(relative.as_path())? else { continue };
let shown: PathBuf = pass.shown(relative.as_path());
preserve_on_descriptor(source.join(&relative).as_path(), anchor.file(), &metadata, !options.update_only, &mut pass.stats.losses)
.map_err(|failure: AttributeFailure| { return attribute_error(failure, shown.as_path()); })?;
}
}
}
pass.stats.unreadable.sort();
pass.stats.conflicts.sort();
pass.stats.conflicts.dedup();
return Ok(pass.stats);
}
#[cfg(test)]
#[path = "subvolume_sync_tests.rs"]
mod tests;