1use crate::fs_primitive_error::PrimitiveError;
25use crate::subvolume_anchor::Anchor;
26use crate::subvolume_error::{Interrupt, MigrateError};
27use crate::subvolume_platform::Platform;
28use crate::subvolume_remove::remove_tree;
29use crate::subvolume_sync_attrs::{
30 AttributeFailure, AttributeLosses, carried_attributes_equal, make_fifo, open_fifo, preserve_on_descriptor, preserve_on_link,
31};
32use std::collections::{HashMap, HashSet};
33use std::ffi::{OsStr, OsString};
34use std::fs::{File, Metadata};
35use std::io::Seek;
36use std::os::unix::fs::{DirBuilderExt, FileTypeExt, MetadataExt, OpenOptionsExt};
37use std::path::{Path, PathBuf};
38
39const PARTIAL_NAME: &str = ".cli-git-migrate-partial";
41
42const COPY_ATTEMPTS: u32 = 3;
44
45#[derive(Clone, Copy, Debug)]
57pub struct SyncOptions<'a> {
58 pub delete: bool,
60 pub update_only: bool,
62 pub excluded: &'a [PathBuf],
64 pub created_since: Option<i64>,
67}
68
69#[derive(Clone, Debug, Default, Eq, PartialEq)]
79pub struct SyncStats {
80 pub entries: u64,
82 pub directories: u64,
84 pub files_copied: u64,
86 pub cloned: u64,
88 pub hard_linked: u64,
90 pub symlinks: u64,
92 pub fifos: u64,
94 pub unchanged: u64,
96 pub deleted: u64,
98 pub vanished: u64,
100 pub unreadable: Vec<PathBuf>,
102 pub special_skipped: Vec<PathBuf>,
104 pub losses: AttributeLosses,
106 pub conflicts: Vec<PathBuf>,
109 pub recopied: u64,
111 pub left_out: u64,
113 pub copied_sample: Vec<PathBuf>,
115}
116
117const SAMPLE_SIZE: usize = 20;
122
123enum Work {
125 Visit(PathBuf),
127 Finish(PathBuf, Metadata),
129}
130
131struct Directory {
133 anchor: Anchor,
135 temporary: OsString,
137}
138
139impl Directory {
140 fn child(&self, name: &OsStr) -> PathBuf {
142 return self.anchor.child(name);
143 }
144
145 fn partial(&self) -> PathBuf {
147 return self.anchor.child(self.temporary.as_os_str());
148 }
149}
150
151struct Pass<'a> {
153 platform: &'a dyn Platform,
154 source: &'a Path,
155 destination: &'a Path,
156 options: SyncOptions<'a>,
157 interrupt: &'a Interrupt,
158 stats: SyncStats,
159 links: HashMap<(u64, u64), Vec<PathBuf>>,
162}
163
164fn file_is_equal(copy: &Metadata, original: &Metadata) -> bool {
167 return copy.is_file()
168 && copy.len() == original.len()
169 && copy.mtime() == original.mtime()
170 && copy.mtime_nsec() == original.mtime_nsec()
171 && copy.mode() & 0o7777 == original.mode() & 0o7777;
172}
173
174fn is_newer(left: &Metadata, right: &Metadata) -> bool {
176 return (left.mtime(), left.mtime_nsec()) > (right.mtime(), right.mtime_nsec());
177}
178
179fn is_settled(before: &Metadata, after: &Metadata) -> bool {
181 return before.len() == after.len()
182 && (before.mtime(), before.mtime_nsec()) == (after.mtime(), after.mtime_nsec())
183 && (before.ctime(), before.ctime_nsec()) == (after.ctime(), after.ctime_nsec());
184}
185
186fn remove_entry(path: &Path, shown: &Path) -> Result<(), MigrateError> {
188 return remove_tree(path).map_err(|error: std::io::Error| { return MigrateError::io("remove", shown, &error); });
189}
190
191fn remove_if_present(path: &Path, shown: &Path) -> Result<(), MigrateError> {
193 return match std::fs::symlink_metadata(path) {
194 Ok(_) => remove_entry(path, shown),
195 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
196 Err(error) => Err(MigrateError::io("examine", shown, &error)),
197 };
198}
199
200fn attribute_error(failure: AttributeFailure, shown: &Path) -> MigrateError {
202 return MigrateError::io(failure.action, shown, &failure.error);
203}
204
205fn is_replaced(error: &std::io::Error) -> bool {
207 return matches!(error.raw_os_error(), Some(libc::ELOOP | libc::ENOTDIR | libc::ENOENT));
208}
209
210fn vanished_is_counted<T>(outcome: std::io::Result<T>, vanished: &mut u64) -> std::io::Result<Option<T>> {
222 return match outcome {
223 Ok(found) => Ok(Some(found)),
224 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
225 *vanished += 1;
226 Ok(None)
227 }
228 Err(error) => Err(error),
229 };
230}
231
232impl Pass<'_> {
233 fn shown(&self, relative: &Path) -> PathBuf {
235 return self.destination.join(relative);
236 }
237
238 fn open_destination(&mut self, relative: &Path) -> Result<Option<Anchor>, MigrateError> {
241 return match Anchor::open_below(self.destination, relative) {
242 Ok(anchor) => Ok(Some(anchor)),
243 Err(error) if is_replaced(&error) && self.options.update_only && !relative.as_os_str().is_empty() => {
244 self.stats.conflicts.push(relative.to_path_buf());
245 Ok(None)
246 }
247 Err(error) => Err(MigrateError::io("open the directory", self.shown(relative).as_path(), &error)),
248 };
249 }
250
251 fn replace_allowed(&mut self, relative: &Path, found: &Metadata, original: &Metadata) -> bool {
254 if !self.options.update_only {
255 return true;
256 }
257 if found.is_dir() || !is_newer(original, found) {
258 self.stats.conflicts.push(relative.to_path_buf());
259 return false;
260 }
261 return true;
262 }
263
264 fn visit(&mut self, relative: &Path, stack: &mut Vec<Work>) -> Result<(), MigrateError> {
266 let source_directory: PathBuf = self.source.join(relative);
267 let listing: std::fs::ReadDir = match vanished_is_counted(std::fs::read_dir(&source_directory), &mut self.stats.vanished) {
268 Ok(Some(found)) => found,
269 Ok(None) => return Ok(()),
270 Err(error) => return Err(MigrateError::UnreadableDirectory { path: source_directory, reason: error.to_string() }),
271 };
272 let Some(anchor) = self.open_destination(relative)? else { return Ok(()) };
273 make_writable(&anchor, self.shown(relative).as_path())?;
274 let mut names: HashSet<OsString> = HashSet::new();
275 let mut children: Vec<(OsString, Metadata)> = Vec::new();
276 for item in listing {
277 let entry: std::fs::DirEntry = item.map_err(|error: std::io::Error| {
278 return MigrateError::UnreadableDirectory { path: source_directory.clone(), reason: error.to_string() };
279 })?;
280 names.insert(entry.file_name());
281 if self.options.excluded.iter().any(|path: &PathBuf| { return *path == relative.join(entry.file_name()); }) {
282 continue;
283 }
284 match vanished_is_counted(entry.metadata(), &mut self.stats.vanished) {
285 Ok(Some(metadata)) => children.push((entry.file_name(), metadata)),
286 Ok(None) => {}
287 Err(error) => return Err(MigrateError::UnreadableDirectory { path: source_directory, reason: error.to_string() }),
288 }
289 }
290 let mut temporary: OsString = OsString::from(PARTIAL_NAME);
291 while names.contains(&temporary) {
292 temporary.push("~");
293 }
294 let directory: Directory = Directory { anchor, temporary };
295 remove_if_present(directory.partial().as_path(), self.shown(relative).join(&directory.temporary).as_path())?;
297 if self.options.delete && !self.options.update_only {
298 self.delete_extras(&directory, relative, &names)?;
299 }
300 for (name, metadata) in children {
301 self.stats.entries += 1;
302 self.handle_child(&directory, relative.join(&name), metadata, stack)?;
303 }
304 return Ok(());
305 }
306
307 fn delete_extras(&mut self, directory: &Directory, relative: &Path, names: &HashSet<OsString>) -> Result<(), MigrateError> {
309 let Ok(listing) = std::fs::read_dir(directory.anchor.path()) else { return Ok(()) };
310 for item in listing.flatten() {
311 if names.contains(&item.file_name()) {
312 continue;
313 }
314 if self.options.excluded.iter().any(|path: &PathBuf| { return *path == relative.join(item.file_name()); }) {
315 continue;
316 }
317 remove_entry(directory.child(item.file_name().as_os_str()).as_path(), self.shown(relative).join(item.file_name()).as_path())?;
318 self.stats.deleted += 1;
319 }
320 return Ok(());
321 }
322
323 fn handle_child(&mut self, directory: &Directory, relative: PathBuf, metadata: Metadata, stack: &mut Vec<Work>) -> Result<(), MigrateError> {
325 let name: OsString = relative.file_name().map(OsStr::to_os_string).unwrap_or_default();
326 let destination: PathBuf = directory.child(name.as_os_str());
327 let existing: Option<Metadata> = std::fs::symlink_metadata(&destination).ok();
328 if existing.is_none() && self.options.update_only && self.options.created_since.is_some_and(|since: i64| { return metadata.mtime() < since; }) {
329 self.stats.left_out += 1;
330 return Ok(());
331 }
332 let file_type: std::fs::FileType = metadata.file_type();
333 if file_type.is_dir() {
334 if let Some(found) = &existing && !found.is_dir() {
335 if !self.replace_allowed(relative.as_path(), found, &metadata) {
336 return Ok(());
337 }
338 remove_entry(destination.as_path(), self.shown(relative.as_path()).as_path())?;
339 }
340 if existing.as_ref().is_none_or(|found: &Metadata| { return !found.is_dir(); }) {
341 std::fs::DirBuilder::new().mode(0o700).create(&destination).map_err(|error: std::io::Error| {
342 return MigrateError::io("create the directory", self.shown(relative.as_path()).as_path(), &error);
343 })?;
344 }
345 self.stats.directories += 1;
346 stack.push(Work::Finish(relative.clone(), metadata));
347 stack.push(Work::Visit(relative));
348 return Ok(());
349 }
350 if file_type.is_symlink() {
351 return self.copy_symlink(directory, &relative, &metadata, existing);
352 }
353 if file_type.is_fifo() {
354 return self.copy_fifo(directory, &relative, &metadata, existing);
355 }
356 if file_type.is_file() {
357 return self.copy_file(directory, &relative, &metadata, existing);
358 }
359 self.stats.special_skipped.push(relative);
360 return Ok(());
361 }
362
363 fn publish(&self, directory: &Directory, relative: &Path, existing: Option<&Metadata>) -> Result<(), MigrateError> {
365 let name: &OsStr = relative.file_name().unwrap_or_default();
366 let destination: PathBuf = directory.child(name);
367 if let Some(found) = existing && found.is_dir() {
368 remove_entry(destination.as_path(), self.shown(relative).as_path())?;
369 }
370 return std::fs::rename(directory.partial(), &destination).map_err(|error: std::io::Error| {
371 return MigrateError::io("put the finished copy in place of", self.shown(relative).as_path(), &error);
372 });
373 }
374
375 fn copy_symlink(&mut self, directory: &Directory, relative: &Path, metadata: &Metadata, existing: Option<Metadata>) -> Result<(), MigrateError> {
377 let source: PathBuf = self.source.join(relative);
378 let destination: PathBuf = directory.child(relative.file_name().unwrap_or_default());
379 let target: PathBuf = std::fs::read_link(&source).map_err(|error: std::io::Error| { return MigrateError::io("read the link", source.as_path(), &error); })?;
380 if let Some(found) = &existing {
381 if found.file_type().is_symlink() && std::fs::read_link(&destination).ok().as_ref() == Some(&target) {
382 if (found.mtime(), found.mtime_nsec()) != (metadata.mtime(), metadata.mtime_nsec()) && !self.options.update_only {
383 preserve_on_link(source.as_path(), destination.as_path(), metadata, &mut self.stats.losses)
384 .map_err(|failure: AttributeFailure| { return attribute_error(failure, self.shown(relative).as_path()); })?;
385 }
386 self.stats.unchanged += 1;
387 return Ok(());
388 }
389 if !self.replace_allowed(relative, found, metadata) {
390 return Ok(());
391 }
392 }
393 let partial: PathBuf = directory.partial();
394 std::os::unix::fs::symlink(&target, &partial).map_err(|error: std::io::Error| { return MigrateError::io("create the link", self.shown(relative).as_path(), &error); })?;
395 preserve_on_link(source.as_path(), partial.as_path(), metadata, &mut self.stats.losses)
396 .map_err(|failure: AttributeFailure| { return attribute_error(failure, self.shown(relative).as_path()); })?;
397 self.publish(directory, relative, existing.as_ref())?;
398 self.stats.symlinks += 1;
399 return Ok(());
400 }
401
402 fn copy_fifo(&mut self, directory: &Directory, relative: &Path, metadata: &Metadata, existing: Option<Metadata>) -> Result<(), MigrateError> {
404 let source: PathBuf = self.source.join(relative);
405 let destination: PathBuf = directory.child(relative.file_name().unwrap_or_default());
406 if let Some(found) = &existing {
407 if found.file_type().is_fifo() {
408 let same: bool = found.mode() & 0o7777 == metadata.mode() & 0o7777 && (found.mtime(), found.mtime_nsec()) == (metadata.mtime(), metadata.mtime_nsec());
409 if !same && !self.options.update_only {
410 let handle: File = open_fifo(destination.as_path()).map_err(|error: std::io::Error| { return MigrateError::io("open the named pipe", self.shown(relative).as_path(), &error); })?;
411 preserve_on_descriptor(source.as_path(), &handle, metadata, true, &mut self.stats.losses)
412 .map_err(|failure: AttributeFailure| { return attribute_error(failure, self.shown(relative).as_path()); })?;
413 }
414 self.stats.unchanged += 1;
415 return Ok(());
416 }
417 if !self.replace_allowed(relative, found, metadata) {
418 return Ok(());
419 }
420 }
421 let partial: PathBuf = directory.partial();
422 make_fifo(partial.as_path(), metadata.mode()).map_err(|error: std::io::Error| { return MigrateError::io("create the named pipe", self.shown(relative).as_path(), &error); })?;
423 let handle: File = open_fifo(partial.as_path()).map_err(|error: std::io::Error| { return MigrateError::io("open the named pipe", self.shown(relative).as_path(), &error); })?;
424 preserve_on_descriptor(source.as_path(), &handle, metadata, true, &mut self.stats.losses)
425 .map_err(|failure: AttributeFailure| { return attribute_error(failure, self.shown(relative).as_path()); })?;
426 self.publish(directory, relative, existing.as_ref())?;
427 self.stats.fifos += 1;
428 return Ok(());
429 }
430
431 fn is_linked_as_recorded(&self, key: (u64, u64), found: &Metadata) -> bool {
433 let Some(copies) = self.links.get(&key) else { return true };
434 return copies.iter().any(|copy: &PathBuf| {
435 return std::fs::symlink_metadata(self.destination.join(copy)).is_ok_and(|copy_meta: Metadata| { return copy_meta.ino() == found.ino(); });
436 });
437 }
438
439 fn record_link(&mut self, key: (u64, u64), relative: &Path) {
441 let copies: &mut Vec<PathBuf> = self.links.entry(key).or_default();
442 if !copies.iter().any(|copy: &PathBuf| { return copy == relative; }) {
443 copies.push(relative.to_path_buf());
444 }
445 }
446
447 fn link_to_earlier_copy(&mut self, directory: &Directory, relative: &Path, key: (u64, u64), existing: Option<&Metadata>) -> Result<bool, MigrateError> {
449 for copy in self.links.get(&key).cloned().unwrap_or_default() {
450 let Ok(earlier) = Anchor::open_below(self.destination, copy.parent().unwrap_or(Path::new(""))) else { continue };
451 let first: PathBuf = earlier.child(copy.file_name().unwrap_or_default());
452 remove_if_present(directory.partial().as_path(), self.shown(relative).as_path())?;
453 if self.platform.hard_link(first.as_path(), directory.partial().as_path()).is_err() {
454 continue;
455 }
456 self.publish(directory, relative, existing)?;
457 self.stats.hard_linked += 1;
458 return Ok(true);
459 }
460 return Ok(false);
461 }
462
463 fn copy_file(&mut self, directory: &Directory, relative: &Path, listing: &Metadata, existing: Option<Metadata>) -> Result<(), MigrateError> {
465 let source: PathBuf = self.source.join(relative);
466 let destination: PathBuf = directory.child(relative.file_name().unwrap_or_default());
467 let shown: PathBuf = self.shown(relative);
468 let listed_key: (u64, u64) = (listing.dev(), listing.ino());
469 if let Some(found) = &existing {
470 if !found.is_file() {
471 if !self.replace_allowed(relative, found, listing) {
472 return Ok(());
473 }
474 } else if self.options.update_only && is_newer(found, listing) {
475 self.stats.unchanged += 1;
477 return Ok(());
478 } else if file_is_equal(found, listing) && (listing.nlink() <= 1 || self.is_linked_as_recorded(listed_key, found)) {
479 if !self.options.update_only && !carried_attributes_equal(source.as_path(), destination.as_path()) {
480 self.refresh_attributes(&source, &destination, listing, shown.as_path())?;
481 }
482 self.stats.unchanged += 1;
483 if listing.nlink() > 1 {
484 self.record_link(listed_key, relative);
485 }
486 return Ok(());
487 }
488 }
489 if listing.nlink() > 1 && self.link_to_earlier_copy(directory, relative, listed_key, existing.as_ref())? {
490 return Ok(());
491 }
492 let mut input: File = match File::open(&source) {
493 Ok(opened) => opened,
494 Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => {
495 self.stats.unreadable.push(relative.to_path_buf());
496 return Ok(());
497 }
498 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
499 self.stats.vanished += 1;
500 return Ok(());
501 }
502 Err(error) => return Err(MigrateError::io("open", source.as_path(), &error)),
503 };
504 let mut attempt: u32 = 0;
505 loop {
506 attempt += 1;
507 let before: Metadata = input.metadata().map_err(|error: std::io::Error| { return MigrateError::io("examine", source.as_path(), &error); })?;
508 if !before.is_file() {
509 self.stats.vanished += 1;
510 return Ok(());
511 }
512 remove_if_present(directory.partial().as_path(), shown.as_path())?;
513 let mut output: File = std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(directory.partial())
514 .map_err(|error: std::io::Error| { return MigrateError::io("create", shown.as_path(), &error); })?;
515 self.interrupt.check("copy-partial")?;
516 input.rewind().map_err(|error: std::io::Error| { return MigrateError::io("rewind", source.as_path(), &error); })?;
517 let cloned: Result<(), PrimitiveError> = self.platform.clone_file(&input, &output, directory.partial().as_path());
518 if cloned.is_err() {
519 std::io::copy(&mut input, &mut output).map_err(|error: std::io::Error| { return MigrateError::io("copy to", shown.as_path(), &error); })?;
520 }
521 let after: Metadata = input.metadata().map_err(|error: std::io::Error| { return MigrateError::io("examine", source.as_path(), &error); })?;
522 if !is_settled(&before, &after) {
523 drop(output);
524 remove_if_present(directory.partial().as_path(), shown.as_path())?;
525 if attempt >= COPY_ATTEMPTS {
526 return Err(MigrateError::Io {
527 action: String::from("copy"),
528 path: source,
529 reason: format!("it changed while it was being copied, {COPY_ATTEMPTS} times in a row; stop the programs writing to it and run the command again"),
530 });
531 }
532 self.stats.recopied += 1;
533 continue;
534 }
535 preserve_on_descriptor(source.as_path(), &output, &before, !self.options.update_only, &mut self.stats.losses)
536 .map_err(|failure: AttributeFailure| { return attribute_error(failure, shown.as_path()); })?;
537 drop(output);
538 self.publish(directory, relative, existing.as_ref())?;
539 if cloned.is_ok() {
540 self.stats.cloned += 1;
541 }
542 self.stats.files_copied += 1;
543 if self.stats.copied_sample.len() < SAMPLE_SIZE {
544 self.stats.copied_sample.push(relative.to_path_buf());
545 }
546 if before.nlink() > 1 {
547 self.record_link((before.dev(), before.ino()), relative);
548 }
549 return self.interrupt.check("copy-entry");
550 }
551 }
552
553 fn refresh_attributes(&mut self, source: &Path, destination: &Path, metadata: &Metadata, shown: &Path) -> Result<(), MigrateError> {
555 let handle: File = std::fs::OpenOptions::new().read(true).custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC).open(destination)
556 .map_err(|error: std::io::Error| { return MigrateError::io("open", shown, &error); })?;
557 return preserve_on_descriptor(source, &handle, metadata, true, &mut self.stats.losses)
558 .map_err(|failure: AttributeFailure| { return attribute_error(failure, shown); });
559 }
560}
561
562fn make_writable(anchor: &Anchor, shown: &Path) -> Result<(), MigrateError> {
564 let metadata: Metadata = anchor.file().metadata().map_err(|error: std::io::Error| { return MigrateError::io("examine", shown, &error); })?;
565 if metadata.mode() & 0o700 != 0o700 {
566 anchor.file().set_permissions(std::os::unix::fs::PermissionsExt::from_mode(metadata.mode() | 0o700))
567 .map_err(|error: std::io::Error| { return MigrateError::io("make writable", shown, &error); })?;
568 }
569 return Ok(());
570}
571
572pub fn sync_tree(
582 platform: &dyn Platform,
583 source: &Path,
584 destination: &Path,
585 options: SyncOptions<'_>,
586 interrupt: &Interrupt,
587) -> Result<SyncStats, MigrateError> {
588 let mut pass: Pass<'_> = Pass { platform, source, destination, options, interrupt, stats: SyncStats::default(), links: HashMap::new() };
589 let root_metadata: Metadata = std::fs::symlink_metadata(source).map_err(|error: std::io::Error| { return MigrateError::io("examine", source, &error); })?;
590 let mut stack: Vec<Work> = vec![Work::Finish(PathBuf::new(), root_metadata), Work::Visit(PathBuf::new())];
591 while let Some(work) = stack.pop() {
592 match work {
593 Work::Visit(relative) => pass.visit(relative.as_path(), &mut stack)?,
594 Work::Finish(relative, metadata) => {
595 let Some(anchor) = pass.open_destination(relative.as_path())? else { continue };
596 let shown: PathBuf = pass.shown(relative.as_path());
597 preserve_on_descriptor(source.join(&relative).as_path(), anchor.file(), &metadata, !options.update_only, &mut pass.stats.losses)
598 .map_err(|failure: AttributeFailure| { return attribute_error(failure, shown.as_path()); })?;
599 }
600 }
601 }
602 pass.stats.unreadable.sort();
603 pass.stats.conflicts.sort();
604 pass.stats.conflicts.dedup();
605 return Ok(pass.stats);
606}
607
608#[cfg(test)]
610#[path = "subvolume_sync_tests.rs"]
611mod tests;