1use super::fs_primitive_error::PrimitiveError;
25use super::subvolume_anchor::Anchor;
26use super::subvolume_error::{Interrupt, MigrateError};
27use super::subvolume_platform::Platform;
28use super::subvolume_remove::remove_tree;
29use super::subvolume_sync_attrs::{
30 AttributeFailure, AttributeLosses, carried_attributes_equal, make_fifo, open_fifo, preserve_on_descriptor, preserve_on_link,
31};
32use std::collections::{HashMap, HashSet};
33use std::ffi::{OsStr, OsString};
34use std::fs::{File, Metadata};
35use std::io::Seek;
36use std::os::unix::fs::{DirBuilderExt, FileTypeExt, MetadataExt, OpenOptionsExt};
37use std::path::{Path, PathBuf};
38
39const PARTIAL_NAME: &str = ".cli-git-migrate-partial";
41
42const COPY_ATTEMPTS: u32 = 3;
44
45#[derive(Clone, Copy, Debug)]
57pub struct SyncOptions<'a> {
58 pub delete: bool,
60 pub update_only: bool,
62 pub excluded: &'a [PathBuf],
64 pub created_since: Option<i64>,
67}
68
69#[derive(Clone, Debug, Default, Eq, PartialEq)]
79pub struct SyncStats {
80 pub entries: u64,
82 pub directories: u64,
84 pub files_copied: u64,
86 pub cloned: u64,
88 pub hard_linked: u64,
90 pub symlinks: u64,
92 pub fifos: u64,
94 pub unchanged: u64,
96 pub deleted: u64,
98 pub vanished: u64,
100 pub unreadable: Vec<PathBuf>,
102 pub special_skipped: Vec<PathBuf>,
104 pub losses: AttributeLosses,
106 pub conflicts: Vec<PathBuf>,
109 pub recopied: u64,
111 pub left_out: u64,
113 pub copied_sample: Vec<PathBuf>,
115}
116
117const SAMPLE_SIZE: usize = 20;
119
120enum Work {
122 Visit(PathBuf),
124 Finish(PathBuf, Metadata),
126}
127
128struct Directory {
130 anchor: Anchor,
132 temporary: OsString,
134}
135
136impl Directory {
137 fn child(&self, name: &OsStr) -> PathBuf {
139 return self.anchor.child(name);
140 }
141
142 fn partial(&self) -> PathBuf {
144 return self.anchor.child(self.temporary.as_os_str());
145 }
146}
147
148struct Pass<'a> {
150 platform: &'a dyn Platform,
151 source: &'a Path,
152 destination: &'a Path,
153 options: SyncOptions<'a>,
154 interrupt: &'a Interrupt,
155 stats: SyncStats,
156 links: HashMap<(u64, u64), Vec<PathBuf>>,
159}
160
161fn file_is_equal(copy: &Metadata, original: &Metadata) -> bool {
164 return copy.is_file()
165 && copy.len() == original.len()
166 && copy.mtime() == original.mtime()
167 && copy.mtime_nsec() == original.mtime_nsec()
168 && copy.mode() & 0o7777 == original.mode() & 0o7777;
169}
170
171fn is_newer(left: &Metadata, right: &Metadata) -> bool {
173 return (left.mtime(), left.mtime_nsec()) > (right.mtime(), right.mtime_nsec());
174}
175
176fn is_settled(before: &Metadata, after: &Metadata) -> bool {
178 return before.len() == after.len()
179 && (before.mtime(), before.mtime_nsec()) == (after.mtime(), after.mtime_nsec())
180 && (before.ctime(), before.ctime_nsec()) == (after.ctime(), after.ctime_nsec());
181}
182
183fn remove_entry(path: &Path, shown: &Path) -> Result<(), MigrateError> {
185 return remove_tree(path).map_err(|error: std::io::Error| { return MigrateError::io("remove", shown, &error); });
186}
187
188fn remove_if_present(path: &Path, shown: &Path) -> Result<(), MigrateError> {
190 return match std::fs::symlink_metadata(path) {
191 Ok(_) => remove_entry(path, shown),
192 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
193 Err(error) => Err(MigrateError::io("examine", shown, &error)),
194 };
195}
196
197fn attribute_error(failure: AttributeFailure, shown: &Path) -> MigrateError {
199 return MigrateError::io(failure.action, shown, &failure.error);
200}
201
202fn is_replaced(error: &std::io::Error) -> bool {
204 return matches!(error.raw_os_error(), Some(libc::ELOOP | libc::ENOTDIR | libc::ENOENT));
205}
206
207impl Pass<'_> {
208 fn shown(&self, relative: &Path) -> PathBuf {
210 return self.destination.join(relative);
211 }
212
213 fn open_destination(&mut self, relative: &Path) -> Result<Option<Anchor>, MigrateError> {
216 return match Anchor::open_below(self.destination, relative) {
217 Ok(anchor) => Ok(Some(anchor)),
218 Err(error) if is_replaced(&error) && self.options.update_only && !relative.as_os_str().is_empty() => {
219 self.stats.conflicts.push(relative.to_path_buf());
220 Ok(None)
221 }
222 Err(error) => Err(MigrateError::io("open the directory", self.shown(relative).as_path(), &error)),
223 };
224 }
225
226 fn replace_allowed(&mut self, relative: &Path, found: &Metadata, original: &Metadata) -> bool {
229 if !self.options.update_only {
230 return true;
231 }
232 if found.is_dir() || !is_newer(original, found) {
233 self.stats.conflicts.push(relative.to_path_buf());
234 return false;
235 }
236 return true;
237 }
238
239 fn visit(&mut self, relative: &Path, stack: &mut Vec<Work>) -> Result<(), MigrateError> {
241 let source_directory: PathBuf = self.source.join(relative);
242 let listing: std::fs::ReadDir = match std::fs::read_dir(&source_directory) {
243 Ok(found) => found,
244 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
245 self.stats.vanished += 1;
246 return Ok(());
247 }
248 Err(error) => return Err(MigrateError::UnreadableDirectory { path: source_directory, reason: error.to_string() }),
249 };
250 let Some(anchor) = self.open_destination(relative)? else { return Ok(()) };
251 make_writable(&anchor, self.shown(relative).as_path())?;
252 let mut names: HashSet<OsString> = HashSet::new();
253 let mut children: Vec<(OsString, Metadata)> = Vec::new();
254 for item in listing {
255 let entry: std::fs::DirEntry = item.map_err(|error: std::io::Error| {
256 return MigrateError::UnreadableDirectory { path: source_directory.clone(), reason: error.to_string() };
257 })?;
258 names.insert(entry.file_name());
259 if self.options.excluded.iter().any(|path: &PathBuf| { return *path == relative.join(entry.file_name()); }) {
260 continue;
261 }
262 match entry.metadata() {
263 Ok(metadata) => children.push((entry.file_name(), metadata)),
264 Err(error) if error.kind() == std::io::ErrorKind::NotFound => self.stats.vanished += 1,
265 Err(error) => return Err(MigrateError::UnreadableDirectory { path: source_directory, reason: error.to_string() }),
266 }
267 }
268 let mut temporary: OsString = OsString::from(PARTIAL_NAME);
269 while names.contains(&temporary) {
270 temporary.push("~");
271 }
272 let directory: Directory = Directory { anchor, temporary };
273 remove_if_present(directory.partial().as_path(), self.shown(relative).join(&directory.temporary).as_path())?;
275 if self.options.delete && !self.options.update_only {
276 self.delete_extras(&directory, relative, &names)?;
277 }
278 for (name, metadata) in children {
279 self.stats.entries += 1;
280 self.handle_child(&directory, relative.join(&name), metadata, stack)?;
281 }
282 return Ok(());
283 }
284
285 fn delete_extras(&mut self, directory: &Directory, relative: &Path, names: &HashSet<OsString>) -> Result<(), MigrateError> {
287 let Ok(listing) = std::fs::read_dir(directory.anchor.path()) else { return Ok(()) };
288 for item in listing.flatten() {
289 if names.contains(&item.file_name()) {
290 continue;
291 }
292 if self.options.excluded.iter().any(|path: &PathBuf| { return *path == relative.join(item.file_name()); }) {
293 continue;
294 }
295 remove_entry(directory.child(item.file_name().as_os_str()).as_path(), self.shown(relative).join(item.file_name()).as_path())?;
296 self.stats.deleted += 1;
297 }
298 return Ok(());
299 }
300
301 fn handle_child(&mut self, directory: &Directory, relative: PathBuf, metadata: Metadata, stack: &mut Vec<Work>) -> Result<(), MigrateError> {
303 let name: OsString = relative.file_name().map(OsStr::to_os_string).unwrap_or_default();
304 let destination: PathBuf = directory.child(name.as_os_str());
305 let existing: Option<Metadata> = std::fs::symlink_metadata(&destination).ok();
306 if existing.is_none() && self.options.update_only && self.options.created_since.is_some_and(|since: i64| { return metadata.mtime() < since; }) {
307 self.stats.left_out += 1;
308 return Ok(());
309 }
310 let file_type: std::fs::FileType = metadata.file_type();
311 if file_type.is_dir() {
312 if let Some(found) = &existing && !found.is_dir() {
313 if !self.replace_allowed(relative.as_path(), found, &metadata) {
314 return Ok(());
315 }
316 remove_entry(destination.as_path(), self.shown(relative.as_path()).as_path())?;
317 }
318 if existing.as_ref().is_none_or(|found: &Metadata| { return !found.is_dir(); }) {
319 std::fs::DirBuilder::new().mode(0o700).create(&destination).map_err(|error: std::io::Error| {
320 return MigrateError::io("create the directory", self.shown(relative.as_path()).as_path(), &error);
321 })?;
322 }
323 self.stats.directories += 1;
324 stack.push(Work::Finish(relative.clone(), metadata));
325 stack.push(Work::Visit(relative));
326 return Ok(());
327 }
328 if file_type.is_symlink() {
329 return self.copy_symlink(directory, &relative, &metadata, existing);
330 }
331 if file_type.is_fifo() {
332 return self.copy_fifo(directory, &relative, &metadata, existing);
333 }
334 if file_type.is_file() {
335 return self.copy_file(directory, &relative, &metadata, existing);
336 }
337 self.stats.special_skipped.push(relative);
338 return Ok(());
339 }
340
341 fn publish(&self, directory: &Directory, relative: &Path, existing: Option<&Metadata>) -> Result<(), MigrateError> {
343 let name: &OsStr = relative.file_name().unwrap_or_default();
344 let destination: PathBuf = directory.child(name);
345 if let Some(found) = existing && found.is_dir() {
346 remove_entry(destination.as_path(), self.shown(relative).as_path())?;
347 }
348 return std::fs::rename(directory.partial(), &destination).map_err(|error: std::io::Error| {
349 return MigrateError::io("put the finished copy in place of", self.shown(relative).as_path(), &error);
350 });
351 }
352
353 fn copy_symlink(&mut self, directory: &Directory, relative: &Path, metadata: &Metadata, existing: Option<Metadata>) -> Result<(), MigrateError> {
355 let source: PathBuf = self.source.join(relative);
356 let destination: PathBuf = directory.child(relative.file_name().unwrap_or_default());
357 let target: PathBuf = std::fs::read_link(&source).map_err(|error: std::io::Error| { return MigrateError::io("read the link", source.as_path(), &error); })?;
358 if let Some(found) = &existing {
359 if found.file_type().is_symlink() && std::fs::read_link(&destination).ok().as_ref() == Some(&target) {
360 if (found.mtime(), found.mtime_nsec()) != (metadata.mtime(), metadata.mtime_nsec()) && !self.options.update_only {
361 preserve_on_link(source.as_path(), destination.as_path(), metadata, &mut self.stats.losses)
362 .map_err(|failure: AttributeFailure| { return attribute_error(failure, self.shown(relative).as_path()); })?;
363 }
364 self.stats.unchanged += 1;
365 return Ok(());
366 }
367 if !self.replace_allowed(relative, found, metadata) {
368 return Ok(());
369 }
370 }
371 let partial: PathBuf = directory.partial();
372 std::os::unix::fs::symlink(&target, &partial).map_err(|error: std::io::Error| { return MigrateError::io("create the link", self.shown(relative).as_path(), &error); })?;
373 preserve_on_link(source.as_path(), partial.as_path(), metadata, &mut self.stats.losses)
374 .map_err(|failure: AttributeFailure| { return attribute_error(failure, self.shown(relative).as_path()); })?;
375 self.publish(directory, relative, existing.as_ref())?;
376 self.stats.symlinks += 1;
377 return Ok(());
378 }
379
380 fn copy_fifo(&mut self, directory: &Directory, relative: &Path, metadata: &Metadata, existing: Option<Metadata>) -> Result<(), MigrateError> {
382 let source: PathBuf = self.source.join(relative);
383 let destination: PathBuf = directory.child(relative.file_name().unwrap_or_default());
384 if let Some(found) = &existing {
385 if found.file_type().is_fifo() {
386 let same: bool = found.mode() & 0o7777 == metadata.mode() & 0o7777 && (found.mtime(), found.mtime_nsec()) == (metadata.mtime(), metadata.mtime_nsec());
387 if !same && !self.options.update_only {
388 let handle: File = open_fifo(destination.as_path()).map_err(|error: std::io::Error| { return MigrateError::io("open the named pipe", self.shown(relative).as_path(), &error); })?;
389 preserve_on_descriptor(source.as_path(), &handle, metadata, true, &mut self.stats.losses)
390 .map_err(|failure: AttributeFailure| { return attribute_error(failure, self.shown(relative).as_path()); })?;
391 }
392 self.stats.unchanged += 1;
393 return Ok(());
394 }
395 if !self.replace_allowed(relative, found, metadata) {
396 return Ok(());
397 }
398 }
399 let partial: PathBuf = directory.partial();
400 make_fifo(partial.as_path(), metadata.mode()).map_err(|error: std::io::Error| { return MigrateError::io("create the named pipe", self.shown(relative).as_path(), &error); })?;
401 let handle: File = open_fifo(partial.as_path()).map_err(|error: std::io::Error| { return MigrateError::io("open the named pipe", self.shown(relative).as_path(), &error); })?;
402 preserve_on_descriptor(source.as_path(), &handle, metadata, true, &mut self.stats.losses)
403 .map_err(|failure: AttributeFailure| { return attribute_error(failure, self.shown(relative).as_path()); })?;
404 self.publish(directory, relative, existing.as_ref())?;
405 self.stats.fifos += 1;
406 return Ok(());
407 }
408
409 fn is_linked_as_recorded(&self, key: (u64, u64), found: &Metadata) -> bool {
411 let Some(copies) = self.links.get(&key) else { return true };
412 return copies.iter().any(|copy: &PathBuf| {
413 return std::fs::symlink_metadata(self.destination.join(copy)).is_ok_and(|copy_meta: Metadata| { return copy_meta.ino() == found.ino(); });
414 });
415 }
416
417 fn record_link(&mut self, key: (u64, u64), relative: &Path) {
419 let copies: &mut Vec<PathBuf> = self.links.entry(key).or_default();
420 if !copies.iter().any(|copy: &PathBuf| { return copy == relative; }) {
421 copies.push(relative.to_path_buf());
422 }
423 }
424
425 fn link_to_earlier_copy(&mut self, directory: &Directory, relative: &Path, key: (u64, u64), existing: Option<&Metadata>) -> Result<bool, MigrateError> {
427 for copy in self.links.get(&key).cloned().unwrap_or_default() {
428 let Ok(earlier) = Anchor::open_below(self.destination, copy.parent().unwrap_or(Path::new(""))) else { continue };
429 let first: PathBuf = earlier.child(copy.file_name().unwrap_or_default());
430 remove_if_present(directory.partial().as_path(), self.shown(relative).as_path())?;
431 if self.platform.hard_link(first.as_path(), directory.partial().as_path()).is_err() {
432 continue;
433 }
434 self.publish(directory, relative, existing)?;
435 self.stats.hard_linked += 1;
436 return Ok(true);
437 }
438 return Ok(false);
439 }
440
441 fn copy_file(&mut self, directory: &Directory, relative: &Path, listing: &Metadata, existing: Option<Metadata>) -> Result<(), MigrateError> {
443 let source: PathBuf = self.source.join(relative);
444 let destination: PathBuf = directory.child(relative.file_name().unwrap_or_default());
445 let shown: PathBuf = self.shown(relative);
446 let listed_key: (u64, u64) = (listing.dev(), listing.ino());
447 if let Some(found) = &existing {
448 if !found.is_file() {
449 if !self.replace_allowed(relative, found, listing) {
450 return Ok(());
451 }
452 } else if self.options.update_only && is_newer(found, listing) {
453 self.stats.unchanged += 1;
455 return Ok(());
456 } else if file_is_equal(found, listing) && (listing.nlink() <= 1 || self.is_linked_as_recorded(listed_key, found)) {
457 if !self.options.update_only && !carried_attributes_equal(source.as_path(), destination.as_path()) {
458 self.refresh_attributes(&source, &destination, listing, shown.as_path())?;
459 }
460 self.stats.unchanged += 1;
461 if listing.nlink() > 1 {
462 self.record_link(listed_key, relative);
463 }
464 return Ok(());
465 }
466 }
467 if listing.nlink() > 1 && self.link_to_earlier_copy(directory, relative, listed_key, existing.as_ref())? {
468 return Ok(());
469 }
470 let mut input: File = match File::open(&source) {
471 Ok(opened) => opened,
472 Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => {
473 self.stats.unreadable.push(relative.to_path_buf());
474 return Ok(());
475 }
476 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
477 self.stats.vanished += 1;
478 return Ok(());
479 }
480 Err(error) => return Err(MigrateError::io("open", source.as_path(), &error)),
481 };
482 let mut attempt: u32 = 0;
483 loop {
484 attempt += 1;
485 let before: Metadata = input.metadata().map_err(|error: std::io::Error| { return MigrateError::io("examine", source.as_path(), &error); })?;
486 if !before.is_file() {
487 self.stats.vanished += 1;
488 return Ok(());
489 }
490 remove_if_present(directory.partial().as_path(), shown.as_path())?;
491 let mut output: File = std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(directory.partial())
492 .map_err(|error: std::io::Error| { return MigrateError::io("create", shown.as_path(), &error); })?;
493 self.interrupt.check("copy-partial")?;
494 input.rewind().map_err(|error: std::io::Error| { return MigrateError::io("rewind", source.as_path(), &error); })?;
495 let cloned: Result<(), PrimitiveError> = self.platform.clone_file(&input, &output, directory.partial().as_path());
496 if cloned.is_err() {
497 std::io::copy(&mut input, &mut output).map_err(|error: std::io::Error| { return MigrateError::io("copy to", shown.as_path(), &error); })?;
498 }
499 let after: Metadata = input.metadata().map_err(|error: std::io::Error| { return MigrateError::io("examine", source.as_path(), &error); })?;
500 if !is_settled(&before, &after) {
501 drop(output);
502 remove_if_present(directory.partial().as_path(), shown.as_path())?;
503 if attempt >= COPY_ATTEMPTS {
504 return Err(MigrateError::Io {
505 action: String::from("copy"),
506 path: source,
507 reason: format!("it changed while it was being copied, {COPY_ATTEMPTS} times in a row; stop the programs writing to it and run the command again"),
508 });
509 }
510 self.stats.recopied += 1;
511 continue;
512 }
513 preserve_on_descriptor(source.as_path(), &output, &before, !self.options.update_only, &mut self.stats.losses)
514 .map_err(|failure: AttributeFailure| { return attribute_error(failure, shown.as_path()); })?;
515 drop(output);
516 self.publish(directory, relative, existing.as_ref())?;
517 if cloned.is_ok() {
518 self.stats.cloned += 1;
519 }
520 self.stats.files_copied += 1;
521 if self.stats.copied_sample.len() < SAMPLE_SIZE {
522 self.stats.copied_sample.push(relative.to_path_buf());
523 }
524 if before.nlink() > 1 {
525 self.record_link((before.dev(), before.ino()), relative);
526 }
527 return self.interrupt.check("copy-entry");
528 }
529 }
530
531 fn refresh_attributes(&mut self, source: &Path, destination: &Path, metadata: &Metadata, shown: &Path) -> Result<(), MigrateError> {
533 let handle: File = std::fs::OpenOptions::new().read(true).custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC).open(destination)
534 .map_err(|error: std::io::Error| { return MigrateError::io("open", shown, &error); })?;
535 return preserve_on_descriptor(source, &handle, metadata, true, &mut self.stats.losses)
536 .map_err(|failure: AttributeFailure| { return attribute_error(failure, shown); });
537 }
538}
539
540fn make_writable(anchor: &Anchor, shown: &Path) -> Result<(), MigrateError> {
542 let metadata: Metadata = anchor.file().metadata().map_err(|error: std::io::Error| { return MigrateError::io("examine", shown, &error); })?;
543 if metadata.mode() & 0o700 != 0o700 {
544 anchor.file().set_permissions(std::os::unix::fs::PermissionsExt::from_mode(metadata.mode() | 0o700))
545 .map_err(|error: std::io::Error| { return MigrateError::io("make writable", shown, &error); })?;
546 }
547 return Ok(());
548}
549
550pub fn sync_tree(
560 platform: &dyn Platform,
561 source: &Path,
562 destination: &Path,
563 options: SyncOptions<'_>,
564 interrupt: &Interrupt,
565) -> Result<SyncStats, MigrateError> {
566 let mut pass: Pass<'_> = Pass { platform, source, destination, options, interrupt, stats: SyncStats::default(), links: HashMap::new() };
567 let root_metadata: Metadata = std::fs::symlink_metadata(source).map_err(|error: std::io::Error| { return MigrateError::io("examine", source, &error); })?;
568 let mut stack: Vec<Work> = vec![Work::Finish(PathBuf::new(), root_metadata), Work::Visit(PathBuf::new())];
569 while let Some(work) = stack.pop() {
570 match work {
571 Work::Visit(relative) => pass.visit(relative.as_path(), &mut stack)?,
572 Work::Finish(relative, metadata) => {
573 let Some(anchor) = pass.open_destination(relative.as_path())? else { continue };
574 let shown: PathBuf = pass.shown(relative.as_path());
575 preserve_on_descriptor(source.join(&relative).as_path(), anchor.file(), &metadata, !options.update_only, &mut pass.stats.losses)
576 .map_err(|failure: AttributeFailure| { return attribute_error(failure, shown.as_path()); })?;
577 }
578 }
579 }
580 pass.stats.unreadable.sort();
581 pass.stats.conflicts.sort();
582 pass.stats.conflicts.dedup();
583 return Ok(pass.stats);
584}
585
586#[cfg(test)]
588#[path = "subvolume_sync_tests.rs"]
589mod tests;