robosync 1.0.10

High-performance file synchronization with intelligent concurrent processing
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

//! Linux-optimized parallel sync for small files

use anyhow::Result;
use rayon::prelude::*;
use std::path::PathBuf;

use crate::file_list::FileOperation;
use crate::options::SyncOptions;
use crate::sync_stats::SyncStats;

#[cfg(target_os = "linux")]
use crate::linux_fast_copy::batch_copy_files;

/// Linux-optimized synchronizer for thousands of small files
pub struct LinuxParallelSyncer {
    worker_threads: usize,
}

impl LinuxParallelSyncer {
    pub fn new(threads: usize) -> Self {
        Self {
            worker_threads: threads,
        }
    }

    /// Synchronize using Linux-specific optimizations
    #[cfg(target_os = "linux")]
    pub fn synchronize_optimized(
        &self,
        source: PathBuf,
        destination: PathBuf,
        options: SyncOptions,
    ) -> Result<SyncStats> {
        use crate::file_list::compare_file_lists_with_roots;
        use crate::file_list::generate_file_list_parallel;

        println!("Linux-optimized sync mode");
        println!("Using {} threads", self.worker_threads);

        // Use parallel directory scanning
        println!("Scanning source directory (parallel)...");
        let source_files = generate_file_list_parallel(&source, &options)?;
        println!("Found {} source files", source_files.len());

        // Scan destination if it exists
        let dest_files = if destination.exists() {
            println!("Scanning destination directory (parallel)...");
            let files = generate_file_list_parallel(&destination, &options)?;
            println!("Found {} destination files", files.len());
            files
        } else {
            println!("Destination does not exist, will create");
            Vec::new()
        };

        // Compare file lists
        println!("Analyzing changes...");
        let operations = compare_file_lists_with_roots(
            &source_files,
            &dest_files,
            &source,
            &destination,
            &options,
        );

        if operations.is_empty() {
            println!("No changes needed.");
            return Ok(SyncStats::default());
        }

        println!("Processing {} operations", operations.len());

        // Convert operations to source/dest pairs
        let copy_operations: Vec<(PathBuf, PathBuf)> = operations
            .into_par_iter()
            .filter_map(|op| match op {
                FileOperation::Create { path } | FileOperation::Update { path, .. } => {
                    let dest_path = destination.join(path.strip_prefix(&source).ok()?);
                    Some((path, dest_path))
                }
                _ => None,
            })
            .collect();

        // Use batched copy for optimal performance
        println!(
            "Starting optimized batch copy of {} files...",
            copy_operations.len()
        );
        let stats = batch_copy_files(copy_operations)?;

        println!("\nCompleted in {:?}", stats.elapsed);
        println!("Files copied: {}/{}", stats.files_copied, stats.total_files);
        println!("Speed: {:.0} files/second", stats.files_per_second());
        println!("Throughput: {:.2} MB/s", stats.throughput_mb_per_sec());

        // Convert to SyncStats
        let sync_stats = SyncStats::new();
        sync_stats.add_bytes_transferred(stats.bytes_copied);

        Ok(sync_stats)
    }

    /// Fall back to standard sync on non-Linux platforms
    #[cfg(not(target_os = "linux"))]
    pub fn synchronize_optimized(
        &self,
        source: PathBuf,
        destination: PathBuf,
        options: SyncOptions,
    ) -> Result<SyncStats> {
        use crate::parallel_sync::{ParallelSyncConfig, ParallelSyncer};

        println!("Using standard parallel sync (non-Linux platform)");

        let config = ParallelSyncConfig {
            worker_threads: self.worker_threads,
            io_threads: self.worker_threads,
            block_size: 1024,
            max_parallel_files: self.worker_threads * 2,
        };

        let syncer = ParallelSyncer::new(config);
        syncer.synchronize_with_options(source, destination, options)
    }
}