use alloc::string::{String, ToString};
use alloc::vec::Vec;
use super::compute::{
apply_delta, compute_delta, create_full_insert_delta, file_checksum, generate_signatures,
optimal_block_size,
};
use super::types::{
DEFAULT_BLOCK_SIZE, Delta, DeltaError, DeltaResult, MIN_BLOCK_SIZE, SyncEntry, SyncPlan,
SyncProgress, SyncStatus,
};
#[derive(Debug, Clone)]
pub struct FileInfo {
pub path: String,
pub size: u64,
pub mtime: u64,
pub checksum: [u64; 4],
}
impl FileInfo {
pub fn new(path: &str, size: u64, mtime: u64, checksum: [u64; 4]) -> Self {
Self {
path: path.to_string(),
size,
mtime,
checksum,
}
}
pub fn matches(&self, other: &FileInfo) -> bool {
self.checksum == other.checksum && self.size == other.size
}
pub fn is_newer_than(&self, other: &FileInfo) -> bool {
self.mtime > other.mtime
}
}
pub fn plan_sync(
source: &str,
dest: &str,
source_files: &[FileInfo],
dest_files: &[FileInfo],
timestamp: u64,
) -> SyncPlan {
let mut plan = SyncPlan::new(source, dest, timestamp);
let dest_lookup: alloc::collections::BTreeMap<&str, &FileInfo> =
dest_files.iter().map(|f| (f.path.as_str(), f)).collect();
let source_by_checksum: alloc::collections::BTreeMap<[u64; 4], &FileInfo> =
source_files.iter().map(|f| (f.checksum, f)).collect();
let mut processed_dest: alloc::collections::BTreeSet<&str> =
alloc::collections::BTreeSet::new();
for src_file in source_files {
if let Some(dest_file) = dest_lookup.get(src_file.path.as_str()) {
processed_dest.insert(&dest_file.path);
if src_file.matches(dest_file) {
plan.add_entry(SyncEntry::new(
&src_file.path,
src_file.size,
src_file.mtime,
src_file.checksum,
SyncStatus::Unchanged,
));
} else {
plan.add_entry(SyncEntry::new(
&src_file.path,
src_file.size,
src_file.mtime,
src_file.checksum,
SyncStatus::Modified,
));
}
} else {
let mut is_rename = false;
for dest_file in dest_files {
if !processed_dest.contains(dest_file.path.as_str())
&& dest_file.checksum == src_file.checksum
&& dest_file.path != src_file.path
{
processed_dest.insert(&dest_file.path);
plan.add_entry(
SyncEntry::new(
&src_file.path,
src_file.size,
src_file.mtime,
src_file.checksum,
SyncStatus::Renamed,
)
.with_original(&dest_file.path),
);
is_rename = true;
break;
}
}
if !is_rename {
plan.add_entry(SyncEntry::new(
&src_file.path,
src_file.size,
src_file.mtime,
src_file.checksum,
SyncStatus::New,
));
}
}
}
for dest_file in dest_files {
if !processed_dest.contains(dest_file.path.as_str()) {
plan.add_entry(SyncEntry::new(
&dest_file.path,
dest_file.size,
dest_file.mtime,
dest_file.checksum,
SyncStatus::Deleted,
));
}
}
plan
}
pub fn compute_file_delta(old_data: &[u8], new_data: &[u8]) -> DeltaResult<Delta> {
let block_size = optimal_block_size(old_data.len() as u64).max(MIN_BLOCK_SIZE);
let sigs = generate_signatures(old_data, block_size)?;
compute_delta(&sigs, new_data)
}
pub fn compute_new_file_delta(data: &[u8]) -> Delta {
create_full_insert_delta(data)
}
pub fn apply_file_delta(old_data: &[u8], delta: &Delta) -> DeltaResult<Vec<u8>> {
apply_delta(old_data, delta)
}
pub fn estimate_transfer_size(plan: &SyncPlan) -> u64 {
plan.entries
.iter()
.map(|e| match e.status {
SyncStatus::New => e.size,
SyncStatus::Modified => {
(e.size as f64 * 0.2) as u64 + 1024 }
SyncStatus::Renamed => 256, _ => 0,
})
.sum()
}
pub trait SyncFilesystem {
fn read_file(&self, path: &str) -> DeltaResult<Vec<u8>>;
fn write_file(&mut self, path: &str, data: &[u8]) -> DeltaResult<()>;
fn delete_file(&mut self, path: &str) -> DeltaResult<()>;
fn rename_file(&mut self, from: &str, to: &str) -> DeltaResult<()>;
fn stat_file(&self, path: &str) -> DeltaResult<FileInfo>;
fn list_files(&self, path: &str) -> DeltaResult<Vec<FileInfo>>;
}
pub fn execute_sync<S, D, F>(
plan: &SyncPlan,
source_fs: &S,
dest_fs: &mut D,
mut progress_cb: Option<F>,
start_time: u64,
) -> DeltaResult<SyncProgress>
where
S: SyncFilesystem,
D: SyncFilesystem,
F: FnMut(&SyncProgress),
{
let total_files = plan.file_count() as u64;
let total_bytes = plan.transfer_bytes();
let mut progress = SyncProgress::new(total_files, total_bytes, start_time);
for entry in &plan.entries {
match entry.status {
SyncStatus::Unchanged => {
}
SyncStatus::New => {
let data = source_fs.read_file(&entry.path)?;
dest_fs.write_file(&entry.path, &data)?;
progress.update(&entry.path, data.len() as u64, start_time);
}
SyncStatus::Modified => {
let old_data = dest_fs.read_file(&entry.path)?;
let new_data = source_fs.read_file(&entry.path)?;
let delta = compute_file_delta(&old_data, &new_data)?;
let result = apply_file_delta(&old_data, &delta)?;
dest_fs.write_file(&entry.path, &result)?;
progress.update(&entry.path, delta.transfer_size(), start_time);
}
SyncStatus::Deleted => {
dest_fs.delete_file(&entry.path)?;
progress.update(&entry.path, 0, start_time);
}
SyncStatus::Renamed => {
if let Some(original) = &entry.original_path {
dest_fs.rename_file(original, &entry.path)?;
}
progress.update(&entry.path, 0, start_time);
}
}
if let Some(ref mut cb) = progress_cb {
cb(&progress);
}
}
Ok(progress)
}
#[derive(Debug, Clone, Default)]
pub struct MemoryFs {
files: alloc::collections::BTreeMap<String, (Vec<u8>, u64)>,
}
impl MemoryFs {
pub fn new() -> Self {
Self {
files: alloc::collections::BTreeMap::new(),
}
}
pub fn add_file(&mut self, path: &str, data: &[u8], mtime: u64) {
self.files.insert(path.to_string(), (data.to_vec(), mtime));
}
pub fn paths(&self) -> Vec<&String> {
self.files.keys().collect()
}
pub fn exists(&self, path: &str) -> bool {
self.files.contains_key(path)
}
}
impl SyncFilesystem for MemoryFs {
fn read_file(&self, path: &str) -> DeltaResult<Vec<u8>> {
self.files
.get(path)
.map(|(data, _)| data.clone())
.ok_or_else(|| DeltaError::SourceNotFound(path.to_string()))
}
fn write_file(&mut self, path: &str, data: &[u8]) -> DeltaResult<()> {
let mtime = self.files.get(path).map(|(_, m)| *m + 1).unwrap_or(1);
self.files.insert(path.to_string(), (data.to_vec(), mtime));
Ok(())
}
fn delete_file(&mut self, path: &str) -> DeltaResult<()> {
self.files.remove(path);
Ok(())
}
fn rename_file(&mut self, from: &str, to: &str) -> DeltaResult<()> {
if let Some(entry) = self.files.remove(from) {
self.files.insert(to.to_string(), entry);
}
Ok(())
}
fn stat_file(&self, path: &str) -> DeltaResult<FileInfo> {
self.files
.get(path)
.map(|(data, mtime)| {
let checksum = file_checksum(data);
FileInfo::new(path, data.len() as u64, *mtime, checksum)
})
.ok_or_else(|| DeltaError::SourceNotFound(path.to_string()))
}
fn list_files(&self, _path: &str) -> DeltaResult<Vec<FileInfo>> {
Ok(self
.files
.iter()
.map(|(path, (data, mtime))| {
let checksum = file_checksum(data);
FileInfo::new(path, data.len() as u64, *mtime, checksum)
})
.collect())
}
}
#[cfg(test)]
mod tests {
use super::*;
use alloc::vec;
fn make_file_info(path: &str, data: &[u8], mtime: u64) -> FileInfo {
FileInfo::new(path, data.len() as u64, mtime, file_checksum(data))
}
#[test]
fn test_file_info_matches() {
let data = b"test data";
let info1 = make_file_info("/file.txt", data, 1000);
let info2 = make_file_info("/file.txt", data, 2000);
assert!(info1.matches(&info2));
let info3 = make_file_info("/file.txt", b"different", 1000);
assert!(!info1.matches(&info3)); }
#[test]
fn test_file_info_is_newer() {
let info1 = make_file_info("/file.txt", b"data", 1000);
let info2 = make_file_info("/file.txt", b"data", 2000);
assert!(info2.is_newer_than(&info1));
assert!(!info1.is_newer_than(&info2));
}
#[test]
fn test_plan_sync_no_changes() {
let data = b"same content";
let source = vec![make_file_info("/file.txt", data, 1000)];
let dest = vec![make_file_info("/file.txt", data, 1000)];
let plan = plan_sync("/src", "/dst", &source, &dest, 0);
assert!(plan.is_empty());
assert_eq!(plan.count_by_status(SyncStatus::Unchanged), 1);
}
#[test]
fn test_plan_sync_new_file() {
let source = vec![make_file_info("/new.txt", b"new file", 1000)];
let dest = vec![];
let plan = plan_sync("/src", "/dst", &source, &dest, 0);
assert_eq!(plan.count_by_status(SyncStatus::New), 1);
assert_eq!(plan.new_files()[0].path, "/new.txt");
}
#[test]
fn test_plan_sync_modified_file() {
let source = vec![make_file_info("/file.txt", b"new content", 2000)];
let dest = vec![make_file_info("/file.txt", b"old content", 1000)];
let plan = plan_sync("/src", "/dst", &source, &dest, 0);
assert_eq!(plan.count_by_status(SyncStatus::Modified), 1);
}
#[test]
fn test_plan_sync_deleted_file() {
let source = vec![];
let dest = vec![make_file_info("/old.txt", b"delete me", 1000)];
let plan = plan_sync("/src", "/dst", &source, &dest, 0);
assert_eq!(plan.count_by_status(SyncStatus::Deleted), 1);
}
#[test]
fn test_plan_sync_renamed_file() {
let data = b"same content different name";
let source = vec![make_file_info("/new_name.txt", data, 2000)];
let dest = vec![make_file_info("/old_name.txt", data, 1000)];
let plan = plan_sync("/src", "/dst", &source, &dest, 0);
assert_eq!(plan.count_by_status(SyncStatus::Renamed), 1);
let renamed = &plan.renamed_files()[0];
assert_eq!(renamed.path, "/new_name.txt");
assert_eq!(renamed.original_path.as_deref(), Some("/old_name.txt"));
}
#[test]
fn test_compute_file_delta() {
let old = b"old file content here";
let new = b"new file content here";
let delta = compute_file_delta(old, new).unwrap();
let result = apply_file_delta(old, &delta).unwrap();
assert_eq!(&result, new);
}
#[test]
fn test_compute_new_file_delta() {
let data = b"brand new file";
let delta = compute_new_file_delta(data);
assert!(delta.is_full_replace());
}
#[test]
fn test_estimate_transfer_size() {
let mut plan = SyncPlan::new("/src", "/dst", 0);
plan.add_entry(SyncEntry::new("/new.txt", 1000, 1, [0; 4], SyncStatus::New));
plan.add_entry(SyncEntry::new(
"/mod.txt",
5000,
2,
[0; 4],
SyncStatus::Modified,
));
let estimate = estimate_transfer_size(&plan);
assert!(estimate > 1000); assert!(estimate < 6000); }
#[test]
fn test_memory_fs_basic() {
let mut fs = MemoryFs::new();
fs.add_file("/test.txt", b"hello world", 1000);
assert!(fs.exists("/test.txt"));
let data = fs.read_file("/test.txt").unwrap();
assert_eq!(&data, b"hello world");
let info = fs.stat_file("/test.txt").unwrap();
assert_eq!(info.size, 11);
}
#[test]
fn test_memory_fs_write() {
let mut fs = MemoryFs::new();
fs.write_file("/new.txt", b"new content").unwrap();
assert!(fs.exists("/new.txt"));
let data = fs.read_file("/new.txt").unwrap();
assert_eq!(&data, b"new content");
}
#[test]
fn test_memory_fs_delete() {
let mut fs = MemoryFs::new();
fs.add_file("/delete_me.txt", b"temp", 1000);
assert!(fs.exists("/delete_me.txt"));
fs.delete_file("/delete_me.txt").unwrap();
assert!(!fs.exists("/delete_me.txt"));
}
#[test]
fn test_memory_fs_rename() {
let mut fs = MemoryFs::new();
fs.add_file("/old.txt", b"content", 1000);
fs.rename_file("/old.txt", "/new.txt").unwrap();
assert!(!fs.exists("/old.txt"));
assert!(fs.exists("/new.txt"));
let data = fs.read_file("/new.txt").unwrap();
assert_eq!(&data, b"content");
}
#[test]
fn test_execute_sync_new_files() {
let mut source = MemoryFs::new();
let mut dest = MemoryFs::new();
source.add_file("/file1.txt", b"content one", 1000);
source.add_file("/file2.txt", b"content two", 1000);
let source_files = source.list_files("/").unwrap();
let dest_files = dest.list_files("/").unwrap();
let plan = plan_sync("/src", "/dst", &source_files, &dest_files, 0);
let progress =
execute_sync(&plan, &source, &mut dest, None::<fn(&SyncProgress)>, 0).unwrap();
assert!(dest.exists("/file1.txt"));
assert!(dest.exists("/file2.txt"));
assert_eq!(progress.files_done, 2);
}
#[test]
fn test_execute_sync_modified_files() {
let mut source = MemoryFs::new();
let mut dest = MemoryFs::new();
source.add_file("/file.txt", b"new content here", 2000);
dest.add_file("/file.txt", b"old content here", 1000);
let source_files = source.list_files("/").unwrap();
let dest_files = dest.list_files("/").unwrap();
let plan = plan_sync("/src", "/dst", &source_files, &dest_files, 0);
execute_sync(&plan, &source, &mut dest, None::<fn(&SyncProgress)>, 0).unwrap();
let result = dest.read_file("/file.txt").unwrap();
assert_eq!(&result, b"new content here");
}
#[test]
fn test_execute_sync_deleted_files() {
let mut source = MemoryFs::new();
let mut dest = MemoryFs::new();
dest.add_file("/delete_me.txt", b"old file", 1000);
let source_files = source.list_files("/").unwrap();
let dest_files = dest.list_files("/").unwrap();
let plan = plan_sync("/src", "/dst", &source_files, &dest_files, 0);
execute_sync(&plan, &source, &mut dest, None::<fn(&SyncProgress)>, 0).unwrap();
assert!(!dest.exists("/delete_me.txt"));
}
#[test]
fn test_execute_sync_renamed_files() {
let mut source = MemoryFs::new();
let mut dest = MemoryFs::new();
let content = b"same content";
source.add_file("/new_name.txt", content, 2000);
dest.add_file("/old_name.txt", content, 1000);
let source_files = source.list_files("/").unwrap();
let dest_files = dest.list_files("/").unwrap();
let plan = plan_sync("/src", "/dst", &source_files, &dest_files, 0);
execute_sync(&plan, &source, &mut dest, None::<fn(&SyncProgress)>, 0).unwrap();
assert!(!dest.exists("/old_name.txt"));
assert!(dest.exists("/new_name.txt"));
}
#[test]
fn test_full_sync_scenario() {
let mut source = MemoryFs::new();
let mut dest = MemoryFs::new();
source.add_file("/unchanged.txt", b"same", 1000);
source.add_file("/modified.txt", b"new version", 2000);
source.add_file("/new.txt", b"brand new", 3000);
source.add_file("/renamed.txt", b"rename this", 2000);
dest.add_file("/unchanged.txt", b"same", 1000);
dest.add_file("/modified.txt", b"old version", 1000);
dest.add_file("/to_delete.txt", b"delete me", 1000);
dest.add_file("/old_name.txt", b"rename this", 1000);
let source_files = source.list_files("/").unwrap();
let dest_files = dest.list_files("/").unwrap();
let plan = plan_sync("/src", "/dst", &source_files, &dest_files, 0);
assert_eq!(plan.count_by_status(SyncStatus::Unchanged), 1);
assert_eq!(plan.count_by_status(SyncStatus::Modified), 1);
assert_eq!(plan.count_by_status(SyncStatus::New), 1);
assert_eq!(plan.count_by_status(SyncStatus::Deleted), 1);
assert_eq!(plan.count_by_status(SyncStatus::Renamed), 1);
execute_sync(&plan, &source, &mut dest, None::<fn(&SyncProgress)>, 0).unwrap();
assert!(dest.exists("/unchanged.txt"));
assert!(dest.exists("/modified.txt"));
assert!(dest.exists("/new.txt"));
assert!(dest.exists("/renamed.txt"));
assert!(!dest.exists("/to_delete.txt"));
assert!(!dest.exists("/old_name.txt"));
assert_eq!(dest.read_file("/modified.txt").unwrap(), b"new version");
assert_eq!(dest.read_file("/new.txt").unwrap(), b"brand new");
}
}