ddup_bak/
repository.rs

1use crate::{
2    archive::{Archive, CompressionFormat, ProgressCallback, entries::Entry},
3    chunks::ChunkIndex,
4};
5use std::{
6    fs::{File, FileTimes},
7    io::{BufWriter, Read, Write},
8    path::{Path, PathBuf},
9    sync::{Arc, RwLock},
10};
11
12pub type DeletionProgressCallback = Option<Arc<dyn Fn(u64, bool) + Send + Sync + 'static>>;
13
14pub struct Repository {
15    pub directory: PathBuf,
16    pub save_on_drop: bool,
17
18    pub chunk_index: ChunkIndex,
19    pub ignored_files: Vec<String>,
20}
21
22impl Repository {
23    /// Opens an existing repository.
24    /// The repository must be initialized with `new` before use.
25    /// The repository directory must contain a `.ddup-bak` directory.
26    pub fn open(directory: &Path) -> std::io::Result<Self> {
27        let chunk_index = ChunkIndex::open(directory.join(".ddup-bak"))?;
28        let mut ignored_files = Vec::new();
29
30        let ignored_files_path = directory.join(".ddup-bak/ignored_files");
31        if ignored_files_path.exists() {
32            let text = std::fs::read_to_string(&ignored_files_path)?;
33            for line in text.lines() {
34                if !line.is_empty() {
35                    ignored_files.push(line.to_string());
36                }
37            }
38        }
39
40        Ok(Self {
41            directory: directory.to_path_buf(),
42            save_on_drop: true,
43            chunk_index,
44            ignored_files,
45        })
46    }
47
48    pub fn new(
49        directory: &Path,
50        chunk_size: usize,
51        max_chunk_count: usize,
52        ignored_files: Vec<String>,
53    ) -> Self {
54        let chunk_index = ChunkIndex::new(directory.join(".ddup-bak"), chunk_size, max_chunk_count);
55
56        std::fs::create_dir_all(directory.join(".ddup-bak/archives")).unwrap();
57        std::fs::create_dir_all(directory.join(".ddup-bak/archives-tmp")).unwrap();
58        std::fs::create_dir_all(directory.join(".ddup-bak/archives-restored")).unwrap();
59        std::fs::create_dir_all(directory.join(".ddup-bak/chunks")).unwrap();
60        std::fs::write(directory.join(".ddup-bak/ignored_files"), "").unwrap();
61
62        Self {
63            directory: directory.to_path_buf(),
64            save_on_drop: true,
65            chunk_index,
66            ignored_files,
67        }
68    }
69
70    #[inline]
71    fn archive_path(&self, name: &str) -> PathBuf {
72        self.directory
73            .join(".ddup-bak/archives")
74            .join(format!("{}.ddup", name))
75    }
76
77    /// Sets the save_on_drop flag.
78    /// If set to true, the repository will save all changes to disk when dropped.
79    /// If set to false, the repository will not save changes when dropped.
80    /// This is useful for testing purposes, where you may want to discard changes.
81    /// By default, this flag is set to true and should NOT be changed.
82    #[inline]
83    pub const fn set_save_on_drop(&mut self, save_on_drop: bool) -> &mut Self {
84        self.save_on_drop = save_on_drop;
85        self.chunk_index.set_save_on_drop(save_on_drop);
86
87        self
88    }
89
90    /// Adds a file to the ignored list.
91    /// If the file is already in the list, it does nothing.
92    /// The file is added as a relative path from the repository directory.
93    #[inline]
94    pub fn add_ignored_file(&mut self, file: &str) -> &mut Self {
95        if !self.ignored_files.contains(&file.to_string()) {
96            self.ignored_files.push(file.to_string());
97        }
98
99        self
100    }
101
102    /// Removes a file from the ignored list.
103    /// If the file is not in the list, it does nothing.
104    #[inline]
105    pub fn remove_ignored_file(&mut self, file: &str) {
106        if let Some(pos) = self.ignored_files.iter().position(|x| x == file) {
107            self.ignored_files.remove(pos);
108        }
109    }
110
111    /// Checks if a file is ignored.
112    /// Returns true if the file is ignored, false otherwise.
113    pub fn is_ignored(&self, file: &str) -> bool {
114        self.ignored_files.contains(&file.to_string())
115    }
116
117    /// Returns a reference to the list of ignored files.
118    #[inline]
119    pub fn get_ignored_files(&self) -> &[String] {
120        &self.ignored_files
121    }
122
123    /// Lists all archives in the repository.
124    /// Returns a vector of archive names without the ".ddup" extension.
125    /// Example: "my_archive" instead of "my_archive.ddup".
126    /// The archives are stored in the ".ddup-bak/archives" directory.
127    pub fn list_archives(&self) -> std::io::Result<Vec<String>> {
128        let mut archives = Vec::new();
129        let archive_dir = self.directory.join(".ddup-bak/archives");
130
131        for entry in std::fs::read_dir(archive_dir)?.flatten() {
132            if let Some(name) = entry.file_name().to_str() {
133                if let Some(stripped) = name.strip_suffix(".ddup") {
134                    archives.push(stripped.to_string());
135                }
136            }
137        }
138
139        Ok(archives)
140    }
141
142    /// Gets an archive by name.
143    /// Do not use this method to extract data, the data is chunked and compressed.
144    /// Use `restore_archive` instead.
145    pub fn get_archive(&self, name: &str) -> std::io::Result<Archive> {
146        let archive_path = self.archive_path(name);
147
148        Archive::open(archive_path.to_str().unwrap())
149    }
150
151    pub fn clean(&self, progress: DeletionProgressCallback) -> std::io::Result<()> {
152        self.chunk_index.clean(progress)?;
153
154        Ok(())
155    }
156
157    fn recursive_create_archive(
158        chunk_index: &ChunkIndex,
159        entry: std::fs::DirEntry,
160        temp_path: &Path,
161        progress_chunking: ProgressCallback,
162        scope: &rayon::Scope,
163        error: Arc<RwLock<Option<std::io::Error>>>,
164    ) -> std::io::Result<()> {
165        let path = entry.path();
166        let destination = temp_path.join(path.file_name().unwrap());
167        let metadata = path.symlink_metadata()?;
168
169        if error.read().unwrap().is_some() {
170            return Ok(());
171        }
172
173        if let Some(f) = progress_chunking.clone() {
174            f(&path)
175        }
176
177        if metadata.is_file() {
178            let chunks = chunk_index.chunk_file(&path, CompressionFormat::Deflate, Some(scope))?;
179
180            let file = File::create(&destination)?;
181
182            let mut writer = BufWriter::new(&file);
183            for id in chunks {
184                writer.write_all(&crate::varint::encode_u64(id))?;
185            }
186
187            writer.flush()?;
188            file.sync_all()?;
189
190            file.set_permissions(metadata.permissions())?;
191            file.set_times(FileTimes::new().set_modified(metadata.modified()?))?;
192
193            #[cfg(unix)]
194            {
195                use std::os::unix::fs::MetadataExt;
196
197                let (uid, gid) = (metadata.uid(), metadata.gid());
198                std::os::unix::fs::lchown(&destination, Some(uid), Some(gid))?;
199            }
200        } else if metadata.is_dir() {
201            std::fs::create_dir_all(&destination)?;
202
203            #[cfg(unix)]
204            {
205                use std::os::unix::fs::MetadataExt;
206
207                let (uid, gid) = (metadata.uid(), metadata.gid());
208                std::os::unix::fs::lchown(&destination, Some(uid), Some(gid))?;
209            }
210
211            for sub_entry in std::fs::read_dir(&path)?.flatten() {
212                scope.spawn({
213                    let error = Arc::clone(&error);
214                    let destination = destination.clone();
215                    let chunk_index = chunk_index.clone();
216                    let progress_chunking = progress_chunking.clone();
217
218                    move |scope| {
219                        if let Err(err) = Self::recursive_create_archive(
220                            &chunk_index,
221                            sub_entry,
222                            &destination,
223                            progress_chunking,
224                            scope,
225                            Arc::clone(&error),
226                        ) {
227                            let mut error = error.write().unwrap();
228                            if error.is_none() {
229                                *error = Some(err);
230                            }
231                        }
232                    }
233                });
234            }
235        } else if metadata.is_symlink() {
236            if let Ok(target) = std::fs::read_link(&path) {
237                #[cfg(unix)]
238                {
239                    use std::os::unix::fs::MetadataExt;
240
241                    std::os::unix::fs::symlink(target, &destination)?;
242
243                    std::fs::set_permissions(&destination, metadata.permissions())?;
244                    let (uid, gid) = (metadata.uid(), metadata.gid());
245                    std::os::unix::fs::lchown(&destination, Some(uid), Some(gid))?;
246                }
247                #[cfg(windows)]
248                {
249                    if target.is_dir() {
250                        std::os::windows::fs::symlink_dir(target, &destination)?;
251                    } else {
252                        std::os::windows::fs::symlink_file(target, &destination)?;
253                    }
254                }
255            }
256        }
257
258        Ok(())
259    }
260
261    pub fn create_archive(
262        &mut self,
263        name: &str,
264        directory: Option<&Path>,
265        progress_chunking: ProgressCallback,
266        progress_archiving: ProgressCallback,
267        threads: usize,
268    ) -> std::io::Result<Archive> {
269        if self.list_archives()?.contains(&name.to_string()) {
270            return Err(std::io::Error::new(
271                std::io::ErrorKind::AlreadyExists,
272                format!("Archive {} already exists", name),
273            ));
274        }
275
276        let archive_path = self.archive_path(name);
277        let archive_tmp_path = self.directory.join(".ddup-bak/archives-tmp").join(name);
278
279        std::fs::create_dir_all(&archive_tmp_path)?;
280
281        let worker_pool = Arc::new(
282            rayon::ThreadPoolBuilder::new()
283                .num_threads(threads)
284                .build()
285                .unwrap(),
286        );
287        let error = Arc::new(RwLock::new(None));
288
289        worker_pool.in_place_scope(|scope| {
290            for entry in std::fs::read_dir(directory.unwrap_or(&self.directory))
291                .unwrap()
292                .flatten()
293            {
294                let path = entry.path();
295                if self.is_ignored(path.to_str().unwrap())
296                    || path.file_name() == Some(".ddup-bak".as_ref())
297                {
298                    continue;
299                }
300
301                scope.spawn({
302                    let error = Arc::clone(&error);
303                    let chunk_index = self.chunk_index.clone();
304                    let archive_tmp_path = archive_tmp_path.to_path_buf();
305                    let progress_chunking = progress_chunking.clone();
306
307                    move |scope| {
308                        if let Err(err) = Self::recursive_create_archive(
309                            &chunk_index,
310                            entry,
311                            &archive_tmp_path,
312                            progress_chunking,
313                            scope,
314                            Arc::clone(&error),
315                        ) {
316                            let mut error = error.write().unwrap();
317                            if error.is_none() {
318                                *error = Some(err);
319                            }
320                        }
321                    }
322                });
323            }
324        });
325
326        if let Some(err) = error.write().unwrap().take() {
327            return Err(err);
328        }
329
330        let mut archive = Archive::new(File::create(&archive_path)?);
331
332        let entries = std::fs::read_dir(&archive_tmp_path)?
333            .flatten()
334            .collect::<Vec<_>>();
335        archive.add_entries(entries, progress_archiving)?;
336
337        std::fs::remove_dir_all(&archive_tmp_path)?;
338
339        Ok(archive)
340    }
341
342    pub fn read_entry_content<S: Write>(
343        &self,
344        entry: Entry,
345        stream: &mut S,
346    ) -> std::io::Result<()> {
347        match entry {
348            Entry::File(mut file_entry) => {
349                let mut buffer = [0; 4096];
350
351                loop {
352                    let chunk_id = crate::varint::decode_u64(&mut file_entry);
353                    if chunk_id == 0 {
354                        break;
355                    }
356
357                    let mut chunk = self
358                        .chunk_index
359                        .read_chunk_id_content(chunk_id)
360                        .map_or_else(
361                            || {
362                                Err(std::io::Error::new(
363                                    std::io::ErrorKind::NotFound,
364                                    format!("Chunk not found: {}", chunk_id),
365                                ))
366                            },
367                            Ok,
368                        )?;
369
370                    loop {
371                        let bytes_read = chunk.read(&mut buffer)?;
372                        if bytes_read == 0 {
373                            break;
374                        }
375
376                        stream.write_all(&buffer[..bytes_read])?;
377                    }
378                }
379
380                Ok(())
381            }
382            _ => Err(std::io::Error::new(
383                std::io::ErrorKind::InvalidData,
384                "Entry is not a file",
385            )),
386        }
387    }
388
389    fn recursive_restore_archive(
390        chunk_index: &ChunkIndex,
391        entry: Entry,
392        directory: &Path,
393        progress: ProgressCallback,
394        scope: &rayon::Scope,
395        error: Arc<RwLock<Option<std::io::Error>>>,
396    ) -> std::io::Result<()> {
397        let path = directory.join(entry.name());
398
399        if error.read().unwrap().is_some() {
400            return Ok(());
401        }
402
403        if let Some(f) = progress.clone() {
404            f(&path)
405        }
406
407        match entry {
408            Entry::File(mut file_entry) => {
409                let mut file = File::create(&path)?;
410                let mut buffer = [0; 4096];
411
412                loop {
413                    let chunk_id = crate::varint::decode_u64(&mut file_entry);
414                    if chunk_id == 0 {
415                        break;
416                    }
417
418                    let mut chunk = chunk_index.read_chunk_id_content(chunk_id).map_or_else(
419                        || {
420                            Err(std::io::Error::new(
421                                std::io::ErrorKind::NotFound,
422                                format!("Chunk not found: {}", chunk_id),
423                            ))
424                        },
425                        Ok,
426                    )?;
427
428                    loop {
429                        let bytes_read = chunk.read(&mut buffer)?;
430                        if bytes_read == 0 {
431                            break;
432                        }
433
434                        file.write_all(&buffer[..bytes_read])?;
435                    }
436                }
437
438                file.set_permissions(file_entry.mode)?;
439                file.set_times(FileTimes::new().set_modified(file_entry.mtime))?;
440
441                #[cfg(unix)]
442                {
443                    let (uid, gid) = file_entry.owner;
444
445                    std::os::unix::fs::lchown(&path, Some(uid), Some(gid))?;
446                }
447            }
448            Entry::Directory(dir_entry) => {
449                std::fs::create_dir_all(&path)?;
450
451                std::fs::set_permissions(&path, dir_entry.mode)?;
452
453                #[cfg(unix)]
454                {
455                    let (uid, gid) = dir_entry.owner;
456                    std::os::unix::fs::chown(&path, Some(uid), Some(gid))?;
457                }
458
459                for sub_entry in dir_entry.entries {
460                    scope.spawn({
461                        let error = Arc::clone(&error);
462                        let chunk_index = chunk_index.clone();
463                        let path = path.to_path_buf();
464                        let progress = progress.clone();
465
466                        move |scope| {
467                            if let Err(err) = Self::recursive_restore_archive(
468                                &chunk_index,
469                                sub_entry,
470                                &path,
471                                progress,
472                                scope,
473                                Arc::clone(&error),
474                            ) {
475                                let mut error = error.write().unwrap();
476                                if error.is_none() {
477                                    *error = Some(err);
478                                }
479                            }
480                        }
481                    });
482                }
483            }
484            #[cfg(unix)]
485            Entry::Symlink(link_entry) => {
486                std::os::unix::fs::symlink(link_entry.target, &path)?;
487                std::fs::set_permissions(&path, link_entry.mode)?;
488
489                let (uid, gid) = link_entry.owner;
490                std::os::unix::fs::lchown(&path, Some(uid), Some(gid))?;
491            }
492            #[cfg(windows)]
493            Entry::Symlink(link_entry) => {
494                if link_entry.target_dir {
495                    std::os::windows::fs::symlink_dir(link_entry.target, &path)?;
496                } else {
497                    std::os::windows::fs::symlink_file(link_entry.target, &path)?;
498                }
499
500                std::fs::set_permissions(&path, link_entry.mode)?;
501            }
502        }
503
504        Ok(())
505    }
506
507    pub fn restore_archive(
508        &self,
509        name: &str,
510        progress: ProgressCallback,
511        threads: usize,
512    ) -> std::io::Result<PathBuf> {
513        if !self.list_archives()?.contains(&name.to_string()) {
514            return Err(std::io::Error::new(
515                std::io::ErrorKind::NotFound,
516                format!("Archive {} not found", name),
517            ));
518        }
519
520        let archive_path = self.archive_path(name);
521        let archive = Archive::open(archive_path.to_str().unwrap())?;
522        let destination = self
523            .directory
524            .join(".ddup-bak/archives-restored")
525            .join(name);
526
527        std::fs::create_dir_all(&destination)?;
528
529        let worker_pool = Arc::new(
530            rayon::ThreadPoolBuilder::new()
531                .num_threads(threads)
532                .build()
533                .unwrap(),
534        );
535        let error = Arc::new(RwLock::new(None));
536
537        worker_pool.in_place_scope(|scope| {
538            for entry in archive.into_entries() {
539                scope.spawn({
540                    let error = Arc::clone(&error);
541                    let chunk_index = self.chunk_index.clone();
542                    let destination = destination.clone();
543                    let progress = progress.clone();
544
545                    move |scope| {
546                        if let Err(err) = Self::recursive_restore_archive(
547                            &chunk_index,
548                            entry,
549                            &destination,
550                            progress,
551                            scope,
552                            Arc::clone(&error),
553                        ) {
554                            let mut error = error.write().unwrap();
555                            if error.is_none() {
556                                *error = Some(err);
557                            }
558                        }
559                    }
560                });
561            }
562        });
563
564        if let Some(err) = error.write().unwrap().take() {
565            return Err(err);
566        }
567
568        Ok(destination)
569    }
570
571    pub fn restore_entries(
572        &self,
573        name: &str,
574        entries: Vec<Entry>,
575        progress: ProgressCallback,
576        threads: usize,
577    ) -> std::io::Result<PathBuf> {
578        if !self.list_archives()?.contains(&name.to_string()) {
579            return Err(std::io::Error::new(
580                std::io::ErrorKind::NotFound,
581                format!("Archive {} not found", name),
582            ));
583        }
584
585        let destination = self
586            .directory
587            .join(".ddup-bak/archives-restored")
588            .join(name);
589
590        std::fs::create_dir_all(&destination)?;
591
592        let worker_pool = Arc::new(
593            rayon::ThreadPoolBuilder::new()
594                .num_threads(threads)
595                .build()
596                .unwrap(),
597        );
598        let error = Arc::new(RwLock::new(None));
599
600        worker_pool.in_place_scope(|scope| {
601            for entry in entries {
602                scope.spawn({
603                    let error = Arc::clone(&error);
604                    let chunk_index = self.chunk_index.clone();
605                    let destination = destination.clone();
606                    let progress = progress.clone();
607
608                    move |scope| {
609                        if let Err(err) = Self::recursive_restore_archive(
610                            &chunk_index,
611                            entry,
612                            &destination,
613                            progress,
614                            scope,
615                            Arc::clone(&error),
616                        ) {
617                            let mut error = error.write().unwrap();
618                            if error.is_none() {
619                                *error = Some(err);
620                            }
621                        }
622                    }
623                });
624            }
625        });
626
627        if let Some(err) = error.write().unwrap().take() {
628            return Err(err);
629        }
630
631        Ok(destination)
632    }
633
634    fn recursive_delete_archive(
635        &mut self,
636        entry: Entry,
637        progress: DeletionProgressCallback,
638    ) -> std::io::Result<()> {
639        match entry {
640            Entry::File(mut file_entry) => loop {
641                let chunk_id = crate::varint::decode_u64(&mut file_entry);
642                if chunk_id == 0 {
643                    break;
644                }
645
646                if let Some(deleted) = self.chunk_index.dereference_chunk_id(chunk_id, true) {
647                    if let Some(f) = progress.clone() {
648                        f(chunk_id, deleted)
649                    }
650                }
651            },
652            Entry::Directory(dir_entry) => {
653                for sub_entry in dir_entry.entries {
654                    self.recursive_delete_archive(sub_entry, progress.clone())?;
655                }
656            }
657            _ => {}
658        }
659
660        Ok(())
661    }
662
663    pub fn delete_archive(
664        &mut self,
665        name: &str,
666        progress: DeletionProgressCallback,
667    ) -> std::io::Result<()> {
668        if !self.list_archives()?.contains(&name.to_string()) {
669            return Err(std::io::Error::new(
670                std::io::ErrorKind::NotFound,
671                format!("Archive {} not found", name),
672            ));
673        }
674
675        let archive_path = self.archive_path(name);
676        let archive = Archive::open(archive_path.to_str().unwrap())?;
677
678        for entry in archive.into_entries() {
679            self.recursive_delete_archive(entry, progress.clone())?;
680        }
681
682        std::fs::remove_file(archive_path)?;
683
684        Ok(())
685    }
686}
687
688impl Drop for Repository {
689    fn drop(&mut self) {
690        if !self.save_on_drop {
691            return;
692        }
693
694        let ignored_files_path = self.directory.join(".ddup-bak/ignored_files");
695        let mut file = File::create(&ignored_files_path).unwrap();
696
697        for entry in &self.ignored_files {
698            writeln!(file, "{}", entry).unwrap();
699        }
700    }
701}
ddup_bak/repository.rs

ddup_bak/
repository.rs
