syncweb_core/fs/
scanner.rs

1use std::{
2    fs::{self, File},
3    io::{self, Read},
4    path::{Path, PathBuf},
5    time::SystemTime,
6};
7
8use globset::{Glob, GlobSet, GlobSetBuilder};
9use rayon::prelude::*;
10
11use crate::error::{Result, SyncwebError};
12
13/// The kind of a filesystem entry returned by a scanner.
14#[derive(Clone, Copy, Debug, Eq, PartialEq)]
15#[non_exhaustive]
16pub enum FileType {
17    File,
18    Directory,
19    Symlink,
20}
21
22/// Metadata and content digest for a regular file.
23#[derive(Clone, Debug)]
24#[non_exhaustive]
25pub struct FileEntry {
26    pub path: PathBuf,
27    pub relative_path: PathBuf,
28    pub size: u64,
29    pub modified: SystemTime,
30    pub hash: blake3::Hash,
31    pub file_type: FileType,
32}
33
34impl FileEntry {
35    #[must_use]
36    pub fn builder() -> FileEntryBuilder {
37        FileEntryBuilder::default()
38    }
39}
40
41#[derive(Clone, Default)]
42pub struct FileEntryBuilder {
43    path: Option<PathBuf>,
44    relative_path: Option<PathBuf>,
45    size: Option<u64>,
46    modified: Option<SystemTime>,
47    hash: Option<blake3::Hash>,
48    file_type: Option<FileType>,
49}
50
51impl FileEntryBuilder {
52    #[must_use]
53    pub fn path(mut self, path: PathBuf) -> Self {
54        self.path = Some(path);
55        self
56    }
57    #[must_use]
58    pub fn relative_path(mut self, relative_path: PathBuf) -> Self {
59        self.relative_path = Some(relative_path);
60        self
61    }
62    #[must_use]
63    pub const fn size(mut self, size: u64) -> Self {
64        self.size = Some(size);
65        self
66    }
67    #[must_use]
68    pub const fn modified(mut self, modified: SystemTime) -> Self {
69        self.modified = Some(modified);
70        self
71    }
72    #[must_use]
73    pub const fn hash(mut self, hash: blake3::Hash) -> Self {
74        self.hash = Some(hash);
75        self
76    }
77    #[must_use]
78    pub const fn file_type(mut self, file_type: FileType) -> Self {
79        self.file_type = Some(file_type);
80        self
81    }
82    /// Builds the `FileEntry`.
83    ///
84    /// # Errors
85    ///
86    /// Returns an error if any of the required fields have not been set.
87    pub fn build(self) -> std::result::Result<FileEntry, &'static str> {
88        Ok(FileEntry {
89            path: self.path.ok_or("path missing")?,
90            relative_path: self.relative_path.ok_or("relative_path missing")?,
91            size: self.size.ok_or("size missing")?,
92            modified: self.modified.ok_or("modified missing")?,
93            hash: self.hash.ok_or("hash missing")?,
94            file_type: self.file_type.ok_or("file_type missing")?,
95        })
96    }
97}
98
99/// Glob patterns used to exclude files and directories from a scan.
100#[derive(Clone, Debug, Default, Eq, PartialEq)]
101pub struct IgnoreFilter {
102    patterns: Vec<String>,
103}
104
105impl IgnoreFilter {
106    #[must_use]
107    pub fn new<I, S>(patterns: I) -> Self
108    where
109        I: IntoIterator<Item = S>,
110        S: Into<String>,
111    {
112        Self {
113            patterns: patterns.into_iter().map(Into::into).collect(),
114        }
115    }
116
117    #[must_use]
118    pub fn patterns(&self) -> &[String] {
119        &self.patterns
120    }
121}
122
123impl From<Vec<String>> for IgnoreFilter {
124    fn from(patterns: Vec<String>) -> Self {
125        Self { patterns }
126    }
127}
128
129/// Accepted thread-count forms for [`ParallelScanner::new`].
130pub trait ThreadCount {
131    fn into_thread_count(self) -> Option<usize>;
132}
133
134impl ThreadCount for usize {
135    fn into_thread_count(self) -> Option<usize> {
136        (self > 0).then_some(self)
137    }
138}
139
140impl ThreadCount for Option<usize> {
141    fn into_thread_count(self) -> Option<usize> {
142        self.filter(|threads| *threads > 0)
143    }
144}
145
146impl FileEntry {
147    #[must_use]
148    pub fn name(&self) -> Option<&str> {
149        self.path.file_name().and_then(|name| name.to_str())
150    }
151}
152
153#[derive(Clone)]
154struct IgnoreSet {
155    set: GlobSet,
156}
157
158impl IgnoreSet {
159    fn new<I, S>(patterns: I) -> Self
160    where
161        I: IntoIterator<Item = S>,
162        S: AsRef<str>,
163    {
164        let mut builder = GlobSetBuilder::new();
165        for pattern in patterns {
166            if let Ok(glob) = Glob::new(pattern.as_ref()) {
167                builder.add(glob);
168            }
169        }
170        let set = builder
171            .build()
172            .unwrap_or_else(|_| GlobSetBuilder::new().build().unwrap_or_default());
173        Self { set }
174    }
175
176    fn matches(&self, relative: &Path) -> bool {
177        self.set.is_match(relative)
178            || relative
179                .components()
180                .any(|component| self.set.is_match(Path::new(component.as_os_str())))
181    }
182}
183
184/// Sequential recursive filesystem scanner.
185#[derive(Clone)]
186pub struct Scanner {
187    root: PathBuf,
188    ignore: IgnoreSet,
189}
190
191/// Name of the per-folder ignore file, resolved relative to a scan root.
192pub const SYNCIGNORE_FILE: &str = ".syncignore";
193
194/// Marker embedded in conflict copies written by materialization
195/// (`<path>.conflict.<short-hash>`).
196///
197/// These are derived artifacts, not folder content; they must never be
198/// re-imported (which would echo them to peers as new keys) or listed as
199/// syncable entries.
200pub const CONFLICT_COPY_MARKER: &str = ".conflict.";
201
202/// Glob matching conflict copies at any depth.
203///
204/// globset's `*` does not cross a path separator, but [`IgnoreSet::matches`]
205/// tests each component separately, so the bare marker suffices for files in
206/// subdirectories too.
207pub const CONFLICT_COPY_GLOB: &str = "*.conflict.*";
208
209/// Merge explicit ignore patterns with any `.syncignore` entries found at the
210/// scan root. The ignore file itself and conflict copies are always excluded
211/// from the scan.
212#[must_use]
213fn with_syncignore_patterns(root: &Path, base: Vec<String>) -> Vec<String> {
214    let mut patterns = base;
215    if root.is_dir() {
216        if let Ok(contents) = fs::read_to_string(root.join(SYNCIGNORE_FILE)) {
217            for line in contents.lines() {
218                let trimmed = line.trim();
219                if trimmed.is_empty() || trimmed.starts_with('#') {
220                    continue;
221                }
222                patterns.push(trimmed.to_owned());
223            }
224        }
225        patterns.push(SYNCIGNORE_FILE.to_owned());
226    }
227    patterns.push(CONFLICT_COPY_GLOB.to_owned());
228    patterns
229}
230
231impl Scanner {
232    #[must_use]
233    pub fn new(root: impl Into<PathBuf>, ignore_patterns_arg: impl Into<IgnoreFilter>) -> Self {
234        let ignore_patterns = ignore_patterns_arg.into();
235        let root_path = root.into();
236        let patterns = with_syncignore_patterns(&root_path, ignore_patterns.patterns().to_vec());
237        Self {
238            root: root_path,
239            ignore: IgnoreSet::new(patterns),
240        }
241    }
242
243    #[must_use]
244    pub fn root(&self) -> &Path {
245        &self.root
246    }
247
248    /// Scan all regular files below the configured root.
249    /// # Errors
250    ///
251    /// Returns an error if the filesystem cannot be accessed or directories cannot be read.
252    pub fn scan(&self) -> Result<Vec<FileEntry>> {
253        if self.root.is_file() {
254            let relative = self.root.file_name().map_or_else(PathBuf::new, PathBuf::from);
255            return Ok(vec![hash_file(&self.root, &relative)?]);
256        }
257        let mut entries = Vec::new();
258        self.scan_dir(&self.root, Path::new(""), &mut entries)?;
259        entries.sort_by(|left, right| left.relative_path.cmp(&right.relative_path));
260        Ok(entries)
261    }
262
263    /// Scan files and filesystem entries below the configured root.
264    ///
265    /// [`Scanner::scan`] intentionally returns only regular files because it
266    /// is used by import and hashing pipelines. This variant is useful for
267    /// metadata queries such as finding directories or symlinks.
268    /// # Errors
269    ///
270    /// Returns an error if the filesystem cannot be accessed or directories cannot be read.
271    pub fn scan_all(&self) -> Result<Vec<FileEntry>> {
272        if self.root.is_file() {
273            let relative = self.root.file_name().map_or_else(PathBuf::new, PathBuf::from);
274            return Ok(vec![metadata_entry(&self.root, &relative)?]);
275        }
276        let mut entries = Vec::new();
277        self.scan_dir_all(&self.root, Path::new(""), &mut entries)?;
278        entries.sort_by(|left, right| left.relative_path.cmp(&right.relative_path));
279        Ok(entries)
280    }
281
282    fn scan_dir(&self, directory: &Path, relative: &Path, output: &mut Vec<FileEntry>) -> Result<()> {
283        let mut children = fs::read_dir(directory)?.collect::<std::result::Result<Vec<_>, io::Error>>()?;
284        children.sort_by_key(std::fs::DirEntry::file_name);
285        for child in children {
286            let path = child.path();
287            let child_relative = relative.join(child.file_name());
288            if self.ignore.matches(&child_relative) {
289                continue;
290            }
291            let file_type = child.file_type()?;
292            if file_type.is_dir() {
293                self.scan_dir(&path, &child_relative, output)?;
294            } else if file_type.is_file() {
295                output.push(hash_file(&path, &child_relative)?);
296            } else {
297                // do nothing
298            }
299        }
300        Ok(())
301    }
302
303    fn scan_dir_all(&self, directory: &Path, relative: &Path, output: &mut Vec<FileEntry>) -> Result<()> {
304        let mut children = fs::read_dir(directory)?.collect::<std::result::Result<Vec<_>, io::Error>>()?;
305        children.sort_by_key(std::fs::DirEntry::file_name);
306        for child in children {
307            let path = child.path();
308            let child_relative = relative.join(child.file_name());
309            if self.ignore.matches(&child_relative) {
310                continue;
311            }
312            let file_type = child.file_type()?;
313            output.push(metadata_entry(&path, &child_relative)?);
314            if file_type.is_dir() {
315                self.scan_dir_all(&path, &child_relative, output)?;
316            }
317        }
318        Ok(())
319    }
320}
321
322/// Rayon-backed recursive filesystem scanner.
323#[derive(Clone)]
324pub struct ParallelScanner {
325    scanner: Scanner,
326    threads: Option<usize>,
327}
328
329impl ParallelScanner {
330    #[must_use]
331    pub fn new<T: ThreadCount>(root: impl Into<PathBuf>, ignore_patterns: impl Into<IgnoreFilter>, threads: T) -> Self {
332        Self {
333            scanner: Scanner::new(root, ignore_patterns),
334            threads: threads.into_thread_count(),
335        }
336    }
337
338    #[must_use]
339    pub fn root(&self) -> &Path {
340        self.scanner.root()
341    }
342
343    /// Scan and hash files concurrently.
344    /// # Errors
345    ///
346    /// Returns an error if the filesystem cannot be accessed or directories cannot be read.
347    pub fn scan(&self) -> Result<Vec<FileEntry>> {
348        if self.scanner.root.is_file() {
349            let relative = self.scanner.root.file_name().map_or_else(PathBuf::new, PathBuf::from);
350            return Ok(vec![hash_file(&self.scanner.root, &relative)?]);
351        }
352        let mut paths = Vec::new();
353        collect_paths(self.scanner.root(), Path::new(""), &self.scanner.ignore, &mut paths)?;
354        let run = || {
355            paths
356                .par_iter()
357                .map(|(path, relative)| hash_file(path, relative))
358                .collect::<Result<Vec<_>>>()
359        };
360        let mut entries = if let Some(threads) = self.threads {
361            rayon::ThreadPoolBuilder::new()
362                .num_threads(threads)
363                .build()
364                .map_err(|error| SyncwebError::operation("failed to create scanner thread pool", error))?
365                .install(run)?
366        } else {
367            run()?
368        };
369        entries.sort_by(|left, right| left.relative_path.cmp(&right.relative_path));
370        Ok(entries)
371    }
372
373    /// Scan and hash files concurrently.
374    /// # Errors
375    ///
376    /// Returns an error if the filesystem cannot be accessed or directories cannot be read.
377    pub fn scan_parallel(&self) -> Result<Vec<FileEntry>> {
378        self.scan()
379    }
380
381    /// Scan files and filesystem entries below the configured root.
382    /// # Errors
383    ///
384    /// Returns an error if the filesystem cannot be accessed or directories cannot be read.
385    pub fn scan_all(&self) -> Result<Vec<FileEntry>> {
386        if self.scanner.root.is_file() {
387            let relative = self.scanner.root.file_name().map_or_else(PathBuf::new, PathBuf::from);
388            return Ok(vec![metadata_entry(&self.scanner.root, &relative)?]);
389        }
390        let mut paths = Vec::new();
391        collect_all_paths(self.scanner.root(), Path::new(""), &self.scanner.ignore, &mut paths)?;
392        let run = || {
393            paths
394                .par_iter()
395                .map(|(path, relative)| metadata_entry(path, relative))
396                .collect::<Result<Vec<_>>>()
397        };
398        let mut entries = if let Some(threads) = self.threads {
399            rayon::ThreadPoolBuilder::new()
400                .num_threads(threads)
401                .build()
402                .map_err(|error| SyncwebError::operation("failed to create scanner thread pool", error))?
403                .install(run)?
404        } else {
405            run()?
406        };
407        entries.sort_by(|left, right| left.relative_path.cmp(&right.relative_path));
408        Ok(entries)
409    }
410}
411
412fn collect_paths(
413    directory: &Path,
414    relative: &Path,
415    ignore: &IgnoreSet,
416    paths: &mut Vec<(PathBuf, PathBuf)>,
417) -> Result<()> {
418    let mut children = fs::read_dir(directory)?.collect::<std::result::Result<Vec<_>, io::Error>>()?;
419    children.sort_by_key(std::fs::DirEntry::file_name);
420    for child in children {
421        let path = child.path();
422        let child_relative = relative.join(child.file_name());
423        if ignore.matches(&child_relative) {
424            continue;
425        }
426        let file_type = child.file_type()?;
427        if file_type.is_dir() {
428            collect_paths(&path, &child_relative, ignore, paths)?;
429        } else if file_type.is_file() {
430            paths.push((path, child_relative));
431        } else {
432            // do nothing
433        }
434    }
435    Ok(())
436}
437
438fn collect_all_paths(
439    directory: &Path,
440    relative: &Path,
441    ignore: &IgnoreSet,
442    paths: &mut Vec<(PathBuf, PathBuf)>,
443) -> Result<()> {
444    let mut children = fs::read_dir(directory)?.collect::<std::result::Result<Vec<_>, io::Error>>()?;
445    children.sort_by_key(std::fs::DirEntry::file_name);
446    for child in children {
447        let path = child.path();
448        let child_relative = relative.join(child.file_name());
449        if ignore.matches(&child_relative) {
450            continue;
451        }
452        let file_type = child.file_type()?;
453        paths.push((path.clone(), child_relative.clone()));
454        if file_type.is_dir() {
455            collect_all_paths(&path, &child_relative, ignore, paths)?;
456        }
457    }
458    Ok(())
459}
460
461fn hash_file(path: &Path, relative: &Path) -> Result<FileEntry> {
462    let mut file = File::open(path)?;
463    let mut hasher = blake3::Hasher::new();
464    let mut buffer = vec![0_u8; 128 * 1024];
465    loop {
466        let read = file.read(&mut buffer)?;
467        if read == 0 {
468            break;
469        }
470        if let Some(chunk) = buffer.get(..read) {
471            hasher.update(chunk);
472        }
473    }
474    let metadata = fs::metadata(path)?;
475    Ok(FileEntry {
476        path: path.to_path_buf(),
477        relative_path: relative.to_path_buf(),
478        size: metadata.len(),
479        modified: metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH),
480        hash: hasher.finalize(),
481        file_type: if path.is_symlink() {
482            FileType::Symlink
483        } else if path.is_dir() {
484            FileType::Directory
485        } else {
486            FileType::File
487        },
488    })
489}
490
491fn metadata_entry(path: &Path, relative: &Path) -> Result<FileEntry> {
492    let metadata = fs::symlink_metadata(path)?;
493    let file_type = if metadata.file_type().is_symlink() {
494        FileType::Symlink
495    } else if metadata.is_dir() {
496        FileType::Directory
497    } else {
498        FileType::File
499    };
500    let hash = if file_type == FileType::File {
501        hash_file(path, relative)?.hash
502    } else {
503        blake3::hash(&[])
504    };
505    Ok(FileEntry {
506        path: path.to_path_buf(),
507        relative_path: relative.to_path_buf(),
508        size: metadata.len(),
509        modified: metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH),
510        hash,
511        file_type,
512    })
513}
syncweb_core/fs/scanner.rs

syncweb_core/fs/
scanner.rs
