syncweb_core/fs/
scanner.rs

1use std::{
2    fs::{self, File},
3    io::{self, Read},
4    path::{Path, PathBuf},
5    time::SystemTime,
6};
7
8use globset::{Glob, GlobSet, GlobSetBuilder};
9use rayon::prelude::*;
10
11use crate::error::{Result, SyncwebError};
12
13/// The kind of a filesystem entry returned by a scanner.
14#[derive(Clone, Copy, Debug, Eq, PartialEq)]
15#[non_exhaustive]
16pub enum FileType {
17    File,
18    Directory,
19    Symlink,
20}
21
22/// Metadata and content digest for a regular file.
23#[derive(Clone, Debug)]
24#[non_exhaustive]
25pub struct FileEntry {
26    pub path: PathBuf,
27    pub relative_path: PathBuf,
28    pub size: u64,
29    pub modified: SystemTime,
30    pub hash: blake3::Hash,
31    pub file_type: FileType,
32}
33
34impl FileEntry {
35    #[must_use]
36    pub fn builder() -> FileEntryBuilder {
37        FileEntryBuilder::default()
38    }
39}
40
41#[derive(Clone, Default)]
42pub struct FileEntryBuilder {
43    path: Option<PathBuf>,
44    relative_path: Option<PathBuf>,
45    size: Option<u64>,
46    modified: Option<SystemTime>,
47    hash: Option<blake3::Hash>,
48    file_type: Option<FileType>,
49}
50
51impl FileEntryBuilder {
52    #[must_use]
53    pub fn path(mut self, path: PathBuf) -> Self {
54        self.path = Some(path);
55        self
56    }
57    #[must_use]
58    pub fn relative_path(mut self, relative_path: PathBuf) -> Self {
59        self.relative_path = Some(relative_path);
60        self
61    }
62    #[must_use]
63    pub const fn size(mut self, size: u64) -> Self {
64        self.size = Some(size);
65        self
66    }
67    #[must_use]
68    pub const fn modified(mut self, modified: SystemTime) -> Self {
69        self.modified = Some(modified);
70        self
71    }
72    #[must_use]
73    pub const fn hash(mut self, hash: blake3::Hash) -> Self {
74        self.hash = Some(hash);
75        self
76    }
77    #[must_use]
78    pub const fn file_type(mut self, file_type: FileType) -> Self {
79        self.file_type = Some(file_type);
80        self
81    }
82    /// Builds the `FileEntry`.
83    ///
84    /// # Errors
85    ///
86    /// Returns an error if any of the required fields have not been set.
87    pub fn build(self) -> std::result::Result<FileEntry, &'static str> {
88        Ok(FileEntry {
89            path: self.path.ok_or("path missing")?,
90            relative_path: self.relative_path.ok_or("relative_path missing")?,
91            size: self.size.ok_or("size missing")?,
92            modified: self.modified.ok_or("modified missing")?,
93            hash: self.hash.ok_or("hash missing")?,
94            file_type: self.file_type.ok_or("file_type missing")?,
95        })
96    }
97}
98
99/// Glob patterns used to exclude files and directories from a scan.
100#[derive(Clone, Debug, Default, Eq, PartialEq)]
101pub struct IgnoreFilter {
102    patterns: Vec<String>,
103}
104
105impl IgnoreFilter {
106    #[must_use]
107    pub fn new<I, S>(patterns: I) -> Self
108    where
109        I: IntoIterator<Item = S>,
110        S: Into<String>,
111    {
112        Self {
113            patterns: patterns.into_iter().map(Into::into).collect(),
114        }
115    }
116
117    #[must_use]
118    pub fn patterns(&self) -> &[String] {
119        &self.patterns
120    }
121}
122
123impl From<Vec<String>> for IgnoreFilter {
124    fn from(patterns: Vec<String>) -> Self {
125        Self { patterns }
126    }
127}
128
129/// Accepted thread-count forms for [`ParallelScanner::new`].
130pub trait ThreadCount {
131    fn into_thread_count(self) -> Option<usize>;
132}
133
134impl ThreadCount for usize {
135    fn into_thread_count(self) -> Option<usize> {
136        (self > 0).then_some(self)
137    }
138}
139
140impl ThreadCount for Option<usize> {
141    fn into_thread_count(self) -> Option<usize> {
142        self.filter(|threads| *threads > 0)
143    }
144}
145
146impl FileEntry {
147    #[must_use]
148    pub fn name(&self) -> Option<&str> {
149        self.path.file_name().and_then(|name| name.to_str())
150    }
151}
152
153#[derive(Clone)]
154struct IgnoreSet {
155    set: GlobSet,
156}
157
158impl IgnoreSet {
159    fn new<I, S>(patterns: I) -> Self
160    where
161        I: IntoIterator<Item = S>,
162        S: AsRef<str>,
163    {
164        let mut builder = GlobSetBuilder::new();
165        for pattern in patterns {
166            if let Ok(glob) = Glob::new(pattern.as_ref()) {
167                builder.add(glob);
168            }
169        }
170        let set = builder
171            .build()
172            .unwrap_or_else(|_| GlobSetBuilder::new().build().unwrap_or_default());
173        Self { set }
174    }
175
176    fn matches(&self, relative: &Path) -> bool {
177        self.set.is_match(relative)
178            || relative
179                .components()
180                .any(|component| self.set.is_match(Path::new(component.as_os_str())))
181    }
182}
183
184/// Sequential recursive filesystem scanner.
185#[derive(Clone)]
186pub struct Scanner {
187    root: PathBuf,
188    ignore: IgnoreSet,
189}
190
191impl Scanner {
192    #[must_use]
193    pub fn new(root: impl Into<PathBuf>, ignore_patterns_arg: impl Into<IgnoreFilter>) -> Self {
194        let ignore_patterns = ignore_patterns_arg.into();
195        Self {
196            root: root.into(),
197            ignore: IgnoreSet::new(ignore_patterns.patterns()),
198        }
199    }
200
201    #[must_use]
202    pub fn root(&self) -> &Path {
203        &self.root
204    }
205
206    /// Scan all regular files below the configured root.
207    /// # Errors
208    ///
209    /// Returns an error if the filesystem cannot be accessed or directories cannot be read.
210    pub fn scan(&self) -> Result<Vec<FileEntry>> {
211        if self.root.is_file() {
212            let relative = self.root.file_name().map_or_else(PathBuf::new, PathBuf::from);
213            return Ok(vec![hash_file(&self.root, &relative)?]);
214        }
215        let mut entries = Vec::new();
216        self.scan_dir(&self.root, Path::new(""), &mut entries)?;
217        entries.sort_by(|left, right| left.relative_path.cmp(&right.relative_path));
218        Ok(entries)
219    }
220
221    /// Scan files and filesystem entries below the configured root.
222    ///
223    /// [`Scanner::scan`] intentionally returns only regular files because it
224    /// is used by import and hashing pipelines. This variant is useful for
225    /// metadata queries such as finding directories or symlinks.
226    /// # Errors
227    ///
228    /// Returns an error if the filesystem cannot be accessed or directories cannot be read.
229    pub fn scan_all(&self) -> Result<Vec<FileEntry>> {
230        if self.root.is_file() {
231            let relative = self.root.file_name().map_or_else(PathBuf::new, PathBuf::from);
232            return Ok(vec![metadata_entry(&self.root, &relative)?]);
233        }
234        let mut entries = Vec::new();
235        self.scan_dir_all(&self.root, Path::new(""), &mut entries)?;
236        entries.sort_by(|left, right| left.relative_path.cmp(&right.relative_path));
237        Ok(entries)
238    }
239
240    fn scan_dir(&self, directory: &Path, relative: &Path, output: &mut Vec<FileEntry>) -> Result<()> {
241        let mut children = fs::read_dir(directory)?.collect::<std::result::Result<Vec<_>, io::Error>>()?;
242        children.sort_by_key(std::fs::DirEntry::file_name);
243        for child in children {
244            let path = child.path();
245            let child_relative = relative.join(child.file_name());
246            if self.ignore.matches(&child_relative) {
247                continue;
248            }
249            let file_type = child.file_type()?;
250            if file_type.is_dir() {
251                self.scan_dir(&path, &child_relative, output)?;
252            } else if file_type.is_file() {
253                output.push(hash_file(&path, &child_relative)?);
254            } else {
255                // do nothing
256            }
257        }
258        Ok(())
259    }
260
261    fn scan_dir_all(&self, directory: &Path, relative: &Path, output: &mut Vec<FileEntry>) -> Result<()> {
262        let mut children = fs::read_dir(directory)?.collect::<std::result::Result<Vec<_>, io::Error>>()?;
263        children.sort_by_key(std::fs::DirEntry::file_name);
264        for child in children {
265            let path = child.path();
266            let child_relative = relative.join(child.file_name());
267            if self.ignore.matches(&child_relative) {
268                continue;
269            }
270            let file_type = child.file_type()?;
271            output.push(metadata_entry(&path, &child_relative)?);
272            if file_type.is_dir() {
273                self.scan_dir_all(&path, &child_relative, output)?;
274            }
275        }
276        Ok(())
277    }
278}
279
280/// Rayon-backed recursive filesystem scanner.
281#[derive(Clone)]
282pub struct ParallelScanner {
283    scanner: Scanner,
284    threads: Option<usize>,
285}
286
287impl ParallelScanner {
288    #[must_use]
289    pub fn new<T: ThreadCount>(root: impl Into<PathBuf>, ignore_patterns: impl Into<IgnoreFilter>, threads: T) -> Self {
290        Self {
291            scanner: Scanner::new(root, ignore_patterns),
292            threads: threads.into_thread_count(),
293        }
294    }
295
296    #[must_use]
297    pub fn root(&self) -> &Path {
298        self.scanner.root()
299    }
300
301    /// Scan and hash files concurrently.
302    /// # Errors
303    ///
304    /// Returns an error if the filesystem cannot be accessed or directories cannot be read.
305    pub fn scan(&self) -> Result<Vec<FileEntry>> {
306        if self.scanner.root.is_file() {
307            let relative = self.scanner.root.file_name().map_or_else(PathBuf::new, PathBuf::from);
308            return Ok(vec![hash_file(&self.scanner.root, &relative)?]);
309        }
310        let mut paths = Vec::new();
311        collect_paths(self.scanner.root(), Path::new(""), &self.scanner.ignore, &mut paths)?;
312        let run = || {
313            paths
314                .par_iter()
315                .map(|(path, relative)| hash_file(path, relative))
316                .collect::<Result<Vec<_>>>()
317        };
318        let mut entries = if let Some(threads) = self.threads {
319            rayon::ThreadPoolBuilder::new()
320                .num_threads(threads)
321                .build()
322                .map_err(|error| SyncwebError::operation("failed to create scanner thread pool", error))?
323                .install(run)?
324        } else {
325            run()?
326        };
327        entries.sort_by(|left, right| left.relative_path.cmp(&right.relative_path));
328        Ok(entries)
329    }
330
331    /// Scan and hash files concurrently.
332    /// # Errors
333    ///
334    /// Returns an error if the filesystem cannot be accessed or directories cannot be read.
335    pub fn scan_parallel(&self) -> Result<Vec<FileEntry>> {
336        self.scan()
337    }
338
339    /// Scan files and filesystem entries below the configured root.
340    /// # Errors
341    ///
342    /// Returns an error if the filesystem cannot be accessed or directories cannot be read.
343    pub fn scan_all(&self) -> Result<Vec<FileEntry>> {
344        if self.scanner.root.is_file() {
345            let relative = self.scanner.root.file_name().map_or_else(PathBuf::new, PathBuf::from);
346            return Ok(vec![metadata_entry(&self.scanner.root, &relative)?]);
347        }
348        let mut paths = Vec::new();
349        collect_all_paths(self.scanner.root(), Path::new(""), &self.scanner.ignore, &mut paths)?;
350        let run = || {
351            paths
352                .par_iter()
353                .map(|(path, relative)| metadata_entry(path, relative))
354                .collect::<Result<Vec<_>>>()
355        };
356        let mut entries = if let Some(threads) = self.threads {
357            rayon::ThreadPoolBuilder::new()
358                .num_threads(threads)
359                .build()
360                .map_err(|error| SyncwebError::operation("failed to create scanner thread pool", error))?
361                .install(run)?
362        } else {
363            run()?
364        };
365        entries.sort_by(|left, right| left.relative_path.cmp(&right.relative_path));
366        Ok(entries)
367    }
368}
369
370fn collect_paths(
371    directory: &Path,
372    relative: &Path,
373    ignore: &IgnoreSet,
374    paths: &mut Vec<(PathBuf, PathBuf)>,
375) -> Result<()> {
376    let mut children = fs::read_dir(directory)?.collect::<std::result::Result<Vec<_>, io::Error>>()?;
377    children.sort_by_key(std::fs::DirEntry::file_name);
378    for child in children {
379        let path = child.path();
380        let child_relative = relative.join(child.file_name());
381        if ignore.matches(&child_relative) {
382            continue;
383        }
384        let file_type = child.file_type()?;
385        if file_type.is_dir() {
386            collect_paths(&path, &child_relative, ignore, paths)?;
387        } else if file_type.is_file() {
388            paths.push((path, child_relative));
389        } else {
390            // do nothing
391        }
392    }
393    Ok(())
394}
395
396fn collect_all_paths(
397    directory: &Path,
398    relative: &Path,
399    ignore: &IgnoreSet,
400    paths: &mut Vec<(PathBuf, PathBuf)>,
401) -> Result<()> {
402    let mut children = fs::read_dir(directory)?.collect::<std::result::Result<Vec<_>, io::Error>>()?;
403    children.sort_by_key(std::fs::DirEntry::file_name);
404    for child in children {
405        let path = child.path();
406        let child_relative = relative.join(child.file_name());
407        if ignore.matches(&child_relative) {
408            continue;
409        }
410        let file_type = child.file_type()?;
411        paths.push((path.clone(), child_relative.clone()));
412        if file_type.is_dir() {
413            collect_all_paths(&path, &child_relative, ignore, paths)?;
414        }
415    }
416    Ok(())
417}
418
419fn hash_file(path: &Path, relative: &Path) -> Result<FileEntry> {
420    let mut file = File::open(path)?;
421    let mut hasher = blake3::Hasher::new();
422    let mut buffer = vec![0_u8; 128 * 1024];
423    loop {
424        let read = file.read(&mut buffer)?;
425        if read == 0 {
426            break;
427        }
428        if let Some(chunk) = buffer.get(..read) {
429            hasher.update(chunk);
430        }
431    }
432    let metadata = fs::metadata(path)?;
433    Ok(FileEntry {
434        path: path.to_path_buf(),
435        relative_path: relative.to_path_buf(),
436        size: metadata.len(),
437        modified: metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH),
438        hash: hasher.finalize(),
439        file_type: if path.is_symlink() {
440            FileType::Symlink
441        } else if path.is_dir() {
442            FileType::Directory
443        } else {
444            FileType::File
445        },
446    })
447}
448
449fn metadata_entry(path: &Path, relative: &Path) -> Result<FileEntry> {
450    let metadata = fs::symlink_metadata(path)?;
451    let file_type = if metadata.file_type().is_symlink() {
452        FileType::Symlink
453    } else if metadata.is_dir() {
454        FileType::Directory
455    } else {
456        FileType::File
457    };
458    let hash = if file_type == FileType::File {
459        hash_file(path, relative)?.hash
460    } else {
461        blake3::hash(&[])
462    };
463    Ok(FileEntry {
464        path: path.to_path_buf(),
465        relative_path: relative.to_path_buf(),
466        size: metadata.len(),
467        modified: metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH),
468        hash,
469        file_type,
470    })
471}
syncweb_core/fs/scanner.rs

syncweb_core/fs/
scanner.rs
