Skip to main content

provenant/scanner/
collect.rs

1// SPDX-FileCopyrightText: Provenant contributors
2// SPDX-License-Identifier: Apache-2.0
3
4use glob::Pattern;
5use std::collections::HashSet;
6use std::fs;
7use std::io;
8use std::path::{Path, PathBuf};
9
10use crate::utils::file::is_path_excluded;
11
12pub struct CollectedPaths {
13    pub files: Vec<(PathBuf, fs::Metadata)>,
14    pub directories: Vec<(PathBuf, fs::Metadata)>,
15    pub excluded_count: usize,
16    pub total_file_bytes: u64,
17    pub collection_errors: Vec<(PathBuf, String)>,
18}
19
20#[derive(Debug, Clone, PartialEq, Eq)]
21pub struct CollectionFrontier {
22    pub path: PathBuf,
23    pub recurse: bool,
24}
25
26struct CollectionAccumulator {
27    files: Vec<(PathBuf, fs::Metadata)>,
28    directories: Vec<(PathBuf, fs::Metadata)>,
29    file_seen: HashSet<PathBuf>,
30    dir_seen: HashSet<PathBuf>,
31    excluded_count: usize,
32    total_file_bytes: u64,
33    collection_errors: Vec<(PathBuf, String)>,
34}
35
36enum TraversalMetadata {
37    File(fs::Metadata),
38    Directory {
39        metadata: fs::Metadata,
40        can_recurse: bool,
41    },
42    Other,
43}
44
45impl CollectedPaths {
46    pub fn file_count(&self) -> usize {
47        self.files.len()
48    }
49
50    pub fn directory_count(&self) -> usize {
51        self.directories.len()
52    }
53
54    pub fn scan_root(&self) -> Option<&Path> {
55        self.directories
56            .first()
57            .map(|(path, _)| path.as_path())
58            .or_else(|| {
59                self.files
60                    .first()
61                    .and_then(|(path, _)| path.parent().or(Some(path.as_path())))
62            })
63    }
64}
65
66pub fn collect_paths<P: AsRef<Path>>(
67    root: P,
68    max_depth: usize,
69    exclude_patterns: &[Pattern],
70) -> CollectedPaths {
71    let depth_limit = depth_limit_from_cli(max_depth);
72    let root = root.as_ref();
73
74    if is_path_excluded(root, exclude_patterns) {
75        return CollectedPaths {
76            files: Vec::new(),
77            directories: Vec::new(),
78            excluded_count: 1,
79            total_file_bytes: 0,
80            collection_errors: Vec::new(),
81        };
82    }
83
84    let traversal_metadata = match classify_for_traversal(root, true) {
85        Ok(traversal_metadata) => traversal_metadata,
86        Err(error) => {
87            return CollectedPaths {
88                files: Vec::new(),
89                directories: Vec::new(),
90                excluded_count: 0,
91                total_file_bytes: 0,
92                collection_errors: vec![(root.to_path_buf(), error.to_string())],
93            };
94        }
95    };
96
97    match traversal_metadata {
98        TraversalMetadata::File(metadata) => CollectedPaths {
99            total_file_bytes: metadata.len(),
100            files: vec![(root.to_path_buf(), metadata)],
101            directories: Vec::new(),
102            excluded_count: 0,
103            collection_errors: Vec::new(),
104        },
105        TraversalMetadata::Directory {
106            metadata,
107            can_recurse,
108        } if can_recurse => collect_all_paths(root, &metadata, depth_limit, exclude_patterns),
109        TraversalMetadata::Directory { metadata, .. } => CollectedPaths {
110            files: Vec::new(),
111            directories: vec![(root.to_path_buf(), metadata)],
112            excluded_count: 0,
113            total_file_bytes: 0,
114            collection_errors: Vec::new(),
115        },
116        TraversalMetadata::Other => CollectedPaths {
117            files: Vec::new(),
118            directories: Vec::new(),
119            excluded_count: 0,
120            total_file_bytes: 0,
121            collection_errors: Vec::new(),
122        },
123    }
124}
125
126pub fn collect_selected_paths(
127    root: &Path,
128    selected: &[CollectionFrontier],
129    max_depth: usize,
130    exclude_patterns: &[Pattern],
131) -> CollectedPaths {
132    let depth_limit = depth_limit_from_cli(max_depth);
133
134    if is_path_excluded(root, exclude_patterns) {
135        return CollectedPaths {
136            files: Vec::new(),
137            directories: Vec::new(),
138            excluded_count: 1,
139            total_file_bytes: 0,
140            collection_errors: Vec::new(),
141        };
142    }
143
144    let root_metadata = match classify_for_traversal(root, true) {
145        Ok(TraversalMetadata::Directory { metadata, .. }) => metadata,
146        Ok(TraversalMetadata::File(metadata)) => metadata,
147        Ok(TraversalMetadata::Other) => {
148            return CollectedPaths {
149                files: Vec::new(),
150                directories: Vec::new(),
151                excluded_count: 0,
152                total_file_bytes: 0,
153                collection_errors: Vec::new(),
154            };
155        }
156        Err(error) => {
157            return CollectedPaths {
158                files: Vec::new(),
159                directories: Vec::new(),
160                excluded_count: 0,
161                total_file_bytes: 0,
162                collection_errors: vec![(root.to_path_buf(), error.to_string())],
163            };
164        }
165    };
166
167    let mut accumulator = CollectionAccumulator {
168        files: Vec::new(),
169        directories: vec![(root.to_path_buf(), root_metadata)],
170        file_seen: HashSet::new(),
171        dir_seen: HashSet::from([root.to_path_buf()]),
172        excluded_count: 0,
173        total_file_bytes: 0,
174        collection_errors: Vec::new(),
175    };
176
177    for frontier in minimize_frontier(selected) {
178        let relative_depth = frontier.path.components().count();
179        if depth_limit.is_some_and(|limit| relative_depth > limit) {
180            continue;
181        }
182
183        let absolute = root.join(&frontier.path);
184        if is_path_or_any_ancestor_excluded(root, &absolute, exclude_patterns) {
185            accumulator.excluded_count += 1;
186            continue;
187        }
188
189        let traversal_metadata = match classify_for_traversal(&absolute, false) {
190            Ok(traversal_metadata) => traversal_metadata,
191            Err(error) => {
192                accumulator
193                    .collection_errors
194                    .push((absolute, error.to_string()));
195                continue;
196            }
197        };
198
199        add_ancestor_directories(root, &absolute, &mut accumulator);
200
201        let collected = match traversal_metadata {
202            TraversalMetadata::File(metadata) => {
203                insert_file(&mut accumulator, absolute, metadata);
204                continue;
205            }
206            TraversalMetadata::Directory {
207                metadata,
208                can_recurse,
209            } if frontier.recurse && can_recurse => {
210                let subtree_depth_limit =
211                    depth_limit.map(|limit| limit.saturating_sub(relative_depth));
212                collect_all_paths(&absolute, &metadata, subtree_depth_limit, exclude_patterns)
213            }
214            TraversalMetadata::Directory { metadata, .. } => CollectedPaths {
215                files: Vec::new(),
216                directories: vec![(absolute, metadata)],
217                excluded_count: 0,
218                total_file_bytes: 0,
219                collection_errors: Vec::new(),
220            },
221            TraversalMetadata::Other => continue,
222        };
223        merge_collected(&mut accumulator, collected);
224    }
225
226    CollectedPaths {
227        files: accumulator.files,
228        directories: accumulator.directories,
229        excluded_count: accumulator.excluded_count,
230        total_file_bytes: accumulator.total_file_bytes,
231        collection_errors: accumulator.collection_errors,
232    }
233}
234
235fn collect_all_paths(
236    root: &Path,
237    root_metadata: &fs::Metadata,
238    depth_limit: Option<usize>,
239    exclude_patterns: &[Pattern],
240) -> CollectedPaths {
241    let mut files = Vec::new();
242    let mut directories = vec![(root.to_path_buf(), root_metadata.clone())];
243    let mut excluded_count = 0;
244    let mut total_file_bytes = 0_u64;
245    let mut collection_errors = Vec::new();
246
247    let mut pending_dirs: Vec<(PathBuf, Option<usize>)> = vec![(root.to_path_buf(), depth_limit)];
248
249    while let Some((dir_path, current_depth)) = pending_dirs.pop() {
250        let entries: Vec<_> = match fs::read_dir(&dir_path) {
251            Ok(entries) => entries.filter_map(Result::ok).collect(),
252            Err(e) => {
253                collection_errors.push((dir_path.clone(), e.to_string()));
254                continue;
255            }
256        };
257
258        for entry in entries {
259            let path = entry.path();
260
261            if is_path_excluded(&path, exclude_patterns) {
262                excluded_count += 1;
263                continue;
264            }
265
266            match classify_for_traversal(&path, false) {
267                Ok(TraversalMetadata::File(metadata)) => {
268                    total_file_bytes += metadata.len();
269                    files.push((path, metadata));
270                }
271                Ok(TraversalMetadata::Directory {
272                    metadata,
273                    can_recurse,
274                }) => {
275                    directories.push((path.clone(), metadata));
276                    let should_recurse = can_recurse && current_depth.is_none_or(|d| d > 0);
277                    if should_recurse {
278                        let next_depth = current_depth.map(|d| d - 1);
279                        pending_dirs.push((path, next_depth));
280                    }
281                }
282                _ => continue,
283            }
284        }
285    }
286
287    CollectedPaths {
288        files,
289        directories,
290        excluded_count,
291        total_file_bytes,
292        collection_errors,
293    }
294}
295
296fn classify_for_traversal(
297    path: &Path,
298    recurse_into_symlinked_directories: bool,
299) -> io::Result<TraversalMetadata> {
300    let link_metadata = fs::symlink_metadata(path)?;
301    if link_metadata.file_type().is_symlink() {
302        let target_metadata = fs::metadata(path)?;
303        return Ok(classify_resolved_metadata(
304            target_metadata,
305            recurse_into_symlinked_directories,
306        ));
307    }
308
309    Ok(classify_resolved_metadata(link_metadata, true))
310}
311
312fn classify_resolved_metadata(metadata: fs::Metadata, can_recurse: bool) -> TraversalMetadata {
313    if metadata.is_file() {
314        TraversalMetadata::File(metadata)
315    } else if metadata.is_dir() {
316        TraversalMetadata::Directory {
317            metadata,
318            can_recurse,
319        }
320    } else {
321        TraversalMetadata::Other
322    }
323}
324
325fn depth_limit_from_cli(max_depth: usize) -> Option<usize> {
326    if max_depth == 0 {
327        None
328    } else {
329        Some(max_depth)
330    }
331}
332
333fn is_path_or_any_ancestor_excluded(
334    path_root: &Path,
335    path: &Path,
336    exclude_patterns: &[Pattern],
337) -> bool {
338    let mut current = Some(path);
339    while let Some(candidate) = current {
340        if is_path_excluded(candidate, exclude_patterns) {
341            return true;
342        }
343        if candidate == path_root {
344            break;
345        }
346        current = candidate.parent();
347    }
348    false
349}
350
351fn minimize_frontier(selected: &[CollectionFrontier]) -> Vec<CollectionFrontier> {
352    let mut ordered = selected.to_vec();
353    ordered.sort_by_key(|entry| (entry.path.components().count(), !entry.recurse));
354
355    let mut minimized = Vec::new();
356    for entry in ordered {
357        let covered = minimized.iter().any(|existing: &CollectionFrontier| {
358            existing.recurse
359                && (entry.path == existing.path || entry.path.starts_with(&existing.path))
360        });
361        if !covered {
362            minimized.push(entry);
363        }
364    }
365    minimized
366}
367
368fn add_ancestor_directories(root: &Path, path: &Path, accumulator: &mut CollectionAccumulator) {
369    let mut current = path.parent();
370    while let Some(dir) = current {
371        if dir == root {
372            break;
373        }
374        if accumulator.dir_seen.insert(dir.to_path_buf()) {
375            match classify_for_traversal(dir, false) {
376                Ok(TraversalMetadata::Directory { metadata, .. }) => {
377                    accumulator.directories.push((dir.to_path_buf(), metadata))
378                }
379                Ok(_) => {}
380                Err(error) => accumulator
381                    .collection_errors
382                    .push((dir.to_path_buf(), error.to_string())),
383            }
384        }
385        current = dir.parent();
386    }
387}
388
389fn insert_file(accumulator: &mut CollectionAccumulator, path: PathBuf, metadata: fs::Metadata) {
390    if accumulator.file_seen.insert(path.clone()) {
391        accumulator.total_file_bytes += metadata.len();
392        accumulator.files.push((path, metadata));
393    }
394}
395
396fn merge_collected(accumulator: &mut CollectionAccumulator, collected: CollectedPaths) {
397    accumulator.excluded_count += collected.excluded_count;
398    accumulator
399        .collection_errors
400        .extend(collected.collection_errors);
401
402    for (path, metadata) in collected.files {
403        insert_file(accumulator, path, metadata);
404    }
405    for (path, metadata) in collected.directories {
406        if accumulator.dir_seen.insert(path.clone()) {
407            accumulator.directories.push((path, metadata));
408        }
409    }
410}
411
412#[cfg(test)]
413mod tests {
414    use super::{CollectionFrontier, collect_paths, collect_selected_paths};
415    use std::fs;
416    use std::path::PathBuf;
417
418    #[test]
419    fn file_scan_root_uses_parent_directory() {
420        let temp_dir = tempfile::tempdir().expect("temp dir");
421        let file_path = temp_dir.path().join("Directory.Packages.props");
422        fs::write(&file_path, "<Project />").expect("write props file");
423
424        let collected = collect_paths(&file_path, 0, &[]);
425        assert_eq!(collected.file_count(), 1);
426        assert_eq!(collected.directory_count(), 0);
427        assert_eq!(collected.scan_root(), Some(temp_dir.path()));
428    }
429
430    #[test]
431    fn collect_paths_recurses_regular_directories() {
432        let temp_dir = tempfile::tempdir().expect("temp dir");
433        let nested = temp_dir.path().join("src/bin");
434        fs::create_dir_all(&nested).expect("create nested directory");
435        fs::write(nested.join("main.rs"), "fn main() {}\n").expect("write nested file");
436
437        let collected = collect_paths(temp_dir.path(), 0, &[]);
438
439        assert!(
440            collected
441                .files
442                .iter()
443                .any(|(path, _)| path == &temp_dir.path().join("src/bin/main.rs"))
444        );
445    }
446
447    #[cfg(unix)]
448    #[test]
449    fn collect_paths_does_not_recurse_into_symlinked_directory_cycle() {
450        use std::os::unix::fs::symlink;
451
452        let temp_dir = tempfile::tempdir().expect("temp dir");
453        let root = temp_dir.path();
454        let real = root.join("real");
455        fs::create_dir_all(&real).expect("create real directory");
456        fs::write(real.join("file.txt"), "content\n").expect("write file");
457        symlink(root, real.join("back")).expect("create symlink cycle");
458
459        let collected = collect_paths(root, 0, &[]);
460
461        assert!(collected.collection_errors.is_empty());
462        assert_eq!(collected.file_count(), 1);
463        assert!(
464            collected
465                .files
466                .iter()
467                .all(|(path, _)| !path.starts_with(real.join("back")))
468        );
469    }
470
471    #[cfg(unix)]
472    #[test]
473    fn collect_paths_recurses_explicit_symlinked_scan_root() {
474        use std::os::unix::fs::symlink;
475
476        let temp_dir = tempfile::tempdir().expect("temp dir");
477        let target = temp_dir.path().join("target");
478        fs::create_dir_all(&target).expect("create target directory");
479        fs::write(target.join("inside.txt"), "content\n").expect("write file");
480        let root_link = temp_dir.path().join("root-link");
481        symlink(&target, &root_link).expect("create root symlink");
482
483        let collected = collect_paths(&root_link, 0, &[]);
484
485        assert!(collected.collection_errors.is_empty());
486        assert!(
487            collected
488                .files
489                .iter()
490                .any(|(path, _)| path == &root_link.join("inside.txt"))
491        );
492    }
493
494    #[cfg(unix)]
495    #[test]
496    fn collect_paths_keeps_symlinked_regular_files_scannable() {
497        use std::os::unix::fs::symlink;
498
499        let temp_dir = tempfile::tempdir().expect("temp dir");
500        let root = temp_dir.path();
501        let target = root.join("target.txt");
502        fs::write(&target, "content\n").expect("write target file");
503        let link = root.join("link.txt");
504        symlink(&target, &link).expect("create file symlink");
505
506        let collected = collect_paths(root, 0, &[]);
507
508        assert!(collected.collection_errors.is_empty());
509        assert!(
510            collected
511                .files
512                .iter()
513                .any(|(path, _)| path == &root.join("link.txt"))
514        );
515    }
516
517    #[cfg(unix)]
518    #[test]
519    fn collect_selected_paths_does_not_recurse_into_symlinked_directory() {
520        use std::os::unix::fs::symlink;
521
522        let temp_dir = tempfile::tempdir().expect("temp dir");
523        let root = temp_dir.path();
524        let target = root.join("target");
525        fs::create_dir_all(&target).expect("create target directory");
526        fs::write(target.join("inside.txt"), "content\n").expect("write file");
527        symlink(&target, root.join("link")).expect("create directory symlink");
528
529        let collected = collect_selected_paths(
530            root,
531            &[CollectionFrontier {
532                path: PathBuf::from("link"),
533                recurse: true,
534            }],
535            0,
536            &[],
537        );
538
539        assert!(collected.collection_errors.is_empty());
540        assert!(collected.files.is_empty());
541        assert!(
542            collected
543                .directories
544                .iter()
545                .any(|(path, _)| path == &root.join("link"))
546        );
547    }
548}