pagers_core/
crawl.rs

1use std::io::{self, BufRead};
2use std::path::{Path, PathBuf};
3use std::sync::atomic::Ordering;
4
5use ignore::WalkBuilder;
6
7use crate::mincore::PageMap;
8use crate::mode::DisplayMode;
9use crate::ops::{FileRange, Op, Stats};
10use crate::par::{InodeSet, SeenInodes as _, par_collect};
11
12#[derive(Debug, Clone, PartialEq)]
13#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
14pub struct CrawlConfig {
15    pub follow_symlinks: bool,
16    pub single_filesystem: bool,
17    pub count_hardlinks: bool,
18    pub ignore_patterns: Vec<String>,
19    pub filter_patterns: Vec<String>,
20    pub max_file_size: Option<u64>,
21    pub batch: Option<PathBuf>,
22    pub nul_delim: bool,
23}
24
25pub fn crawl_and_process<O: Op, PM: PageMap + Send + Sync, D: DisplayMode<PM>>(
26    paths: &[PathBuf],
27    crawl_config: &CrawlConfig,
28    op: &O,
29    range: &FileRange,
30    stats: &Stats,
31    display: &D,
32) -> crate::Result<Vec<O::Output>> {
33    let seen_inodes = InodeSet::default();
34    let file_paths = collect_file_paths(paths, crawl_config, &seen_inodes, stats);
35
36    let outputs = par_collect(&file_paths, |path| {
37        display.process_one::<O>(op, path, range, stats)
38    });
39
40    display.finish();
41    op.finish()?;
42
43    Ok(outputs)
44}
45
46fn collect_file_paths(
47    paths: &[PathBuf],
48    crawl_config: &CrawlConfig,
49    seen_inodes: &InodeSet,
50    stats: &Stats,
51) -> Vec<PathBuf> {
52    let mut all_paths: Vec<PathBuf> = paths.to_vec();
53
54    if let Some(batch_path) = &crawl_config.batch {
55        match read_batch_paths(batch_path, crawl_config.nul_delim) {
56            Ok(batch_paths) => all_paths.extend(batch_paths),
57            Err(e) => tracing::warn!("batch file: {e}"),
58        }
59    }
60
61    let needs_meta = crawl_config.max_file_size.is_some() || !crawl_config.count_hardlinks;
62    let mut file_paths = Vec::new();
63
64    for path in &all_paths {
65        if path.is_dir() {
66            stats.total_dirs.fetch_add(1, Ordering::Relaxed);
67            collect_dir_entries(path, crawl_config, needs_meta, seen_inodes, &mut file_paths);
68        } else if path.is_file() {
69            file_paths.push(path.clone());
70        } else {
71            tracing::warn!("skipping {}: not a file or directory", path.display());
72        }
73    }
74
75    file_paths
76}
77
78fn collect_dir_entries(
79    root: &Path,
80    config: &CrawlConfig,
81    needs_meta: bool,
82    seen_inodes: &InodeSet,
83    out: &mut Vec<PathBuf>,
84) {
85    let mut builder = WalkBuilder::new(root);
86    builder
87        .follow_links(config.follow_symlinks)
88        .same_file_system(config.single_filesystem)
89        .hidden(false)
90        .git_ignore(false)
91        .git_global(false)
92        .git_exclude(false);
93
94    if !config.ignore_patterns.is_empty() || !config.filter_patterns.is_empty() {
95        let mut overrides = ignore::overrides::OverrideBuilder::new(root);
96        for pat in &config.ignore_patterns {
97            let _ = overrides.add(&format!("!{pat}"));
98        }
99        for pat in &config.filter_patterns {
100            let _ = overrides.add(pat);
101        }
102        if let Ok(ov) = overrides.build() {
103            builder.overrides(ov);
104        }
105    }
106
107    for entry in builder.build() {
108        let Ok(entry) = entry.inspect_err(|e| tracing::warn!("{e}")) else {
109            continue;
110        };
111
112        let Some(ft) = entry.file_type() else {
113            continue;
114        };
115
116        if !ft.is_file() {
117            continue;
118        }
119
120        let entry_path = entry.path();
121        let meta = if needs_meta {
122            entry_path.metadata().ok()
123        } else {
124            None
125        };
126
127        if let Some(max_size) = config.max_file_size
128            && let Some(ref m) = meta
129            && m.len() > max_size
130        {
131            continue;
132        }
133
134        if !config.count_hardlinks {
135            #[cfg(unix)]
136            {
137                use std::os::unix::fs::MetadataExt;
138                if let Some(ref m) = meta
139                    && m.nlink() > 1
140                    && seen_inodes.already_seen((m.dev(), m.ino()))
141                {
142                    continue;
143                }
144            }
145        }
146
147        out.push(entry_path.to_path_buf());
148    }
149}
150
151pub fn read_batch_paths(path: &Path, nul_delim: bool) -> io::Result<Vec<PathBuf>> {
152    use std::os::unix::ffi::OsStrExt;
153
154    let reader: Box<dyn BufRead> = if path == Path::new("-") {
155        Box::new(io::stdin().lock())
156    } else {
157        Box::new(io::BufReader::new(std::fs::File::open(path)?))
158    };
159
160    let delim = if nul_delim { b'\0' } else { b'\n' };
161    reader
162        .split(delim)
163        .filter_map(|r| match r {
164            Ok(buf) if !buf.is_empty() => {
165                Some(Ok(PathBuf::from(std::ffi::OsStr::from_bytes(&buf))))
166            }
167            Ok(_) => None,
168            Err(e) => Some(Err(e)),
169        })
170        .collect()
171}
pagers_core/crawl.rs

pagers_core/
crawl.rs
