greplm_core/
walk.rs

1//! Filesystem traversal.
2//!
3//! Uses ripgrep's `ignore` crate, which honors `.gitignore`/`.ignore`, prunes
4//! ignored directories early, and uses `d_type` from `getdents64` to avoid an
5//! extra `stat` per entry. We walk sequentially (the walk is rarely the
6//! bottleneck) and feed the resulting file list to a parallel read/parse stage.
7
8use ignore::overrides::OverrideBuilder;
9use ignore::{WalkBuilder, WalkState};
10use std::fs::Metadata;
11use std::path::PathBuf;
12use std::sync::mpsc::channel;
13
14use crate::config::Config;
15use crate::error::{Error, Result};
16use crate::paths::Paths;
17
18/// A file discovered during the walk.
19#[derive(Debug, Clone)]
20pub struct WalkEntry {
21    /// Absolute path on disk.
22    pub path: PathBuf,
23    /// Path relative to the project root, using `/` separators.
24    pub rel: String,
25    pub metadata: Metadata,
26}
27
28/// Walk the project root and return all candidate text files.
29pub fn walk(paths: &Paths, config: &Config) -> Result<Vec<WalkEntry>> {
30    let mut overrides = OverrideBuilder::new(&paths.root);
31    for pat in &config.include {
32        overrides
33            .add(pat)
34            .map_err(|e| Error::other(format!("bad include glob {pat:?}: {e}")))?;
35    }
36    for pat in &config.exclude {
37        overrides
38            .add(&format!("!{pat}"))
39            .map_err(|e| Error::other(format!("bad exclude glob {pat:?}: {e}")))?;
40    }
41    let overrides = overrides
42        .build()
43        .map_err(|e| Error::other(format!("invalid overrides: {e}")))?;
44
45    let mut builder = WalkBuilder::new(&paths.root);
46    builder
47        .hidden(!config.index_hidden)
48        .git_ignore(config.respect_gitignore)
49        .git_global(config.respect_gitignore)
50        .git_exclude(config.respect_gitignore)
51        .ignore(config.respect_gitignore)
52        .parents(config.respect_gitignore)
53        .overrides(overrides)
54        .follow_links(false);
55
56    // Parallel walk: worker threads send entries over a channel so traversal
57    // overlaps with downstream processing and uses all cores.
58    let (tx, rx) = channel::<WalkEntry>();
59    let root = paths.root.clone();
60    let max_size = config.max_file_size;
61    builder.build_parallel().run(|| {
62        let tx = tx.clone();
63        let root = root.clone();
64        Box::new(move |result| {
65            let dent = match result {
66                Ok(d) => d,
67                Err(_) => return WalkState::Continue,
68            };
69            match dent.file_type() {
70                Some(ft) if ft.is_file() => {}
71                _ => return WalkState::Continue,
72            }
73            let metadata = match dent.metadata() {
74                Ok(m) => m,
75                Err(_) => return WalkState::Continue,
76            };
77            if metadata.len() > max_size || metadata.len() == 0 {
78                return WalkState::Continue;
79            }
80            let path = dent.into_path();
81            let rel = match path.strip_prefix(&root) {
82                Ok(r) => r.to_string_lossy().replace('\\', "/"),
83                Err(_) => path.to_string_lossy().to_string(),
84            };
85            let _ = tx.send(WalkEntry {
86                path,
87                rel,
88                metadata,
89            });
90            WalkState::Continue
91        })
92    });
93    drop(tx);
94    Ok(rx.into_iter().collect())
95}
greplm_core/walk.rs

greplm_core/
walk.rs
