sift_core/search/candidates/
walk.rs

1use std::collections::HashSet;
2use std::path::{Path, PathBuf};
3use std::sync::{Arc, Mutex};
4
5use ignore::{DirEntry, Error as IgnoreError, WalkBuilder, WalkState};
6
7use crate::Candidate;
8use crate::search::filter::{CandidateFilter, HiddenMode, IgnoreSources};
9use crate::search::request::{LinkTraversal, WalkOptions};
10
11/// Per-scope parallel walk that produces candidates.
12///
13/// Also serves as [`ignore::ParallelVisitorBuilder`] — each worker thread
14/// receives a thread-local [`CandidateCollector`].
15struct CandidateWalk<'a> {
16    filter: &'a CandidateFilter,
17    root: PathBuf,
18    filter_root: PathBuf,
19    walk_error: Arc<Mutex<Option<crate::Error>>>,
20    consolidated: Arc<Mutex<Vec<Candidate>>>,
21}
22
23impl<'a> CandidateWalk<'a> {
24    fn new(root: &Path, filter: &'a CandidateFilter) -> crate::Result<Self> {
25        let root = root.canonicalize()?;
26        let filter_root = filter
27            .root()
28            .canonicalize()
29            .unwrap_or_else(|_| filter.root().to_path_buf());
30        Ok(Self {
31            filter,
32            root,
33            filter_root,
34            walk_error: Arc::new(Mutex::new(None)),
35            consolidated: Arc::new(Mutex::new(Vec::new())),
36        })
37    }
38
39    fn walk_builder(&self) -> WalkBuilder {
40        let visibility = self.filter.visibility();
41        let sources = visibility.ignore.sources;
42        let mut builder = WalkBuilder::new(&self.root);
43        builder
44            .follow_links(self.filter.follow_links())
45            .same_file_system(self.filter.one_file_system())
46            .hidden(matches!(visibility.hidden, HiddenMode::Respect))
47            .parents(sources.contains(IgnoreSources::PARENT))
48            .ignore(sources.contains(IgnoreSources::DOT))
49            .git_ignore(sources.contains(IgnoreSources::VCS))
50            .git_exclude(sources.contains(IgnoreSources::EXCLUDE))
51            .git_global(sources.contains(IgnoreSources::GLOBAL))
52            .require_git(visibility.ignore.require_git);
53        if let Some(d) = self.filter.max_depth() {
54            builder.max_depth(Some(d + 1));
55        }
56        builder
57    }
58
59    fn collect(&mut self) -> crate::Result<Vec<Candidate>> {
60        self.walk_builder().build_parallel().visit(self);
61
62        {
63            let mut guard = self.walk_error.lock().expect("walk error lock");
64            if let Some(err) = guard.take() {
65                return Err(err);
66            }
67        }
68
69        Ok(std::mem::take(
70            &mut *self.consolidated.lock().expect("candidate lock"),
71        ))
72    }
73}
74
75impl<'a> ignore::ParallelVisitorBuilder<'a> for CandidateWalk<'_> {
76    fn build(&mut self) -> Box<dyn ignore::ParallelVisitor + 'a> {
77        Box::new(CandidateCollector {
78            filter_root: self.filter_root.clone(),
79            max_filesize: self.filter.max_filesize(),
80            cache_metadata: true,
81            thread_candidates: Vec::new(),
82            walk_error: Arc::clone(&self.walk_error),
83            consolidated: Arc::clone(&self.consolidated),
84        })
85    }
86}
87
88/// Per-thread collector for the parallel walk.
89struct CandidateCollector {
90    filter_root: PathBuf,
91    max_filesize: Option<u64>,
92    cache_metadata: bool,
93    thread_candidates: Vec<Candidate>,
94    walk_error: Arc<Mutex<Option<crate::Error>>>,
95    consolidated: Arc<Mutex<Vec<Candidate>>>,
96}
97
98impl Drop for CandidateCollector {
99    fn drop(&mut self) {
100        if self.thread_candidates.is_empty() {
101            return;
102        }
103        let mut guard = self.consolidated.lock().expect("candidate lock");
104        guard.append(&mut self.thread_candidates);
105    }
106}
107
108impl ignore::ParallelVisitor for CandidateCollector {
109    fn visit(&mut self, entry: Result<DirEntry, IgnoreError>) -> WalkState {
110        let entry = match entry {
111            Ok(e) => e,
112            Err(err) => {
113                let mut guard = self.walk_error.lock().expect("walk error lock");
114                if guard.is_none() {
115                    *guard = Some(crate::Error::Ignore(err));
116                }
117                drop(guard);
118                return WalkState::Quit;
119            }
120        };
121        if !entry.file_type().is_some_and(|ft| ft.is_file()) {
122            return WalkState::Continue;
123        }
124        let metadata = entry.metadata().ok();
125        if let Some(limit) = self.max_filesize
126            && metadata.as_ref().is_some_and(|m| m.len() > limit)
127        {
128            return WalkState::Continue;
129        }
130        let (cached_size, cached_depth) = if self.cache_metadata {
131            (
132                metadata.map(|m| m.len()),
133                Some(entry.depth().saturating_sub(1)),
134            )
135        } else {
136            (None, None)
137        };
138        let abs_path = entry.path().to_path_buf();
139        let rel_path = abs_path
140            .strip_prefix(&self.filter_root)
141            .unwrap_or(&abs_path)
142            .to_path_buf();
143        self.thread_candidates.push(Candidate::with_metadata(
144            rel_path,
145            abs_path,
146            cached_size,
147            cached_depth,
148        ));
149        WalkState::Continue
150    }
151}
152
153/// Collect candidate files across all scopes by walking the filesystem.
154impl CandidateFilter {
155    /// # Errors
156    ///
157    /// Returns an error if walking the corpus fails.
158    pub fn collect(&self) -> crate::Result<Vec<Candidate>> {
159        let filter_root = self
160            .root()
161            .canonicalize()
162            .unwrap_or_else(|_| self.root().to_path_buf());
163        let mut out = Vec::new();
164        for scope in self.scopes() {
165            let path = if scope.as_os_str().is_empty() {
166                filter_root.clone()
167            } else {
168                filter_root.join(scope)
169            };
170            if !path.exists() {
171                continue;
172            }
173            let path = path.canonicalize().unwrap_or(path);
174            if path.is_file() {
175                let rel_path = path
176                    .strip_prefix(&filter_root)
177                    .unwrap_or(&path)
178                    .to_path_buf();
179                out.push(Candidate::new(rel_path, path));
180            } else if path.is_dir() {
181                let mut walk = CandidateWalk::new(&path, self)?;
182                out.extend(walk.collect()?);
183            }
184        }
185        out.sort_by(|a, b| a.rel_path().cmp(b.rel_path()));
186        out.dedup();
187        Ok(out)
188    }
189}
190
191impl WalkOptions {
192    /// Discovers files under the given root matching these walk options.
193    ///
194    /// # Errors
195    ///
196    /// Returns an error if the root path cannot be canonicalized or
197    /// the walk encounters an inaccessible directory.
198    pub fn discover_files(&self, root: &Path) -> crate::Result<HashSet<PathBuf>> {
199        let root = root.canonicalize()?;
200        let mut set = HashSet::new();
201        let follow = matches!(self.links, LinkTraversal::Follow);
202        let mut builder = ignore::WalkBuilder::new(&root);
203        builder.follow_links(follow);
204        if let Some(depth) = self.max_depth {
205            builder.max_depth(Some(depth + 1));
206        }
207        builder.same_file_system(self.one_file_system);
208        let walker = builder.build();
209        for entry in walker {
210            let entry = entry.map_err(crate::Error::Ignore)?;
211            if !entry.file_type().is_some_and(|ft| ft.is_file()) {
212                continue;
213            }
214            let path = entry.path();
215            if self
216                .max_filesize
217                .is_some_and(|limit| std::fs::metadata(path).is_ok_and(|m| m.len() > limit))
218            {
219                continue;
220            }
221            let display = path.strip_prefix(&root).unwrap_or(path).to_path_buf();
222            set.insert(display);
223        }
224        Ok(set)
225    }
226}
sift_core/search/candidates/walk.rs

sift_core/search/candidates/
walk.rs
