#![deny(unsafe_code)]
#![warn(missing_docs)]
#![doc = "Portable filesystem walking."]
use std::{
borrow::Cow,
error::Error,
ffi::{OsStr, OsString},
fmt, fs,
path::{Path, PathBuf},
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
};
use ferralk_glob::{Pattern, PatternError, PatternOptions, WalkerPathViability};
pub use ferralk_glob;
#[cfg(all(feature = "native-linux", target_os = "linux"))]
#[allow(unsafe_code)]
mod linux_native;
#[cfg(all(feature = "native-macos", target_os = "macos"))]
#[allow(unsafe_code)]
mod macos_native;
#[cfg(all(
test,
any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
)
))]
mod native_parity;
#[cfg(all(feature = "native-linux", target_os = "linux"))]
#[doc(hidden)]
pub use linux_native::fuzz_validate_records as fuzz_validate_linux_dirent_records;
#[cfg(all(feature = "native-macos", target_os = "macos"))]
#[doc(hidden)]
pub use macos_native::fuzz_validate_bulk_record as fuzz_validate_macos_bulk_record;
#[cfg(all(feature = "native-macos", target_os = "macos"))]
#[doc(hidden)]
pub use macos_native::fuzz_validate_records as fuzz_validate_macos_dirent_records;
mod absolute;
#[doc(hidden)]
pub fn corpus_rewrite_absolute_pattern(
pattern: &[u8],
root: &[u8],
windows_paths: bool,
) -> Result<Option<Vec<u8>>, PatternError> {
let syntax = if windows_paths {
absolute::Syntax::Windows
} else {
absolute::Syntax::Posix
};
rewrite_pattern_for_root(pattern, root, syntax)
}
mod classify;
mod gitignore;
mod ignore_rules;
#[doc(hidden)]
pub use ignore_rules::fuzz_rule as fuzz_ignore_rule;
#[doc(hidden)]
pub use ignore_rules::fuzz_rule_bytes as fuzz_ignore_rule_bytes;
mod parallel;
mod scheduler;
use classify::{DirectoryTask, EmittedEntry, EntryAction, TraversalContext, classify_entry};
use gitignore::{IgnoreReadError, IgnoreScope};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum ErrorPolicy {
Abort,
Skip,
#[default]
Collect,
}
#[derive(Debug, Clone, Default)]
pub struct CancellationToken {
cancelled: Arc<AtomicBool>,
}
impl CancellationToken {
pub fn cancel(&self) {
self.cancelled.store(true, Ordering::Release);
}
#[must_use]
pub fn is_cancelled(&self) -> bool {
self.cancelled.load(Ordering::Acquire)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum WildcardMode {
#[default]
ComponentScoped,
SeparatorCrossing,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct WalkOptions {
follow_symlinks: bool,
sort: bool,
metadata: bool,
directories_only: bool,
files_only: bool,
resolve_symlink_kind: bool,
skip_hidden: bool,
keep_git_dir: bool,
max_depth: Option<usize>,
}
impl WalkOptions {
#[must_use]
pub const fn follow_symlinks(mut self, enabled: bool) -> Self {
self.follow_symlinks = enabled;
self
}
#[must_use]
pub const fn sort(mut self, enabled: bool) -> Self {
self.sort = enabled;
self
}
#[must_use]
pub const fn metadata(mut self, enabled: bool) -> Self {
self.metadata = enabled;
self
}
#[must_use]
pub const fn directories_only(mut self, enabled: bool) -> Self {
self.directories_only = enabled;
self
}
#[must_use]
pub const fn files_only(mut self, enabled: bool) -> Self {
self.files_only = enabled;
self
}
#[must_use]
pub const fn resolve_symlink_kind(mut self, enabled: bool) -> Self {
self.resolve_symlink_kind = enabled;
self
}
#[must_use]
pub const fn skip_hidden(mut self, enabled: bool) -> Self {
self.skip_hidden = enabled;
self
}
#[must_use]
pub const fn keep_git_dir(mut self, enabled: bool) -> Self {
self.keep_git_dir = enabled;
self
}
#[must_use]
pub const fn max_depth(mut self, max_depth: usize) -> Self {
self.max_depth = Some(max_depth);
self
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WalkEntryKind {
File,
Directory,
Symlink,
}
#[derive(Debug, Clone)]
pub struct WalkEntry {
path: PathBuf,
root: Arc<Path>,
is_dir: bool,
is_symlink: bool,
depth: usize,
metadata: Option<Box<fs::Metadata>>,
}
impl WalkEntry {
#[must_use]
pub fn path(&self) -> &Path {
&self.path
}
#[must_use]
pub fn path_bytes(&self) -> &[u8] {
self.path.as_os_str().as_encoded_bytes()
}
#[must_use]
pub fn root(&self) -> &Path {
&self.root
}
#[must_use]
pub const fn is_dir(&self) -> bool {
self.is_dir
}
#[must_use]
pub const fn is_symlink(&self) -> bool {
self.is_symlink
}
#[must_use]
pub const fn kind(&self) -> WalkEntryKind {
if self.is_symlink {
WalkEntryKind::Symlink
} else if self.is_dir {
WalkEntryKind::Directory
} else {
WalkEntryKind::File
}
}
#[must_use]
pub const fn depth(&self) -> usize {
self.depth
}
#[must_use]
pub fn basename(&self) -> Option<&std::ffi::OsStr> {
self.path.file_name()
}
#[must_use]
pub fn metadata(&self) -> Option<&fs::Metadata> {
self.metadata.as_deref()
}
}
#[derive(Debug)]
pub struct WalkError {
operation: &'static str,
path: PathBuf,
source: std::io::Error,
}
impl WalkError {
fn new(operation: &'static str, path: PathBuf, source: std::io::Error) -> Self {
Self {
operation,
path,
source,
}
}
#[must_use]
pub const fn operation(&self) -> &'static str {
self.operation
}
#[must_use]
pub fn path(&self) -> &Path {
&self.path
}
}
impl fmt::Display for WalkError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
formatter,
"{} {}: {}",
self.operation,
self.path.display(),
self.source
)
}
}
impl Error for WalkError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
Some(&self.source)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Verdict {
Keep,
Skip,
Stop,
}
#[derive(Debug)]
pub struct WalkResult {
entries: Vec<WalkEntry>,
errors: Vec<WalkError>,
cancelled: bool,
}
impl WalkResult {
#[must_use]
pub fn entries(&self) -> &[WalkEntry] {
&self.entries
}
#[must_use]
pub fn errors(&self) -> &[WalkError] {
&self.errors
}
#[must_use]
pub fn into_parts(self) -> (Vec<WalkEntry>, Vec<WalkError>) {
(self.entries, self.errors)
}
#[must_use]
pub const fn was_cancelled(&self) -> bool {
self.cancelled
}
}
pub(crate) type EntryVisitor<'a> = &'a (dyn Fn(&WalkEntry) -> Verdict + Sync + 'a);
pub(crate) fn keep_every_entry(_: &WalkEntry) -> Verdict {
Verdict::Keep
}
#[derive(Debug, Clone)]
struct RootPlan {
path: PathBuf,
shared_path: Arc<Path>,
relative_start: usize,
includes: Vec<TraversalPattern>,
excludes: Vec<TraversalPattern>,
}
#[derive(Debug, Clone)]
pub struct Walker {
roots: Vec<RootPlan>,
include_sources: Vec<Vec<u8>>,
exclude_sources: Vec<Vec<u8>>,
match_hidden: bool,
options: WalkOptions,
error_policy: ErrorPolicy,
cancellation: Option<CancellationToken>,
respect_git_ignore: bool,
git_ignore_case: Option<bool>,
git_precompose_unicode: Option<bool>,
wildcard_mode: WildcardMode,
threads: usize,
}
const MAX_WORKERS: usize = 256;
impl RootPlan {
fn relative_start(root: &Path) -> usize {
let mut probe = root.to_path_buf();
probe.push("x");
probe.as_os_str().as_encoded_bytes().len() - 1
}
fn new(path: PathBuf) -> Self {
Self {
relative_start: Self::relative_start(&path),
shared_path: Arc::from(path.as_path()),
path,
includes: Vec::new(),
excludes: Vec::new(),
}
}
}
impl Walker {
#[must_use]
pub fn new(root: impl Into<PathBuf>) -> Self {
Self {
roots: vec![RootPlan::new(root.into())],
include_sources: Vec::new(),
exclude_sources: Vec::new(),
match_hidden: false,
options: WalkOptions::default(),
error_policy: ErrorPolicy::default(),
cancellation: None,
respect_git_ignore: false,
git_ignore_case: None,
git_precompose_unicode: None,
wildcard_mode: WildcardMode::default(),
threads: std::thread::available_parallelism()
.map(std::num::NonZeroUsize::get)
.unwrap_or(1)
.min(MAX_WORKERS),
}
}
pub fn include(mut self, pattern: impl AsRef<[u8]>) -> Result<Self, PatternError> {
self.try_include(pattern)?;
Ok(self)
}
pub fn try_include(&mut self, pattern: impl AsRef<[u8]>) -> Result<&mut Self, PatternError> {
let pattern = pattern.as_ref();
let compiled = self.compile_for_every_root(pattern)?;
for (root, pattern) in self.roots.iter_mut().zip(compiled) {
root.includes.push(pattern);
}
self.include_sources.push(pattern.to_vec());
Ok(self)
}
pub fn exclude(mut self, pattern: impl AsRef<[u8]>) -> Result<Self, PatternError> {
self.try_exclude(pattern)?;
Ok(self)
}
pub fn try_exclude(&mut self, pattern: impl AsRef<[u8]>) -> Result<&mut Self, PatternError> {
let pattern = pattern.as_ref();
let compiled = self.compile_for_every_root(pattern)?;
for (root, pattern) in self.roots.iter_mut().zip(compiled) {
root.excludes.push(pattern);
}
self.exclude_sources.push(pattern.to_vec());
Ok(self)
}
pub fn add_root(mut self, root: impl Into<PathBuf>) -> Result<Self, PatternError> {
self.try_add_root(root)?;
Ok(self)
}
pub fn try_add_root(&mut self, root: impl Into<PathBuf>) -> Result<&mut Self, PatternError> {
let mut plan = RootPlan::new(root.into());
let options = traversal_pattern_options(self.match_hidden);
let root_bytes = glob_path_bytes(&plan.path);
for source in &self.include_sources {
plan.includes
.push(compile_for_root(source, root_bytes.as_ref(), options)?);
}
for source in &self.exclude_sources {
plan.excludes
.push(compile_for_root(source, root_bytes.as_ref(), options)?);
}
drop(root_bytes);
self.roots.push(plan);
Ok(self)
}
pub fn add_roots<P: Into<PathBuf>>(
mut self,
roots: impl IntoIterator<Item = P>,
) -> Result<Self, PatternError> {
for root in roots {
self = self.add_root(root)?;
}
Ok(self)
}
#[must_use]
pub fn roots(&self) -> impl ExactSizeIterator<Item = &Path> {
self.roots.iter().map(|root| root.path.as_path())
}
fn compile_for_every_root(
&self,
pattern: &[u8],
) -> Result<Vec<TraversalPattern>, PatternError> {
let options = traversal_pattern_options(self.match_hidden);
self.roots
.iter()
.map(|root| compile_for_root(pattern, glob_path_bytes(&root.path).as_ref(), options))
.collect()
}
#[must_use]
pub fn match_hidden(mut self, enabled: bool) -> Self {
if self.match_hidden == enabled {
return self;
}
self.match_hidden = enabled;
let options = traversal_pattern_options(enabled);
for root in &mut self.roots {
for pattern in root.includes.iter_mut().chain(root.excludes.iter_mut()) {
pattern.recompile(options);
}
}
self
}
#[must_use]
pub const fn wildcard_mode(mut self, mode: WildcardMode) -> Self {
self.wildcard_mode = mode;
self
}
#[must_use]
pub const fn options(mut self, options: WalkOptions) -> Self {
self.options = options;
self
}
#[must_use]
pub const fn error_policy(mut self, error_policy: ErrorPolicy) -> Self {
self.error_policy = error_policy;
self
}
#[must_use]
pub fn cancellation(mut self, cancellation: CancellationToken) -> Self {
self.cancellation = Some(cancellation);
self
}
#[must_use]
pub const fn respect_git_ignore(mut self, enabled: bool) -> Self {
self.respect_git_ignore = enabled;
self
}
#[must_use]
pub const fn git_ignore_case(mut self, enabled: bool) -> Self {
self.git_ignore_case = Some(enabled);
self
}
#[must_use]
pub const fn clear_git_ignore_case(mut self) -> Self {
self.git_ignore_case = None;
self
}
#[must_use]
pub const fn git_precompose_unicode(mut self, enabled: bool) -> Self {
self.git_precompose_unicode = Some(enabled);
self
}
#[must_use]
pub const fn clear_git_precompose_unicode(mut self) -> Self {
self.git_precompose_unicode = None;
self
}
#[must_use]
pub const fn threads(mut self, threads: usize) -> Self {
self.threads = if threads == 0 {
1
} else if threads > MAX_WORKERS {
MAX_WORKERS
} else {
threads
};
self
}
pub fn collect(self) -> Result<WalkResult, WalkError> {
self.collect_with(&SystemBackend)
}
pub fn visit<V>(self, visitor: V) -> Result<WalkResult, WalkError>
where
V: Fn(&WalkEntry) -> Verdict + Sync,
{
self.walk(&SystemBackend, &visitor)
}
fn collect_with<B: DirectoryBackend + Sync>(
self,
backend: &B,
) -> Result<WalkResult, WalkError> {
self.walk(backend, &keep_every_entry)
}
fn walk<B: DirectoryBackend + Sync>(
self,
backend: &B,
visitor: EntryVisitor<'_>,
) -> Result<WalkResult, WalkError> {
if self.threads > 1 {
return parallel::collect(self, backend, visitor);
}
let mut state = WalkState::new(&self, visitor);
let scheduler = scheduler::Scheduler::new();
for task in self.root_tasks(backend) {
scheduler.push(task);
}
let worker = scheduler.worker();
while let Some(task) = scheduler.steal_into(&worker).or_else(|| worker.pop()) {
state.walk_directory(backend, task)?;
}
if self.options.sort {
state
.entries
.sort_by(|left, right| left.path.cmp(&right.path));
}
Ok(WalkResult {
entries: state.entries,
errors: state.errors,
cancelled: state.cancelled,
})
}
#[must_use]
pub fn stream(self) -> WalkStream {
let mut pending_directories = self.root_tasks(&SystemBackend);
pending_directories.reverse();
WalkStream {
pending_directories,
walker: self,
listing: Listing::default(),
glob_bytes: Vec::new(),
next_entry: 0,
path: PathBuf::new(),
directory: PathBuf::new(),
ancestors: AncestorChain::default(),
ignores: IgnoreScope::default(),
depth: 0,
root: 0,
pending_errors: Vec::new(),
cancelled: false,
stopped: false,
}
}
fn may_descend_into(&self, root: usize, relative: &[u8]) -> bool {
let includes = &self.roots[root].includes;
includes.is_empty()
|| includes
.iter()
.any(|pattern| pattern.could_match_descendant(relative))
}
fn may_descend_at(&self, root: usize, depth: usize, bytes: &[u8]) -> bool {
self.options
.max_depth
.is_none_or(|max_depth| depth < max_depth)
&& self.may_descend_into(root, bytes)
}
fn may_include_file(&self, root: usize, relative: &[u8]) -> bool {
let includes = &self.roots[root].includes;
includes.is_empty()
|| includes
.iter()
.any(|pattern| pattern.matches_extension(relative))
}
fn root_tasks<B: DirectoryBackend + ?Sized>(&self, backend: &B) -> Vec<DirectoryTask> {
self.roots
.iter()
.enumerate()
.map(|(index, plan)| {
let (ignores, ignore_errors) = IgnoreScope::for_root(self, backend, &plan.path);
DirectoryTask {
path: plan.path.clone(),
open: DirectoryOpen::default(),
depth: 0,
root: index,
ancestors: AncestorChain::default(),
ignores,
ignore_errors,
}
})
.collect()
}
}
fn has_hidden_component(path: &[u8]) -> bool {
path.split(is_path_separator)
.any(|component| component.first() == Some(&b'.'))
}
pub(crate) fn glob_path_bytes(path: &Path) -> Cow<'_, [u8]> {
glob_bytes(path.as_os_str().as_encoded_bytes())
}
pub(crate) fn glob_bytes(bytes: &[u8]) -> Cow<'_, [u8]> {
#[cfg(windows)]
{
Cow::Owned(
bytes
.iter()
.map(|&byte| if byte == b'\\' { b'/' } else { byte })
.collect(),
)
}
#[cfg(not(windows))]
{
Cow::Borrowed(bytes)
}
}
#[cfg(windows)]
pub(crate) fn glob_bytes_into<'a>(bytes: &[u8], scratch: &'a mut Vec<u8>) -> &'a [u8] {
scratch.clear();
scratch.extend(
bytes
.iter()
.map(|&byte| if byte == b'\\' { b'/' } else { byte }),
);
scratch
}
fn compile_for_root(
pattern: &[u8],
root: &[u8],
options: PatternOptions,
) -> Result<TraversalPattern, PatternError> {
match walker_pattern_for_root(pattern, root, absolute::Syntax::NATIVE, options)? {
Some(usable) => TraversalPattern::compile(&usable, options),
None => {
let mut compiled = TraversalPattern::compile(pattern, options)?;
compiled.never_matches = true;
Ok(compiled)
}
}
}
fn walker_pattern_for_root(
pattern: &[u8],
root: &[u8],
syntax: absolute::Syntax,
options: PatternOptions,
) -> Result<Option<Vec<u8>>, PatternError> {
let Some(rewritten) = rewrite_pattern_for_root(pattern, root, syntax)? else {
return Ok(None);
};
let parsed = Pattern::compile(pattern_without_directory_marker(&rewritten), options)?;
reject_unwalkable_relative_pattern(
parsed.walker_path_viability(),
parsed.walker_path_problem_offset(),
)?;
Ok(Some(rewritten))
}
fn rewrite_pattern_for_root(
pattern: &[u8],
root: &[u8],
syntax: absolute::Syntax,
) -> Result<Option<Vec<u8>>, PatternError> {
match absolute::rewrite_in(pattern, root, syntax)? {
absolute::Rewrite::Relative => {
absolute::reject_path_shaped(pattern, syntax)?;
Ok(Some(pattern.to_vec()))
}
absolute::Rewrite::Rooted(rooted) => {
absolute::reject_path_shaped(&rooted, syntax)?;
Ok(Some(rooted))
}
absolute::Rewrite::Outside => Ok(None),
}
}
fn pattern_without_directory_marker(pattern: &[u8]) -> &[u8] {
if pattern.len() > 1 {
pattern.strip_suffix(b"/").unwrap_or(pattern)
} else {
pattern
}
}
fn reject_unwalkable_relative_pattern(
viability: WalkerPathViability,
offset: Option<usize>,
) -> Result<(), PatternError> {
let message = match viability {
WalkerPathViability::Viable => return Ok(()),
WalkerPathViability::ParentComponent => {
"`..` in a walker-relative pattern is not resolved, because resolving it lexically would be wrong across a symlink"
}
WalkerPathViability::Root => {
"a walker-relative pattern that names the walk root itself selects nothing; add `/**` to select what is inside it"
}
WalkerPathViability::TrailingDot => {
"a walker-relative pattern ending in `/.` selects that directory itself; add `/**` to select what is inside it"
}
WalkerPathViability::DotComponent => {
"a walker-relative pattern with a `.` component is not normalized; remove `/.` to name the entry below that directory"
}
};
Err(PatternError::new(offset.unwrap_or(0), message))
}
fn traversal_pattern_options(match_hidden: bool) -> PatternOptions {
PatternOptions::default()
.braces(true)
.recursive_double_star(true)
.extglob(true)
.match_hidden(match_hidden)
}
#[derive(Debug, Clone)]
struct TraversalPattern {
source: Vec<u8>,
matcher: Pattern,
directories_only: bool,
subtree_root: Option<Pattern>,
literal_roots: Option<Vec<Vec<u8>>>,
extensions: Option<Vec<Vec<u8>>>,
never_matches: bool,
}
impl TraversalPattern {
fn compile(source: &[u8], options: PatternOptions) -> Result<Self, PatternError> {
let pattern = source.strip_prefix(b"./").unwrap_or(source);
let directories_only = pattern.len() > 1 && pattern.ends_with(b"/");
let pattern = if directories_only {
&pattern[..pattern.len() - 1]
} else {
pattern
};
let subtree_root = pattern
.strip_suffix(b"/**")
.map(|root| Pattern::compile(root, options))
.transpose()?;
let alternatives = ferralk_glob::expand_braces(pattern, options)?;
Ok(Self {
source: source.to_vec(),
matcher: Pattern::compile(pattern, options)?,
directories_only,
subtree_root,
literal_roots: prefilter_of_every_alternative(&alternatives, literal_pattern_root),
extensions: prefilter_of_every_alternative(&alternatives, literal_extension),
never_matches: false,
})
}
fn recompile(&mut self, options: PatternOptions) {
let source = std::mem::take(&mut self.source);
let never_matches = self.never_matches;
*self = Self::compile(&source, options)
.expect("a compiled pattern stays valid when only match_hidden changes");
self.never_matches = never_matches;
}
fn matches(&self, path: &[u8], is_dir: bool, mode: WildcardMode) -> bool {
if self.never_matches || (self.directories_only && !is_dir) {
return false;
}
match mode {
WildcardMode::ComponentScoped => self.matcher.is_match_glob_path(path),
WildcardMode::SeparatorCrossing => self.matcher.is_match(path),
}
}
fn covers_subtree(&self, path: &[u8], mode: WildcardMode) -> bool {
if self.never_matches || self.directories_only {
return false;
}
self.subtree_root.as_ref().is_some_and(|root| match mode {
WildcardMode::ComponentScoped => root.is_match_glob_path(path),
WildcardMode::SeparatorCrossing => root.is_match(path),
})
}
fn could_match_descendant(&self, path: &[u8]) -> bool {
if self.never_matches {
return false;
}
let Some(roots) = &self.literal_roots else {
return true;
};
roots
.iter()
.any(|root| shares_a_line_of_descent(root, path))
}
fn matches_extension(&self, path: &[u8]) -> bool {
if self.never_matches {
return false;
}
let Some(extensions) = &self.extensions else {
return true;
};
final_extension(path)
.is_some_and(|candidate| extensions.iter().any(|extension| extension == candidate))
}
}
fn shares_a_line_of_descent(root: &[u8], path: &[u8]) -> bool {
root == path
|| root
.strip_prefix(path)
.is_some_and(|suffix| suffix.starts_with(b"/"))
|| path
.strip_prefix(root)
.is_some_and(|suffix| suffix.starts_with(b"/"))
}
fn prefilter_of_every_alternative(
alternatives: &[Vec<u8>],
of_alternative: impl Fn(&[u8]) -> Option<Vec<u8>>,
) -> Option<Vec<Vec<u8>>> {
if alternatives.is_empty() {
return None;
}
let mut values = alternatives
.iter()
.map(|alternative| of_alternative(alternative))
.collect::<Option<Vec<_>>>()?;
values.sort_unstable();
values.dedup();
Some(values)
}
pub(crate) fn first_metacharacter(pattern: &[u8]) -> Option<usize> {
pattern.iter().enumerate().position(|(index, byte)| {
matches!(byte, b'*' | b'?' | b'[')
|| (*byte == b'\\')
|| (*byte == b'{' && has_closing_brace(pattern, index))
|| (matches!(byte, b'@' | b'+' | b'!')
&& pattern.get(index + 1) == Some(&b'(')
&& has_closing_parenthesis(pattern, index + 1))
})
}
fn literal_pattern_root(pattern: &[u8]) -> Option<Vec<u8>> {
let magic = first_metacharacter(pattern);
let prefix = &pattern[..magic.unwrap_or(pattern.len())];
let root = if magic.is_some() {
match prefix.strip_suffix(b"/") {
Some(complete) => complete,
None => {
let separator = prefix.iter().rposition(|byte| *byte == b'/')?;
&prefix[..separator]
}
}
} else {
prefix
};
(!root.is_empty()).then(|| root.to_vec())
}
fn literal_extension(pattern: &[u8]) -> Option<Vec<u8>> {
let extension = final_extension(pattern)?;
if extension.is_empty()
|| extension.iter().any(|byte| {
matches!(
byte,
b'*' | b'?' | b'[' | b']' | b'{' | b'}' | b'\\' | b'(' | b')' | b'|'
)
})
{
return None;
}
Some(extension.to_vec())
}
fn final_extension(path: &[u8]) -> Option<&[u8]> {
let name = path.rsplit(is_path_separator).next().unwrap_or(path);
let dot = name.iter().rposition(|byte| *byte == b'.')?;
name.get(dot + 1..)
}
fn is_path_separator(byte: &u8) -> bool {
*byte == b'/' || (cfg!(windows) && *byte == b'\\')
}
fn has_closing_brace(pattern: &[u8], open: usize) -> bool {
let mut depth = 0_usize;
let mut index = open;
while index < pattern.len() {
if pattern[index] == b'\\' {
index += 2;
continue;
}
match pattern[index] {
b'{' => depth += 1,
b'}' => {
depth -= 1;
if depth == 0 {
return true;
}
}
_ => {}
}
index += 1;
}
false
}
fn has_closing_parenthesis(pattern: &[u8], open: usize) -> bool {
let mut depth = 0_usize;
let mut index = open;
while index < pattern.len() {
if pattern[index] == b'\\' {
index += 2;
continue;
}
match pattern[index] {
b'(' => depth += 1,
b')' => {
depth -= 1;
if depth == 0 {
return true;
}
}
_ => {}
}
index += 1;
}
false
}
#[derive(Debug, Clone, Default)]
pub(crate) struct DirectoryOpen {
#[cfg(all(feature = "native-macos", target_os = "macos"))]
relative: Option<macos_native::RelativeDirectoryOpen>,
}
trait DirectoryBackend {
fn read_directory(
&self,
path: &Path,
follow_symlinks: bool,
refuse_final_symlink: bool,
listing: &mut Listing,
) -> std::io::Result<()>;
fn read_scheduled_directory(
&self,
path: &Path,
open: &DirectoryOpen,
follow_symlinks: bool,
refuse_final_symlink: bool,
listing: &mut Listing,
) -> std::io::Result<()> {
let _ = open;
self.read_directory(path, follow_symlinks, refuse_final_symlink, listing)
}
fn child_directory_open(&self, listing: &Listing, name: &OsStr) -> DirectoryOpen {
let _ = (listing, name);
DirectoryOpen::default()
}
fn metadata(&self, path: &Path) -> std::io::Result<fs::Metadata> {
fs::metadata(path)
}
fn symlink_metadata(&self, path: &Path) -> std::io::Result<fs::Metadata> {
fs::symlink_metadata(path)
}
#[cfg(not(unix))]
fn canonicalize(&self, path: &Path) -> std::io::Result<PathBuf> {
fs::canonicalize(path)
}
fn read_ignore_file(&self, path: &Path) -> std::io::Result<Vec<u8>> {
read_in_tree_ignore_file(path)
}
fn read_repository_file(&self, path: &Path) -> std::io::Result<Vec<u8>> {
read_bounded_file(path)
}
fn cycle_key(&self, path: &Path) -> std::io::Result<CycleKey> {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let metadata = self.metadata(path)?;
Ok((metadata.dev(), metadata.ino()))
}
#[cfg(not(unix))]
{
self.canonicalize(path)
}
}
}
pub(crate) const MAX_IGNORE_FILE_BYTES: u64 = 8 * 1024 * 1024;
pub(crate) fn read_bounded_file(path: &Path) -> std::io::Result<Vec<u8>> {
read_bounded(fs::File::open(path)?)
}
fn read_bounded(file: fs::File) -> std::io::Result<Vec<u8>> {
use std::io::Read;
let mut contents = Vec::new();
file.take(MAX_IGNORE_FILE_BYTES + 1)
.read_to_end(&mut contents)?;
if contents.len() as u64 > MAX_IGNORE_FILE_BYTES {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"configuration file exceeds the 8 MiB safety limit",
));
}
Ok(contents)
}
#[cfg(unix)]
fn read_in_tree_ignore_file(path: &Path) -> std::io::Result<Vec<u8>> {
use std::{fs::OpenOptions, os::unix::fs::OpenOptionsExt};
let file = OpenOptions::new()
.read(true)
.custom_flags(libc::O_NOFOLLOW)
.open(path)?;
read_bounded(file)
}
#[cfg(not(unix))]
fn read_in_tree_ignore_file(path: &Path) -> std::io::Result<Vec<u8>> {
if fs::symlink_metadata(path)?.file_type().is_symlink() {
return Err(std::io::Error::from(std::io::ErrorKind::InvalidInput));
}
read_bounded_file(path)
}
#[cfg(unix)]
type CycleKey = (u64, u64);
#[cfg(not(unix))]
type CycleKey = PathBuf;
#[derive(Debug, Default, Clone)]
pub(crate) struct AncestorChain(Option<Arc<AncestorLink>>);
#[derive(Debug)]
struct AncestorLink {
key: CycleKey,
parent: Option<Arc<AncestorLink>>,
}
impl AncestorChain {
fn enter(&self, key: CycleKey) -> Option<Self> {
let mut ancestor = self.0.as_deref();
while let Some(link) = ancestor {
if link.key == key {
return None;
}
ancestor = link.parent.as_deref();
}
Some(Self(Some(Arc::new(AncestorLink {
key,
parent: self.0.clone(),
}))))
}
}
#[cfg(unix)]
const CYCLE_KEY_OPERATION: &str = "metadata";
#[cfg(not(unix))]
const CYCLE_KEY_OPERATION: &str = "canonicalize";
const IGNORE_FILE_OPERATION: &str = "read_ignore";
#[derive(Debug, Default)]
pub(crate) struct Listing {
entries: Vec<ListedEntry>,
#[cfg(all(feature = "native-macos", target_os = "macos"))]
native_directory: Option<Arc<macos_native::RetainedDirectory>>,
len: usize,
deferred_errors: Vec<DeferredListingError>,
}
#[derive(Debug)]
pub(crate) struct DeferredListingError {
path: PathBuf,
source: DeferredIoError,
}
#[derive(Debug)]
struct DeferredIoError {
kind: std::io::ErrorKind,
message: String,
}
impl From<std::io::Error> for DeferredIoError {
fn from(source: std::io::Error) -> Self {
Self {
kind: source.kind(),
message: source.to_string(),
}
}
}
impl DeferredIoError {
fn into_io_error(self) -> std::io::Error {
std::io::Error::new(self.kind, self.message)
}
}
#[derive(Debug, Default, Clone)]
pub(crate) struct ListedEntry {
name: OsString,
is_dir: bool,
is_symlink: bool,
}
impl ListedEntry {
pub(crate) fn name(&self) -> &OsStr {
&self.name
}
pub(crate) const fn is_dir(&self) -> bool {
self.is_dir
}
pub(crate) const fn is_symlink(&self) -> bool {
self.is_symlink
}
}
impl Listing {
pub(crate) fn clear(&mut self) {
self.len = 0;
#[cfg(all(feature = "native-macos", target_os = "macos"))]
{
self.native_directory = None;
}
self.deferred_errors.clear();
}
pub(crate) fn push(&mut self, name: &OsStr, is_dir: bool, is_symlink: bool) {
if self.len == self.entries.len() {
self.entries.push(ListedEntry::default());
}
let entry = &mut self.entries[self.len];
entry.name.clear();
entry.name.push(name);
entry.is_dir = is_dir;
entry.is_symlink = is_symlink;
self.len += 1;
}
pub(crate) fn entries(&self) -> &[ListedEntry] {
&self.entries[..self.len]
}
#[cfg(all(feature = "native-macos", target_os = "macos"))]
pub(crate) fn remove_entry(&mut self, index: usize) {
debug_assert!(index < self.len);
let entry = self.entries.remove(index);
self.len -= 1;
self.entries.insert(self.len, entry);
}
pub(crate) fn defer_error(&mut self, path: PathBuf, source: std::io::Error) {
self.deferred_errors.push(DeferredListingError {
path,
source: source.into(),
});
}
pub(crate) fn take_deferred_error(&mut self) -> Option<DeferredListingError> {
(!self.deferred_errors.is_empty()).then(|| self.deferred_errors.remove(0))
}
pub(crate) fn contains(&self, name: &str) -> bool {
self.entries().iter().any(|entry| entry.name == *name)
}
pub(crate) fn contains_git_ignore_name(&self, name: &str) -> bool {
self.contains(name)
|| self.entries().iter().any(|entry| {
entry
.name
.as_encoded_bytes()
.eq_ignore_ascii_case(name.as_bytes())
})
}
}
#[cfg_attr(
any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
),
allow(dead_code)
)]
struct StdBackend;
impl DirectoryBackend for StdBackend {
fn read_directory(
&self,
path: &Path,
_follow_symlinks: bool,
_refuse_final_symlink: bool,
listing: &mut Listing,
) -> std::io::Result<()> {
read_portable_directory(path, path, listing)
}
}
#[cfg_attr(
any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
),
allow(dead_code)
)]
fn read_portable_directory(
directory: &Path,
reported_path: &Path,
listing: &mut Listing,
) -> std::io::Result<()> {
listing.clear();
for entry in fs::read_dir(directory)? {
let entry = match entry {
Ok(entry) => entry,
Err(error) => {
defer_entry_stat_error(listing, reported_path.to_path_buf(), error)?;
continue;
}
};
let name = entry.file_name();
let file_type = match entry.file_type() {
Ok(file_type) => file_type,
Err(error) => {
defer_entry_stat_error(listing, reported_path.join(&name), error)?;
continue;
}
};
listing.push(&name, file_type.is_dir(), file_type.is_symlink());
}
Ok(())
}
pub(crate) fn defer_entry_stat_error(
listing: &mut Listing,
path: PathBuf,
error: std::io::Error,
) -> std::io::Result<()> {
match error.kind() {
std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory => Ok(()),
std::io::ErrorKind::PermissionDenied => {
listing.defer_error(path, error);
Ok(())
}
_ => Err(error),
}
}
struct SystemBackend;
#[cfg(any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
))]
#[cfg_attr(
any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
),
allow(dead_code)
)]
fn read_native_or_portable(
listing: &mut Listing,
native: impl FnOnce(&mut Listing) -> std::io::Result<()>,
fallback: impl FnOnce(&mut Listing) -> std::io::Result<()>,
) -> std::io::Result<()> {
match native(listing) {
Ok(()) => Ok(()),
Err(error) if error.kind() == std::io::ErrorKind::Unsupported => fallback(listing),
Err(error) => Err(error),
}
}
impl DirectoryBackend for SystemBackend {
fn read_directory(
&self,
path: &Path,
follow_symlinks: bool,
refuse_final_symlink: bool,
listing: &mut Listing,
) -> std::io::Result<()> {
#[cfg(all(feature = "native-macos", target_os = "macos"))]
{
let _ = follow_symlinks;
macos_native::read_directory(path, None, refuse_final_symlink, listing)
.map_err(macos_native::NativeDirectoryReadError::into_io_error)
}
#[cfg(all(
feature = "native-linux",
target_os = "linux",
not(all(feature = "native-macos", target_os = "macos"))
))]
{
let _ = follow_symlinks;
linux_native::read_directory(path, refuse_final_symlink, listing)
}
#[cfg(not(any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
)))]
StdBackend.read_directory(path, follow_symlinks, refuse_final_symlink, listing)
}
fn read_scheduled_directory(
&self,
path: &Path,
_open: &DirectoryOpen,
follow_symlinks: bool,
refuse_final_symlink: bool,
listing: &mut Listing,
) -> std::io::Result<()> {
#[cfg(all(feature = "native-macos", target_os = "macos"))]
{
let _ = follow_symlinks;
macos_native::read_directory(
path,
_open.relative.as_ref(),
refuse_final_symlink,
listing,
)
.map_err(macos_native::NativeDirectoryReadError::into_io_error)
}
#[cfg(not(all(feature = "native-macos", target_os = "macos")))]
self.read_directory(path, follow_symlinks, refuse_final_symlink, listing)
}
fn child_directory_open(&self, listing: &Listing, name: &OsStr) -> DirectoryOpen {
#[cfg(all(feature = "native-macos", target_os = "macos"))]
{
DirectoryOpen {
relative: listing.native_directory.as_ref().map(|directory| {
macos_native::RelativeDirectoryOpen {
parent: Arc::clone(directory),
name: name.to_os_string(),
}
}),
}
}
#[cfg(not(all(feature = "native-macos", target_os = "macos")))]
{
let _ = (listing, name);
DirectoryOpen::default()
}
}
}
#[derive(Debug)]
pub struct WalkStream {
walker: Walker,
pending_directories: Vec<DirectoryTask>,
listing: Listing,
glob_bytes: Vec<u8>,
next_entry: usize,
path: PathBuf,
directory: PathBuf,
ancestors: AncestorChain,
ignores: IgnoreScope,
depth: usize,
root: usize,
pending_errors: Vec<PendingWalkError>,
cancelled: bool,
stopped: bool,
}
#[derive(Debug)]
struct PendingWalkError {
operation: &'static str,
path: PathBuf,
kind: std::io::ErrorKind,
message: String,
}
impl PendingWalkError {
fn from_ignore(error: IgnoreReadError) -> Self {
let (path, source) = error.into_parts();
Self {
operation: IGNORE_FILE_OPERATION,
path,
kind: source.kind(),
message: source.to_string(),
}
}
fn into_walk_error(self) -> WalkError {
WalkError::new(
self.operation,
self.path,
std::io::Error::new(self.kind, self.message),
)
}
}
impl WalkStream {
#[must_use]
pub const fn was_cancelled(&self) -> bool {
self.cancelled
}
fn check_cancellation(&mut self) -> bool {
self.cancelled |= self
.walker
.cancellation
.as_ref()
.is_some_and(CancellationToken::is_cancelled);
self.cancelled
}
fn error(
&mut self,
operation: &'static str,
path: PathBuf,
source: std::io::Error,
is_root: bool,
) -> Option<Result<WalkEntry, WalkError>> {
let error = WalkError::new(operation, path, source);
match self.walker.error_policy {
ErrorPolicy::Abort => {
self.stopped = true;
Some(Err(error))
}
ErrorPolicy::Skip if !is_root => None,
ErrorPolicy::Skip | ErrorPolicy::Collect => Some(Err(error)),
}
}
fn queue_ignore_errors(
&mut self,
errors: Vec<IgnoreReadError>,
) -> Option<Result<WalkEntry, WalkError>> {
match self.walker.error_policy {
ErrorPolicy::Skip => None,
ErrorPolicy::Abort => errors.into_iter().next().map(|error| {
self.stopped = true;
let (path, source) = error.into_parts();
Err(WalkError::new(IGNORE_FILE_OPERATION, path, source))
}),
ErrorPolicy::Collect => {
self.pending_errors
.extend(errors.into_iter().rev().map(PendingWalkError::from_ignore));
self.pending_errors
.pop()
.map(PendingWalkError::into_walk_error)
.map(Err)
}
}
}
fn prepare_directory(&mut self, task: DirectoryTask) -> Option<Result<WalkEntry, WalkError>> {
let DirectoryTask {
path,
open,
depth,
root,
ancestors,
ignores,
mut ignore_errors,
} = task;
let ancestors = if self.walker.options.follow_symlinks {
match SystemBackend.cycle_key(&path) {
Ok(key) => ancestors.enter(key)?,
Err(source) => return self.error(CYCLE_KEY_OPERATION, path, source, depth == 0),
}
} else {
ancestors
};
match SystemBackend.read_scheduled_directory(
&path,
&open,
self.walker.options.follow_symlinks,
!self.walker.options.follow_symlinks && depth > 0,
&mut self.listing,
) {
Ok(()) => {
let (ignores, mut entered_errors) =
ignores.enter(&self.walker, &SystemBackend, &path, &self.listing);
self.ignores = ignores;
ignore_errors.append(&mut entered_errors);
self.depth = depth;
self.root = root;
self.ancestors = ancestors;
self.next_entry = 0;
self.directory = path;
reset_to_directory(&mut self.path, &self.directory);
self.queue_ignore_errors(ignore_errors)
}
Err(source) => {
self.listing.clear();
self.next_entry = 0;
self.error("read_dir", path, source, depth == 0)
}
}
}
fn process_entry(&mut self, index: usize) -> Option<Result<WalkEntry, WalkError>> {
self.path.push(self.listing.entries()[index].name());
let action = classify_entry(
&self.walker,
&SystemBackend,
&self.path,
&self.listing.entries()[index],
&self.ignores,
self.depth,
TraversalContext {
root: self.root,
ancestors: &self.ancestors,
listing: &self.listing,
glob_bytes_scratch: &mut self.glob_bytes,
},
);
let emitted = match action {
EntryAction::Skip => None,
EntryAction::Descend(task) => {
self.pending_directories.push(task);
None
}
EntryAction::Emit(entry) => Some(Ok(entry.with_path(self.path.clone()))),
EntryAction::DescendAndEmit(entry, task) => {
let entry = entry.with_path(self.path.clone());
self.pending_directories.push(task);
Some(Ok(entry))
}
EntryAction::Failed { failure, descend } => {
if let Some(task) = descend {
self.pending_directories.push(task);
}
self.error(failure.operation, failure.path, failure.source, false)
}
};
reset_to_directory(&mut self.path, &self.directory);
emitted
}
}
impl Iterator for WalkStream {
type Item = Result<WalkEntry, WalkError>;
fn next(&mut self) -> Option<Self::Item> {
while !self.stopped {
if self.check_cancellation() {
self.stopped = true;
return None;
}
if let Some(error) = self.pending_errors.pop() {
return Some(Err(error.into_walk_error()));
}
if self.next_entry < self.listing.entries().len() {
let index = self.next_entry;
self.next_entry += 1;
if let Some(result) = self.process_entry(index) {
return Some(result);
}
continue;
}
if let Some(error) = self.listing.take_deferred_error() {
if let Some(result) =
self.error("read_dir", error.path, error.source.into_io_error(), false)
{
return Some(result);
}
continue;
}
let task = self.pending_directories.pop()?;
if let Some(result) = self.prepare_directory(task) {
return Some(result);
}
}
None
}
}
struct WalkState<'walker> {
walker: &'walker Walker,
visitor: EntryVisitor<'walker>,
entries: Vec<WalkEntry>,
errors: Vec<WalkError>,
scratch: Vec<DirectoryScratch>,
spare: PathBuf,
cancelled: bool,
}
#[derive(Default)]
struct DirectoryScratch {
listing: Listing,
path: PathBuf,
glob_bytes: Vec<u8>,
}
struct DirectoryFrame {
task: DirectoryTask,
ignores: IgnoreScope,
scratch: DirectoryScratch,
next_entry: usize,
}
enum SerialTask {
Directory(DirectoryTask),
Resume(DirectoryFrame),
Emit(WalkEntry),
}
const CANCELLATION_STRIDE: usize = 64;
fn reset_to_directory(path: &mut PathBuf, directory: &Path) {
path.clear();
path.as_mut_os_string().push(directory.as_os_str());
}
fn own_path(spare: &mut PathBuf, path: &Path) -> PathBuf {
let mut owned = std::mem::take(spare);
if owned.capacity() == 0 {
return path.to_path_buf();
}
owned.clear();
owned.as_mut_os_string().push(path.as_os_str());
owned
}
impl<'walker> WalkState<'walker> {
fn new(walker: &'walker Walker, visitor: EntryVisitor<'walker>) -> Self {
Self {
walker,
visitor,
entries: Vec::new(),
errors: Vec::new(),
scratch: Vec::new(),
spare: PathBuf::new(),
cancelled: false,
}
}
fn emit(&mut self, path: &Path, emitted: EmittedEntry) {
let entry = emitted.with_path(own_path(&mut self.spare, path));
self.emit_owned(entry);
}
fn emit_owned(&mut self, entry: WalkEntry) {
match (self.visitor)(&entry) {
Verdict::Keep => self.entries.push(entry),
Verdict::Skip => self.spare = entry.path,
Verdict::Stop => {
self.spare = entry.path;
self.cancelled = true;
}
}
}
fn walk_directory(
&mut self,
backend: &impl DirectoryBackend,
task: DirectoryTask,
) -> Result<(), WalkError> {
let mut pending = vec![SerialTask::Directory(task)];
while let Some(task) = pending.pop() {
match task {
SerialTask::Directory(task) => self.start_directory(backend, task, &mut pending)?,
SerialTask::Resume(frame) => self.resume_directory(backend, frame, &mut pending)?,
SerialTask::Emit(entry) => self.emit_owned(entry),
}
}
Ok(())
}
fn start_directory(
&mut self,
backend: &impl DirectoryBackend,
mut task: DirectoryTask,
pending: &mut Vec<SerialTask>,
) -> Result<(), WalkError> {
if self.check_cancellation() {
return Ok(());
}
let is_root = task.depth == 0;
for error in std::mem::take(&mut task.ignore_errors) {
let (path, source) = error.into_parts();
self.handle_error(IGNORE_FILE_OPERATION, path, source, false)?;
}
if self.walker.options.follow_symlinks {
let Some(ancestors) =
self.enter_directory(backend, &task.ancestors, &task.path, is_root)?
else {
return Ok(());
};
task.ancestors = ancestors;
}
let mut scratch = self.scratch.pop().unwrap_or_default();
let path = task.path.as_path();
let depth = task.depth;
let is_root = depth == 0;
if let Err(source) = backend.read_scheduled_directory(
path,
&task.open,
self.walker.options.follow_symlinks,
!self.walker.options.follow_symlinks && depth > 0,
&mut scratch.listing,
) {
let result = self.handle_error("read_dir", path.to_path_buf(), source, is_root);
scratch.listing.clear();
self.scratch.push(scratch);
return result;
}
let (ignores, ignore_errors) =
std::mem::take(&mut task.ignores).enter(self.walker, backend, path, &scratch.listing);
for error in ignore_errors {
let (path, source) = error.into_parts();
if let Err(error) = self.handle_error(IGNORE_FILE_OPERATION, path, source, false) {
scratch.listing.clear();
self.scratch.push(scratch);
return Err(error);
}
}
scratch.path.clear();
scratch.path.push(path);
pending.push(SerialTask::Resume(DirectoryFrame {
task,
ignores,
scratch,
next_entry: 0,
}));
Ok(())
}
fn resume_directory(
&mut self,
backend: &impl DirectoryBackend,
mut frame: DirectoryFrame,
pending: &mut Vec<SerialTask>,
) -> Result<(), WalkError> {
let path = frame.task.path.as_path();
let depth = frame.task.depth;
while frame.next_entry < frame.scratch.listing.entries().len() {
if self.cancelled
|| (frame.next_entry.is_multiple_of(CANCELLATION_STRIDE)
&& self.check_cancellation())
{
return self.finish_directory(frame);
}
let index = frame.next_entry;
frame.next_entry += 1;
frame
.scratch
.path
.push(frame.scratch.listing.entries()[index].name());
let action = classify_entry(
self.walker,
backend,
&frame.scratch.path,
&frame.scratch.listing.entries()[index],
&frame.ignores,
depth,
TraversalContext {
root: frame.task.root,
ancestors: &frame.task.ancestors,
listing: &frame.scratch.listing,
glob_bytes_scratch: &mut frame.scratch.glob_bytes,
},
);
match action {
EntryAction::Skip => reset_to_directory(&mut frame.scratch.path, path),
EntryAction::Emit(entry) => {
self.emit(&frame.scratch.path, entry);
reset_to_directory(&mut frame.scratch.path, path);
}
EntryAction::Descend(task) => {
reset_to_directory(&mut frame.scratch.path, path);
pending.push(SerialTask::Resume(frame));
pending.push(SerialTask::Directory(task));
return Ok(());
}
EntryAction::DescendAndEmit(entry, task) => {
let entry = entry.with_path(own_path(&mut self.spare, &frame.scratch.path));
reset_to_directory(&mut frame.scratch.path, path);
pending.push(SerialTask::Emit(entry));
pending.push(SerialTask::Resume(frame));
pending.push(SerialTask::Directory(task));
return Ok(());
}
EntryAction::Failed { failure, descend } => {
reset_to_directory(&mut frame.scratch.path, path);
if let Err(error) =
self.handle_error(failure.operation, failure.path, failure.source, false)
{
self.finish_directory(frame)?;
return Err(error);
}
if let Some(task) = descend {
pending.push(SerialTask::Resume(frame));
pending.push(SerialTask::Directory(task));
return Ok(());
}
}
}
}
while let Some(error) = frame.scratch.listing.take_deferred_error() {
if let Err(error) =
self.handle_error("read_dir", error.path, error.source.into_io_error(), false)
{
self.finish_directory(frame)?;
return Err(error);
}
}
self.finish_directory(frame)
}
fn finish_directory(&mut self, mut frame: DirectoryFrame) -> Result<(), WalkError> {
frame.scratch.listing.clear();
self.scratch.push(frame.scratch);
Ok(())
}
fn enter_directory(
&mut self,
backend: &impl DirectoryBackend,
ancestors: &AncestorChain,
directory: &Path,
is_root: bool,
) -> Result<Option<AncestorChain>, WalkError> {
match backend.cycle_key(directory) {
Ok(key) => Ok(ancestors.enter(key)),
Err(source) => {
self.handle_error(
CYCLE_KEY_OPERATION,
directory.to_path_buf(),
source,
is_root,
)?;
Ok(None)
}
}
}
fn handle_error(
&mut self,
operation: &'static str,
path: PathBuf,
source: std::io::Error,
is_root: bool,
) -> Result<(), WalkError> {
let error = WalkError::new(operation, path, source);
match self.walker.error_policy {
ErrorPolicy::Abort => Err(error),
ErrorPolicy::Skip if !is_root => Ok(()),
ErrorPolicy::Skip | ErrorPolicy::Collect => {
self.errors.push(error);
Ok(())
}
}
}
fn check_cancellation(&mut self) -> bool {
self.cancelled |= self
.walker
.cancellation
.as_ref()
.is_some_and(CancellationToken::is_cancelled);
self.cancelled
}
}
fn should_skip_git_directory(walker: &Walker, name: &OsStr) -> bool {
walker.respect_git_ignore && !walker.options.keep_git_dir && name == ".git"
}
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
#[cfg(test)]
mod tests {
use std::{
cell::RefCell,
collections::{HashMap, HashSet},
fs,
path::{Path, PathBuf},
process::Command,
sync::{
Mutex,
atomic::{AtomicUsize, Ordering},
},
time::{SystemTime, UNIX_EPOCH},
};
use super::{
CancellationToken, ErrorPolicy, Pattern, PatternOptions, TraversalPattern, Verdict,
WalkEntry, WalkEntryKind, WalkOptions, WalkStream, Walker, WildcardMode, glob_path_bytes,
literal_extension, literal_pattern_root, traversal_pattern_options,
};
static NEXT_FIXTURE: AtomicUsize = AtomicUsize::new(0);
const HOSTILE_GIT_CONFIG_FIXTURE: &str = "FERRALK_HOSTILE_GIT_CONFIG_FIXTURE";
#[test]
fn walk_stream_keeps_its_unwind_auto_traits() {
fn assert_unwind_safe<T: std::panic::UnwindSafe + std::panic::RefUnwindSafe>() {}
assert_unwind_safe::<WalkStream>();
}
#[test]
fn worker_budget_is_bounded_at_both_ends() {
assert_eq!(Walker::new(".").threads(0).threads, 1);
assert_eq!(
Walker::new(".").threads(usize::MAX).threads,
super::MAX_WORKERS
);
}
fn traversal_pattern(pattern: &[u8]) -> TraversalPattern {
TraversalPattern::compile(pattern, traversal_pattern_options(false))
.expect("valid walker pattern")
}
struct Fixture {
root: PathBuf,
}
impl Fixture {
fn new() -> Self {
let unique = format!(
"ferralk-test-{}-{}",
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("system clock is after unix epoch")
.as_nanos()
+ NEXT_FIXTURE.fetch_add(1, Ordering::Relaxed) as u128
);
let root = std::env::temp_dir().join(unique);
fs::create_dir_all(&root).expect("create fixture root");
Self { root }
}
fn write(&self, relative: impl AsRef<Path>) {
let path = self.root.join(relative);
fs::create_dir_all(path.parent().expect("fixture file has parent"))
.expect("create fixture parent");
fs::write(path, b"fixture").expect("write fixture file");
}
}
impl Fixture {
fn absolute(&self, suffix: &str) -> String {
let root = String::from_utf8(glob_path_bytes(&self.root).into_owned())
.expect("the temporary directory is UTF-8 on a test host");
format!("{root}{suffix}")
}
}
impl Drop for Fixture {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.root);
}
}
#[derive(Debug, PartialEq, Eq)]
enum FrontendOutcome {
Completed {
entries: Vec<PathBuf>,
errors: Vec<(&'static str, PathBuf)>,
},
Aborted(&'static str),
}
fn error_multiset(errors: &[super::WalkError], root: &Path) -> Vec<(&'static str, PathBuf)> {
let mut errors = errors
.iter()
.map(|error| {
(
error.operation(),
error
.path()
.strip_prefix(root)
.unwrap_or(error.path())
.to_path_buf(),
)
})
.collect::<Vec<_>>();
errors.sort_unstable();
errors
}
fn collect_outcome(
result: Result<super::WalkResult, super::WalkError>,
root: &Path,
) -> FrontendOutcome {
match result {
Ok(result) => {
let mut entries = relative_paths(result.entries(), root);
entries.sort_unstable();
FrontendOutcome::Completed {
entries,
errors: error_multiset(result.errors(), root),
}
}
Err(error) => FrontendOutcome::Aborted(error.operation()),
}
}
fn stream_outcome(
stream: super::WalkStream,
root: &Path,
policy: ErrorPolicy,
) -> FrontendOutcome {
let mut entries = Vec::new();
let mut errors = Vec::new();
for item in stream {
match item {
Ok(entry) => entries.push(
entry
.path()
.strip_prefix(root)
.expect("entry is rooted in fixture")
.to_path_buf(),
),
Err(error) => {
if policy == ErrorPolicy::Abort {
return FrontendOutcome::Aborted(error.operation());
}
errors.push(error);
}
}
}
entries.sort_unstable();
FrontendOutcome::Completed {
entries,
errors: error_multiset(&errors, root),
}
}
fn assert_frontends_agree(label: &str, root: &Path, build: impl Fn() -> Walker) {
for policy in [ErrorPolicy::Collect, ErrorPolicy::Skip, ErrorPolicy::Abort] {
let serial = collect_outcome(build().threads(1).error_policy(policy).collect(), root);
let parallel = collect_outcome(build().threads(4).error_policy(policy).collect(), root);
let streamed = stream_outcome(build().error_policy(policy).stream(), root, policy);
assert_eq!(
parallel, serial,
"{label}: parallel and serial disagree under {policy:?}"
);
assert_eq!(
streamed, serial,
"{label}: stream and serial disagree under {policy:?}"
);
for threads in [1, 4] {
let visited = collect_outcome(
build()
.threads(threads)
.error_policy(policy)
.visit(|_| Verdict::Keep),
root,
);
assert_eq!(
visited, serial,
"{label}: keep-everything visit on {threads} threads disagrees under {policy:?}"
);
let keeps = |entry: &WalkEntry| {
entry
.path()
.to_string_lossy()
.bytes()
.filter(|byte| *byte == b'a')
.count()
% 2
== 0
};
let filtered = collect_outcome(
build()
.threads(threads)
.error_policy(policy)
.visit(|entry| {
if keeps(entry) {
Verdict::Keep
} else {
Verdict::Skip
}
}),
root,
);
let expected = match &serial {
FrontendOutcome::Aborted(operation) => FrontendOutcome::Aborted(operation),
FrontendOutcome::Completed { errors, .. } => {
let mut entries = collect_outcome(
build().threads(threads).error_policy(policy).collect(),
root,
);
if let FrontendOutcome::Completed {
entries: collected, ..
} = &mut entries
{
collected.retain(|path| keeps_relative(path, root, keeps));
}
match entries {
FrontendOutcome::Completed { entries, .. } => {
FrontendOutcome::Completed {
entries,
errors: errors.clone(),
}
}
aborted => aborted,
}
}
};
assert_eq!(
filtered, expected,
"{label}: filtering visit on {threads} threads disagrees under {policy:?}"
);
}
}
}
fn keeps_relative(relative: &Path, root: &Path, keeps: impl Fn(&WalkEntry) -> bool) -> bool {
let entry = WalkEntry {
path: root.join(relative),
root: std::sync::Arc::from(root),
is_dir: false,
is_symlink: false,
depth: 0,
metadata: None,
};
keeps(&entry)
}
fn directory_task(
walker: &Walker,
backend: &impl super::DirectoryBackend,
path: PathBuf,
) -> super::DirectoryTask {
let (ignores, ignore_errors) = super::IgnoreScope::for_root(walker, backend, &path);
super::DirectoryTask {
path: path.clone(),
open: super::DirectoryOpen::default(),
depth: 0,
root: 0,
ancestors: super::AncestorChain::default(),
ignores,
ignore_errors,
}
}
#[test]
fn serial_collect_handles_a_deep_directory_chain_without_recursion() {
const DEPTH: usize = 4_096;
struct DeepChainBackend {
root: PathBuf,
}
impl super::DirectoryBackend for DeepChainBackend {
fn read_directory(
&self,
path: &Path,
_follow_symlinks: bool,
_refuse_final_symlink: bool,
listing: &mut super::Listing,
) -> std::io::Result<()> {
listing.clear();
let depth = path
.strip_prefix(&self.root)
.expect("walk only reads descendants of its root")
.components()
.count();
if depth < DEPTH {
listing.push("child".as_ref(), true, false);
}
Ok(())
}
}
let backend = DeepChainBackend {
root: PathBuf::from("/serial-deep-chain"),
};
let result = Walker::new(&backend.root)
.threads(1)
.options(WalkOptions::default().files_only(true))
.collect_with(&backend)
.expect("the serial walker does not consume one call frame per directory");
assert!(
result.entries().is_empty(),
"the mock tree has only directories"
);
assert!(result.errors().is_empty());
}
#[test]
fn stream_discards_partial_listing_after_a_directory_read_error() {
let fixture = Fixture::new();
let missing = fixture.root.join("missing");
let mut stream = Walker::new(&missing)
.error_policy(ErrorPolicy::Collect)
.stream();
stream
.listing
.push(std::ffi::OsStr::new("must-not-leak"), false, false);
let task = stream
.pending_directories
.pop()
.expect("the root is queued for reading");
let error = stream
.prepare_directory(task)
.expect("the failed root produces an error")
.expect_err("a missing root cannot yield an entry");
assert_eq!(error.operation(), "read_dir");
assert!(stream.listing.entries().is_empty());
assert!(stream.next().is_none(), "partial entries must not leak");
}
#[derive(Default)]
struct CountingBackend {
ignore_reads: std::sync::Mutex<HashMap<PathBuf, usize>>,
}
impl CountingBackend {
fn ignore_reads(&self) -> Vec<(PathBuf, usize)> {
let mut reads = self
.ignore_reads
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.iter()
.map(|(path, count)| (path.clone(), *count))
.collect::<Vec<_>>();
reads.sort_unstable();
reads
}
}
impl super::DirectoryBackend for CountingBackend {
fn read_directory(
&self,
path: &Path,
follow_symlinks: bool,
refuse_final_symlink: bool,
listing: &mut super::Listing,
) -> std::io::Result<()> {
super::StdBackend.read_directory(path, follow_symlinks, refuse_final_symlink, listing)
}
fn read_ignore_file(&self, path: &Path) -> std::io::Result<Vec<u8>> {
*self
.ignore_reads
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.entry(path.to_path_buf())
.or_default() += 1;
fs::read(path)
}
}
fn relative_paths(entries: &[WalkEntry], root: &Path) -> Vec<PathBuf> {
entries
.iter()
.map(|entry| {
entry
.path()
.strip_prefix(root)
.expect("entry is rooted in fixture")
.to_path_buf()
})
.collect()
}
fn relative_paths_and_depths(entries: &[WalkEntry], root: &Path) -> Vec<(PathBuf, usize)> {
entries
.iter()
.map(|entry| {
(
entry
.path()
.strip_prefix(root)
.expect("entry is rooted in fixture")
.to_path_buf(),
entry.depth(),
)
})
.collect()
}
fn rooted_relative_paths(entries: &[WalkEntry]) -> Vec<(PathBuf, PathBuf)> {
let mut paths = entries
.iter()
.map(|entry| {
(
entry.root().to_path_buf(),
entry
.path()
.strip_prefix(entry.root())
.expect("entry is rooted in its declared root")
.to_path_buf(),
)
})
.collect::<Vec<_>>();
paths.sort_unstable();
paths
}
type RootFilterSources = (Vec<Vec<u8>>, Vec<Vec<u8>>);
fn configured_filter_sources(walker: &Walker) -> Vec<RootFilterSources> {
walker
.roots
.iter()
.map(|root| {
(
root.includes
.iter()
.map(|pattern| pattern.source.clone())
.collect(),
root.excludes
.iter()
.map(|pattern| pattern.source.clone())
.collect(),
)
})
.collect()
}
#[test]
fn include_exclude_and_sort_are_applied_to_relative_paths() {
let fixture = Fixture::new();
fixture.write("src/main.rs");
fixture.write("src/lib.txt");
fixture.write("target/generated.rs");
let result = Walker::new(&fixture.root)
.include("**/*.rs")
.expect("valid include")
.exclude("**/target/**")
.expect("valid exclude")
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds");
assert_eq!(
relative_paths(result.entries(), &fixture.root),
vec![PathBuf::from("src/main.rs")]
);
assert!(result.errors().is_empty());
}
#[test]
fn borrowed_pattern_lists_keep_the_builder_and_filter_every_frontend() {
let fixture = Fixture::new();
let alpha = fixture.root.join("alpha");
let beta = fixture.root.join("beta");
for root in ["alpha", "beta"] {
fixture.write(format!("{root}/src/keep.rs"));
fixture.write(format!("{root}/src/remove.rs"));
fixture.write(format!("{root}/src/ignore.txt"));
fixture.write(format!("{root}/.hidden.rs"));
}
let mut walker = Walker::new(&alpha)
.add_root(&beta)
.expect("valid second root")
.match_hidden(true)
.wildcard_mode(WildcardMode::SeparatorCrossing)
.options(WalkOptions::default().files_only(true).sort(true));
walker
.try_include("*.rs")
.expect("first supplied include is valid");
let before_bad_include = walker.clone();
assert!(walker.try_include("[a").is_err());
assert_eq!(walker.include_sources, before_bad_include.include_sources);
assert_eq!(walker.exclude_sources, before_bad_include.exclude_sources);
assert_eq!(
configured_filter_sources(&walker),
configured_filter_sources(&before_bad_include),
"a rejected include must not update any root"
);
walker
.try_include("**/also.rs")
.expect("a later supplied include still composes");
walker
.try_exclude("**/remove.rs")
.expect("first supplied exclude is valid");
let before_bad_exclude = walker.clone();
assert!(walker.try_exclude("[a").is_err());
assert_eq!(walker.include_sources, before_bad_exclude.include_sources);
assert_eq!(walker.exclude_sources, before_bad_exclude.exclude_sources);
assert_eq!(
configured_filter_sources(&walker),
configured_filter_sources(&before_bad_exclude),
"a rejected exclude must not update any root"
);
walker
.try_exclude("**/generated/**")
.expect("a later supplied exclude still composes");
let expected = vec![
(alpha.clone(), PathBuf::from(".hidden.rs")),
(alpha.clone(), PathBuf::from("src/keep.rs")),
(beta.clone(), PathBuf::from(".hidden.rs")),
(beta.clone(), PathBuf::from("src/keep.rs")),
];
for threads in [1, 4] {
let collected = walker
.clone()
.threads(threads)
.collect()
.expect("collect succeeds");
assert_eq!(rooted_relative_paths(collected.entries()), expected);
let visited = walker
.clone()
.threads(threads)
.visit(|_| Verdict::Keep)
.expect("visit succeeds");
assert_eq!(rooted_relative_paths(visited.entries()), expected);
}
let streamed = walker
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream succeeds");
assert_eq!(rooted_relative_paths(&streamed), expected);
}
#[test]
fn source_walk_glob_patterns_filter_recursive_anchored_and_brace_paths() {
let fixture = Fixture::new();
fixture.write("src/a.rs");
fixture.write("src/b.txt");
fixture.write("src/deep/e.rs");
fixture.write("lib/c.rs");
fixture.write("docs/d.md");
fixture.write("top.rs");
let options = WalkOptions::default().sort(true);
let recursive = Walker::new(&fixture.root)
.include("**/*.rs")
.expect("valid recursive include")
.options(options)
.collect()
.expect("recursive walk succeeds");
assert_eq!(
relative_paths(recursive.entries(), &fixture.root),
vec![
PathBuf::from("lib/c.rs"),
PathBuf::from("src/a.rs"),
PathBuf::from("src/deep/e.rs"),
PathBuf::from("top.rs"),
]
);
let anchored = Walker::new(&fixture.root)
.include("src/**")
.expect("valid anchored include")
.options(options)
.collect()
.expect("anchored walk succeeds");
assert_eq!(
relative_paths(anchored.entries(), &fixture.root),
vec![
PathBuf::from("src"),
PathBuf::from("src/a.rs"),
PathBuf::from("src/b.txt"),
PathBuf::from("src/deep"),
PathBuf::from("src/deep/e.rs"),
]
);
let brace = Walker::new(&fixture.root)
.include("**/*.{md,txt}")
.expect("valid brace include")
.options(options)
.collect()
.expect("brace walk succeeds");
assert_eq!(
relative_paths(brace.entries(), &fixture.root),
vec![PathBuf::from("docs/d.md"), PathBuf::from("src/b.txt")]
);
}
#[test]
fn relative_patterns_that_name_no_candidate_are_rejected_with_guidance() {
let fixture = Fixture::new();
fixture.write("src/main.rs");
fixture.write("src/a.rs");
fixture.write("src/].rs");
#[cfg(not(windows))]
{
fixture.write("prefix../bar");
fixture.write("foo/..suffix/bar");
}
for mode in [
WildcardMode::ComponentScoped,
WildcardMode::SeparatorCrossing,
] {
for add in [
|walker: Walker| walker.include("."),
|walker: Walker| walker.include("./"),
|walker: Walker| walker.include("src/."),
|walker: Walker| walker.include("src/./main.rs"),
|walker: Walker| walker.include("././src/**"),
|walker: Walker| walker.include("src/../main.rs"),
|walker: Walker| walker.include("+(dead/../branch)"),
|walker: Walker| walker.exclude("."),
|walker: Walker| walker.exclude("./"),
|walker: Walker| walker.exclude("src/."),
|walker: Walker| walker.exclude("src/./main.rs"),
|walker: Walker| walker.exclude("././src/**"),
|walker: Walker| walker.exclude("src/../main.rs"),
|walker: Walker| walker.exclude("+(dead/../branch)"),
] {
let error = add(Walker::new(&fixture.root).wildcard_mode(mode))
.expect_err("unwalkable relative pattern is refused");
assert!(
error.message().contains("select")
|| error.message().contains("not normalized")
|| error.message().starts_with("`..`"),
"the rejection explains why the pattern cannot select: {error}"
);
}
}
for mode in [
WildcardMode::ComponentScoped,
WildcardMode::SeparatorCrossing,
] {
assert!(
Walker::new(".")
.wildcard_mode(mode)
.include("/bar")
.is_err(),
"include rejects an unselectable leading empty component under {mode:?}"
);
assert!(
Walker::new(".")
.wildcard_mode(mode)
.exclude("/bar")
.is_err(),
"exclude rejects an unselectable leading empty component under {mode:?}"
);
}
for pattern in [
"./src/**",
"...",
".hidden",
r"\.\.",
r"src\..\main.rs",
"[.]",
"{..,src}",
"@(..)",
"src/[[:alpha:]/../].rs",
"src/[]/../].rs",
"{dead/../branch,src/main.rs}",
"@(dead/../branch|src/main.rs)",
"src/[{],a}/../].rs",
"@(dead/{),x}/../branch|src/main.rs)",
"prefix@(../bar)",
"@(foo/..)suffix/bar",
] {
for mode in [
WildcardMode::ComponentScoped,
WildcardMode::SeparatorCrossing,
] {
Walker::new(&fixture.root)
.wildcard_mode(mode)
.include(pattern)
.unwrap_or_else(|error| panic!("{pattern} must stay matcher text: {error}"));
}
}
let brace_dot_components = "{.,..}";
assert!(
Walker::new(&fixture.root)
.include(brace_dot_components)
.is_err(),
"brace-expanded {brace_dot_components:?} is an unwalkable path spelling"
);
for pattern in ["@(.)", "@(..)/x", "?(.)/x"] {
Walker::new(&fixture.root)
.include(pattern)
.unwrap_or_else(|error| panic!("{pattern} remains opaque matcher text: {error}"));
}
for (pattern, matching_paths, component_scoped_paths) in [
(
"{dead/../branch,src/main.rs}",
&["src/main.rs"][..],
&["src/main.rs"][..],
),
(
"@(dead/../branch|src/main.rs)",
&["src/main.rs"][..],
&[][..],
),
(
"src/[[:alpha:]/../].rs",
&["src/a.rs"][..],
&["src/a.rs"][..],
),
("src/[]/../].rs", &["src/].rs"][..], &["src/].rs"][..]),
(
"src/[{],a}/../].rs",
&["src/a.rs", "src/].rs"][..],
&["src/a.rs", "src/].rs"][..],
),
(
"@(dead/{),x}/../branch|src/main.rs)",
&["src/main.rs"][..],
&[][..],
),
("prefix@(../bar)", &["prefix../bar"][..], &[][..]),
("@(foo/..)suffix/bar", &["foo/..suffix/bar"][..], &[][..]),
] {
let matcher = Pattern::compile(pattern, traversal_pattern_options(false))
.expect("the reviewer regression is valid matcher syntax");
for matching_path in matching_paths {
assert!(matcher.is_match(matching_path));
}
let matching_paths_on_disk =
if cfg!(windows) && matches!(pattern, "prefix@(../bar)" | "@(foo/..)suffix/bar") {
&[][..]
} else {
matching_paths
};
for mode in [
WildcardMode::ComponentScoped,
WildcardMode::SeparatorCrossing,
] {
Walker::new(&fixture.root)
.wildcard_mode(mode)
.exclude(pattern)
.expect("the viable group arm is accepted for excludes");
for threads in [1, 4] {
let result = Walker::new(&fixture.root)
.wildcard_mode(mode)
.threads(threads)
.include(pattern)
.expect("the viable group arm is accepted")
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("walk succeeds");
let mut expected = if mode == WildcardMode::ComponentScoped {
component_scoped_paths
} else {
matching_paths_on_disk
}
.iter()
.map(PathBuf::from)
.collect::<Vec<_>>();
expected.sort();
assert_eq!(
relative_paths(result.entries(), &fixture.root),
expected,
"{pattern} must keep its viable arm under {mode:?} on {threads} threads"
);
}
let streamed = Walker::new(&fixture.root)
.wildcard_mode(mode)
.include(pattern)
.expect("the viable group arm is accepted")
.options(WalkOptions::default().files_only(true))
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream succeeds");
let mut actual = relative_paths(&streamed, &fixture.root);
actual.sort();
let mut expected = if mode == WildcardMode::ComponentScoped {
component_scoped_paths
} else {
matching_paths_on_disk
}
.iter()
.map(PathBuf::from)
.collect::<Vec<_>>();
expected.sort();
assert_eq!(actual, expected);
}
}
for pattern in [
"src/../main.rs",
"src/{live,dead}/../main.rs",
"{dead/[}]]/../x,src/main.rs}",
"@(dead/[)]]/../x|src/main.rs)",
"{dead/../branch}",
"@(dead/../branch)",
"@(dead|src)/../main.rs",
"src/@(./a.rs)",
"@(./a.rs)",
"{./a.rs}",
"src/?(./a.rs)",
"src/*(./a.rs)",
"src/?(./a.rs)/bar",
"src/*(./a.rs)/bar",
"src/?()/bar",
"src/*()/bar",
"?()/bar",
"*()/bar",
"src//bar",
"src/{}/bar",
"src/{,./a.rs}/bar",
] {
assert!(
Walker::new(&fixture.root).include(pattern).is_err(),
"a parser-top-level `..` component stays invalid: {pattern}"
);
}
for mode in [
WildcardMode::ComponentScoped,
WildcardMode::SeparatorCrossing,
] {
for pattern in [
"{dead/[}]]/../x,src/main.rs}",
"@(dead/[)]]/../x|src/main.rs)",
"{dead/../branch}",
"@(dead/../branch)",
"@(dead|src)/../main.rs",
"src/@(./a.rs)",
"@(./a.rs)",
"{./a.rs}",
"src/?(./a.rs)",
"src/*(./a.rs)",
"src/?(./a.rs)/bar",
"src/*(./a.rs)/bar",
"src/?()/bar",
"src/*()/bar",
"?()/bar",
"*()/bar",
"src//bar",
"src/{}/bar",
"src/{,./a.rs}/bar",
] {
assert!(
Walker::new(&fixture.root)
.wildcard_mode(mode)
.include(pattern)
.is_err(),
"include rejects parser-top-level `..` under {mode:?}: {pattern}"
);
assert!(
Walker::new(&fixture.root)
.wildcard_mode(mode)
.exclude(pattern)
.is_err(),
"exclude rejects parser-top-level `..` under {mode:?}: {pattern}"
);
}
}
}
#[test]
fn relative_pattern_errors_keep_determinate_component_offsets() {
for (pattern, offset) in [
("src/../main.rs", 4),
("src/./main.rs", 4),
("src/.", 4),
("é/../main.rs", 3),
("src/@(./a.rs)", 6),
("{src}/../main.rs", 6),
("{a,b}/../main.rs", 6),
("src/{x}/.", 8),
] {
for mode in [
WildcardMode::ComponentScoped,
WildcardMode::SeparatorCrossing,
] {
for error in [
Walker::new(".")
.wildcard_mode(mode)
.include(pattern)
.expect_err("unwalkable include is refused"),
Walker::new(".")
.wildcard_mode(mode)
.exclude(pattern)
.expect_err("unwalkable exclude is refused"),
] {
assert_eq!(
error.offset(),
offset,
"{pattern} keeps its component offset under {mode:?}"
);
}
}
}
for pattern in [r"src/\./main.rs", r"src/\../main.rs", "src/@(..)suffix"] {
Walker::new(".")
.include(pattern)
.unwrap_or_else(|error| panic!("{pattern} remains matcher text: {error}"));
}
}
#[test]
fn source_rust_glob_patterns_replay_as_walker_filters() {
let fixture = Fixture::new();
for path in [
"xyz/x",
"xyz/y",
"xyz/z",
"aaa/tomato/tomato.txt",
"aaa/tomato/tomoto.txt",
"bbb/specials/[",
"bbb/specials/!",
"bbb/specials/]",
] {
fixture.write(path);
}
for path in ["aaa/apple", "aaa/orange"] {
fs::create_dir_all(fixture.root.join(path)).expect("create source fixture directory");
}
let paths_for = |pattern: &str| {
let result = Walker::new(&fixture.root)
.threads(1)
.include(pattern)
.expect("valid source pattern")
.options(WalkOptions::default().sort(true))
.collect()
.expect("source walk succeeds");
relative_paths(result.entries(), &fixture.root)
};
assert_eq!(paths_for("aaa"), vec![PathBuf::from("aaa")]);
assert_eq!(paths_for("./aaa"), vec![PathBuf::from("aaa")]);
assert_eq!(paths_for("aaa/"), vec![PathBuf::from("aaa")]);
assert!(paths_for("aaa/tomato/tomato.txt/").is_empty());
assert!(paths_for("nope").is_empty());
assert_eq!(paths_for("a*"), vec![PathBuf::from("aaa")]);
assert_eq!(paths_for("@(a*)"), vec![PathBuf::from("aaa")]);
assert_eq!(paths_for("a*a"), vec![PathBuf::from("aaa")]);
assert_eq!(paths_for("*a*a*a*"), vec![PathBuf::from("aaa")]);
assert_eq!(paths_for("aaa/apple"), vec![PathBuf::from("aaa/apple")]);
assert_eq!(paths_for("./*"), paths_for("*"));
assert_eq!(
paths_for("???/"),
vec![
PathBuf::from("aaa"),
PathBuf::from("bbb"),
PathBuf::from("xyz"),
]
);
assert_eq!(
paths_for("xyz/?"),
vec![
PathBuf::from("xyz/x"),
PathBuf::from("xyz/y"),
PathBuf::from("xyz/z"),
]
);
assert_eq!(
paths_for("aaa/tomato/tom?to.txt"),
vec![
PathBuf::from("aaa/tomato/tomato.txt"),
PathBuf::from("aaa/tomato/tomoto.txt"),
]
);
assert_eq!(
paths_for("aaa/*"),
vec![
PathBuf::from("aaa/apple"),
PathBuf::from("aaa/orange"),
PathBuf::from("aaa/tomato"),
]
);
let component_local = paths_for("aaa/*");
let parallel = Walker::new(&fixture.root)
.threads(4)
.include("aaa/*")
.expect("valid source pattern")
.options(WalkOptions::default().sort(true))
.collect()
.expect("parallel source walk succeeds");
assert_eq!(
relative_paths(parallel.entries(), &fixture.root),
component_local
);
let mut streamed = Walker::new(&fixture.root)
.include("aaa/*")
.expect("valid source pattern")
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream source walk succeeds");
streamed.sort_by(|left, right| left.path().cmp(right.path()));
assert_eq!(relative_paths(&streamed, &fixture.root), component_local);
let trailing_directory = paths_for("aaa/");
let trailing_parallel = Walker::new(&fixture.root)
.threads(4)
.include("aaa/")
.expect("valid trailing directory pattern")
.options(WalkOptions::default().sort(true))
.collect()
.expect("parallel trailing directory walk succeeds");
assert_eq!(
relative_paths(trailing_parallel.entries(), &fixture.root),
trailing_directory
);
let mut trailing_streamed = Walker::new(&fixture.root)
.include("aaa/")
.expect("valid trailing directory pattern")
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream trailing directory walk succeeds");
trailing_streamed.sort_by(|left, right| left.path().cmp(right.path()));
assert_eq!(
relative_paths(&trailing_streamed, &fixture.root),
trailing_directory
);
assert_eq!(
paths_for("*/*/*.txt"),
vec![
PathBuf::from("aaa/tomato/tomato.txt"),
PathBuf::from("aaa/tomato/tomoto.txt"),
]
);
assert_eq!(paths_for("aa[a]"), vec![PathBuf::from("aaa")]);
assert_eq!(paths_for("aa[!b]"), vec![PathBuf::from("aaa")]);
assert!(paths_for("aa[b]").is_empty());
assert_eq!(
paths_for("*/*/t[aob]m?to[.]t[!y]t"),
vec![
PathBuf::from("aaa/tomato/tomato.txt"),
PathBuf::from("aaa/tomato/tomoto.txt"),
]
);
assert_eq!(
paths_for("bbb/specials/[[]"),
vec![PathBuf::from("bbb/specials/["),]
);
assert_eq!(
paths_for("bbb/specials/[]]"),
vec![PathBuf::from("bbb/specials/]"),]
);
}
#[test]
fn source_walk_entry_depths_are_relative_component_counts() {
let fixture = Fixture::new();
fixture.write("a.txt");
fixture.write("src/b.txt");
fixture.write("src/sub/c.txt");
let options = WalkOptions::default().sort(true);
let expected = vec![
(PathBuf::from("a.txt"), 1),
(PathBuf::from("src"), 1),
(PathBuf::from("src/b.txt"), 2),
(PathBuf::from("src/sub"), 2),
(PathBuf::from("src/sub/c.txt"), 3),
];
let serial = Walker::new(&fixture.root)
.threads(1)
.options(options)
.collect()
.expect("serial walk succeeds");
let parallel = Walker::new(&fixture.root)
.threads(4)
.options(options)
.collect()
.expect("parallel walk succeeds");
let mut streamed = Walker::new(&fixture.root)
.options(options)
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream succeeds");
streamed.sort_by(|left, right| left.path.cmp(&right.path));
assert_eq!(
relative_paths_and_depths(serial.entries(), &fixture.root),
expected
);
assert_eq!(
relative_paths_and_depths(parallel.entries(), &fixture.root),
expected
);
assert_eq!(
relative_paths_and_depths(&streamed, &fixture.root),
expected
);
let a = serial
.entries()
.iter()
.find(|entry| entry.basename() == Some(std::ffi::OsStr::new("a.txt")))
.expect("a.txt is present");
assert!(!a.is_dir());
assert_eq!(a.kind(), WalkEntryKind::File);
assert_eq!(a.depth(), 1);
let src = serial
.entries()
.iter()
.find(|entry| entry.basename() == Some(std::ffi::OsStr::new("src")))
.expect("src is present");
assert!(src.is_dir());
assert_eq!(src.kind(), WalkEntryKind::Directory);
assert_eq!(src.depth(), 1);
let c = serial
.entries()
.iter()
.find(|entry| entry.basename() == Some(std::ffi::OsStr::new("c.txt")))
.expect("c.txt is present");
assert!(!c.is_dir());
assert_eq!(c.kind(), WalkEntryKind::File);
assert_eq!(c.depth(), 3);
}
#[test]
fn carried_depth_matches_the_component_count_it_replaced() {
let fixture = Fixture::new();
fixture.write("a.txt");
fixture.write("one/b.txt");
fixture.write("one/two/c.txt");
fixture.write("one/two/three/d.txt");
fixture.write("one/two/three/four/e.txt");
for threads in [1, 4] {
let walked = Walker::new(&fixture.root)
.threads(threads)
.collect()
.expect("walk succeeds");
assert!(!walked.entries().is_empty());
for entry in walked.entries() {
let counted = entry
.path()
.strip_prefix(&fixture.root)
.expect("entry is rooted in the fixture")
.components()
.count();
assert_eq!(
entry.depth(),
counted,
"{} on {threads} thread(s)",
entry.path().display()
);
}
}
}
#[test]
fn a_root_with_a_trailing_separator_walks_like_one_without() {
let fixture = Fixture::new();
fixture.write("src/main.rs");
fixture.write("src/nested/lib.rs");
let options = WalkOptions::default().sort(true);
let plain = Walker::new(&fixture.root)
.threads(1)
.include("src/**/*.rs")
.expect("valid include")
.options(options)
.collect()
.expect("walk succeeds");
let mut trailing_root = fixture.root.clone().into_os_string();
trailing_root.push(std::path::MAIN_SEPARATOR_STR);
let trailing = Walker::new(PathBuf::from(trailing_root))
.threads(1)
.include("src/**/*.rs")
.expect("valid include")
.options(options)
.collect()
.expect("walk succeeds");
let paths = |result: &super::WalkResult| {
result
.entries()
.iter()
.map(|entry| entry.path().to_path_buf())
.collect::<Vec<_>>()
};
assert_eq!(paths(&trailing), paths(&plain));
assert_eq!(plain.entries().len(), 2);
}
#[test]
fn entries_after_a_subdirectory_keep_their_own_paths() {
let fixture = Fixture::new();
for index in 0..4 {
fixture.write(format!("outer/dir-{index}/inner.txt"));
fixture.write(format!("outer/file-{index}.txt"));
}
let options = WalkOptions::default().sort(true).files_only(true);
for threads in [1, 4] {
let walked = Walker::new(&fixture.root)
.threads(threads)
.options(options)
.collect()
.expect("walk succeeds");
let mut relative = relative_paths(walked.entries(), &fixture.root);
relative.sort();
let mut expected = (0..4)
.flat_map(|index| {
[
PathBuf::from(format!("outer/dir-{index}/inner.txt")),
PathBuf::from(format!("outer/file-{index}.txt")),
]
})
.collect::<Vec<_>>();
expected.sort();
assert_eq!(relative, expected, "on {threads} thread(s)");
}
}
#[test]
fn a_short_directory_after_a_long_one_reports_only_its_own_entries() {
let fixture = Fixture::new();
for index in 0..40 {
fixture.write(format!("crowded/file-{index:02}.txt"));
}
fixture.write("sparse/only.txt");
let walked = Walker::new(&fixture.root)
.threads(1)
.include("sparse/**")
.expect("valid include")
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("walk succeeds");
assert_eq!(
relative_paths(walked.entries(), &fixture.root),
vec![PathBuf::from("sparse/only.txt")]
);
}
#[test]
fn directories_only_filters_results_without_pruning_descendants() {
let fixture = Fixture::new();
fixture.write("src/main.rs");
fixture.write("src/nested/lib.rs");
let options = WalkOptions::default().directories_only(true).sort(true);
let serial = Walker::new(&fixture.root)
.threads(1)
.options(options)
.collect()
.expect("serial walk succeeds");
let parallel = Walker::new(&fixture.root)
.threads(4)
.options(options)
.collect()
.expect("parallel walk succeeds");
let streamed = Walker::new(&fixture.root)
.options(options)
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream succeeds");
let expected = vec![PathBuf::from("src"), PathBuf::from("src/nested")];
assert_eq!(relative_paths(serial.entries(), &fixture.root), expected);
assert_eq!(relative_paths(parallel.entries(), &fixture.root), expected);
assert_eq!(relative_paths(&streamed, &fixture.root), expected);
assert!(streamed.iter().all(WalkEntry::is_dir));
}
#[test]
fn max_depth_keeps_boundary_entries_without_descending() {
let fixture = Fixture::new();
fixture.write("top.txt");
fixture.write("d1/mid.txt");
fixture.write("d1/d2/bottom.txt");
for (max_depth, expected) in [
(0, vec![]),
(1, vec![PathBuf::from("d1"), PathBuf::from("top.txt")]),
(
2,
vec![
PathBuf::from("d1"),
PathBuf::from("d1/d2"),
PathBuf::from("d1/mid.txt"),
PathBuf::from("top.txt"),
],
),
] {
let options = WalkOptions::default().max_depth(max_depth).sort(true);
let serial = Walker::new(&fixture.root)
.threads(1)
.options(options)
.collect()
.expect("serial walk succeeds");
let parallel = Walker::new(&fixture.root)
.threads(4)
.options(options)
.collect()
.expect("parallel walk succeeds");
let mut streamed = Walker::new(&fixture.root)
.options(options)
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream succeeds");
streamed.sort_by(|left, right| left.path.cmp(&right.path));
assert_eq!(relative_paths(serial.entries(), &fixture.root), expected);
assert_eq!(relative_paths(parallel.entries(), &fixture.root), expected);
assert_eq!(relative_paths(&streamed, &fixture.root), expected);
}
}
#[test]
fn parallel_collect_matches_the_serial_result_multiset() {
let fixture = Fixture::new();
fixture.write("wide/a.txt");
fixture.write("wide/b.txt");
fixture.write("deep/one/two/three/leaf.txt");
fixture.write("ignored.tmp");
fs::write(fixture.root.join(".gitignore"), b"*.tmp\n").expect("write gitignore");
let serial = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(1)
.options(WalkOptions::default().sort(true))
.collect()
.expect("serial walk succeeds");
let parallel = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(4)
.options(WalkOptions::default().sort(true))
.collect()
.expect("parallel walk succeeds");
assert_eq!(
relative_paths(parallel.entries(), &fixture.root),
relative_paths(serial.entries(), &fixture.root)
);
assert!(parallel.errors().is_empty());
assert!(serial.errors().is_empty());
}
#[test]
fn parallel_collect_stress_covers_empty_shallow_and_imbalanced_trees() {
let empty = Fixture::new();
assert!(
Walker::new(&empty.root)
.threads(8)
.collect()
.expect("empty parallel walk succeeds")
.entries()
.is_empty()
);
let fixture = Fixture::new();
fixture.write("shallow.txt");
for branch in 0..8 {
fixture.write(format!("wide/{branch}/leaf.txt"));
}
for depth in 0..20 {
fixture.write(format!("deep/{depth}/next/leaf.txt"));
}
let serial = Walker::new(&fixture.root)
.threads(1)
.options(WalkOptions::default().sort(true))
.collect()
.expect("serial walk succeeds");
let expected = relative_paths(serial.entries(), &fixture.root);
for _ in 0..32 {
let actual = Walker::new(&fixture.root)
.threads(8)
.options(WalkOptions::default().sort(true))
.collect()
.expect("parallel stress walk succeeds");
assert_eq!(relative_paths(actual.entries(), &fixture.root), expected);
assert!(actual.errors().is_empty());
}
}
#[test]
fn source_walk_metadata_preserves_portable_and_unix_fields() {
let fixture = Fixture::new();
fs::write(fixture.root.join("five.bin"), b"12345").expect("write metadata fixture");
let without_metadata = Walker::new(&fixture.root)
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds");
assert!(
without_metadata
.entries()
.iter()
.all(|entry| entry.metadata().is_none())
);
let with_metadata = Walker::new(&fixture.root)
.options(WalkOptions::default().sort(true).metadata(true))
.collect()
.expect("walk succeeds");
let metadata = with_metadata
.entries()
.iter()
.find(|entry| entry.path().ends_with("five.bin"))
.expect("fixture file is returned")
.metadata()
.expect("metadata is requested");
assert_eq!(metadata.len(), 5);
assert!(metadata.is_file());
assert!(metadata.modified().is_ok());
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
assert_ne!(metadata.ino(), 0);
assert_ne!(metadata.mode() & 0o400, 0);
}
}
#[test]
fn walk_entries_are_cloneable_with_their_metadata_snapshot() {
let fixture = Fixture::new();
fs::write(fixture.root.join("five.bin"), b"12345").expect("write clone fixture");
let result = Walker::new(&fixture.root)
.options(WalkOptions::default().sort(true).metadata(true))
.collect()
.expect("walk succeeds");
let entry = result
.entries()
.iter()
.find(|entry| entry.path().ends_with("five.bin"))
.expect("fixture file is returned");
let clone = entry.clone();
assert_eq!(clone.path(), entry.path());
assert_eq!(clone.root(), entry.root());
assert_eq!(clone.kind(), entry.kind());
assert_eq!(clone.depth(), entry.depth());
assert_eq!(clone.metadata().map(fs::Metadata::len), Some(5));
}
#[test]
fn root_gitignore_rules_and_negation_apply_to_collect_and_stream() {
let fixture = Fixture::new();
fixture.write("generated.tmp");
fixture.write("keep.tmp");
fixture.write("src/main.rs");
fixture.write("src/keep.tmp");
fixture.write("build/keep.txt");
fs::write(
fixture.root.join(".gitignore"),
b"*.tmp\n!keep.tmp\nbuild/\n",
)
.expect("write root gitignore");
fs::write(fixture.root.join("src/.gitignore"), b"!keep.tmp\n")
.expect("write nested gitignore");
fs::write(fixture.root.join("build/.gitignore"), b"!keep.txt\n")
.expect("write nested re-include");
let collected = Walker::new(&fixture.root)
.respect_git_ignore(true)
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds");
let collected_paths = relative_paths(collected.entries(), &fixture.root);
assert!(!collected_paths.contains(&PathBuf::from("generated.tmp")));
assert!(collected_paths.contains(&PathBuf::from("keep.tmp")));
assert!(collected_paths.contains(&PathBuf::from("src/keep.tmp")));
assert!(!collected_paths.contains(&PathBuf::from("build/keep.txt")));
assert!(!collected_paths.contains(&PathBuf::from("build")));
let streamed = Walker::new(&fixture.root)
.respect_git_ignore(true)
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream has no I/O errors");
let streamed_paths = relative_paths(&streamed, &fixture.root);
assert!(!streamed_paths.contains(&PathBuf::from("generated.tmp")));
assert!(streamed_paths.contains(&PathBuf::from("keep.tmp")));
assert!(streamed_paths.contains(&PathBuf::from("src/keep.tmp")));
assert!(!streamed_paths.contains(&PathBuf::from("build/keep.txt")));
assert!(!streamed_paths.contains(&PathBuf::from("build")));
}
#[test]
fn subtree_walk_inherits_repository_ignore_rules_excludes_and_config() {
let fixture = Fixture::new();
for path in [
"src/debug.LOG",
"src/secret.txt",
"src/ignored-by-info",
"src/deep/trace.log",
"src/kept.txt",
] {
fixture.write(path);
}
let initialized = git_command()
.args(["init", "--quiet"])
.current_dir(&fixture.root)
.status()
.expect("initialize Git fixture");
assert!(initialized.success());
let configured = git_command()
.args(["config", "core.ignoreCase", "true"])
.current_dir(&fixture.root)
.status()
.expect("configure Git fixture");
assert!(configured.success());
fs::write(fixture.root.join(".gitignore"), b"*.log\n/src/secret.txt\n")
.expect("write repository ignore rules");
fs::write(fixture.root.join(".git/info/exclude"), b"ignored-by-info\n")
.expect("write repository excludes");
let root = fixture.root.join("src");
let walked = Walker::new(&root)
.respect_git_ignore(true)
.options(WalkOptions::default().sort(true))
.collect()
.expect("subtree walk succeeds");
let paths = relative_paths(walked.entries(), &root);
for candidate in [
"src/debug.LOG",
"src/secret.txt",
"src/ignored-by-info",
"src/deep/trace.log",
] {
let relative = Path::new(candidate)
.strip_prefix("src")
.expect("candidate is below subtree root")
.to_path_buf();
assert_eq!(
!paths.contains(&relative),
git_check_ignore(&fixture.root, candidate),
"subtree walk must agree with Git for {candidate}",
);
}
assert!(paths.contains(&PathBuf::from("kept.txt")));
}
fn git_command() -> Command {
let mut command = Command::new("git");
command
.env("GIT_CONFIG_GLOBAL", "/dev/null")
.env("GIT_CONFIG_SYSTEM", "/dev/null")
.env("GIT_CONFIG_NOSYSTEM", "1");
command
}
fn git_check_ignore(root: &Path, candidate: &str) -> bool {
let status = git_command()
.args(["check-ignore", "--no-index", "--quiet", "--", candidate])
.current_dir(root)
.status()
.expect("run Git ignore oracle");
match status.code() {
Some(0) => true,
Some(1) => false,
other => panic!("git check-ignore failed with {other:?}"),
}
}
fn git_config_bool(root: &Path, key: &str) -> bool {
let output = git_command()
.args(["config", "--type=bool", "--get", key])
.current_dir(root)
.output()
.expect("run Git config boolean oracle");
assert!(
output.status.success(),
"Git config boolean oracle failed: {}",
String::from_utf8_lossy(&output.stderr)
);
match String::from_utf8(output.stdout)
.expect("Git config boolean output is UTF-8")
.trim()
{
"true" => true,
"false" => false,
value => panic!("unexpected Git config boolean output: {value:?}"),
}
}
fn git_config_file_bool(config: &Path, key: &str) -> bool {
let output = git_command()
.args(["config", "--file"])
.arg(config)
.args(["--type=bool", "--get", key])
.output()
.expect("run Git config file boolean oracle");
assert!(
output.status.success(),
"Git config file boolean oracle failed: {}",
String::from_utf8_lossy(&output.stderr)
);
match String::from_utf8(output.stdout)
.expect("Git config boolean output is UTF-8")
.trim()
{
"true" => true,
"false" => false,
value => panic!("unexpected Git config boolean output: {value:?}"),
}
}
#[test]
fn git_oracle_ignores_an_inherited_global_excludes_file() {
if let Some(root) = std::env::var_os(HOSTILE_GIT_CONFIG_FIXTURE) {
let root = PathBuf::from(root);
assert!(
!git_check_ignore(&root, "unignored.log"),
"the oracle must not read the hostile inherited excludes file"
);
return;
}
let fixture = Fixture::new();
fixture.write("unignored.log");
let initialized = git_command()
.args(["init", "--quiet"])
.current_dir(&fixture.root)
.status()
.expect("initialize hostile-config Git fixture");
assert!(initialized.success());
let excludes = fixture.root.join("hostile-excludes");
fs::write(&excludes, b"*.log\n").expect("write hostile global excludes file");
let config = fixture.root.join("hostile-gitconfig");
let excludes = excludes.display().to_string();
let excludes = excludes.replace('\\', "\\\\").replace('"', "\\\"");
fs::write(
&config,
format!("[core]\n\texcludesFile = \"{excludes}\"\n"),
)
.expect("write hostile global Git config");
let status = Command::new(std::env::current_exe().expect("locate test binary"))
.args([
"tests::git_oracle_ignores_an_inherited_global_excludes_file",
"--exact",
])
.env(HOSTILE_GIT_CONFIG_FIXTURE, &fixture.root)
.env("GIT_CONFIG_GLOBAL", &config)
.status()
.expect("run isolated Git oracle regression test");
assert!(
status.success(),
"hostile-config child test failed: {status}"
);
}
#[test]
fn repository_ignorecase_matches_git_for_rules_negation_and_anchors() {
let fixture = Fixture::new();
fs::create_dir_all(fixture.root.join(".git")).expect("create Git metadata");
fs::write(
fixture.root.join(".git/config"),
b"[CoRe]\nignoreCase = YeS\n",
)
.expect("write local config");
fs::write(
fixture.root.join(".gitignore"),
b"Build.LOG\nDist/\n!Kept.LOG\n/src/Anchored.LOG\n",
)
.expect("write ignore rules");
for path in [
"BUILD.log",
"DIST/deep.txt",
"kept.log",
"SRC/ANCHORED.log",
"other/ANCHORED.log",
] {
fixture.write(path);
}
let initialized = git_command()
.args(["init", "--quiet"])
.current_dir(&fixture.root)
.status()
.expect("initialize Git oracle");
assert!(initialized.success());
let configured = git_command()
.args(["config", "core.ignoreCase", "true"])
.current_dir(&fixture.root)
.status()
.expect("configure Git oracle");
assert!(configured.success());
let serial = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(1)
.options(WalkOptions::default().sort(true))
.collect()
.expect("serial walk succeeds");
let parallel = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(4)
.options(WalkOptions::default().sort(true))
.collect()
.expect("parallel walk succeeds");
let streamed = Walker::new(&fixture.root)
.respect_git_ignore(true)
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream succeeds");
for paths in [
relative_paths(serial.entries(), &fixture.root),
relative_paths(parallel.entries(), &fixture.root),
relative_paths(&streamed, &fixture.root),
] {
for candidate in [
"BUILD.log",
"DIST/deep.txt",
"kept.log",
"SRC/ANCHORED.log",
"other/ANCHORED.log",
] {
assert_eq!(
!paths.contains(&PathBuf::from(candidate)),
git_check_ignore(&fixture.root, candidate),
"Ferralk must agree with Git for {candidate}",
);
}
}
}
#[test]
fn non_utf8_unrelated_config_value_does_not_hide_repository_ignorecase() {
let fixture = Fixture::new();
fixture.write("BUILD.log");
fs::write(fixture.root.join(".gitignore"), b"build.log\n").expect("write rule");
let initialized = git_command()
.args(["init", "--quiet"])
.current_dir(&fixture.root)
.status()
.expect("initialize Git oracle");
assert!(initialized.success());
fs::write(
fixture.root.join(".git/config"),
b"[user]\nname = Jos\xe9\n[core]\nignorecase = true\n",
)
.expect("write Latin-1 local config");
assert!(git_config_bool(&fixture.root, "core.ignoreCase"));
assert!(git_check_ignore(&fixture.root, "BUILD.log"));
let walked = Walker::new(&fixture.root)
.respect_git_ignore(true)
.collect()
.expect("walk with non-UTF-8 config");
assert!(
!relative_paths(walked.entries(), &fixture.root).contains(&PathBuf::from("BUILD.log")),
"Ferralk must retain core.ignoreCase and agree with Git"
);
}
#[test]
fn explicit_ignorecase_false_and_walker_override_take_precedence() {
let fixture = Fixture::new();
fixture.write("BUILD.log");
fs::create_dir_all(fixture.root.join(".git")).expect("create Git metadata");
fs::write(
fixture.root.join(".git/config"),
b"[core]\nignorecase = true\nIGNORECASE = off\n",
)
.expect("write last-value local config");
fs::write(fixture.root.join(".gitignore"), b"build.log\n").expect("write rule");
let local_false = Walker::new(&fixture.root)
.respect_git_ignore(true)
.collect()
.expect("walk local false");
assert!(
relative_paths(local_false.entries(), &fixture.root)
.contains(&PathBuf::from("BUILD.log"))
);
let override_true = Walker::new(&fixture.root)
.respect_git_ignore(true)
.git_ignore_case(true)
.collect()
.expect("walk override true");
assert!(
!relative_paths(override_true.entries(), &fixture.root)
.contains(&PathBuf::from("BUILD.log"))
);
let cleared = Walker::new(&fixture.root)
.respect_git_ignore(true)
.git_ignore_case(true)
.clear_git_ignore_case()
.collect()
.expect("cleared override resumes local config");
assert!(
relative_paths(cleared.entries(), &fixture.root).contains(&PathBuf::from("BUILD.log")),
"clearing an override restores the repository-local false value"
);
}
#[test]
fn numeric_and_empty_ignorecase_values_match_the_git_oracle() {
let fixture = Fixture::new();
fixture.write("BUILD.log");
let initialized = git_command()
.args(["init", "--quiet"])
.current_dir(&fixture.root)
.status()
.expect("initialize Git oracle");
assert!(initialized.success());
fs::write(fixture.root.join(".gitignore"), b"build.log\n").expect("write rule");
let config = fixture.root.join(".git/config");
for (value, expected) in [
("", false),
("\"\"", false),
("+0", false),
("-0", false),
("+2", true),
("-7", true),
(" +2 # whitespace and comment", true),
] {
fs::write(
&config,
format!("[core]\nignoreCase = {}\nIGNORECASE = {value}\n", !expected),
)
.expect("write duplicate config");
assert_eq!(
git_config_bool(&fixture.root, "core.ignoreCase"),
expected,
"Git must accept {value:?}",
);
let walked = Walker::new(&fixture.root)
.respect_git_ignore(true)
.collect()
.expect("walk with Git boolean value");
assert_eq!(
!relative_paths(walked.entries(), &fixture.root)
.contains(&PathBuf::from("BUILD.log")),
expected,
"the later {value:?} value must override the earlier opposite value",
);
}
fs::write(&config, b"[core]\nignoreCase = false\nIGNORECASE\n")
.expect("write bare boolean config");
assert!(git_config_bool(&fixture.root, "core.ignoreCase"));
let bare_true = Walker::new(&fixture.root)
.respect_git_ignore(true)
.collect()
.expect("walk with bare true");
assert!(
!relative_paths(bare_true.entries(), &fixture.root)
.contains(&PathBuf::from("BUILD.log")),
"a bare key must retain Git's true behavior"
);
fs::write(
&config,
b"[core]\nignoreCase = false\nIGNORECASE = t\\\nr\\\nue\n\
[core \"unrelated\"]\nignoreCase = false\n",
)
.expect("write continued and subsection config");
assert!(git_config_bool(&fixture.root, "core.ignoreCase"));
let continued_true = Walker::new(&fixture.root)
.respect_git_ignore(true)
.collect()
.expect("walk with continued boolean config");
assert!(
!relative_paths(continued_true.entries(), &fixture.root)
.contains(&PathBuf::from("BUILD.log")),
"a later subsection must not override the top-level continued true value"
);
}
#[test]
fn case_variant_ignore_file_follows_the_filesystem_canonical_open() {
let fixture = Fixture::new();
fixture.write("build.log");
fs::write(fixture.root.join(".GITIGNORE"), b"build.log\n")
.expect("write case variant ignore file");
let canonical_open_resolves = fs::read(fixture.root.join(".gitignore")).is_ok();
let walked = Walker::new(&fixture.root)
.respect_git_ignore(true)
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds");
let paths = relative_paths(walked.entries(), &fixture.root);
assert!(paths.contains(&PathBuf::from(".GITIGNORE")));
assert_eq!(
!paths.contains(&PathBuf::from("build.log")),
canonical_open_resolves,
"a case variant is a rule file only if Git's canonical open resolves it"
);
}
#[test]
fn multi_root_walks_keep_each_repository_ignorecase_setting() {
let fixture = Fixture::new();
let nested = fixture.root.join("nested");
fixture.write("nested/BUILD.log");
fs::create_dir_all(fixture.root.join(".git")).expect("create outer Git metadata");
fs::create_dir_all(nested.join(".git")).expect("create nested Git metadata");
fs::write(
fixture.root.join(".git/config"),
b"[core]\nignorecase = true\n",
)
.expect("write outer config");
fs::write(nested.join(".git/config"), b"[core]\nignorecase = false\n")
.expect("write nested config");
fs::write(fixture.root.join(".gitignore"), b"build.log\n").expect("write outer rule");
fs::write(nested.join(".gitignore"), b"build.log\n").expect("write nested rule");
let walked = Walker::new(&fixture.root)
.add_root(&nested)
.expect("add nested root")
.add_root(&nested)
.expect("add duplicate nested root")
.respect_git_ignore(true)
.collect()
.expect("multi-root walk succeeds");
let emitted = walked
.entries()
.iter()
.filter(|entry| entry.path() == nested.join("BUILD.log"))
.count();
assert_eq!(emitted, 2);
}
#[test]
fn try_add_root_keeps_the_builder_after_a_rejected_root() {
let fixture = Fixture::new();
let first_extra = fixture.root.clone();
let last_extra = fixture.root.clone();
let mut walker = Walker::new(&fixture.root)
.include(fixture.absolute("/**"))
.expect("absolute pattern applies to the first absolute root");
walker
.try_add_root(&first_extra)
.expect("the absolute pattern applies to an equal root");
let error = walker
.try_add_root("relative-root")
.expect_err("an absolute pattern needs an absolute added root");
assert_eq!(
error.message(),
"an absolute pattern needs an absolute walk root"
);
assert_eq!(
walker.roots().collect::<Vec<_>>(),
vec![fixture.root.as_path(), first_extra.as_path()],
"a rejected borrowed addition cannot partially append a root"
);
walker
.try_add_root(&last_extra)
.expect("the retained builder can consider later roots");
assert_eq!(
walker.roots().collect::<Vec<_>>(),
vec![
fixture.root.as_path(),
first_extra.as_path(),
last_extra.as_path(),
]
);
}
#[cfg(target_os = "macos")]
#[test]
fn repository_precomposeunicode_matches_the_macos_git_oracle() {
let fixture = Fixture::new();
let decomposed = "cafe\u{301}.txt";
fs::write(fixture.root.join(".gitignore"), "caf\u{e9}.txt\n").expect("write NFC rule");
fixture.write(decomposed);
let initialized = git_command()
.args(["init", "--quiet"])
.current_dir(&fixture.root)
.status()
.expect("initialize Git oracle");
assert!(initialized.success());
let config = fixture.root.join(".git/config");
fs::write(
&config,
b"[core]\nprecomposeUnicode = false\nPRECOMPOSEUNICODE = t\\\nr\\\nue\n\
[core \"unrelated\"]\nprecomposeUnicode = false\n",
)
.expect("write continued precompose Unicode config");
assert!(git_config_bool(&fixture.root, "core.precomposeUnicode"));
assert!(git_check_ignore(&fixture.root, decomposed));
let enabled = Walker::new(&fixture.root)
.respect_git_ignore(true)
.collect()
.expect("walk with NFC adaptation");
assert!(
!relative_paths(enabled.entries(), &fixture.root).contains(&PathBuf::from(decomposed))
);
let forced_false = Walker::new(&fixture.root)
.respect_git_ignore(true)
.git_precompose_unicode(false)
.collect()
.expect("walk with explicit false override");
assert!(
relative_paths(forced_false.entries(), &fixture.root)
.contains(&PathBuf::from(decomposed))
);
fs::write(
&config,
b"[core]\nprecomposeUnicode = \"\"\n\
[core \"unrelated\"]\nprecomposeUnicode = true\n",
)
.expect("write empty precompose Unicode config");
assert!(!git_config_bool(&fixture.root, "core.precomposeUnicode"));
assert!(!git_check_ignore(&fixture.root, decomposed));
let disabled = Walker::new(&fixture.root)
.respect_git_ignore(true)
.collect()
.expect("walk without NFC adaptation");
assert!(
relative_paths(disabled.entries(), &fixture.root).contains(&PathBuf::from(decomposed))
);
let forced_true = Walker::new(&fixture.root)
.respect_git_ignore(true)
.git_precompose_unicode(true)
.collect()
.expect("walk with explicit true override");
assert!(
!relative_paths(forced_true.entries(), &fixture.root)
.contains(&PathBuf::from(decomposed))
);
let cleared = Walker::new(&fixture.root)
.respect_git_ignore(true)
.git_precompose_unicode(true)
.clear_git_precompose_unicode()
.collect()
.expect("cleared override resumes local config");
assert!(
relative_paths(cleared.entries(), &fixture.root).contains(&PathBuf::from(decomposed)),
"clearing an override restores the repository-local false value"
);
}
#[test]
fn clearing_git_adaptation_overrides_restores_the_unset_state() {
let walker = Walker::new("workspace")
.git_ignore_case(true)
.clear_git_ignore_case()
.git_precompose_unicode(true)
.clear_git_precompose_unicode();
assert_eq!(walker.git_ignore_case, None);
assert_eq!(walker.git_precompose_unicode, None);
}
#[cfg(target_os = "linux")]
#[test]
fn source_walk_gitignore_matches_non_utf8_file_names_on_linux() {
use std::os::unix::ffi::OsStringExt;
let fixture = Fixture::new();
let name = std::ffi::OsString::from_vec(b"\xE9latin1.txt".to_vec());
fs::write(fixture.root.join(".gitignore"), b"\xE9latin1.txt\n")
.expect("write byte-pattern gitignore");
fixture.write(Path::new(&name));
let ignored = fixture.root.join(&name);
let result = Walker::new(&fixture.root)
.respect_git_ignore(true)
.collect()
.expect("walk succeeds");
assert!(
!result
.entries()
.iter()
.any(|entry| entry.path() == ignored.as_path()),
"the byte-pattern rule has to hide its byte-named file"
);
}
#[test]
fn source_walk_nested_gitignore_overrides_and_skips_dot_git() {
let fixture = Fixture::new();
for path in [
"build/artifact.o",
"root.log",
"keep.txt",
"sub/important.log",
"sub/other.log",
"sub/temp/scratch.txt",
".git/config",
] {
fixture.write(path);
}
fs::write(fixture.root.join(".gitignore"), b"*.log\nbuild/\n")
.expect("write root gitignore");
fs::write(
fixture.root.join("sub/.gitignore"),
b"!important.log\ntemp/\n",
)
.expect("write nested gitignore");
let result = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(1)
.options(WalkOptions::default().sort(true))
.collect()
.expect("Gitignore walk succeeds");
let paths = relative_paths(result.entries(), &fixture.root);
for ignored in [
"build",
"build/artifact.o",
"root.log",
"sub/other.log",
"sub/temp",
"sub/temp/scratch.txt",
".git",
".git/config",
] {
assert!(
!paths.contains(&PathBuf::from(ignored)),
"ignored source path {ignored} was returned"
);
}
for kept in ["keep.txt", "sub", "sub/.gitignore", "sub/important.log"] {
assert!(
paths.contains(&PathBuf::from(kept)),
"kept source path {kept} was omitted"
);
}
let parallel = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(4)
.options(WalkOptions::default().sort(true))
.collect()
.expect("parallel Gitignore walk succeeds");
assert_eq!(relative_paths(parallel.entries(), &fixture.root), paths);
let mut streamed = Walker::new(&fixture.root)
.respect_git_ignore(true)
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream Gitignore walk succeeds");
streamed.sort_by(|left, right| left.path().cmp(right.path()));
assert_eq!(relative_paths(&streamed, &fixture.root), paths);
let without_ignore = Walker::new(&fixture.root)
.threads(1)
.options(WalkOptions::default().sort(true))
.collect()
.expect("unfiltered source walk succeeds");
let unfiltered_paths = relative_paths(without_ignore.entries(), &fixture.root);
assert!(unfiltered_paths.contains(&PathBuf::from(".git/config")));
assert!(unfiltered_paths.len() > paths.len());
}
#[test]
fn source_walk_ignore_file_overrides_gitignore_rules() {
let fixture = Fixture::new();
for path in [
"app.log",
"keep.log",
"scratch.tmp",
"important.tmp",
"old.bak",
"src/main.rs",
"src/old.bak",
"build/artifact.txt",
] {
fixture.write(path);
}
fs::write(
fixture.root.join(".gitignore"),
b"*.log\nbuild/\n!keep.log\n*.tmp\n",
)
.expect("write root gitignore");
fs::write(fixture.root.join(".ignore"), b"!important.tmp\n")
.expect("write root ignore supplement");
fs::write(fixture.root.join("src/.gitignore"), b"*.bak\n").expect("write nested gitignore");
let expected = vec![
PathBuf::from(".gitignore"),
PathBuf::from(".ignore"),
PathBuf::from("important.tmp"),
PathBuf::from("keep.log"),
PathBuf::from("old.bak"),
PathBuf::from("src"),
PathBuf::from("src/.gitignore"),
PathBuf::from("src/main.rs"),
];
let options = WalkOptions::default().sort(true);
let serial = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(1)
.options(options)
.collect()
.expect("serial ignore walk succeeds");
let parallel = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(4)
.options(options)
.collect()
.expect("parallel ignore walk succeeds");
let mut streamed = Walker::new(&fixture.root)
.respect_git_ignore(true)
.options(options)
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream ignore walk succeeds");
streamed.sort_by(|left, right| left.path().cmp(right.path()));
assert_eq!(relative_paths(serial.entries(), &fixture.root), expected);
assert_eq!(relative_paths(parallel.entries(), &fixture.root), expected);
assert_eq!(relative_paths(&streamed, &fixture.root), expected);
}
#[cfg(unix)]
#[test]
fn symlinked_in_tree_ignore_files_do_not_apply_but_repository_excludes_can_follow_links() {
use std::os::unix::fs::symlink;
for ignore_file in [".gitignore", ".ignore"] {
let fixture = Fixture::new();
fixture.write("linked.tmp");
fs::write(fixture.root.join("rules-source"), b"linked.tmp\n")
.expect("write linked ignore source");
symlink("rules-source", fixture.root.join(ignore_file))
.expect("create linked in-tree ignore file");
let result = Walker::new(&fixture.root)
.respect_git_ignore(true)
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds when a linked ignore file is refused");
assert!(
relative_paths(result.entries(), &fixture.root)
.contains(&PathBuf::from("linked.tmp")),
"{ignore_file} must not redirect in-tree rules through its target"
);
}
let fixture = Fixture::new();
fixture.write("repository-only.tmp");
fs::write(fixture.root.join("rules-source"), b"repository-only.tmp\n")
.expect("write repository exclude source");
fs::create_dir_all(fixture.root.join(".git/info")).expect("create git info directory");
symlink("../../rules-source", fixture.root.join(".git/info/exclude"))
.expect("link repository exclude");
let result = Walker::new(&fixture.root)
.respect_git_ignore(true)
.collect()
.expect("walk succeeds when repository exclude is linked");
assert!(
!relative_paths(result.entries(), &fixture.root)
.contains(&PathBuf::from("repository-only.tmp")),
".git/info/exclude remains allowed to follow links"
);
}
#[test]
fn linked_worktree_and_submodule_gitdir_pointers_read_the_right_info_exclude() {
let fixture = Fixture::new();
let worktree = fixture.root.join("linked-worktree");
let common_git = fixture.root.join("main/.git");
let private_git = common_git.join("worktrees/linked-worktree");
fs::create_dir_all(private_git.join("refs")).expect("create private git directory");
fs::create_dir_all(common_git.join("info")).expect("create common git info");
fs::write(common_git.join("info/exclude"), b"worktree-secret.txt\n")
.expect("write common exclude");
fs::write(private_git.join("commondir"), b"../..\n").expect("write commondir");
fs::create_dir_all(&worktree).expect("create linked worktree");
fs::write(
worktree.join(".git"),
b"gitdir: ../main/.git/worktrees/linked-worktree\n",
)
.expect("write linked-worktree pointer");
fs::write(worktree.join("worktree-secret.txt"), b"fixture")
.expect("write linked-worktree secret");
let worktree_result = Walker::new(&worktree)
.respect_git_ignore(true)
.collect()
.expect("walk linked worktree");
assert!(
!relative_paths(worktree_result.entries(), &worktree)
.contains(&PathBuf::from("worktree-secret.txt")),
"the linked worktree must use its common info/exclude"
);
let submodule = fixture.root.join("super/dependency");
let submodule_git = fixture.root.join("super/.git/modules/dependency");
fs::create_dir_all(submodule_git.join("info")).expect("create submodule git info");
fs::write(
submodule_git.join("info/exclude"),
b"submodule-secret.txt\n",
)
.expect("write submodule exclude");
fs::create_dir_all(&submodule).expect("create submodule checkout");
fs::write(
submodule.join(".git"),
b"gitdir: ../.git/modules/dependency\n",
)
.expect("write submodule pointer");
fs::write(submodule.join("submodule-secret.txt"), b"fixture")
.expect("write submodule secret");
let submodule_result = Walker::new(&submodule)
.respect_git_ignore(true)
.collect()
.expect("walk submodule checkout");
assert!(
!relative_paths(submodule_result.entries(), &submodule)
.contains(&PathBuf::from("submodule-secret.txt")),
"the submodule pointer must use its own info/exclude"
);
}
#[test]
fn linked_worktree_config_overrides_the_common_repository_setting() {
let fixture = Fixture::new();
let checkout = fixture.root.join("linked-worktree");
let common_git = fixture.root.join("main/.git");
let private_git = common_git.join("worktrees/linked-worktree");
fs::create_dir_all(&checkout).expect("create linked checkout");
fs::create_dir_all(&private_git).expect("create private Git directory");
fs::write(private_git.join("commondir"), b"../..\n").expect("write commondir");
fs::create_dir_all(&common_git).expect("create common Git directory");
fs::write(
common_git.join("config"),
b"[extensions]\nworktreeConfig = f\\\nalse\nWORKTREECONFIG = t\\\nr\\\nue\n\
[extensions \"unrelated\"]\nworktreeConfig = false\n\
[core]\nignorecase = false\n",
)
.expect("write continued common config");
fs::write(
private_git.join("config.worktree"),
b"[CORE]\nignoreCASE = t\\\nr\\\nue\n\
[core \"unrelated\"]\nignoreCASE = false\n",
)
.expect("write continued private worktree config");
fs::write(
checkout.join(".git"),
b"gitdir: ../main/.git/worktrees/linked-worktree\n",
)
.expect("write worktree pointer");
fs::write(checkout.join(".gitignore"), b"build.log\n").expect("write ignore rule");
fs::write(checkout.join("BUILD.log"), b"fixture").expect("write mixed-case candidate");
assert!(git_config_file_bool(
&common_git.join("config"),
"extensions.worktreeConfig"
));
let walked = Walker::new(&checkout)
.respect_git_ignore(true)
.collect()
.expect("walk linked worktree");
assert!(
!relative_paths(walked.entries(), &checkout).contains(&PathBuf::from("BUILD.log")),
"private config.worktree must win after the common config"
);
}
#[test]
fn absolute_and_malformed_gitdir_pointers_are_handled_without_rules() {
let fixture = Fixture::new();
let checkout = fixture.root.join("absolute-worktree");
let git_directory = fixture.root.join("separate-git-directory");
fs::create_dir_all(git_directory.join("info")).expect("create separate git info");
fs::write(git_directory.join("info/exclude"), b"absolute-secret.txt\n")
.expect("write absolute exclude");
fs::create_dir_all(&checkout).expect("create absolute checkout");
fs::write(
checkout.join(".git"),
format!("gitdir: {}\n", git_directory.display()),
)
.expect("write absolute pointer");
fs::write(checkout.join("absolute-secret.txt"), b"fixture").expect("write absolute secret");
let absolute_result = Walker::new(&checkout)
.respect_git_ignore(true)
.collect()
.expect("walk absolute pointer checkout");
assert!(
!relative_paths(absolute_result.entries(), &checkout)
.contains(&PathBuf::from("absolute-secret.txt"))
);
let malformed = fixture.root.join("malformed-pointer");
fs::create_dir_all(&malformed).expect("create malformed checkout");
fs::write(malformed.join(".git"), b"gitdir: \nextra data\n")
.expect("write malformed pointer");
fs::write(malformed.join("not-excluded.txt"), b"fixture")
.expect("write malformed candidate");
let malformed_result = Walker::new(&malformed)
.respect_git_ignore(true)
.collect()
.expect("malformed metadata is skipped");
assert!(
relative_paths(malformed_result.entries(), &malformed)
.contains(&PathBuf::from("not-excluded.txt")),
"a malformed pointer must add no repository rules"
);
}
#[cfg(unix)]
#[test]
fn unreadable_gitdir_pointer_is_skipped_like_unreadable_repository_metadata() {
use std::os::unix::fs::PermissionsExt;
let fixture = Fixture::new();
let checkout = fixture.root.join("unreadable-pointer");
fs::create_dir_all(&checkout).expect("create checkout");
let pointer = checkout.join(".git");
fs::write(&pointer, b"gitdir: ../missing-git-directory\n").expect("write pointer file");
fs::write(checkout.join("not-excluded.txt"), b"fixture").expect("write candidate");
let original_permissions = fs::metadata(&pointer)
.expect("read pointer metadata")
.permissions();
fs::set_permissions(&pointer, fs::Permissions::from_mode(0o000))
.expect("make pointer unreadable");
if fs::read(&pointer).is_ok() {
fs::set_permissions(&pointer, original_permissions)
.expect("restore pointer permissions");
return;
}
let result = Walker::new(&checkout)
.respect_git_ignore(true)
.collect()
.expect("unreadable metadata is skipped rather than reported");
fs::set_permissions(&pointer, original_permissions).expect("restore pointer permissions");
assert!(
result.errors().is_empty(),
"unreadable repository metadata follows the existing silent policy"
);
assert!(
relative_paths(result.entries(), &checkout)
.contains(&PathBuf::from("not-excluded.txt")),
"an unreadable pointer must add no repository rules"
);
}
#[test]
fn nested_repositories_remain_traversed_with_the_outer_ignore_chain() {
let fixture = Fixture::new();
fixture.write("nested/.git/config");
fixture.write("nested/outer.tmp");
fixture.write("nested/keep.tmp");
fixture.write("nested/inner-only.tmp");
fs::write(fixture.root.join(".gitignore"), b"*.tmp\n").expect("write outer ignore rules");
fs::write(
fixture.root.join("nested/.gitignore"),
b"!keep.tmp\ninner-only.tmp\n",
)
.expect("write nested repository rules");
let result = Walker::new(&fixture.root)
.respect_git_ignore(true)
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk nested repository");
let paths = relative_paths(result.entries(), &fixture.root);
assert!(paths.contains(&PathBuf::from("nested/keep.tmp")));
assert!(!paths.contains(&PathBuf::from("nested/outer.tmp")));
assert!(!paths.contains(&PathBuf::from("nested/inner-only.tmp")));
assert!(
!paths.contains(&PathBuf::from("nested/.git/config")),
"the nested repository's control directory remains skipped"
);
}
#[test]
fn source_walk_allowlist_gitignore_descends_into_reincluded_directories() {
let fixture = Fixture::new();
for path in [
"main.rs",
"Makefile",
".keep",
"src/lib.rs",
"src/noext",
"src/deep/a.txt",
"dir.d/x.md",
"dir.d/noext",
"plain/y.txt",
".git/config",
] {
fixture.write(path);
}
fs::write(
fixture.root.join(".gitignore"),
b"# Ignore all\n*\n\n# Unignore all with extensions\n!*.*\n\n# Unignore all dirs\n!/**/\n",
)
.expect("write root gitignore");
let expected = vec![
PathBuf::from(".gitignore"),
PathBuf::from(".keep"),
PathBuf::from("dir.d"),
PathBuf::from("dir.d/x.md"),
PathBuf::from("main.rs"),
PathBuf::from("plain"),
PathBuf::from("plain/y.txt"),
PathBuf::from("src"),
PathBuf::from("src/deep"),
PathBuf::from("src/deep/a.txt"),
PathBuf::from("src/lib.rs"),
];
let options = WalkOptions::default().sort(true);
let serial = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(1)
.options(options)
.collect()
.expect("serial Gitignore walk succeeds");
let parallel = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(4)
.options(options)
.collect()
.expect("parallel Gitignore walk succeeds");
let mut streamed = Walker::new(&fixture.root)
.respect_git_ignore(true)
.options(options)
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream Gitignore walk succeeds");
streamed.sort_by(|left, right| left.path().cmp(right.path()));
assert_eq!(relative_paths(serial.entries(), &fixture.root), expected);
assert_eq!(relative_paths(parallel.entries(), &fixture.root), expected);
assert_eq!(relative_paths(&streamed, &fixture.root), expected);
}
#[test]
fn every_ignore_file_is_read_once_per_walk() {
let fixture = Fixture::new();
fixture.write(".gitignore");
fixture.write("src/.gitignore");
fixture.write("src/nested/.gitignore");
for branch in 0..6 {
fixture.write(format!("src/nested/branch-{branch}/leaf.txt"));
fixture.write(format!("docs/branch-{branch}/leaf.md"));
}
for threads in [1, 4] {
let backend = CountingBackend::default();
Walker::new(&fixture.root)
.threads(threads)
.respect_git_ignore(true)
.collect_with(&backend)
.expect("walk succeeds");
let repeated = backend
.ignore_reads()
.into_iter()
.filter(|(_, reads)| *reads > 1)
.collect::<Vec<_>>();
assert!(
repeated.is_empty(),
"with {threads} threads these ignore files were read more than once: {repeated:?}"
);
assert!(
backend
.ignore_reads()
.iter()
.any(|(path, _)| path.ends_with("src/.gitignore")),
"the walk has to read the nested ignore files through the backend"
);
}
}
#[test]
fn oversized_ignore_files_follow_the_configured_error_policy() {
let fixture = Fixture::new();
fixture.write("visible.txt");
let ignore_path = fixture.root.join(".gitignore");
let ignore_file = fs::File::create(&ignore_path).expect("create oversized ignore file");
ignore_file
.set_len(super::MAX_IGNORE_FILE_BYTES + 1)
.expect("size oversized ignore file");
for threads in [1, 4] {
let result = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(threads)
.error_policy(ErrorPolicy::Collect)
.collect()
.expect("collect policy keeps walking");
assert_eq!(result.errors().len(), 1);
assert_eq!(result.errors()[0].operation(), "read_ignore");
assert_eq!(result.errors()[0].path(), ignore_path);
assert!(
relative_paths(result.entries(), &fixture.root)
.contains(&PathBuf::from("visible.txt")),
"an unreadable rule file cannot silently hide unrelated entries"
);
}
let skipped = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(1)
.error_policy(ErrorPolicy::Skip)
.collect()
.expect("skip policy keeps walking");
assert!(skipped.errors().is_empty());
let aborted = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(1)
.error_policy(ErrorPolicy::Abort)
.collect()
.expect_err("abort policy reports the ignore failure immediately");
assert_eq!(aborted.operation(), "read_ignore");
assert_eq!(aborted.path(), ignore_path);
let streamed = Walker::new(&fixture.root)
.respect_git_ignore(true)
.error_policy(ErrorPolicy::Collect)
.stream()
.collect::<Vec<_>>();
assert_eq!(
streamed.iter().filter(|item| item.is_err()).count(),
1,
"the stream yields the ignore failure exactly once"
);
assert!(streamed.iter().any(|item| {
item.as_ref()
.is_ok_and(|entry| entry.path().ends_with("visible.txt"))
}));
}
#[test]
fn root_rules_reach_deep_entries_and_deeper_rules_win() {
let fixture = Fixture::new();
fixture.write("a/b/c/deep.log");
fixture.write("a/b/other.log");
fixture.write("a/b/c/keep.txt");
fs::write(fixture.root.join(".gitignore"), b"*.log\n").expect("write root gitignore");
fs::write(fixture.root.join("a/b/c/.gitignore"), b"!deep.log\n")
.expect("write nested gitignore");
let walked = Walker::new(&fixture.root)
.respect_git_ignore(true)
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds");
let paths = relative_paths(walked.entries(), &fixture.root);
assert!(
!paths.contains(&PathBuf::from("a/b/other.log")),
"a root rule has to reach entries below it"
);
assert!(
paths.contains(&PathBuf::from("a/b/c/deep.log")),
"the ignore file closest to the entry decides"
);
assert!(paths.contains(&PathBuf::from("a/b/c/keep.txt")));
}
#[test]
fn directory_rules_and_spanning_rules_apply_per_directory() {
let fixture = Fixture::new();
fixture.write("logs");
fixture.write("build/main.o");
fixture.write("a/b/temp/c/note.txt");
fixture.write("a/b/kept.txt");
fs::write(
fixture.root.join(".gitignore"),
b"logs/\nbuild/\n**/temp/**\n",
)
.expect("write root gitignore");
let walked = Walker::new(&fixture.root)
.respect_git_ignore(true)
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds");
let paths = relative_paths(walked.entries(), &fixture.root);
assert!(
paths.contains(&PathBuf::from("logs")),
"a directory-only rule must not match a file of the same name"
);
assert!(!paths.contains(&PathBuf::from("build")));
assert!(!paths.contains(&PathBuf::from("build/main.o")));
assert!(!paths.contains(&PathBuf::from("a/b/temp/c/note.txt")));
assert!(paths.contains(&PathBuf::from("a/b/kept.txt")));
}
#[test]
fn the_three_frontends_agree_on_nested_ignore_files() {
let fixture = Fixture::new();
fixture.write("src/main.rs");
fixture.write("src/debug.log");
fixture.write("src/keep.log");
fixture.write("src/nested/deep.log");
fixture.write("docs/guide.md");
fixture.write("build/main.o");
fixture.write("build/keep.txt");
fs::write(fixture.root.join(".gitignore"), b"*.log\nbuild/\n")
.expect("write root gitignore");
fs::write(fixture.root.join("src/.gitignore"), b"!keep.log\n")
.expect("write nested gitignore");
fs::write(fixture.root.join("build/.gitignore"), b"!keep.txt\n").expect("write re-include");
fs::create_dir_all(fixture.root.join(".git/info")).expect("create git directory");
fs::write(fixture.root.join(".git/info/exclude"), b"*.md\n")
.expect("write repository excludes");
assert_frontends_agree("nested ignore files", &fixture.root, || {
Walker::new(&fixture.root).respect_git_ignore(true)
});
let walked = Walker::new(&fixture.root)
.respect_git_ignore(true)
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds");
let paths = relative_paths(walked.entries(), &fixture.root);
assert!(
!paths.contains(&PathBuf::from("docs/guide.md")),
"the repository excludes have to apply"
);
assert!(paths.contains(&PathBuf::from("src/keep.log")));
assert!(!paths.contains(&PathBuf::from("src/nested/deep.log")));
}
const KNOWN_WALKER_GAPS: &[&str] = &[];
#[test]
fn git_ignore_corpus_replays_through_the_walker() {
let corpus_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus/ignore.jsonl");
for line in fs::read_to_string(corpus_path)
.expect("read ignore corpus")
.lines()
.filter(|line| !line.trim().is_empty())
{
let case: corpus::Case = serde_json::from_str(line).expect("valid ignore corpus case");
if KNOWN_WALKER_GAPS.contains(&case.id.as_str()) {
continue;
}
let fixture = Fixture::new();
fs::write(
fixture.root.join(".gitignore"),
case.ignore_rules.join("\n").as_bytes(),
)
.expect("write fixture gitignore");
for nested in &case.nested_ignore_rules {
let directory = fixture.root.join(&nested.directory);
fs::create_dir_all(&directory).expect("create nested ignore directory");
fs::write(
directory.join(".gitignore"),
nested.rules.join("\n").as_bytes(),
)
.expect("write nested fixture gitignore");
}
if !case.exclude_rules.is_empty() {
let info = fixture.root.join(".git/info");
fs::create_dir_all(&info).expect("create repository info directory");
fs::write(
info.join("exclude"),
case.exclude_rules.join("\n").as_bytes(),
)
.expect("write repository excludes");
}
fixture.write(&case.path);
let result = Walker::new(&fixture.root)
.respect_git_ignore(true)
.collect()
.expect("walk succeeds");
let returned =
relative_paths(result.entries(), &fixture.root).contains(&PathBuf::from(case.path));
assert_eq!(
!returned, case.expected,
"walker verdict for corpus case {}",
case.id
);
}
}
#[test]
fn brace_alternatives_carry_the_planner_prefilters() {
let sources = traversal_pattern(b"**/*.{ts,tsx}");
assert!(sources.matches_extension(b"src/app.ts"));
assert!(sources.matches_extension(b"src/app.tsx"));
assert!(!sources.matches_extension(b"src/app.js"));
assert!(!sources.matches_extension(b"src/app"));
let scoped = traversal_pattern(b"{src,lib}/**/*.ts");
assert!(scoped.could_match_descendant(b"src"));
assert!(scoped.could_match_descendant(b"lib"));
assert!(scoped.could_match_descendant(b"src/nested"));
assert!(!scoped.could_match_descendant(b"docs"));
assert!(!scoped.could_match_descendant(b"node_modules"));
let nested = traversal_pattern(b"{src/{a,b},lib}/**");
assert!(nested.could_match_descendant(b"src"));
assert!(nested.could_match_descendant(b"src/a"));
assert!(!nested.could_match_descendant(b"src/c"));
let partial_root = traversal_pattern(b"{src,*}/**/*.ts");
assert!(partial_root.could_match_descendant(b"docs"));
let partial_extension = traversal_pattern(b"**/*.{ts,*}");
assert!(partial_extension.matches_extension(b"src/app.js"));
}
#[test]
fn match_hidden_widens_the_matcher_without_moving_the_planner_prefilters() {
let hidden = traversal_pattern_options(true);
let scoped = TraversalPattern::compile(b"site/**/*.ts", hidden).expect("valid include");
assert!(scoped.could_match_descendant(b"site/.react-router"));
assert!(scoped.matches(
b"site/.react-router/routes.ts",
false,
WildcardMode::ComponentScoped
));
assert!(scoped.matches_extension(b"site/.react-router/routes.ts"));
assert!(!scoped.could_match_descendant(b".react-router"));
let default = traversal_pattern(b"site/**/*.ts");
assert!(default.could_match_descendant(b"site/.react-router"));
assert!(!default.matches(
b"site/.react-router/routes.ts",
false,
WildcardMode::ComponentScoped
));
assert_eq!(default.literal_roots, scoped.literal_roots);
assert_eq!(default.extensions, scoped.extensions);
let literal = TraversalPattern::compile(b".claude/**/*.ts", hidden).expect("valid include");
assert!(literal.could_match_descendant(b".claude"));
assert!(traversal_pattern(b".claude/**/*.ts").matches(
b".claude/agents/run.ts",
false,
WildcardMode::ComponentScoped
));
}
#[test]
fn match_hidden_applies_to_patterns_added_before_and_after_it() {
let fixture = Fixture::new();
fixture.write(".react-router/types.ts");
fixture.write("src/app.ts");
let walk = |walker: Walker| {
relative_paths(
walker
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds")
.entries(),
&fixture.root,
)
};
let before = walk(
Walker::new(&fixture.root)
.match_hidden(true)
.include("**/*.ts")
.expect("valid include"),
);
let after = walk(
Walker::new(&fixture.root)
.include("**/*.ts")
.expect("valid include")
.match_hidden(true),
);
assert_eq!(
before,
vec![
PathBuf::from(".react-router/types.ts"),
PathBuf::from("src/app.ts"),
]
);
assert_eq!(after, before);
assert_eq!(
walk(
Walker::new(&fixture.root)
.match_hidden(true)
.include("**/*.ts")
.expect("valid include")
.match_hidden(false),
),
vec![PathBuf::from("src/app.ts")]
);
}
#[test]
fn an_unexpandable_include_is_reported_as_a_pattern_error() {
let beyond = "{a,b}".repeat(13);
let error = Walker::new(".")
.include(&beyond)
.expect_err("the expansion budget rejects this pattern");
assert_eq!(error.message(), "too many brace alternatives");
assert_eq!(
error.offset(),
ferralk_glob::Pattern::compile(&beyond, traversal_pattern_options(false))
.expect_err("the matcher rejects it the same way")
.offset()
);
}
#[test]
fn a_brace_include_returns_the_union_of_its_alternatives() {
let fixture = Fixture::new();
fixture.write("src/app.ts");
fixture.write("src/app.tsx");
fixture.write("src/app.js");
fixture.write("src/nested/deep.ts");
fixture.write("lib/util.ts");
fixture.write("lib/util.rs");
fixture.write("docs/guide.md");
fixture.write("docs/nested/notes.md");
fixture.write("node_modules/pkg/index.ts");
let walk = |pattern: &str| -> Vec<PathBuf> {
let result = Walker::new(&fixture.root)
.include(pattern)
.expect("valid include")
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds");
relative_paths(result.entries(), &fixture.root)
};
for pattern in [
"**/*.{ts,tsx}",
"{src,lib}/**/*.ts",
"{src,docs}/**",
"src/{app,nested}*",
"{src,lib}/**/*.{ts,rs}",
] {
let alternatives =
ferralk_glob::expand_braces(pattern, traversal_pattern_options(false))
.expect("expandable pattern");
let mut union = alternatives
.iter()
.flat_map(|alternative| {
walk(std::str::from_utf8(alternative).expect("ASCII fixture pattern"))
})
.collect::<Vec<_>>();
union.sort_unstable();
union.dedup();
assert_eq!(walk(pattern), union, "{pattern}");
}
}
#[test]
fn the_wildcard_mode_decides_how_far_a_wildcard_reaches() {
let fixture = Fixture::new();
fixture.write("main.ts");
fixture.write("src/app.ts");
fixture.write("src/deep/nested.ts");
fixture.write("other/stray.ts");
let walk = |mode: WildcardMode, pattern: &str| -> Vec<PathBuf> {
let result = Walker::new(&fixture.root)
.wildcard_mode(mode)
.include(pattern)
.expect("valid include")
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("walk succeeds");
relative_paths(result.entries(), &fixture.root)
};
assert_eq!(
walk(WildcardMode::ComponentScoped, "*.ts"),
vec![PathBuf::from("main.ts")],
"the default keeps a wildcard inside its component"
);
assert_eq!(
walk(WildcardMode::SeparatorCrossing, "*.ts"),
vec![
PathBuf::from("main.ts"),
PathBuf::from("other/stray.ts"),
PathBuf::from("src/app.ts"),
PathBuf::from("src/deep/nested.ts"),
],
"crossing reads the pattern the way globset does"
);
assert_eq!(
walk(WildcardMode::SeparatorCrossing, "src/*.ts"),
vec![
PathBuf::from("src/app.ts"),
PathBuf::from("src/deep/nested.ts")
],
"crossing reaches below the prefix, never outside it"
);
assert_eq!(
walk(WildcardMode::ComponentScoped, "src/*.ts"),
vec![PathBuf::from("src/app.ts")]
);
}
#[test]
fn the_wildcard_mode_governs_excludes_too() {
let fixture = Fixture::new();
fixture.write("keep.rs");
fixture.write("drop.tmp");
fixture.write("src/drop.tmp");
let walk = |mode: WildcardMode| -> Vec<PathBuf> {
let result = Walker::new(&fixture.root)
.wildcard_mode(mode)
.exclude("*.tmp")
.expect("valid exclude")
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("walk succeeds");
relative_paths(result.entries(), &fixture.root)
};
assert_eq!(
walk(WildcardMode::ComponentScoped),
vec![PathBuf::from("keep.rs"), PathBuf::from("src/drop.tmp")],
"a component-scoped exclude only reaches the root component"
);
assert_eq!(
walk(WildcardMode::SeparatorCrossing),
vec![PathBuf::from("keep.rs")],
"a crossing exclude reaches every level"
);
}
#[test]
fn the_wildcard_mode_leaves_the_hidden_policy_alone() {
let fixture = Fixture::new();
fixture.write("visible.ts");
fixture.write(".hidden.ts");
fixture.write("src/visible.ts");
fixture.write("src/.hidden.ts");
fixture.write(".config/inside.ts");
let walk = |match_hidden: bool| -> Vec<PathBuf> {
let result = Walker::new(&fixture.root)
.wildcard_mode(WildcardMode::SeparatorCrossing)
.match_hidden(match_hidden)
.include("*.ts")
.expect("valid include")
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("walk succeeds");
relative_paths(result.entries(), &fixture.root)
};
assert_eq!(
walk(false),
vec![PathBuf::from("src/visible.ts"), PathBuf::from("visible.ts")],
"crossing reaches deeper, not into hidden names"
);
assert_eq!(
walk(true),
vec![
PathBuf::from(".config/inside.ts"),
PathBuf::from(".hidden.ts"),
PathBuf::from("src/.hidden.ts"),
PathBuf::from("src/visible.ts"),
PathBuf::from("visible.ts"),
],
"match_hidden opens hidden names at every level a crossing wildcard reaches"
);
let skipped = Walker::new(&fixture.root)
.wildcard_mode(WildcardMode::SeparatorCrossing)
.match_hidden(true)
.include("*.ts")
.expect("valid include")
.options(
WalkOptions::default()
.sort(true)
.files_only(true)
.skip_hidden(true),
)
.collect()
.expect("walk succeeds");
assert_eq!(
relative_paths(skipped.entries(), &fixture.root),
vec![PathBuf::from("src/visible.ts"), PathBuf::from("visible.ts")],
"skip_hidden keeps hidden entries away from the matcher under either mode"
);
}
#[test]
fn an_absolute_pattern_means_what_its_relative_spelling_means() {
let fixture = Fixture::new();
fixture.write("src/a.ts");
fixture.write("src/deep/b.ts");
fixture.write("other/c.ts");
let walk = |pattern: &str| -> Vec<PathBuf> {
let result = Walker::new(&fixture.root)
.include(pattern)
.expect("valid include")
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("walk succeeds");
relative_paths(result.entries(), &fixture.root)
};
for (absolute, relative) in [
("/src/*.ts", "src/*.ts"),
("/**/*.ts", "**/*.ts"),
("/src/**", "src/**"),
("/{src,other}/*.ts", "{src,other}/*.ts"),
("//src/*.ts", "src/*.ts"),
("/./src/*.ts", "src/*.ts"),
] {
assert_eq!(
walk(&fixture.absolute(absolute)),
walk(relative),
"absolute {absolute} must select what {relative} selects"
);
}
assert_eq!(
walk(&fixture.absolute("/src/*.ts")),
vec![PathBuf::from("src/a.ts")]
);
}
#[test]
fn an_absolute_pattern_is_read_under_the_walk_s_wildcard_mode() {
let fixture = Fixture::new();
fixture.write("src/a.ts");
fixture.write("src/deep/b.ts");
let walk = |mode: WildcardMode| -> Vec<PathBuf> {
let result = Walker::new(&fixture.root)
.wildcard_mode(mode)
.include(fixture.absolute("/src/*.ts"))
.expect("valid include")
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("walk succeeds");
relative_paths(result.entries(), &fixture.root)
};
assert_eq!(
walk(WildcardMode::ComponentScoped),
vec![PathBuf::from("src/a.ts")]
);
assert_eq!(
walk(WildcardMode::SeparatorCrossing),
vec![PathBuf::from("src/a.ts"), PathBuf::from("src/deep/b.ts")]
);
}
#[test]
fn an_absolute_pattern_outside_the_root_selects_and_prunes_nothing() {
let fixture = Fixture::new();
fixture.write("src/a.ts");
fixture.write("keep/b.ts");
let elsewhere = format!("{}-elsewhere/**", fixture.absolute(""));
let included = Walker::new(&fixture.root)
.include(&elsewhere)
.expect("an unrelated tree is not an error")
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("walk succeeds");
assert!(
relative_paths(included.entries(), &fixture.root).is_empty(),
"an include about another tree selects nothing here"
);
let excluded = Walker::new(&fixture.root)
.exclude(&elsewhere)
.expect("an unrelated tree is not an error")
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("walk succeeds");
assert_eq!(
relative_paths(excluded.entries(), &fixture.root),
vec![PathBuf::from("keep/b.ts"), PathBuf::from("src/a.ts")],
"an exclude about another tree removes nothing here"
);
}
#[test]
fn the_planner_prefilters_still_apply_to_a_rewritten_pattern() {
let fixture = Fixture::new();
let walker = Walker::new(&fixture.root)
.include(fixture.absolute("/src/**/*.ts"))
.expect("valid include");
let pattern = &walker.roots[0].includes[0];
assert_eq!(
pattern.literal_roots,
Some(vec![b"src".to_vec()]),
"the root prefilter survives the rewrite"
);
assert_eq!(
pattern.extensions,
Some(vec![b"ts".to_vec()]),
"the extension prefilter survives the rewrite"
);
assert!(pattern.could_match_descendant(b"src"));
assert!(!pattern.could_match_descendant(b"node_modules"));
assert!(pattern.matches_extension(b"src/app.ts"));
assert!(!pattern.matches_extension(b"src/app.js"));
let outside = Walker::new(&fixture.root)
.include(format!("{}-elsewhere/**/*.ts", fixture.absolute("")))
.expect("an unrelated tree is not an error");
assert!(!outside.roots[0].includes[0].could_match_descendant(b"src"));
assert!(
!outside.roots[0].includes[0].covers_subtree(b"src", WildcardMode::ComponentScoped)
);
}
#[test]
fn an_unprovable_absolute_pattern_is_rejected() {
let fixture = Fixture::new();
let parent = fixture
.root
.parent()
.expect("the fixture root has a parent")
.to_path_buf();
let parent = String::from_utf8(glob_path_bytes(&parent).into_owned())
.expect("the temporary directory is UTF-8 on a test host");
let above = Walker::new(&fixture.root)
.include(format!("{parent}/*/x.ts"))
.expect_err("a wildcard above the root is rejected");
assert!(
above
.message()
.starts_with("a wildcard at or above the walk root"),
);
let dot_dot = Walker::new(&fixture.root)
.include(fixture.absolute("/../x.ts"))
.expect_err("`..` is rejected");
assert!(dot_dot.message().starts_with("`..`"));
let root_itself = Walker::new(&fixture.root)
.exclude(fixture.absolute(""))
.expect_err("naming the root is rejected");
assert!(
root_itself
.message()
.starts_with("an absolute pattern that names the walk root itself")
);
let relative_root = Walker::new("relative/dir")
.include(fixture.absolute("/x.ts"))
.expect_err("a relative root is rejected");
assert_eq!(
relative_root.message(),
"an absolute pattern needs an absolute walk root"
);
}
#[derive(Debug, PartialEq, Eq)]
struct RootedOutcome {
entries: Vec<(PathBuf, PathBuf, usize)>,
errors: Vec<(&'static str, PathBuf)>,
}
impl RootedOutcome {
fn of(result: &super::WalkResult) -> Self {
let mut entries = result
.entries()
.iter()
.map(|entry| {
(
entry.root().to_path_buf(),
entry.path().to_path_buf(),
entry.depth(),
)
})
.collect::<Vec<_>>();
let mut errors = result
.errors()
.iter()
.map(|error| (error.operation(), error.path().to_path_buf()))
.collect::<Vec<_>>();
entries.sort_unstable();
errors.sort_unstable();
Self { entries, errors }
}
fn concatenated(parts: impl IntoIterator<Item = Self>) -> Self {
let mut joined = Self {
entries: Vec::new(),
errors: Vec::new(),
};
for part in parts {
joined.entries.extend(part.entries);
joined.errors.extend(part.errors);
}
joined.entries.sort_unstable();
joined.errors.sort_unstable();
joined
}
}
#[derive(Debug, Clone, Copy)]
enum Frontend {
Collect,
Visit,
Stream,
}
fn multi_root_outcome(
roots: &[PathBuf],
include: Option<&str>,
threads: usize,
frontend: Frontend,
) -> RootedOutcome {
multi_root_outcome_with_following(roots, include, threads, frontend, false)
}
fn multi_root_outcome_with_following(
roots: &[PathBuf],
include: Option<&str>,
threads: usize,
frontend: Frontend,
follow_symlinks: bool,
) -> RootedOutcome {
let (first, rest) = roots.split_first().expect("at least one root");
let mut walker = Walker::new(first).threads(threads).options(
WalkOptions::default()
.files_only(true)
.follow_symlinks(follow_symlinks),
);
for root in rest {
walker = walker.add_root(root).expect("the root takes the patterns");
}
if let Some(pattern) = include {
walker = walker.include(pattern).expect("valid include");
}
match frontend {
Frontend::Collect => {
RootedOutcome::of(&walker.collect().expect("collect walks under Collect"))
}
Frontend::Visit => RootedOutcome::of(
&walker
.visit(|_| Verdict::Keep)
.expect("visit walks under Collect"),
),
Frontend::Stream => {
let mut entries = Vec::new();
let mut errors = Vec::new();
for item in walker.stream() {
match item {
Ok(entry) => entries.push(entry),
Err(error) => errors.push(error),
}
}
RootedOutcome::of(&super::WalkResult {
entries,
errors,
cancelled: false,
})
}
}
}
#[test]
fn a_multi_root_walk_is_the_concatenation_of_the_single_root_walks() {
let fixture = Fixture::new();
fixture.write("alpha/src/one.rs");
fixture.write("alpha/src/deep/two.rs");
fixture.write("alpha/notes.txt");
fixture.write("beta/src/three.rs");
fixture.write("beta/four.txt");
fixture.write("gamma/src/five.rs");
let roots = [
fixture.root.join("alpha"),
fixture.root.join("beta"),
fixture.root.join("gamma"),
fixture.root.join("missing"),
];
for include in [None, Some("src/**/*.rs"), Some("**/*.rs")] {
for threads in [1, 4] {
for frontend in [Frontend::Collect, Frontend::Visit, Frontend::Stream] {
let together = multi_root_outcome(&roots, include, threads, frontend);
let separately = RootedOutcome::concatenated(roots.iter().map(|root| {
multi_root_outcome(std::slice::from_ref(root), include, threads, frontend)
}));
assert_eq!(
together, separately,
"{frontend:?} on {threads} threads with include {include:?}"
);
}
}
}
let all = multi_root_outcome(&roots, None, 4, Frontend::Collect);
assert_eq!(all.entries.len(), 6);
assert_eq!(all.errors.len(), 1);
assert_eq!(all.errors[0].0, "read_dir");
}
#[test]
fn overlapping_roots_deliver_their_overlap_once_per_root() {
let fixture = Fixture::new();
fixture.write("outer/inner/shared.rs");
fixture.write("outer/own.rs");
let outer = fixture.root.join("outer");
let inner = outer.join("inner");
let result = Walker::new(&outer)
.add_root(&inner)
.expect("nested roots are allowed")
.threads(1)
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("walk succeeds");
let shared = inner.join("shared.rs");
let copies = result
.entries()
.iter()
.filter(|entry| entry.path() == shared)
.collect::<Vec<_>>();
assert_eq!(copies.len(), 2, "the overlap is delivered once per root");
let mut seen = copies
.iter()
.map(|entry| (entry.root().to_path_buf(), entry.depth()))
.collect::<Vec<_>>();
seen.sort_unstable();
assert_eq!(
seen,
vec![(outer.clone(), 2), (inner.clone(), 1)],
"each copy is one level below the root it came from"
);
}
#[cfg(unix)]
#[test]
fn following_links_keeps_overlapping_and_duplicate_roots_independent() {
use std::os::unix::fs::symlink;
let fixture = Fixture::new();
fixture.write("outer/own.txt");
fixture.write("outer/inner/shared.txt");
fixture.write("outer/inner/deep/leaf.txt");
symlink("..", fixture.root.join("outer/inner/back")).expect("create cycle");
let outer = fixture.root.join("outer");
let inner = outer.join("inner");
let alias = fixture.root.join("outer-alias");
symlink("outer", &alias).expect("create root alias");
let overlapping = [outer.clone(), inner.clone()];
let duplicate = [outer.clone(), outer.clone()];
let aliases = [outer.clone(), alias];
for roots in [&overlapping[..], &duplicate[..], &aliases[..]] {
for (frontend, threads) in [
(Frontend::Collect, 1),
(Frontend::Collect, 4),
(Frontend::Visit, 1),
(Frontend::Visit, 4),
(Frontend::Stream, 1),
] {
let together =
multi_root_outcome_with_following(roots, None, threads, frontend, true);
let separately = RootedOutcome::concatenated(roots.iter().map(|root| {
multi_root_outcome_with_following(
std::slice::from_ref(root),
None,
threads,
frontend,
true,
)
}));
assert_eq!(
together, separately,
"{frontend:?} with {threads} thread(s) and roots {roots:?}"
);
}
}
}
#[cfg(unix)]
#[test]
fn parallel_following_links_keeps_root_attribution_stable_under_stress() {
use std::os::unix::fs::symlink;
let fixture = Fixture::new();
let outer = fixture.root.join("outer");
let inner = outer.join("inner");
for branch in 0..32 {
for leaf in 0..8 {
fixture.write(format!("outer/inner/branch-{branch}/leaf-{leaf}.txt"));
}
}
symlink("..", inner.join("back")).expect("create cycle");
let roots = [outer, inner];
let expected = RootedOutcome::concatenated(roots.iter().map(|root| {
multi_root_outcome_with_following(
std::slice::from_ref(root),
None,
1,
Frontend::Collect,
true,
)
}));
for run in 0..24 {
assert_eq!(
multi_root_outcome_with_following(&roots, None, 4, Frontend::Collect, true),
expected,
"parallel run {run} changed overlap attribution"
);
}
}
#[test]
fn patterns_are_read_once_per_root() {
let fixture = Fixture::new();
fixture.write("alpha/src/one.rs");
fixture.write("alpha/other/two.rs");
fixture.write("beta/src/three.rs");
fixture.write("beta/other/four.rs");
let alpha = fixture.root.join("alpha");
let beta = fixture.root.join("beta");
let walk = |pattern: String| -> Vec<PathBuf> {
let result = Walker::new(&alpha)
.add_root(&beta)
.expect("the root takes the patterns")
.include(pattern)
.expect("valid include")
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("walk succeeds");
result
.entries()
.iter()
.map(|entry| entry.path().to_path_buf())
.collect()
};
assert_eq!(
walk("src/*.rs".to_owned()),
vec![alpha.join("src/one.rs"), beta.join("src/three.rs")],
);
let alpha_glob = String::from_utf8(glob_path_bytes(&alpha).into_owned())
.expect("the temporary directory is UTF-8 on a test host");
assert_eq!(
walk(format!("{alpha_glob}/src/*.rs")),
vec![alpha.join("src/one.rs")],
);
}
#[test]
fn a_root_and_a_pattern_meet_whichever_order_they_arrive_in() {
let fixture = Fixture::new();
fixture.write("alpha/src/one.rs");
fixture.write("beta/src/two.rs");
let alpha = fixture.root.join("alpha");
let beta = fixture.root.join("beta");
let entries = |walker: Walker| -> Vec<PathBuf> {
walker
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("walk succeeds")
.entries()
.iter()
.map(|entry| entry.path().to_path_buf())
.collect()
};
let root_first = entries(
Walker::new(&alpha)
.add_root(&beta)
.expect("root")
.include("src/*.rs")
.expect("include"),
);
let pattern_first = entries(
Walker::new(&alpha)
.include("src/*.rs")
.expect("include")
.add_root(&beta)
.expect("root"),
);
assert_eq!(root_first, pattern_first);
assert_eq!(root_first.len(), 2);
let unprovable = fixture.absolute("/../x.rs");
assert!(
Walker::new(&alpha)
.add_root(&beta)
.expect("root")
.include(&unprovable)
.is_err()
);
assert!(
Walker::new(&alpha)
.include(&unprovable)
.expect_err("`..` is refused for the first root already")
.message()
.starts_with("`..`")
);
}
#[test]
fn an_unreadable_root_does_not_stop_the_other_roots() {
let fixture = Fixture::new();
fixture.write("alpha/one.rs");
fixture.write("gamma/two.rs");
let missing = fixture.root.join("beta");
for threads in [1, 4] {
let result = Walker::new(fixture.root.join("alpha"))
.add_root(&missing)
.expect("root")
.add_root(fixture.root.join("gamma"))
.expect("root")
.threads(threads)
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("Collect keeps walking");
assert_eq!(
relative_paths(result.entries(), &fixture.root),
vec![PathBuf::from("alpha/one.rs"), PathBuf::from("gamma/two.rs")],
"the readable roots are walked on {threads} threads"
);
assert_eq!(result.errors().len(), 1);
assert_eq!(result.errors()[0].operation(), "read_dir");
assert_eq!(result.errors()[0].path(), missing);
}
}
#[test]
fn skip_reports_a_failed_root_while_walking_the_other_roots() {
let fixture = Fixture::new();
fixture.write("alpha/one.rs");
fixture.write("gamma/two.rs");
let missing = fixture.root.join("beta");
let build = || {
Walker::new(fixture.root.join("alpha"))
.add_root(&missing)
.expect("root")
.add_root(fixture.root.join("gamma"))
.expect("root")
.error_policy(ErrorPolicy::Skip)
.options(WalkOptions::default().sort(true).files_only(true))
};
for threads in [1, 4] {
let result = build()
.threads(threads)
.collect()
.expect("Skip preserves the other roots");
assert_eq!(
relative_paths(result.entries(), &fixture.root),
vec![PathBuf::from("alpha/one.rs"), PathBuf::from("gamma/two.rs")]
);
assert_eq!(result.errors().len(), 1);
assert_eq!(result.errors()[0].path(), missing);
}
let mut stream = build().stream();
let mut entries = Vec::new();
let mut errors = Vec::new();
for item in &mut stream {
match item {
Ok(entry) => entries.push(entry),
Err(error) => errors.push(error),
}
}
entries.sort_by(|left, right| left.path().cmp(right.path()));
assert_eq!(
relative_paths(&entries, &fixture.root),
vec![PathBuf::from("alpha/one.rs"), PathBuf::from("gamma/two.rs")]
);
assert_eq!(errors.len(), 1);
assert_eq!(errors[0].path(), missing);
}
#[cfg(windows)]
#[test]
fn a_windows_path_handed_over_as_a_pattern_is_refused() {
let fixture = Fixture::new();
fixture.write("src/main.ts");
fixture.write("src/deep/other.ts");
let root = fixture.root.display().to_string();
for pattern in [
format!(r"{root}\src\**\*.ts"),
format!(r"{root}\src\*.ts"),
r"src\*.ts".to_owned(),
r"src\**\*.ts".to_owned(),
] {
let refused = Walker::new(&fixture.root)
.include(&pattern)
.expect_err("a path spelled as a pattern is refused");
assert!(
refused
.message()
.starts_with("this looks like a Windows path"),
"{pattern:?} reported {refused}"
);
assert!(Walker::new(&fixture.root).exclude(&pattern).is_err());
}
for pattern in [
"src/**/*.ts",
&format!("{}/src/**/*.ts", root.replace('\\', "/")),
] {
let result = Walker::new(&fixture.root)
.include(pattern)
.expect("valid include")
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("walk succeeds");
assert_eq!(result.entries().len(), 2, "{pattern} selects both files");
}
for pattern in [
r"src/[m\*]ain.ts",
r"src/{main,\*}.ts",
r"src/@(main|\*).ts",
] {
let result = Walker::new(&fixture.root)
.include(pattern)
.unwrap_or_else(|error| panic!("{pattern} can match, got {error}"))
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("walk succeeds");
assert_eq!(
relative_paths(result.entries(), &fixture.root),
vec![PathBuf::from("src/main.ts")],
"{pattern} selects through its group"
);
}
}
#[test]
fn subtree_pruning_agrees_with_the_exclude_it_came_from() {
let fixture = Fixture::new();
fixture.write("a/b.tmp/keep.rs");
fixture.write("b.tmp/gone.rs");
fixture.write("a/plain/keep.rs");
let walk = |mode: WildcardMode| -> Vec<PathBuf> {
let result = Walker::new(&fixture.root)
.wildcard_mode(mode)
.exclude("*.tmp/**")
.expect("valid exclude")
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("walk succeeds");
relative_paths(result.entries(), &fixture.root)
};
assert_eq!(
walk(WildcardMode::ComponentScoped),
vec![
PathBuf::from("a/b.tmp/keep.rs"),
PathBuf::from("a/plain/keep.rs"),
],
"a nested `.tmp` directory is out of a component-scoped exclude's reach"
);
assert_eq!(
walk(WildcardMode::SeparatorCrossing),
vec![PathBuf::from("a/plain/keep.rs")],
"a crossing exclude reaches the nested one, and pruning may follow it"
);
}
#[test]
fn excludes_keep_descending_for_an_explicitly_included_descendant() {
let fixture = Fixture::new();
fixture.write("a/keep.txt");
fixture.write("a/drop.txt");
let result = Walker::new(&fixture.root)
.include("a/keep.txt")
.expect("valid include")
.exclude("a")
.expect("valid exclude")
.options(WalkOptions::default().files_only(true).sort(true))
.collect()
.expect("walk succeeds");
assert_eq!(
relative_paths(result.entries(), &fixture.root),
[PathBuf::from("a/keep.txt")]
);
}
#[test]
fn a_partial_component_literal_does_not_prune_its_siblings() {
let fixture = Fixture::new();
fixture.write("src/x.ts");
fixture.write("srcfoo/x.ts");
fixture.write("other/x.ts");
for mode in [
WildcardMode::ComponentScoped,
WildcardMode::SeparatorCrossing,
] {
let result = Walker::new(&fixture.root)
.wildcard_mode(mode)
.include("src*/x.ts")
.expect("valid include")
.options(WalkOptions::default().sort(true).files_only(true))
.collect()
.expect("walk succeeds");
assert_eq!(
relative_paths(result.entries(), &fixture.root),
vec![PathBuf::from("src/x.ts"), PathBuf::from("srcfoo/x.ts")],
"{mode:?}: a partial-component literal must not prune a sibling it matches"
);
}
assert_eq!(
literal_pattern_root(b"src/*.ts"),
Some(b"src".to_vec()),
"a complete component is still a root"
);
assert_eq!(
literal_pattern_root(b"src*/x.ts"),
None,
"a partial component is not"
);
assert_eq!(
literal_pattern_root(b"docs/api*/x.ts"),
Some(b"docs".to_vec()),
"what precedes the last separator is still proven"
);
}
#[test]
fn prune_planner_only_accepts_explicit_whole_subtree_excludes() {
let scoped = WildcardMode::ComponentScoped;
let subtree = traversal_pattern(b"src/**");
assert!(subtree.covers_subtree(b"src", scoped));
assert!(!subtree.covers_subtree(b"src/nested", scoped));
let suffix = traversal_pattern(b"*.tmp");
assert!(!suffix.covers_subtree(b"cache", scoped));
let nested = traversal_pattern(b"**/target/**");
assert!(nested.covers_subtree(b"target", scoped));
assert!(nested.covers_subtree(b"crates/ferralk/target", scoped));
let wildcard_subtree = traversal_pattern(b"*.tmp/**");
assert!(!wildcard_subtree.covers_subtree(b"a/b.tmp", scoped));
assert!(wildcard_subtree.covers_subtree(b"b.tmp", scoped));
assert!(wildcard_subtree.covers_subtree(b"a/b.tmp", WildcardMode::SeparatorCrossing));
assert_eq!(
literal_pattern_root(b"src/foo/*.rs"),
Some(b"src/foo".to_vec())
);
assert_eq!(literal_pattern_root(b"src/foo*.rs"), Some(b"src".to_vec()));
assert_eq!(literal_pattern_root(b"**/*.rs"), None);
assert_eq!(
literal_pattern_root(b"foo+bar/**/*.rs"),
Some(b"foo+bar".to_vec())
);
assert_eq!(
literal_pattern_root(b"foo@(bar/**/*.rs"),
Some(b"foo@(bar".to_vec())
);
let rust_sources = traversal_pattern(b"src/**/*.rs");
assert!(rust_sources.matches_extension(b"src/lib.rs"));
assert!(!rust_sources.matches_extension(b"src/lib.txt"));
assert_eq!(literal_extension(b"src/**/*.{rs,ts}"), None);
assert_eq!(literal_extension(b"src/**/*.rs"), Some(b"rs".to_vec()));
}
#[test]
fn a_visitor_skip_drops_the_entry_without_pruning_the_subtree() {
let fixture = Fixture::new();
fixture.write("keep/inside.txt");
for threads in [1, 4] {
let result = Walker::new(&fixture.root)
.threads(threads)
.visit(|entry| {
if entry.path().file_name().is_some_and(|name| name == "keep") {
Verdict::Skip
} else {
Verdict::Keep
}
})
.expect("visited walk succeeds");
let paths = relative_paths(result.entries(), &fixture.root);
assert_eq!(
paths,
vec![PathBuf::from("keep/inside.txt")],
"the skipped directory must still have been descended into"
);
assert!(!result.was_cancelled());
}
}
#[test]
fn a_visitor_stop_ends_the_walk_and_is_reported() {
let fixture = Fixture::new();
for index in 0..64 {
fixture.write(format!("file-{index}.txt"));
}
for threads in [1, 4] {
let seen = AtomicUsize::new(0);
let result = Walker::new(&fixture.root)
.threads(threads)
.visit(|_| {
if seen.fetch_add(1, Ordering::AcqRel) >= 8 {
Verdict::Stop
} else {
Verdict::Keep
}
})
.expect("visited walk succeeds");
assert!(
result.was_cancelled(),
"a stop must be reported the way a cancellation is"
);
assert!(result.entries().len() <= 64);
}
}
#[test]
fn a_visitor_stop_leaves_a_caller_owned_cancellation_token_alone() {
let fixture = Fixture::new();
fixture.write("only.txt");
let cancellation = CancellationToken::default();
for threads in [1, 4] {
let result = Walker::new(&fixture.root)
.threads(threads)
.cancellation(cancellation.clone())
.visit(|_| Verdict::Stop)
.expect("visited walk succeeds");
assert!(result.was_cancelled());
assert!(
!cancellation.is_cancelled(),
"a stop must not cancel the caller's token"
);
}
}
#[test]
fn a_visitor_panic_is_resumed_on_the_caller() {
let fixture = Fixture::new();
for branch in 0..12 {
fixture.write(format!("branch-{branch}/file.txt"));
}
for threads in [1, 4] {
let root = fixture.root.clone();
let panicked = std::panic::catch_unwind(move || {
let _ = Walker::new(&root)
.threads(threads)
.visit(|_| panic!("visitor panic"));
});
assert!(
panicked.is_err(),
"a panic inside the visitor must reach the caller on {threads} threads"
);
}
}
#[test]
fn a_small_tree_stays_on_one_thread() {
let fixture = Fixture::new();
for index in 0..12 {
fixture.write(format!("one/file-{index}.txt"));
}
fixture.write("two/only.txt");
let threads = Mutex::new(HashSet::new());
let result = Walker::new(&fixture.root)
.threads(4)
.visit(|_| {
threads
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(std::thread::current().id());
Verdict::Keep
})
.expect("visited walk succeeds");
let observed = threads
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len();
assert_eq!(
observed, 1,
"a tree below the size floor must not pay for helper threads"
);
assert_eq!(result.entries().len(), 2 + 13);
}
#[test]
fn the_three_frontends_agree_on_entries_and_errors() {
let fixture = Fixture::new();
fixture.write("src/lib.rs");
fixture.write("src/nested/mod.rs");
fixture.write("docs/guide.md");
fixture.write("docs/notes/todo.md");
fs::create_dir_all(fixture.root.join("empty")).expect("create empty fixture directory");
assert_frontends_agree("plain tree", &fixture.root, || Walker::new(&fixture.root));
assert_frontends_agree("with metadata", &fixture.root, || {
Walker::new(&fixture.root).options(WalkOptions::default().metadata(true))
});
assert_frontends_agree("include filtered", &fixture.root, || {
Walker::new(&fixture.root)
.options(WalkOptions::default().metadata(true))
.include("**/*.md")
.expect("valid include")
});
assert_frontends_agree("directories only", &fixture.root, || {
Walker::new(&fixture.root).options(
WalkOptions::default()
.metadata(true)
.directories_only(true)
.max_depth(2),
)
});
}
#[cfg(unix)]
#[test]
fn the_three_frontends_agree_when_stat_fails() {
use std::os::unix::fs::symlink;
let fixture = Fixture::new();
fixture.write("src/lib.rs");
fixture.write("docs/guide.md");
symlink("missing-target", fixture.root.join("src/dangling"))
.expect("create dangling symlink");
symlink("missing-target", fixture.root.join("docs/dangling"))
.expect("create second dangling symlink");
assert_frontends_agree("dangling symlinks", &fixture.root, || {
Walker::new(&fixture.root).options(
WalkOptions::default()
.metadata(true)
.follow_symlinks(true)
.sort(true),
)
});
}
#[cfg(unix)]
#[test]
fn a_filtered_entry_is_never_stat_ed() {
use std::os::unix::fs::PermissionsExt;
let fixture = Fixture::new();
fixture.write("keep/note.md");
fixture.write("blocked/hidden.txt");
let blocked = fixture.root.join("blocked");
fs::set_permissions(&blocked, fs::Permissions::from_mode(0o400))
.expect("restrict fixture directory");
let listable = fs::read_dir(&blocked).is_ok_and(|entries| {
entries
.into_iter()
.all(|entry| entry.is_ok_and(|entry| entry.file_type().is_ok()))
});
let stat_fails = fs::symlink_metadata(blocked.join("hidden.txt")).is_err();
let outcomes = (listable && stat_fails).then(|| {
let directories_only = || {
Walker::new(&fixture.root)
.options(WalkOptions::default().metadata(true).directories_only(true))
};
let included = || {
Walker::new(&fixture.root)
.options(WalkOptions::default().metadata(true))
.include("**/keep/**")
.expect("valid include")
};
assert_frontends_agree(
"unreadable file, directories only",
&fixture.root,
directories_only,
);
assert_frontends_agree("unreadable file, not included", &fixture.root, included);
(
collect_outcome(directories_only().threads(1).collect(), &fixture.root),
collect_outcome(included().threads(1).collect(), &fixture.root),
)
});
fs::set_permissions(&blocked, fs::Permissions::from_mode(0o700))
.expect("restore fixture directory");
let Some((directories_only, included)) = outcomes else {
return;
};
assert_eq!(
directories_only,
FrontendOutcome::Completed {
entries: vec![PathBuf::from("blocked"), PathBuf::from("keep")],
errors: Vec::new(),
},
"a file dropped by directories_only must not be stat-ed"
);
assert_eq!(
included,
FrontendOutcome::Completed {
entries: vec![PathBuf::from("keep"), PathBuf::from("keep/note.md")],
errors: Vec::new(),
},
"an entry dropped by the include patterns must not be stat-ed"
);
}
#[test]
fn a_mock_backend_drives_the_parallel_walker() {
struct InjectingBackend {
failing_stat: PathBuf,
reads: std::sync::Mutex<Vec<PathBuf>>,
}
impl super::DirectoryBackend for InjectingBackend {
fn read_directory(
&self,
path: &Path,
follow_symlinks: bool,
refuse_final_symlink: bool,
listing: &mut super::Listing,
) -> std::io::Result<()> {
self.reads
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.push(path.to_path_buf());
super::StdBackend.read_directory(
path,
follow_symlinks,
refuse_final_symlink,
listing,
)
}
fn symlink_metadata(&self, path: &Path) -> std::io::Result<fs::Metadata> {
if path == self.failing_stat {
return Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied));
}
fs::symlink_metadata(path)
}
}
let fixture = Fixture::new();
fixture.write("src/lib.rs");
fixture.write("src/nested/mod.rs");
fixture.write("docs/guide.md");
let failing_stat = fixture.root.join("src/nested/mod.rs");
let backend = InjectingBackend {
failing_stat: failing_stat.clone(),
reads: std::sync::Mutex::new(Vec::new()),
};
let result = Walker::new(&fixture.root)
.threads(4)
.options(WalkOptions::default().metadata(true).sort(true))
.collect_with(&backend)
.expect("collect policy retains the injected error");
let reads = backend
.reads
.into_inner()
.unwrap_or_else(|poisoned| poisoned.into_inner());
assert!(
reads.contains(&fixture.root.join("src")),
"the parallel walker has to read through the injected backend"
);
assert_eq!(result.errors().len(), 1);
assert_eq!(result.errors()[0].operation(), "symlink_metadata");
assert_eq!(result.errors()[0].path(), failing_stat);
assert!(
!result
.entries()
.iter()
.any(|entry| entry.path() == failing_stat)
);
let backend = InjectingBackend {
failing_stat: failing_stat.clone(),
reads: std::sync::Mutex::new(Vec::new()),
};
let filtered = Walker::new(&fixture.root)
.threads(4)
.options(
WalkOptions::default()
.metadata(true)
.directories_only(true)
.sort(true),
)
.collect_with(&backend)
.expect("filtered walk succeeds");
assert!(
filtered.errors().is_empty(),
"a filtered entry must not be stat-ed on the parallel path either"
);
assert_eq!(
relative_paths(filtered.entries(), &fixture.root),
vec![
PathBuf::from("docs"),
PathBuf::from("src"),
PathBuf::from("src/nested")
]
);
}
#[test]
fn deferred_entry_stat_failures_keep_siblings_and_report_permissions() {
struct UnknownTypeBackend {
root: PathBuf,
failure: std::io::ErrorKind,
}
impl super::DirectoryBackend for UnknownTypeBackend {
fn read_directory(
&self,
path: &Path,
_follow_symlinks: bool,
_refuse_final_symlink: bool,
listing: &mut super::Listing,
) -> std::io::Result<()> {
listing.clear();
if path == self.root {
listing.push("before".as_ref(), false, false);
super::defer_entry_stat_error(
listing,
self.root.join("unknown"),
std::io::Error::from(self.failure),
)?;
listing.push("after".as_ref(), false, false);
}
Ok(())
}
}
let fixture = Fixture::new();
let permission = UnknownTypeBackend {
root: fixture.root.clone(),
failure: std::io::ErrorKind::PermissionDenied,
};
let walker = Walker::new(&fixture.root)
.threads(1)
.error_policy(ErrorPolicy::Collect);
let mut state = super::WalkState::new(&walker, &super::keep_every_entry);
state
.walk_directory(
&permission,
directory_task(&walker, &permission, fixture.root.clone()),
)
.expect("collect keeps the usable listing");
assert_eq!(
relative_paths(&state.entries, &fixture.root),
[PathBuf::from("before"), PathBuf::from("after")]
);
assert_eq!(state.errors.len(), 1);
assert_eq!(state.errors[0].operation(), "read_dir");
assert_eq!(state.errors[0].path(), fixture.root.join("unknown"));
assert_eq!(
state.errors[0].source.kind(),
std::io::ErrorKind::PermissionDenied
);
let parallel = UnknownTypeBackend {
root: fixture.root.clone(),
failure: std::io::ErrorKind::PermissionDenied,
};
let result = Walker::new(&fixture.root)
.threads(4)
.error_policy(ErrorPolicy::Collect)
.collect_with(¶llel)
.expect("parallel collect keeps the usable listing");
assert_eq!(
relative_paths(result.entries(), &fixture.root),
[PathBuf::from("before"), PathBuf::from("after")]
);
assert_eq!(result.errors().len(), 1);
assert_eq!(result.errors()[0].path(), fixture.root.join("unknown"));
for race in [
std::io::ErrorKind::NotFound,
std::io::ErrorKind::NotADirectory,
] {
let backend = UnknownTypeBackend {
root: fixture.root.clone(),
failure: race,
};
let mut state = super::WalkState::new(&walker, &super::keep_every_entry);
state
.walk_directory(
&backend,
directory_task(&walker, &backend, fixture.root.clone()),
)
.expect("a changed entry costs only that entry");
assert_eq!(
relative_paths(&state.entries, &fixture.root),
[PathBuf::from("before"), PathBuf::from("after")]
);
assert!(state.errors.is_empty(), "{race:?} is a replacement race");
}
}
#[test]
fn stream_delivers_deferred_entry_errors_after_listing_siblings() {
let fixture = Fixture::new();
let mut stream = Walker::new(&fixture.root)
.error_policy(ErrorPolicy::Collect)
.stream();
stream.pending_directories.clear();
stream.directory = fixture.root.clone();
stream.path = fixture.root.clone();
stream.listing.push("before".as_ref(), false, false);
super::defer_entry_stat_error(
&mut stream.listing,
fixture.root.join("unknown"),
std::io::Error::from(std::io::ErrorKind::PermissionDenied),
)
.expect("permission failure is deferred");
stream.listing.push("after".as_ref(), false, false);
let delivered = stream
.map(|item| match item {
Ok(entry) => Ok(entry
.path()
.strip_prefix(&fixture.root)
.expect("entry belongs to fixture")
.to_path_buf()),
Err(error) => Err((
error.operation(),
error.path().to_path_buf(),
error.source.kind(),
)),
})
.collect::<Vec<_>>();
assert_eq!(
delivered,
vec![
Ok(PathBuf::from("before")),
Ok(PathBuf::from("after")),
Err((
"read_dir",
fixture.root.join("unknown"),
std::io::ErrorKind::PermissionDenied,
)),
],
"the stream must not let one deferred entry failure hide its siblings"
);
}
#[test]
fn literal_include_roots_prune_unrelated_sibling_directories() {
type MockDirectory = Vec<(&'static str, bool)>;
struct RecordingBackend {
entries: HashMap<PathBuf, MockDirectory>,
reads: RefCell<Vec<PathBuf>>,
}
impl super::DirectoryBackend for RecordingBackend {
fn read_directory(
&self,
path: &Path,
_follow_symlinks: bool,
_refuse_final_symlink: bool,
listing: &mut super::Listing,
) -> std::io::Result<()> {
self.reads.borrow_mut().push(path.to_path_buf());
listing.clear();
for &(name, is_dir) in self.entries.get(path).into_iter().flatten() {
listing.push(name.as_ref(), is_dir, false);
}
Ok(())
}
}
let root = PathBuf::from("/fixture");
let source = root.join("src");
let mut entries = HashMap::new();
entries.insert(root.clone(), vec![("src", true), ("docs", true)]);
entries.insert(source.clone(), vec![("main.rs", false)]);
let backend = RecordingBackend {
entries,
reads: RefCell::new(Vec::new()),
};
let walker = Walker::new(&root)
.include("src/**/*.rs")
.expect("valid include");
let mut state = super::WalkState::new(&walker, &super::keep_every_entry);
state
.walk_directory(&backend, directory_task(&walker, &backend, root.clone()))
.expect("backend walk succeeds");
assert_eq!(backend.reads.into_inner(), vec![root, source]);
assert_eq!(state.entries.len(), 1);
assert_eq!(state.entries[0].path(), Path::new("/fixture/src/main.rs"));
}
#[test]
fn metadata_error_is_retained_when_a_dirent_disappears_before_stat() {
struct DisappearingFileBackend {
root: PathBuf,
disappeared: PathBuf,
}
impl super::DirectoryBackend for DisappearingFileBackend {
fn read_directory(
&self,
path: &Path,
_follow_symlinks: bool,
_refuse_final_symlink: bool,
listing: &mut super::Listing,
) -> std::io::Result<()> {
listing.clear();
if path == self.root {
let name = self
.disappeared
.file_name()
.expect("the disappearing entry has a name");
listing.push(name, false, false);
}
Ok(())
}
}
let fixture = Fixture::new();
let disappeared = fixture.root.join("gone.rs");
let walker = Walker::new(&fixture.root)
.threads(1)
.options(WalkOptions::default().metadata(true));
let backend = DisappearingFileBackend {
root: fixture.root.clone(),
disappeared: disappeared.clone(),
};
let mut state = super::WalkState::new(&walker, &super::keep_every_entry);
state
.walk_directory(
&backend,
directory_task(&walker, &backend, fixture.root.clone()),
)
.expect("collect policy retains the metadata error");
assert!(state.entries.is_empty());
assert_eq!(state.errors.len(), 1);
assert_eq!(state.errors[0].operation(), "symlink_metadata");
assert_eq!(state.errors[0].path(), disappeared);
}
#[test]
fn an_unreadable_symlink_target_is_reported_rather_than_silently_dropped() {
struct UnreadableLinkBackend {
root: PathBuf,
kind: std::io::ErrorKind,
}
impl super::DirectoryBackend for UnreadableLinkBackend {
fn read_directory(
&self,
path: &Path,
_follow_symlinks: bool,
_refuse_final_symlink: bool,
listing: &mut super::Listing,
) -> std::io::Result<()> {
listing.clear();
if path == self.root {
listing.push("link".as_ref(), false, true);
}
Ok(())
}
fn metadata(&self, _path: &Path) -> std::io::Result<fs::Metadata> {
Err(std::io::Error::from(self.kind))
}
}
let fixture = Fixture::new();
let walker = Walker::new(&fixture.root).threads(1).options(
WalkOptions::default()
.files_only(true)
.resolve_symlink_kind(true),
);
let backend = UnreadableLinkBackend {
root: fixture.root.clone(),
kind: std::io::ErrorKind::NotFound,
};
let mut state = super::WalkState::new(&walker, &super::keep_every_entry);
state
.walk_directory(
&backend,
directory_task(&walker, &backend, fixture.root.clone()),
)
.expect("a broken link does not end the walk");
assert!(state.entries.is_empty(), "a broken link is not a file");
assert!(state.errors.is_empty(), "a broken link is not an error");
let backend = UnreadableLinkBackend {
root: fixture.root.clone(),
kind: std::io::ErrorKind::PermissionDenied,
};
let mut state = super::WalkState::new(&walker, &super::keep_every_entry);
state
.walk_directory(
&backend,
directory_task(&walker, &backend, fixture.root.clone()),
)
.expect("the collect policy retains the metadata error");
assert!(state.entries.is_empty());
assert_eq!(state.errors.len(), 1);
assert_eq!(state.errors[0].operation(), "metadata");
assert_eq!(state.errors[0].path(), fixture.root.join("link"));
}
#[test]
fn resolving_stats_only_symlinks_that_a_kind_filter_asks_about() {
struct CountingLinkBackend {
root: PathBuf,
stats: std::cell::Cell<usize>,
}
impl super::DirectoryBackend for CountingLinkBackend {
fn read_directory(
&self,
path: &Path,
_follow_symlinks: bool,
_refuse_final_symlink: bool,
listing: &mut super::Listing,
) -> std::io::Result<()> {
listing.clear();
if path == self.root {
listing.push("plain.txt".as_ref(), false, false);
listing.push("link".as_ref(), false, true);
}
Ok(())
}
fn metadata(&self, _path: &Path) -> std::io::Result<fs::Metadata> {
self.stats.set(self.stats.get() + 1);
fs::metadata(&self.root)
}
}
let count = |options: WalkOptions| {
let fixture = Fixture::new();
let walker = Walker::new(&fixture.root).threads(1).options(options);
let backend = CountingLinkBackend {
root: fixture.root.clone(),
stats: std::cell::Cell::new(0),
};
let mut state = super::WalkState::new(&walker, &super::keep_every_entry);
state
.walk_directory(
&backend,
directory_task(&walker, &backend, fixture.root.clone()),
)
.expect("walk succeeds");
backend.stats.get()
};
assert_eq!(
count(WalkOptions::default().resolve_symlink_kind(true)),
0,
"no kind filter is asking, so there is nothing to resolve"
);
assert_eq!(
count(WalkOptions::default().files_only(true)),
0,
"the option is off"
);
assert_eq!(
count(
WalkOptions::default()
.files_only(true)
.resolve_symlink_kind(true)
),
1,
"one stat, for the symlink only"
);
}
#[cfg(any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
))]
#[test]
fn native_backend_matches_portable_across_walker_option_matrix() {
use std::os::unix::fs::symlink;
let fixture = Fixture::new();
fixture.write("src/lib.rs");
fixture.write("src/nested/mod.rs");
fixture.write("src/generated.tmp");
fixture.write("ignored/skip.rs");
fixture.write(".hidden/skip.rs");
for index in 0..192 {
fixture.write(format!("many/{index:03}-{}", "x".repeat(180)));
}
fixture.write(".gitignore");
fs::write(fixture.root.join(".gitignore"), b"ignored/\n").expect("write ignore rule");
symlink("src", fixture.root.join("source-link")).expect("create directory symlink");
symlink("missing-target", fixture.root.join("dangling-link"))
.expect("create dangling symlink");
let cases = [
("baseline", WalkOptions::default().sort(true)),
("metadata", WalkOptions::default().sort(true).metadata(true)),
(
"directories_only",
WalkOptions::default().sort(true).directories_only(true),
),
(
"files_only",
WalkOptions::default().sort(true).files_only(true),
),
(
"skip_hidden",
WalkOptions::default().sort(true).skip_hidden(true),
),
(
"follow_symlinks",
WalkOptions::default().sort(true).follow_symlinks(true),
),
("max_depth", WalkOptions::default().sort(true).max_depth(1)),
];
for (name, options) in cases {
let walker = Walker::new(&fixture.root)
.threads(1)
.include("**/*")
.expect("valid include")
.exclude("**/*.tmp")
.expect("valid exclude")
.respect_git_ignore(true)
.error_policy(ErrorPolicy::Collect)
.options(options);
let native = walker.clone().collect().expect("native walk succeeds");
let (portable_entries, portable_errors) = collect_with_portable_backend(&walker);
assert_eq!(
describe_entries(native.entries(), &fixture.root),
describe_entries(&portable_entries, &fixture.root),
"native {name} differs from portable"
);
assert_eq!(
describe_errors(native.errors(), &fixture.root),
describe_errors(&portable_errors, &fixture.root),
"native {name} errors differ from portable"
);
if name == "follow_symlinks" {
assert_eq!(
describe_errors(native.errors(), &fixture.root),
vec![(PathBuf::from("dangling-link"), "metadata")]
);
} else {
assert!(native.errors().is_empty(), "native {name} errors");
}
}
}
#[cfg(any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
))]
#[test]
fn native_backend_matches_portable_unreadable_directory_error() {
use std::os::unix::fs::{MetadataExt, PermissionsExt};
let fixture = Fixture::new();
if fs::metadata(&fixture.root)
.expect("fixture root metadata")
.uid()
== 0
{
return;
}
fixture.write("visible.rs");
fixture.write("locked/secret.rs");
let locked = fixture.root.join("locked");
let original_permissions = fs::metadata(&locked)
.expect("locked directory metadata")
.permissions();
fs::set_permissions(&locked, fs::Permissions::from_mode(0o0))
.expect("make locked directory unreadable");
let walker = Walker::new(&fixture.root)
.threads(1)
.error_policy(ErrorPolicy::Collect)
.options(WalkOptions::default().sort(true));
let native = walker.clone().collect().expect("native walk succeeds");
let (portable_entries, portable_errors) = collect_with_portable_backend(&walker);
fs::set_permissions(&locked, original_permissions)
.expect("restore locked directory permissions");
assert_eq!(
describe_entries(native.entries(), &fixture.root),
describe_entries(&portable_entries, &fixture.root),
"native unreadable-directory entries differ from portable"
);
assert_eq!(
describe_errors(native.errors(), &fixture.root),
describe_errors(&portable_errors, &fixture.root),
"native unreadable-directory errors differ from portable"
);
assert_eq!(
describe_errors(native.errors(), &fixture.root),
vec![(PathBuf::from("locked"), "read_dir")]
);
}
#[cfg(any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
))]
type DescribedEntry = (PathBuf, bool, bool, usize, Option<(u64, bool, bool)>);
#[cfg(any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
))]
fn collect_with_portable_backend(walker: &Walker) -> (Vec<WalkEntry>, Vec<super::WalkError>) {
let mut state = super::WalkState::new(walker, &super::keep_every_entry);
state
.walk_directory(
&super::StdBackend,
directory_task(
walker,
&super::StdBackend,
walker
.roots()
.next()
.expect("a walk has a root")
.to_path_buf(),
),
)
.expect("portable walk succeeds");
if walker.options.sort {
state
.entries
.sort_by(|left, right| left.path.cmp(&right.path));
}
(state.entries, state.errors)
}
#[cfg(any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
))]
fn describe_entries(entries: &[WalkEntry], root: &Path) -> Vec<DescribedEntry> {
entries
.iter()
.map(|entry| {
(
entry
.path()
.strip_prefix(root)
.expect("entry belongs to fixture")
.to_path_buf(),
entry.is_dir(),
entry.is_symlink(),
entry.depth(),
entry.metadata().map(|metadata| {
(
metadata.len(),
metadata.file_type().is_dir(),
metadata.file_type().is_symlink(),
)
}),
)
})
.collect()
}
#[cfg(any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
))]
fn describe_errors(errors: &[super::WalkError], root: &Path) -> Vec<(PathBuf, &'static str)> {
errors
.iter()
.map(|error| {
(
error
.path()
.strip_prefix(root)
.expect("error belongs to fixture")
.to_path_buf(),
error.operation(),
)
})
.collect()
}
#[test]
fn a_failed_root_is_reported_under_every_error_policy() {
let missing = std::env::temp_dir().join(format!(
"ferralk-missing-{}",
NEXT_FIXTURE.fetch_add(1, Ordering::Relaxed)
));
let collected = Walker::new(&missing)
.error_policy(ErrorPolicy::Collect)
.collect()
.expect("collect policy retains the error");
assert_eq!(collected.errors().len(), 1);
for threads in [1, 4] {
let skipped = Walker::new(&missing)
.threads(threads)
.error_policy(ErrorPolicy::Skip)
.collect()
.expect("Skip keeps walking after a caller-supplied root failure");
assert_eq!(skipped.errors().len(), 1);
assert_eq!(skipped.errors()[0].operation(), "read_dir");
assert_eq!(skipped.errors()[0].path(), missing);
}
let streamed = Walker::new(&missing)
.error_policy(ErrorPolicy::Skip)
.stream()
.collect::<Result<Vec<_>, _>>()
.expect_err("stream reports a failed caller-supplied root under Skip");
assert_eq!(streamed.operation(), "read_dir");
assert_eq!(streamed.path(), missing);
assert!(
Walker::new(&missing)
.error_policy(ErrorPolicy::Abort)
.collect()
.is_err()
);
}
#[test]
fn a_plain_file_root_reports_not_a_directory_without_emitting_it() {
let fixture = Fixture::new();
let file = fixture.root.join("not-a-directory.txt");
fs::write(&file, b"fixture").expect("write file root");
for threads in [1, 4] {
let result = Walker::new(&file)
.threads(threads)
.error_policy(ErrorPolicy::Skip)
.collect()
.expect("root error is collected while the walk completes");
assert!(result.entries().is_empty());
assert_eq!(result.errors().len(), 1);
assert_eq!(result.errors()[0].operation(), "read_dir");
assert_eq!(result.errors()[0].path(), file);
}
}
#[cfg(unix)]
#[test]
fn a_directory_symlink_root_is_traversed_even_without_following_descendants() {
use std::os::unix::fs::symlink;
let fixture = Fixture::new();
fixture.write("real/inside.txt");
let link = fixture.root.join("linked-root");
symlink("real", &link).expect("create directory symlink root");
for threads in [1, 4] {
for follow_symlinks in [false, true] {
let result = Walker::new(&link)
.threads(threads)
.options(WalkOptions::default().follow_symlinks(follow_symlinks))
.collect()
.expect("directory symlink root is opened");
assert_eq!(
relative_paths(result.entries(), &link),
vec![PathBuf::from("inside.txt")]
);
}
}
}
#[test]
fn cancellation_returns_a_partial_result_without_an_io_error() {
let fixture = Fixture::new();
fixture.write("src/main.rs");
let cancellation = CancellationToken::default();
cancellation.cancel();
let result = Walker::new(&fixture.root)
.cancellation(cancellation)
.collect()
.expect("cancellation is a normal partial result");
assert!(result.was_cancelled());
assert!(result.entries().is_empty());
assert!(result.errors().is_empty());
}
#[test]
fn stream_yields_filtered_entries_incrementally_and_honours_cancellation() {
let fixture = Fixture::new();
fixture.write("src/main.rs");
fixture.write("src/lib.txt");
let mut stream = Walker::new(&fixture.root)
.include("**/*.rs")
.expect("valid include")
.stream();
let entries = stream
.by_ref()
.map(|entry| entry.expect("fixture has no I/O errors"))
.collect::<Vec<_>>();
assert_eq!(
relative_paths(&entries, &fixture.root),
vec![PathBuf::from("src/main.rs")]
);
assert!(!stream.was_cancelled());
let cancellation = CancellationToken::default();
cancellation.cancel();
let mut cancelled = Walker::new(&fixture.root)
.cancellation(cancellation)
.stream();
assert!(cancelled.next().is_none());
assert!(cancelled.was_cancelled());
}
#[test]
fn path_bytes_bridge_entries_to_byte_first_matchers() {
let fixture = Fixture::new();
fixture.write("src/main.rs");
let result = Walker::new(&fixture.root)
.collect()
.expect("fixture has no I/O errors");
let entry = result
.entries()
.iter()
.find(|entry| entry.path().ends_with("src/main.rs"))
.expect("walk reports the fixture file");
let matcher = Pattern::compile(
"**/main.rs",
PatternOptions::default().recursive_double_star(true),
)
.expect("valid pattern");
assert_eq!(
entry.path_bytes(),
entry.path().as_os_str().as_encoded_bytes()
);
assert!(matcher.is_match(entry.path_bytes()));
}
#[cfg(unix)]
#[test]
fn symlink_aliases_are_entered_by_every_frontend() {
use std::os::unix::fs::symlink;
let fixture = Fixture::new();
fixture.write("real/inside.txt");
symlink("real", fixture.root.join("first")).expect("create first directory symlink");
symlink("real", fixture.root.join("second")).expect("create second directory symlink");
let options = WalkOptions::default().follow_symlinks(true).sort(true);
let inside_paths = |paths: Vec<PathBuf>| {
let mut paths = paths
.into_iter()
.filter(|path| path.file_name().is_some_and(|name| name == "inside.txt"))
.collect::<Vec<_>>();
paths.sort_unstable();
paths
};
let expected = vec![
PathBuf::from("first/inside.txt"),
PathBuf::from("real/inside.txt"),
PathBuf::from("second/inside.txt"),
];
for threads in [1, 4] {
let result = Walker::new(&fixture.root)
.threads(threads)
.options(options)
.collect()
.expect("walk succeeds");
assert_eq!(
inside_paths(relative_paths(result.entries(), &fixture.root)),
expected,
"collect with {threads} thread(s) enters every acyclic alias"
);
}
let streamed = Walker::new(&fixture.root)
.options(options)
.stream()
.map(|entry| entry.expect("fixture has no I/O errors"))
.collect::<Vec<_>>();
assert_eq!(
inside_paths(relative_paths(&streamed, &fixture.root)),
expected,
"the stream enters every acyclic alias"
);
}
#[cfg(unix)]
#[test]
fn followed_directory_symlinks_use_the_target_kind_for_excludes_and_ignores() {
use std::os::unix::fs::symlink;
let excluded = Fixture::new();
excluded.write("real/artifact.o");
symlink("real", excluded.root.join("linked")).expect("create directory symlink");
let follow = WalkOptions::default().follow_symlinks(true).sort(true);
assert_frontends_agree(
"followed symlink excluded as directory",
&excluded.root,
|| {
Walker::new(&excluded.root)
.exclude("linked/")
.expect("valid directory exclusion")
.options(follow)
},
);
let result = Walker::new(&excluded.root)
.exclude("linked/")
.expect("valid directory exclusion")
.options(follow)
.collect()
.expect("walk succeeds");
assert_eq!(
relative_paths(result.entries(), &excluded.root),
vec![PathBuf::from("real"), PathBuf::from("real/artifact.o")]
);
let ignored = Fixture::new();
ignored.write("real/artifact.o");
fs::write(ignored.root.join(".gitignore"), b"build/\n").expect("write gitignore");
symlink("real", ignored.root.join("build")).expect("create ignored directory symlink");
assert_frontends_agree(
"followed symlink ignored as directory",
&ignored.root,
|| {
Walker::new(&ignored.root)
.respect_git_ignore(true)
.options(follow)
},
);
let result = Walker::new(&ignored.root)
.respect_git_ignore(true)
.options(follow)
.collect()
.expect("walk succeeds");
assert_eq!(
relative_paths(result.entries(), &ignored.root),
vec![
PathBuf::from(".gitignore"),
PathBuf::from("real"),
PathBuf::from("real/artifact.o"),
]
);
}
#[cfg(unix)]
#[test]
fn followed_symlinks_skipped_by_path_rules_do_not_need_a_target() {
use std::os::unix::fs::symlink;
let excluded = Fixture::new();
symlink("missing-target", excluded.root.join("hidden-link"))
.expect("create dangling symlink");
let follow = WalkOptions::default().follow_symlinks(true).sort(true);
assert_frontends_agree(
"path-excluded dangling followed symlink",
&excluded.root,
|| {
Walker::new(&excluded.root)
.exclude("hidden-link")
.expect("valid path exclusion")
.options(follow)
},
);
let result = Walker::new(&excluded.root)
.exclude("hidden-link")
.expect("valid path exclusion")
.options(follow)
.error_policy(ErrorPolicy::Collect)
.collect()
.expect("excluded dangling link is not an error");
assert!(result.entries().is_empty());
assert!(result.errors().is_empty());
let ignored = Fixture::new();
fs::write(ignored.root.join(".gitignore"), b"hidden-link\n").expect("write gitignore");
symlink("missing-target", ignored.root.join("hidden-link"))
.expect("create dangling symlink");
assert_frontends_agree(
"path-ignored dangling followed symlink",
&ignored.root,
|| {
Walker::new(&ignored.root)
.respect_git_ignore(true)
.options(follow)
},
);
let result = Walker::new(&ignored.root)
.respect_git_ignore(true)
.options(follow)
.error_policy(ErrorPolicy::Collect)
.collect()
.expect("ignored dangling link is not an error");
assert_eq!(
relative_paths(result.entries(), &ignored.root),
vec![PathBuf::from(".gitignore")]
);
assert!(result.errors().is_empty());
}
#[cfg(unix)]
#[test]
fn symlink_policy_prevents_directory_cycles_without_pruning_aliases() {
use std::os::unix::fs::symlink;
let fixture = Fixture::new();
fixture.write("real/inside.txt");
symlink("..", fixture.root.join("real/back")).expect("create cycle symlink");
let without_following = Walker::new(&fixture.root)
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds");
assert!(
!relative_paths(without_following.entries(), &fixture.root)
.contains(&PathBuf::from("real/back/real/inside.txt"))
);
let link = without_following
.entries()
.iter()
.find(|entry| entry.basename() == Some(std::ffi::OsStr::new("back")))
.expect("symlink is reported");
assert!(link.is_symlink());
assert_eq!(link.kind(), WalkEntryKind::Symlink);
let with_following = Walker::new(&fixture.root)
.options(WalkOptions::default().follow_symlinks(true).sort(true))
.collect()
.expect("walk succeeds");
assert_eq!(
relative_paths(with_following.entries(), &fixture.root)
.iter()
.filter(|path| path.file_name().is_some_and(|name| name == "inside.txt"))
.count(),
1
);
}
#[cfg(unix)]
#[test]
fn parallel_collect_retains_concurrent_metadata_errors() {
use std::os::unix::fs::symlink;
let fixture = Fixture::new();
fixture.write("left/ok.txt");
fixture.write("right/ok.txt");
symlink("missing-left", fixture.root.join("left/dangling"))
.expect("create left dangling symlink");
symlink("missing-right", fixture.root.join("right/dangling"))
.expect("create right dangling symlink");
let options = WalkOptions::default().follow_symlinks(true).sort(true);
let serial = Walker::new(&fixture.root)
.threads(1)
.options(options)
.error_policy(ErrorPolicy::Collect)
.collect()
.expect("serial walk retains errors");
let parallel = Walker::new(&fixture.root)
.threads(4)
.options(options)
.error_policy(ErrorPolicy::Collect)
.collect()
.expect("parallel walk retains errors");
let error_paths = |result: &super::WalkResult| {
let mut errors = result
.errors()
.iter()
.map(|error| {
(
error.operation(),
error
.path()
.strip_prefix(&fixture.root)
.expect("error is rooted in fixture")
.to_path_buf(),
)
})
.collect::<Vec<_>>();
errors.sort_unstable();
errors
};
assert_eq!(error_paths(¶llel), error_paths(&serial));
assert_eq!(parallel.errors().len(), 2);
}
#[cfg(unix)]
#[test]
fn parallel_abort_returns_an_error_without_cancelling_the_caller_token() {
use std::os::unix::fs::symlink;
let fixture = Fixture::new();
fixture.write("left/ok.txt");
fixture.write("right/ok.txt");
symlink("missing-left", fixture.root.join("left/dangling"))
.expect("create left dangling symlink");
symlink("missing-right", fixture.root.join("right/dangling"))
.expect("create right dangling symlink");
let cancellation = CancellationToken::default();
let serial_cancellation = CancellationToken::default();
let serial_error = Walker::new(&fixture.root)
.threads(1)
.options(WalkOptions::default().follow_symlinks(true))
.error_policy(ErrorPolicy::Abort)
.cancellation(serial_cancellation.clone())
.collect()
.expect_err("serial abort returns the first metadata error");
let error = Walker::new(&fixture.root)
.threads(4)
.options(WalkOptions::default().follow_symlinks(true))
.error_policy(ErrorPolicy::Abort)
.cancellation(cancellation.clone())
.collect()
.expect_err("abort policy returns the first metadata error");
assert_eq!(error.operation(), "metadata");
assert_eq!(serial_error.operation(), error.operation());
assert!(
!serial_cancellation.is_cancelled(),
"serial abort leaves the caller-owned token alone"
);
assert!(
!cancellation.is_cancelled(),
"parallel abort must match serial token ownership"
);
let reused = Walker::new(&fixture.root)
.threads(4)
.cancellation(cancellation.clone())
.collect()
.expect("the same token can drive a later walk");
assert!(!reused.was_cancelled());
}
#[cfg(target_os = "linux")]
#[test]
fn preserves_non_utf8_native_paths() {
use std::os::unix::ffi::OsStringExt;
let fixture = Fixture::new();
let name = std::ffi::OsString::from_vec(vec![b'n', 0xFF]);
fixture.write(PathBuf::from(&name));
let result = Walker::new(&fixture.root)
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds");
assert_eq!(
relative_paths(result.entries(), &fixture.root),
vec![PathBuf::from(name)]
);
}
}