#![deny(unsafe_code)]
#![doc = "Portable filesystem walking."]
use std::{
borrow::Cow,
collections::VecDeque,
collections::{HashMap, HashSet},
error::Error,
fmt, fs,
path::{Path, PathBuf},
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
};
use ferralk_glob::{Pattern, PatternError, PatternOptions};
use ignore::gitignore::{Gitignore, GitignoreBuilder};
pub use ferralk_glob;
#[cfg(all(feature = "native-linux", target_os = "linux"))]
#[allow(unsafe_code)]
mod linux_native;
#[cfg(all(feature = "native-macos", target_os = "macos"))]
#[allow(unsafe_code)]
mod macos_native;
#[cfg(all(feature = "native-linux", target_os = "linux"))]
#[doc(hidden)]
pub use linux_native::fuzz_validate_records as fuzz_validate_linux_dirent_records;
#[cfg(all(feature = "native-macos", target_os = "macos"))]
#[doc(hidden)]
pub use macos_native::fuzz_validate_bulk_record as fuzz_validate_macos_bulk_record;
#[cfg(all(feature = "native-macos", target_os = "macos"))]
#[doc(hidden)]
pub use macos_native::fuzz_validate_records as fuzz_validate_macos_dirent_records;
mod parallel;
mod scheduler;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum ErrorPolicy {
Abort,
Skip,
#[default]
Collect,
}
#[derive(Debug, Clone, Default)]
pub struct CancellationToken {
cancelled: Arc<AtomicBool>,
}
impl CancellationToken {
pub fn cancel(&self) {
self.cancelled.store(true, Ordering::Release);
}
#[must_use]
pub fn is_cancelled(&self) -> bool {
self.cancelled.load(Ordering::Acquire)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct WalkOptions {
follow_symlinks: bool,
sort: bool,
metadata: bool,
directories_only: bool,
files_only: bool,
skip_hidden: bool,
keep_git_dir: bool,
max_depth: Option<usize>,
}
impl WalkOptions {
#[must_use]
pub const fn follow_symlinks(mut self, enabled: bool) -> Self {
self.follow_symlinks = enabled;
self
}
#[must_use]
pub const fn sort(mut self, enabled: bool) -> Self {
self.sort = enabled;
self
}
#[must_use]
pub const fn metadata(mut self, enabled: bool) -> Self {
self.metadata = enabled;
self
}
#[must_use]
pub const fn directories_only(mut self, enabled: bool) -> Self {
self.directories_only = enabled;
self
}
#[must_use]
pub const fn files_only(mut self, enabled: bool) -> Self {
self.files_only = enabled;
self
}
#[must_use]
pub const fn skip_hidden(mut self, enabled: bool) -> Self {
self.skip_hidden = enabled;
self
}
#[must_use]
pub const fn keep_git_dir(mut self, enabled: bool) -> Self {
self.keep_git_dir = enabled;
self
}
#[must_use]
pub const fn max_depth(mut self, max_depth: usize) -> Self {
self.max_depth = Some(max_depth);
self
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WalkEntryKind {
File,
Directory,
Symlink,
}
#[derive(Debug)]
pub struct WalkEntry {
path: PathBuf,
is_dir: bool,
is_symlink: bool,
depth: usize,
metadata: Option<fs::Metadata>,
}
impl WalkEntry {
#[must_use]
pub fn path(&self) -> &Path {
&self.path
}
#[must_use]
pub const fn is_dir(&self) -> bool {
self.is_dir
}
#[must_use]
pub const fn is_symlink(&self) -> bool {
self.is_symlink
}
#[must_use]
pub const fn kind(&self) -> WalkEntryKind {
if self.is_symlink {
WalkEntryKind::Symlink
} else if self.is_dir {
WalkEntryKind::Directory
} else {
WalkEntryKind::File
}
}
#[must_use]
pub const fn depth(&self) -> usize {
self.depth
}
#[must_use]
pub fn basename(&self) -> Option<&std::ffi::OsStr> {
self.path.file_name()
}
#[must_use]
pub fn metadata(&self) -> Option<&fs::Metadata> {
self.metadata.as_ref()
}
}
#[derive(Debug)]
pub struct WalkError {
operation: &'static str,
path: PathBuf,
source: std::io::Error,
}
impl WalkError {
fn new(operation: &'static str, path: PathBuf, source: std::io::Error) -> Self {
Self {
operation,
path,
source,
}
}
#[must_use]
pub const fn operation(&self) -> &'static str {
self.operation
}
#[must_use]
pub fn path(&self) -> &Path {
&self.path
}
}
impl fmt::Display for WalkError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
formatter,
"{} {}: {}",
self.operation,
self.path.display(),
self.source
)
}
}
impl Error for WalkError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
Some(&self.source)
}
}
#[derive(Debug)]
pub struct WalkResult {
entries: Vec<WalkEntry>,
errors: Vec<WalkError>,
cancelled: bool,
}
impl WalkResult {
#[must_use]
pub fn entries(&self) -> &[WalkEntry] {
&self.entries
}
#[must_use]
pub fn errors(&self) -> &[WalkError] {
&self.errors
}
#[must_use]
pub fn into_parts(self) -> (Vec<WalkEntry>, Vec<WalkError>) {
(self.entries, self.errors)
}
#[must_use]
pub const fn was_cancelled(&self) -> bool {
self.cancelled
}
}
#[derive(Debug, Clone)]
pub struct Walker {
root: PathBuf,
includes: Vec<TraversalPattern>,
excludes: Vec<TraversalPattern>,
options: WalkOptions,
error_policy: ErrorPolicy,
cancellation: Option<CancellationToken>,
respect_git_ignore: bool,
threads: usize,
}
impl Walker {
#[must_use]
pub fn new(root: impl Into<PathBuf>) -> Self {
Self {
root: root.into(),
includes: Vec::new(),
excludes: Vec::new(),
options: WalkOptions::default(),
error_policy: ErrorPolicy::default(),
cancellation: None,
respect_git_ignore: false,
threads: std::thread::available_parallelism()
.map(std::num::NonZeroUsize::get)
.unwrap_or(1),
}
}
pub fn include(mut self, pattern: impl AsRef<[u8]>) -> Result<Self, PatternError> {
self.includes
.push(TraversalPattern::compile(pattern.as_ref())?);
Ok(self)
}
pub fn exclude(mut self, pattern: impl AsRef<[u8]>) -> Result<Self, PatternError> {
self.excludes
.push(TraversalPattern::compile(pattern.as_ref())?);
Ok(self)
}
#[must_use]
pub const fn options(mut self, options: WalkOptions) -> Self {
self.options = options;
self
}
#[must_use]
pub const fn error_policy(mut self, error_policy: ErrorPolicy) -> Self {
self.error_policy = error_policy;
self
}
#[must_use]
pub fn cancellation(mut self, cancellation: CancellationToken) -> Self {
self.cancellation = Some(cancellation);
self
}
#[must_use]
pub const fn respect_git_ignore(mut self, enabled: bool) -> Self {
self.respect_git_ignore = enabled;
self
}
#[must_use]
pub const fn threads(mut self, threads: usize) -> Self {
self.threads = if threads == 0 { 1 } else { threads };
self
}
pub fn collect(self) -> Result<WalkResult, WalkError> {
if self.threads > 1 {
return parallel::collect(self);
}
let backend = SystemBackend;
let mut state = WalkState::new(&self);
let scheduler = scheduler::Scheduler::new();
scheduler.push(self.root.clone());
let worker = scheduler.worker();
while let Some(directory) = scheduler.steal_into(&worker).or_else(|| worker.pop()) {
state.walk_directory(&backend, directory)?;
}
if self.options.sort {
state
.entries
.sort_by(|left, right| left.path.cmp(&right.path));
}
Ok(WalkResult {
entries: state.entries,
errors: state.errors,
cancelled: state.cancelled,
})
}
#[must_use]
pub fn stream(self) -> WalkStream {
WalkStream {
pending_directories: vec![self.root.clone()],
walker: self,
pending_entries: VecDeque::new(),
visited_directories: HashSet::new(),
gitignore_cache: HashMap::new(),
cancelled: false,
stopped: false,
}
}
fn may_descend_into(&self, relative: &[u8]) -> bool {
self.includes.is_empty()
|| self
.includes
.iter()
.any(|pattern| pattern.could_match_descendant(relative))
}
fn may_descend_path(&self, relative: &Path, bytes: &[u8]) -> bool {
self.includes_depth(relative)
&& self
.options
.max_depth
.is_none_or(|max_depth| relative.components().count() < max_depth)
&& self.may_descend_into(bytes)
}
fn includes_depth(&self, relative: &Path) -> bool {
self.options
.max_depth
.is_none_or(|max_depth| relative.components().count() <= max_depth)
}
fn may_include_file(&self, relative: &[u8]) -> bool {
self.includes.is_empty()
|| self
.includes
.iter()
.any(|pattern| pattern.matches_extension(relative))
}
}
fn has_hidden_component(path: &[u8]) -> bool {
path.split(is_path_separator)
.any(|component| component.first() == Some(&b'.'))
}
pub(crate) fn glob_path_bytes(path: &Path) -> Cow<'_, [u8]> {
let bytes = path.as_os_str().as_encoded_bytes();
#[cfg(windows)]
{
Cow::Owned(
bytes
.iter()
.map(|&byte| if byte == b'\\' { b'/' } else { byte })
.collect(),
)
}
#[cfg(not(windows))]
{
Cow::Borrowed(bytes)
}
}
fn traversal_pattern_options() -> PatternOptions {
PatternOptions::default()
.braces(true)
.recursive_double_star(true)
.extglob(true)
}
#[derive(Debug, Clone)]
struct TraversalPattern {
matcher: Pattern,
directories_only: bool,
subtree_root: Option<Pattern>,
literal_root: Option<Vec<u8>>,
extension: Option<Vec<u8>>,
}
impl TraversalPattern {
fn compile(pattern: &[u8]) -> Result<Self, PatternError> {
let pattern = pattern.strip_prefix(b"./").unwrap_or(pattern);
let directories_only = pattern.len() > 1 && pattern.ends_with(b"/");
let pattern = if directories_only {
&pattern[..pattern.len() - 1]
} else {
pattern
};
let options = traversal_pattern_options();
let subtree_root = pattern
.strip_suffix(b"/**")
.map(|root| Pattern::compile(root, options))
.transpose()?;
Ok(Self {
matcher: Pattern::compile(pattern, options)?,
directories_only,
subtree_root,
literal_root: literal_pattern_root(pattern),
extension: literal_extension(pattern),
})
}
fn matches(&self, path: &[u8], is_dir: bool) -> bool {
(!self.directories_only || is_dir) && self.matcher.is_match_glob_path(path)
}
fn covers_subtree(&self, path: &[u8]) -> bool {
self.subtree_root
.as_ref()
.is_some_and(|root| root.is_match(path))
}
fn could_match_descendant(&self, path: &[u8]) -> bool {
let Some(root) = &self.literal_root else {
return true;
};
root == path
|| root
.strip_prefix(path)
.is_some_and(|suffix| suffix.starts_with(b"/"))
|| path
.strip_prefix(root.as_slice())
.is_some_and(|suffix| suffix.starts_with(b"/"))
}
fn matches_extension(&self, path: &[u8]) -> bool {
let Some(extension) = &self.extension else {
return true;
};
final_extension(path).is_some_and(|candidate| candidate == extension)
}
}
fn literal_pattern_root(pattern: &[u8]) -> Option<Vec<u8>> {
let magic = pattern.iter().enumerate().position(|(index, byte)| {
matches!(byte, b'*' | b'?' | b'[')
|| (*byte == b'\\')
|| (*byte == b'{' && has_closing_brace(pattern, index))
|| (matches!(byte, b'@' | b'+' | b'!')
&& pattern.get(index + 1) == Some(&b'(')
&& has_closing_parenthesis(pattern, index + 1))
});
let prefix = &pattern[..magic.unwrap_or(pattern.len())];
let root = if magic.is_some() {
if let Some(prefix) = prefix.strip_suffix(b"/") {
prefix
} else {
prefix
.iter()
.rposition(|byte| *byte == b'/')
.map_or(prefix, |separator| &prefix[..separator])
}
} else {
prefix
};
(!root.is_empty()).then(|| root.to_vec())
}
fn literal_extension(pattern: &[u8]) -> Option<Vec<u8>> {
let extension = final_extension(pattern)?;
if extension.is_empty()
|| extension.iter().any(|byte| {
matches!(
byte,
b'*' | b'?' | b'[' | b']' | b'{' | b'}' | b'\\' | b'(' | b')' | b'|'
)
})
{
return None;
}
Some(extension.to_vec())
}
fn final_extension(path: &[u8]) -> Option<&[u8]> {
let name = path.rsplit(is_path_separator).next().unwrap_or(path);
let dot = name.iter().rposition(|byte| *byte == b'.')?;
name.get(dot + 1..)
}
fn is_path_separator(byte: &u8) -> bool {
*byte == b'/' || (cfg!(windows) && *byte == b'\\')
}
fn has_closing_brace(pattern: &[u8], open: usize) -> bool {
let mut depth = 0_usize;
let mut index = open;
while index < pattern.len() {
if pattern[index] == b'\\' {
index += 2;
continue;
}
match pattern[index] {
b'{' => depth += 1,
b'}' => {
depth -= 1;
if depth == 0 {
return true;
}
}
_ => {}
}
index += 1;
}
false
}
fn has_closing_parenthesis(pattern: &[u8], open: usize) -> bool {
let mut depth = 0_usize;
let mut index = open;
while index < pattern.len() {
if pattern[index] == b'\\' {
index += 2;
continue;
}
match pattern[index] {
b'(' => depth += 1,
b')' => {
depth -= 1;
if depth == 0 {
return true;
}
}
_ => {}
}
index += 1;
}
false
}
trait DirectoryBackend {
fn read_directory(&self, path: &Path) -> std::io::Result<Vec<BackendEntry>>;
}
#[derive(Debug, Clone)]
struct BackendEntry {
path: PathBuf,
is_dir: bool,
is_symlink: bool,
}
struct StdBackend;
impl DirectoryBackend for StdBackend {
fn read_directory(&self, path: &Path) -> std::io::Result<Vec<BackendEntry>> {
fs::read_dir(path)?
.map(|entry| {
let entry = entry?;
let file_type = entry.file_type()?;
Ok(BackendEntry {
path: entry.path(),
is_dir: file_type.is_dir(),
is_symlink: file_type.is_symlink(),
})
})
.collect()
}
}
struct SystemBackend;
impl DirectoryBackend for SystemBackend {
fn read_directory(&self, path: &Path) -> std::io::Result<Vec<BackendEntry>> {
#[cfg(all(feature = "native-macos", target_os = "macos"))]
{
match macos_native::read_directory(path) {
Ok(entries) => Ok(entries),
Err(error) if error.kind() == std::io::ErrorKind::Unsupported => {
StdBackend.read_directory(path)
}
Err(error) => Err(error),
}
}
#[cfg(all(
feature = "native-linux",
target_os = "linux",
not(all(feature = "native-macos", target_os = "macos"))
))]
{
match linux_native::read_directory(path) {
Ok(entries) => Ok(entries),
Err(error) if error.kind() == std::io::ErrorKind::Unsupported => {
StdBackend.read_directory(path)
}
Err(error) => Err(error),
}
}
#[cfg(not(any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
)))]
StdBackend.read_directory(path)
}
}
#[derive(Debug)]
pub struct WalkStream {
walker: Walker,
pending_directories: Vec<PathBuf>,
pending_entries: VecDeque<BackendEntry>,
visited_directories: HashSet<PathBuf>,
gitignore_cache: HashMap<PathBuf, Arc<GitIgnoreNode>>,
cancelled: bool,
stopped: bool,
}
impl WalkStream {
#[must_use]
pub const fn was_cancelled(&self) -> bool {
self.cancelled
}
fn check_cancellation(&mut self) -> bool {
self.cancelled |= self
.walker
.cancellation
.as_ref()
.is_some_and(CancellationToken::is_cancelled);
self.cancelled
}
fn error(
&mut self,
operation: &'static str,
path: PathBuf,
source: std::io::Error,
) -> Option<Result<WalkEntry, WalkError>> {
let error = WalkError::new(operation, path, source);
match self.walker.error_policy {
ErrorPolicy::Abort => {
self.stopped = true;
Some(Err(error))
}
ErrorPolicy::Skip => None,
ErrorPolicy::Collect => Some(Err(error)),
}
}
fn prepare_directory(&mut self, directory: PathBuf) -> Option<Result<WalkEntry, WalkError>> {
if self.walker.options.follow_symlinks {
match fs::canonicalize(&directory) {
Ok(canonical) => {
if !self.visited_directories.insert(canonical) {
return None;
}
}
Err(source) => return self.error("canonicalize", directory, source),
}
}
match SystemBackend.read_directory(&directory) {
Ok(entries) => {
self.pending_entries = entries.into();
None
}
Err(source) => self.error("read_dir", directory, source),
}
}
fn process_entry(&mut self, mut entry: BackendEntry) -> Option<Result<WalkEntry, WalkError>> {
let relative = entry
.path
.strip_prefix(&self.walker.root)
.unwrap_or(entry.path.as_path());
let depth = relative.components().count();
if !self.walker.includes_depth(relative) {
return None;
}
let bytes = glob_path_bytes(relative);
if self.walker.options.skip_hidden && has_hidden_component(bytes.as_ref()) {
return None;
}
if should_skip_git_directory(&self.walker, &entry.path) {
return None;
}
if self
.walker
.excludes
.iter()
.any(|pattern| pattern.matches(bytes.as_ref(), entry.is_dir))
{
return None;
}
let git_ignored = is_git_ignored(
&self.walker,
&entry.path,
entry.is_dir,
&mut self.gitignore_cache,
);
if git_ignored && !entry.is_dir {
return None;
}
if entry.is_symlink && self.walker.options.follow_symlinks {
match fs::metadata(&entry.path) {
Ok(metadata) => entry.is_dir = metadata.is_dir(),
Err(source) => return self.error("metadata", entry.path, source),
}
}
if !entry.is_dir && !self.walker.may_include_file(bytes.as_ref()) {
return None;
}
if entry.is_dir
&& !self
.walker
.excludes
.iter()
.any(|pattern| pattern.covers_subtree(bytes.as_ref()))
&& self.walker.may_descend_path(relative, bytes.as_ref())
{
self.pending_directories.push(entry.path.clone());
}
if !self.walker.includes.is_empty()
&& !self
.walker
.includes
.iter()
.any(|pattern| pattern.matches(bytes.as_ref(), entry.is_dir))
{
return None;
}
if git_ignored {
return None;
}
if self.walker.options.directories_only && !entry.is_dir {
return None;
}
if self.walker.options.files_only && entry.is_dir {
return None;
}
let metadata = if self.walker.options.metadata {
match fs::symlink_metadata(&entry.path) {
Ok(metadata) => Some(metadata),
Err(source) => return self.error("symlink_metadata", entry.path, source),
}
} else {
None
};
Some(Ok(WalkEntry {
path: entry.path,
is_dir: entry.is_dir,
is_symlink: entry.is_symlink,
depth,
metadata,
}))
}
}
impl Iterator for WalkStream {
type Item = Result<WalkEntry, WalkError>;
fn next(&mut self) -> Option<Self::Item> {
while !self.stopped {
if self.check_cancellation() {
self.stopped = true;
return None;
}
if let Some(entry) = self.pending_entries.pop_front() {
if let Some(result) = self.process_entry(entry) {
return Some(result);
}
continue;
}
let directory = self.pending_directories.pop()?;
if let Some(result) = self.prepare_directory(directory) {
return Some(result);
}
}
None
}
}
struct WalkState<'walker> {
walker: &'walker Walker,
entries: Vec<WalkEntry>,
errors: Vec<WalkError>,
visited_directories: HashSet<PathBuf>,
gitignore_cache: HashMap<PathBuf, Arc<GitIgnoreNode>>,
cancelled: bool,
}
impl<'walker> WalkState<'walker> {
fn new(walker: &'walker Walker) -> Self {
Self {
walker,
entries: Vec::new(),
errors: Vec::new(),
visited_directories: HashSet::new(),
gitignore_cache: HashMap::new(),
cancelled: false,
}
}
fn walk_directory(
&mut self,
backend: &impl DirectoryBackend,
directory: PathBuf,
) -> Result<(), WalkError> {
if self.check_cancellation() {
return Ok(());
}
if self.walker.options.follow_symlinks && !self.mark_directory(&directory)? {
return Ok(());
}
let entries = match backend.read_directory(&directory) {
Ok(entries) => entries,
Err(source) => return self.handle_error("read_dir", directory, source),
};
for entry in entries {
if self.check_cancellation() {
return Ok(());
}
self.visit_entry(backend, entry)?;
}
Ok(())
}
fn mark_directory(&mut self, directory: &Path) -> Result<bool, WalkError> {
match fs::canonicalize(directory) {
Ok(canonical) => Ok(self.visited_directories.insert(canonical)),
Err(source) => {
self.handle_error("canonicalize", directory.to_path_buf(), source)?;
Ok(false)
}
}
}
fn visit_entry(
&mut self,
backend: &impl DirectoryBackend,
mut entry: BackendEntry,
) -> Result<(), WalkError> {
if self.check_cancellation() {
return Ok(());
}
let relative = entry
.path
.strip_prefix(&self.walker.root)
.unwrap_or(entry.path.as_path());
let depth = relative.components().count();
if !self.walker.includes_depth(relative) {
return Ok(());
}
let bytes = glob_path_bytes(relative);
if self.walker.options.skip_hidden && has_hidden_component(bytes.as_ref()) {
return Ok(());
}
if should_skip_git_directory(self.walker, &entry.path) {
return Ok(());
}
if self
.walker
.excludes
.iter()
.any(|pattern| pattern.matches(bytes.as_ref(), entry.is_dir))
{
return Ok(());
}
let git_ignored = is_git_ignored(
self.walker,
&entry.path,
entry.is_dir,
&mut self.gitignore_cache,
);
if git_ignored && !entry.is_dir {
return Ok(());
}
if entry.is_symlink && self.walker.options.follow_symlinks {
match fs::metadata(&entry.path) {
Ok(metadata) => entry.is_dir = metadata.is_dir(),
Err(source) => {
self.handle_error("metadata", entry.path.clone(), source)?;
return Ok(());
}
}
}
if !entry.is_dir && !self.walker.may_include_file(bytes.as_ref()) {
return Ok(());
}
if entry.is_dir
&& !self
.walker
.excludes
.iter()
.any(|pattern| pattern.covers_subtree(bytes.as_ref()))
&& self.walker.may_descend_path(relative, bytes.as_ref())
{
self.walk_directory(backend, entry.path.clone())?;
}
if self.walker.options.files_only && entry.is_dir {
return Ok(());
}
let metadata = if self.walker.options.metadata {
match fs::symlink_metadata(&entry.path) {
Ok(metadata) => Some(metadata),
Err(source) => {
self.handle_error("symlink_metadata", entry.path.clone(), source)?;
return Ok(());
}
}
} else {
None
};
if (!self.walker.options.directories_only || entry.is_dir)
&& (self.walker.includes.is_empty()
|| self
.walker
.includes
.iter()
.any(|pattern| pattern.matches(bytes.as_ref(), entry.is_dir)))
{
if git_ignored {
return Ok(());
}
self.entries.push(WalkEntry {
path: entry.path,
is_dir: entry.is_dir,
is_symlink: entry.is_symlink,
depth,
metadata,
});
}
Ok(())
}
fn handle_error(
&mut self,
operation: &'static str,
path: PathBuf,
source: std::io::Error,
) -> Result<(), WalkError> {
let error = WalkError::new(operation, path, source);
match self.walker.error_policy {
ErrorPolicy::Abort => Err(error),
ErrorPolicy::Skip => Ok(()),
ErrorPolicy::Collect => {
self.errors.push(error);
Ok(())
}
}
}
fn check_cancellation(&mut self) -> bool {
self.cancelled |= self
.walker
.cancellation
.as_ref()
.is_some_and(CancellationToken::is_cancelled);
self.cancelled
}
}
fn is_git_ignored(
walker: &Walker,
path: &Path,
is_dir: bool,
cache: &mut HashMap<PathBuf, Arc<GitIgnoreNode>>,
) -> bool {
if !walker.respect_git_ignore {
return false;
}
let directory = path
.parent()
.filter(|parent| parent.starts_with(&walker.root));
directory
.is_some_and(|directory| gitignore_node(walker, directory, cache).is_ignored(path, is_dir))
}
fn should_skip_git_directory(walker: &Walker, path: &Path) -> bool {
walker.respect_git_ignore
&& !walker.options.keep_git_dir
&& path.file_name().is_some_and(|name| name == ".git")
}
struct GitIgnoreNode {
rules: Gitignore,
parent: Option<Arc<Self>>,
}
impl fmt::Debug for GitIgnoreNode {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("GitIgnoreNode")
.field("has_parent", &self.parent.is_some())
.finish_non_exhaustive()
}
}
impl GitIgnoreNode {
fn is_ignored(&self, path: &Path, is_dir: bool) -> bool {
let mut ignored = self
.parent
.as_ref()
.is_some_and(|parent| parent.is_ignored(path, is_dir));
let matched = self.rules.matched_path_or_any_parents(path, is_dir);
if !matched.is_none() {
ignored = matched.is_ignore();
}
ignored
}
}
fn gitignore_node(
walker: &Walker,
directory: &Path,
cache: &mut HashMap<PathBuf, Arc<GitIgnoreNode>>,
) -> Arc<GitIgnoreNode> {
if let Some(node) = cache.get(directory) {
return Arc::clone(node);
}
let parent = (directory != walker.root)
.then(|| {
directory
.parent()
.filter(|parent| parent.starts_with(&walker.root))
})
.flatten()
.map(|parent| gitignore_node(walker, parent, cache));
let node = Arc::new(GitIgnoreNode {
rules: gitignore_rules(directory),
parent,
});
cache.insert(directory.to_path_buf(), Arc::clone(&node));
node
}
fn gitignore_rules(directory: &Path) -> Gitignore {
let mut builder = GitignoreBuilder::new(directory);
for file_name in [".gitignore", ".ignore"] {
let path = directory.join(file_name);
if path.is_file() {
let _ = builder.add(path);
}
}
builder.build().unwrap_or_else(|_| Gitignore::empty())
}
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
#[cfg(test)]
mod tests {
use std::{
cell::RefCell,
collections::HashMap,
fs,
path::{Path, PathBuf},
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
time::{SystemTime, UNIX_EPOCH},
};
use super::{
CancellationToken, ErrorPolicy, TraversalPattern, WalkEntry, WalkEntryKind, WalkOptions,
Walker, gitignore_node, literal_extension, literal_pattern_root,
};
static NEXT_FIXTURE: AtomicUsize = AtomicUsize::new(0);
struct Fixture {
root: PathBuf,
}
impl Fixture {
fn new() -> Self {
let unique = format!(
"ferralk-test-{}-{}",
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("system clock is after unix epoch")
.as_nanos()
+ NEXT_FIXTURE.fetch_add(1, Ordering::Relaxed) as u128
);
let root = std::env::temp_dir().join(unique);
fs::create_dir_all(&root).expect("create fixture root");
Self { root }
}
fn write(&self, relative: impl AsRef<Path>) {
let path = self.root.join(relative);
fs::create_dir_all(path.parent().expect("fixture file has parent"))
.expect("create fixture parent");
fs::write(path, b"fixture").expect("write fixture file");
}
}
impl Drop for Fixture {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.root);
}
}
fn relative_paths(entries: &[WalkEntry], root: &Path) -> Vec<PathBuf> {
entries
.iter()
.map(|entry| {
entry
.path()
.strip_prefix(root)
.expect("entry is rooted in fixture")
.to_path_buf()
})
.collect()
}
fn relative_paths_and_depths(entries: &[WalkEntry], root: &Path) -> Vec<(PathBuf, usize)> {
entries
.iter()
.map(|entry| {
(
entry
.path()
.strip_prefix(root)
.expect("entry is rooted in fixture")
.to_path_buf(),
entry.depth(),
)
})
.collect()
}
#[test]
fn include_exclude_and_sort_are_applied_to_relative_paths() {
let fixture = Fixture::new();
fixture.write("src/main.rs");
fixture.write("src/lib.txt");
fixture.write("target/generated.rs");
let result = Walker::new(&fixture.root)
.include("**/*.rs")
.expect("valid include")
.exclude("**/target/**")
.expect("valid exclude")
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds");
assert_eq!(
relative_paths(result.entries(), &fixture.root),
vec![PathBuf::from("src/main.rs")]
);
assert!(result.errors().is_empty());
}
#[test]
fn source_walk_glob_patterns_filter_recursive_anchored_and_brace_paths() {
let fixture = Fixture::new();
fixture.write("src/a.rs");
fixture.write("src/b.txt");
fixture.write("src/deep/e.rs");
fixture.write("lib/c.rs");
fixture.write("docs/d.md");
fixture.write("top.rs");
let options = WalkOptions::default().sort(true);
let recursive = Walker::new(&fixture.root)
.include("**/*.rs")
.expect("valid recursive include")
.options(options)
.collect()
.expect("recursive walk succeeds");
assert_eq!(
relative_paths(recursive.entries(), &fixture.root),
vec![
PathBuf::from("lib/c.rs"),
PathBuf::from("src/a.rs"),
PathBuf::from("src/deep/e.rs"),
PathBuf::from("top.rs"),
]
);
let anchored = Walker::new(&fixture.root)
.include("src/**")
.expect("valid anchored include")
.options(options)
.collect()
.expect("anchored walk succeeds");
assert_eq!(
relative_paths(anchored.entries(), &fixture.root),
vec![
PathBuf::from("src"),
PathBuf::from("src/a.rs"),
PathBuf::from("src/b.txt"),
PathBuf::from("src/deep"),
PathBuf::from("src/deep/e.rs"),
]
);
let brace = Walker::new(&fixture.root)
.include("**/*.{md,txt}")
.expect("valid brace include")
.options(options)
.collect()
.expect("brace walk succeeds");
assert_eq!(
relative_paths(brace.entries(), &fixture.root),
vec![PathBuf::from("docs/d.md"), PathBuf::from("src/b.txt")]
);
}
#[test]
fn source_rust_glob_patterns_replay_as_walker_filters() {
let fixture = Fixture::new();
for path in [
"xyz/x",
"xyz/y",
"xyz/z",
"aaa/tomato/tomato.txt",
"aaa/tomato/tomoto.txt",
"bbb/specials/[",
"bbb/specials/!",
"bbb/specials/]",
] {
fixture.write(path);
}
for path in ["aaa/apple", "aaa/orange"] {
fs::create_dir_all(fixture.root.join(path)).expect("create source fixture directory");
}
let paths_for = |pattern: &str| {
let result = Walker::new(&fixture.root)
.threads(1)
.include(pattern)
.expect("valid source pattern")
.options(WalkOptions::default().sort(true))
.collect()
.expect("source walk succeeds");
relative_paths(result.entries(), &fixture.root)
};
assert_eq!(paths_for("aaa"), vec![PathBuf::from("aaa")]);
assert_eq!(paths_for("./aaa"), vec![PathBuf::from("aaa")]);
assert_eq!(paths_for("aaa/"), vec![PathBuf::from("aaa")]);
assert!(paths_for("aaa/tomato/tomato.txt/").is_empty());
assert!(paths_for("nope").is_empty());
assert_eq!(paths_for("a*"), vec![PathBuf::from("aaa")]);
assert_eq!(paths_for("@(a*)"), vec![PathBuf::from("aaa")]);
assert_eq!(paths_for("a*a"), vec![PathBuf::from("aaa")]);
assert_eq!(paths_for("*a*a*a*"), vec![PathBuf::from("aaa")]);
assert_eq!(paths_for("aaa/apple"), vec![PathBuf::from("aaa/apple")]);
assert_eq!(paths_for("./*"), paths_for("*"));
assert_eq!(
paths_for("???/"),
vec![
PathBuf::from("aaa"),
PathBuf::from("bbb"),
PathBuf::from("xyz"),
]
);
assert_eq!(
paths_for("xyz/?"),
vec![
PathBuf::from("xyz/x"),
PathBuf::from("xyz/y"),
PathBuf::from("xyz/z"),
]
);
assert_eq!(
paths_for("aaa/tomato/tom?to.txt"),
vec![
PathBuf::from("aaa/tomato/tomato.txt"),
PathBuf::from("aaa/tomato/tomoto.txt"),
]
);
assert_eq!(
paths_for("aaa/*"),
vec![
PathBuf::from("aaa/apple"),
PathBuf::from("aaa/orange"),
PathBuf::from("aaa/tomato"),
]
);
let component_local = paths_for("aaa/*");
let parallel = Walker::new(&fixture.root)
.threads(4)
.include("aaa/*")
.expect("valid source pattern")
.options(WalkOptions::default().sort(true))
.collect()
.expect("parallel source walk succeeds");
assert_eq!(
relative_paths(parallel.entries(), &fixture.root),
component_local
);
let mut streamed = Walker::new(&fixture.root)
.include("aaa/*")
.expect("valid source pattern")
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream source walk succeeds");
streamed.sort_by(|left, right| left.path().cmp(right.path()));
assert_eq!(relative_paths(&streamed, &fixture.root), component_local);
let trailing_directory = paths_for("aaa/");
let trailing_parallel = Walker::new(&fixture.root)
.threads(4)
.include("aaa/")
.expect("valid trailing directory pattern")
.options(WalkOptions::default().sort(true))
.collect()
.expect("parallel trailing directory walk succeeds");
assert_eq!(
relative_paths(trailing_parallel.entries(), &fixture.root),
trailing_directory
);
let mut trailing_streamed = Walker::new(&fixture.root)
.include("aaa/")
.expect("valid trailing directory pattern")
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream trailing directory walk succeeds");
trailing_streamed.sort_by(|left, right| left.path().cmp(right.path()));
assert_eq!(
relative_paths(&trailing_streamed, &fixture.root),
trailing_directory
);
assert_eq!(
paths_for("*/*/*.txt"),
vec![
PathBuf::from("aaa/tomato/tomato.txt"),
PathBuf::from("aaa/tomato/tomoto.txt"),
]
);
assert_eq!(paths_for("aa[a]"), vec![PathBuf::from("aaa")]);
assert_eq!(paths_for("aa[!b]"), vec![PathBuf::from("aaa")]);
assert!(paths_for("aa[b]").is_empty());
assert_eq!(
paths_for("*/*/t[aob]m?to[.]t[!y]t"),
vec![
PathBuf::from("aaa/tomato/tomato.txt"),
PathBuf::from("aaa/tomato/tomoto.txt"),
]
);
assert_eq!(
paths_for("bbb/specials/[[]"),
vec![PathBuf::from("bbb/specials/["),]
);
assert_eq!(
paths_for("bbb/specials/[]]"),
vec![PathBuf::from("bbb/specials/]"),]
);
}
#[test]
fn source_walk_entry_depths_are_relative_component_counts() {
let fixture = Fixture::new();
fixture.write("a.txt");
fixture.write("src/b.txt");
fixture.write("src/sub/c.txt");
let options = WalkOptions::default().sort(true);
let expected = vec![
(PathBuf::from("a.txt"), 1),
(PathBuf::from("src"), 1),
(PathBuf::from("src/b.txt"), 2),
(PathBuf::from("src/sub"), 2),
(PathBuf::from("src/sub/c.txt"), 3),
];
let serial = Walker::new(&fixture.root)
.threads(1)
.options(options)
.collect()
.expect("serial walk succeeds");
let parallel = Walker::new(&fixture.root)
.threads(4)
.options(options)
.collect()
.expect("parallel walk succeeds");
let mut streamed = Walker::new(&fixture.root)
.options(options)
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream succeeds");
streamed.sort_by(|left, right| left.path.cmp(&right.path));
assert_eq!(
relative_paths_and_depths(serial.entries(), &fixture.root),
expected
);
assert_eq!(
relative_paths_and_depths(parallel.entries(), &fixture.root),
expected
);
assert_eq!(
relative_paths_and_depths(&streamed, &fixture.root),
expected
);
let a = serial
.entries()
.iter()
.find(|entry| entry.basename() == Some(std::ffi::OsStr::new("a.txt")))
.expect("a.txt is present");
assert!(!a.is_dir());
assert_eq!(a.kind(), WalkEntryKind::File);
assert_eq!(a.depth(), 1);
let src = serial
.entries()
.iter()
.find(|entry| entry.basename() == Some(std::ffi::OsStr::new("src")))
.expect("src is present");
assert!(src.is_dir());
assert_eq!(src.kind(), WalkEntryKind::Directory);
assert_eq!(src.depth(), 1);
let c = serial
.entries()
.iter()
.find(|entry| entry.basename() == Some(std::ffi::OsStr::new("c.txt")))
.expect("c.txt is present");
assert!(!c.is_dir());
assert_eq!(c.kind(), WalkEntryKind::File);
assert_eq!(c.depth(), 3);
}
#[test]
fn directories_only_filters_results_without_pruning_descendants() {
let fixture = Fixture::new();
fixture.write("src/main.rs");
fixture.write("src/nested/lib.rs");
let options = WalkOptions::default().directories_only(true).sort(true);
let serial = Walker::new(&fixture.root)
.threads(1)
.options(options)
.collect()
.expect("serial walk succeeds");
let parallel = Walker::new(&fixture.root)
.threads(4)
.options(options)
.collect()
.expect("parallel walk succeeds");
let streamed = Walker::new(&fixture.root)
.options(options)
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream succeeds");
let expected = vec![PathBuf::from("src"), PathBuf::from("src/nested")];
assert_eq!(relative_paths(serial.entries(), &fixture.root), expected);
assert_eq!(relative_paths(parallel.entries(), &fixture.root), expected);
assert_eq!(relative_paths(&streamed, &fixture.root), expected);
assert!(streamed.iter().all(WalkEntry::is_dir));
}
#[test]
fn max_depth_keeps_boundary_entries_without_descending() {
let fixture = Fixture::new();
fixture.write("top.txt");
fixture.write("d1/mid.txt");
fixture.write("d1/d2/bottom.txt");
for (max_depth, expected) in [
(0, vec![]),
(1, vec![PathBuf::from("d1"), PathBuf::from("top.txt")]),
(
2,
vec![
PathBuf::from("d1"),
PathBuf::from("d1/d2"),
PathBuf::from("d1/mid.txt"),
PathBuf::from("top.txt"),
],
),
] {
let options = WalkOptions::default().max_depth(max_depth).sort(true);
let serial = Walker::new(&fixture.root)
.threads(1)
.options(options)
.collect()
.expect("serial walk succeeds");
let parallel = Walker::new(&fixture.root)
.threads(4)
.options(options)
.collect()
.expect("parallel walk succeeds");
let mut streamed = Walker::new(&fixture.root)
.options(options)
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream succeeds");
streamed.sort_by(|left, right| left.path.cmp(&right.path));
assert_eq!(relative_paths(serial.entries(), &fixture.root), expected);
assert_eq!(relative_paths(parallel.entries(), &fixture.root), expected);
assert_eq!(relative_paths(&streamed, &fixture.root), expected);
}
}
#[test]
fn parallel_collect_matches_the_serial_result_multiset() {
let fixture = Fixture::new();
fixture.write("wide/a.txt");
fixture.write("wide/b.txt");
fixture.write("deep/one/two/three/leaf.txt");
fixture.write("ignored.tmp");
fs::write(fixture.root.join(".gitignore"), b"*.tmp\n").expect("write gitignore");
let serial = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(1)
.options(WalkOptions::default().sort(true))
.collect()
.expect("serial walk succeeds");
let parallel = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(4)
.options(WalkOptions::default().sort(true))
.collect()
.expect("parallel walk succeeds");
assert_eq!(
relative_paths(parallel.entries(), &fixture.root),
relative_paths(serial.entries(), &fixture.root)
);
assert!(parallel.errors().is_empty());
assert!(serial.errors().is_empty());
}
#[test]
fn parallel_collect_stress_covers_empty_shallow_and_imbalanced_trees() {
let empty = Fixture::new();
assert!(
Walker::new(&empty.root)
.threads(8)
.collect()
.expect("empty parallel walk succeeds")
.entries()
.is_empty()
);
let fixture = Fixture::new();
fixture.write("shallow.txt");
for branch in 0..8 {
fixture.write(format!("wide/{branch}/leaf.txt"));
}
for depth in 0..20 {
fixture.write(format!("deep/{depth}/next/leaf.txt"));
}
let serial = Walker::new(&fixture.root)
.threads(1)
.options(WalkOptions::default().sort(true))
.collect()
.expect("serial walk succeeds");
let expected = relative_paths(serial.entries(), &fixture.root);
for _ in 0..32 {
let actual = Walker::new(&fixture.root)
.threads(8)
.options(WalkOptions::default().sort(true))
.collect()
.expect("parallel stress walk succeeds");
assert_eq!(relative_paths(actual.entries(), &fixture.root), expected);
assert!(actual.errors().is_empty());
}
}
#[test]
fn source_walk_metadata_preserves_portable_and_unix_fields() {
let fixture = Fixture::new();
fs::write(fixture.root.join("five.bin"), b"12345").expect("write metadata fixture");
let without_metadata = Walker::new(&fixture.root)
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds");
assert!(
without_metadata
.entries()
.iter()
.all(|entry| entry.metadata().is_none())
);
let with_metadata = Walker::new(&fixture.root)
.options(WalkOptions::default().sort(true).metadata(true))
.collect()
.expect("walk succeeds");
let metadata = with_metadata
.entries()
.iter()
.find(|entry| entry.path().ends_with("five.bin"))
.expect("fixture file is returned")
.metadata()
.expect("metadata is requested");
assert_eq!(metadata.len(), 5);
assert!(metadata.is_file());
assert!(metadata.modified().is_ok());
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
assert_ne!(metadata.ino(), 0);
assert_ne!(metadata.mode() & 0o400, 0);
}
}
#[test]
fn root_gitignore_rules_and_negation_apply_to_collect_and_stream() {
let fixture = Fixture::new();
fixture.write("generated.tmp");
fixture.write("keep.tmp");
fixture.write("src/main.rs");
fixture.write("src/keep.tmp");
fixture.write("build/keep.txt");
fs::write(
fixture.root.join(".gitignore"),
b"*.tmp\n!keep.tmp\nbuild/\n",
)
.expect("write root gitignore");
fs::write(fixture.root.join("src/.gitignore"), b"!keep.tmp\n")
.expect("write nested gitignore");
fs::write(fixture.root.join("build/.gitignore"), b"!keep.txt\n")
.expect("write nested re-include");
let collected = Walker::new(&fixture.root)
.respect_git_ignore(true)
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds");
let collected_paths = relative_paths(collected.entries(), &fixture.root);
assert!(!collected_paths.contains(&PathBuf::from("generated.tmp")));
assert!(collected_paths.contains(&PathBuf::from("keep.tmp")));
assert!(collected_paths.contains(&PathBuf::from("src/keep.tmp")));
assert!(collected_paths.contains(&PathBuf::from("build/keep.txt")));
assert!(!collected_paths.contains(&PathBuf::from("build")));
let streamed = Walker::new(&fixture.root)
.respect_git_ignore(true)
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream has no I/O errors");
let streamed_paths = relative_paths(&streamed, &fixture.root);
assert!(!streamed_paths.contains(&PathBuf::from("generated.tmp")));
assert!(streamed_paths.contains(&PathBuf::from("keep.tmp")));
assert!(streamed_paths.contains(&PathBuf::from("src/keep.tmp")));
assert!(streamed_paths.contains(&PathBuf::from("build/keep.txt")));
assert!(!streamed_paths.contains(&PathBuf::from("build")));
}
#[test]
fn source_walk_nested_gitignore_overrides_and_skips_dot_git() {
let fixture = Fixture::new();
for path in [
"build/artifact.o",
"root.log",
"keep.txt",
"sub/important.log",
"sub/other.log",
"sub/temp/scratch.txt",
".git/config",
] {
fixture.write(path);
}
fs::write(fixture.root.join(".gitignore"), b"*.log\nbuild/\n")
.expect("write root gitignore");
fs::write(
fixture.root.join("sub/.gitignore"),
b"!important.log\ntemp/\n",
)
.expect("write nested gitignore");
let result = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(1)
.options(WalkOptions::default().sort(true))
.collect()
.expect("Gitignore walk succeeds");
let paths = relative_paths(result.entries(), &fixture.root);
for ignored in [
"build",
"build/artifact.o",
"root.log",
"sub/other.log",
"sub/temp",
"sub/temp/scratch.txt",
".git",
".git/config",
] {
assert!(
!paths.contains(&PathBuf::from(ignored)),
"ignored source path {ignored} was returned"
);
}
for kept in ["keep.txt", "sub", "sub/.gitignore", "sub/important.log"] {
assert!(
paths.contains(&PathBuf::from(kept)),
"kept source path {kept} was omitted"
);
}
let parallel = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(4)
.options(WalkOptions::default().sort(true))
.collect()
.expect("parallel Gitignore walk succeeds");
assert_eq!(relative_paths(parallel.entries(), &fixture.root), paths);
let mut streamed = Walker::new(&fixture.root)
.respect_git_ignore(true)
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream Gitignore walk succeeds");
streamed.sort_by(|left, right| left.path().cmp(right.path()));
assert_eq!(relative_paths(&streamed, &fixture.root), paths);
let without_ignore = Walker::new(&fixture.root)
.threads(1)
.options(WalkOptions::default().sort(true))
.collect()
.expect("unfiltered source walk succeeds");
let unfiltered_paths = relative_paths(without_ignore.entries(), &fixture.root);
assert!(unfiltered_paths.contains(&PathBuf::from(".git/config")));
assert!(unfiltered_paths.len() > paths.len());
}
#[test]
fn source_walk_ignore_file_overrides_gitignore_rules() {
let fixture = Fixture::new();
for path in [
"app.log",
"keep.log",
"scratch.tmp",
"important.tmp",
"old.bak",
"src/main.rs",
"src/old.bak",
"build/artifact.txt",
] {
fixture.write(path);
}
fs::write(
fixture.root.join(".gitignore"),
b"*.log\nbuild/\n!keep.log\n*.tmp\n",
)
.expect("write root gitignore");
fs::write(fixture.root.join(".ignore"), b"!important.tmp\n")
.expect("write root ignore supplement");
fs::write(fixture.root.join("src/.gitignore"), b"*.bak\n").expect("write nested gitignore");
let expected = vec![
PathBuf::from(".gitignore"),
PathBuf::from(".ignore"),
PathBuf::from("important.tmp"),
PathBuf::from("keep.log"),
PathBuf::from("old.bak"),
PathBuf::from("src"),
PathBuf::from("src/.gitignore"),
PathBuf::from("src/main.rs"),
];
let options = WalkOptions::default().sort(true);
let serial = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(1)
.options(options)
.collect()
.expect("serial ignore walk succeeds");
let parallel = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(4)
.options(options)
.collect()
.expect("parallel ignore walk succeeds");
let mut streamed = Walker::new(&fixture.root)
.respect_git_ignore(true)
.options(options)
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream ignore walk succeeds");
streamed.sort_by(|left, right| left.path().cmp(right.path()));
assert_eq!(relative_paths(serial.entries(), &fixture.root), expected);
assert_eq!(relative_paths(parallel.entries(), &fixture.root), expected);
assert_eq!(relative_paths(&streamed, &fixture.root), expected);
}
#[test]
fn source_walk_allowlist_gitignore_descends_into_reincluded_directories() {
let fixture = Fixture::new();
for path in [
"main.rs",
"Makefile",
".keep",
"src/lib.rs",
"src/noext",
"src/deep/a.txt",
"dir.d/x.md",
"dir.d/noext",
"plain/y.txt",
".git/config",
] {
fixture.write(path);
}
fs::write(
fixture.root.join(".gitignore"),
b"# Ignore all\n*\n\n# Unignore all with extensions\n!*.*\n\n# Unignore all dirs\n!/**/\n",
)
.expect("write root gitignore");
let expected = vec![
PathBuf::from(".gitignore"),
PathBuf::from(".keep"),
PathBuf::from("dir.d"),
PathBuf::from("dir.d/x.md"),
PathBuf::from("main.rs"),
PathBuf::from("plain"),
PathBuf::from("plain/y.txt"),
PathBuf::from("src"),
PathBuf::from("src/deep"),
PathBuf::from("src/deep/a.txt"),
PathBuf::from("src/lib.rs"),
];
let options = WalkOptions::default().sort(true);
let serial = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(1)
.options(options)
.collect()
.expect("serial Gitignore walk succeeds");
let parallel = Walker::new(&fixture.root)
.respect_git_ignore(true)
.threads(4)
.options(options)
.collect()
.expect("parallel Gitignore walk succeeds");
let mut streamed = Walker::new(&fixture.root)
.respect_git_ignore(true)
.options(options)
.stream()
.collect::<Result<Vec<_>, _>>()
.expect("stream Gitignore walk succeeds");
streamed.sort_by(|left, right| left.path().cmp(right.path()));
assert_eq!(relative_paths(serial.entries(), &fixture.root), expected);
assert_eq!(relative_paths(parallel.entries(), &fixture.root), expected);
assert_eq!(relative_paths(&streamed, &fixture.root), expected);
}
#[test]
fn gitignore_nodes_share_their_immutable_parent_chain() {
let fixture = Fixture::new();
let walker = Walker::new(&fixture.root).respect_git_ignore(true);
let mut cache = HashMap::new();
let left = gitignore_node(&walker, &fixture.root.join("left/nested"), &mut cache);
let right = gitignore_node(&walker, &fixture.root.join("left/sibling"), &mut cache);
let left_parent = left.parent.as_ref().expect("nested left parent");
let right_parent = right.parent.as_ref().expect("nested right parent");
assert!(Arc::ptr_eq(left_parent, right_parent));
assert_eq!(cache.len(), 4, "root, left, and two child nodes are cached");
}
#[test]
fn git_ignore_corpus_replays_through_the_walker() {
let corpus_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus/ignore.jsonl");
for line in fs::read_to_string(corpus_path)
.expect("read ignore corpus")
.lines()
.filter(|line| !line.trim().is_empty())
{
let case: corpus::Case = serde_json::from_str(line).expect("valid ignore corpus case");
let fixture = Fixture::new();
fs::write(
fixture.root.join(".gitignore"),
case.ignore_rules.join("\n").as_bytes(),
)
.expect("write fixture gitignore");
fixture.write(&case.path);
let result = Walker::new(&fixture.root)
.respect_git_ignore(true)
.collect()
.expect("walk succeeds");
let returned =
relative_paths(result.entries(), &fixture.root).contains(&PathBuf::from(case.path));
assert_eq!(
!returned, case.expected,
"walker verdict for corpus case {}",
case.id
);
}
}
#[test]
fn prune_planner_only_accepts_explicit_whole_subtree_excludes() {
let subtree = TraversalPattern::compile(b"src/**").expect("valid subtree pattern");
assert!(subtree.covers_subtree(b"src"));
assert!(!subtree.covers_subtree(b"src/nested"));
let suffix = TraversalPattern::compile(b"*.tmp").expect("valid suffix pattern");
assert!(!suffix.covers_subtree(b"cache"));
let nested =
TraversalPattern::compile(b"**/target/**").expect("valid recursive subtree pattern");
assert!(nested.covers_subtree(b"target"));
assert!(nested.covers_subtree(b"crates/ferralk/target"));
assert_eq!(
literal_pattern_root(b"src/foo/*.rs"),
Some(b"src/foo".to_vec())
);
assert_eq!(literal_pattern_root(b"src/foo*.rs"), Some(b"src".to_vec()));
assert_eq!(literal_pattern_root(b"**/*.rs"), None);
assert_eq!(
literal_pattern_root(b"foo+bar/**/*.rs"),
Some(b"foo+bar".to_vec())
);
assert_eq!(
literal_pattern_root(b"foo@(bar/**/*.rs"),
Some(b"foo@(bar".to_vec())
);
let rust_sources = TraversalPattern::compile(b"src/**/*.rs").expect("valid suffix");
assert!(rust_sources.matches_extension(b"src/lib.rs"));
assert!(!rust_sources.matches_extension(b"src/lib.txt"));
assert_eq!(literal_extension(b"src/**/*.{rs,ts}"), None);
assert_eq!(literal_extension(b"src/**/*.rs"), Some(b"rs".to_vec()));
}
#[test]
fn literal_include_roots_prune_unrelated_sibling_directories() {
struct RecordingBackend {
entries: HashMap<PathBuf, Vec<super::BackendEntry>>,
reads: RefCell<Vec<PathBuf>>,
}
impl super::DirectoryBackend for RecordingBackend {
fn read_directory(&self, path: &Path) -> std::io::Result<Vec<super::BackendEntry>> {
self.reads.borrow_mut().push(path.to_path_buf());
Ok(self.entries.get(path).cloned().unwrap_or_default())
}
}
let root = PathBuf::from("/fixture");
let source = root.join("src");
let docs = root.join("docs");
let mut entries = HashMap::new();
entries.insert(
root.clone(),
vec![
super::BackendEntry {
path: source.clone(),
is_dir: true,
is_symlink: false,
},
super::BackendEntry {
path: docs.clone(),
is_dir: true,
is_symlink: false,
},
],
);
entries.insert(
source.clone(),
vec![super::BackendEntry {
path: source.join("main.rs"),
is_dir: false,
is_symlink: false,
}],
);
let backend = RecordingBackend {
entries,
reads: RefCell::new(Vec::new()),
};
let walker = Walker::new(&root)
.include("src/**/*.rs")
.expect("valid include");
let mut state = super::WalkState::new(&walker);
state
.walk_directory(&backend, root.clone())
.expect("backend walk succeeds");
assert_eq!(backend.reads.into_inner(), vec![root, source]);
assert_eq!(state.entries.len(), 1);
assert_eq!(state.entries[0].path(), Path::new("/fixture/src/main.rs"));
}
#[test]
fn metadata_error_is_retained_when_a_dirent_disappears_before_stat() {
struct DisappearingFileBackend {
root: PathBuf,
disappeared: PathBuf,
}
impl super::DirectoryBackend for DisappearingFileBackend {
fn read_directory(&self, path: &Path) -> std::io::Result<Vec<super::BackendEntry>> {
if path == self.root {
Ok(vec![super::BackendEntry {
path: self.disappeared.clone(),
is_dir: false,
is_symlink: false,
}])
} else {
Ok(Vec::new())
}
}
}
let fixture = Fixture::new();
let disappeared = fixture.root.join("gone.rs");
let walker = Walker::new(&fixture.root)
.threads(1)
.options(WalkOptions::default().metadata(true));
let backend = DisappearingFileBackend {
root: fixture.root.clone(),
disappeared: disappeared.clone(),
};
let mut state = super::WalkState::new(&walker);
state
.walk_directory(&backend, fixture.root.clone())
.expect("collect policy retains the metadata error");
assert!(state.entries.is_empty());
assert_eq!(state.errors.len(), 1);
assert_eq!(state.errors[0].operation(), "symlink_metadata");
assert_eq!(state.errors[0].path(), disappeared);
}
#[cfg(any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
))]
#[test]
fn native_backend_matches_portable_across_walker_option_matrix() {
use std::os::unix::fs::symlink;
let fixture = Fixture::new();
fixture.write("src/lib.rs");
fixture.write("src/nested/mod.rs");
fixture.write("src/generated.tmp");
fixture.write("ignored/skip.rs");
fixture.write(".hidden/skip.rs");
for index in 0..192 {
fixture.write(format!("many/{index:03}-{}", "x".repeat(180)));
}
fixture.write(".gitignore");
fs::write(fixture.root.join(".gitignore"), b"ignored/\n").expect("write ignore rule");
symlink("src", fixture.root.join("source-link")).expect("create directory symlink");
symlink("missing-target", fixture.root.join("dangling-link"))
.expect("create dangling symlink");
let cases = [
("baseline", WalkOptions::default().sort(true)),
("metadata", WalkOptions::default().sort(true).metadata(true)),
(
"directories_only",
WalkOptions::default().sort(true).directories_only(true),
),
(
"files_only",
WalkOptions::default().sort(true).files_only(true),
),
(
"skip_hidden",
WalkOptions::default().sort(true).skip_hidden(true),
),
(
"follow_symlinks",
WalkOptions::default().sort(true).follow_symlinks(true),
),
("max_depth", WalkOptions::default().sort(true).max_depth(1)),
];
for (name, options) in cases {
let walker = Walker::new(&fixture.root)
.threads(1)
.include("**/*")
.expect("valid include")
.exclude("**/*.tmp")
.expect("valid exclude")
.respect_git_ignore(true)
.error_policy(ErrorPolicy::Collect)
.options(options);
let native = walker.clone().collect().expect("native walk succeeds");
let (portable_entries, portable_errors) = collect_with_portable_backend(&walker);
assert_eq!(
describe_entries(native.entries(), &fixture.root),
describe_entries(&portable_entries, &fixture.root),
"native {name} differs from portable"
);
assert_eq!(
describe_errors(native.errors(), &fixture.root),
describe_errors(&portable_errors, &fixture.root),
"native {name} errors differ from portable"
);
if name == "follow_symlinks" {
assert_eq!(
describe_errors(native.errors(), &fixture.root),
vec![(PathBuf::from("dangling-link"), "metadata")]
);
} else {
assert!(native.errors().is_empty(), "native {name} errors");
}
}
}
#[cfg(any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
))]
#[test]
fn native_backend_matches_portable_unreadable_directory_error() {
use std::os::unix::fs::{MetadataExt, PermissionsExt};
let fixture = Fixture::new();
if fs::metadata(&fixture.root)
.expect("fixture root metadata")
.uid()
== 0
{
return;
}
fixture.write("visible.rs");
fixture.write("locked/secret.rs");
let locked = fixture.root.join("locked");
let original_permissions = fs::metadata(&locked)
.expect("locked directory metadata")
.permissions();
fs::set_permissions(&locked, fs::Permissions::from_mode(0o0))
.expect("make locked directory unreadable");
let walker = Walker::new(&fixture.root)
.threads(1)
.error_policy(ErrorPolicy::Collect)
.options(WalkOptions::default().sort(true));
let native = walker.clone().collect().expect("native walk succeeds");
let (portable_entries, portable_errors) = collect_with_portable_backend(&walker);
fs::set_permissions(&locked, original_permissions)
.expect("restore locked directory permissions");
assert_eq!(
describe_entries(native.entries(), &fixture.root),
describe_entries(&portable_entries, &fixture.root),
"native unreadable-directory entries differ from portable"
);
assert_eq!(
describe_errors(native.errors(), &fixture.root),
describe_errors(&portable_errors, &fixture.root),
"native unreadable-directory errors differ from portable"
);
assert_eq!(
describe_errors(native.errors(), &fixture.root),
vec![(PathBuf::from("locked"), "read_dir")]
);
}
#[cfg(any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
))]
type DescribedEntry = (PathBuf, bool, bool, usize, Option<(u64, bool, bool)>);
#[cfg(any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
))]
fn collect_with_portable_backend(walker: &Walker) -> (Vec<WalkEntry>, Vec<super::WalkError>) {
let mut state = super::WalkState::new(walker);
state
.walk_directory(&super::StdBackend, walker.root.clone())
.expect("portable walk succeeds");
if walker.options.sort {
state
.entries
.sort_by(|left, right| left.path.cmp(&right.path));
}
(state.entries, state.errors)
}
#[cfg(any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
))]
fn describe_entries(entries: &[WalkEntry], root: &Path) -> Vec<DescribedEntry> {
entries
.iter()
.map(|entry| {
(
entry
.path()
.strip_prefix(root)
.expect("entry belongs to fixture")
.to_path_buf(),
entry.is_dir(),
entry.is_symlink(),
entry.depth(),
entry.metadata().map(|metadata| {
(
metadata.len(),
metadata.file_type().is_dir(),
metadata.file_type().is_symlink(),
)
}),
)
})
.collect()
}
#[cfg(any(
all(feature = "native-macos", target_os = "macos"),
all(feature = "native-linux", target_os = "linux")
))]
fn describe_errors(errors: &[super::WalkError], root: &Path) -> Vec<(PathBuf, &'static str)> {
errors
.iter()
.map(|error| {
(
error
.path()
.strip_prefix(root)
.expect("error belongs to fixture")
.to_path_buf(),
error.operation(),
)
})
.collect()
}
#[test]
fn collect_and_skip_distinguish_recoverable_root_errors() {
let missing = std::env::temp_dir().join(format!(
"ferralk-missing-{}",
NEXT_FIXTURE.fetch_add(1, Ordering::Relaxed)
));
let collected = Walker::new(&missing)
.error_policy(ErrorPolicy::Collect)
.collect()
.expect("collect policy retains the error");
assert_eq!(collected.errors().len(), 1);
assert!(
Walker::new(&missing)
.error_policy(ErrorPolicy::Skip)
.collect()
.expect("skip policy ignores the error")
.errors()
.is_empty()
);
assert!(
Walker::new(&missing)
.error_policy(ErrorPolicy::Abort)
.collect()
.is_err()
);
}
#[test]
fn cancellation_returns_a_partial_result_without_an_io_error() {
let fixture = Fixture::new();
fixture.write("src/main.rs");
let cancellation = CancellationToken::default();
cancellation.cancel();
let result = Walker::new(&fixture.root)
.cancellation(cancellation)
.collect()
.expect("cancellation is a normal partial result");
assert!(result.was_cancelled());
assert!(result.entries().is_empty());
assert!(result.errors().is_empty());
}
#[test]
fn stream_yields_filtered_entries_incrementally_and_honours_cancellation() {
let fixture = Fixture::new();
fixture.write("src/main.rs");
fixture.write("src/lib.txt");
let mut stream = Walker::new(&fixture.root)
.include("**/*.rs")
.expect("valid include")
.stream();
let entries = stream
.by_ref()
.map(|entry| entry.expect("fixture has no I/O errors"))
.collect::<Vec<_>>();
assert_eq!(
relative_paths(&entries, &fixture.root),
vec![PathBuf::from("src/main.rs")]
);
assert!(!stream.was_cancelled());
let cancellation = CancellationToken::default();
cancellation.cancel();
let mut cancelled = Walker::new(&fixture.root)
.cancellation(cancellation)
.stream();
assert!(cancelled.next().is_none());
assert!(cancelled.was_cancelled());
}
#[cfg(unix)]
#[test]
fn symlink_policy_prevents_or_deduplicates_directory_cycles() {
use std::os::unix::fs::symlink;
let fixture = Fixture::new();
fixture.write("real/inside.txt");
symlink("real", fixture.root.join("linked")).expect("create directory symlink");
let without_following = Walker::new(&fixture.root)
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds");
assert!(
!relative_paths(without_following.entries(), &fixture.root)
.contains(&PathBuf::from("linked/inside.txt"))
);
let link = without_following
.entries()
.iter()
.find(|entry| entry.basename() == Some(std::ffi::OsStr::new("linked")))
.expect("symlink is reported");
assert!(link.is_symlink());
assert_eq!(link.kind(), WalkEntryKind::Symlink);
let with_following = Walker::new(&fixture.root)
.options(WalkOptions::default().follow_symlinks(true).sort(true))
.collect()
.expect("walk succeeds");
assert_eq!(
relative_paths(with_following.entries(), &fixture.root)
.iter()
.filter(|path| path.file_name().is_some_and(|name| name == "inside.txt"))
.count(),
1
);
}
#[cfg(unix)]
#[test]
fn parallel_collect_retains_concurrent_metadata_errors() {
use std::os::unix::fs::symlink;
let fixture = Fixture::new();
fixture.write("left/ok.txt");
fixture.write("right/ok.txt");
symlink("missing-left", fixture.root.join("left/dangling"))
.expect("create left dangling symlink");
symlink("missing-right", fixture.root.join("right/dangling"))
.expect("create right dangling symlink");
let options = WalkOptions::default().follow_symlinks(true).sort(true);
let serial = Walker::new(&fixture.root)
.threads(1)
.options(options)
.error_policy(ErrorPolicy::Collect)
.collect()
.expect("serial walk retains errors");
let parallel = Walker::new(&fixture.root)
.threads(4)
.options(options)
.error_policy(ErrorPolicy::Collect)
.collect()
.expect("parallel walk retains errors");
let error_paths = |result: &super::WalkResult| {
let mut errors = result
.errors()
.iter()
.map(|error| {
(
error.operation(),
error
.path()
.strip_prefix(&fixture.root)
.expect("error is rooted in fixture")
.to_path_buf(),
)
})
.collect::<Vec<_>>();
errors.sort_unstable();
errors
};
assert_eq!(error_paths(¶llel), error_paths(&serial));
assert_eq!(parallel.errors().len(), 2);
}
#[cfg(unix)]
#[test]
fn parallel_abort_returns_an_error_and_cancels_the_shared_token() {
use std::os::unix::fs::symlink;
let fixture = Fixture::new();
fixture.write("left/ok.txt");
fixture.write("right/ok.txt");
symlink("missing-left", fixture.root.join("left/dangling"))
.expect("create left dangling symlink");
symlink("missing-right", fixture.root.join("right/dangling"))
.expect("create right dangling symlink");
let cancellation = CancellationToken::default();
let error = Walker::new(&fixture.root)
.threads(4)
.options(WalkOptions::default().follow_symlinks(true))
.error_policy(ErrorPolicy::Abort)
.cancellation(cancellation.clone())
.collect()
.expect_err("abort policy returns the first metadata error");
assert_eq!(error.operation(), "metadata");
assert!(cancellation.is_cancelled());
}
#[cfg(target_os = "linux")]
#[test]
fn preserves_non_utf8_native_paths() {
use std::os::unix::ffi::OsStringExt;
let fixture = Fixture::new();
let name = std::ffi::OsString::from_vec(vec![b'n', 0xFF]);
fixture.write(PathBuf::from(&name));
let result = Walker::new(&fixture.root)
.options(WalkOptions::default().sort(true))
.collect()
.expect("walk succeeds");
assert_eq!(
relative_paths(result.entries(), &fixture.root),
vec![PathBuf::from(name)]
);
}
}