use crate::walker::WalkerItem;
use crate::{
config::Config,
error::{Error, Result},
fs_utils::security::{FileIdentity, get_stdout_identity, output_protection_threshold},
walker::WalkerEntry,
};
use log::debug;
use std::path::PathBuf;
use super::builder::configured_walk_builder;
use super::output_guard;
const TOTAL_ENTRIES_MINIMUM: usize = 10_000;
const TOTAL_ENTRIES_SAFETY_FACTOR: usize = 2;
pub struct DirectoryWalker {
inner: ignore::Walk,
root: PathBuf,
stdout_identity: Option<FileIdentity>,
max_files: usize,
processed_count: usize,
total_yielded: usize,
max_total_entries: usize,
max_files_warned: bool,
max_total_warned: bool,
output_threshold: std::time::Duration,
}
impl DirectoryWalker {
pub fn new(config: &Config) -> Result<Self> {
debug_assert!(
config.root().is_absolute(),
"Config root must be absolute path"
);
let builder = configured_walk_builder(config.root(), config);
let stdout_identity = get_stdout_identity();
let max_files = config.max_files();
let max_total_entries = max_files
.saturating_mul(TOTAL_ENTRIES_SAFETY_FACTOR)
.max(TOTAL_ENTRIES_MINIMUM);
Ok(Self {
inner: builder.build(),
root: config.root().to_path_buf(),
stdout_identity,
max_files,
processed_count: 0,
total_yielded: 0,
max_total_entries,
max_files_warned: false,
max_total_warned: false,
output_threshold: output_protection_threshold(),
})
}
#[cfg(test)]
#[must_use]
pub const fn processed_count(&self) -> usize {
self.processed_count
}
#[cfg(test)]
#[must_use]
pub const fn total_yielded(&self) -> usize {
self.total_yielded
}
fn warn_max_files_once(&mut self) {
if !self.max_files_warned {
self.max_files_warned = true;
eprintln!(
"\nâš Warning: Reached maximum file limit ({} files)",
self.max_files
);
eprintln!(" Remaining files will be skipped.");
eprintln!(
" Consider using --max-files, --max-depth, \
or more specific patterns.\n"
);
}
}
fn warn_max_total_once(&mut self) {
if !self.max_total_warned {
self.max_total_warned = true;
eprintln!(
"\nâš Warning: Reached total entry safety limit ({} entries)",
self.max_total_entries
);
eprintln!(
" This typically indicates an unusually large \
directory structure."
);
eprintln!(
" Consider using --max-depth or more specific \
patterns to narrow the scope.\n"
);
}
}
}
impl Iterator for DirectoryWalker {
type Item = crate::walker::WalkerItem;
fn next(&mut self) -> Option<Self::Item> {
if self.total_yielded >= self.max_total_entries {
self.warn_max_total_once();
return None;
}
while let Some(entry_result) = self.inner.next() {
let entry = match entry_result {
Ok(e) => e,
Err(e) => {
return Some(WalkerItem::Error(Error::Walker {
message: format!("Directory walk error: {e}"),
}));
}
};
let file_type = entry.file_type();
let is_file = file_type.is_some_and(|ft| ft.is_file());
let is_dir = file_type.is_some_and(|ft| ft.is_dir());
if !is_file && !is_dir {
continue;
}
if entry.path() == self.root {
debug!("Skipping root directory entry: {}", entry.path().display());
continue;
}
if is_file {
if self.processed_count >= self.max_files {
self.warn_max_files_once();
continue;
}
if let Some(ref stdout_id) = self.stdout_identity {
match entry.metadata() {
Ok(metadata) => {
if output_guard::matches_stdout(entry.path(), &metadata, stdout_id) {
continue;
}
}
Err(e) => {
debug!(
"Failed to get metadata for {}: {}",
entry.path().display(),
e
);
}
}
}
if let Ok(metadata) = entry.metadata() {
if output_guard::is_recently_created_empty(
entry.path(),
&metadata,
self.output_threshold,
) {
continue;
}
}
self.processed_count += 1;
}
let path = entry.path().to_path_buf();
let relative_path = path.strip_prefix(&self.root).unwrap_or(&path).to_path_buf();
self.total_yielded += 1;
return Some(WalkerItem::Entry(WalkerEntry {
path,
relative_path,
is_dir,
}));
}
None
}
fn size_hint(&self) -> (usize, Option<usize>) {
(
0,
Some(self.max_total_entries.saturating_sub(self.total_yielded)),
)
}
}
impl std::iter::FusedIterator for DirectoryWalker {}
#[cfg(test)]
#[cfg(feature = "cli")]
mod tests {
use super::*;
use crate::{cli::Args, config::Config};
use clap::Parser;
use proptest::prelude::*;
use serial_test::serial;
use std::fs;
use tempfile::TempDir;
proptest! {
#[test]
#[serial]
fn prop_never_exceeds_max_files(
max_files in 1usize..20,
num_files in 10usize..30
) {
let temp = TempDir::new().unwrap();
std::env::set_current_dir(temp.path()).unwrap();
for i in 0..num_files {
fs::write(
temp.path().join(format!("file_{i}.txt")),
"content",
).unwrap();
}
let args = Args::parse_from([
"test", "--max-files", &max_files.to_string(),
]);
let config = Config::from_args(&args).unwrap();
let walker = DirectoryWalker::new(&config).unwrap();
let file_count = walker
.filter_map(WalkerItem::into_entry)
.filter(|entry| !entry.is_dir)
.count();
assert!(
file_count <= max_files,
"Walker yielded {file_count} files but max_files \
is {max_files}"
);
}
#[test]
#[serial]
fn prop_size_hint_upper_bound_valid(num_files in 1usize..20) {
let temp = TempDir::new().unwrap();
std::env::set_current_dir(temp.path()).unwrap();
for i in 0..num_files {
fs::write(
temp.path().join(format!("file_{i}.txt")),
"content",
).unwrap();
}
let args = Args::parse_from(["test"]);
let config = Config::from_args(&args).unwrap();
let mut walker = DirectoryWalker::new(&config).unwrap();
let (lower, upper) = walker.size_hint();
assert_eq!(lower, 0, "Lower bound should be 0 (unknown)");
assert!(upper.is_some(), "Upper bound should be Some");
let initial_upper = upper.unwrap();
let _ = walker.next();
let (_, upper_after) = walker.size_hint();
if let Some(upper_val) = upper_after {
assert!(
upper_val <= initial_upper,
"Upper bound should not increase after consuming \
items"
);
}
}
#[test]
#[serial]
fn prop_relative_paths_start_from_root(
num_files in 1usize..10
) {
let temp = TempDir::new().unwrap();
let root = temp.path().canonicalize().unwrap();
std::env::set_current_dir(&root).unwrap();
fs::create_dir_all(root.join("a/b/c")).unwrap();
for i in 0..num_files {
fs::write(
root.join(format!("a/b/c/file_{i}.txt")),
"content",
).unwrap();
}
let args = Args::parse_from(["test"]);
let config = Config::from_args(&args).unwrap();
let walker = DirectoryWalker::new(&config).unwrap();
for item in walker {
if let Some(entry) = item.into_entry() {
assert!(
entry.relative_path.is_relative(),
"Relative path should be relative, got: {:?}",
entry.relative_path
);
assert!(
!entry.relative_path.as_os_str().is_empty(),
"Relative path should not be empty (root entry \
should be skipped), got entry: {:?}",
entry.path
);
assert!(
entry.path.is_absolute(),
"Absolute path should be absolute, got: {:?}",
entry.path
);
}
}
}
#[test]
#[serial]
fn prop_directories_dont_count_toward_limit(
max_files in 1usize..5,
num_dirs in 5usize..10
) {
let temp = TempDir::new().unwrap();
std::env::set_current_dir(temp.path()).unwrap();
for i in 0..num_dirs {
fs::create_dir_all(
temp.path().join(format!("dir_{i}")),
).unwrap();
}
for i in 0..max_files {
fs::write(
temp.path().join(format!("file_{i}.txt")),
"content",
).unwrap();
}
let args = Args::parse_from([
"test", "--max-files", &max_files.to_string(),
]);
let config = Config::from_args(&args).unwrap();
let mut walker = DirectoryWalker::new(&config).unwrap();
let mut file_count = 0;
let mut dir_count = 0;
for item in &mut walker {
if let Some(entry) = item.into_entry() {
if entry.is_dir {
dir_count += 1;
} else {
file_count += 1;
}
}
}
assert_eq!(
file_count, max_files,
"Should process exactly max_files files"
);
assert!(
dir_count > 0,
"Should process directories without counting them"
);
}
#[test]
#[serial]
fn prop_never_yields_root_entry(
num_files in 0usize..10,
num_dirs in 0usize..5
) {
let temp = TempDir::new().unwrap();
let root = temp.path().canonicalize().unwrap();
std::env::set_current_dir(&root).unwrap();
for i in 0..num_dirs {
fs::create_dir_all(
root.join(format!("dir_{i}")),
).unwrap();
}
for i in 0..num_files {
fs::write(
root.join(format!("file_{i}.txt")),
"content",
).unwrap();
}
let args = Args::parse_from(["test"]);
let config = Config::from_args(&args).unwrap();
let walker = DirectoryWalker::new(&config).unwrap();
for item in walker {
if let Some(entry) = item.into_entry() {
assert_ne!(
entry.path, root,
"Walker should never yield the root directory \
itself"
);
assert!(
!entry.relative_path.as_os_str().is_empty(),
"Walker should never yield an entry with an \
empty relative path"
);
}
}
}
#[test]
#[serial]
fn prop_total_entries_bounded(
max_files in 1usize..10,
num_files in 5usize..20,
num_dirs in 5usize..20
) {
let temp = TempDir::new().unwrap();
std::env::set_current_dir(temp.path()).unwrap();
for i in 0..num_dirs {
fs::create_dir_all(
temp.path().join(format!("dir_{i}")),
).unwrap();
}
for i in 0..num_files {
fs::write(
temp.path().join(format!("file_{i}.txt")),
"content",
).unwrap();
}
let args = Args::parse_from([
"test", "--max-files", &max_files.to_string(),
]);
let config = Config::from_args(&args).unwrap();
let mut walker = DirectoryWalker::new(&config).unwrap();
let max_total = walker.max_total_entries;
let mut total_count = 0;
for item in &mut walker {
if item.into_entry().is_some() {
total_count += 1;
}
}
assert!(
total_count <= max_total,
"Total entries ({total_count}) should not exceed \
safety cap ({max_total})"
);
assert_eq!(
walker.total_yielded(),
total_count,
"Internal counter should match actual yielded count"
);
}
}
#[test]
#[serial]
fn test_directory_walker_creation() {
let temp = TempDir::new().unwrap();
std::env::set_current_dir(temp.path()).unwrap();
let args = Args::parse_from(["test"]);
let config = Config::from_args(&args).unwrap();
let walker = DirectoryWalker::new(&config).unwrap();
assert_eq!(walker.max_files, 1_000_000);
assert!(!walker.max_files_warned);
assert!(!walker.max_total_warned);
assert_eq!(walker.processed_count(), 0);
assert_eq!(walker.total_yielded(), 0);
}
#[test]
#[serial]
fn test_max_files_one_sets_warned_flag() {
let temp = TempDir::new().unwrap();
std::env::set_current_dir(temp.path()).unwrap();
fs::write(temp.path().join("first.txt"), "content").unwrap();
fs::write(temp.path().join("second.txt"), "content").unwrap();
let args = Args::parse_from(["test", "--max-files", "1"]);
let config = Config::from_args(&args).unwrap();
let mut walker = DirectoryWalker::new(&config).unwrap();
assert!(!walker.max_files_warned);
let entries: Vec<_> = walker.by_ref().filter_map(WalkerItem::into_entry).collect();
let file_count = entries.iter().filter(|e| !e.is_dir).count();
assert_eq!(file_count, 1, "max_files=1 should yield exactly one file");
assert!(
walker.max_files_warned,
"max_files_warned should be true when max_files is 1 and \
a second file was encountered"
);
}
#[test]
#[serial]
fn test_max_files_optimization_skips_iterator() {
let temp = TempDir::new().unwrap();
fs::create_dir_all(temp.path().join("a/b/c")).unwrap();
fs::write(temp.path().join("a/file1.txt"), "1").unwrap();
fs::write(temp.path().join("a/b/file2.txt"), "2").unwrap();
fs::write(temp.path().join("a/b/c/file3.txt"), "3").unwrap();
std::env::set_current_dir(temp.path()).unwrap();
let args = Args::parse_from(["bench", "--max-files", "1"]);
let config = Config::from_args(&args).unwrap();
let mut walker = DirectoryWalker::new(&config).unwrap();
let mut file_count = 0;
while let Some(item) = walker.next() {
if let Some(entry) = item.into_entry() {
if !entry.is_dir {
file_count += 1;
}
}
if walker.processed_count() >= walker.max_files {
break;
}
}
assert_eq!(walker.processed_count(), 1);
assert_eq!(file_count, 1);
let remaining: Vec<_> = walker.by_ref().filter_map(WalkerItem::into_entry).collect();
let remaining_files = remaining.iter().filter(|e| !e.is_dir).count();
assert_eq!(
remaining_files, 0,
"No more files should be yielded after the limit"
);
}
#[test]
#[serial]
fn test_processed_count_increases_only_for_files() {
let temp = TempDir::new().unwrap();
fs::create_dir_all(temp.path().join("dir1/dir2")).unwrap();
fs::write(temp.path().join("dir1/file1.txt"), "content").unwrap();
fs::write(temp.path().join("dir1/dir2/file2.txt"), "content").unwrap();
std::env::set_current_dir(temp.path()).unwrap();
let args = Args::parse_from(["test"]);
let config = Config::from_args(&args).unwrap();
let mut walker = DirectoryWalker::new(&config).unwrap();
let mut file_entries = 0;
let mut dir_entries = 0;
for item in &mut walker {
if let Some(entry) = item.into_entry() {
if entry.is_dir {
dir_entries += 1;
} else {
file_entries += 1;
}
}
}
assert_eq!(walker.processed_count(), file_entries);
assert_eq!(
walker.processed_count(),
2,
"Should have processed exactly 2 files"
);
assert!(dir_entries > 0, "Should have encountered directories");
}
#[test]
#[serial]
fn test_size_hint_decreases_as_items_consumed() {
let temp = TempDir::new().unwrap();
fs::write(temp.path().join("file1.txt"), "content").unwrap();
fs::write(temp.path().join("file2.txt"), "content").unwrap();
std::env::set_current_dir(temp.path()).unwrap();
let args = Args::parse_from(["test", "--max-files", "10"]);
let config = Config::from_args(&args).unwrap();
let mut walker = DirectoryWalker::new(&config).unwrap();
let (_, upper1) = walker.size_hint();
let _ = walker.next(); let (_, upper2) = walker.size_hint();
assert!(upper1.is_some() && upper2.is_some());
assert!(
upper2.unwrap() <= upper1.unwrap(),
"Upper bound should not increase after consuming items"
);
}
#[test]
#[serial]
fn test_early_termination_on_max_files() {
let temp = TempDir::new().unwrap();
for i in 0..10 {
fs::write(temp.path().join(format!("file_{i}.txt")), "content").unwrap();
}
std::env::set_current_dir(temp.path()).unwrap();
let args = Args::parse_from(["test", "--max-files", "3"]);
let config = Config::from_args(&args).unwrap();
let walker = DirectoryWalker::new(&config).unwrap();
let file_count = walker
.filter_map(WalkerItem::into_entry)
.filter(|entry| !entry.is_dir)
.count();
assert_eq!(file_count, 3, "Should stop at max_files limit");
}
#[test]
#[serial]
fn test_walker_handles_empty_directory() {
let temp = TempDir::new().unwrap();
std::env::set_current_dir(temp.path()).unwrap();
let args = Args::parse_from(["test"]);
let config = Config::from_args(&args).unwrap();
let walker = DirectoryWalker::new(&config).unwrap();
let count = walker.count();
assert_eq!(count, 0, "Empty directory should yield no entries");
}
#[test]
#[serial]
fn test_walker_skips_root_directory_entry() {
let temp = TempDir::new().unwrap();
let root = temp.path().canonicalize().unwrap();
std::env::set_current_dir(&root).unwrap();
fs::write(root.join("file.txt"), "content").unwrap();
let args = Args::parse_from(["test"]);
let config = Config::from_args(&args).unwrap();
let walker = DirectoryWalker::new(&config).unwrap();
let entries: Vec<_> = walker.filter_map(WalkerItem::into_entry).collect();
assert_eq!(entries.len(), 1, "Should yield exactly the one file");
assert!(
!entries[0].is_dir,
"The single entry should be a file, not the root dir"
);
assert_eq!(
entries[0].relative_path,
PathBuf::from("file.txt"),
"Relative path should be the filename"
);
}
#[test]
#[serial]
fn test_walker_yields_subdirectories_but_not_root() {
let temp = TempDir::new().unwrap();
let root = temp.path().canonicalize().unwrap();
std::env::set_current_dir(&root).unwrap();
fs::create_dir_all(root.join("subdir")).unwrap();
fs::write(root.join("subdir/file.txt"), "content").unwrap();
let args = Args::parse_from(["test"]);
let config = Config::from_args(&args).unwrap();
let walker = DirectoryWalker::new(&config).unwrap();
let entries: Vec<_> = walker.filter_map(WalkerItem::into_entry).collect();
let dir_entries: Vec<_> = entries.iter().filter(|e| e.is_dir).collect();
let file_entries = entries.iter().filter(|e| !e.is_dir);
assert_eq!(dir_entries.len(), 1, "Should yield one subdirectory");
assert_eq!(
dir_entries[0].relative_path,
PathBuf::from("subdir"),
"Subdirectory relative path should be 'subdir'"
);
assert_eq!(
file_entries.filter(|e| !e.is_dir).count(),
1,
"Should yield one file"
);
for entry in &entries {
assert_ne!(
entry.path, root,
"No entry should be the root directory itself"
);
assert!(
!entry.relative_path.as_os_str().is_empty(),
"No entry should have an empty relative path"
);
}
}
#[test]
#[serial]
fn test_walker_respects_dotfiles_setting() {
let temp = TempDir::new().unwrap();
fs::write(temp.path().join(".hidden"), "content").unwrap();
fs::write(temp.path().join("visible.txt"), "content").unwrap();
std::env::set_current_dir(temp.path()).unwrap();
let args = Args::parse_from(["test", "--no-dotfiles"]);
let config = Config::from_args(&args).unwrap();
let walker = DirectoryWalker::new(&config).unwrap();
let entries: Vec<_> = walker.filter_map(WalkerItem::into_entry).collect();
let has_hidden = entries
.iter()
.any(|e| e.path.file_name().unwrap().to_str().unwrap() == ".hidden");
assert!(!has_hidden, "Should not include hidden files");
let args = Args::parse_from(["test", "--dotfiles"]);
let config = Config::from_args(&args).unwrap();
let walker = DirectoryWalker::new(&config).unwrap();
let entries: Vec<_> = walker.filter_map(WalkerItem::into_entry).collect();
let has_hidden = entries
.iter()
.any(|e| e.path.file_name().unwrap().to_str().unwrap() == ".hidden");
assert!(has_hidden, "Should include hidden files with --dotfiles");
}
#[test]
#[serial]
fn test_directories_yielded_after_file_limit() {
let temp = TempDir::new().unwrap();
for i in 0..5 {
fs::create_dir_all(temp.path().join(format!("dir_{i}"))).unwrap();
}
for i in 0..5 {
fs::write(temp.path().join(format!("file_{i}.txt")), "content").unwrap();
}
std::env::set_current_dir(temp.path()).unwrap();
let args = Args::parse_from(["test", "--max-files", "1"]);
let config = Config::from_args(&args).unwrap();
let walker = DirectoryWalker::new(&config).unwrap();
let entries: Vec<_> = walker.filter_map(WalkerItem::into_entry).collect();
let file_count = entries.iter().filter(|e| !e.is_dir).count();
let dir_count = entries.iter().filter(|e| e.is_dir).count();
assert_eq!(file_count, 1, "Should yield exactly max_files files");
assert_eq!(
dir_count, 5,
"Should yield ALL directories regardless of file limit"
);
}
#[test]
#[serial]
fn test_max_files_one_yields_one_file_plus_directories() {
let temp = TempDir::new().unwrap();
fs::create_dir_all(temp.path().join("dir1")).unwrap();
fs::create_dir_all(temp.path().join("dir2")).unwrap();
fs::write(temp.path().join("file.txt"), "content").unwrap();
fs::write(temp.path().join("dir1/nested.txt"), "content").unwrap();
std::env::set_current_dir(temp.path()).unwrap();
let args = Args::parse_from(["test", "--max-files", "1"]);
let config = Config::from_args(&args).unwrap();
let walker = DirectoryWalker::new(&config).unwrap();
let entries: Vec<_> = walker.filter_map(WalkerItem::into_entry).collect();
let file_count = entries.iter().filter(|e| !e.is_dir).count();
let dir_count = entries.iter().filter(|e| e.is_dir).count();
assert_eq!(file_count, 1, "max_files=1 should yield exactly one file");
assert_eq!(dir_count, 2, "Directories should still be yielded");
}
#[test]
#[serial]
fn test_max_files_rejects_zero() {
let temp = TempDir::new().unwrap();
std::env::set_current_dir(temp.path()).unwrap();
let args = Args::parse_from(["test", "--max-files", "0"]);
let result = Config::from_args(&args);
assert!(
result.is_err(),
"max_files=0 should be rejected by validation"
);
}
#[test]
#[serial]
fn test_total_yielded_tracks_both_files_and_dirs() {
let temp = TempDir::new().unwrap();
fs::create_dir_all(temp.path().join("dir1")).unwrap();
fs::write(temp.path().join("file1.txt"), "content").unwrap();
fs::write(temp.path().join("dir1/file2.txt"), "content").unwrap();
std::env::set_current_dir(temp.path()).unwrap();
let args = Args::parse_from(["test"]);
let config = Config::from_args(&args).unwrap();
let mut walker = DirectoryWalker::new(&config).unwrap();
let mut count = 0;
for item in &mut walker {
if item.into_entry().is_some() {
count += 1;
}
}
assert_eq!(
walker.total_yielded(),
count,
"total_yielded must equal actual entries yielded"
);
assert!(
walker.total_yielded() >= walker.processed_count(),
"total_yielded must be >= processed_count (files only)"
);
}
#[test]
#[serial]
fn test_max_total_entries_computation() {
let temp = TempDir::new().unwrap();
std::env::set_current_dir(temp.path()).unwrap();
let args = Args::parse_from(["test", "--max-files", "1"]);
let config = Config::from_args(&args).unwrap();
let walker = DirectoryWalker::new(&config).unwrap();
assert_eq!(
walker.max_total_entries, TOTAL_ENTRIES_MINIMUM,
"Small max_files should use TOTAL_ENTRIES_MINIMUM"
);
let args = Args::parse_from(["test", "--max-files", "100000"]);
let config = Config::from_args(&args).unwrap();
let walker = DirectoryWalker::new(&config).unwrap();
assert_eq!(
walker.max_total_entries,
100_000 * TOTAL_ENTRIES_SAFETY_FACTOR,
"Large max_files should use max_files * safety factor"
);
}
#[test]
#[serial]
fn test_warning_flags_initially_false() {
let temp = TempDir::new().unwrap();
std::env::set_current_dir(temp.path()).unwrap();
let args = Args::parse_from(["test"]);
let config = Config::from_args(&args).unwrap();
let walker = DirectoryWalker::new(&config).unwrap();
assert!(!walker.max_files_warned);
assert!(!walker.max_total_warned);
}
#[test]
#[serial]
fn test_fused_iterator_returns_none_after_exhaustion() {
let temp = TempDir::new().unwrap();
std::env::set_current_dir(temp.path()).unwrap();
fs::write(temp.path().join("file.txt"), "content").unwrap();
let args = Args::parse_from(["test"]);
let config = Config::from_args(&args).unwrap();
let mut walker = DirectoryWalker::new(&config).unwrap();
while walker.next().is_some() {}
assert!(walker.next().is_none());
assert!(walker.next().is_none());
}
}