use crate::filters::compare_entries;
use crate::formatter::OutputFormat as FormatterOutputFormat;
use crate::stats::{calculate_dir_size, count_lines};
use crate::{Error, FilterOptions, Result};
use colored::*;
use ignore::gitignore::{Gitignore, GitignoreBuilder};
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use std::fs::{self, Metadata};
use std::io::{self, BufWriter, Write};
use std::path::{Path, PathBuf};
use std::time::SystemTime;
#[cfg(unix)]
use libc;
#[cfg(unix)]
use std::ffi::{CStr, CString, OsStr};
#[cfg(unix)]
use std::os::unix::ffi::OsStrExt;
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
#[cfg(windows)]
use std::os::windows::ffi::OsStrExt;
#[cfg(windows)]
use winapi::um::fileapi::{FindClose, FindFirstFileW, FindNextFileW};
#[cfg(windows)]
use winapi::um::minwinbase::WIN32_FIND_DATAW;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TreeEntry {
pub name: String,
pub path: PathBuf,
pub size: u64,
pub line_count: u64,
pub modified: SystemTime,
pub is_dir: bool,
pub is_symlink: bool,
pub is_executable: bool,
pub children: Vec<TreeEntry>,
pub depth: usize,
}
#[derive(Debug, Clone, Copy)]
enum WalkerMode {
FastPath,
Standard,
Full,
}
pub struct Walker {
root: PathBuf,
filter_opts: FilterOptions,
gitignore: Option<Gitignore>,
thread_count: usize,
max_file_size: u64,
show_lines: bool,
dir_sizes: bool,
mode: WalkerMode,
}
impl Walker {
pub fn new(root: &Path, filter_opts: FilterOptions, thread_count: usize) -> Result<Self> {
let root = root.canonicalize()?;
let gitignore = if filter_opts.gitignore {
load_gitignore(&root)?
} else {
None
};
let mode = Self::determine_mode(&filter_opts, &gitignore);
log::debug!("Walker mode selected: {:?}", mode);
Ok(Self {
root,
filter_opts,
gitignore,
thread_count: if thread_count == 0 {
num_cpus::get()
} else {
thread_count
},
max_file_size: 1_073_741_824, show_lines: false,
dir_sizes: false,
mode,
})
}
pub fn set_max_file_size(&mut self, size: u64) {
self.max_file_size = size;
}
pub fn enable_line_counting(&mut self) {
self.show_lines = true;
if matches!(self.mode, WalkerMode::FastPath) {
self.mode = WalkerMode::Standard;
}
}
pub fn enable_dir_sizes(&mut self) {
self.dir_sizes = true;
self.mode = WalkerMode::Full;
}
fn determine_mode(filter_opts: &FilterOptions, gitignore: &Option<Gitignore>) -> WalkerMode {
if filter_opts.search.is_some()
|| filter_opts.min_size.is_some()
|| filter_opts.max_size.is_some()
|| filter_opts.newer_than.is_some()
|| filter_opts.older_than.is_some()
|| gitignore.is_some()
{
return WalkerMode::Full;
}
if filter_opts.include.is_some()
|| filter_opts.exclude.is_some()
|| filter_opts.only_dirs
|| filter_opts.only_files
|| filter_opts.sort_by.is_some()
|| filter_opts.max_dirs.is_some()
|| filter_opts.max_files.is_some()
{
return WalkerMode::Standard;
}
WalkerMode::FastPath
}
pub fn walk(&mut self) -> Result<Vec<TreeEntry>> {
match self.mode {
WalkerMode::FastPath => self.walk_fast_path(),
WalkerMode::Standard => self.walk_standard(),
WalkerMode::Full => self.walk_full(),
}
}
#[cfg(unix)]
fn walk_fast_path(&mut self) -> Result<Vec<TreeEntry>> {
unsafe { self.walk_fast_unix() }
}
#[cfg(not(unix))]
fn walk_fast_path(&mut self) -> Result<Vec<TreeEntry>> {
self.walk_standard()
}
#[cfg(unix)]
unsafe fn walk_fast_unix(&mut self) -> Result<Vec<TreeEntry>> {
let root = self.root.clone();
self.walk_fast_unix_recursive(&root, 0)
}
#[cfg(unix)]
unsafe fn walk_fast_unix_recursive(
&mut self,
path: &Path,
depth: usize,
) -> Result<Vec<TreeEntry>> {
if let Some(max_depth) = self.filter_opts.max_depth {
if depth > max_depth {
return Ok(vec![]);
}
}
let mut entry = self.create_entry_from_path(path, depth)?;
if entry.is_dir && depth < self.filter_opts.max_depth.unwrap_or(usize::MAX) {
let path_cstr = path_to_cstring(path)?;
let dir_handle = libc::opendir(path_cstr.as_ptr());
if dir_handle.is_null() {
return Ok(vec![entry]);
}
loop {
errno::set_errno(errno::Errno(0));
let dir_entry = libc::readdir(dir_handle);
if dir_entry.is_null() {
let err = errno::errno();
if err.0 != 0 {
libc::closedir(dir_handle);
return Err(Error::IoError(io::Error::last_os_error()));
}
break;
}
let d_name = (*dir_entry).d_name.as_ptr();
let name_bytes = CStr::from_ptr(d_name).to_bytes();
if name_bytes == b"." || name_bytes == b".." {
continue;
}
if !self.filter_opts.show_hidden && name_bytes.first() == Some(&b'.') {
continue;
}
let name = OsStr::from_bytes(name_bytes);
let child_path = path.join(name);
if let Ok(mut child_entries) = self.walk_fast_unix_recursive(&child_path, depth + 1)
{
if !child_entries.is_empty() {
entry.children.push(child_entries.remove(0));
}
}
}
libc::closedir(dir_handle);
}
Ok(vec![entry])
}
fn walk_standard(&mut self) -> Result<Vec<TreeEntry>> {
let metadata = fs::symlink_metadata(&self.root)?;
let mut root_entry = self.create_entry(&self.root, &metadata, 0)?;
if root_entry.is_dir {
self.process_directory_children(&mut root_entry)?;
}
if self.should_include(&self.root, &metadata) || !root_entry.children.is_empty() {
Ok(vec![root_entry])
} else {
Ok(vec![])
}
}
fn process_directory_children(&mut self, parent: &mut TreeEntry) -> Result<()> {
if let Some(max_depth) = self.filter_opts.max_depth {
if parent.depth >= max_depth {
return Ok(());
}
}
let children_paths = self.read_directory(&parent.path, parent.depth + 1)?;
for child_path in children_paths {
match self.process_entry(&child_path, parent.depth + 1) {
Ok(Some(mut child_entry)) => {
if child_entry.is_dir {
self.process_directory_children(&mut child_entry)?;
}
parent.children.push(child_entry);
}
Ok(None) => {} Err(e) => {
log::warn!("Error processing {:?}: {}", child_path, e);
if !self.filter_opts.gitignore {
return Err(e);
}
}
}
}
Ok(())
}
fn walk_full(&mut self) -> Result<Vec<TreeEntry>> {
let mut entries = self.walk_standard()?;
if self.dir_sizes {
self.calculate_dir_sizes(&mut entries)?;
}
Ok(entries)
}
fn create_entry_from_path(&self, path: &Path, depth: usize) -> Result<TreeEntry> {
let metadata = fs::symlink_metadata(path)?;
self.create_entry(path, &metadata, depth)
}
fn process_entry(&self, path: &Path, depth: usize) -> Result<Option<TreeEntry>> {
let metadata = fs::symlink_metadata(path)?;
if !self.should_include(path, &metadata) {
return Ok(None);
}
Ok(Some(self.create_entry(path, &metadata, depth)?))
}
fn create_entry(&self, path: &Path, metadata: &Metadata, depth: usize) -> Result<TreeEntry> {
let name = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("")
.to_string();
let size = if metadata.is_dir() {
0 } else {
metadata.len()
};
let line_count = if self.show_lines && metadata.is_file() && size <= self.max_file_size {
count_lines(path, self.max_file_size).unwrap_or(0)
} else {
0
};
let modified = metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH);
let is_symlink = metadata.is_symlink();
let is_executable = is_executable(metadata);
Ok(TreeEntry {
name,
path: path.to_path_buf(),
size,
line_count,
modified,
is_dir: metadata.is_dir(),
is_symlink,
is_executable,
children: Vec::new(),
depth,
})
}
fn read_directory(&self, path: &Path, depth: usize) -> Result<Vec<PathBuf>> {
let mut entries = Vec::new();
let mut dirs = Vec::new();
let mut files = Vec::new();
for entry in fs::read_dir(path)? {
let entry = entry?;
let path = entry.path();
let metadata = entry.metadata()?;
if !self.should_include(&path, &metadata) {
continue;
}
if metadata.is_dir() {
dirs.push((path, metadata));
} else {
files.push((path, metadata));
}
}
if let Some(sort_by) = self.filter_opts.sort_by {
let mut sort_entries: Vec<TreeEntry> = dirs
.iter()
.chain(files.iter())
.map(|(path, metadata)| TreeEntry {
name: path.file_name().unwrap().to_string_lossy().to_string(),
path: path.clone(),
size: metadata.len(),
line_count: 0,
modified: metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH),
is_dir: metadata.is_dir(),
is_symlink: metadata.is_symlink(),
is_executable: is_executable(metadata),
children: Vec::new(),
depth,
})
.collect();
sort_entries
.sort_by(|a, b| compare_entries(a, b, sort_by, self.filter_opts.reverse_sort));
entries = sort_entries.into_iter().map(|e| e.path).collect();
} else {
entries.extend(dirs.into_iter().map(|(p, _)| p));
entries.extend(files.into_iter().map(|(p, _)| p));
}
let mut limited = Vec::new();
let mut dir_count = 0;
let mut file_count = 0;
for path in entries {
let is_dir = path.is_dir();
if is_dir {
if let Some(max_dirs) = self.filter_opts.max_dirs {
if dir_count >= max_dirs {
continue;
}
}
dir_count += 1;
} else {
if let Some(max_files) = self.filter_opts.max_files {
if file_count >= max_files {
continue;
}
}
file_count += 1;
}
limited.push(path);
}
Ok(limited)
}
fn should_include(&self, path: &Path, metadata: &Metadata) -> bool {
if let Some(gitignore) = &self.gitignore {
if gitignore.matched(path, metadata.is_dir()).is_ignore() {
return false;
}
}
self.filter_opts.should_include(path, metadata)
}
fn calculate_dir_sizes(&self, entries: &mut [TreeEntry]) -> Result<()> {
if self.thread_count > 1 {
entries.par_iter_mut().try_for_each(|entry| {
if entry.is_dir {
entry.size = calculate_dir_size(&entry.path)?;
}
Walker::calculate_dir_sizes_recursive(&mut entry.children)
})?;
} else {
for entry in entries {
if entry.is_dir {
entry.size = calculate_dir_size(&entry.path)?;
}
Walker::calculate_dir_sizes_recursive(&mut entry.children)?;
}
}
Ok(())
}
fn calculate_dir_sizes_recursive(entries: &mut [TreeEntry]) -> Result<()> {
for entry in entries {
if entry.is_dir {
entry.size = calculate_dir_size(&entry.path)?;
}
Walker::calculate_dir_sizes_recursive(&mut entry.children)?;
}
Ok(())
}
}
pub struct StreamWalker<'a> {
filter_opts: FilterOptions,
stdout: BufWriter<Box<dyn Write + 'a>>,
format: FormatterOutputFormat,
show_size: bool,
show_lines: bool,
unicode: bool,
max_file_size: u64,
color_enabled: bool,
file_count: usize,
dir_count: usize,
}
impl<'a> StreamWalker<'a> {
pub fn new(
filter_opts: FilterOptions,
format: FormatterOutputFormat,
show_size: bool,
show_lines: bool,
unicode: bool,
) -> Self {
let stdout: Box<dyn Write> = Box::new(io::stdout());
let color_enabled = atty::is(atty::Stream::Stdout);
Self {
filter_opts,
stdout: BufWriter::with_capacity(8192, stdout),
format,
show_size,
show_lines,
unicode,
max_file_size: 1_073_741_824, color_enabled,
file_count: 0,
dir_count: 0,
}
}
pub fn with_writer(
filter_opts: FilterOptions,
format: FormatterOutputFormat,
show_size: bool,
show_lines: bool,
unicode: bool,
color_enabled: bool,
writer: Box<dyn Write + 'a>,
) -> Self {
Self {
filter_opts,
stdout: BufWriter::with_capacity(8192, writer),
format,
show_size,
show_lines,
unicode,
max_file_size: 1_073_741_824,
color_enabled,
file_count: 0,
dir_count: 0,
}
}
pub fn stream(&mut self, root: &Path) -> Result<()> {
match self.format {
FormatterOutputFormat::Plain => self.stream_plain(root),
FormatterOutputFormat::Tree => self.stream_tree(root),
FormatterOutputFormat::Json | FormatterOutputFormat::Csv => {
Err(Error::general(
"JSON/CSV output requires full tree building",
))
}
}
}
fn stream_plain(&mut self, root: &Path) -> Result<()> {
let walker = Walker::new(root, self.filter_opts.clone(), 1)?;
self.walk_and_print_plain(&walker, root, 0)?;
self.stdout.flush()?;
Ok(())
}
fn stream_tree(&mut self, root: &Path) -> Result<()> {
let walker = Walker::new(root, self.filter_opts.clone(), 1)?;
self.walk_and_print_tree(&walker, root, 0, &mut Vec::new())?;
writeln!(self.stdout)?;
writeln!(
self.stdout,
"{} {}, {} {}",
self.dir_count,
if self.dir_count == 1 {
"directory"
} else {
"directories"
},
self.file_count,
if self.file_count == 1 {
"file"
} else {
"files"
}
)?;
self.stdout.flush()?;
Ok(())
}
fn walk_and_print_plain(&mut self, walker: &Walker, path: &Path, depth: usize) -> Result<()> {
if let Some(max_depth) = walker.filter_opts.max_depth {
if depth > max_depth {
return Ok(());
}
}
writeln!(self.stdout, "{}", path.display())?;
if path.is_dir() && depth < walker.filter_opts.max_depth.unwrap_or(usize::MAX) {
let children = walker.read_directory(path, depth + 1)?;
for child in children {
self.walk_and_print_plain(walker, &child, depth + 1)?;
}
}
Ok(())
}
fn walk_and_print_tree(
&mut self,
walker: &Walker,
path: &Path,
depth: usize,
prefix: &mut Vec<bool>,
) -> Result<()> {
if let Some(max_depth) = walker.filter_opts.max_depth {
if depth > max_depth {
return Ok(());
}
}
if depth > 0 {
for (i, &is_last) in prefix.iter().enumerate() {
if i == prefix.len() - 1 {
write!(
self.stdout,
"{}",
if is_last {
if self.unicode { "└── " } else { "`-- " }
} else if self.unicode {
"├── "
} else {
"|-- "
}
)?;
} else {
write!(
self.stdout,
"{}",
if is_last {
" "
} else if self.unicode {
"│ "
} else {
"| "
}
)?;
}
}
}
let metadata = fs::symlink_metadata(path).ok();
let is_dir = metadata.as_ref().is_some_and(|m| m.is_dir());
let is_symlink = metadata.as_ref().is_some_and(|m| m.is_symlink());
let is_executable = metadata.as_ref().is_some_and(is_executable);
let size = metadata.as_ref().map_or(0, |m| m.len());
if is_dir {
self.dir_count += 1;
} else {
self.file_count += 1;
}
let name = if depth == 0 {
path.to_string_lossy().to_string()
} else {
path.file_name()
.map(|n| n.to_string_lossy().to_string())
.unwrap_or_else(|| path.to_string_lossy().to_string())
};
let colored_name = if self.color_enabled {
if is_dir {
name.blue().bold().to_string()
} else if is_symlink {
name.cyan().to_string()
} else if is_executable {
name.green().to_string()
} else {
name
}
} else {
name
};
let mut output = colored_name;
if metadata.is_some() {
let mut details = Vec::new();
if self.show_size && !is_dir {
details.push(crate::formatter::format_size(size));
}
if self.show_lines && !is_dir && size <= self.max_file_size {
if let Ok(lines) = count_lines(path, self.max_file_size) {
if lines > 0 {
details.push(format!("{} lines", lines));
}
}
}
if !details.is_empty() {
let details_str = format!(" ({})", details.join(", "));
output.push_str(&if self.color_enabled {
details_str.dimmed().to_string()
} else {
details_str
});
}
}
writeln!(self.stdout, "{}", output)?;
if path.is_dir() && depth < walker.filter_opts.max_depth.unwrap_or(usize::MAX) {
let children = walker.read_directory(path, depth + 1)?;
let child_count = children.len();
for (i, child) in children.into_iter().enumerate() {
let is_last = i == child_count - 1;
prefix.push(is_last);
self.walk_and_print_tree(walker, &child, depth + 1, prefix)?;
prefix.pop();
}
}
Ok(())
}
}
fn load_gitignore(root: &Path) -> Result<Option<Gitignore>> {
let mut builder = GitignoreBuilder::new(root);
let gitignore_path = root.join(".gitignore");
if gitignore_path.exists() {
builder.add(&gitignore_path);
}
match builder.build() {
Ok(gitignore) => Ok(Some(gitignore)),
Err(e) => {
log::warn!("Failed to load gitignore: {}", e);
Ok(None)
}
}
}
#[cfg(unix)]
#[inline(always)]
fn path_to_cstring(path: &Path) -> Result<CString> {
CString::new(path.as_os_str().as_bytes()).map_err(|_| Error::path("Invalid path"))
}
#[cfg(unix)]
fn is_executable(metadata: &Metadata) -> bool {
metadata.permissions().mode() & 0o111 != 0
}
#[cfg(not(unix))]
fn is_executable(_metadata: &Metadata) -> bool {
false
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs::{self, File};
use tempfile::TempDir;
#[test]
fn test_walker_basic() {
let temp_dir = TempDir::new().unwrap();
let root = temp_dir.path();
fs::create_dir(root.join("subdir")).unwrap();
File::create(root.join("file1.txt")).unwrap();
File::create(root.join("subdir/file2.txt")).unwrap();
let filter_opts = FilterOptions {
show_hidden: true,
..Default::default()
};
let mut walker = Walker::new(root, filter_opts, 1).unwrap();
let entries = walker.walk().unwrap();
assert_eq!(entries.len(), 1); assert_eq!(entries[0].children.len(), 2); }
#[test]
fn test_walker_depth_limit() {
let temp_dir = TempDir::new().unwrap();
let root = temp_dir.path();
fs::create_dir_all(root.join("a/b/c/d")).unwrap();
let filter_opts = FilterOptions {
max_depth: Some(2),
show_hidden: true,
..Default::default()
};
let mut walker = Walker::new(root, filter_opts, 1).unwrap();
let entries = walker.walk().unwrap();
fn check_max_depth(entry: &TreeEntry, max: usize) -> bool {
if entry.depth > max {
return false;
}
entry.children.iter().all(|c| check_max_depth(c, max))
}
assert!(entries.iter().all(|e| check_max_depth(e, 2)));
}
}