use std::{
env,
fs::{self, DirEntry},
io,
path::{Path, PathBuf},
rc::Rc,
sync::atomic::{AtomicBool, Ordering},
sync::Arc,
};
use crossbeam::channel;
use crossbeam::sync::WaitGroup;
use futures::executor::ThreadPool;
use log::{debug, error, info, warn};
use crate::utils::display::Display;
use crate::utils::filters::Filters;
use crate::utils::grep::Grep;
use crate::utils::lines::Zero;
use crate::utils::mapped::Mapped;
use crate::utils::matcher::Matcher;
use crate::utils::patterns::{Patterns, ToPatterns};
use crate::utils::writer::BufferedWriter;
static GIT_IGNORE: &str = ".gitignore";
pub const GIT_DIR: &str = ".git";
#[derive(Clone)]
pub struct Walker {
tpool: Option<ThreadPool>,
ignore_patterns: Arc<Patterns>,
force_ignore_patterns: Arc<Patterns>,
file_filters: Arc<Filters>,
grep: Grep,
matcher: Matcher,
ignore_symlinks: bool,
display: Arc<dyn Display>,
print_file_separator: bool,
file_separator_printed: Rc<AtomicBool>,
}
struct GrepResult {
seq: usize,
writer: Arc<BufferedWriter>,
}
pub struct WalkerBuilder(Walker);
impl WalkerBuilder {
pub fn new(grep: Grep, matcher: Matcher, display: Arc<dyn Display>) -> Self {
WalkerBuilder(Walker::new(grep, matcher, display))
}
pub fn thread_pool(mut self, tpool: ThreadPool) -> WalkerBuilder {
self.0.tpool = Some(tpool);
self
}
pub fn ignore_patterns(mut self, ignore_patterns: Patterns) -> WalkerBuilder {
self.0.ignore_patterns = Arc::new(ignore_patterns);
self
}
pub fn force_ignore_patterns(mut self, force_ignore_patterns: Patterns) -> WalkerBuilder {
self.0.force_ignore_patterns = Arc::new(force_ignore_patterns);
self
}
pub fn file_filters(mut self, file_filters: Filters) -> WalkerBuilder {
self.0.file_filters = Arc::new(file_filters);
self
}
pub fn ignore_symlinks(mut self, ignore_symlinks: bool) -> WalkerBuilder {
self.0.ignore_symlinks = ignore_symlinks;
self
}
pub fn print_file_separator(mut self, print_file_separator: bool) -> WalkerBuilder {
self.0.print_file_separator = print_file_separator;
self
}
pub fn build(self) -> Walker {
self.0
}
}
impl Walker {
pub fn new(grep: Grep, matcher: Matcher, display: Arc<dyn Display>) -> Self {
Walker {
tpool: None,
ignore_patterns: Default::default(),
force_ignore_patterns: Default::default(),
file_filters: Default::default(),
grep,
matcher,
ignore_symlinks: false,
display,
print_file_separator: false,
file_separator_printed: Default::default(),
}
}
fn is_ignore_file(&self, entry: &DirEntry) -> bool {
Some(GIT_IGNORE) == entry.file_name().to_str()
}
fn is_excluded(&self, path: &Path, is_dir: bool) -> bool {
let path = path.to_str().unwrap();
let skip = self.force_ignore_patterns.is_excluded(path, is_dir);
if skip {
info!("Skipping [forced] {:?}", path);
return true;
}
let skip = self.ignore_patterns.is_excluded(path, is_dir);
if skip {
info!("Skipping {:?}", path);
}
skip
}
fn process_gitignore(path: &Path) -> Option<Patterns> {
let ifile = {
let mut ifile = path.to_path_buf();
ifile.push(GIT_IGNORE);
ifile
};
match ifile.to_patterns() {
Ok(ignore_patterns) => Some(ignore_patterns),
Err(e) => {
match e.downcast_ref::<io::Error>() {
Some(e) if e.kind() == io::ErrorKind::NotFound => {}
_ => error!("Failed to process path '{}': {:?}", ifile.display(), e),
};
None
}
}
}
fn contains_git_dir(path: &Path) -> bool {
let mut path = path.to_path_buf();
path.push(GIT_DIR);
path.exists()
}
fn collect_files_in_dir(&self, path: &Path, parents: &[PathBuf], files: &mut Vec<PathBuf>) {
let walker = {
let mut walker = self.clone();
if let Some(mut ignore_patterns) = Self::process_gitignore(path) {
ignore_patterns.extend(&walker.ignore_patterns);
walker.ignore_patterns = Arc::new(ignore_patterns);
}
walker
};
let mut to_dive = Vec::new();
let mut to_grep = Vec::new();
let read_dir = match fs::read_dir(path) {
Ok(read_dir) => read_dir,
Err(e) => {
error!("Failed to read directory '{}': {}", path.display(), e);
return;
}
};
let mut entries: Vec<_> = read_dir
.filter_map(|entry| entry.ok())
.filter(|entry| !self.is_ignore_file(entry))
.filter_map(|entry| match entry.file_type() {
Ok(file_type) => Some((entry.path(), file_type)),
Err(e) => {
error!("Failed to get path '{}' file type: {}", path.display(), e);
None
}
})
.filter(|(entry, file_type)| !walker.is_excluded(entry, file_type.is_dir()))
.collect();
entries.sort_unstable_by(|(left, _), (right, _)| left.cmp(right));
for (path, file_type) in entries {
if file_type.is_file() {
if !self.file_filters.matches(path.to_str().unwrap()) {
continue;
}
to_grep.push(path);
} else {
to_dive.push((path, file_type));
}
}
let parents = {
let mut parents = parents.to_owned();
parents.push(path.to_path_buf());
parents
};
for (entry, file_type) in to_dive {
walker.collect_files_with_parents(&entry, Some(file_type), &parents, files);
}
files.extend(to_grep);
}
fn grep(grep: Grep, entry: Arc<PathBuf>, matcher: Matcher, display: Arc<dyn Display>) {
match Mapped::open(&entry) {
Ok(Some(mapped)) => {
if content_inspector::inspect(&mapped).is_binary() {
debug!("Skipping binary file '{}'", entry.display());
return;
}
(grep)(Arc::new(mapped), matcher, display);
}
Ok(None) => {
(grep)(Arc::new(Zero::new((*entry).clone())), matcher, display);
}
Err(_) => {
warn!("Failed to map file '{}'", entry.display());
(grep)(entry, matcher, display);
}
}
}
fn flush_writer(
&self,
writer: &Arc<BufferedWriter>,
output: &Arc<dyn crate::utils::writer::Writer>,
) {
if self.print_file_separator
&& writer.has_some()
&& self.file_separator_printed.swap(true, Ordering::Relaxed)
{
self.display.file_separator();
}
writer.flush(output);
}
fn grep_many(&self, entries: &[PathBuf]) {
let output = self.display.writer();
if entries.len() < 3 || self.tpool.is_none() {
for entry in entries {
let entry = Arc::new(entry.clone());
let matcher = self.matcher.clone();
let writer = Arc::new(BufferedWriter::new());
let display = self.display.with_writer(writer.clone());
Walker::grep(self.grep.clone(), entry, matcher, display);
self.flush_writer(&writer, &output);
}
return;
}
let (tx, rx) = channel::unbounded();
let mut pending = std::collections::BTreeMap::new();
let wg = WaitGroup::new();
let tpool = self.tpool.as_ref().unwrap();
for (seq, entry) in entries.iter().enumerate() {
let entry = Arc::new(entry.clone());
let matcher = self.matcher.clone();
let writer = Arc::new(BufferedWriter::new());
let display = self.display.with_writer(writer.clone());
let grep = self.grep.clone();
let tx = tx.clone();
let wg = wg.clone();
tpool.spawn_ok(async move {
Walker::grep(grep, entry, matcher, display);
tx.send(GrepResult { seq, writer }).ok();
drop(wg);
});
}
drop(tx);
let mut next = 0;
for result in rx {
pending.insert(result.seq, result.writer);
while let Some(writer) = pending.remove(&next) {
self.flush_writer(&writer, &output);
next += 1;
}
}
wg.wait();
}
fn canonicalize(&self, orig: &Path, resolved: &Path) -> anyhow::Result<PathBuf> {
let cwd = env::current_dir()?;
let parent = orig
.parent()
.ok_or_else(|| anyhow::Error::msg("no parent"))?;
env::set_current_dir(parent)?;
let path = resolved
.canonicalize()
.map_err(|e| anyhow::Error::new(e).context(format!("cwd {}", parent.display())));
env::set_current_dir(&cwd)?;
path
}
fn collect_symlink_files(
&self,
orig: &Path,
resolved: &Path,
parents: &[PathBuf],
files: &mut Vec<PathBuf>,
) {
let path = self.canonicalize(orig, resolved);
if let Err(e) = path {
error!("Failed to canonicalize '{}': {}", resolved.display(), e);
return;
}
let path = path.unwrap();
if let Some(level) = parents.iter().position(|parent| *parent == path) {
error!(
"Symlink '{}' -> '{}' (dereferenced to '{}') loop detected at level {}",
orig.display(),
resolved.display(),
path.display(),
level,
);
return;
}
if parents.iter().any(|parent| path.starts_with(parent)) {
info!(
"Skipping symlink '{}' -> '{}' (dereferenced to '{}')",
orig.display(),
resolved.display(),
path.display(),
);
return;
}
self.collect_files_with_parents(&path, None, &{
let mut parents = parents.to_owned();
parents.push(path.clone());
parents
}, files);
}
fn collect_files_with_parents(
&self,
path: &Path,
file_type: Option<fs::FileType>,
parents: &[PathBuf],
files: &mut Vec<PathBuf>,
) {
let file_type = file_type.or_else(|| match fs::symlink_metadata(path) {
Ok(meta) => Some(meta.file_type()),
Err(e) => {
error!("Failed to get path '{}' metadata: {}", path.display(), e);
None
}
});
let file_type = match file_type {
Some(file_type) => file_type,
_ => return,
};
if file_type.is_dir() {
self.collect_files_in_dir(path, parents, files);
} else if file_type.is_file() {
files.push(path.to_path_buf());
} else if file_type.is_symlink() {
if self.ignore_symlinks {
info!("Skipping symlink '{}'", path.display());
return;
}
match fs::read_link(path) {
Ok(resolved) => self.collect_symlink_files(path, &resolved, parents, files),
Err(e) => error!("Failed to read link '{}': {}", path.display(), e),
}
} else {
warn!("Unhandled path '{}': {:?}", path.display(), file_type)
}
}
pub fn find_ignore_patterns_in_parents(path: &Path) -> Option<Patterns> {
if Self::contains_git_dir(path) {
return None;
}
let mut patterns = Vec::new();
let mut path = path.to_path_buf();
while path.pop() {
if let Some(ignore_patterns) = Self::process_gitignore(&path) {
debug!("Found .gitignore in {}", path.display());
patterns.push(ignore_patterns);
}
if Self::contains_git_dir(&path) {
break;
}
}
if patterns.is_empty() {
return None;
}
let mut ignore_patterns = Patterns::default();
for pattern in patterns {
ignore_patterns.extend(&pattern);
}
Some(ignore_patterns)
}
pub fn walk(&self, path: &Path) {
let mut files = Vec::new();
self.collect_files_with_parents(path, None, &[], &mut files);
files.sort();
self.grep_many(&files);
}
}
#[cfg(test)]
mod tests {
use std::fs;
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
#[cfg(unix)]
use std::os::unix::fs::symlink;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::{SystemTime, UNIX_EPOCH};
use super::*;
use crate::utils::display::DisplayTerminal;
use crate::utils::display::{Format, PathFormat};
use crate::utils::matcher::Match;
use crate::utils::writer::Writer;
#[derive(Clone)]
struct TestWriter {
writes: Arc<Mutex<Vec<String>>>,
}
impl TestWriter {
fn new() -> Self {
Self {
writes: Arc::new(Mutex::new(Vec::new())),
}
}
fn lines(&self) -> Vec<String> {
self.writes.lock().unwrap().clone()
}
}
impl Writer for TestWriter {
fn write(&self, content: &str) {
self.writes.lock().unwrap().push(content.to_owned());
}
}
struct TempDir {
path: PathBuf,
}
impl TempDir {
fn new() -> Self {
let mut path = std::env::temp_dir();
let unique = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
path.push(format!("tgrep-walker-test-{}", unique));
fs::create_dir_all(&path).unwrap();
Self { path }
}
fn path(&self) -> &Path {
&self.path
}
fn write(&self, relative: &str, contents: &[u8]) {
let path = self.path.join(relative);
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).unwrap();
}
fs::write(path, contents).unwrap();
}
fn mkdir(&self, relative: &str) {
fs::create_dir_all(self.path.join(relative)).unwrap();
}
#[cfg(unix)]
fn symlink_dir(&self, target: &Path, link: &str) {
symlink(target, self.path.join(link)).unwrap();
}
}
impl Drop for TempDir {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.path);
}
}
fn matcher() -> Matcher {
Arc::new(Box::new(|_, _| Some(vec![Match::new(0, 0)])))
}
fn display(writer: Arc<dyn Writer>) -> Arc<dyn Display> {
let path_format: PathFormat = Arc::new(Box::new(|path: &Path| {
path.file_name().unwrap().to_string_lossy().into_owned()
}));
Arc::new(DisplayTerminal::new(
120,
Format::PathOnly { colour: false },
path_format,
writer,
))
}
fn grep_recorder() -> Grep {
Arc::new(Box::new(|reader, _, display| {
display.display(reader.path(), None);
}))
}
#[test]
fn finds_parent_gitignore_until_git_dir() {
let temp = TempDir::new();
temp.mkdir(".git");
temp.write(".gitignore", b"root-ignored.txt\n");
temp.mkdir("nested/deep");
temp.write("nested/.gitignore", b"nested-ignored.txt\n");
let patterns = Walker::find_ignore_patterns_in_parents(&temp.path().join("nested/deep"))
.expect("expected parent ignore patterns");
let root_ignored = temp.path().join("root-ignored.txt");
let nested_ignored = temp.path().join("nested/nested-ignored.txt");
let outside = temp.path().join("nested/deep/visible.txt");
assert!(patterns.is_excluded(root_ignored.to_str().unwrap(), false));
assert!(patterns.is_excluded(nested_ignored.to_str().unwrap(), false));
assert!(!patterns.is_excluded(outside.to_str().unwrap(), false));
}
#[test]
fn does_not_search_beyond_repository_root_for_parent_gitignores() {
let outer = TempDir::new();
outer.write(".gitignore", b"outside.txt\n");
outer.mkdir("repo/.git");
outer.mkdir("repo/nested");
let patterns = Walker::find_ignore_patterns_in_parents(&outer.path().join("repo/nested"));
let outside = outer.path().join("outside.txt");
assert!(
patterns.is_none()
|| !patterns
.unwrap()
.is_excluded(outside.to_str().unwrap(), false)
);
}
#[test]
fn walk_honors_gitignore_and_file_filters() {
let temp = TempDir::new();
temp.write(".gitignore", b"ignored.txt\n");
temp.write("visible.rs", b"fn main() {}\n");
temp.write("ignored.txt", b"secret\n");
temp.write("notes.md", b"# notes\n");
let writer = TestWriter::new();
let walker = WalkerBuilder::new(
grep_recorder(),
matcher(),
display(Arc::new(writer.clone())),
)
.ignore_patterns(Patterns::new(temp.path().to_str().unwrap(), &[]))
.force_ignore_patterns(Patterns::new(temp.path().to_str().unwrap(), &[]))
.file_filters(Filters::new(&["*.rs".to_string()]).unwrap())
.build();
walker.walk(temp.path());
assert_eq!(vec!["visible.rs"], writer.lines());
}
#[test]
fn walk_outputs_files_in_global_lexicographic_order() {
let temp = TempDir::new();
temp.write("a.txt", b"match\n");
temp.write("b/c.txt", b"match\n");
let writer = TestWriter::new();
let walker = WalkerBuilder::new(
grep_recorder(),
matcher(),
display(Arc::new(writer.clone())),
)
.thread_pool(ThreadPool::new().unwrap())
.file_filters(Filters::new(&["*".to_string()]).unwrap())
.build();
walker.walk(temp.path());
assert_eq!(vec!["a.txt", "c.txt"], writer.lines());
}
#[test]
fn force_ignore_patterns_override_walk_results() {
let temp = TempDir::new();
temp.write("visible.txt", b"ok\n");
temp.write("forced.txt", b"skip\n");
let writer = TestWriter::new();
let walker = WalkerBuilder::new(
grep_recorder(),
matcher(),
display(Arc::new(writer.clone())),
)
.ignore_patterns(Patterns::new(temp.path().to_str().unwrap(), &[]))
.force_ignore_patterns(Patterns::new(
temp.path().to_str().unwrap(),
&["forced.txt".to_string()],
))
.file_filters(Filters::new(&["*".to_string()]).unwrap())
.build();
walker.walk(temp.path());
assert_eq!(vec!["visible.txt"], writer.lines());
}
#[cfg(unix)]
#[test]
fn walk_skips_symlinks_when_configured() {
let temp = TempDir::new();
let external = TempDir::new();
external.write("linked.txt", b"external\n");
temp.symlink_dir(external.path(), "external-link");
let writer = TestWriter::new();
let walker = WalkerBuilder::new(
grep_recorder(),
matcher(),
display(Arc::new(writer.clone())),
)
.ignore_patterns(Patterns::new(temp.path().to_str().unwrap(), &[]))
.force_ignore_patterns(Patterns::new(temp.path().to_str().unwrap(), &[]))
.file_filters(Filters::new(&["*".to_string()]).unwrap())
.ignore_symlinks(true)
.build();
walker.walk(temp.path());
assert!(writer.lines().is_empty());
}
#[cfg(unix)]
#[test]
fn walk_follows_external_directory_symlinks() {
let temp = TempDir::new();
let external = TempDir::new();
external.write("linked.txt", b"external\n");
temp.symlink_dir(external.path(), "external-link");
let writer = TestWriter::new();
let walker = WalkerBuilder::new(
grep_recorder(),
matcher(),
display(Arc::new(writer.clone())),
)
.ignore_patterns(Patterns::new(temp.path().to_str().unwrap(), &[]))
.force_ignore_patterns(Patterns::new(temp.path().to_str().unwrap(), &[]))
.file_filters(Filters::new(&["*".to_string()]).unwrap())
.build();
walker.walk(temp.path());
assert_eq!(vec!["linked.txt"], writer.lines());
}
#[cfg(unix)]
#[test]
fn walk_skips_unreadable_directories_without_panicking() {
let temp = TempDir::new();
temp.write("visible.txt", b"visible\n");
temp.mkdir("private");
temp.write("private/hidden.txt", b"hidden\n");
let private = temp.path().join("private");
let original_permissions = fs::metadata(&private).unwrap().permissions();
let mut unreadable_permissions = original_permissions.clone();
unreadable_permissions.set_mode(0o000);
fs::set_permissions(&private, unreadable_permissions).unwrap();
let writer = TestWriter::new();
let walker = WalkerBuilder::new(
grep_recorder(),
matcher(),
display(Arc::new(writer.clone())),
)
.ignore_patterns(Patterns::new(temp.path().to_str().unwrap(), &[]))
.force_ignore_patterns(Patterns::new(temp.path().to_str().unwrap(), &[]))
.file_filters(Filters::new(&["*".to_string()]).unwrap())
.build();
walker.walk(temp.path());
fs::set_permissions(&private, original_permissions).unwrap();
assert_eq!(vec!["visible.txt"], writer.lines());
}
}