use std::cmp;
use std::ffi::OsStr;
use std::fmt;
use std::fs::{self, FileType, Metadata};
use std::io;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::thread;
use std::time::Duration;
use std::vec;
use channel;
use same_file::Handle;
use walkdir::{self, WalkDir};
use dir::{Ignore, IgnoreBuilder};
use gitignore::GitignoreBuilder;
use overrides::Override;
use types::Types;
use {Error, PartialErrorBuilder};
#[derive(Clone, Debug)]
pub struct DirEntry {
dent: DirEntryInner,
err: Option<Error>,
}
impl DirEntry {
pub fn path(&self) -> &Path {
self.dent.path()
}
pub fn into_path(self) -> PathBuf {
self.dent.into_path()
}
pub fn path_is_symlink(&self) -> bool {
self.dent.path_is_symlink()
}
pub fn is_stdin(&self) -> bool {
self.dent.is_stdin()
}
pub fn metadata(&self) -> Result<Metadata, Error> {
self.dent.metadata()
}
pub fn file_type(&self) -> Option<FileType> {
self.dent.file_type()
}
pub fn file_name(&self) -> &OsStr {
self.dent.file_name()
}
pub fn depth(&self) -> usize {
self.dent.depth()
}
#[cfg(unix)]
pub fn ino(&self) -> Option<u64> {
self.dent.ino()
}
pub fn error(&self) -> Option<&Error> {
self.err.as_ref()
}
pub(crate) fn is_dir(&self) -> bool {
self.dent.is_dir()
}
fn new_stdin() -> DirEntry {
DirEntry {
dent: DirEntryInner::Stdin,
err: None,
}
}
fn new_walkdir(dent: walkdir::DirEntry, err: Option<Error>) -> DirEntry {
DirEntry {
dent: DirEntryInner::Walkdir(dent),
err: err,
}
}
fn new_raw(dent: DirEntryRaw, err: Option<Error>) -> DirEntry {
DirEntry {
dent: DirEntryInner::Raw(dent),
err: err,
}
}
}
#[derive(Clone, Debug)]
enum DirEntryInner {
Stdin,
Walkdir(walkdir::DirEntry),
Raw(DirEntryRaw),
}
impl DirEntryInner {
fn path(&self) -> &Path {
use self::DirEntryInner::*;
match *self {
Stdin => Path::new("<stdin>"),
Walkdir(ref x) => x.path(),
Raw(ref x) => x.path(),
}
}
fn into_path(self) -> PathBuf {
use self::DirEntryInner::*;
match self {
Stdin => PathBuf::from("<stdin>"),
Walkdir(x) => x.into_path(),
Raw(x) => x.into_path(),
}
}
fn path_is_symlink(&self) -> bool {
use self::DirEntryInner::*;
match *self {
Stdin => false,
Walkdir(ref x) => x.path_is_symlink(),
Raw(ref x) => x.path_is_symlink(),
}
}
fn is_stdin(&self) -> bool {
match *self {
DirEntryInner::Stdin => true,
_ => false,
}
}
fn metadata(&self) -> Result<Metadata, Error> {
use self::DirEntryInner::*;
match *self {
Stdin => {
let err = Error::Io(io::Error::new(
io::ErrorKind::Other, "<stdin> has no metadata"));
Err(err.with_path("<stdin>"))
}
Walkdir(ref x) => {
x.metadata().map_err(|err| {
Error::Io(io::Error::from(err)).with_path(x.path())
})
}
Raw(ref x) => x.metadata(),
}
}
fn file_type(&self) -> Option<FileType> {
use self::DirEntryInner::*;
match *self {
Stdin => None,
Walkdir(ref x) => Some(x.file_type()),
Raw(ref x) => Some(x.file_type()),
}
}
fn file_name(&self) -> &OsStr {
use self::DirEntryInner::*;
match *self {
Stdin => OsStr::new("<stdin>"),
Walkdir(ref x) => x.file_name(),
Raw(ref x) => x.file_name(),
}
}
fn depth(&self) -> usize {
use self::DirEntryInner::*;
match *self {
Stdin => 0,
Walkdir(ref x) => x.depth(),
Raw(ref x) => x.depth(),
}
}
#[cfg(unix)]
fn ino(&self) -> Option<u64> {
use walkdir::DirEntryExt;
use self::DirEntryInner::*;
match *self {
Stdin => None,
Walkdir(ref x) => Some(x.ino()),
Raw(ref x) => Some(x.ino()),
}
}
fn is_dir(&self) -> bool {
self.file_type().map(|ft| ft.is_dir()).unwrap_or(false)
}
}
#[derive(Clone)]
struct DirEntryRaw {
path: PathBuf,
ty: FileType,
follow_link: bool,
depth: usize,
#[cfg(unix)]
ino: u64,
#[cfg(windows)]
metadata: fs::Metadata,
}
impl fmt::Debug for DirEntryRaw {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_struct("DirEntryRaw")
.field("path", &self.path)
.field("follow_link", &self.follow_link)
.field("depth", &self.depth)
.finish()
}
}
impl DirEntryRaw {
fn path(&self) -> &Path {
&self.path
}
fn into_path(self) -> PathBuf {
self.path
}
fn path_is_symlink(&self) -> bool {
self.ty.is_symlink() || self.follow_link
}
fn metadata(&self) -> Result<Metadata, Error> {
self.metadata_internal()
}
#[cfg(windows)]
fn metadata_internal(&self) -> Result<fs::Metadata, Error> {
if self.follow_link {
fs::metadata(&self.path)
} else {
Ok(self.metadata.clone())
}.map_err(|err| Error::Io(io::Error::from(err)).with_path(&self.path))
}
#[cfg(not(windows))]
fn metadata_internal(&self) -> Result<fs::Metadata, Error> {
if self.follow_link {
fs::metadata(&self.path)
} else {
fs::symlink_metadata(&self.path)
}.map_err(|err| Error::Io(io::Error::from(err)).with_path(&self.path))
}
fn file_type(&self) -> FileType {
self.ty
}
fn file_name(&self) -> &OsStr {
self.path.file_name().unwrap_or_else(|| self.path.as_os_str())
}
fn depth(&self) -> usize {
self.depth
}
#[cfg(unix)]
fn ino(&self) -> u64 {
self.ino
}
fn from_entry(
depth: usize,
ent: &fs::DirEntry,
) -> Result<DirEntryRaw, Error> {
let ty = ent.file_type().map_err(|err| {
let err = Error::Io(io::Error::from(err)).with_path(ent.path());
Error::WithDepth {
depth: depth,
err: Box::new(err),
}
})?;
DirEntryRaw::from_entry_os(depth, ent, ty)
}
#[cfg(windows)]
fn from_entry_os(
depth: usize,
ent: &fs::DirEntry,
ty: fs::FileType,
) -> Result<DirEntryRaw, Error> {
let md = ent.metadata().map_err(|err| {
let err = Error::Io(io::Error::from(err)).with_path(ent.path());
Error::WithDepth {
depth: depth,
err: Box::new(err),
}
})?;
Ok(DirEntryRaw {
path: ent.path(),
ty: ty,
follow_link: false,
depth: depth,
metadata: md,
})
}
#[cfg(unix)]
fn from_entry_os(
depth: usize,
ent: &fs::DirEntry,
ty: fs::FileType,
) -> Result<DirEntryRaw, Error> {
use std::os::unix::fs::DirEntryExt;
Ok(DirEntryRaw {
path: ent.path(),
ty: ty,
follow_link: false,
depth: depth,
ino: ent.ino(),
})
}
#[cfg(not(unix))]
fn from_path(
depth: usize,
pb: PathBuf,
link: bool,
) -> Result<DirEntryRaw, Error> {
let md = fs::metadata(&pb).map_err(|err| {
Error::Io(err).with_path(&pb)
})?;
Ok(DirEntryRaw {
path: pb,
ty: md.file_type(),
follow_link: link,
depth: depth,
metadata: md,
})
}
#[cfg(unix)]
fn from_path(
depth: usize,
pb: PathBuf,
link: bool,
) -> Result<DirEntryRaw, Error> {
use std::os::unix::fs::MetadataExt;
let md = fs::metadata(&pb).map_err(|err| {
Error::Io(err).with_path(&pb)
})?;
Ok(DirEntryRaw {
path: pb,
ty: md.file_type(),
follow_link: link,
depth: depth,
ino: md.ino(),
})
}
}
#[derive(Clone)]
pub struct WalkBuilder {
paths: Vec<PathBuf>,
ig_builder: IgnoreBuilder,
max_depth: Option<usize>,
max_filesize: Option<u64>,
follow_links: bool,
same_file_system: bool,
sorter: Option<Sorter>,
threads: usize,
skip: Option<Arc<Handle>>,
}
#[derive(Clone)]
enum Sorter {
ByName(Arc<Fn(&OsStr, &OsStr) -> cmp::Ordering + Send + Sync + 'static>),
ByPath(Arc<Fn(&Path, &Path) -> cmp::Ordering + Send + Sync + 'static>),
}
impl fmt::Debug for WalkBuilder {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_struct("WalkBuilder")
.field("paths", &self.paths)
.field("ig_builder", &self.ig_builder)
.field("max_depth", &self.max_depth)
.field("max_filesize", &self.max_filesize)
.field("follow_links", &self.follow_links)
.field("threads", &self.threads)
.field("skip", &self.skip)
.finish()
}
}
impl WalkBuilder {
pub fn new<P: AsRef<Path>>(path: P) -> WalkBuilder {
WalkBuilder {
paths: vec![path.as_ref().to_path_buf()],
ig_builder: IgnoreBuilder::new(),
max_depth: None,
max_filesize: None,
follow_links: false,
same_file_system: false,
sorter: None,
threads: 0,
skip: None,
}
}
pub fn build(&self) -> Walk {
let follow_links = self.follow_links;
let max_depth = self.max_depth;
let sorter = self.sorter.clone();
let its = self.paths.iter().map(move |p| {
if p == Path::new("-") {
(p.to_path_buf(), None)
} else {
let mut wd = WalkDir::new(p);
wd = wd.follow_links(follow_links || p.is_file());
wd = wd.same_file_system(self.same_file_system);
if let Some(max_depth) = max_depth {
wd = wd.max_depth(max_depth);
}
if let Some(ref sorter) = sorter {
match sorter.clone() {
Sorter::ByName(cmp) => {
wd = wd.sort_by(move |a, b| {
cmp(a.file_name(), b.file_name())
});
}
Sorter::ByPath(cmp) => {
wd = wd.sort_by(move |a, b| {
cmp(a.path(), b.path())
});
}
}
}
(p.to_path_buf(), Some(WalkEventIter::from(wd)))
}
}).collect::<Vec<_>>().into_iter();
let ig_root = self.ig_builder.build();
Walk {
its: its,
it: None,
ig_root: ig_root.clone(),
ig: ig_root.clone(),
max_filesize: self.max_filesize,
skip: self.skip.clone(),
}
}
pub fn build_parallel(&self) -> WalkParallel {
WalkParallel {
paths: self.paths.clone().into_iter(),
ig_root: self.ig_builder.build(),
max_depth: self.max_depth,
max_filesize: self.max_filesize,
follow_links: self.follow_links,
same_file_system: self.same_file_system,
threads: self.threads,
skip: self.skip.clone(),
}
}
pub fn add<P: AsRef<Path>>(&mut self, path: P) -> &mut WalkBuilder {
self.paths.push(path.as_ref().to_path_buf());
self
}
pub fn max_depth(&mut self, depth: Option<usize>) -> &mut WalkBuilder {
self.max_depth = depth;
self
}
pub fn follow_links(&mut self, yes: bool) -> &mut WalkBuilder {
self.follow_links = yes;
self
}
pub fn max_filesize(&mut self, filesize: Option<u64>) -> &mut WalkBuilder {
self.max_filesize = filesize;
self
}
pub fn threads(&mut self, n: usize) -> &mut WalkBuilder {
self.threads = n;
self
}
pub fn add_ignore<P: AsRef<Path>>(&mut self, path: P) -> Option<Error> {
let mut builder = GitignoreBuilder::new("");
let mut errs = PartialErrorBuilder::default();
errs.maybe_push(builder.add(path));
match builder.build() {
Ok(gi) => { self.ig_builder.add_ignore(gi); }
Err(err) => { errs.push(err); }
}
errs.into_error_option()
}
pub fn add_custom_ignore_filename<S: AsRef<OsStr>>(
&mut self,
file_name: S
) -> &mut WalkBuilder {
self.ig_builder.add_custom_ignore_filename(file_name);
self
}
pub fn overrides(&mut self, overrides: Override) -> &mut WalkBuilder {
self.ig_builder.overrides(overrides);
self
}
pub fn types(&mut self, types: Types) -> &mut WalkBuilder {
self.ig_builder.types(types);
self
}
pub fn standard_filters(&mut self, yes: bool) -> &mut WalkBuilder {
self.hidden(yes)
.parents(yes)
.ignore(yes)
.git_ignore(yes)
.git_global(yes)
.git_exclude(yes)
}
pub fn hidden(&mut self, yes: bool) -> &mut WalkBuilder {
self.ig_builder.hidden(yes);
self
}
pub fn parents(&mut self, yes: bool) -> &mut WalkBuilder {
self.ig_builder.parents(yes);
self
}
pub fn ignore(&mut self, yes: bool) -> &mut WalkBuilder {
self.ig_builder.ignore(yes);
self
}
pub fn git_global(&mut self, yes: bool) -> &mut WalkBuilder {
self.ig_builder.git_global(yes);
self
}
pub fn git_ignore(&mut self, yes: bool) -> &mut WalkBuilder {
self.ig_builder.git_ignore(yes);
self
}
pub fn git_exclude(&mut self, yes: bool) -> &mut WalkBuilder {
self.ig_builder.git_exclude(yes);
self
}
pub fn ignore_case_insensitive(&mut self, yes: bool) -> &mut WalkBuilder {
self.ig_builder.ignore_case_insensitive(yes);
self
}
pub fn sort_by_file_path<F>(
&mut self,
cmp: F,
) -> &mut WalkBuilder
where F: Fn(&Path, &Path) -> cmp::Ordering + Send + Sync + 'static
{
self.sorter = Some(Sorter::ByPath(Arc::new(cmp)));
self
}
pub fn sort_by_file_name<F>(&mut self, cmp: F) -> &mut WalkBuilder
where F: Fn(&OsStr, &OsStr) -> cmp::Ordering + Send + Sync + 'static
{
self.sorter = Some(Sorter::ByName(Arc::new(cmp)));
self
}
pub fn same_file_system(&mut self, yes: bool) -> &mut WalkBuilder {
self.same_file_system = yes;
self
}
pub fn skip_stdout(&mut self, yes: bool) -> &mut WalkBuilder {
if yes {
self.skip = stdout_handle().map(Arc::new);
} else {
self.skip = None;
}
self
}
}
pub struct Walk {
its: vec::IntoIter<(PathBuf, Option<WalkEventIter>)>,
it: Option<WalkEventIter>,
ig_root: Ignore,
ig: Ignore,
max_filesize: Option<u64>,
skip: Option<Arc<Handle>>,
}
impl Walk {
pub fn new<P: AsRef<Path>>(path: P) -> Walk {
WalkBuilder::new(path).build()
}
fn skip_entry(&self, ent: &DirEntry) -> Result<bool, Error> {
if ent.depth() == 0 {
return Ok(false);
}
if let Some(ref stdout) = self.skip {
if path_equals(ent, stdout)? {
return Ok(true);
}
}
if should_skip_entry(&self.ig, ent) {
return Ok(true);
}
if self.max_filesize.is_some() && !ent.is_dir() {
return Ok(skip_filesize(
self.max_filesize.unwrap(),
ent.path(),
&ent.metadata().ok(),
));
}
Ok(false)
}
}
impl Iterator for Walk {
type Item = Result<DirEntry, Error>;
#[inline(always)]
fn next(&mut self) -> Option<Result<DirEntry, Error>> {
loop {
let ev = match self.it.as_mut().and_then(|it| it.next()) {
Some(ev) => ev,
None => {
match self.its.next() {
None => return None,
Some((_, None)) => {
return Some(Ok(DirEntry::new_stdin()));
}
Some((path, Some(it))) => {
self.it = Some(it);
if path.is_dir() {
let (ig, err) = self.ig_root.add_parents(path);
self.ig = ig;
if let Some(err) = err {
return Some(Err(err));
}
} else {
self.ig = self.ig_root.clone();
}
}
}
continue;
}
};
match ev {
Err(err) => {
return Some(Err(Error::from_walkdir(err)));
}
Ok(WalkEvent::Exit) => {
self.ig = self.ig.parent().unwrap();
}
Ok(WalkEvent::Dir(ent)) => {
let mut ent = DirEntry::new_walkdir(ent, None);
let should_skip = match self.skip_entry(&ent) {
Err(err) => return Some(Err(err)),
Ok(should_skip) => should_skip,
};
if should_skip {
self.it.as_mut().unwrap().it.skip_current_dir();
let (igtmp, _) = self.ig.add_child(ent.path());
self.ig = igtmp;
continue;
}
let (igtmp, err) = self.ig.add_child(ent.path());
self.ig = igtmp;
ent.err = err;
return Some(Ok(ent));
}
Ok(WalkEvent::File(ent)) => {
let ent = DirEntry::new_walkdir(ent, None);
let should_skip = match self.skip_entry(&ent) {
Err(err) => return Some(Err(err)),
Ok(should_skip) => should_skip,
};
if should_skip {
continue;
}
return Some(Ok(ent));
}
}
}
}
}
struct WalkEventIter {
depth: usize,
it: walkdir::IntoIter,
next: Option<Result<walkdir::DirEntry, walkdir::Error>>,
}
#[derive(Debug)]
enum WalkEvent {
Dir(walkdir::DirEntry),
File(walkdir::DirEntry),
Exit,
}
impl From<WalkDir> for WalkEventIter {
fn from(it: WalkDir) -> WalkEventIter {
WalkEventIter { depth: 0, it: it.into_iter(), next: None }
}
}
impl Iterator for WalkEventIter {
type Item = walkdir::Result<WalkEvent>;
#[inline(always)]
fn next(&mut self) -> Option<walkdir::Result<WalkEvent>> {
let dent = self.next.take().or_else(|| self.it.next());
let depth = match dent {
None => 0,
Some(Ok(ref dent)) => dent.depth(),
Some(Err(ref err)) => err.depth(),
};
if depth < self.depth {
self.depth -= 1;
self.next = dent;
return Some(Ok(WalkEvent::Exit));
}
self.depth = depth;
match dent {
None => None,
Some(Err(err)) => Some(Err(err)),
Some(Ok(dent)) => {
if dent.file_type().is_dir() {
self.depth += 1;
Some(Ok(WalkEvent::Dir(dent)))
} else {
Some(Ok(WalkEvent::File(dent)))
}
}
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum WalkState {
Continue,
Skip,
Quit,
}
impl WalkState {
fn is_quit(&self) -> bool {
*self == WalkState::Quit
}
}
pub struct WalkParallel {
paths: vec::IntoIter<PathBuf>,
ig_root: Ignore,
max_filesize: Option<u64>,
max_depth: Option<usize>,
follow_links: bool,
same_file_system: bool,
threads: usize,
skip: Option<Arc<Handle>>,
}
impl WalkParallel {
pub fn run<F>(
self,
mut mkf: F,
) where F: FnMut() -> Box<FnMut(Result<DirEntry, Error>) -> WalkState + Send + 'static> {
let mut f = mkf();
let threads = self.threads();
let (tx, rx) = channel::unbounded();
let mut any_work = false;
for path in self.paths {
let (dent, root_device) =
if path == Path::new("-") {
(DirEntry::new_stdin(), None)
} else {
let root_device =
if !self.same_file_system {
None
} else {
match device_num(&path) {
Ok(root_device) => Some(root_device),
Err(err) => {
let err = Error::Io(err).with_path(path);
if f(Err(err)).is_quit() {
return;
}
continue;
}
}
};
match DirEntryRaw::from_path(0, path, false) {
Ok(dent) => {
(DirEntry::new_raw(dent, None), root_device)
}
Err(err) => {
if f(Err(err)).is_quit() {
return;
}
continue;
}
}
};
tx.send(Message::Work(Work {
dent: dent,
ignore: self.ig_root.clone(),
root_device: root_device,
})).unwrap();
any_work = true;
}
if !any_work {
return;
}
let num_waiting = Arc::new(AtomicUsize::new(0));
let num_quitting = Arc::new(AtomicUsize::new(0));
let quit_now = Arc::new(AtomicBool::new(false));
let mut handles = vec![];
for _ in 0..threads {
let worker = Worker {
f: mkf(),
tx: tx.clone(),
rx: rx.clone(),
quit_now: quit_now.clone(),
is_waiting: false,
is_quitting: false,
num_waiting: num_waiting.clone(),
num_quitting: num_quitting.clone(),
threads: threads,
max_depth: self.max_depth,
max_filesize: self.max_filesize,
follow_links: self.follow_links,
skip: self.skip.clone(),
};
handles.push(thread::spawn(|| worker.run()));
}
drop(tx);
drop(rx);
for handle in handles {
handle.join().unwrap();
}
}
fn threads(&self) -> usize {
if self.threads == 0 {
2
} else {
self.threads
}
}
}
enum Message {
Work(Work),
Quit,
}
struct Work {
dent: DirEntry,
ignore: Ignore,
root_device: Option<u64>,
}
impl Work {
fn is_dir(&self) -> bool {
self.dent.is_dir()
}
fn is_symlink(&self) -> bool {
self.dent.file_type().map_or(false, |ft| ft.is_symlink())
}
fn add_parents(&mut self) -> Option<Error> {
if self.dent.depth() > 0 {
return None;
}
let (ig, err) = self.ignore.add_parents(self.dent.path());
self.ignore = ig;
err
}
fn read_dir(&mut self) -> Result<fs::ReadDir, Error> {
let readdir = match fs::read_dir(self.dent.path()) {
Ok(readdir) => readdir,
Err(err) => {
let err = Error::from(err)
.with_path(self.dent.path())
.with_depth(self.dent.depth());
return Err(err);
}
};
let (ig, err) = self.ignore.add_child(self.dent.path());
self.ignore = ig;
self.dent.err = err;
Ok(readdir)
}
}
struct Worker {
f: Box<FnMut(Result<DirEntry, Error>) -> WalkState + Send + 'static>,
tx: channel::Sender<Message>,
rx: channel::Receiver<Message>,
quit_now: Arc<AtomicBool>,
is_waiting: bool,
is_quitting: bool,
num_waiting: Arc<AtomicUsize>,
num_quitting: Arc<AtomicUsize>,
threads: usize,
max_depth: Option<usize>,
max_filesize: Option<u64>,
follow_links: bool,
skip: Option<Arc<Handle>>,
}
impl Worker {
fn run(mut self) {
while let Some(mut work) = self.get_work() {
if work.is_symlink() || !work.is_dir() {
if (self.f)(Ok(work.dent)).is_quit() {
self.quit_now();
return;
}
continue;
}
if let Some(err) = work.add_parents() {
if (self.f)(Err(err)).is_quit() {
self.quit_now();
return;
}
}
let readdir = match work.read_dir() {
Ok(readdir) => readdir,
Err(err) => {
if (self.f)(Err(err)).is_quit() {
self.quit_now();
return;
}
continue;
}
};
let descend =
if let Some(root_device) = work.root_device {
match is_same_file_system(root_device, work.dent.path()) {
Ok(true) => true,
Ok(false) => false,
Err(err) => {
if (self.f)(Err(err)).is_quit() {
self.quit_now();
return;
}
false
}
}
} else {
true
};
let depth = work.dent.depth();
match (self.f)(Ok(work.dent)) {
WalkState::Continue => {}
WalkState::Skip => continue,
WalkState::Quit => {
self.quit_now();
return;
}
}
if !descend {
continue;
}
if self.max_depth.map_or(false, |max| depth >= max) {
continue;
}
for result in readdir {
let state = self.run_one(
&work.ignore,
depth + 1,
work.root_device,
result,
);
if state.is_quit() {
self.quit_now();
return;
}
}
}
}
fn run_one(
&mut self,
ig: &Ignore,
depth: usize,
root_device: Option<u64>,
result: Result<fs::DirEntry, io::Error>,
) -> WalkState {
let fs_dent = match result {
Ok(fs_dent) => fs_dent,
Err(err) => {
return (self.f)(Err(Error::from(err).with_depth(depth)));
}
};
let mut dent = match DirEntryRaw::from_entry(depth, &fs_dent) {
Ok(dent) => DirEntry::new_raw(dent, None),
Err(err) => {
return (self.f)(Err(err));
}
};
let is_symlink = dent.file_type().map_or(false, |ft| ft.is_symlink());
if self.follow_links && is_symlink {
let path = dent.path().to_path_buf();
dent = match DirEntryRaw::from_path(depth, path, true) {
Ok(dent) => DirEntry::new_raw(dent, None),
Err(err) => {
return (self.f)(Err(err));
}
};
if dent.is_dir() {
if let Err(err) = check_symlink_loop(ig, dent.path(), depth) {
return (self.f)(Err(err));
}
}
}
if let Some(ref stdout) = self.skip {
let is_stdout = match path_equals(&dent, stdout) {
Ok(is_stdout) => is_stdout,
Err(err) => return (self.f)(Err(err)),
};
if is_stdout {
return WalkState::Continue;
}
}
let should_skip_path = should_skip_entry(ig, &dent);
let should_skip_filesize =
if self.max_filesize.is_some() && !dent.is_dir() {
skip_filesize(
self.max_filesize.unwrap(),
dent.path(),
&dent.metadata().ok(),
)
} else {
false
};
if !should_skip_path && !should_skip_filesize {
self.tx.send(Message::Work(Work {
dent: dent,
ignore: ig.clone(),
root_device: root_device,
})).unwrap();
}
WalkState::Continue
}
fn get_work(&mut self) -> Option<Work> {
loop {
if self.is_quit_now() {
return None;
}
match self.rx.try_recv() {
Ok(Message::Work(work)) => {
self.waiting(false);
self.quitting(false);
return Some(work);
}
Ok(Message::Quit) => {
self.waiting(true);
self.quitting(true);
while !self.is_quit_now() {
let nwait = self.num_waiting();
let nquit = self.num_quitting();
if nwait < self.threads {
break;
}
if nquit == self.threads {
return None;
}
}
}
Err(_) => {
self.waiting(true);
self.quitting(false);
if self.num_waiting() == self.threads {
for _ in 0..self.threads {
self.tx.send(Message::Quit).unwrap();
}
} else {
thread::sleep(Duration::from_millis(1));
}
}
}
}
}
fn quit_now(&self) {
self.quit_now.store(true, Ordering::SeqCst);
}
fn is_quit_now(&self) -> bool {
self.quit_now.load(Ordering::SeqCst)
}
fn num_waiting(&self) -> usize {
self.num_waiting.load(Ordering::SeqCst)
}
fn num_quitting(&self) -> usize {
self.num_quitting.load(Ordering::SeqCst)
}
fn quitting(&mut self, yes: bool) {
if yes {
if !self.is_quitting {
self.is_quitting = true;
self.num_quitting.fetch_add(1, Ordering::SeqCst);
}
} else {
if self.is_quitting {
self.is_quitting = false;
self.num_quitting.fetch_sub(1, Ordering::SeqCst);
}
}
}
fn waiting(&mut self, yes: bool) {
if yes {
if !self.is_waiting {
self.is_waiting = true;
self.num_waiting.fetch_add(1, Ordering::SeqCst);
}
} else {
if self.is_waiting {
self.is_waiting = false;
self.num_waiting.fetch_sub(1, Ordering::SeqCst);
}
}
}
}
fn check_symlink_loop(
ig_parent: &Ignore,
child_path: &Path,
child_depth: usize,
) -> Result<(), Error> {
let hchild = Handle::from_path(child_path).map_err(|err| {
Error::from(err).with_path(child_path).with_depth(child_depth)
})?;
for ig in ig_parent.parents().take_while(|ig| !ig.is_absolute_parent()) {
let h = Handle::from_path(ig.path()).map_err(|err| {
Error::from(err).with_path(child_path).with_depth(child_depth)
})?;
if hchild == h {
return Err(Error::Loop {
ancestor: ig.path().to_path_buf(),
child: child_path.to_path_buf(),
}.with_depth(child_depth));
}
}
Ok(())
}
fn skip_filesize(
max_filesize: u64,
path: &Path,
ent: &Option<Metadata>
) -> bool {
let filesize = match *ent {
Some(ref md) => Some(md.len()),
None => None
};
if let Some(fs) = filesize {
if fs > max_filesize {
debug!("ignoring {}: {} bytes", path.display(), fs);
true
} else {
false
}
} else {
false
}
}
fn should_skip_entry(
ig: &Ignore,
dent: &DirEntry,
) -> bool {
let m = ig.matched_dir_entry(dent);
if m.is_ignore() {
debug!("ignoring {}: {:?}", dent.path().display(), m);
true
} else if m.is_whitelist() {
debug!("whitelisting {}: {:?}", dent.path().display(), m);
false
} else {
false
}
}
fn stdout_handle() -> Option<Handle> {
let h = match Handle::stdout() {
Err(_) => return None,
Ok(h) => h,
};
let md = match h.as_file().metadata() {
Err(_) => return None,
Ok(md) => md,
};
if !md.is_file() {
return None;
}
Some(h)
}
fn path_equals(dent: &DirEntry, handle: &Handle) -> Result<bool, Error> {
#[cfg(unix)]
fn never_equal(dent: &DirEntry, handle: &Handle) -> bool {
dent.ino() != Some(handle.ino())
}
#[cfg(not(unix))]
fn never_equal(_: &DirEntry, _: &Handle) -> bool {
false
}
if dent.is_stdin() || never_equal(dent, handle) {
return Ok(false);
}
Handle::from_path(dent.path())
.map(|h| &h == handle)
.map_err(|err| Error::Io(err).with_path(dent.path()))
}
fn is_same_file_system(root_device: u64, path: &Path) -> Result<bool, Error> {
let dent_device = device_num(path)
.map_err(|err| Error::Io(err).with_path(path))?;
Ok(root_device == dent_device)
}
#[cfg(unix)]
fn device_num<P: AsRef<Path>>(path: P)-> io::Result<u64> {
use std::os::unix::fs::MetadataExt;
path.as_ref().metadata().map(|md| md.dev())
}
#[cfg(windows)]
fn device_num<P: AsRef<Path>>(path: P) -> io::Result<u64> {
use winapi_util::{Handle, file};
let h = Handle::from_path_any(path)?;
file::information(h).map(|info| info.volume_serial_number())
}
#[cfg(not(any(unix, windows)))]
fn device_num<P: AsRef<Path>>(_: P)-> io::Result<u64> {
Err(io::Error::new(
io::ErrorKind::Other,
"walkdir: same_file_system option not supported on this platform",
))
}
#[cfg(test)]
mod tests {
use std::fs::{self, File};
use std::io::Write;
use std::path::Path;
use std::sync::{Arc, Mutex};
use tempfile::{self, TempDir};
use super::{DirEntry, WalkBuilder, WalkState};
fn wfile<P: AsRef<Path>>(path: P, contents: &str) {
let mut file = File::create(path).unwrap();
file.write_all(contents.as_bytes()).unwrap();
}
fn wfile_size<P: AsRef<Path>>(path: P, size: u64) {
let file = File::create(path).unwrap();
file.set_len(size).unwrap();
}
#[cfg(unix)]
fn symlink<P: AsRef<Path>, Q: AsRef<Path>>(src: P, dst: Q) {
use std::os::unix::fs::symlink;
symlink(src, dst).unwrap();
}
fn mkdirp<P: AsRef<Path>>(path: P) {
fs::create_dir_all(path).unwrap();
}
fn normal_path(unix: &str) -> String {
if cfg!(windows) {
unix.replace("\\", "/")
} else {
unix.to_string()
}
}
fn walk_collect(prefix: &Path, builder: &WalkBuilder) -> Vec<String> {
let mut paths = vec![];
for result in builder.build() {
let dent = match result {
Err(_) => continue,
Ok(dent) => dent,
};
let path = dent.path().strip_prefix(prefix).unwrap();
if path.as_os_str().is_empty() {
continue;
}
paths.push(normal_path(path.to_str().unwrap()));
}
paths.sort();
paths
}
fn walk_collect_parallel(
prefix: &Path,
builder: &WalkBuilder,
) -> Vec<String> {
let mut paths = vec![];
for dent in walk_collect_entries_parallel(builder) {
let path = dent.path().strip_prefix(prefix).unwrap();
if path.as_os_str().is_empty() {
continue;
}
paths.push(normal_path(path.to_str().unwrap()));
}
paths.sort();
paths
}
fn walk_collect_entries_parallel(builder: &WalkBuilder) -> Vec<DirEntry> {
let dents = Arc::new(Mutex::new(vec![]));
builder.build_parallel().run(|| {
let dents = dents.clone();
Box::new(move |result| {
if let Ok(dent) = result {
dents.lock().unwrap().push(dent);
}
WalkState::Continue
})
});
let dents = dents.lock().unwrap();
dents.to_vec()
}
fn mkpaths(paths: &[&str]) -> Vec<String> {
let mut paths: Vec<_> = paths.iter().map(|s| s.to_string()).collect();
paths.sort();
paths
}
fn tmpdir(prefix: &str) -> TempDir {
tempfile::Builder::new().prefix(prefix).tempdir().unwrap()
}
fn assert_paths(
prefix: &Path,
builder: &WalkBuilder,
expected: &[&str],
) {
let got = walk_collect(prefix, builder);
assert_eq!(got, mkpaths(expected), "single threaded");
let got = walk_collect_parallel(prefix, builder);
assert_eq!(got, mkpaths(expected), "parallel");
}
#[test]
fn no_ignores() {
let td = tmpdir("walk-test-");
mkdirp(td.path().join("a/b/c"));
mkdirp(td.path().join("x/y"));
wfile(td.path().join("a/b/foo"), "");
wfile(td.path().join("x/y/foo"), "");
assert_paths(td.path(), &WalkBuilder::new(td.path()), &[
"x", "x/y", "x/y/foo", "a", "a/b", "a/b/foo", "a/b/c",
]);
}
#[test]
fn custom_ignore() {
let td = tmpdir("walk-test-");
let custom_ignore = ".customignore";
mkdirp(td.path().join("a"));
wfile(td.path().join(custom_ignore), "foo");
wfile(td.path().join("foo"), "");
wfile(td.path().join("a/foo"), "");
wfile(td.path().join("bar"), "");
wfile(td.path().join("a/bar"), "");
let mut builder = WalkBuilder::new(td.path());
builder.add_custom_ignore_filename(&custom_ignore);
assert_paths(td.path(), &builder, &["bar", "a", "a/bar"]);
}
#[test]
fn custom_ignore_exclusive_use() {
let td = tmpdir("walk-test-");
let custom_ignore = ".customignore";
mkdirp(td.path().join("a"));
wfile(td.path().join(custom_ignore), "foo");
wfile(td.path().join("foo"), "");
wfile(td.path().join("a/foo"), "");
wfile(td.path().join("bar"), "");
wfile(td.path().join("a/bar"), "");
let mut builder = WalkBuilder::new(td.path());
builder.ignore(false);
builder.git_ignore(false);
builder.git_global(false);
builder.git_exclude(false);
builder.add_custom_ignore_filename(&custom_ignore);
assert_paths(td.path(), &builder, &["bar", "a", "a/bar"]);
}
#[test]
fn gitignore() {
let td = tmpdir("walk-test-");
mkdirp(td.path().join(".git"));
mkdirp(td.path().join("a"));
wfile(td.path().join(".gitignore"), "foo");
wfile(td.path().join("foo"), "");
wfile(td.path().join("a/foo"), "");
wfile(td.path().join("bar"), "");
wfile(td.path().join("a/bar"), "");
assert_paths(td.path(), &WalkBuilder::new(td.path()), &[
"bar", "a", "a/bar",
]);
}
#[test]
fn explicit_ignore() {
let td = tmpdir("walk-test-");
let igpath = td.path().join(".not-an-ignore");
mkdirp(td.path().join("a"));
wfile(&igpath, "foo");
wfile(td.path().join("foo"), "");
wfile(td.path().join("a/foo"), "");
wfile(td.path().join("bar"), "");
wfile(td.path().join("a/bar"), "");
let mut builder = WalkBuilder::new(td.path());
assert!(builder.add_ignore(&igpath).is_none());
assert_paths(td.path(), &builder, &["bar", "a", "a/bar"]);
}
#[test]
fn explicit_ignore_exclusive_use() {
let td = tmpdir("walk-test-");
let igpath = td.path().join(".not-an-ignore");
mkdirp(td.path().join("a"));
wfile(&igpath, "foo");
wfile(td.path().join("foo"), "");
wfile(td.path().join("a/foo"), "");
wfile(td.path().join("bar"), "");
wfile(td.path().join("a/bar"), "");
let mut builder = WalkBuilder::new(td.path());
builder.standard_filters(false);
assert!(builder.add_ignore(&igpath).is_none());
assert_paths(td.path(), &builder,
&[".not-an-ignore", "bar", "a", "a/bar"]);
}
#[test]
fn gitignore_parent() {
let td = tmpdir("walk-test-");
mkdirp(td.path().join(".git"));
mkdirp(td.path().join("a"));
wfile(td.path().join(".gitignore"), "foo");
wfile(td.path().join("a/foo"), "");
wfile(td.path().join("a/bar"), "");
let root = td.path().join("a");
assert_paths(&root, &WalkBuilder::new(&root), &["bar"]);
}
#[test]
fn max_depth() {
let td = tmpdir("walk-test-");
mkdirp(td.path().join("a/b/c"));
wfile(td.path().join("foo"), "");
wfile(td.path().join("a/foo"), "");
wfile(td.path().join("a/b/foo"), "");
wfile(td.path().join("a/b/c/foo"), "");
let mut builder = WalkBuilder::new(td.path());
assert_paths(td.path(), &builder, &[
"a", "a/b", "a/b/c", "foo", "a/foo", "a/b/foo", "a/b/c/foo",
]);
assert_paths(td.path(), builder.max_depth(Some(0)), &[]);
assert_paths(td.path(), builder.max_depth(Some(1)), &["a", "foo"]);
assert_paths(td.path(), builder.max_depth(Some(2)), &[
"a", "a/b", "foo", "a/foo",
]);
}
#[test]
fn max_filesize() {
let td = tmpdir("walk-test-");
mkdirp(td.path().join("a/b"));
wfile_size(td.path().join("foo"), 0);
wfile_size(td.path().join("bar"), 400);
wfile_size(td.path().join("baz"), 600);
wfile_size(td.path().join("a/foo"), 600);
wfile_size(td.path().join("a/bar"), 500);
wfile_size(td.path().join("a/baz"), 200);
let mut builder = WalkBuilder::new(td.path());
assert_paths(td.path(), &builder, &[
"a", "a/b", "foo", "bar", "baz", "a/foo", "a/bar", "a/baz",
]);
assert_paths(td.path(), builder.max_filesize(Some(0)), &[
"a", "a/b", "foo"
]);
assert_paths(td.path(), builder.max_filesize(Some(500)), &[
"a", "a/b", "foo", "bar", "a/bar", "a/baz"
]);
assert_paths(td.path(), builder.max_filesize(Some(50000)), &[
"a", "a/b", "foo", "bar", "baz", "a/foo", "a/bar", "a/baz",
]);
}
#[cfg(unix)]
#[test]
fn symlinks() {
let td = tmpdir("walk-test-");
mkdirp(td.path().join("a/b"));
symlink(td.path().join("a/b"), td.path().join("z"));
wfile(td.path().join("a/b/foo"), "");
let mut builder = WalkBuilder::new(td.path());
assert_paths(td.path(), &builder, &[
"a", "a/b", "a/b/foo", "z",
]);
assert_paths(td.path(), &builder.follow_links(true), &[
"a", "a/b", "a/b/foo", "z", "z/foo",
]);
}
#[cfg(unix)]
#[test]
fn first_path_not_symlink() {
let td = tmpdir("walk-test-");
mkdirp(td.path().join("foo"));
let dents = WalkBuilder::new(td.path().join("foo"))
.build()
.into_iter()
.collect::<Result<Vec<_>, _>>()
.unwrap();
assert_eq!(1, dents.len());
assert!(!dents[0].path_is_symlink());
let dents = walk_collect_entries_parallel(
&WalkBuilder::new(td.path().join("foo")),
);
assert_eq!(1, dents.len());
assert!(!dents[0].path_is_symlink());
}
#[cfg(unix)]
#[test]
fn symlink_loop() {
let td = tmpdir("walk-test-");
mkdirp(td.path().join("a/b"));
symlink(td.path().join("a"), td.path().join("a/b/c"));
let mut builder = WalkBuilder::new(td.path());
assert_paths(td.path(), &builder, &[
"a", "a/b", "a/b/c",
]);
assert_paths(td.path(), &builder.follow_links(true), &[
"a", "a/b",
]);
}
#[test]
#[cfg(target_os = "linux")]
fn same_file_system() {
use super::device_num;
if !Path::new("/sys").is_dir() {
return;
}
let td = tmpdir("walk-test-");
if device_num(td.path()).unwrap() == device_num("/sys").unwrap() {
return;
}
mkdirp(td.path().join("same_file"));
symlink("/sys", td.path().join("same_file").join("alink"));
let mut builder = WalkBuilder::new(td.path());
builder.follow_links(true).same_file_system(true);
assert_paths(td.path(), &builder, &[
"same_file", "same_file/alink",
]);
}
}