dirwalk 1.1.1

Platform-optimized recursive directory walker with metadata
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84

use crate::entry::Entry;
use crate::error::Error;
use crate::filter::Filter;
use crate::scan::scan_dir_prefixed;
use crate::walk::StorageHint;
use rayon::prelude::*;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::Arc;

type DirResult = Result<(Vec<Entry>, Vec<Error>), Error>;

pub(crate) struct ParallelConfig {
    pub max_depth: u32,
    pub include_hidden: bool,
    pub follow_links: bool,
    pub filter: Filter,
    pub has_filter: bool,
    pub storage_hint: StorageHint,
}

pub(crate) fn parallel_walk(
    root: &Path,
    config: &ParallelConfig,
    threads: usize,
    pool: Option<Arc<rayon::ThreadPool>>,
) -> DirResult {
    let pool = match pool {
        Some(p) => p,
        None => Arc::new(
            rayon::ThreadPoolBuilder::new()
                .num_threads(threads)
                .build()
                .map_err(|e| Error::Io {
                    path: root.to_path_buf(),
                    source: io::Error::other(e),
                })?,
        ),
    };
    pool.install(|| walk_dir(root, "", 0, config))
}

fn walk_dir(abs: &Path, rel: &str, depth: u32, config: &ParallelConfig) -> DirResult {
    let raw = scan_dir_prefixed(abs, rel, config.storage_hint)?;
    let child_depth = depth + 1;

    let mut entries = Vec::with_capacity(raw.len());
    let mut subdirs: Vec<(PathBuf, String)> = Vec::with_capacity(raw.len() / 8); // ~1 in 8 entries are subdirs on typical trees

    for mut e in raw {
        if !config.include_hidden && e.is_hidden {
            continue;
        }

        e.depth = child_depth;

        let should_descend = e.is_dir || (e.is_symlink && config.follow_links);
        if should_descend && child_depth < config.max_depth {
            subdirs.push((abs.join(e.name()), e.relative_path.clone()));
        }

        if !config.has_filter || config.filter.matches(&e, abs) {
            entries.push(e);
        }
    }

    let sub_results: Vec<DirResult> = subdirs
        .into_par_iter()
        .map(|(child_abs, child_rel)| walk_dir(&child_abs, &child_rel, child_depth, config))
        .collect();

    let mut errors = Vec::new();
    for result in sub_results {
        match result {
            Ok((mut sub_entries, mut sub_errors)) => {
                entries.append(&mut sub_entries);
                errors.append(&mut sub_errors);
            }
            Err(e) => errors.push(e),
        }
    }

    Ok((entries, errors))
}