Struct Crawler 

pub struct Crawler<A, C> { /* private fields */ }

The core of this library. Create one with Crawler::new or Crawler::new_async to get started. Also see the examples.

Implementations

impl Crawler<NonAsync, NoContext>

pub fn new() -> Self

Create a new non-async, parallel Crawler without any context

let parallel_crawler=Crawler::new();

impl<C> Crawler<NonAsync, C>

where C: Send + Sync,

pub fn start_dir<P: AsRef<Path>>(self, path: P) -> Self

Sets the directory the crawler should start in. Default is the current directory, resolved when Crawler::run is called, if that fails, it panics before doing anything.

 use std::collections::HashSet;
 use std::path::PathBuf;
 use std::sync::Mutex;

 //Assuming that the content of C:\foo isn't changing during execution
 //and this program is executed in that same folder
 let crawler_1_result =
 Crawler::new()
    .start_dir("C:\\foo")
    .context(Mutex::new(Vec::new()))
    .run(|ctx: Mutex<Vec<String>>, path| {
        ctx.lock().unwrap().insert(path.display());
    });
 let crawler_2_result =
 Crawler::new()
    .context(Mutex::new(Vec::new()))
    .run(|ctx: Mutex<HashSet<String>>, path| {
        ctx.lock().unwrap().insert(path.display());
    })?;

 //then (not guaranteed when using a Vec instead of a HashSet by design)
 assert_eq!(crawler_1_result, crawler_2_result);

pub fn file_regex<STR: AsRef<str>>(self, regex: STR) -> Self

Only applies the closure in run to a file if the given regex matches

//prints all text files in the current directory (and all its subfolders)
Crawler::new()
    .file_regex(r"^.*\.txt$")
    .run(|_, path| {
        println!("{}", path.display());
    })?;

pub fn folder_regex<STR: AsRef<str>>(self, regex: STR) -> Self

Only go into a folder if matches the given regex (meaning all files and subfolders etc. will not be traversed)

//given this folder structure:
//foo
// |--bar
// |   |--foo.txt
// |--foobar
// |    |---barbar
// |           |---baz.txt
// |--foo
//     |--baz.txt

//this prints *only* baz.txt because the regex matches "foo", but not "bar" or "barbar" AND "foobar"
Crawler::new()
    .start_dir("path\\to\\foo")
    .folder_regex("foo")
    .run(|_, path| {
        println!("{}", path.display());
    })?;

pub fn search_depth(self, depth: u32) -> Self

How deep (in terms of folder layers over each other) the Crawler should go

//prints all text files in the current directory, but not its subfolders
Crawler::new()
    //exchanging the 0 with a 1 mean that it also traverses the subfolders, but not their subfolders
    .search_depth(0)
    .file_regex(r"^.*\.txt$")
    .run(|_, path| {
        println!("{}", path.display());
    })?;

pub fn context<CNEW: Send + Sync>( self, context: CNEW, ) -> Crawler<NonAsync, CNEW>

Adds a context ( = a value that is passed to the closure on every invocation via an Arc) with the type CNEW. It is returned from the run function after execution. Defaults to the zero-sized NoContext.

 use std::sync::atomic::AtomicU16;

 //bind the context to a variable
 let result =
 Crawler::new()
    //adds a counter (for everything not representable with Atomics, a Mutex is recommended)
     .context(AtomicU16::new(0))
     .run(|_, path| {
         println!("{}", path.display());
     })?;
 println!("{} files in the current directory")

pub fn run<A, E>(self, action: A) -> Result<C, Box<dyn Error + Send + 'static>>

where A: FnMut(Arc<C>, PathBuf) -> Result<(), E> + Clone + Send + Sync, E: Error + Send + 'static,

Runs the (modified) Crawler returned from Crawler::new, execution a closure that’s passed a Context and the path of the file for every file in the specified directory. For exceptions, see search_depth, file_regex and folder_regex.

 use file_crawler::prelude::*;

 use std::path::PathBuf;

 Crawler::new()
    .start_dir("C\\user\\foo")
    .file_regex(r"^.*\.txt$")
    .search_depth(3)
    .run(|_, path| {
        println!("{}", path.display());
        Ok(())
    })?;

impl Crawler<Async, NoContext>

pub fn new_async() -> Self

Create a new async (parallel)[^async_disclaimer] Crawler without any context

let async_crawler=Crawler::new_async();

impl<C> Crawler<Async, C>

where C: Send + Sync + 'static,

pub fn start_dir<P: AsRef<Path>>(self, path: P) -> Self

See Crawler::search_depth.

pub fn file_regex<STR: AsRef<str>>(self, regex: STR) -> Self

See Crawler::file_regex.

pub fn folder_regex<STR: AsRef<str>>(self, regex: STR) -> Self

See Crawler::folder_regex.

pub fn search_depth(self, depth: u32) -> Self

See Crawler::search_depth.

pub fn context<CNEW: Send + Sync + 'static>( self, context: CNEW, ) -> Crawler<Async, CNEW>

See Crawler::context.

pub async fn run<Fun, Fut, E>( self, action: Fun, ) -> Result<C, Box<dyn Error + Send + 'static>>

where E: Send + Error + 'static, Fun: Fn(Arc<C>, PathBuf) -> Fut + Send + 'static + Clone, Fut: Future<Output = Result<(), E>> + Send + 'static,

Runs a (modified) asynchronous file crawler from Crawler::new_async using tokio. Requires an at least two-threaded runtime (3). Otherwise, the same as the synchronous version. It is recommended to use the exposed tokio (through the prelude) dependency instead of std when possible.

 use file_crawler::prelude::*;

 use std::path::PathBuf;

 Crawler::new()
    .start_dir("C\\user\\foo")
    .file_regex(r"^.*\.txt$")
    .run(|_, path| {
        let contents=String::new();
        let file=tokio::fs::File::open(&path).await?;
        file.read_to_string(&mut contents).await?;
        println!("{}:\n{}", path.display(), contents);
        Ok(())
    })?;

Trait Implementations

impl<A: Clone, C: Clone> Clone for Crawler<A, C>

fn clone(&self) -> Crawler<A, C>

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<A: Debug, C: Debug> Debug for Crawler<A, C>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<A: Default, C: Default> Default for Crawler<A, C>

fn default() -> Crawler<A, C>

Returns the “default value” for a type.