crusty-core 0.82.0

Library for creating blazing fast and configurable web crawlers
Documentation
#[macro_use]
extern crate lazy_static;
#[macro_use]
mod macro_helpers;

pub use flume;
#[cfg(feature = "select_rs")]
pub use select;
#[cfg(feature = "select_rs")]
pub mod select_task_expanders;

mod _prelude;
pub mod prelude;

mod hyper_utils;
mod task_processor;
mod task_scheduler;

pub mod config;
pub mod resolver;
pub mod types;

mod crawler;
pub use crawler::*;
mod parser_processor;
pub use parser_processor::*;
pub mod status_filters;
pub use status_filters::Filter as StatusFilter;
pub mod load_filters;
pub use load_filters::Filter as LoadFilter;
pub mod task_filters;
pub use task_filters::Filter as TaskFilter;
pub mod task_expanders;
pub use task_expanders::Expander as TaskExpander;