website_crawler 0.5.0

gRPC web crawler that is very fast
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

extern crate tokio;
extern crate tonic;
extern crate ua_generator;

// packages mainly for spider
extern crate hashbrown;
extern crate log;
extern crate num_cpus;
extern crate rayon;
extern crate reqwest;
extern crate scraper;
extern crate url;
#[macro_use]
extern crate lazy_static;

pub mod interface;
pub mod packages;
pub mod rpc;
pub mod scanner; // internal packages.

pub use packages::spider;
pub use rpc::handlers::grpc_start;