Struct CrawlerBuilder 

pub struct CrawlerBuilder<S, D>
where
    S: Spider,
    D: Downloader,
{ /* private fields */ }

Implementations

impl<S> CrawlerBuilder<S, ReqwestClientDownloader>

where S: Spider,

pub fn new(spider: S) -> CrawlerBuilder<S, ReqwestClientDownloader>

Creates a new CrawlerBuilder for a given spider with the default ReqwestClientDownloader.

impl<S, D> CrawlerBuilder<S, D>

where S: Spider, D: Downloader,

pub fn max_concurrent_downloads(self, limit: usize) -> CrawlerBuilder<S, D>

pub fn max_parser_workers(self, limit: usize) -> CrawlerBuilder<S, D>

pub fn max_concurrent_pipelines(self, limit: usize) -> CrawlerBuilder<S, D>

pub fn channel_capacity(self, capacity: usize) -> CrawlerBuilder<S, D>

pub fn downloader(self, downloader: D) -> CrawlerBuilder<S, D>

pub fn add_middleware<M>(self, middleware: M) -> CrawlerBuilder<S, D>

where M: Middleware<<D as Downloader>::Client> + Send + Sync + 'static,

pub fn add_pipeline<P>(self, pipeline: P) -> CrawlerBuilder<S, D>

where P: Pipeline<<S as Spider>::Item> + 'static,

pub fn with_checkpoint_path<P>(self, path: P) -> CrawlerBuilder<S, D>

where P: AsRef<Path>,

pub fn with_checkpoint_interval( self, interval: Duration, ) -> CrawlerBuilder<S, D>

pub async fn build( self, ) -> Result<Crawler<S, <D as Downloader>::Client>, SpiderError>

where D: Downloader + Send + Sync + 'static, <D as Downloader>::Client: Send + Sync + Clone, <S as Spider>::Item: Send + Sync + 'static,

Trait Implementations

impl<S> Default for CrawlerBuilder<S, ReqwestClientDownloader>

where S: Spider,

fn default() -> CrawlerBuilder<S, ReqwestClientDownloader>

Returns the “default value” for a type.