Struct Crawler 

pub struct Crawler<S: Spider, C> { /* private fields */ }

Implementations

impl<S, C> Crawler<S, C>

where S: Spider + 'static, S::Item: ScrapedItem, C: Send + Sync + 'static,

pub async fn start_crawl(self) -> Result<(), SpiderError>

Starts the crawl.

Examples found in repository

examples/quotes_scraper.rs (line 115)

async fn main() -> Result<(), SpiderError> {
    tracing_subscriber::fmt()
        .with_env_filter("info,spider_lib=debug")
        .without_time()
        .init();

    let crawler = CrawlerBuilder::<_, ReqwestClientDownloader>::new(QuotesSpider)
        .add_pipeline(DeduplicationPipeline::new(&["text"]))
        .add_pipeline(CsvExporterPipeline::<QuoteItem>::new("output/quotes.csv")?)
        .add_middleware(HttpCacheMiddleware::builder().build()?)
        .add_middleware(
            RateLimitMiddleware::builder()
                .use_token_bucket_limiter(5)
                .build(),
        )
        .add_middleware(RetryMiddleware::default().max_retries(2))
        .add_middleware(UserAgentMiddleware::builder().build()?)
        .add_middleware(RobotsTxtMiddleware::new())
        .add_middleware(
            RefererMiddleware::new()
                .same_origin_only(true)
                .max_chain_length(100)
                .include_fragment(false),
        )
        .with_checkpoint_path("output/quotes.bin")
        .with_checkpoint_interval(Duration::from_secs(15))
        .max_concurrent_downloads(5)
        .max_parser_workers(2)
        .max_concurrent_pipelines(2)
        .build()
        .await?;

    crawler.start_crawl().await?;

    Ok(())
}