Struct HttpCacheMiddlewareBuilder 

pub struct HttpCacheMiddlewareBuilder { /* private fields */ }

Builder for HttpCacheMiddleware.

Implementations

impl HttpCacheMiddlewareBuilder

pub fn cache_dir(self, path: PathBuf) -> Self

Sets the directory where cache files will be stored.

pub fn build(self) -> Result<HttpCacheMiddleware, SpiderError>

Builds the HttpCacheMiddleware. This can fail if the cache directory cannot be created or determined.

Examples found in repository

examples/quotes_scraper.rs (line 82)

async fn main() -> Result<(), SpiderError> {
    tracing_subscriber::fmt()
        .with_env_filter("info,spider_lib=debug")
        .without_time()
        .init();

    let crawler = CrawlerBuilder::<_, ReqwestClientDownloader>::new(QuotesSpider)
        .add_pipeline(DeduplicationPipeline::new(&["text"]))
        .add_pipeline(CsvExporterPipeline::<QuoteItem>::new("output/quotes.csv")?)
        .add_middleware(HttpCacheMiddleware::builder().build()?)
        .add_middleware(UserAgentMiddleware::builder().build()?)
        .add_middleware(RobotsTxtMiddleware::new())
        .add_middleware(
            RefererMiddleware::new()
                .same_origin_only(true)
                .max_chain_length(100)
                .include_fragment(false),
        )
        .with_checkpoint_path("output/quotes.bin")
        .with_checkpoint_interval(Duration::from_secs(15))
        .max_concurrent_downloads(5)
        .max_parser_workers(2)
        .max_concurrent_pipelines(2)
        .build()
        .await?;

    crawler.start_crawl().await?;

    Ok(())
}

Trait Implementations

impl Default for HttpCacheMiddlewareBuilder

fn default() -> HttpCacheMiddlewareBuilder

Returns the “default value” for a type.