Struct CrawlerBuilder

Source

pub struct CrawlerBuilder<S: Spider, D = ReqwestClientDownloader>where
    D: Downloader,
{ /* private fields */ }

Implementations§

Source §

impl<S: Spider, D: Downloader> CrawlerBuilder<S, D>

Source

pub fn new(spider: S) -> Self
where D: Default,

Creates a new CrawlerBuilder for a given spider.

Examples found in repository ?

examples/quotes_scraper.rs (line 89)

83async fn main() -> Result<(), SpiderError> {
84    tracing_subscriber::fmt()
85        .with_env_filter("info,spider_lib=debug")
86        .without_time()
87        .init();
88
89    let crawler = CrawlerBuilder::<_, ReqwestClientDownloader>::new(QuotesSpider)
90        .add_pipeline(DeduplicationPipeline::new(&["text"]))
91        .add_pipeline(CsvExporterPipeline::<QuoteItem>::new("output/quotes.csv")?)
92        .add_middleware(HttpCacheMiddleware::builder().build()?)
93        .add_middleware(
94            RateLimitMiddleware::builder()
95                .use_token_bucket_limiter(5)
96                .build(),
97        )
98        .add_middleware(RetryMiddleware::default().max_retries(2))
99        .add_middleware(UserAgentMiddleware::builder().build()?)
100        .add_middleware(RobotsTxtMiddleware::new())
101        .add_middleware(
102            RefererMiddleware::new()
103                .same_origin_only(true)
104                .max_chain_length(100)
105                .include_fragment(false),
106        )
107        .with_checkpoint_path("output/quotes.bin")
108        .with_checkpoint_interval(Duration::from_secs(15))
109        .max_concurrent_downloads(5)
110        .max_parser_workers(2)
111        .max_concurrent_pipelines(2)
112        .build()
113        .await?;
114
115    crawler.start_crawl().await?;
116
117    Ok(())
118}

Source

pub fn max_concurrent_downloads(self, limit: usize) -> Self

Sets the maximum number of concurrent downloads.

Examples found in repository ?

examples/quotes_scraper.rs (line 109)

83async fn main() -> Result<(), SpiderError> {
84    tracing_subscriber::fmt()
85        .with_env_filter("info,spider_lib=debug")
86        .without_time()
87        .init();
88
89    let crawler = CrawlerBuilder::<_, ReqwestClientDownloader>::new(QuotesSpider)
90        .add_pipeline(DeduplicationPipeline::new(&["text"]))
91        .add_pipeline(CsvExporterPipeline::<QuoteItem>::new("output/quotes.csv")?)
92        .add_middleware(HttpCacheMiddleware::builder().build()?)
93        .add_middleware(
94            RateLimitMiddleware::builder()
95                .use_token_bucket_limiter(5)
96                .build(),
97        )
98        .add_middleware(RetryMiddleware::default().max_retries(2))
99        .add_middleware(UserAgentMiddleware::builder().build()?)
100        .add_middleware(RobotsTxtMiddleware::new())
101        .add_middleware(
102            RefererMiddleware::new()
103                .same_origin_only(true)
104                .max_chain_length(100)
105                .include_fragment(false),
106        )
107        .with_checkpoint_path("output/quotes.bin")
108        .with_checkpoint_interval(Duration::from_secs(15))
109        .max_concurrent_downloads(5)
110        .max_parser_workers(2)
111        .max_concurrent_pipelines(2)
112        .build()
113        .await?;
114
115    crawler.start_crawl().await?;
116
117    Ok(())
118}

Source

pub fn max_parser_workers(self, limit: usize) -> Self

Sets the maximum number of concurrent parser workers.

Examples found in repository ?

examples/quotes_scraper.rs (line 110)

83async fn main() -> Result<(), SpiderError> {
84    tracing_subscriber::fmt()
85        .with_env_filter("info,spider_lib=debug")
86        .without_time()
87        .init();
88
89    let crawler = CrawlerBuilder::<_, ReqwestClientDownloader>::new(QuotesSpider)
90        .add_pipeline(DeduplicationPipeline::new(&["text"]))
91        .add_pipeline(CsvExporterPipeline::<QuoteItem>::new("output/quotes.csv")?)
92        .add_middleware(HttpCacheMiddleware::builder().build()?)
93        .add_middleware(
94            RateLimitMiddleware::builder()
95                .use_token_bucket_limiter(5)
96                .build(),
97        )
98        .add_middleware(RetryMiddleware::default().max_retries(2))
99        .add_middleware(UserAgentMiddleware::builder().build()?)
100        .add_middleware(RobotsTxtMiddleware::new())
101        .add_middleware(
102            RefererMiddleware::new()
103                .same_origin_only(true)
104                .max_chain_length(100)
105                .include_fragment(false),
106        )
107        .with_checkpoint_path("output/quotes.bin")
108        .with_checkpoint_interval(Duration::from_secs(15))
109        .max_concurrent_downloads(5)
110        .max_parser_workers(2)
111        .max_concurrent_pipelines(2)
112        .build()
113        .await?;
114
115    crawler.start_crawl().await?;
116
117    Ok(())
118}

Source

pub fn max_concurrent_pipelines(self, limit: usize) -> Self

Sets the maximum number of concurrent pipelines.

Examples found in repository ?

examples/quotes_scraper.rs (line 111)

83async fn main() -> Result<(), SpiderError> {
84    tracing_subscriber::fmt()
85        .with_env_filter("info,spider_lib=debug")
86        .without_time()
87        .init();
88
89    let crawler = CrawlerBuilder::<_, ReqwestClientDownloader>::new(QuotesSpider)
90        .add_pipeline(DeduplicationPipeline::new(&["text"]))
91        .add_pipeline(CsvExporterPipeline::<QuoteItem>::new("output/quotes.csv")?)
92        .add_middleware(HttpCacheMiddleware::builder().build()?)
93        .add_middleware(
94            RateLimitMiddleware::builder()
95                .use_token_bucket_limiter(5)
96                .build(),
97        )
98        .add_middleware(RetryMiddleware::default().max_retries(2))
99        .add_middleware(UserAgentMiddleware::builder().build()?)
100        .add_middleware(RobotsTxtMiddleware::new())
101        .add_middleware(
102            RefererMiddleware::new()
103                .same_origin_only(true)
104                .max_chain_length(100)
105                .include_fragment(false),
106        )
107        .with_checkpoint_path("output/quotes.bin")
108        .with_checkpoint_interval(Duration::from_secs(15))
109        .max_concurrent_downloads(5)
110        .max_parser_workers(2)
111        .max_concurrent_pipelines(2)
112        .build()
113        .await?;
114
115    crawler.start_crawl().await?;
116
117    Ok(())
118}

Source

pub fn downloader(self, downloader: D) -> Self

Sets a custom downloader for the crawler.

Source

pub fn add_middleware<M>(self, middleware: M) -> Self
where D: Downloader, M: Middleware<D::Client> + Send + Sync + 'static,

Adds a middleware to the crawler.

Examples found in repository ?

examples/quotes_scraper.rs (line 92)

83async fn main() -> Result<(), SpiderError> {
84    tracing_subscriber::fmt()
85        .with_env_filter("info,spider_lib=debug")
86        .without_time()
87        .init();
88
89    let crawler = CrawlerBuilder::<_, ReqwestClientDownloader>::new(QuotesSpider)
90        .add_pipeline(DeduplicationPipeline::new(&["text"]))
91        .add_pipeline(CsvExporterPipeline::<QuoteItem>::new("output/quotes.csv")?)
92        .add_middleware(HttpCacheMiddleware::builder().build()?)
93        .add_middleware(
94            RateLimitMiddleware::builder()
95                .use_token_bucket_limiter(5)
96                .build(),
97        )
98        .add_middleware(RetryMiddleware::default().max_retries(2))
99        .add_middleware(UserAgentMiddleware::builder().build()?)
100        .add_middleware(RobotsTxtMiddleware::new())
101        .add_middleware(
102            RefererMiddleware::new()
103                .same_origin_only(true)
104                .max_chain_length(100)
105                .include_fragment(false),
106        )
107        .with_checkpoint_path("output/quotes.bin")
108        .with_checkpoint_interval(Duration::from_secs(15))
109        .max_concurrent_downloads(5)
110        .max_parser_workers(2)
111        .max_concurrent_pipelines(2)
112        .build()
113        .await?;
114
115    crawler.start_crawl().await?;
116
117    Ok(())
118}

Source

pub fn add_pipeline(self, pipeline: P) -> Self
where P: Pipeline<S::Item> + 'static,

Adds an item pipeline to the crawler.

Examples found in repository ?

examples/quotes_scraper.rs (line 90)

83async fn main() -> Result<(), SpiderError> {
84    tracing_subscriber::fmt()
85        .with_env_filter("info,spider_lib=debug")
86        .without_time()
87        .init();
88
89    let crawler = CrawlerBuilder::<_, ReqwestClientDownloader>::new(QuotesSpider)
90        .add_pipeline(DeduplicationPipeline::new(&["text"]))
91        .add_pipeline(CsvExporterPipeline::<QuoteItem>::new("output/quotes.csv")?)
92        .add_middleware(HttpCacheMiddleware::builder().build()?)
93        .add_middleware(
94            RateLimitMiddleware::builder()
95                .use_token_bucket_limiter(5)
96                .build(),
97        )
98        .add_middleware(RetryMiddleware::default().max_retries(2))
99        .add_middleware(UserAgentMiddleware::builder().build()?)
100        .add_middleware(RobotsTxtMiddleware::new())
101        .add_middleware(
102            RefererMiddleware::new()
103                .same_origin_only(true)
104                .max_chain_length(100)
105                .include_fragment(false),
106        )
107        .with_checkpoint_path("output/quotes.bin")
108        .with_checkpoint_interval(Duration::from_secs(15))
109        .max_concurrent_downloads(5)
110        .max_parser_workers(2)
111        .max_concurrent_pipelines(2)
112        .build()
113        .await?;
114
115    crawler.start_crawl().await?;
116
117    Ok(())
118}

Source

pub fn with_checkpoint_path<P: AsRef<Path>>(self, path: P) -> Self

Enables checkpointing and sets the path for the checkpoint file.

Examples found in repository ?

examples/quotes_scraper.rs (line 107)

83async fn main() -> Result<(), SpiderError> {
84    tracing_subscriber::fmt()
85        .with_env_filter("info,spider_lib=debug")
86        .without_time()
87        .init();
88
89    let crawler = CrawlerBuilder::<_, ReqwestClientDownloader>::new(QuotesSpider)
90        .add_pipeline(DeduplicationPipeline::new(&["text"]))
91        .add_pipeline(CsvExporterPipeline::<QuoteItem>::new("output/quotes.csv")?)
92        .add_middleware(HttpCacheMiddleware::builder().build()?)
93        .add_middleware(
94            RateLimitMiddleware::builder()
95                .use_token_bucket_limiter(5)
96                .build(),
97        )
98        .add_middleware(RetryMiddleware::default().max_retries(2))
99        .add_middleware(UserAgentMiddleware::builder().build()?)
100        .add_middleware(RobotsTxtMiddleware::new())
101        .add_middleware(
102            RefererMiddleware::new()
103                .same_origin_only(true)
104                .max_chain_length(100)
105                .include_fragment(false),
106        )
107        .with_checkpoint_path("output/quotes.bin")
108        .with_checkpoint_interval(Duration::from_secs(15))
109        .max_concurrent_downloads(5)
110        .max_parser_workers(2)
111        .max_concurrent_pipelines(2)
112        .build()
113        .await?;
114
115    crawler.start_crawl().await?;
116
117    Ok(())
118}

Source

pub fn with_checkpoint_interval(self, interval: Duration) -> Self

Sets the interval for periodic checkpointing.

Examples found in repository ?

examples/quotes_scraper.rs (line 108)

83async fn main() -> Result<(), SpiderError> {
84    tracing_subscriber::fmt()
85        .with_env_filter("info,spider_lib=debug")
86        .without_time()
87        .init();
88
89    let crawler = CrawlerBuilder::<_, ReqwestClientDownloader>::new(QuotesSpider)
90        .add_pipeline(DeduplicationPipeline::new(&["text"]))
91        .add_pipeline(CsvExporterPipeline::<QuoteItem>::new("output/quotes.csv")?)
92        .add_middleware(HttpCacheMiddleware::builder().build()?)
93        .add_middleware(
94            RateLimitMiddleware::builder()
95                .use_token_bucket_limiter(5)
96                .build(),
97        )
98        .add_middleware(RetryMiddleware::default().max_retries(2))
99        .add_middleware(UserAgentMiddleware::builder().build()?)
100        .add_middleware(RobotsTxtMiddleware::new())
101        .add_middleware(
102            RefererMiddleware::new()
103                .same_origin_only(true)
104                .max_chain_length(100)
105                .include_fragment(false),
106        )
107        .with_checkpoint_path("output/quotes.bin")
108        .with_checkpoint_interval(Duration::from_secs(15))
109        .max_concurrent_downloads(5)
110        .max_parser_workers(2)
111        .max_concurrent_pipelines(2)
112        .build()
113        .await?;
114
115    crawler.start_crawl().await?;
116
117    Ok(())
118}

Source

pub async fn build(self) -> Result<Crawler<S, D::Client>, SpiderError>
where D: Downloader + Send + Sync + 'static, D::Client: Send + Sync,

Builds the Crawler instance.

Examples found in repository ?

examples/quotes_scraper.rs (line 112)

83async fn main() -> Result<(), SpiderError> {
84    tracing_subscriber::fmt()
85        .with_env_filter("info,spider_lib=debug")
86        .without_time()
87        .init();
88
89    let crawler = CrawlerBuilder::<_, ReqwestClientDownloader>::new(QuotesSpider)
90        .add_pipeline(DeduplicationPipeline::new(&["text"]))
91        .add_pipeline(CsvExporterPipeline::<QuoteItem>::new("output/quotes.csv")?)
92        .add_middleware(HttpCacheMiddleware::builder().build()?)
93        .add_middleware(
94            RateLimitMiddleware::builder()
95                .use_token_bucket_limiter(5)
96                .build(),
97        )
98        .add_middleware(RetryMiddleware::default().max_retries(2))
99        .add_middleware(UserAgentMiddleware::builder().build()?)
100        .add_middleware(RobotsTxtMiddleware::new())
101        .add_middleware(
102            RefererMiddleware::new()
103                .same_origin_only(true)
104                .max_chain_length(100)
105                .include_fragment(false),
106        )
107        .with_checkpoint_path("output/quotes.bin")
108        .with_checkpoint_interval(Duration::from_secs(15))
109        .max_concurrent_downloads(5)
110        .max_parser_workers(2)
111        .max_concurrent_pipelines(2)
112        .build()
113        .await?;
114
115    crawler.start_crawl().await?;
116
117    Ok(())
118}