Struct CrawlerBuilder 

pub struct CrawlerBuilder<S: Spider, D>
where
    D: Downloader,
{ /* private fields */ }

Implementations

impl<S: Spider> CrawlerBuilder<S, ReqwestClientDownloader>

pub fn new(spider: S) -> Self

Creates a new CrawlerBuilder for a given spider with the default ReqwestClientDownloader.

impl<S: Spider, D: Downloader> CrawlerBuilder<S, D>

pub fn max_concurrent_downloads(self, limit: usize) -> Self

Sets the maximum number of concurrent downloads.

pub fn max_parser_workers(self, limit: usize) -> Self

Sets the maximum number of concurrent parser workers.

pub fn max_concurrent_pipelines(self, limit: usize) -> Self

Sets the maximum number of concurrent pipelines.

pub fn channel_capacity(self, capacity: usize) -> Self

Sets the capacity of communication channels between components.

pub fn downloader(self, downloader: D) -> Self

Sets a custom downloader for the crawler.

pub fn add_middleware<M>(self, middleware: M) -> Self

where M: Middleware<D::Client> + Send + Sync + 'static,

Adds a middleware to the crawler.

pub fn add_pipeline<P>(self, pipeline: P) -> Self

where P: Pipeline<S::Item> + 'static,

Adds an item pipeline to the crawler.

pub fn with_checkpoint_path<P: AsRef<Path>>(self, path: P) -> Self

Enables checkpointing and sets the path for the checkpoint file.

pub fn with_checkpoint_interval(self, interval: Duration) -> Self

Sets the interval for periodic checkpointing.

pub async fn build(self) -> Result<Crawler<S, D::Client>, SpiderError>

where D: Downloader + Send + Sync + 'static, D::Client: Send + Sync + Clone, S::Item: Send + Sync + 'static,

Builds the Crawler instance, initializing and passing the StatCollector along with other configured components.

Trait Implementations

impl<S: Spider> Default for CrawlerBuilder<S, ReqwestClientDownloader>

fn default() -> Self

Returns the “default value” for a type.