Struct CrawlerConfigBuilder 

pub struct CrawlerConfigBuilder { /* private fields */ }

Builder for CrawlerConfig

Following api-builder and api-must-use.

Implementations

impl CrawlerConfigBuilder

pub fn new(seed_url: Url) -> Self

Create a new builder with seed URL

pub fn max_depth(self, depth: u8) -> Self

Set maximum crawl depth

pub fn max_pages(self, pages: usize) -> Self

Set maximum number of pages

pub fn include_pattern(self, pattern: impl Into<String>) -> Self

Add an include pattern

pub fn include_patterns(self, patterns: Vec<String>) -> Self

Add multiple include patterns

pub fn exclude_pattern(self, pattern: impl Into<String>) -> Self

Add an exclude pattern

pub fn exclude_patterns(self, patterns: Vec<String>) -> Self

Add multiple exclude patterns

pub fn concurrency(self, level: usize) -> Self

Set concurrency level

pub fn delay_ms(self, ms: u64) -> Self

Set delay between requests in milliseconds

pub fn user_agent(self, ua: impl Into<String>) -> Self

Set user agent string

pub fn timeout_secs(self, secs: u64) -> Self

Set request timeout in seconds

pub fn use_sitemap(self, use_sitemap: bool) -> Self

Set use_sitemap flag (FASE 3)

pub fn sitemap_url(self, url: impl Into<String>) -> Self

Set explicit sitemap URL (FASE 3)

pub fn build(self) -> CrawlerConfig

Trait Implementations

impl Debug for CrawlerConfigBuilder

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.