Struct CrawlOptions 

pub struct CrawlOptions { /* private fields */ }

Options for crawling a site.

Implementations

impl CrawlOptions

pub fn new(url: &str) -> Self

Create crawl options for the given seed URL.

pub fn limit(self, n: usize) -> Self

Maximum number of pages to crawl (default: 50).

pub fn max_depth(self, n: usize) -> Self

Maximum link depth from the seed URL (default: 3).

pub fn timeout(self, timeout: Duration) -> Self

Page load timeout per page (default: 30s).

pub fn settle(self, settle: Duration) -> Self

Extra wait after load event per page (default: 0).

pub fn include(self, patterns: &[&str]) -> Self

URL path glob patterns to include (e.g. "/docs/**").

pub fn exclude(self, patterns: &[&str]) -> Self

URL path glob patterns to exclude (e.g. "/docs/archive/**").

pub fn json(self, json: bool) -> Self

Output crawled content as JSON instead of Markdown.

pub fn selector(self, selector: impl Into<String>) -> Self

CSS selector to extract a specific section per page.

pub fn user_agent(self, ua: impl Into<String>) -> Self

Override the User-Agent string for all pages in this crawl.

Trait Implementations

impl Clone for CrawlOptions

fn clone(&self) -> CrawlOptions

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for CrawlOptions

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.