Struct DiscoveryConfig 

pub struct DiscoveryConfig {
    pub mode: DiscoveryMode,
    pub discover_sitemaps: bool,
    pub max_sitemap_depth: usize,
    pub extract_page_metadata: bool,
    pub link_extract_options: LinkExtractOptions,
    pub rules: Vec<DiscoveryRule>,
}

Core runtime types and traits used to define and run a crawl. Discovery-specific runtime configuration.

Fields

mode: DiscoveryMode

How the runtime should discover follow-up work from responses.

discover_sitemaps: bool

Whether sitemap XML should be parsed into follow-up requests.

max_sitemap_depth: usize

Maximum recursion depth for nested sitemap indexes.

extract_page_metadata: bool

Whether page metadata should be extracted and attached to response metadata.

link_extract_options: LinkExtractOptions

Base link extraction options used for HTML discovery.

rules: Vec<DiscoveryRule>

Optional rule-like link discovery behavior matched against source responses.

Implementations

impl DiscoveryConfig

pub fn new() -> DiscoveryConfig

Creates a new discovery config with default values.

pub fn with_mode(self, mode: DiscoveryMode) -> DiscoveryConfig

Sets the discovery mode.

pub fn with_sitemaps(self, enabled: bool) -> DiscoveryConfig

Enables or disables sitemap parsing.

pub fn with_max_sitemap_depth(self, depth: usize) -> DiscoveryConfig

Sets the maximum nested sitemap depth.

pub fn with_page_metadata(self, enabled: bool) -> DiscoveryConfig

Enables or disables page metadata extraction.

pub fn with_link_extract_options( self, options: LinkExtractOptions, ) -> DiscoveryConfig

Replaces the base link extraction options.

pub fn with_rules( self, rules: impl IntoIterator<Item = DiscoveryRule>, ) -> DiscoveryConfig

Replaces the configured discovery rules.

pub fn with_rule(self, rule: DiscoveryRule) -> DiscoveryConfig

Adds a single discovery rule.

pub fn with_same_site_only(self, enabled: bool) -> DiscoveryConfig

Sets whether only same-site links should be discovered.

pub fn with_text_links(self, enabled: bool) -> DiscoveryConfig

Sets whether text content should be scanned for plain-text URLs.

pub fn with_allow_patterns( self, patterns: impl IntoIterator<Item = impl Into<String>>, ) -> DiscoveryConfig

Restricts discovery to URLs that match at least one glob-style pattern.

pub fn with_deny_patterns( self, patterns: impl IntoIterator<Item = impl Into<String>>, ) -> DiscoveryConfig

Excludes URLs that match any glob-style pattern.

pub fn with_allow_domains( self, domains: impl IntoIterator<Item = impl Into<String>>, ) -> DiscoveryConfig

Restricts discovery to the given domains or subdomains.

pub fn with_deny_domains( self, domains: impl IntoIterator<Item = impl Into<String>>, ) -> DiscoveryConfig

Excludes discovery for the given domains or subdomains.

pub fn with_allow_path_prefixes( self, prefixes: impl IntoIterator<Item = impl Into<String>>, ) -> DiscoveryConfig

Restricts discovery to URL paths with one of the provided prefixes.

pub fn with_deny_path_prefixes( self, prefixes: impl IntoIterator<Item = impl Into<String>>, ) -> DiscoveryConfig

Excludes URL paths with one of the provided prefixes.

pub fn with_allowed_tags( self, tags: impl IntoIterator<Item = impl Into<String>>, ) -> DiscoveryConfig

Restricts attribute extraction to specific HTML tags.

pub fn with_allowed_attributes( self, attributes: impl IntoIterator<Item = impl Into<String>>, ) -> DiscoveryConfig

Restricts attribute extraction to specific attributes.

pub fn with_allowed_link_types( self, link_types: impl IntoIterator<Item = LinkType>, ) -> DiscoveryConfig

Restricts discovery to the provided link types.

pub fn with_denied_link_types( self, link_types: impl IntoIterator<Item = LinkType>, ) -> DiscoveryConfig

Excludes the provided link types from discovery.

pub fn effective_link_extract_options(&self) -> Option<LinkExtractOptions>

Returns the effective link extraction options for the configured mode.

pub fn effective_link_extract_options_for( &self, options: LinkExtractOptions, ) -> Option<LinkExtractOptions>

Returns the effective link extraction options for a specific rule or override.

pub fn should_extract_metadata(&self) -> bool

Returns true when metadata extraction should run.

Trait Implementations

impl Clone for DiscoveryConfig

fn clone(&self) -> DiscoveryConfig

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for DiscoveryConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl Default for DiscoveryConfig

fn default() -> DiscoveryConfig

Returns the “default value” for a type.

impl PartialEq for DiscoveryConfig

fn eq(&self, other: &DiscoveryConfig) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Eq for DiscoveryConfig

impl StructuralPartialEq for DiscoveryConfig