Struct CrawlScrapeOptions

pub struct CrawlScrapeOptions {
    pub formats: Option<Vec<CrawlScrapeFormats>>,
    pub only_main_content: Option<bool>,
    pub include_tags: Option<Vec<String>>,
    pub exclude_tags: Option<Vec<String>>,
    pub headers: Option<HashMap<String, String>>,
    pub wait_for: Option<u32>,
    pub timeout: Option<u32>,
}

Fields

formats: Option<Vec<CrawlScrapeFormats>>

Formats to extract from the page. (default: [ Markdown ])

only_main_content: Option<bool>

Only extract the main content of the page, excluding navigation and other miscellaneous content. (default: true)

include_tags: Option<Vec<String>>

HTML tags to exclusively include.

For example, if you pass div, you will only get content from <div>s and their children.

exclude_tags: Option<Vec<String>>

HTML tags to exclude.

For example, if you pass img, you will never get image URLs in your results.

headers: Option<HashMap<String, String>>

Additional HTTP headers to use when loading the page.

wait_for: Option<u32>

Amount of time to wait after loading the page, and before grabbing the content, in milliseconds. (default: 0)

timeout: Option<u32>

Timeout before returning an error, in milliseconds. (default: 60000)

Trait Implementations

impl Clone for CrawlScrapeOptions

fn clone(&self) -> CrawlScrapeOptions

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for CrawlScrapeOptions

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for CrawlScrapeOptions

fn default() -> CrawlScrapeOptions

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for CrawlScrapeOptions

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl From<CrawlScrapeOptions> for ScrapeOptions

fn from(value: CrawlScrapeOptions) -> Self

Converts to this type from the input type.

impl Serialize for CrawlScrapeOptions

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.