Struct ScrapeRequest 

pub struct ScrapeRequest {
    pub website_url: String,
    pub clean: Option<bool>,
    pub parse_mode: Option<ParseMode>,
    pub tag_truncate: Option<bool>,
    pub extract_links: Option<bool>,
    pub include_tags: Option<Vec<String>>,
    pub exclude_tags: Option<Vec<String>>,
    pub headers: Option<HashMap<String, String>>,
    pub max_age: Option<u64>,
    pub stealth: Option<bool>,
}

Request body for Client::scrape.

Build with ScrapeRequest::new and chain the option setters; only fields you touch are serialized.

use webscrape_ai::ScrapeRequest;
let req = ScrapeRequest::new("https://example.com").clean(true).extract_links(true);
let json = serde_json::to_value(&req).unwrap();
assert_eq!(json["website_url"], "https://example.com");
assert_eq!(json["clean"], true);
assert!(json.get("tag_truncate").is_none()); // untouched -> absent

Fields

website_url: String

The URL to fetch. Required.

clean: Option<bool>

Convert HTML to cleaned markdown.

parse_mode: Option<ParseMode>

Cleaner mode when clean.

tag_truncate: Option<bool>

Replace inline images with alt text when clean (tri-state).

extract_links: Option<bool>

Include a deduplicated list of outbound links.

include_tags: Option<Vec<String>>

Whitelist of tags to keep when clean.

exclude_tags: Option<Vec<String>>

Blacklist of tags to drop when clean.

headers: Option<HashMap<String, String>>

Custom request headers; providing any disables URL caching.

max_age: Option<u64>

URL-cache opt-in in seconds (tri-state; omitted = fetch fresh).

stealth: Option<bool>

Browser-based stealth fetch. +2 credits.

Implementations

impl ScrapeRequest

pub fn new(website_url: impl Into<String>) -> Self

Start a request for website_url.

pub fn clean(self, clean: bool) -> Self

Convert HTML to cleaned markdown.

pub fn parse_mode(self, mode: ParseMode) -> Self

Set the cleaner mode.

pub fn tag_truncate(self, tag_truncate: bool) -> Self

Replace inline images with alt text when cleaning.

pub fn extract_links(self, extract_links: bool) -> Self

Include a deduplicated list of outbound links.

pub fn include_tags<I, S>(self, tags: I) -> Self

where I: IntoIterator<Item = S>, S: Into<String>,

Set the tag whitelist.

pub fn exclude_tags<I, S>(self, tags: I) -> Self

where I: IntoIterator<Item = S>, S: Into<String>,

Set the tag blacklist.

pub fn header(self, key: impl Into<String>, value: impl Into<String>) -> Self

Add a single custom request header.

pub fn headers(self, headers: HashMap<String, String>) -> Self

Replace all custom request headers.

pub fn max_age(self, seconds: u64) -> Self

Opt into URL caching: accept cache entries fresher than seconds.

pub fn stealth(self, stealth: bool) -> Self

Use browser-based stealth fetch (+2 credits).

Trait Implementations

impl Clone for ScrapeRequest

fn clone(&self) -> ScrapeRequest

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for ScrapeRequest

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for ScrapeRequest

fn default() -> ScrapeRequest

Returns the “default value” for a type.

impl Serialize for ScrapeRequest

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.