webscrape_ai/
request.rs

1//! Request models — builder-pattern, with every unset option omitted from the
2//! serialized JSON.
3
4use std::collections::HashMap;
5
6use serde::Serialize;
7use serde_json::Value;
8
9/// Cleaner mode used when `clean: true`.
10#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
11#[serde(rename_all = "lowercase")]
12pub enum ParseMode {
13    /// More forgiving on malformed pages (default).
14    Accurate,
15    /// Faster, less forgiving.
16    Speed,
17}
18
19/// Page-complexity hint for [`SmartScraperRequest`].
20#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
21#[serde(rename_all = "lowercase")]
22pub enum PageComplexity {
23    /// Faster and cheaper (default).
24    Low,
25    /// For visually busy pages or deeply nested schemas.
26    High,
27}
28
29/// How exhaustively [`SmartScraperRequest`] should populate the result.
30#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
31#[serde(rename_all = "lowercase")]
32pub enum DetailLevel {
33    /// Minimal.
34    Low,
35    /// Balanced (default).
36    Medium,
37    /// Exhaustive.
38    High,
39}
40
41/// Request body for [`Client::scrape`](crate::Client::scrape).
42///
43/// Build with [`ScrapeRequest::new`] and chain the option setters; only fields
44/// you touch are serialized.
45///
46/// ```
47/// use webscrape_ai::ScrapeRequest;
48/// let req = ScrapeRequest::new("https://example.com").clean(true).extract_links(true);
49/// let json = serde_json::to_value(&req).unwrap();
50/// assert_eq!(json["website_url"], "https://example.com");
51/// assert_eq!(json["clean"], true);
52/// assert!(json.get("tag_truncate").is_none()); // untouched -> absent
53/// ```
54#[derive(Debug, Clone, Default, Serialize)]
55pub struct ScrapeRequest {
56    /// The URL to fetch. Required.
57    pub website_url: String,
58    /// Convert HTML to cleaned markdown.
59    #[serde(skip_serializing_if = "Option::is_none")]
60    pub clean: Option<bool>,
61    /// Cleaner mode when `clean`.
62    #[serde(skip_serializing_if = "Option::is_none")]
63    pub parse_mode: Option<ParseMode>,
64    /// Replace inline images with alt text when `clean` (tri-state).
65    #[serde(skip_serializing_if = "Option::is_none")]
66    pub tag_truncate: Option<bool>,
67    /// Include a deduplicated list of outbound links.
68    #[serde(skip_serializing_if = "Option::is_none")]
69    pub extract_links: Option<bool>,
70    /// Whitelist of tags to keep when `clean`.
71    #[serde(skip_serializing_if = "Option::is_none")]
72    pub include_tags: Option<Vec<String>>,
73    /// Blacklist of tags to drop when `clean`.
74    #[serde(skip_serializing_if = "Option::is_none")]
75    pub exclude_tags: Option<Vec<String>>,
76    /// Custom request headers; providing any disables URL caching.
77    #[serde(skip_serializing_if = "Option::is_none")]
78    pub headers: Option<HashMap<String, String>>,
79    /// URL-cache opt-in in seconds (tri-state; omitted = fetch fresh).
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub max_age: Option<u64>,
82    /// Browser-based stealth fetch. +2 credits.
83    #[serde(skip_serializing_if = "Option::is_none")]
84    pub stealth: Option<bool>,
85}
86
87impl ScrapeRequest {
88    /// Start a request for `website_url`.
89    pub fn new(website_url: impl Into<String>) -> Self {
90        Self {
91            website_url: website_url.into(),
92            ..Default::default()
93        }
94    }
95
96    /// Convert HTML to cleaned markdown.
97    pub fn clean(mut self, clean: bool) -> Self {
98        self.clean = Some(clean);
99        self
100    }
101
102    /// Set the cleaner mode.
103    pub fn parse_mode(mut self, mode: ParseMode) -> Self {
104        self.parse_mode = Some(mode);
105        self
106    }
107
108    /// Replace inline images with alt text when cleaning.
109    pub fn tag_truncate(mut self, tag_truncate: bool) -> Self {
110        self.tag_truncate = Some(tag_truncate);
111        self
112    }
113
114    /// Include a deduplicated list of outbound links.
115    pub fn extract_links(mut self, extract_links: bool) -> Self {
116        self.extract_links = Some(extract_links);
117        self
118    }
119
120    /// Set the tag whitelist.
121    pub fn include_tags<I, S>(mut self, tags: I) -> Self
122    where
123        I: IntoIterator<Item = S>,
124        S: Into<String>,
125    {
126        self.include_tags = Some(tags.into_iter().map(Into::into).collect());
127        self
128    }
129
130    /// Set the tag blacklist.
131    pub fn exclude_tags<I, S>(mut self, tags: I) -> Self
132    where
133        I: IntoIterator<Item = S>,
134        S: Into<String>,
135    {
136        self.exclude_tags = Some(tags.into_iter().map(Into::into).collect());
137        self
138    }
139
140    /// Add a single custom request header.
141    pub fn header(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
142        self.headers
143            .get_or_insert_with(HashMap::new)
144            .insert(key.into(), value.into());
145        self
146    }
147
148    /// Replace all custom request headers.
149    pub fn headers(mut self, headers: HashMap<String, String>) -> Self {
150        self.headers = Some(headers);
151        self
152    }
153
154    /// Opt into URL caching: accept cache entries fresher than `seconds`.
155    pub fn max_age(mut self, seconds: u64) -> Self {
156        self.max_age = Some(seconds);
157        self
158    }
159
160    /// Use browser-based stealth fetch (+2 credits).
161    pub fn stealth(mut self, stealth: bool) -> Self {
162        self.stealth = Some(stealth);
163        self
164    }
165}
166
167/// Request body for [`Client::smartscraper`](crate::Client::smartscraper).
168///
169/// Build with [`SmartScraperRequest::new`]; only fields you touch are serialized.
170#[derive(Debug, Clone, Default, Serialize)]
171pub struct SmartScraperRequest {
172    /// The URL to extract from. Required.
173    pub website_url: String,
174    /// Plain-English description of what to extract. Required.
175    /// (Wire field name is `user_prompt`, not `prompt`.)
176    pub user_prompt: String,
177    /// JSON Schema the result is validated against (one repair attempt).
178    #[serde(skip_serializing_if = "Option::is_none")]
179    pub output_schema: Option<Value>,
180    /// Page-complexity hint.
181    #[serde(skip_serializing_if = "Option::is_none")]
182    pub page_complexity: Option<PageComplexity>,
183    /// How exhaustively to populate the result.
184    #[serde(skip_serializing_if = "Option::is_none")]
185    pub detail_level: Option<DetailLevel>,
186    /// Cleaner mode.
187    #[serde(skip_serializing_if = "Option::is_none")]
188    pub parse_mode: Option<ParseMode>,
189    /// Return raw text under `result` (bypasses schema validation).
190    #[serde(skip_serializing_if = "Option::is_none")]
191    pub plain_text: Option<bool>,
192    /// Whitelist of tags to keep before extraction.
193    #[serde(skip_serializing_if = "Option::is_none")]
194    pub include_tags: Option<Vec<String>>,
195    /// Blacklist of tags to drop before extraction.
196    #[serde(skip_serializing_if = "Option::is_none")]
197    pub exclude_tags: Option<Vec<String>>,
198    /// Trim long content before extraction (tri-state; server default when unset).
199    #[serde(skip_serializing_if = "Option::is_none")]
200    pub reduce_content: Option<bool>,
201    /// Opt in to an alternate extraction path that can do better on
202    /// hard-to-parse pages. Behavior may change without notice.
203    #[serde(skip_serializing_if = "Option::is_none")]
204    pub experimental: Option<bool>,
205    /// Custom request headers forwarded to the fetcher. Providing headers
206    /// disables URL caching for this request.
207    #[serde(skip_serializing_if = "Option::is_none")]
208    pub headers: Option<HashMap<String, String>>,
209    /// URL-cache opt-in in seconds (same semantics as `/scrape`).
210    #[serde(skip_serializing_if = "Option::is_none")]
211    pub max_age: Option<u64>,
212    /// Browser-based stealth fetch. +5 credits.
213    #[serde(skip_serializing_if = "Option::is_none")]
214    pub stealth: Option<bool>,
215}
216
217impl SmartScraperRequest {
218    /// Start a request for `website_url` with the given `user_prompt`.
219    pub fn new(website_url: impl Into<String>, user_prompt: impl Into<String>) -> Self {
220        Self {
221            website_url: website_url.into(),
222            user_prompt: user_prompt.into(),
223            ..Default::default()
224        }
225    }
226
227    /// Set the JSON Schema for the output.
228    pub fn output_schema(mut self, schema: Value) -> Self {
229        self.output_schema = Some(schema);
230        self
231    }
232
233    /// Set the page-complexity hint.
234    pub fn page_complexity(mut self, complexity: PageComplexity) -> Self {
235        self.page_complexity = Some(complexity);
236        self
237    }
238
239    /// Set the detail level.
240    pub fn detail_level(mut self, level: DetailLevel) -> Self {
241        self.detail_level = Some(level);
242        self
243    }
244
245    /// Set the cleaner mode.
246    pub fn parse_mode(mut self, mode: ParseMode) -> Self {
247        self.parse_mode = Some(mode);
248        self
249    }
250
251    /// Return raw text instead of parsed JSON (bypasses schema validation).
252    pub fn plain_text(mut self, plain_text: bool) -> Self {
253        self.plain_text = Some(plain_text);
254        self
255    }
256
257    /// Set the tag whitelist.
258    pub fn include_tags<I, S>(mut self, tags: I) -> Self
259    where
260        I: IntoIterator<Item = S>,
261        S: Into<String>,
262    {
263        self.include_tags = Some(tags.into_iter().map(Into::into).collect());
264        self
265    }
266
267    /// Set the tag blacklist.
268    pub fn exclude_tags<I, S>(mut self, tags: I) -> Self
269    where
270        I: IntoIterator<Item = S>,
271        S: Into<String>,
272    {
273        self.exclude_tags = Some(tags.into_iter().map(Into::into).collect());
274        self
275    }
276
277    /// Trim long content before extraction.
278    pub fn reduce_content(mut self, reduce_content: bool) -> Self {
279        self.reduce_content = Some(reduce_content);
280        self
281    }
282
283    /// Opt in to an alternate extraction path that can do better on
284    /// hard-to-parse pages. Behavior may change without notice.
285    pub fn experimental(mut self, experimental: bool) -> Self {
286        self.experimental = Some(experimental);
287        self
288    }
289
290    /// Add a single custom request header.
291    pub fn header(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
292        self.headers
293            .get_or_insert_with(HashMap::new)
294            .insert(key.into(), value.into());
295        self
296    }
297
298    /// Replace all custom request headers.
299    pub fn headers(mut self, headers: HashMap<String, String>) -> Self {
300        self.headers = Some(headers);
301        self
302    }
303
304    /// Opt into URL caching: accept cache entries fresher than `seconds`.
305    pub fn max_age(mut self, seconds: u64) -> Self {
306        self.max_age = Some(seconds);
307        self
308    }
309
310    /// Use browser-based stealth fetch (+5 credits).
311    pub fn stealth(mut self, stealth: bool) -> Self {
312        self.stealth = Some(stealth);
313        self
314    }
315}
webscrape_ai/request.rs

webscrape_ai/
request.rs
