webscrape_ai/
client.rs

1//! Async-first client.
2
3use std::sync::Arc;
4use std::time::{Duration, Instant};
5
6use reqwest::Method;
7use serde::de::DeserializeOwned;
8use serde_json::Value;
9
10use crate::error::{Error, Result};
11use crate::request::{ScrapeRequest, SmartScraperRequest};
12use crate::response::{
13    Response, RunStatus, ScrapeData, SmartBrowseDispatch, SmartBrowseRun, SmartBrowseUsage,
14    SmartScraperData, WaitOptions,
15};
16use crate::transport;
17use crate::{user_agent, DEFAULT_BASE_URL};
18
19const DEFAULT_TIMEOUT: Duration = Duration::from_secs(180);
20const DEFAULT_MAX_RETRIES: u32 = 2;
21const MAX_POLL_INTERVAL: Duration = Duration::from_secs(10);
22
23struct Inner {
24    http: reqwest::Client,
25    base_url: String,
26    api_key: String,
27    user_agent: String,
28    timeout: Duration,
29    max_retries: u32,
30}
31
32/// Async webscrape.ai API client. Cheap to [`Clone`] (reference-counted).
33#[derive(Clone)]
34pub struct Client(Arc<Inner>);
35
36impl Client {
37    /// Construct a client with an explicit API key and otherwise default config.
38    ///
39    /// Returns [`Error::Config`] if `api_key` is empty.
40    pub fn new(api_key: impl Into<String>) -> Result<Self> {
41        Self::builder().api_key(api_key).build()
42    }
43
44    /// Construct a client reading the API key from the `WEBSCRAPE_API_KEY`
45    /// environment variable.
46    ///
47    /// Returns [`Error::Config`] if the variable is unset or empty.
48    pub fn from_env() -> Result<Self> {
49        Self::builder().build()
50    }
51
52    /// Start a [`ClientBuilder`] for custom configuration.
53    pub fn builder() -> ClientBuilder {
54        ClientBuilder::default()
55    }
56
57    /// Fetch a URL as HTML, cleaned markdown, or links. Costs 1 credit (+2 with
58    /// stealth).
59    pub async fn scrape(&self, request: ScrapeRequest) -> Result<Response<ScrapeData>> {
60        let body = to_value(&request)?;
61        self.0
62            .execute(Method::POST, "/scrape".to_string(), Some(body))
63            .await
64    }
65
66    /// LLM structured extraction returning JSON. Costs 5 credits (+5 with
67    /// stealth).
68    pub async fn smartscraper(
69        &self,
70        request: SmartScraperRequest,
71    ) -> Result<Response<SmartScraperData>> {
72        let body = to_value(&request)?;
73        self.0
74            .execute(Method::POST, "/smartscraper".to_string(), Some(body))
75            .await
76    }
77
78    /// Access the SmartBrowse namespace (run recipes, poll runs, usage).
79    pub fn smartbrowse(&self) -> SmartBrowse<'_> {
80        SmartBrowse { client: self }
81    }
82}
83
84impl std::fmt::Debug for Client {
85    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
86        f.debug_struct("Client")
87            .field("base_url", &self.0.base_url)
88            .field("timeout", &self.0.timeout)
89            .field("max_retries", &self.0.max_retries)
90            .finish_non_exhaustive()
91    }
92}
93
94impl Inner {
95    async fn execute<T: DeserializeOwned>(
96        &self,
97        method: Method,
98        path: String,
99        body: Option<Value>,
100    ) -> Result<Response<T>> {
101        let url = format!("{}{}", self.base_url, path);
102        let mut attempt = 0u32;
103
104        loop {
105            let mut rb = self
106                .http
107                .request(method.clone(), &url)
108                .header("X-API-Key", &self.api_key)
109                .header(reqwest::header::USER_AGENT, &self.user_agent)
110                .header(reqwest::header::ACCEPT, "application/json")
111                .timeout(self.timeout);
112            if let Some(ref b) = body {
113                rb = rb.json(b);
114            }
115
116            match rb.send().await {
117                Ok(resp) => {
118                    let status = resp.status().as_u16();
119                    let request_id_header = transport::header_str(resp.headers(), "x-request-id");
120                    let retry_after = transport::parse_retry_after(
121                        transport::header_str(resp.headers(), "retry-after").as_deref(),
122                    );
123                    // Body-read failures are NOT retried: the request may have
124                    // completed server-side, and a completed request must not be
125                    // replayed.
126                    let bytes = resp.bytes().await.map_err(Error::Transport)?;
127
128                    if transport::is_retryable_status(status) && attempt < self.max_retries {
129                        attempt += 1;
130                        tokio::time::sleep(transport::backoff_delay(attempt, retry_after)).await;
131                        continue;
132                    }
133                    return transport::process_response::<T>(
134                        status,
135                        request_id_header,
136                        retry_after,
137                        &bytes,
138                    );
139                }
140                Err(e) => {
141                    if e.is_timeout() {
142                        return Err(Error::Timeout(self.timeout));
143                    }
144                    if transport::is_retryable_send_error(&e) && attempt < self.max_retries {
145                        attempt += 1;
146                        tokio::time::sleep(transport::backoff_delay(attempt, None)).await;
147                        continue;
148                    }
149                    return Err(Error::Transport(e));
150                }
151            }
152        }
153    }
154}
155
156/// The SmartBrowse method namespace, obtained via [`Client::smartbrowse`].
157pub struct SmartBrowse<'a> {
158    client: &'a Client,
159}
160
161impl SmartBrowse<'_> {
162    /// Dispatch a recipe replay run (async). Returns immediately with a run id
163    /// to poll. Costs 2 credits per page, billed on completion.
164    pub async fn run(&self, recipe_id: &str) -> Result<Response<SmartBrowseDispatch>> {
165        let path = format!("/smartbrowse/recipes/{recipe_id}/run");
166        self.client.0.execute(Method::POST, path, None).await
167    }
168
169    /// Poll a run's current state (free).
170    pub async fn get_run(&self, run_id: &str) -> Result<Response<SmartBrowseRun>> {
171        let path = format!("/smartbrowse/runs/{run_id}");
172        self.client.0.execute(Method::GET, path, None).await
173    }
174
175    /// Plan caps + rolling-30-day usage (free).
176    pub async fn usage(&self) -> Result<Response<SmartBrowseUsage>> {
177        self.client
178            .0
179            .execute(Method::GET, "/smartbrowse/usage".to_string(), None)
180            .await
181    }
182
183    /// Poll `run_id` until it reaches a terminal state.
184    ///
185    /// - `completed` → returns the run response.
186    /// - `failed` / `cancelled` → [`Error::RunFailed`] carrying the run.
187    /// - deadline exceeded → [`Error::WaitTimeout`] carrying the last-seen run.
188    pub async fn wait_for_run(
189        &self,
190        run_id: &str,
191        opts: WaitOptions,
192    ) -> Result<Response<SmartBrowseRun>> {
193        let start = Instant::now();
194        let mut interval = opts.poll_interval;
195
196        loop {
197            let resp = self.get_run(run_id).await?;
198            match resp.data.run_status {
199                RunStatus::Completed => return Ok(resp),
200                RunStatus::Failed | RunStatus::Cancelled => {
201                    return Err(Error::RunFailed { run: resp.data })
202                }
203                _ => {}
204            }
205
206            if start.elapsed() >= opts.timeout {
207                return Err(Error::WaitTimeout { last: resp.data });
208            }
209            let remaining = opts.timeout.saturating_sub(start.elapsed());
210            tokio::time::sleep(interval.min(remaining)).await;
211            interval = interval.mul_f64(1.5).min(MAX_POLL_INTERVAL);
212        }
213    }
214
215    /// [`run`](Self::run) followed by [`wait_for_run`](Self::wait_for_run).
216    pub async fn run_and_wait(
217        &self,
218        recipe_id: &str,
219        opts: WaitOptions,
220    ) -> Result<Response<SmartBrowseRun>> {
221        let dispatch = self.run(recipe_id).await?;
222        self.wait_for_run(&dispatch.data.run_id, opts).await
223    }
224}
225
226fn to_value<T: serde::Serialize>(request: &T) -> Result<Value> {
227    serde_json::to_value(request)
228        .map_err(|e| Error::Decode(format!("failed to serialize request: {e}")))
229}
230
231/// Builder for [`Client`].
232#[derive(Default)]
233pub struct ClientBuilder {
234    api_key: Option<String>,
235    base_url: Option<String>,
236    timeout: Option<Duration>,
237    max_retries: Option<u32>,
238    http_client: Option<reqwest::Client>,
239}
240
241impl ClientBuilder {
242    /// Set the API key explicitly. Overrides the `WEBSCRAPE_API_KEY` env fallback.
243    pub fn api_key(mut self, api_key: impl Into<String>) -> Self {
244        self.api_key = Some(api_key.into());
245        self
246    }
247
248    /// Override the base URL (default `https://api.webscrape.ai/v1`). A trailing
249    /// slash is accepted and normalized away.
250    pub fn base_url(mut self, base_url: impl Into<String>) -> Self {
251        self.base_url = Some(base_url.into());
252        self
253    }
254
255    /// Set the per-request timeout (default 180s).
256    pub fn timeout(mut self, timeout: Duration) -> Self {
257        self.timeout = Some(timeout);
258        self
259    }
260
261    /// Set the maximum number of retries (default 2; 0 disables).
262    pub fn max_retries(mut self, max_retries: u32) -> Self {
263        self.max_retries = Some(max_retries);
264        self
265    }
266
267    /// Supply a preconfigured [`reqwest::Client`] (proxies, custom TLS, etc.).
268    pub fn http_client(mut self, http_client: reqwest::Client) -> Self {
269        self.http_client = Some(http_client);
270        self
271    }
272
273    /// Build the [`Client`]. Resolves the API key from [`api_key`](Self::api_key)
274    /// or `WEBSCRAPE_API_KEY`; returns [`Error::Config`] if neither is set.
275    pub fn build(self) -> Result<Client> {
276        let api_key = resolve_api_key(self.api_key)?;
277        let base_url = self
278            .base_url
279            .unwrap_or_else(|| DEFAULT_BASE_URL.to_string())
280            .trim_end_matches('/')
281            .to_string();
282        let timeout = self.timeout.unwrap_or(DEFAULT_TIMEOUT);
283        let max_retries = self.max_retries.unwrap_or(DEFAULT_MAX_RETRIES);
284        let http = match self.http_client {
285            Some(c) => c,
286            None => reqwest::Client::builder()
287                .build()
288                .map_err(|e| Error::Config(format!("failed to build HTTP client: {e}")))?,
289        };
290
291        Ok(Client(Arc::new(Inner {
292            http,
293            base_url,
294            api_key,
295            user_agent: user_agent(),
296            timeout,
297            max_retries,
298        })))
299    }
300}
301
302/// Resolve the API key from an explicit value or the environment, failing fast
303/// when neither is present.
304pub(crate) fn resolve_api_key(explicit: Option<String>) -> Result<String> {
305    let key = explicit
306        .filter(|k| !k.is_empty())
307        .or_else(|| std::env::var(crate::API_KEY_ENV).ok())
308        .filter(|k| !k.is_empty());
309    key.ok_or_else(|| {
310        Error::Config(format!(
311            "no API key: pass one to the builder or set {}",
312            crate::API_KEY_ENV
313        ))
314    })
315}
webscrape_ai/client.rs

webscrape_ai/
client.rs
