webscrape_ai/
blocking.rs

1//! Synchronous client, enabled by the `blocking` cargo feature. Mirrors the
2//! async [`Client`](crate::Client) surface, wrapping `reqwest::blocking` and
3//! using `std::thread::sleep` for backoff and the wait helper.
4
5use std::sync::Arc;
6use std::time::{Duration, Instant};
7
8use reqwest::Method;
9use serde::de::DeserializeOwned;
10use serde_json::Value;
11
12use crate::client::resolve_api_key;
13use crate::error::{Error, Result};
14use crate::request::{ScrapeRequest, SmartScraperRequest};
15use crate::response::{
16    Response, RunStatus, ScrapeData, SmartBrowseDispatch, SmartBrowseRun, SmartBrowseUsage,
17    SmartScraperData, WaitOptions,
18};
19use crate::transport;
20use crate::{user_agent, DEFAULT_BASE_URL};
21
22const DEFAULT_TIMEOUT: Duration = Duration::from_secs(180);
23const DEFAULT_MAX_RETRIES: u32 = 2;
24const MAX_POLL_INTERVAL: Duration = Duration::from_secs(10);
25
26struct Inner {
27    http: reqwest::blocking::Client,
28    base_url: String,
29    api_key: String,
30    user_agent: String,
31    timeout: Duration,
32    max_retries: u32,
33}
34
35/// Synchronous webscrape.ai API client. Cheap to [`Clone`].
36#[derive(Clone)]
37pub struct Client(Arc<Inner>);
38
39impl Client {
40    /// Construct a client with an explicit API key and default config.
41    pub fn new(api_key: impl Into<String>) -> Result<Self> {
42        Self::builder().api_key(api_key).build()
43    }
44
45    /// Construct a client reading `WEBSCRAPE_API_KEY` from the environment.
46    pub fn from_env() -> Result<Self> {
47        Self::builder().build()
48    }
49
50    /// Start a [`ClientBuilder`].
51    pub fn builder() -> ClientBuilder {
52        ClientBuilder::default()
53    }
54
55    /// Fetch a URL as HTML, cleaned markdown, or links.
56    pub fn scrape(&self, request: ScrapeRequest) -> Result<Response<ScrapeData>> {
57        let body = to_value(&request)?;
58        self.0
59            .execute(Method::POST, "/scrape".to_string(), Some(body))
60    }
61
62    /// LLM structured extraction returning JSON.
63    pub fn smartscraper(&self, request: SmartScraperRequest) -> Result<Response<SmartScraperData>> {
64        let body = to_value(&request)?;
65        self.0
66            .execute(Method::POST, "/smartscraper".to_string(), Some(body))
67    }
68
69    /// Access the SmartBrowse namespace.
70    pub fn smartbrowse(&self) -> SmartBrowse<'_> {
71        SmartBrowse { client: self }
72    }
73}
74
75impl std::fmt::Debug for Client {
76    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
77        f.debug_struct("blocking::Client")
78            .field("base_url", &self.0.base_url)
79            .field("timeout", &self.0.timeout)
80            .field("max_retries", &self.0.max_retries)
81            .finish_non_exhaustive()
82    }
83}
84
85impl Inner {
86    fn execute<T: DeserializeOwned>(
87        &self,
88        method: Method,
89        path: String,
90        body: Option<Value>,
91    ) -> Result<Response<T>> {
92        let url = format!("{}{}", self.base_url, path);
93        let mut attempt = 0u32;
94
95        loop {
96            let mut rb = self
97                .http
98                .request(method.clone(), &url)
99                .header("X-API-Key", &self.api_key)
100                .header(reqwest::header::USER_AGENT, &self.user_agent)
101                .header(reqwest::header::ACCEPT, "application/json")
102                .timeout(self.timeout);
103            if let Some(ref b) = body {
104                rb = rb.json(b);
105            }
106
107            match rb.send() {
108                Ok(resp) => {
109                    let status = resp.status().as_u16();
110                    let request_id_header = transport::header_str(resp.headers(), "x-request-id");
111                    let retry_after = transport::parse_retry_after(
112                        transport::header_str(resp.headers(), "retry-after").as_deref(),
113                    );
114                    let bytes = resp.bytes().map_err(Error::Transport)?;
115
116                    if transport::is_retryable_status(status) && attempt < self.max_retries {
117                        attempt += 1;
118                        std::thread::sleep(transport::backoff_delay(attempt, retry_after));
119                        continue;
120                    }
121                    return transport::process_response::<T>(
122                        status,
123                        request_id_header,
124                        retry_after,
125                        &bytes,
126                    );
127                }
128                Err(e) => {
129                    if e.is_timeout() {
130                        return Err(Error::Timeout(self.timeout));
131                    }
132                    if transport::is_retryable_send_error(&e) && attempt < self.max_retries {
133                        attempt += 1;
134                        std::thread::sleep(transport::backoff_delay(attempt, None));
135                        continue;
136                    }
137                    return Err(Error::Transport(e));
138                }
139            }
140        }
141    }
142}
143
144/// The SmartBrowse method namespace, obtained via [`Client::smartbrowse`].
145pub struct SmartBrowse<'a> {
146    client: &'a Client,
147}
148
149impl SmartBrowse<'_> {
150    /// Dispatch a recipe replay run.
151    pub fn run(&self, recipe_id: &str) -> Result<Response<SmartBrowseDispatch>> {
152        let path = format!("/smartbrowse/recipes/{recipe_id}/run");
153        self.client.0.execute(Method::POST, path, None)
154    }
155
156    /// Poll a run's current state.
157    pub fn get_run(&self, run_id: &str) -> Result<Response<SmartBrowseRun>> {
158        let path = format!("/smartbrowse/runs/{run_id}");
159        self.client.0.execute(Method::GET, path, None)
160    }
161
162    /// Plan caps + rolling-30-day usage.
163    pub fn usage(&self) -> Result<Response<SmartBrowseUsage>> {
164        self.client
165            .0
166            .execute(Method::GET, "/smartbrowse/usage".to_string(), None)
167    }
168
169    /// Poll `run_id` until it reaches a terminal state (see the async
170    /// [`wait_for_run`](crate::SmartBrowse::wait_for_run) for semantics).
171    pub fn wait_for_run(
172        &self,
173        run_id: &str,
174        opts: WaitOptions,
175    ) -> Result<Response<SmartBrowseRun>> {
176        let start = Instant::now();
177        let mut interval = opts.poll_interval;
178
179        loop {
180            let resp = self.get_run(run_id)?;
181            match resp.data.run_status {
182                RunStatus::Completed => return Ok(resp),
183                RunStatus::Failed | RunStatus::Cancelled => {
184                    return Err(Error::RunFailed { run: resp.data })
185                }
186                _ => {}
187            }
188
189            if start.elapsed() >= opts.timeout {
190                return Err(Error::WaitTimeout { last: resp.data });
191            }
192            let remaining = opts.timeout.saturating_sub(start.elapsed());
193            std::thread::sleep(interval.min(remaining));
194            interval = interval.mul_f64(1.5).min(MAX_POLL_INTERVAL);
195        }
196    }
197
198    /// [`run`](Self::run) followed by [`wait_for_run`](Self::wait_for_run).
199    pub fn run_and_wait(
200        &self,
201        recipe_id: &str,
202        opts: WaitOptions,
203    ) -> Result<Response<SmartBrowseRun>> {
204        let dispatch = self.run(recipe_id)?;
205        self.wait_for_run(&dispatch.data.run_id, opts)
206    }
207}
208
209fn to_value<T: serde::Serialize>(request: &T) -> Result<Value> {
210    serde_json::to_value(request)
211        .map_err(|e| Error::Decode(format!("failed to serialize request: {e}")))
212}
213
214/// Builder for the blocking [`Client`].
215#[derive(Default)]
216pub struct ClientBuilder {
217    api_key: Option<String>,
218    base_url: Option<String>,
219    timeout: Option<Duration>,
220    max_retries: Option<u32>,
221    http_client: Option<reqwest::blocking::Client>,
222}
223
224impl ClientBuilder {
225    /// Set the API key explicitly.
226    pub fn api_key(mut self, api_key: impl Into<String>) -> Self {
227        self.api_key = Some(api_key.into());
228        self
229    }
230
231    /// Override the base URL.
232    pub fn base_url(mut self, base_url: impl Into<String>) -> Self {
233        self.base_url = Some(base_url.into());
234        self
235    }
236
237    /// Set the per-request timeout (default 180s).
238    pub fn timeout(mut self, timeout: Duration) -> Self {
239        self.timeout = Some(timeout);
240        self
241    }
242
243    /// Set the maximum number of retries (default 2; 0 disables).
244    pub fn max_retries(mut self, max_retries: u32) -> Self {
245        self.max_retries = Some(max_retries);
246        self
247    }
248
249    /// Supply a preconfigured [`reqwest::blocking::Client`].
250    pub fn http_client(mut self, http_client: reqwest::blocking::Client) -> Self {
251        self.http_client = Some(http_client);
252        self
253    }
254
255    /// Build the blocking [`Client`].
256    pub fn build(self) -> Result<Client> {
257        let api_key = resolve_api_key(self.api_key)?;
258        let base_url = self
259            .base_url
260            .unwrap_or_else(|| DEFAULT_BASE_URL.to_string())
261            .trim_end_matches('/')
262            .to_string();
263        let timeout = self.timeout.unwrap_or(DEFAULT_TIMEOUT);
264        let max_retries = self.max_retries.unwrap_or(DEFAULT_MAX_RETRIES);
265        let http = match self.http_client {
266            Some(c) => c,
267            None => reqwest::blocking::Client::builder()
268                .build()
269                .map_err(|e| Error::Config(format!("failed to build HTTP client: {e}")))?,
270        };
271
272        Ok(Client(Arc::new(Inner {
273            http,
274            base_url,
275            api_key,
276            user_agent: user_agent(),
277            timeout,
278            max_retries,
279        })))
280    }
281}
webscrape_ai/blocking.rs

webscrape_ai/
blocking.rs
