webscrape_ai/
lib.rs

1//! Official Rust SDK for the [webscrape.ai](https://webscrape.ai) API.
2//!
3//! Covers the public API-key surface: [`Client::scrape`], [`Client::smartscraper`],
4//! and the [`SmartBrowse`] namespace (dispatch a recipe run, poll it, wait for
5//! completion, read usage). See <https://webscrape.ai/docs> for API docs.
6//!
7//! # Quick start
8//!
9//! ```no_run
10//! # async fn run() -> Result<(), webscrape_ai::Error> {
11//! use webscrape_ai::{Client, ScrapeRequest};
12//!
13//! // Reads WEBSCRAPE_API_KEY from the environment.
14//! let client = Client::from_env()?;
15//!
16//! let resp = client
17//!     .scrape(ScrapeRequest::new("https://example.com").clean(true))
18//!     .await?;
19//!
20//! println!("credits used: {:?}", resp.credits_used);
21//! println!("markdown: {:?}", resp.data.html);
22//! # Ok(()) }
23//! ```
24//!
25//! # Structured extraction
26//!
27//! ```no_run
28//! # async fn run() -> Result<(), webscrape_ai::Error> {
29//! use webscrape_ai::{Client, SmartScraperRequest};
30//! use serde_json::json;
31//!
32//! let client = Client::new("wsg_live_...")?;
33//! let resp = client
34//!     .smartscraper(
35//!         SmartScraperRequest::new(
36//!             "https://news.ycombinator.com",
37//!             "Extract the front-page stories with title, url, and score.",
38//!         )
39//!         .output_schema(json!({
40//!             "type": "object",
41//!             "properties": { "stories": { "type": "array" } }
42//!         })),
43//!     )
44//!     .await?;
45//!
46//! // `data.result` is a serde_json::Value; decode it into your own type.
47//! let value = resp.data.result;
48//! println!("{value:?}");
49//! # Ok(()) }
50//! ```
51//!
52//! # Error handling
53//!
54//! Every method returns [`Result<T>`](crate::Result). Failures surface as
55//! [`Error`], which distinguishes API errors ([`Error::Api`] carrying an
56//! [`ApiError`] you branch on via [`ApiError::kind`]) from transport, timeout,
57//! and wait-helper failures.
58
59#![forbid(unsafe_code)]
60// `Error` intentionally carries rich payloads (`ApiError`, and a full
61// `SmartBrowseRun` in `RunFailed`/`WaitTimeout`) so callers can inspect the
62// terminal run without a second request. That makes the enum large; boxing would
63// change the public contract, so we accept the size rather than deviate.
64#![allow(clippy::result_large_err)]
65
66mod client;
67mod error;
68mod request;
69mod response;
70mod transport;
71
72#[cfg(feature = "blocking")]
73pub mod blocking;
74
75pub use client::{Client, ClientBuilder, SmartBrowse};
76pub use error::{ApiError, Error, ErrorCode, Result};
77pub use request::{DetailLevel, PageComplexity, ParseMode, ScrapeRequest, SmartScraperRequest};
78pub use response::{
79    LinkInfo, PageMetadata, Response, RunStatus, ScrapeData, SmartBrowseDispatch,
80    SmartBrowseLastRun, SmartBrowseRun, SmartBrowseUsage, SmartScraperData, WaitOptions,
81};
82
83/// Default production base URL (`https://api.webscrape.ai/v1`).
84pub const DEFAULT_BASE_URL: &str = "https://api.webscrape.ai/v1";
85
86/// Environment variable consulted for the API key when none is passed explicitly.
87pub const API_KEY_ENV: &str = "WEBSCRAPE_API_KEY";
88
89/// SDK version, sourced from `CARGO_PKG_VERSION` and used in the `User-Agent`.
90pub const VERSION: &str = env!("CARGO_PKG_VERSION");
91
92/// `User-Agent` sent on every request, e.g. `webscrape-ai-rust/0.1.0`.
93pub(crate) fn user_agent() -> String {
94    format!("webscrape-ai-rust/{VERSION}")
95}
webscrape_ai/lib.rs

webscrape_ai/
lib.rs
