use std::fmt;
use std::sync::Arc;
use std::time::Duration;
use reqwest::header::{HeaderMap, HeaderValue, ACCEPT};
use reqwest::Url;
use serde_json::Value;
use crate::error::{api_error, Error, Result};
use crate::models::{CapturedPage, PageResult, ParsedPage, Step};
use crate::request::{executable, Call, Decode};
pub const DEFAULT_BASE_URL: &str = "https://api.scrapeunblocker.com";
pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(180);
pub const DEFAULT_MAX_RETRIES: u32 = 2;
pub const API_KEY_ENV: &str = "SCRAPEUNBLOCKER_KEY";
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
const API_KEY_HEADER: &str = "x-scrapeunblocker-key";
const RETRYABLE_STATUS: [u16; 4] = [429, 502, 503, 504];
#[derive(Clone)]
pub struct Client {
inner: Arc<Inner>,
}
struct Inner {
http: reqwest::Client,
base_url: String,
timeout: Duration,
max_retries: u32,
}
impl fmt::Debug for Client {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Client")
.field("base_url", &self.inner.base_url)
.field("timeout", &self.inner.timeout)
.field("max_retries", &self.inner.max_retries)
.finish_non_exhaustive()
}
}
#[derive(Debug, Clone)]
#[must_use]
pub struct ClientBuilder {
api_key: Option<String>,
base_url: String,
timeout: Duration,
max_retries: u32,
app: Option<String>,
}
impl Default for ClientBuilder {
fn default() -> Self {
ClientBuilder {
api_key: None,
base_url: DEFAULT_BASE_URL.to_owned(),
timeout: DEFAULT_TIMEOUT,
max_retries: DEFAULT_MAX_RETRIES,
app: None,
}
}
}
impl ClientBuilder {
pub fn api_key(mut self, api_key: impl Into<String>) -> Self {
self.api_key = Some(api_key.into());
self
}
pub fn base_url(mut self, base_url: impl Into<String>) -> Self {
self.base_url = base_url.into();
self
}
pub fn timeout(mut self, timeout: Duration) -> Self {
self.timeout = timeout;
self
}
pub fn max_retries(mut self, max_retries: u32) -> Self {
self.max_retries = max_retries;
self
}
pub fn user_agent(mut self, app: impl Into<String>) -> Self {
self.app = Some(app.into());
self
}
pub fn build(self) -> Result<Client> {
let api_key = self
.api_key
.or_else(|| std::env::var(API_KEY_ENV).ok())
.map(|k| k.trim().to_owned())
.filter(|k| !k.is_empty())
.ok_or(Error::MissingApiKey)?;
let mut key = HeaderValue::from_str(&api_key).map_err(|_| {
Error::Config("the API key holds characters an HTTP header cannot carry".into())
})?;
key.set_sensitive(true);
let base_url = self.base_url.trim_end_matches('/').to_owned();
Url::parse(&base_url).map_err(|e| Error::Config(format!("base URL {base_url:?}: {e}")))?;
let mut headers = HeaderMap::new();
headers.insert(API_KEY_HEADER, key);
headers.insert(ACCEPT, HeaderValue::from_static("*/*"));
let http = reqwest::Client::builder()
.default_headers(headers)
.user_agent(user_agent(self.app.as_deref()))
.timeout(self.timeout)
.build()
.map_err(|e| Error::Config(e.to_string()))?;
Ok(Client {
inner: Arc::new(Inner {
http,
base_url,
timeout: self.timeout,
max_retries: self.max_retries,
}),
})
}
}
fn user_agent(app: Option<&str>) -> String {
let own = format!("scrapeunblocker-rust/{VERSION}");
let app: String = app
.unwrap_or("")
.chars()
.map(|c| if c.is_control() { ' ' } else { c })
.collect();
let app = app.split_whitespace().collect::<Vec<_>>().join(" ");
if app.is_empty() {
own
} else {
format!("{own} {app}")
}
}
fn backoff(attempt: u32) -> Duration {
Duration::from_secs_f64((0.5 * 2f64.powi(attempt.min(10) as i32)).min(8.0))
}
impl Client {
pub fn new(api_key: impl Into<String>) -> Result<Self> {
ClientBuilder::default().api_key(api_key).build()
}
pub fn from_env() -> Result<Self> {
ClientBuilder::default().build()
}
pub fn builder() -> ClientBuilder {
ClientBuilder::default()
}
pub(crate) async fn execute<T: Decode>(&self, call: Call) -> Result<T> {
let mut url = Url::parse(&format!("{}{}", self.inner.base_url, call.path))
.map_err(|e| Error::Config(format!("endpoint {:?}: {e}", call.path)))?;
if !call.query.pairs().is_empty() {
url.query_pairs_mut().extend_pairs(call.query.pairs());
}
let mut attempt = 0;
loop {
let sent = self.inner.http.post(url.clone()).body("").send().await;
let response = match sent {
Ok(response) => response,
Err(e) if e.is_timeout() => return Err(Error::Timeout(e)),
Err(_) if attempt < self.inner.max_retries => {
tokio::time::sleep(backoff(attempt)).await;
attempt += 1;
continue;
}
Err(e) => return Err(Error::Connection(e)),
};
let status = response.status().as_u16();
if RETRYABLE_STATUS.contains(&status) && attempt < self.inner.max_retries {
drop(response);
tokio::time::sleep(backoff(attempt)).await;
attempt += 1;
continue;
}
let header = |name: &str| {
response
.headers()
.get(name)
.and_then(|v| v.to_str().ok())
.map(str::to_owned)
};
let origin_status = header("x-origin-status");
let destination_url = header("x-destination-url");
let success = response.status().is_success();
let body = response.bytes().await.map_err(|e| {
if e.is_timeout() {
Error::Timeout(e)
} else {
Error::Connection(e)
}
})?;
if success {
return T::decode(body.to_vec());
}
let text = String::from_utf8_lossy(&body).into_owned();
return Err(api_error(
status,
Some(text),
origin_status.as_deref(),
destination_url,
)
.into());
}
}
pub fn get_page_source(&self, url: impl Into<String>) -> GetPageSource {
GetPageSource::new(self.clone(), url.into())
}
pub fn list_elements(&self, url: impl Into<String>) -> ListElements {
ListElements::new(self.clone(), url.into())
}
pub fn get_markdown(&self, url: impl Into<String>) -> GetMarkdown {
GetMarkdown::new(self.clone(), url.into())
}
pub fn get_parsed(&self, url: impl Into<String>) -> GetParsed {
GetParsed::new(self.clone(), url.into())
}
pub fn get_page_with_cookies(&self, url: impl Into<String>) -> GetPageWithCookies {
GetPageWithCookies::new(self.clone(), url.into())
}
pub fn get_page_with_xhr(
&self,
url: impl Into<String>,
capture_xhr: impl Into<String>,
) -> GetPageWithXhr {
GetPageWithXhr::new(self.clone(), url.into(), capture_xhr.into())
}
pub fn serp(&self, keyword: impl Into<String>) -> Serp {
Serp::new(self.clone(), keyword.into())
}
pub fn get_image(&self, url: impl Into<String>) -> GetImage {
GetImage::new(self.clone(), url.into())
}
pub fn plugin(&self, path: impl Into<String>) -> Plugin {
Plugin::new(self.clone(), path.into())
}
}
macro_rules! page_setters {
() => {
pub fn proxy_country(mut self, country: impl Into<String>) -> Self {
self.call.query.set("proxy_country", country.into());
self
}
pub fn time_sleep(mut self, seconds: u32) -> Self {
self.call.query.set("time_sleep", seconds);
self
}
pub fn method(mut self, method: impl Into<String>) -> Self {
self.call.query.set("method", method.into());
self
}
pub fn value(mut self, value: impl Into<String>) -> Self {
self.call.query.set("value", value.into());
self
}
pub fn method_timeout(mut self, seconds: u32) -> Self {
self.call.query.set("method_timeout", seconds);
self
}
};
}
macro_rules! steps_setter {
() => {
pub fn steps(mut self, steps: impl IntoIterator<Item = Step>) -> Self {
let steps: Vec<Step> = steps.into_iter().collect();
let json = serde_json::to_string(&steps).expect("steps serialize to JSON");
self.call.query.set("steps", json);
self
}
};
}
#[derive(Debug, Clone)]
#[must_use = "a request does nothing until it is awaited or sent"]
pub struct GetPageSource<C = Client> {
client: C,
call: Call,
}
impl<C> GetPageSource<C> {
pub(crate) fn new(client: C, url: String) -> Self {
let mut call = Call::new("/getPageSource");
call.query.set("url", url);
GetPageSource { client, call }
}
page_setters!();
steps_setter!();
}
executable!(GetPageSource -> String);
#[derive(Debug, Clone)]
#[must_use = "a request does nothing until it is awaited or sent"]
pub struct ListElements<C = Client> {
client: C,
call: Call,
}
impl<C> ListElements<C> {
pub(crate) fn new(client: C, url: String) -> Self {
let mut call = Call::new("/getPageSource");
call.query.set("url", url);
call.query.set("list_elements", true);
ListElements { client, call }
}
page_setters!();
}
executable!(ListElements -> Value);
#[derive(Debug, Clone)]
#[must_use = "a request does nothing until it is awaited or sent"]
pub struct GetMarkdown<C = Client> {
client: C,
call: Call,
}
impl<C> GetMarkdown<C> {
pub(crate) fn new(client: C, url: String) -> Self {
let mut call = Call::new("/getPageSource");
call.query.set("url", url);
call.query.set("markdown", true);
GetMarkdown { client, call }
}
pub fn main_content_only(mut self, main_content_only: bool) -> Self {
self.call.query.set("main_content_only", main_content_only);
self
}
page_setters!();
steps_setter!();
}
executable!(GetMarkdown -> String);
#[derive(Debug, Clone)]
#[must_use = "a request does nothing until it is awaited or sent"]
pub struct GetParsed<C = Client> {
client: C,
call: Call,
}
impl<C> GetParsed<C> {
pub(crate) fn new(client: C, url: String) -> Self {
let mut call = Call::new("/getPageSource");
call.query.set("url", url);
call.query.set("parsed_data", true);
GetParsed { client, call }
}
pub fn refresh_rules(mut self, refresh: bool) -> Self {
self.call.query.set("refresh_rules", refresh);
self
}
pub fn rules_hint(mut self, hint: impl Into<String>) -> Self {
self.call.query.set("rules_hint", hint.into());
self
}
pub fn proxy_country(mut self, country: impl Into<String>) -> Self {
self.call.query.set("proxy_country", country.into());
self
}
pub fn time_sleep(mut self, seconds: u32) -> Self {
self.call.query.set("time_sleep", seconds);
self
}
}
executable!(GetParsed -> ParsedPage);
#[derive(Debug, Clone)]
#[must_use = "a request does nothing until it is awaited or sent"]
pub struct GetPageWithCookies<C = Client> {
client: C,
call: Call,
}
impl<C> GetPageWithCookies<C> {
pub(crate) fn new(client: C, url: String) -> Self {
let mut call = Call::new("/getPageSource");
call.query.set("url", url);
call.query.set("get_cookies", true);
GetPageWithCookies { client, call }
}
pub fn proxy_country(mut self, country: impl Into<String>) -> Self {
self.call.query.set("proxy_country", country.into());
self
}
pub fn time_sleep(mut self, seconds: u32) -> Self {
self.call.query.set("time_sleep", seconds);
self
}
}
executable!(GetPageWithCookies -> PageResult);
#[derive(Debug, Clone)]
#[must_use = "a request does nothing until it is awaited or sent"]
pub struct GetPageWithXhr<C = Client> {
client: C,
call: Call,
}
impl<C> GetPageWithXhr<C> {
pub(crate) fn new(client: C, url: String, capture_xhr: String) -> Self {
let mut call = Call::new("/getPageSource");
call.query.set("url", url);
call.query.set("capture_xhr", capture_xhr);
GetPageWithXhr { client, call }
}
pub fn proxy_country(mut self, country: impl Into<String>) -> Self {
self.call.query.set("proxy_country", country.into());
self
}
steps_setter!();
}
executable!(GetPageWithXhr -> CapturedPage);
#[derive(Debug, Clone)]
#[must_use = "a request does nothing until it is awaited or sent"]
pub struct Serp<C = Client> {
client: C,
call: Call,
}
impl<C> Serp<C> {
pub(crate) fn new(client: C, keyword: String) -> Self {
let mut call = Call::new("/serpApi");
call.query.set("keyword", keyword);
Serp { client, call }
}
pub fn proxy_country(mut self, country: impl Into<String>) -> Self {
self.call.query.set("proxy_country", country.into());
self
}
pub fn pages_to_check(mut self, pages: u32) -> Self {
self.call.query.set("pages_to_check", pages);
self
}
pub fn wait_after_load(mut self, seconds: u32) -> Self {
self.call.query.set("wait_after_load", seconds);
self
}
pub fn full_urls(mut self, full_urls: bool) -> Self {
self.call.query.set("full_urls", full_urls);
self
}
}
executable!(Serp -> Value);
#[derive(Debug, Clone)]
#[must_use = "a request does nothing until it is awaited or sent"]
pub struct GetImage<C = Client> {
client: C,
call: Call,
}
impl<C> GetImage<C> {
pub(crate) fn new(client: C, url: String) -> Self {
let mut call = Call::new("/getImage");
call.query.set("url", url);
GetImage { client, call }
}
pub fn proxy_country(mut self, country: impl Into<String>) -> Self {
self.call.query.set("proxy_country", country.into());
self
}
}
executable!(GetImage -> Vec<u8>);
#[derive(Debug, Clone)]
#[must_use = "a request does nothing until it is awaited or sent"]
pub struct Plugin<C = Client> {
client: C,
call: Call,
}
impl<C> Plugin<C> {
pub(crate) fn new(client: C, path: String) -> Self {
let path = if path.starts_with('/') {
path
} else {
format!("/{path}")
};
Plugin {
client,
call: Call::new(path),
}
}
pub fn param(mut self, name: &str, value: impl ToString) -> Self {
self.call.query.set(name, value);
self
}
pub fn param_each<T: ToString>(
mut self,
name: &str,
values: impl IntoIterator<Item = T>,
) -> Self {
self.call.query.set_each(name, values);
self
}
}
executable!(Plugin -> Value);