use std::fmt;
use std::sync::atomic::{AtomicI64, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use bytes::Bytes;
use serde::de::DeserializeOwned;
use serde::Serialize;
use url::Url;
use crate::error::{AuthCause, Error};
use crate::params::ReturnFormat;
use crate::response::content::MultiBody;
use crate::response::costs::Costs;
use crate::response::metadata::Metadata;
use crate::response::page::{PageParts, PageResult, Pages};
use crate::response::Body;
use crate::status::{ApiStatus, PageStatus};
use crate::Result;
pub const DEFAULT_BASE_URL: &str = "https://api.spider.cloud";
pub const BASE_URL_ENV: &str = "SPIDER_API_URL";
pub const USER_AGENT: &str = concat!(
"spider-cloud-agent/",
env!("CARGO_PKG_VERSION"),
" (+https://spider.cloud)"
);
const REDACTED: &str = "<redacted>";
pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(30);
pub(crate) fn validate_base(base: &Url, insecure: bool) -> Result<()> {
let loopback = match base.host() {
Some(url::Host::Ipv4(ip)) => ip.is_loopback(),
Some(url::Host::Ipv6(ip)) => ip.is_loopback(),
_ => false,
};
if base.scheme() == "https" || (base.scheme() == "http" && (insecure || loopback)) {
Ok(())
} else {
Err(Error::Config(
"the API base must use https; allow_insecure_http is only for test services".into(),
))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum Method {
Get,
Post,
}
impl Method {
pub const fn as_str(self) -> &'static str {
match self {
Method::Get => "GET",
Method::Post => "POST",
}
}
}
impl fmt::Display for Method {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct Route {
pub method: Method,
pub path: &'static str,
}
impl Route {
pub const fn get(path: &'static str) -> Route {
Route {
method: Method::Get,
path,
}
}
pub const fn post(path: &'static str) -> Route {
Route {
method: Method::Post,
path,
}
}
pub fn slots(&self) -> usize {
self.path.matches('{').count()
}
pub fn render(&self, args: &[&str]) -> Result<String> {
if args.len() != self.slots() {
return Err(Error::Config(format!(
"{} takes {} path arguments, got {}",
self.path,
self.slots(),
args.len()
)));
}
for arg in args {
if arg
.split('/')
.any(|segment| segment == "." || segment == "..")
{
return Err(Error::Config(format!(
"{} was given the path argument {arg:?}, which carries a dot \
segment and would resolve to somewhere outside the route",
self.path
)));
}
}
let mut out = String::with_capacity(self.path.len() + 32);
let mut rest = self.path;
let mut args = args.iter();
while let Some(open) = rest.find('{') {
let Some(tail) = rest.get(open..) else { break };
let close = match tail.find('}') {
Some(offset) => open + offset,
None => {
return Err(Error::Config(format!(
"route path {} has an unclosed placeholder",
self.path
)))
}
};
let (Some(head), Some(next)) = (rest.get(..open), rest.get(close + 1..)) else {
break;
};
out.push_str(head);
let arg = args.next().unwrap_or(&"");
out.push_str(&encode_path(arg));
rest = next;
}
out.push_str(rest);
Ok(out)
}
}
impl fmt::Display for Route {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{} {}", self.method, self.path)
}
}
fn encode_path(value: &str) -> String {
let mut out = String::with_capacity(value.len());
for byte in value.bytes() {
match byte {
b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' | b'/' => {
out.push(byte as char)
}
other => out.push_str(&format!("%{other:02X}")),
}
}
out
}
pub mod route {
use super::Route;
pub const SCRAPE: Route = Route::post("/scrape");
pub const CRAWL: Route = Route::post("/crawl");
pub const LINKS: Route = Route::post("/links");
pub const SEARCH: Route = Route::post("/search");
pub const SCREENSHOT: Route = Route::post("/screenshot");
pub const TRANSFORM: Route = Route::post("/transform");
pub const V1_SCRAPE: Route = Route::post("/v1/scrape");
pub const V1_CRAWL: Route = Route::post("/v1/crawl");
pub const V1_LINKS: Route = Route::post("/v1/links");
pub const V1_SEARCH: Route = Route::post("/v1/search");
pub const V1_SCREENSHOT: Route = Route::post("/v1/screenshot");
pub const V1_TRANSFORM: Route = Route::post("/v1/transform");
pub const UNLIMITED_SCRAPE: Route = Route::post("/unlimited/scrape");
pub const UNLIMITED_CRAWL: Route = Route::post("/unlimited/crawl");
pub const UNLIMITED_LINKS: Route = Route::post("/unlimited/links");
pub const AI_SCRAPE: Route = Route::post("/ai/scrape");
pub const AI_CRAWL: Route = Route::post("/ai/crawl");
pub const AI_SEARCH: Route = Route::post("/ai/search");
pub const AI_BROWSER: Route = Route::post("/ai/browser");
pub const AI_LINKS: Route = Route::post("/ai/links");
pub const FETCH: Route = Route::post("/fetch/{domain}/{path}");
pub const DATA_CREDITS: Route = Route::get("/data/credits");
pub const DATA_CRAWL_LOGS: Route = Route::get("/data/crawl_logs");
pub const DATA_TABLE: Route = Route::get("/data/{table}");
}
pub const ROUTES: &[Route] = &[
route::SCRAPE,
route::CRAWL,
route::LINKS,
route::SEARCH,
route::SCREENSHOT,
route::TRANSFORM,
route::V1_SCRAPE,
route::V1_CRAWL,
route::V1_LINKS,
route::V1_SEARCH,
route::V1_SCREENSHOT,
route::V1_TRANSFORM,
route::UNLIMITED_SCRAPE,
route::UNLIMITED_CRAWL,
route::UNLIMITED_LINKS,
route::AI_SCRAPE,
route::AI_CRAWL,
route::AI_SEARCH,
route::AI_BROWSER,
route::AI_LINKS,
route::FETCH,
route::DATA_CREDITS,
route::DATA_CRAWL_LOGS,
route::DATA_TABLE,
];
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct RateLimit {
pub limit: Option<u32>,
pub remaining: Option<u32>,
pub reset: Option<Duration>,
pub retry_after: Option<Duration>,
}
impl RateLimit {
pub fn is_known(&self) -> bool {
self != &RateLimit::default()
}
fn from_headers(headers: &reqwest::header::HeaderMap) -> RateLimit {
let seconds = |name: &str| header_number(headers, name).map(Duration::from_secs);
RateLimit {
limit: header_number(headers, "ratelimit-limit").map(count),
remaining: header_number(headers, "ratelimit-remaining").map(count),
reset: seconds("ratelimit-reset"),
retry_after: seconds("retry-after"),
}
}
pub fn wait(&self) -> Option<Duration> {
match (self.retry_after, self.reset) {
(Some(a), Some(b)) => Some(a.max(b)),
(Some(a), None) => Some(a),
(None, Some(b)) => Some(b),
(None, None) => None,
}
}
}
#[derive(Debug, Default)]
struct RateLimitCell {
limit: AtomicI64,
remaining: AtomicI64,
reset: AtomicI64,
retry_after: AtomicI64,
}
const UNSET: i64 = -1;
impl RateLimitCell {
fn new() -> RateLimitCell {
RateLimitCell {
limit: AtomicI64::new(UNSET),
remaining: AtomicI64::new(UNSET),
reset: AtomicI64::new(UNSET),
retry_after: AtomicI64::new(UNSET),
}
}
fn store(&self, slot: &AtomicI64, value: Option<u64>) {
if let Some(value) = value {
slot.store(i64::try_from(value).unwrap_or(i64::MAX), Ordering::Relaxed);
}
}
fn read(slot: &AtomicI64) -> Option<u64> {
match slot.load(Ordering::Relaxed) {
UNSET => None,
value => u64::try_from(value).ok(),
}
}
fn snapshot(&self) -> RateLimit {
let seconds = |slot: &AtomicI64| RateLimitCell::read(slot).map(Duration::from_secs);
RateLimit {
limit: RateLimitCell::read(&self.limit).map(count),
remaining: RateLimitCell::read(&self.remaining).map(count),
reset: seconds(&self.reset),
retry_after: seconds(&self.retry_after),
}
}
}
fn count(value: u64) -> u32 {
u32::try_from(value).unwrap_or(u32::MAX)
}
fn header_number(headers: &reqwest::header::HeaderMap, name: &str) -> Option<u64> {
let raw = headers.get(name)?.to_str().ok()?.trim();
let value = raw.parse::<u64>().ok()?;
Some(value.min(i64::MAX as u64))
}
pub struct Transport {
client: reqwest::Client,
base: Url,
key: String,
limits: Arc<RateLimitCell>,
insecure_http: bool,
}
impl fmt::Debug for Transport {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Transport")
.field("base", &self.base.as_str())
.field("key", &REDACTED)
.field("rate_limit", &self.rate_limit())
.finish()
}
}
impl Transport {
pub fn new(key: impl Into<String>) -> Result<Transport> {
Transport::with_base(key, Transport::base_url_from_env()?)
}
pub fn with_base(key: impl Into<String>, base: Url) -> Result<Transport> {
Self::with_base_options(key, base, false)
}
pub(crate) fn with_base_options(
key: impl Into<String>,
base: Url,
insecure: bool,
) -> Result<Transport> {
validate_base(&base, insecure)?;
let client = Self::default_client()?;
Ok(Transport::with_client(key, base, client).allow_insecure_http(insecure))
}
pub fn default_client() -> Result<reqwest::Client> {
reqwest::Client::builder()
.user_agent(USER_AGENT)
.redirect(reqwest::redirect::Policy::custom(|attempt| {
attempt.error("API redirects are not supported")
}))
.connect_timeout(CONNECT_TIMEOUT)
.build()
.map_err(Error::Transport)
}
pub fn with_client(key: impl Into<String>, base: Url, client: reqwest::Client) -> Transport {
let mut base = base;
if !base.cannot_be_a_base() && !base.path().ends_with('/') {
let path = format!("{}/", base.path());
base.set_path(&path);
}
Transport {
client,
base,
key: key.into(),
limits: Arc::new(RateLimitCell::new()),
insecure_http: false,
}
}
pub(crate) fn allow_insecure_http(mut self, allow: bool) -> Self {
self.insecure_http = allow;
self
}
pub fn base_url_from_env() -> Result<Url> {
Self::base_url_from_env_options(false)
}
pub(crate) fn base_url_from_env_options(insecure: bool) -> Result<Url> {
let base = match std::env::var("SPIDER_API_URL") {
Ok(value) if !value.trim().is_empty() => Url::parse(value.trim())
.map_err(|e| Error::Config(format!("{BASE_URL_ENV} is not a url: {e}"))),
_ => Url::parse(DEFAULT_BASE_URL)
.map_err(|e| Error::Config(format!("the default address is not a url: {e}"))),
}?;
validate_base(&base, insecure)?;
Ok(base)
}
pub fn base_url(&self) -> &Url {
&self.base
}
pub fn rate_limit(&self) -> RateLimit {
self.limits.snapshot()
}
pub async fn post<B: Serialize>(&self, route: Route, args: &[&str], body: &B) -> Result<Reply> {
self.post_with_limit(route, args, body, None).await
}
pub(crate) async fn post_with_limit<B: Serialize>(
&self,
route: Route,
args: &[&str],
body: &B,
limit: Option<usize>,
) -> Result<Reply> {
if route.method != Method::Post {
return Err(Error::Config(format!("{route} is not a post route")));
}
let url = self.url_for(route, args, &[])?;
let request = self.client.post(url).json(body);
self.execute(request, limit).await
}
pub async fn get(&self, route: Route, args: &[&str], query: &[(&str, &str)]) -> Result<Reply> {
self.get_with_limit(route, args, query, None).await
}
pub(crate) async fn get_with_limit(
&self,
route: Route,
args: &[&str],
query: &[(&str, &str)],
limit: Option<usize>,
) -> Result<Reply> {
if route.method != Method::Get {
return Err(Error::Config(format!("{route} is not a get route")));
}
let url = self.url_for(route, args, query)?;
let request = self.client.get(url);
self.execute(request, limit).await
}
pub fn url_for(&self, route: Route, args: &[&str], query: &[(&str, &str)]) -> Result<Url> {
let path = route.render(args)?;
let mut url = self
.base
.join(path.trim_start_matches('/'))
.map_err(|e| Error::Config(format!("cannot build an address for {route}: {e}")))?;
if !query.is_empty() {
let mut pairs = url.query_pairs_mut();
for (key, value) in query {
pairs.append_pair(key, value);
}
}
Ok(url)
}
async fn execute(
&self,
request: reqwest::RequestBuilder,
limit: Option<usize>,
) -> Result<Reply> {
validate_base(&self.base, self.insecure_http)?;
let started = Instant::now();
let request = if self.key.is_empty() {
request
} else {
request.header(reqwest::header::AUTHORIZATION, self.bearer())
};
let mut response = request.send().await.map_err(Error::Transport)?;
let status = ApiStatus::new(response.status().as_u16());
let headers = response.headers().clone();
self.record_limits(&headers);
let content_type = headers
.get(reqwest::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.map(|v| v.to_ascii_lowercase());
let body = match limit {
None => response.bytes().await.map_err(Error::Transport)?,
Some(limit) => {
if response
.content_length()
.is_some_and(|size| size > limit as u64)
{
return Err(Error::ResponseTooLarge { limit, status });
}
let mut bytes = Vec::new();
while let Some(chunk) = response.chunk().await.map_err(Error::Transport)? {
if chunk.len() > limit.saturating_sub(bytes.len()) {
return Err(Error::ResponseTooLarge { limit, status });
}
bytes.extend_from_slice(&chunk);
}
Bytes::from(bytes)
}
};
let rate_limit = RateLimit::from_headers(&headers);
Ok(Reply {
status,
retry_after: rate_limit.retry_after,
rate_limit,
elapsed: started.elapsed(),
content_type,
body,
})
}
pub fn has_key(&self) -> bool {
!self.key.is_empty()
}
fn bearer(&self) -> String {
format!("Bearer {}", self.key)
}
fn record_limits(&self, headers: &reqwest::header::HeaderMap) {
let cell = &self.limits;
cell.store(&cell.limit, header_number(headers, "ratelimit-limit"));
cell.store(
&cell.remaining,
header_number(headers, "ratelimit-remaining"),
);
cell.store(&cell.reset, header_number(headers, "ratelimit-reset"));
cell.store(&cell.retry_after, header_number(headers, "retry-after"));
}
}
#[derive(Debug, Clone)]
pub struct Reply {
pub status: ApiStatus,
pub rate_limit: RateLimit,
pub retry_after: Option<Duration>,
pub elapsed: Duration,
pub content_type: Option<String>,
pub body: Bytes,
}
impl Reply {
pub(crate) fn operation_status(&self) -> ApiStatus {
if self.is_mirrored_page_status() {
ApiStatus::new(0)
} else {
self.status
}
}
pub fn is_success(&self) -> bool {
self.status.is_success()
}
pub fn text(&self) -> Option<&str> {
std::str::from_utf8(&self.body).ok()
}
pub fn json<T: DeserializeOwned>(&self) -> Result<T> {
serde_json::from_slice(&self.body).map_err(Error::Decode)
}
pub fn as_error(&self) -> Option<Error> {
if self.is_success() || self.is_mirrored_page_status() {
return None;
}
let message = self.message();
if self.asks_for_an_account() {
return Some(Error::Auth {
cause: AuthCause::SignInRequired,
message: message.unwrap_or_else(|| "this call needs an account".to_string()),
});
}
Some(match self.status.code() {
402 => Error::InsufficientCredits,
401 => Error::Auth {
cause: AuthCause::Refused,
message: message.unwrap_or_else(|| "the key was missing or rejected".to_string()),
},
_ => Error::Api {
status: self.status,
message,
retry_after: self.retry_after.or(self.rate_limit.reset),
},
})
}
pub fn into_result(self) -> Result<Reply> {
match self.as_error() {
Some(error) => Err(error),
None => Ok(self),
}
}
fn asks_for_an_account(&self) -> bool {
if !matches!(self.status.code(), 401 | 402) {
return false;
}
let Ok(value) = serde_json::from_slice::<serde_json::Value>(&self.body) else {
return false;
};
value
.get("code")
.and_then(|code| code.as_str())
.is_some_and(|code| code == "auth_required" || code.starts_with("free_tier_"))
}
pub fn message(&self) -> Option<String> {
let value: serde_json::Value = serde_json::from_slice(&self.body).ok()?;
for key in ["error", "message", "detail"] {
if let Some(text) = value.get(key).and_then(|v| v.as_str()) {
if !text.trim().is_empty() {
return Some(text.trim().to_string());
}
}
}
None
}
pub fn is_mirrored_page_status(&self) -> bool {
if self.status.code() < 400 {
return false;
}
let value: serde_json::Value = match serde_json::from_slice(&self.body) {
Ok(value) => value,
Err(_) => return false,
};
let first = match &value {
serde_json::Value::Array(items) => match items.first() {
Some(first) => first,
None => return false,
},
other => other,
};
let has_url = first.get("url").and_then(|v| v.as_str()).is_some();
let same_status = first
.get("status")
.and_then(|v| v.as_u64())
.is_some_and(|code| code == u64::from(self.status.code()));
has_url && same_status
}
pub fn read(&self, requested: &Url, format: Option<ReturnFormat>) -> Result<Pages> {
self.read_pages(requested, format, false)
}
pub(crate) fn read_pages(
&self,
requested: &Url,
format: Option<ReturnFormat>,
transform: bool,
) -> Result<Pages> {
if self.status.code() == 204 && self.body.is_empty() {
return Ok(Pages::default());
}
let value: serde_json::Value = self.json()?;
let items = match value {
serde_json::Value::Array(items) => items,
serde_json::Value::Null => Vec::new(),
other if transform => converted_documents(other),
other => vec![other],
};
let mut out = Vec::with_capacity(items.len());
for item in items {
out.push(self.read_one(item, requested, format, transform)?);
}
Ok(Pages(out))
}
fn read_one(
&self,
value: serde_json::Value,
requested: &Url,
format: Option<ReturnFormat>,
transform: bool,
) -> Result<PageResult> {
let wire: WirePage = serde_json::from_value(value).map_err(Error::Decode)?;
let url = wire
.url
.as_deref()
.and_then(|u| Url::parse(u).ok())
.unwrap_or_else(|| requested.clone());
let status = PageStatus::new(wire.status.unwrap_or(0));
let links = wire.links.map(|links| {
links
.iter()
.filter_map(|link| Url::parse(link).ok().or_else(|| url.join(link).ok()))
.collect()
});
let parts = PageParts {
url,
status,
body: match wire.css_extracted {
Some(fields) => match body_from(wire.content, format) {
Body::Empty => Body::Fields(fields),
content => Body::WithFields {
content: Box::new(content),
fields,
},
},
None => body_from(wire.content, format),
},
duration: wire
.duration_elasped_ms
.filter(|ms| ms.is_finite() && *ms >= 0.0)
.and_then(|ms| Duration::try_from_secs_f64(ms / 1000.0).ok()),
call_elapsed: self.elapsed,
json_data: wire.json_data,
request_map: wire.request_map,
response_map: wire.response_map,
trace: wire.trace,
costs: wire.costs.unwrap_or_default(),
metadata: wire.metadata,
links,
headers: wire.headers.or(wire.response_headers),
cookies: wire
.cookies
.or(wire.response_cookies)
.map(WireCookies::into_map),
error: wire.error,
};
Ok(if transform && status.is_unknown() {
PageResult::Ok(crate::response::Page::document_from_parts(parts))
} else {
PageResult::from_parts(parts)
})
}
}
fn converted_documents(value: serde_json::Value) -> Vec<serde_json::Value> {
let mut object = match value {
serde_json::Value::Object(object) => object,
other => return vec![other],
};
let documents = match object.get("content") {
Some(serde_json::Value::Array(items))
if !items.is_empty() && items.iter().all(serde_json::Value::is_string) =>
{
items.clone()
}
_ => return vec![serde_json::Value::Object(object)],
};
let mut out = Vec::with_capacity(documents.len());
for (position, document) in documents.into_iter().enumerate() {
let mut item = match position {
0 => std::mem::take(&mut object),
_ => serde_json::Map::new(),
};
item.insert("content".to_string(), document);
out.push(serde_json::Value::Object(item));
}
out
}
#[derive(Debug, Default, serde::Deserialize)]
struct WirePage {
#[serde(default)]
url: Option<String>,
#[serde(default)]
status: Option<u16>,
#[serde(default)]
content: Option<MultiBody>,
#[serde(default)]
css_extracted: Option<std::collections::BTreeMap<String, serde_json::Value>>,
#[serde(default)]
links: Option<Vec<String>>,
#[serde(default)]
metadata: Option<Metadata>,
#[serde(default)]
costs: Option<Costs>,
#[serde(default)]
error: Option<String>,
#[serde(default)]
duration_elasped_ms: Option<f64>,
#[serde(default)]
json_data: Option<serde_json::Value>,
#[serde(default)]
request_map: Option<serde_json::Value>,
#[serde(default)]
response_map: Option<serde_json::Value>,
#[serde(default)]
trace: Option<serde_json::Value>,
#[serde(default)]
headers: Option<std::collections::BTreeMap<String, String>>,
#[serde(default)]
response_headers: Option<std::collections::BTreeMap<String, String>>,
#[serde(default)]
cookies: Option<WireCookies>,
#[serde(default)]
response_cookies: Option<WireCookies>,
}
#[derive(Debug, serde::Deserialize)]
#[serde(untagged)]
enum WireCookies {
Text(String),
Map(std::collections::BTreeMap<String, String>),
}
impl WireCookies {
fn into_map(self) -> std::collections::BTreeMap<String, String> {
match self {
Self::Map(map) => map,
Self::Text(text) => text
.split(';')
.filter_map(|pair| {
let (name, value) = pair.trim().split_once('=')?;
Some((name.trim().to_string(), value.trim().to_string()))
})
.collect(),
}
}
}
fn body_from(content: Option<MultiBody>, format: Option<ReturnFormat>) -> Body {
let multi = match content {
Some(multi) => multi,
None => return Body::Empty,
};
if multi.is_empty() && multi.present().is_empty() {
return Body::Empty;
}
let present = multi.present();
let [only] = present[..] else {
return Body::Multi(multi);
};
match only {
"text" => multi.text.map(Body::Text).unwrap_or(Body::Empty),
"markdown" => multi.markdown.map(Body::Markdown).unwrap_or(Body::Empty),
"html2text" => multi.html2text.map(Body::Text).unwrap_or(Body::Empty),
"bytes" => multi.bytes.map(Body::Bytes).unwrap_or(Body::Empty),
"screenshot" => multi
.screenshot
.map(Body::Screenshot)
.unwrap_or(Body::Empty),
"raw" => match multi.raw {
Some(raw) => match format {
Some(ReturnFormat::Markdown) | Some(ReturnFormat::Commonmark) => {
Body::Markdown(raw)
}
Some(ReturnFormat::Text) => Body::Text(raw),
Some(ReturnFormat::Xml) => Body::Xml(raw),
Some(ReturnFormat::Bytes) => Body::Bytes(Bytes::from(raw.into_bytes())),
Some(ReturnFormat::Empty) => Body::Empty,
Some(ReturnFormat::Raw) | None => Body::Html(raw),
},
None => Body::Empty,
},
_ => Body::Multi(multi),
}
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::indexing_slicing,
clippy::string_slice
)]
use super::*;
fn reply(code: u16, body: &str) -> Reply {
Reply {
status: ApiStatus::new(code),
rate_limit: RateLimit::default(),
retry_after: None,
elapsed: Duration::from_millis(40),
content_type: Some("application/json".into()),
body: Bytes::from(body.to_string()),
}
}
fn fixture(name: &str) -> Reply {
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures")
.join(name);
let text = std::fs::read_to_string(&path)
.unwrap_or_else(|e| panic!("cannot read {}: {e}", path.display()));
reply(200, &text)
}
#[test]
fn a_recorded_page_becomes_a_typed_page() {
let pages = fixture("scrape_markdown.json")
.read(
&Url::parse("https://example.com/pricing").unwrap(),
Some(ReturnFormat::Markdown),
)
.expect("pages");
let page = pages.first_ok().expect("a page");
assert_eq!(page.url.as_str(), "https://example.com/pricing");
assert_eq!(page.status.code(), 200);
assert!(matches!(page.body, Body::Markdown(_)));
assert!(page.text().expect("text").starts_with("# Pricing"));
assert_eq!(page.cost(), crate::credits::Credits::from_usd(0.6));
assert_eq!(
page.metadata.as_ref().expect("metadata").title.as_deref(),
Some("Pricing")
);
}
#[test]
fn a_recorded_crawl_sorts_itself_into_pages_and_refusals() {
let pages = fixture("crawl_mixed.json")
.read(&Url::parse("https://example.org/").unwrap(), None)
.expect("pages");
assert_eq!(pages.len(), 3);
assert_eq!(pages.ok().count(), 2);
assert_eq!(pages.failed().count(), 1);
let failed = pages.failed().next().expect("a refusal");
assert_eq!(failed.status.code(), 403);
assert!(failed.was_billed());
assert_eq!(failed.hint, crate::response::Hint::TryResidentialProxy);
assert_eq!(pages.ok().filter(|p| p.is_blank()).count(), 1);
assert!((pages.total_cost().get() - 12_000.0).abs() < 1e-6);
}
#[test]
fn a_recorded_refusal_never_becomes_a_page() {
let pages = fixture("blocked_page.json")
.read(&Url::parse("https://example.net/checkout").unwrap(), None)
.expect("pages");
assert!(pages.first_ok().is_none());
let failed = pages.failed().next().expect("a refusal");
assert_eq!(failed.error.as_deref(), Some("the site refused the fetch"));
}
#[test]
fn a_recorded_picture_arrives_as_bytes() {
let pages = fixture("screenshot.json")
.read(&Url::parse("https://example.com/").unwrap(), None)
.expect("pages");
let page = pages.first_ok().expect("a page");
assert!(matches!(page.body, Body::Screenshot(_)));
assert_eq!(page.body.len(), 8);
assert_eq!(page.text(), None);
}
#[test]
fn a_recorded_page_keeps_its_headers_and_cookies() {
let pages = fixture("multi_format.json")
.read(&Url::parse("https://httpbin.org/html").unwrap(), None)
.expect("pages");
let page = pages.first_ok().expect("a page");
assert!(matches!(page.body, Body::Multi(_)));
assert_eq!(page.text(), Some("Herman Melville"));
assert_eq!(
page.headers.as_ref().expect("headers").get("content-type"),
Some(&"text/html; charset=utf-8".to_string())
);
assert!(page
.cookies
.as_ref()
.expect("cookies")
.contains_key("session"));
}
#[test]
fn the_route_table_has_no_duplicates_and_no_pipeline() {
let mut seen: Vec<String> = ROUTES.iter().map(|r| r.to_string()).collect();
let before = seen.len();
seen.sort();
seen.dedup();
assert_eq!(seen.len(), before, "a route is listed twice");
assert!(
!ROUTES.iter().any(|r| r.path.contains("pipeline")),
"that path is not part of the public api"
);
}
#[test]
fn a_route_fills_its_placeholders() {
assert_eq!(
route::FETCH
.render(&["example.com", "blog/a post"])
.unwrap(),
"/fetch/example.com/blog/a%20post"
);
assert_eq!(
route::DATA_TABLE.render(&["crawl_logs"]).unwrap(),
"/data/crawl_logs"
);
assert_eq!(route::SCRAPE.render(&[]).unwrap(), "/scrape");
}
#[test]
fn a_route_refuses_the_wrong_number_of_arguments() {
assert!(route::FETCH.render(&["example.com"]).is_err());
assert!(route::SCRAPE.render(&["extra"]).is_err());
}
#[test]
fn a_reply_keeps_the_two_planes_apart() {
let pages = reply(
200,
r#"[{"url":"https://example.com/a","status":403,"content":null}]"#,
)
.read(&Url::parse("https://example.com/a").unwrap(), None)
.expect("pages");
assert_eq!(pages.len(), 1);
let failed = pages.failed().next().expect("a failure");
assert_eq!(failed.status.code(), 403);
}
#[test]
fn a_body_status_mirrored_onto_the_call_stays_on_the_target_plane() {
let mirrored = reply(503, r#"{"url":"https://example.com/a","status":503}"#);
assert!(mirrored.is_mirrored_page_status());
let service_down = reply(503, r#"{"error":"try again shortly"}"#);
assert!(!service_down.is_mirrored_page_status());
}
#[test]
fn a_mismatched_body_status_does_not_explain_the_envelope() {
for code in [401u16, 402] {
let r = reply(code, r#"{"url":"https://example.com/a","status":403}"#);
assert!(
!r.is_mirrored_page_status(),
"code {code} must stay on the call plane"
);
}
}
#[test]
fn f1_mirrored_account_codes_are_target_statuses() {
for code in [401u16, 402] {
let r = reply(
code,
&format!(r#"{{"url":"https://example.com","status":{code}}}"#),
);
assert!(r.is_mirrored_page_status());
assert!(r.as_error().is_none());
assert_eq!(r.operation_status().code(), 0);
assert_eq!(r.status.code(), code);
let pages = r
.read(&Url::parse("https://example.com").unwrap(), None)
.unwrap();
assert_eq!(pages.failed().next().unwrap().status.code(), code);
}
}
#[test]
fn f1_empty_204_decodes_without_json() {
let pages = reply(204, "")
.read(&Url::parse("https://example.com").unwrap(), None)
.unwrap();
assert!(pages.is_empty());
}
#[test]
fn f1_missing_status_never_borrows_the_envelope() {
let r = reply(200, r#"{"content":"hello"}"#);
let url = Url::parse("https://example.com").unwrap();
let pages = r.read(&url, None).unwrap();
assert!(pages.failed().next().unwrap().status.is_unknown());
let docs = r.read_pages(&url, None, true).unwrap();
assert!(docs.first_ok().unwrap().status.is_unknown());
}
#[test]
fn running_out_of_credits_is_its_own_error() {
let error = reply(402, r#"{"error":"balance is zero"}"#)
.as_error()
.expect("an error");
assert!(matches!(error, Error::InsufficientCredits));
assert!(!error.is_retryable());
}
#[test]
fn a_call_with_no_key_that_needs_an_account_says_to_sign_in() {
for (code, body) in [
(
402,
r#"{"error":"This route requires an account.","code":"free_tier_route_blocked"}"#,
),
(
401,
r#"{"error":"Authentication required","code":"auth_required","hint":"x"}"#,
),
(
401,
r#"{"message":"Unauthorized","code":"auth_required","hint":"x"}"#,
),
] {
match reply(code, body).as_error().expect("an error") {
Error::Auth { cause, message } => {
assert_eq!(cause, AuthCause::SignInRequired, "{body}");
assert!(!message.is_empty());
}
other => panic!("{body} became {other:?}"),
}
}
}
#[test]
fn only_the_service_marking_it_makes_a_refusal_a_sign_in() {
let spent = reply(
402,
r#"{"error":"balance is zero","code":"insufficient_credits"}"#,
)
.as_error()
.expect("an error");
assert!(matches!(spent, Error::InsufficientCredits));
let limited = reply(
429,
r#"{"error":"Free tier rate limit reached.","code":"free_tier_rate_limited"}"#,
)
.as_error()
.expect("an error");
assert!(matches!(limited, Error::Api { .. }), "{limited:?}");
}
#[test]
fn a_transport_with_no_key_says_so() {
let base = Url::parse("https://api.spider.cloud").expect("an address");
let client = Transport::default_client().expect("a client");
assert!(!Transport::with_client("", base.clone(), client.clone()).has_key());
assert!(Transport::with_client("sk-test", base, client).has_key());
}
#[test]
fn a_rejected_key_reports_what_the_service_said_and_nothing_else() {
let error = reply(401, r#"{"error":"invalid api key"}"#)
.as_error()
.expect("an error");
match error {
Error::Auth { cause, message } => {
assert_eq!(cause, AuthCause::Refused);
assert_eq!(message, "invalid api key");
}
other => panic!("expected an auth error, got {other:?}"),
}
}
#[test]
fn a_success_is_not_an_error() {
assert!(reply(200, "[]").as_error().is_none());
assert!(reply(204, "").as_error().is_none());
}
#[test]
fn the_format_that_was_asked_for_names_the_body() {
let json = r#"[{"url":"https://example.com/a","status":200,"content":{"raw":"Heading"}}]"#;
let url = Url::parse("https://example.com/a").unwrap();
let markdown = reply(200, json)
.read(&url, Some(ReturnFormat::Markdown))
.unwrap();
assert!(matches!(
markdown.first_ok().unwrap().body,
Body::Markdown(_)
));
let raw = reply(200, json).read(&url, None).unwrap();
assert!(matches!(raw.first_ok().unwrap().body, Body::Html(_)));
}
#[test]
fn several_formats_at_once_stay_together() {
let json = r#"[{"url":"https://example.com/a","status":200,
"content":{"raw":"<p>a</p>","markdown":"a"}}]"#;
let pages = reply(200, json)
.read(&Url::parse("https://example.com/a").unwrap(), None)
.unwrap();
assert!(matches!(pages.first_ok().unwrap().body, Body::Multi(_)));
}
#[test]
fn a_page_with_no_content_reads_as_blank_rather_than_failing() {
let pages = reply(200, r#"[{"url":"https://example.com/a","status":200}]"#)
.read(&Url::parse("https://example.com/a").unwrap(), None)
.unwrap();
assert!(pages.first_ok().expect("a page").is_blank());
}
#[test]
fn a_single_object_and_an_array_read_the_same() {
let url = Url::parse("https://example.com/a").unwrap();
let one = reply(200, r#"{"url":"https://example.com/a","status":200}"#)
.read(&url, None)
.unwrap();
let many = reply(200, r#"[{"url":"https://example.com/a","status":200}]"#)
.read(&url, None)
.unwrap();
assert_eq!(one.len(), many.len());
}
#[test]
fn links_are_resolved_against_the_page() {
let json = r#"[{"url":"https://example.com/a/b","status":200,
"links":["/c","https://example.org/d","http://"]}]"#;
let pages = reply(200, json)
.read(&Url::parse("https://example.com/a/b").unwrap(), None)
.unwrap();
let links = pages.first_ok().unwrap().links.clone().expect("links");
assert_eq!(links.len(), 2);
assert_eq!(links[0].as_str(), "https://example.com/c");
}
#[test]
fn rate_limit_headers_land_in_the_snapshot() {
let mut headers = reqwest::header::HeaderMap::new();
headers.insert("ratelimit-limit", "100".parse().unwrap());
headers.insert("ratelimit-remaining", "0".parse().unwrap());
headers.insert("ratelimit-reset", "30".parse().unwrap());
headers.insert("retry-after", "12".parse().unwrap());
let transport = Transport::with_client(
"not-a-real-key",
Url::parse(DEFAULT_BASE_URL).unwrap(),
reqwest::Client::new(),
);
transport.record_limits(&headers);
let snapshot = transport.rate_limit();
assert_eq!(snapshot.limit, Some(100));
assert_eq!(snapshot.remaining, Some(0));
assert_eq!(snapshot.reset, Some(Duration::from_secs(30)));
assert_eq!(snapshot.retry_after, Some(Duration::from_secs(12)));
assert_eq!(snapshot.wait(), Some(Duration::from_secs(30)));
assert!(snapshot.is_known());
}
#[test]
fn a_rate_limit_past_the_field_saturates_rather_than_wrapping_to_zero() {
let mut headers = reqwest::header::HeaderMap::new();
headers.insert("ratelimit-limit", "4294967296".parse().unwrap());
headers.insert("ratelimit-remaining", "4294967296".parse().unwrap());
let transport = Transport::with_client(
"not-a-real-key",
Url::parse(DEFAULT_BASE_URL).unwrap(),
reqwest::Client::new(),
);
transport.record_limits(&headers);
let snapshot = transport.rate_limit();
assert_eq!(snapshot.limit, Some(u32::MAX));
assert_eq!(snapshot.remaining, Some(u32::MAX));
}
#[test]
fn a_missing_rate_limit_header_reads_as_unknown_not_as_zero() {
let transport = Transport::with_client(
"not-a-real-key",
Url::parse(DEFAULT_BASE_URL).unwrap(),
reqwest::Client::new(),
);
assert!(!transport.rate_limit().is_known());
assert_eq!(transport.rate_limit().remaining, None);
assert_eq!(transport.rate_limit().wait(), None);
}
#[test]
fn the_transport_never_prints_the_key() {
let transport = Transport::with_client(
"sk-a-secret-value",
Url::parse(DEFAULT_BASE_URL).unwrap(),
reqwest::Client::new(),
);
let printed = format!("{transport:?}");
assert!(!printed.contains("sk-a-secret-value"), "{printed}");
assert!(printed.contains(REDACTED));
}
#[test]
fn an_address_is_built_from_the_route_and_the_base() {
let transport = Transport::with_client(
"not-a-real-key",
Url::parse("https://api.spider.cloud").unwrap(),
reqwest::Client::new(),
);
assert_eq!(
transport.url_for(route::SCRAPE, &[], &[]).unwrap().as_str(),
"https://api.spider.cloud/scrape"
);
assert_eq!(
transport
.url_for(route::DATA_TABLE, &["crawl_logs"], &[("limit", "5")])
.unwrap()
.as_str(),
"https://api.spider.cloud/data/crawl_logs?limit=5"
);
}
#[test]
fn a_base_with_a_path_keeps_its_path_whether_or_not_it_ends_in_a_slash() {
for base in [
"https://proxy.example/spider",
"https://proxy.example/spider/",
] {
let transport = Transport::with_client(
"not-a-real-key",
Url::parse(base).unwrap(),
reqwest::Client::new(),
);
assert_eq!(
transport.url_for(route::SCRAPE, &[], &[]).unwrap().as_str(),
"https://proxy.example/spider/scrape",
"from base {base}"
);
assert_eq!(
transport
.url_for(route::FETCH, &["example.com", "a/b"], &[])
.unwrap()
.as_str(),
"https://proxy.example/spider/fetch/example.com/a/b",
"from base {base}"
);
}
}
#[tokio::test]
async fn a_route_refuses_the_wrong_verb() {
let transport = Transport::with_client(
"not-a-real-key",
Url::parse(DEFAULT_BASE_URL).unwrap(),
reqwest::Client::new(),
);
assert!(matches!(
transport.get(route::SCRAPE, &[], &[]).await,
Err(Error::Config(_))
));
assert!(matches!(
transport.post(route::DATA_CREDITS, &[], &()).await,
Err(Error::Config(_))
));
}
}