use std::fmt;
use std::sync::Arc;
use spider_route::{HeuristicRouter, Router};
use url::Url;
use crate::auth::{router::StoredRouter, Credentials};
use crate::credits::Credits;
use crate::error::{AuthCause, Error};
use crate::memory::SiteMemoryStore;
use crate::ops::crawl::Crawl;
use crate::ops::data::{CrawlLogs, Table};
use crate::ops::fetch::Fetch;
use crate::ops::links::Links;
use crate::ops::scrape::Scrape;
use crate::ops::screenshot::Screenshot;
use crate::ops::search::Search;
use crate::ops::transform::{Document, Transform};
use crate::policy::{Budget, Policy};
use crate::record::Recorder;
use crate::routing::Explorer;
use crate::Result;
pub use crate::transport::{
route, Method, RateLimit, Reply, Route, Transport, BASE_URL_ENV, DEFAULT_BASE_URL, ROUTES,
USER_AGENT,
};
pub use crate::auth::{API_KEY_ENV, API_KEY_ENV_ALT, CREDENTIALS_PATH};
const REDACTED: &str = "<redacted>";
pub trait IntoUrl {
fn into_url(self) -> std::result::Result<Url, url::ParseError>;
}
impl IntoUrl for Url {
fn into_url(self) -> std::result::Result<Url, url::ParseError> {
Ok(self)
}
}
impl IntoUrl for &Url {
fn into_url(self) -> std::result::Result<Url, url::ParseError> {
Ok(self.clone())
}
}
impl IntoUrl for &str {
fn into_url(self) -> std::result::Result<Url, url::ParseError> {
Url::parse(self)
}
}
impl IntoUrl for String {
fn into_url(self) -> std::result::Result<Url, url::ParseError> {
Url::parse(&self)
}
}
impl IntoUrl for &String {
fn into_url(self) -> std::result::Result<Url, url::ParseError> {
Url::parse(self)
}
}
#[derive(Clone)]
pub struct Spider {
transport: Arc<Transport>,
budget: Budget,
pub(crate) run_budget: Option<RunBudget>,
pub(crate) jitter_seed: u64,
pub(crate) read_wall: Option<std::time::Duration>,
pub(crate) response_limit: Option<usize>,
policy: Option<Policy>,
router: Arc<dyn Router>,
recorder: Option<Arc<dyn Recorder>>,
explorer: Explorer,
memory: Arc<SiteMemoryStore>,
provider_router: Option<Arc<StoredRouter>>,
#[cfg(feature = "optimize")]
pub(crate) optimize: crate::optimize::Hooks,
}
impl fmt::Debug for Spider {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let provider_router = PolicyPresence(self.provider_router.is_some());
let mut out = f.debug_struct("Spider");
out.field("base", &self.transport.base_url().as_str())
.field("key", &REDACTED)
.field("budget", &self.budget)
.field("policy", &PolicyPresence(self.policy.is_some()))
.field("router", &self.router.version())
.field("recorder", &PolicyPresence(self.recorder.is_some()))
.field("explore", &self.explorer.rate)
.field("provider_router", &provider_router);
#[cfg(feature = "optimize")]
out.field("optimize", &self.optimize);
out.finish()
}
}
impl Spider {
pub fn new() -> Result<Spider> {
let key = resolve_key().ok_or_else(no_key)?;
SpiderBuilder::new().key(key).build()
}
pub fn with_key(key: impl Into<String>) -> Result<Spider> {
let base = Transport::base_url_from_env()?;
let transport = Transport::with_base(key, base)?;
Ok(Spider {
transport: Arc::new(transport),
budget: Budget::default(),
run_budget: None,
jitter_seed: client_seed(),
read_wall: Some(crate::ops::data::DEFAULT_READ_WALL),
response_limit: None,
policy: None,
router: Arc::new(HeuristicRouter::new()),
recorder: None,
explorer: Explorer::default(),
memory: Arc::new(SiteMemoryStore::default()),
provider_router: None,
#[cfg(feature = "optimize")]
optimize: crate::optimize::Hooks::default(),
})
}
pub fn builder() -> SpiderBuilder {
SpiderBuilder::new()
}
pub fn scrape(&self, url: impl IntoUrl) -> Scrape<'_> {
Scrape::new(self, url)
}
pub fn fetch(&self, domain: impl Into<String>, path: impl Into<String>) -> Fetch<'_> {
Fetch::new(self, domain, path)
}
pub fn crawl(&self, url: impl IntoUrl) -> Crawl<'_> {
Crawl::new(self, url)
}
pub fn links(&self, url: impl IntoUrl) -> Links<'_> {
Links::new(self, url)
}
pub fn search(&self, query: impl Into<String>) -> Search<'_> {
Search::new(self, query)
}
pub fn screenshot(&self, url: impl IntoUrl) -> Screenshot<'_> {
Screenshot::new(self, url)
}
pub fn transform(&self, docs: Vec<Document>) -> Transform<'_> {
Transform::new(self, docs)
}
pub async fn credits(&self) -> Result<Credits> {
crate::ops::data::read_under_wall(
self,
self.transport
.get_with_limit(route::DATA_CREDITS, &[], &[], self.response_limit),
crate::ops::data::credits_from,
)
.await
}
pub fn crawl_logs(&self) -> CrawlLogs<'_> {
CrawlLogs::new(self)
}
pub fn table(&self, name: impl Into<String>) -> Table<'_> {
Table::new(self, name)
}
pub fn raw(&self) -> &Transport {
&self.transport
}
pub fn rate_limit(&self) -> RateLimit {
self.transport.rate_limit()
}
pub fn budget(&self) -> Budget {
self.budget
}
pub fn run_budget(&self) -> Option<&RunBudget> {
self.run_budget.as_ref()
}
pub fn policy(&self) -> Option<&Policy> {
self.policy.as_ref()
}
pub fn router(&self) -> &dyn Router {
self.router.as_ref()
}
pub fn recorder(&self) -> Option<&dyn Recorder> {
self.recorder.as_deref()
}
pub fn explorer(&self) -> Explorer {
self.explorer
}
pub fn site_memory(&self) -> &SiteMemoryStore {
&self.memory
}
pub fn provider_router(&self) -> Option<&StoredRouter> {
self.provider_router.as_deref()
}
}
#[derive(Debug, Clone)]
pub struct RunBudget(Arc<RunAccount>);
#[derive(Debug)]
struct RunAccount {
cap: crate::Credits,
committed: std::sync::atomic::AtomicU64,
spent: std::sync::atomic::AtomicU64,
attempts: std::sync::atomic::AtomicU64,
unknown: std::sync::atomic::AtomicU64,
}
#[derive(Debug, Clone, Copy)]
pub struct RunSpend {
pub attempts: u64,
pub spent: crate::Credits,
pub unknown: u64,
}
impl RunBudget {
pub fn new(cap: crate::Credits) -> Self {
use std::sync::atomic::AtomicU64;
Self(Arc::new(RunAccount {
cap: crate::Credits(if cap.get().is_finite() {
cap.get().max(0.0)
} else {
0.0
}),
committed: AtomicU64::new(0),
spent: AtomicU64::new(0),
attempts: AtomicU64::new(0),
unknown: AtomicU64::new(0),
}))
}
pub fn remaining(&self) -> crate::Credits {
use std::sync::atomic::Ordering::SeqCst;
crate::Credits((self.0.cap.get() - f64::from_bits(self.0.committed.load(SeqCst))).max(0.0))
}
pub fn snapshot(&self) -> RunSpend {
use std::sync::atomic::Ordering::SeqCst;
RunSpend {
attempts: self.0.attempts.load(SeqCst),
spent: crate::Credits(f64::from_bits(self.0.spent.load(SeqCst))),
unknown: self.0.unknown.load(SeqCst),
}
}
pub(crate) fn reserve(&self, estimate: crate::Credits) -> Option<Reservation> {
use std::sync::atomic::Ordering::SeqCst;
let amount = Budget::floor(estimate).get();
let previous = self
.0
.committed
.fetch_update(SeqCst, SeqCst, |bits| {
let next = f64::from_bits(bits) + amount;
(next <= self.0.cap.get()).then_some(next.to_bits())
})
.ok()?;
self.0.attempts.fetch_add(1, SeqCst);
self.0.unknown.fetch_add(1, SeqCst);
Some(Reservation {
run: self.clone(),
estimate: amount,
allowance: crate::Credits((self.0.cap.get() - f64::from_bits(previous)).max(0.0)),
})
}
}
#[cfg(test)]
mod run_budget_tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use crate::Credits;
#[test]
fn f2_reservations_exclude_other_calls_and_settle_the_bill() {
let run = RunBudget::new(Credits(1.0));
let held = run.reserve(Credits(0.75)).unwrap();
assert!(run.reserve(Credits(0.5)).is_none());
assert_eq!(run.snapshot().attempts, 1);
assert!(held.settle(Credits(0.25), false).is_none());
assert_eq!(run.remaining(), Credits(0.75));
let held = run.reserve(Credits(0.5)).unwrap();
let overrun = held.settle(Credits(1.0), false).unwrap();
assert_eq!(overrun.scope, crate::response::BudgetScope::Run);
assert_eq!(overrun.spent, Credits(1.25));
assert_eq!(run.remaining(), Credits::ZERO);
}
#[test]
fn f2_unknown_and_cancelled_calls_keep_their_reservations() {
let run = RunBudget::new(Credits(1.0));
let held = run.reserve(Credits(0.5)).unwrap();
let _ = held.settle(Credits(0.25), true);
{
let _cancelled = run.reserve(Credits(0.5)).unwrap();
}
assert_eq!(run.remaining(), Credits::ZERO);
assert_eq!(run.snapshot().spent, Credits(0.25));
assert_eq!(run.snapshot().unknown, 2);
assert!(run.reserve(Credits(0.1)).is_none());
}
#[test]
fn f2_concurrent_bills_report_a_run_overrun() {
let run = RunBudget::new(Credits(1.0));
let a = run.reserve(Credits(0.1)).unwrap();
let b = run.reserve(Credits(0.1)).unwrap();
assert!(a.settle(Credits(0.6), false).is_none());
let overrun = b.settle(Credits(0.6), false).unwrap();
assert_eq!(overrun.spent, Credits(1.2));
}
#[test]
fn f2_atomic_admission_has_one_winner_per_available_slot() {
let run = RunBudget::new(Credits(1.0));
let admitted = std::thread::scope(|scope| {
let workers: Vec<_> = (0..32)
.map(|_| scope.spawn(|| run.reserve(Credits(0.25))))
.collect();
workers
.into_iter()
.filter_map(|worker| worker.join().unwrap())
.count()
});
assert_eq!(admitted, 4);
assert_eq!(run.snapshot().attempts, 4);
assert_eq!(run.remaining(), Credits::ZERO);
}
}
pub(crate) struct Reservation {
pub(crate) allowance: crate::Credits,
run: RunBudget,
estimate: f64,
}
impl Reservation {
pub(crate) fn settle(
self,
cost: crate::Credits,
unknown: bool,
) -> Option<crate::response::BudgetOverrun> {
use std::sync::atomic::Ordering::SeqCst;
let cost = cost.get().max(0.0);
let retained = if unknown {
self.estimate.max(cost)
} else {
cost
};
atomic_add(&self.run.0.committed, retained - self.estimate);
atomic_add(&self.run.0.spent, cost);
if !unknown {
self.run.0.unknown.fetch_sub(1, SeqCst);
}
let spent = self.run.snapshot().spent;
(spent > self.run.0.cap).then_some(crate::response::BudgetOverrun {
scope: crate::response::BudgetScope::Run,
cap: self.run.0.cap,
spent,
})
}
}
fn atomic_add(value: &std::sync::atomic::AtomicU64, amount: f64) {
use std::sync::atomic::Ordering::SeqCst;
let _ = value.fetch_update(SeqCst, SeqCst, |bits| {
Some((f64::from_bits(bits) + amount).max(0.0).to_bits())
});
}
#[derive(Default)]
pub struct SpiderBuilder {
key: Option<String>,
base_url: Option<Url>,
budget: Option<Budget>,
run_budget: Option<RunBudget>,
jitter_seed: Option<u64>,
policy: Option<Policy>,
http_client: Option<reqwest::Client>,
router: Option<Arc<dyn Router>>,
recorder: Option<Arc<dyn Recorder>>,
explorer: Explorer,
memory_capacity: Option<usize>,
allow_insecure_http: bool,
without_read_wall: bool,
response_limit: Option<usize>,
provider_router: Option<StoredRouter>,
stored_router: bool,
keyless: bool,
#[cfg(feature = "optimize")]
pub(crate) optimize: crate::optimize::Hooks,
}
impl fmt::Debug for SpiderBuilder {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let mut out = f.debug_struct("SpiderBuilder");
out.field("key", &self.key.as_ref().map(|_| REDACTED))
.field("base_url", &self.base_url.as_ref().map(Url::as_str))
.field("budget", &self.budget)
.field("policy", &PolicyPresence(self.policy.is_some()))
.field("http_client", &self.http_client.as_ref().map(|_| "set"))
.field("router", &PolicyPresence(self.router.is_some()))
.field("recorder", &PolicyPresence(self.recorder.is_some()))
.field("explore", &self.explorer.rate)
.field("provider_router", &self.provider_router)
.field("stored_router", &self.stored_router)
.field("keyless", &self.keyless);
#[cfg(feature = "optimize")]
out.field("optimize", &self.optimize);
out.finish()
}
}
struct PolicyPresence(bool);
impl fmt::Debug for PolicyPresence {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(if self.0 { "set" } else { "default" })
}
}
impl SpiderBuilder {
pub fn new() -> SpiderBuilder {
SpiderBuilder::default()
}
pub fn key(mut self, key: impl Into<String>) -> SpiderBuilder {
self.key = Some(key.into());
self
}
pub fn base_url(mut self, base_url: Url) -> SpiderBuilder {
self.base_url = Some(base_url);
self
}
pub fn allow_insecure_http(mut self, allow: bool) -> SpiderBuilder {
self.allow_insecure_http = allow;
self
}
pub fn without_wall(mut self) -> SpiderBuilder {
self.budget = Some(self.budget.unwrap_or_default().without_wall());
self.without_read_wall = true;
self
}
pub fn budget(mut self, budget: Budget) -> SpiderBuilder {
self.budget = Some(budget);
self.without_read_wall = false;
self
}
pub fn run_budget(mut self, budget: RunBudget) -> SpiderBuilder {
self.run_budget = Some(budget);
self
}
pub fn jitter_seed(mut self, seed: u64) -> SpiderBuilder {
self.jitter_seed = Some(seed);
self
}
pub fn policy(mut self, policy: Policy) -> SpiderBuilder {
self.policy = Some(policy);
self
}
pub fn http_client(mut self, client: reqwest::Client) -> SpiderBuilder {
self.http_client = Some(client);
self
}
pub fn max_response_bytes(mut self, bytes: usize) -> SpiderBuilder {
self.response_limit = Some(bytes);
self
}
pub fn router(mut self, router: impl Router + 'static) -> SpiderBuilder {
self.router = Some(Arc::new(router));
self
}
pub fn recorder(mut self, recorder: impl Recorder + 'static) -> SpiderBuilder {
self.recorder = Some(Arc::new(recorder));
self
}
pub fn explore(mut self, rate: f32) -> SpiderBuilder {
self.explorer.rate = rate.clamp(0.0, 1.0);
self
}
pub fn explore_seed(mut self, seed: u64) -> SpiderBuilder {
self.explorer.seed = seed;
self
}
pub fn site_memory_capacity(mut self, sites: usize) -> SpiderBuilder {
self.memory_capacity = Some(sites);
self
}
pub fn provider_router(mut self, router: StoredRouter) -> SpiderBuilder {
self.provider_router = Some(router);
self
}
pub fn stored_router(mut self, on: bool) -> SpiderBuilder {
self.stored_router = on;
self
}
pub fn keyless(mut self, on: bool) -> SpiderBuilder {
self.keyless = on;
self
}
pub fn build(self) -> Result<Spider> {
let (key, without_key) = match self.key {
Some(key) => (key, false),
None => match resolve_key() {
Some(key) => (key, false),
None if self.keyless => (String::new(), true),
None => return Err(no_key()),
},
};
if !without_key && key.trim().is_empty() {
return Err(Error::Auth {
cause: AuthCause::EmptyKey,
message: "the api key is empty".to_string(),
});
}
let base = match self.base_url {
Some(base) => base,
None => Transport::base_url_from_env_options(self.allow_insecure_http)?,
};
crate::transport::validate_base(&base, self.allow_insecure_http)?;
let transport = match self.http_client {
Some(client) => Transport::with_client(key, base, client),
None => Transport::with_base_options(key, base, self.allow_insecure_http)?,
};
let transport = transport.allow_insecure_http(self.allow_insecure_http);
let memory = match self.memory_capacity {
Some(capacity) => SiteMemoryStore::new(capacity),
None => SiteMemoryStore::default(),
};
let provider_router = match self.provider_router {
Some(router) => Some(router),
None if self.stored_router => crate::auth::router::load()?,
None => None,
};
if let Some(router) = &provider_router {
router.validate()?;
}
Ok(Spider {
transport: Arc::new(transport),
budget: self.budget.unwrap_or_default(),
run_budget: self.run_budget,
jitter_seed: self.jitter_seed.unwrap_or_else(client_seed),
read_wall: if self.without_read_wall {
None
} else {
let cap = crate::ops::data::DEFAULT_READ_WALL;
Some(
self.budget
.and_then(|budget| budget.wall)
.unwrap_or(cap)
.min(cap),
)
},
response_limit: self.response_limit,
policy: self.policy,
router: self
.router
.unwrap_or_else(|| Arc::new(HeuristicRouter::new())),
recorder: self.recorder,
explorer: self.explorer,
memory: Arc::new(memory),
provider_router: provider_router.map(Arc::new),
#[cfg(feature = "optimize")]
optimize: self.optimize,
})
}
}
fn client_seed() -> u64 {
use std::hash::{BuildHasher, Hasher};
std::collections::hash_map::RandomState::new()
.build_hasher()
.finish()
}
fn resolve_key() -> Option<String> {
Credentials::resolve().map(Credentials::into_key)
}
fn no_key() -> Error {
Error::Auth {
cause: AuthCause::NoKey,
message: format!(
"no api key. Set {API_KEY_ENV}, set {API_KEY_ENV_ALT}, or sign in so the key is written to ~/{CREDENTIALS_PATH}"
),
}
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::indexing_slicing,
clippy::string_slice
)]
use super::*;
const SECRET: &str = "sk-live-not-a-real-key-0123456789";
#[test]
fn a_client_never_prints_its_key() {
let spider = Spider::with_key(SECRET).expect("a client");
let printed = format!("{spider:?}");
assert!(!printed.contains(SECRET), "{printed}");
assert!(printed.contains(REDACTED));
let raw = format!("{:?}", spider.raw());
assert!(!raw.contains(SECRET), "{raw}");
}
#[test]
fn a_builder_never_prints_its_key() {
let builder = SpiderBuilder::new().key(SECRET);
let printed = format!("{builder:?}");
assert!(!printed.contains(SECRET), "{printed}");
assert!(printed.contains(REDACTED));
}
#[test]
fn an_address_that_is_not_a_url_fails_the_build() {
let built = SpiderBuilder::new()
.key(SECRET)
.base_url(Url::parse("https://api.spider.cloud").expect("a url"));
assert!(built.build().is_ok());
}
#[test]
fn a_missing_key_names_the_places_it_looked_and_no_values() {
let message = no_key().to_string();
assert!(message.contains("SPIDER_API_KEY"));
assert!(message.contains(".spider/credentials"));
}
#[test]
fn an_address_can_be_named_on_the_builder() {
let spider = SpiderBuilder::new()
.key(SECRET)
.base_url(Url::parse("https://spider.cloud/api/").expect("a url"))
.build()
.expect("a client");
assert_eq!(
spider.raw().base_url().as_str(),
"https://spider.cloud/api/"
);
}
fn every_error(spider: &Spider) -> Vec<Error> {
use crate::error::BudgetKind;
use crate::policy::StopReason;
use crate::status::ApiStatus;
use crate::transport::{RateLimit, Reply};
use bytes::Bytes;
use std::time::Duration;
let reply = |code: u16, body: &str| Reply {
status: ApiStatus::new(code),
rate_limit: RateLimit::default(),
retry_after: Some(Duration::from_secs(3)),
elapsed: Duration::from_millis(12),
content_type: Some("application/json".into()),
body: Bytes::from(body.to_string()),
};
let mut out = Vec::new();
out.extend(reply(500, r#"{"error":"something gave way"}"#).as_error());
out.extend(reply(429, r#"{"error":"slow down"}"#).as_error());
out.extend(reply(401, r#"{"error":"invalid api key"}"#).as_error());
out.extend(reply(402, r#"{"error":"balance is zero"}"#).as_error());
out.extend(
reply(
402,
r#"{"error":"This route requires an account.","code":"free_tier_route_blocked"}"#,
)
.as_error(),
);
out.push(
reply(200, "not json")
.json::<serde_json::Value>()
.expect_err("a decode error"),
);
out.push(
futures_now(spider.scrape("not a url").send_all()).expect_err("a configuration error"),
);
out.push(Error::BudgetExceeded {
kind: BudgetKind::Credits,
attempts: Vec::new(),
});
out.push(Error::Exhausted {
attempts: Vec::new(),
last: None,
reason: StopReason::LadderExhausted,
source: None,
});
out.push(Error::Exhausted {
attempts: Vec::new(),
last: None,
reason: StopReason::RetriesExhausted,
source: reply(429, r#"{"error":"slow down"}"#)
.as_error()
.map(Box::new),
});
out.push(Error::InsufficientCredits);
out.push(no_key());
out.extend(SpiderBuilder::new().key(" ").build().err());
out.extend(Credentials::store(" ").err());
out.push(Error::Auth {
cause: AuthCause::SignInFailed,
message: "the browser did not come back".to_string(),
});
out.push(Error::Auth {
cause: AuthCause::Local,
message: "no home directory to write to".to_string(),
});
out
}
fn futures_now<T>(future: impl std::future::Future<Output = T>) -> T {
use std::task::{Context, Poll, Waker};
let waker = Waker::noop();
let mut cx = Context::from_waker(waker);
let mut future = Box::pin(future);
match future.as_mut().poll(&mut cx) {
Poll::Ready(value) => value,
Poll::Pending => panic!("this path was expected to finish before a call was made"),
}
}
#[test]
fn no_error_the_crate_makes_carries_the_key() {
let spider = Spider::with_key(SECRET).expect("a client");
let errors = every_error(&spider);
assert_eq!(errors.len(), 16, "an error path stopped being covered");
let causes: Vec<AuthCause> = errors
.iter()
.filter_map(|error| match error {
Error::Auth { cause, .. } => Some(*cause),
_ => None,
})
.collect();
for cause in [
AuthCause::NoKey,
AuthCause::EmptyKey,
AuthCause::Refused,
AuthCause::SignInFailed,
AuthCause::Local,
AuthCause::SignInRequired,
] {
assert!(causes.contains(&cause), "no error path makes {cause:?}");
}
for error in errors {
let debug = format!("{error:?}");
let display = format!("{error}");
assert!(!debug.contains(SECRET), "key in Debug: {debug}");
assert!(!display.contains(SECRET), "key in Display: {display}");
assert!(!debug.contains("Bearer"), "authorization header in {debug}");
}
}
#[test]
fn a_budget_set_on_the_builder_reaches_the_client() {
let spider = SpiderBuilder::new()
.key(SECRET)
.budget(Budget::default().with_attempts(2))
.build()
.expect("a client");
assert_eq!(spider.budget().attempts, 2);
}
#[test]
fn f1_response_cap_is_off_unless_asked() {
let builder = || Spider::builder().key(SECRET);
assert_eq!(builder().build().unwrap().response_limit, None);
assert_eq!(
builder()
.max_response_bytes(4096)
.build()
.unwrap()
.response_limit,
Some(4096)
);
}
#[test]
fn f1_account_read_wall_requires_an_explicit_opt_out() {
let builder = || Spider::builder().key(SECRET);
let minute = Some(std::time::Duration::from_secs(60));
for budget in [Budget::unlimited(), Budget::default().with_attempts(1)] {
assert_eq!(builder().budget(budget).build().unwrap().read_wall, minute);
}
assert_eq!(builder().build().unwrap().read_wall, minute);
assert_eq!(builder().without_wall().build().unwrap().read_wall, None);
let short = std::time::Duration::from_millis(20);
assert_eq!(
builder()
.budget(Budget::default().with_wall(short))
.build()
.unwrap()
.read_wall,
Some(short)
);
assert_eq!(
builder()
.without_wall()
.budget(Budget::default())
.build()
.unwrap()
.read_wall,
minute
);
}
#[test]
fn a_stored_router_reaches_the_client_only_when_asked_for() {
use crate::params::Router as RouterParam;
let plain = SpiderBuilder::new().key(SECRET).build().expect("a client");
assert!(plain.provider_router().is_none());
let stored = StoredRouter {
router: RouterParam {
mode: Some("fallback".to_string()),
..RouterParam::default()
},
provider_options: None,
};
let spider = SpiderBuilder::new()
.key(SECRET)
.provider_router(stored.clone())
.build()
.expect("a client");
assert_eq!(spider.provider_router(), Some(&stored));
assert!(format!("{spider:?}").contains("provider_router: set"));
let mut sideways = stored;
sideways.router.mode = Some("sideways".to_string());
let refused = SpiderBuilder::new()
.key(SECRET)
.provider_router(sideways)
.build();
assert!(matches!(refused, Err(Error::Config(_))), "{refused:?}");
}
#[test]
fn no_error_or_debug_output_carries_a_provider_token() {
use crate::params::{RequestParams, Router as RouterParam};
use std::collections::BTreeMap;
const TOKEN: &str = "synthetic-provider-token";
const PASSWORD: &str = "synthetic-provider-password";
let stored = StoredRouter {
router: RouterParam {
mode: Some("first".to_string()),
provider: Some("oxylabs".to_string()),
token: Some(TOKEN.to_string()),
credentials: Some(BTreeMap::from([
("oxylabs_username".to_string(), "synthetic-name".to_string()),
("oxylabs_password".to_string(), PASSWORD.to_string()),
])),
funding: Some("own".to_string()),
},
provider_options: Some(BTreeMap::from([(
"oxylabs".to_string(),
serde_json::json!({ "geo_location": TOKEN }),
)])),
};
let builder = SpiderBuilder::new()
.key(SECRET)
.provider_router(stored.clone());
let mut printed = vec![
format!("{stored:?}"),
format!("{stored:#?}"),
format!("{builder:?}"),
];
let spider = builder.build().expect("a client");
let mut params = RequestParams::url("https://example.com");
let caller = params.clone();
crate::ops::apply_stored_router(spider.provider_router(), &mut params, &caller);
assert_eq!(params.router.as_ref(), Some(&stored.router));
printed.push(format!("{spider:?}"));
printed.push(format!("{params:?}"));
printed.push(format!("{:?}", spider.provider_router()));
printed.push(format!("{:?}", spider.scrape("https://example.com")));
let dir = std::env::temp_dir().join(format!(
"spider-client-router-{}-{}",
std::process::id(),
client_seed()
));
std::fs::create_dir_all(&dir).expect("a directory");
let mut errors = crate::auth::router::tests::every_error(&dir);
let mut blank = stored.clone();
blank.router.token = Some(" ".to_string());
errors.extend(
SpiderBuilder::new()
.key(SECRET)
.provider_router(blank)
.build()
.err(),
);
let _ = std::fs::remove_dir_all(&dir);
assert_eq!(errors.len(), 19, "an error path stopped being covered");
for error in &errors {
printed.push(format!("{error:?}"));
printed.push(format!("{error}"));
}
for one in printed {
assert!(!one.contains(TOKEN), "provider token in {one}");
assert!(!one.contains(PASSWORD), "provider credential in {one}");
}
}
#[test]
fn no_builder_prints_the_key() {
let spider = Spider::with_key(SECRET).expect("a client");
let printed = [
format!("{:?}", spider.scrape("https://example.com")),
format!("{:?}", spider.fetch("example.com", "/")),
format!("{:?}", spider.crawl("https://example.com")),
format!("{:?}", spider.links("https://example.com")),
format!("{:?}", spider.search("a query")),
format!("{:?}", spider.screenshot("https://example.com")),
format!("{:?}", spider.transform(Vec::new())),
format!("{:?}", spider.crawl_logs()),
format!("{:?}", spider.table("pages")),
];
for one in printed {
assert!(!one.contains(SECRET), "key in Debug: {one}");
assert!(!one.contains("Bearer"), "authorization header in {one}");
}
}
#[test]
fn no_builder_prints_caller_credentials() {
let spider = Spider::with_key(SECRET).expect("a client");
let cookie = "session=synthetic-cookie-value";
let authorization = "Bearer synthetic-header-value";
let password = "synthetic-proxy-password";
let proxy = format!("http://user:{password}@example.com:8080");
macro_rules! check {
($builder:expr) => {{
let mut builder = $builder;
let params = builder.params_mut();
params.cookies = Some(cookie.into());
params.headers = Some(
[
("Authorization".into(), authorization.into()),
("X-Custom".into(), password.into()),
]
.into(),
);
params.remote_proxy = Some(proxy.clone());
let wire = serde_json::to_value(&*params).expect("serialize");
assert!(wire["cookies"] == cookie);
assert!(wire["headers"]["Authorization"] == authorization);
assert!(wire["remote_proxy"] == proxy);
let direct = format!("{params:?}");
for printed in [direct, format!("{builder:?}"), format!("{builder:#?}")] {
for secret in [SECRET, cookie, authorization, password] {
assert!(!printed.contains(secret), "caller credential in Debug");
}
assert!(printed.contains("Authorization"));
assert!(printed.contains("X-Custom"));
assert!(printed.contains("<redacted>"));
}
}};
}
check!(spider.scrape("https://example.com"));
check!(spider.crawl("https://example.com"));
check!(spider.fetch("example.com", "/"));
check!(spider.links("https://example.com"));
check!(spider.search("a query"));
check!(spider.screenshot("https://example.com"));
check!(spider.transform(Vec::new()));
}
#[test]
fn builder_debug_redacts_copies_of_url_and_document_credentials() {
let spider = Spider::with_key(SECRET).expect("a client");
let secret = "synthetic-url-password";
let url = format!("https://user:{secret}@example.com/?token={secret}");
let document = crate::ops::transform::Document::html(secret).from_url(&url);
for printed in [
format!("{:?}", spider.scrape(&url)),
format!(
"{:?}",
spider.fetch(format!("user:{secret}@example.com"), secret)
),
format!("{:?}", spider.search(&url)),
format!("{document:?}"),
format!("{:?}", spider.transform(vec![document])),
] {
assert!(!printed.contains(secret), "credential in builder Debug");
}
}
#[tokio::test]
async fn a_failed_call_reports_nothing_about_the_key() {
let client = reqwest::Client::builder()
.connect_timeout(std::time::Duration::from_millis(40))
.timeout(std::time::Duration::from_millis(80))
.build()
.expect("a client");
let spider = SpiderBuilder::new()
.key(SECRET)
.base_url(Url::parse("https://192.0.2.1:9/").expect("a url"))
.budget(Budget::default().with_attempts(1))
.http_client(client)
.build()
.expect("a client");
let error = spider
.scrape("https://example.com")
.send()
.await
.expect_err("nothing answers at that address");
let printed = format!("{error:?} {error}");
assert!(!printed.contains(SECRET), "{printed}");
}
#[test]
fn addresses_come_from_several_shapes() {
let url = Url::parse("https://example.com/a").expect("a url");
assert!("https://example.com/a".into_url().is_ok());
assert!(String::from("https://example.com/a").into_url().is_ok());
assert!((&url).into_url().is_ok());
assert!(url.into_url().is_ok());
assert!("not a url".into_url().is_err());
}
#[test]
fn an_empty_key_is_refused_before_a_call_is_made() {
let built = SpiderBuilder::new().key(" ").build();
assert!(matches!(
built,
Err(Error::Auth {
cause: AuthCause::EmptyKey,
..
})
));
}
}