pub mod crawl;
pub mod data;
pub mod fetch;
pub mod links;
pub mod scrape;
pub mod screenshot;
pub mod search;
pub mod transform;
use std::future::Future;
use std::time::{Duration, Instant};
use serde::Serialize;
use url::Url;
use crate::auth::router::StoredRouter;
use crate::client::{IntoUrl, Spider};
use crate::credits::Credits;
use crate::error::{BudgetKind, Error};
use crate::params::{RequestParams, ReturnFormat, ReturnFormatHandling};
use crate::policy::engine::Reached;
use crate::policy::{AttemptState, Budget, Next, Observed, Policy, Step, StopReason};
use crate::response::{Attempt, Body, FailedPage, Hint, Outcome, Page, PageResult, Pages};
use crate::routing;
use crate::status::{ApiStatus, PageStatus};
use crate::thrift::plan::Endpoint;
use crate::thrift::tokens::{approx_tokens, TokenBudget};
use crate::thrift::trim::Trimmer;
use crate::thrift::{Need, Plan, ThriftReport};
use crate::transport::{route, Route};
use crate::Result;
use spider_route::featurize;
use tokio::time::{sleep, timeout_at, Instant as DeadlineInstant};
const SERVICE_URL: &str = "https://spider.cloud";
#[derive(Clone, Copy)]
pub(crate) struct Deadline(Option<DeadlineInstant>);
impl Deadline {
pub(crate) fn new(wall: Option<Duration>) -> Result<Self> {
let end = wall
.map(|wall| {
DeadlineInstant::now()
.checked_add(wall)
.ok_or_else(|| Error::Config("wall is too large for the clock".into()))
})
.transpose()?;
Ok(Self(end))
}
pub(crate) fn expired(self) -> bool {
self.0.is_some_and(|end| DeadlineInstant::now() >= end)
}
fn allows_wait(self, after: Duration) -> bool {
self.0
.is_none_or(|end| after < end.saturating_duration_since(DeadlineInstant::now()))
}
}
pub(crate) async fn within<T, F>(deadline: Deadline, call: F) -> Option<Result<T>>
where
F: Future<Output = Result<T>>,
{
if deadline.expired() {
return None;
}
match deadline.0 {
Some(end) => timeout_at(end, call).await.ok(),
None => Some(call.await),
}
}
pub(crate) struct Call<'a> {
pub(crate) spider: &'a Spider,
pub(crate) params: crate::params::RequestParams,
pub(crate) budget: Budget,
pub(crate) need: Option<Need>,
pub(crate) max_tokens: Option<usize>,
url: Option<Url>,
url_error: Option<String>,
}
impl std::fmt::Debug for Call<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Call")
.field("spider", &self.spider)
.field("params", &self.params)
.field("budget", &self.budget)
.field("need", &self.need.as_ref().map(|_| "<redacted>"))
.field("max_tokens", &self.max_tokens)
.field("url", &self.url.as_ref().map(|_| "<redacted>"))
.field("url_error", &self.url_error.as_ref().map(|_| "<redacted>"))
.finish_non_exhaustive()
}
}
impl<'a> Call<'a> {
fn admit(
&mut self,
attempts: &[Attempt],
estimate: Credits,
) -> Result<Option<crate::client::Reservation>> {
let spent: Credits = attempts
.iter()
.map(|a| Credits(a.cost.get().max(a.reserved.get())))
.sum();
self.budget
.preflight(attempts.len().min(255) as u8, spent, estimate)
.map_err(|kind| {
let error = Error::BudgetExceeded {
kind,
attempts: attempts.to_vec(),
};
match self.spider.run_budget() {
Some(run) => error.accounted(attempts.to_vec(), Some(run.snapshot())),
None => error,
}
})?;
match self.spider.run_budget() {
None => Ok(None),
Some(run) => run
.reserve(estimate)
.map(|reservation| {
let cap = spent + reservation.allowance;
self.budget.credits = Some(
self.budget
.credits
.map_or(cap, |old| Credits(old.get().min(cap.get()))),
);
Some(reservation)
})
.ok_or_else(|| {
Error::BudgetExceeded {
kind: BudgetKind::Credits,
attempts: attempts.to_vec(),
}
.accounted(attempts.to_vec(), Some(run.snapshot()))
}),
}
}
fn cap_for_run(&mut self) {
if let Some(run) = self.spider.run_budget() {
let left = run.remaining();
self.budget.credits = Some(
self.budget
.credits
.map_or(left, |cap| Credits(cap.get().min(left.get()))),
);
}
}
fn call_failure(&self, error: Error, attempts: Vec<Attempt>) -> Error {
error.accounted(
attempts,
self.spider.run_budget().map(crate::RunBudget::snapshot),
)
}
pub(crate) fn new(spider: &'a Spider, url: impl IntoUrl) -> Call<'a> {
let mut call = Call::bare(spider);
match url.into_url() {
Ok(url) => {
call.params.url = Some(url.to_string());
call.url = Some(url);
}
Err(e) => call.url_error = Some(format!("that is not an address: {e}")),
}
call
}
pub(crate) fn bare(spider: &'a Spider) -> Call<'a> {
Call {
spider,
params: crate::params::RequestParams::default(),
budget: spider.budget(),
need: None,
max_tokens: None,
url: None,
url_error: None,
}
}
fn target(&self) -> Result<Url> {
if let Some(message) = &self.url_error {
return Err(Error::Config(message.clone()));
}
match &self.url {
Some(url) => Ok(url.clone()),
None => Url::parse(SERVICE_URL).map_err(|e| {
Error::Config(format!(
"the built in address {SERVICE_URL} does not parse: {e}"
))
}),
}
}
fn format(&self) -> Option<ReturnFormat> {
match self.params.return_format.as_ref()? {
ReturnFormatHandling::Single(format) => Some(*format),
ReturnFormatHandling::Multi(formats) if formats.len() == 1 => formats.first().copied(),
ReturnFormatHandling::Multi(_) => None,
}
}
pub(crate) async fn run<B, F>(&mut self, route: Route, make_body: F) -> Result<Outcome<Pages>>
where
B: Serialize,
F: Fn(&crate::params::RequestParams) -> B,
{
self.run_at(route, &[], make_body).await
}
pub(crate) async fn run_at<B, F>(
&mut self,
route: Route,
args: &[&str],
make_body: F,
) -> Result<Outcome<Pages>>
where
B: Serialize,
F: Fn(&crate::params::RequestParams) -> B,
{
self.run_at_inner(route, args, make_body)
.await
.map_err(|error| match self.spider.run_budget() {
Some(run) => error.with_run(run.snapshot()),
None => error,
})
}
async fn run_at_inner<B, F>(
&mut self,
route: Route,
args: &[&str],
make_body: F,
) -> Result<Outcome<Pages>>
where
B: Serialize,
F: Fn(&crate::params::RequestParams) -> B,
{
self.cap_for_run();
let deadline = Deadline::new(self.budget.wall)?;
let target = self.target()?;
let policy = match self.spider.policy() {
Some(policy) => policy.clone(),
None => {
let mut policy = Policy::for_target(target.as_str());
policy.backoff = crate::policy::Backoff::seeded(self.spider.jitter_seed);
policy
}
};
let plan = self.plan();
let caller = self.params.clone();
let remembered = self.spider.site_memory().get(&target);
let input = routing::input(
&target,
routing::declared_need(self.need.as_ref()),
routing::pins_from(&caller),
remembered.as_ref(),
);
let picked = self.spider.router().route(&input);
let decision = if input.pins.any() {
picked
} else {
self.spider
.explorer()
.choose(&target, &picked, &self.budget)
.unwrap_or(picked)
};
routing::apply_decision(&decision, &mut self.params, &caller);
plan.apply_over(&mut self.params, &caller);
apply_stored_router(self.spider.provider_router(), &mut self.params, &caller);
#[cfg(feature = "optimize")]
let optimized = crate::optimize::decide(self, &input, &caller, &decision);
let format = self.format();
let mut state = AttemptState::new(self.budget);
state.step = usize::from(decision.start_rung);
let features = self.spider.recorder().map(|_| featurize(&input));
let mut routed_attempt = true;
let mut attempts: Vec<Attempt> = Vec::new();
let mut pages = Pages::default();
let mut pages_came_back = false;
let mut last_failed: Option<FailedPage> = None;
let mut call_error: Option<Error>;
let current = endpoint_for(&plan, route);
let mut wire_bytes = 0usize;
let mut estimate = Budget::floor(Credits::ZERO);
let mut run_overrun = None;
let mut page_overrun = None;
loop {
let mut attempt_bytes = 0u32;
if deadline.expired() {
return Err(out_of_time(attempts));
}
if attempts.len() >= usize::from(policy.max_attempts) {
return Err(Error::BudgetExceeded {
kind: BudgetKind::Attempts,
attempts,
});
}
let reservation = self.admit(&attempts, estimate)?;
state.budget = self.budget;
let mut wire_budget = self.budget;
wire_budget.credits = self.budget.remaining_credits(state.spent);
wire_budget.apply(&mut self.params);
let body = make_body(&self.params);
let before = Instant::now();
let sent = within(
deadline,
self.spider
.raw()
.post_with_limit(current, args, &body, self.spider.response_limit),
)
.await;
let mut wall_ran_out = false;
let mut charge_unknown = false;
let observed = match sent {
None => {
charge_unknown = true;
call_error = None;
pages = Pages::default();
wall_ran_out = true;
Observed::timed_out().taking(before.elapsed())
}
Some(Ok(reply)) => {
wire_bytes = reply.body.len();
attempt_bytes = wire_bytes.min(u32::MAX as usize) as u32;
let elapsed = reply.elapsed;
let status = reply.status;
let wait = reply.retry_after.or(reply.rate_limit.reset);
if reply.is_success() || reply.is_mirrored_page_status() {
charge_unknown = !reply.body.is_empty()
&& serde_json::from_slice::<serde_json::Value>(&reply.body)
.ok()
.as_ref()
.is_none_or(|body| !has_charge(body));
call_error = None;
pages = match reply.read_pages(&target, format, current == route::TRANSFORM)
{
Ok(pages) => pages,
Err(error) => {
let body =
serde_json::from_slice::<serde_json::Value>(&reply.body).ok();
let cost = body.as_ref().map(charged).unwrap_or(Credits::ZERO);
let mut attempt = Attempt::new(elapsed, status, None, cost);
attempt.charge_unknown =
body.as_ref().is_none_or(|body| !has_charge(body));
if let Some(reservation) = reservation {
let _ = reservation.settle(cost, attempt.charge_unknown);
}
attempts.push(attempt);
return Err(self.call_failure(error, attempts));
}
};
pages.retain_requested(&self.params);
if !pages.is_empty() {
pages_came_back = true;
}
if let Some(cap) = self.budget.per_page_credits {
if let Some(page) = pages.0.iter().find(|page| page.cost() > cap) {
page_overrun = Some(crate::response::BudgetOverrun {
scope: crate::response::BudgetScope::Page,
cap,
spent: page.cost(),
});
}
}
if let Some(failed) = pages.failed().next() {
last_failed = Some(failed.clone());
}
let api = reply.operation_status();
let page = representative(&pages);
let mut observed = Observed::seen(0, None);
observed.api = Reached::Api(api);
observed.page = if current == route::TRANSFORM {
page.filter(|s| !s.is_unknown())
} else {
page
};
let mut observed = observed.costing(pages.total_cost()).taking(elapsed);
if nothing_came_back(&pages) && !body_was_declined(&self.params) {
observed = observed.blank();
}
match wait {
Some(after) => observed.retry_after(after),
None => observed,
}
} else {
call_error = reply.as_error();
pages = Pages::default();
let cost = serde_json::from_slice::<serde_json::Value>(&reply.body)
.ok()
.as_ref()
.map(charged)
.unwrap_or(Credits::ZERO);
let observed = Observed::seen(status.code(), None)
.taking(elapsed)
.costing(cost);
match wait {
Some(after) => observed.retry_after(after),
None => observed,
}
}
}
Some(Err(Error::Transport(e))) if e.is_timeout() => {
charge_unknown = true;
call_error = Some(Error::Transport(e));
Observed::timed_out().taking(before.elapsed())
}
Some(Err(Error::Transport(e))) if e.is_connect() => {
call_error = Some(Error::Transport(e));
Observed::connect_failed().taking(before.elapsed())
}
Some(Err(Error::ResponseTooLarge { limit, status })) => {
let mut attempt = Attempt::new(before.elapsed(), status, None, Credits::ZERO);
attempt.charge_unknown = true;
attempts.push(attempt);
return Err(Error::Exhausted {
attempts,
last: last_failed.map(Box::new),
reason: StopReason::Unhandled,
source: Some(Box::new(Error::ResponseTooLarge { limit, status })),
});
}
Some(Err(other)) => {
let mut attempt =
Attempt::new(before.elapsed(), ApiStatus::new(0), None, Credits::ZERO);
attempt.charge_unknown =
matches!(&other, Error::Transport(e) if !e.is_connect() && !e.is_builder());
if let Some(reservation) = reservation {
let _ = reservation.settle(attempt.cost, attempt.charge_unknown);
}
attempts.push(attempt);
return Err(self.call_failure(other, attempts));
}
}
.for_request(&self.params);
state.record(&observed);
if let Some(reservation) = reservation {
run_overrun = reservation
.settle(observed.cost, charge_unknown)
.or(run_overrun);
}
attempts.push(Attempt::new(
observed.elapsed,
api_status_of(&observed),
observed.page,
observed.cost,
));
if let Some(attempt) = attempts.last_mut() {
attempt.charge_unknown = charge_unknown;
if charge_unknown {
attempt.reserved = Credits(estimate.get().max(attempt.cost.get()));
state.spent += Credits((estimate.get() - attempt.cost.get()).max(0.0));
}
}
let next = if wall_ran_out || deadline.expired() {
Next::Stop(StopReason::Budget(BudgetKind::Time))
} else {
policy.decide(&observed, &state)
};
let result = routing::outcome_of(&observed, next == Next::Accept, attempt_bytes);
if routing::describes_the_site(&observed) {
self.spider.site_memory().observe(&target, &result);
}
self.spider.router().observe(&input, &result);
#[cfg(feature = "optimize")]
crate::optimize::compare(self, optimized.as_ref(), &next, &result, &attempts, &pages);
if routed_attempt {
if let (Some(recorder), Some(features)) = (self.spider.recorder(), &features) {
recorder.observe(features, &decision, &result);
}
routed_attempt = false;
}
state.follow(&next);
if deadline.expired() {
return Err(out_of_time(attempts));
}
match next {
Next::Accept => {
let report = self.settle(&mut pages, wire_bytes);
if deadline.expired() {
return Err(out_of_time(attempts));
}
let mut outcome = Outcome::new(pages, attempts)
.with_cap(self.budget.credits)
.reporting(report)
.routed(decision);
outcome.overrun = run_overrun.or(outcome.overrun).or(page_overrun);
return Ok(outcome);
}
Next::Retry { after } => {
if !deadline.allows_wait(after) {
return Err(out_of_time(attempts));
}
estimate = Budget::floor(if observed.was_billed() {
state.last_cost
} else {
Credits::ZERO
});
if within(deadline, async {
sleep(after).await;
Ok(())
})
.await
.is_none()
{
return Err(out_of_time(attempts));
}
}
Next::Escalate { step, after, .. } => {
if !deadline.allows_wait(after) {
return Err(out_of_time(attempts));
}
estimate = step.estimate(if observed.was_billed() {
state.last_cost
} else {
Credits::ZERO
});
if within(deadline, async {
sleep(after).await;
Ok(())
})
.await
.is_none()
{
return Err(out_of_time(attempts));
}
escalate(&step, &mut self.params, &plan, &caller);
}
Next::Stop(reason) => {
return Err(stopped(
reason,
attempts,
pages_came_back,
last_failed,
call_error,
));
}
}
}
}
pub(crate) fn plan(&self) -> Plan {
match &self.need {
Some(need) => Plan::for_need(need),
None => Plan::none(),
}
}
fn settle(&self, pages: &mut Pages, wire_bytes: usize) -> ThriftReport {
let bodies: Vec<String> = pages
.0
.iter()
.map(|result| match result {
PageResult::Ok(page) => page.text().unwrap_or_default().to_string(),
PageResult::Failed(_) => String::new(),
})
.collect();
let tokens_in: usize = bodies.iter().map(|body| approx_tokens(body)).sum();
if self.need.is_none() && self.max_tokens.is_none() {
return ThriftReport::untrimmed(wire_bytes, tokens_in);
}
let mut trimmer = match &self.need {
Some(need) => Trimmer::for_need(need),
None => Trimmer::default(),
};
trimmer.learn(&bodies);
let budget = match self.max_tokens {
Some(total) => TokenBudget::total(total),
None => TokenBudget::unlimited(),
};
let first_pass: Vec<String> = bodies
.iter()
.map(|body| trimmer.trim(body, None).text)
.collect();
let costs: Vec<usize> = first_pass.iter().map(|text| approx_tokens(text)).collect();
let shares = budget.split(&costs);
let mut reasons = Vec::new();
let mut returned_bytes = 0usize;
let mut tokens_out = 0usize;
for ((result, before), ceiling) in pages.0.iter_mut().zip(&first_pass).zip(shares) {
let PageResult::Ok(page) = result else {
continue;
};
let trimmed = trimmer.trim(before, budget.total.map(|_| ceiling));
returned_bytes += trimmed.text.len();
tokens_out += approx_tokens(&trimmed.text);
reasons.extend(trimmed.reasons);
replace_text(&mut page.body, trimmed.text);
let aside = payload_beside_the_text(page);
returned_bytes += aside.0;
tokens_out += aside.1;
}
ThriftReport {
wire_bytes,
returned_bytes,
approx_tokens_in: tokens_in,
approx_tokens_out: tokens_out,
trimmed: reasons,
}
}
pub(crate) async fn run_json<B, T, F>(
&mut self,
route: Route,
make_body: F,
) -> Result<Outcome<T>>
where
B: Serialize,
T: serde::de::DeserializeOwned,
F: Fn(&crate::params::RequestParams) -> B,
{
self.run_json_inner(route, make_body)
.await
.map_err(|error| match self.spider.run_budget() {
Some(run) => error.with_run(run.snapshot()),
None => error,
})
}
async fn run_json_inner<B, T, F>(&mut self, route: Route, make_body: F) -> Result<Outcome<T>>
where
B: Serialize,
T: serde::de::DeserializeOwned,
F: Fn(&crate::params::RequestParams) -> B,
{
self.cap_for_run();
let deadline = Deadline::new(self.budget.wall)?;
if deadline.expired() {
return Err(out_of_time(Vec::new()));
}
let reservation = self.admit(&[], Budget::floor(Credits::ZERO))?;
self.budget.apply(&mut self.params);
let body = make_body(&self.params);
let before = Instant::now();
let reply = match within(
deadline,
self.spider
.raw()
.post_with_limit(route, &[], &body, self.spider.response_limit),
)
.await
.unwrap_or_else(|| {
let mut attempt =
Attempt::new(before.elapsed(), ApiStatus::new(0), None, Credits::ZERO);
attempt.charge_unknown = true;
Err(out_of_time(vec![attempt]))
}) {
Ok(reply) => reply,
Err(Error::ResponseTooLarge { limit, status }) => {
return Err(Error::Exhausted {
attempts: vec![Attempt {
charge_unknown: true,
..Attempt::new(before.elapsed(), status, None, Credits::ZERO)
}],
last: None,
reason: StopReason::Unhandled,
source: Some(Box::new(Error::ResponseTooLarge { limit, status })),
});
}
Err(other) => {
if matches!(other, Error::BudgetExceeded { .. }) {
return Err(other);
}
let mut attempt =
Attempt::new(before.elapsed(), ApiStatus::new(0), None, Credits::ZERO);
attempt.charge_unknown =
matches!(&other, Error::Transport(e) if !e.is_connect() && !e.is_builder());
if let Some(reservation) = reservation {
let _ = reservation.settle(attempt.cost, attempt.charge_unknown);
}
return Err(self.call_failure(other, vec![attempt]));
}
};
let body = reply.json::<serde_json::Value>();
let cost = body.as_ref().map(charged).unwrap_or(Credits::ZERO);
let mut attempt = Attempt::new(reply.elapsed, reply.status, None, cost);
attempt.charge_unknown =
reply.is_success() && body.as_ref().map_or(true, |body| !has_charge(body));
let run_overrun =
reservation.and_then(|reservation| reservation.settle(cost, attempt.charge_unknown));
if let Some(error) = reply.as_error() {
return Err(self.call_failure(error, vec![attempt]));
}
let body = body.map_err(|error| self.call_failure(error, vec![attempt.clone()]))?;
let value: T = serde_json::from_value(body)
.map_err(|error| self.call_failure(Error::Decode(error), vec![attempt.clone()]))?;
if deadline.expired() {
return Err(out_of_time(vec![attempt]));
}
let mut outcome = Outcome::new(value, vec![attempt]).with_cap(self.budget.credits);
outcome.overrun = run_overrun.or(outcome.overrun);
Ok(outcome)
}
}
fn has_charge(body: &serde_json::Value) -> bool {
match body {
serde_json::Value::Array(items) => !items.is_empty() && items.iter().all(has_charge),
other => reported_charge(other).is_some(),
}
}
fn reported_charge(body: &serde_json::Value) -> Option<Credits> {
let total = body.get("costs")?.get("total_cost")?;
serde_json::from_value::<crate::credits::Usd>(total.clone())
.ok()
.map(Credits::from)
}
fn charged(body: &serde_json::Value) -> Credits {
match body {
serde_json::Value::Array(items) => items.iter().map(charged).sum(),
other => reported_charge(other).unwrap_or(Credits::ZERO),
}
}
fn escalate(step: &Step, params: &mut RequestParams, plan: &Plan, caller: &RequestParams) {
step.apply(params);
if let Some(mode) = caller.request {
params.request = Some(mode);
}
plan.apply_over(params, caller);
}
pub(crate) fn apply_stored_router(
store: Option<&StoredRouter>,
params: &mut RequestParams,
caller: &RequestParams,
) {
let Some(store) = store else { return };
if caller.router.is_none() {
params.router = Some(store.router.clone());
}
if caller.provider_options.is_none() && store.provider_options.is_some() {
params.provider_options.clone_from(&store.provider_options);
}
}
fn endpoint_for(plan: &Plan, requested: Route) -> Route {
if requested != route::SCRAPE {
return requested;
}
match plan.endpoint {
Some(Endpoint::Links) => route::LINKS,
Some(Endpoint::Screenshot) => route::SCREENSHOT,
Some(Endpoint::Scrape) | None => requested,
}
}
fn replace_text(body: &mut Body, text: String) {
match body {
Body::WithFields { content, .. } => replace_text(content, text),
Body::Text(held) | Body::Markdown(held) | Body::Html(held) | Body::Xml(held) => {
*held = text
}
_ => {}
}
}
fn api_status_of(observed: &Observed) -> ApiStatus {
match observed.api {
Reached::Api(status) => status,
Reached::TimedOut | Reached::ConnectFailed => ApiStatus::new(0),
}
}
fn representative(pages: &Pages) -> Option<PageStatus> {
pages
.first_ok()
.map(|page| page.status)
.or_else(|| pages.0.first().map(|result| result.status()))
}
fn payload_beside_the_text(page: &Page) -> (usize, usize) {
let mut bytes = 0usize;
if let Some(fields) = page.body.fields() {
if let Ok(encoded) = serde_json::to_string(fields) {
bytes += encoded.len();
}
}
if let Some(meta) = &page.metadata {
if let Ok(encoded) = serde_json::to_string(meta) {
bytes += encoded.len();
}
}
for value in [
&page.json_data,
&page.request_map,
&page.response_map,
&page.trace,
]
.into_iter()
.flatten()
{
if let Ok(encoded) = serde_json::to_string(value) {
bytes += encoded.len();
}
}
for value in [&page.headers, &page.cookies].into_iter().flatten() {
if let Ok(encoded) = serde_json::to_string(value) {
bytes += encoded.len();
}
}
if let Some(links) = &page.links {
bytes += links.iter().map(|link| link.as_str().len()).sum::<usize>();
}
(bytes, bytes.div_ceil(4))
}
fn body_was_declined(params: &RequestParams) -> bool {
match params.return_format.as_ref() {
Some(ReturnFormatHandling::Single(format)) => *format == ReturnFormat::Empty,
Some(ReturnFormatHandling::Multi(formats)) => {
!formats.is_empty() && formats.iter().all(|f| *f == ReturnFormat::Empty)
}
None => false,
}
}
fn nothing_came_back(pages: &Pages) -> bool {
if pages.is_empty() {
return true;
}
let mut served = pages.ok().peekable();
served.peek().is_some() && pages.ok().all(Page::is_blank)
}
pub(crate) fn out_of_time(attempts: Vec<Attempt>) -> Error {
Error::BudgetExceeded {
kind: BudgetKind::Time,
attempts,
}
}
fn stopped(
reason: StopReason,
attempts: Vec<Attempt>,
pages_came_back: bool,
last_failed: Option<FailedPage>,
call_error: Option<Error>,
) -> Error {
match reason {
StopReason::OutOfCredits => Error::InsufficientCredits.accounted(attempts, None),
StopReason::Budget(kind) => Error::BudgetExceeded { kind, attempts },
_ => match call_error {
Some(error) if !pages_came_back => error.accounted(attempts, None),
source => Error::Exhausted {
attempts,
last: last_failed.map(Box::new),
reason,
source: source.map(Box::new),
},
},
}
}
pub(crate) fn first_page(outcome: Outcome<Pages>) -> Result<Outcome<Page>> {
let Outcome {
overrun,
value,
attempts,
cost,
thrift,
route,
} = outcome;
let last = value.failed().next().cloned();
match value.into_ok().into_iter().next() {
Some(page) => Ok(Outcome {
overrun,
value: page,
attempts,
cost,
thrift,
route,
}),
None => Err(Error::Exhausted {
attempts,
reason: StopReason::Rejected {
hint: last.as_ref().map_or(Hint::Permanent, |failed| failed.hint),
},
last: last.map(Box::new),
source: None,
}),
}
}
const MAX_TIMEOUT_SECS: u64 = u8::MAX as u64;
pub(crate) fn timeout_secs(timeout: Duration) -> u8 {
timeout.as_secs().clamp(1, MAX_TIMEOUT_SECS) as u8
}
macro_rules! curated_surface {
($builder:ident, page_links) => {
curated_surface!($builder);
impl<'a> $builder<'a> {
pub fn page_links(mut self, on: bool) -> Self {
self.call.params.return_page_links = Some(on);
self
}
}
};
($builder:ident) => {
impl<'a> $builder<'a> {
pub fn mode(mut self, mode: $crate::params::RequestMode) -> Self {
self.call.params.request = Some(mode);
self
}
pub fn proxy(mut self, pool: $crate::params::ProxyPool) -> Self {
self.call.params.proxy = Some(pool);
self
}
pub fn country(mut self, country: $crate::params::Country) -> Self {
self.call.params.country_code = Some(country);
self
}
pub fn wait_for(mut self, wait: $crate::params::WaitFor) -> Self {
self.call.params.wait_for = Some(wait);
self
}
pub fn profile(mut self, profile: $crate::params::Profile) -> Self {
self.call.params.apply_profile(profile);
self
}
pub fn timeout(mut self, timeout: std::time::Duration) -> Self {
self.call.params.request_timeout = Some($crate::ops::timeout_secs(timeout));
self
}
pub fn session(mut self, on: bool) -> Self {
self.call.params.session = Some(on);
self
}
pub fn budget(mut self, budget: $crate::policy::Budget) -> Self {
self.call.budget = budget;
self
}
pub fn need(mut self, need: $crate::thrift::Need) -> Self {
self.call.need = Some(need);
self
}
pub fn max_tokens(mut self, tokens: usize) -> Self {
self.call.max_tokens = Some(tokens);
self
}
pub fn params_mut(&mut self) -> &mut $crate::params::RequestParams {
&mut self.call.params
}
}
};
}
pub(crate) use curated_surface;
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::indexing_slicing,
clippy::string_slice
)]
use super::*;
use crate::params::ReturnFormat;
use crate::policy::Ladder;
use crate::thrift::Need;
#[test]
fn curated_surface_membership_is_explicit() {
let source = include_str!("mod.rs");
let surface = source
.split_once("macro_rules! curated_surface {")
.unwrap()
.1
.split_once("pub(crate) use curated_surface;")
.unwrap()
.0;
let (page_links, common) = surface.split_once("($builder:ident) =>").unwrap();
let methods = |arm: &str| -> Vec<String> {
let code = arm
.lines()
.map(|line| line.split_once("//").map_or(line, |(code, _)| code))
.collect::<Vec<_>>()
.join("\n");
let tokens: Vec<_> = code
.split(|c: char| !c.is_alphanumeric() && c != '_')
.filter(|token| !token.is_empty())
.collect();
tokens
.windows(2)
.filter(|pair| pair[0] == "fn")
.map(|pair| pair[1].to_owned())
.collect()
};
assert_eq!(methods(page_links), ["page_links"]);
let expected = [
"mode",
"proxy",
"country",
"wait_for",
"profile",
"timeout",
"session",
"budget",
"need",
"max_tokens",
"params_mut",
];
assert_eq!(methods(common), expected);
let vocabulary = include_str!("../../../docs/action-vocabulary.md");
for name in expected.into_iter().chain(["page_links"]) {
assert!(
vocabulary
.lines()
.any(|line| line.starts_with(&format!("| `{name}` |"))),
"{name} needs a line in docs/action-vocabulary.md"
);
}
}
#[test]
fn f1_default_wall_reaches_page_and_search_calls() {
let spider = Spider::builder()
.key("not-a-real-key")
.base_url(Url::parse("https://example.com").unwrap())
.build()
.unwrap();
for call in [
Call::new(&spider, "https://example.com"),
Call::bare(&spider),
] {
assert_eq!(call.budget.wall, Some(Duration::from_secs(900)));
assert!(Deadline::new(call.budget.wall).unwrap().0.is_some());
}
}
#[test]
fn f2_a_malformed_breakdown_does_not_erase_a_readable_total() {
let body = serde_json::json!({
"costs": { "total_cost": 0.0002, "compute_cost": {} },
"status": "not a status"
});
assert!(has_charge(&body));
assert_eq!(charged(&body), Credits(2.0));
assert!(!has_charge(&serde_json::json!({"costs": {}})));
}
#[test]
fn a_caller_that_declined_a_body_is_not_treated_as_a_blank_page() {
let mut params = RequestParams {
return_format: Some(ReturnFormatHandling::Single(ReturnFormat::Empty)),
..RequestParams::default()
};
assert!(body_was_declined(¶ms));
params.return_format = Some(ReturnFormatHandling::Multi(vec![ReturnFormat::Empty]));
assert!(body_was_declined(¶ms));
}
#[test]
fn a_caller_that_wanted_a_body_still_notices_a_blank_page() {
let mut params = RequestParams::default();
assert!(!body_was_declined(¶ms));
params.return_format = Some(ReturnFormatHandling::Single(ReturnFormat::Markdown));
assert!(!body_was_declined(¶ms));
params.return_format = Some(ReturnFormatHandling::Multi(vec![
ReturnFormat::Markdown,
ReturnFormat::Empty,
]));
assert!(!body_was_declined(¶ms));
params.return_format = Some(ReturnFormatHandling::Multi(vec![]));
assert!(!body_was_declined(¶ms));
}
#[test]
fn a_step_up_the_ladder_keeps_the_plan_the_need_asked_for() {
let caller = RequestParams::url("https://example.com");
let plan = Plan::for_need(&Need::Markdown);
let mut params = caller.clone();
plan.apply_over(&mut params, &caller);
for step in Ladder::standard().0 {
params.return_format = Some(ReturnFormat::Raw.into());
params.metadata = Some(true);
escalate(&step, &mut params, &plan, &caller);
assert_eq!(
params.return_format,
Some(ReturnFormat::Markdown.into()),
"the {} step left a fat return format behind",
step.label
);
assert_eq!(params.metadata, Some(false), "after {}", step.label);
assert!(params.request.is_some(), "after {}", step.label);
}
}
#[test]
fn a_step_never_argues_with_a_setting_the_caller_made() {
let mut caller = RequestParams::url("https://example.com");
caller.return_format = Some(ReturnFormat::Raw.into());
let plan = Plan::for_need(&Need::Markdown);
let mut params = caller.clone();
for step in Ladder::standard().0 {
escalate(&step, &mut params, &plan, &caller);
}
assert_eq!(params.return_format, Some(ReturnFormat::Raw.into()));
}
#[test]
fn only_a_page_request_is_moved_to_another_endpoint() {
let links = Plan::for_need(&Need::Links);
assert_eq!(endpoint_for(&links, route::SCRAPE), route::LINKS);
assert_eq!(endpoint_for(&links, route::CRAWL), route::CRAWL);
let shot = Plan::for_need(&Need::screenshot());
assert_eq!(endpoint_for(&shot, route::SCRAPE), route::SCREENSHOT);
let markdown = Plan::for_need(&Need::Markdown);
assert_eq!(endpoint_for(&markdown, route::SCRAPE), route::SCRAPE);
assert_eq!(endpoint_for(&Plan::none(), route::SCRAPE), route::SCRAPE);
}
#[test]
fn trimmed_text_goes_back_in_the_shape_it_came_out_of() {
let mut body = Body::Markdown("# one".into());
replace_text(&mut body, "# two".into());
assert_eq!(body, Body::Markdown("# two".into()));
let mut shot = Body::Screenshot(bytes::Bytes::from_static(b"png"));
replace_text(&mut shot, "nonsense".into());
assert_eq!(shot, Body::Screenshot(bytes::Bytes::from_static(b"png")));
}
#[tokio::test(start_paused = true)]
async fn f2_wait_admission_uses_time_spent_outside_the_policy() {
let deadline = Deadline::new(Some(Duration::from_millis(600))).unwrap();
let wait = Duration::from_millis(500);
assert!(deadline.allows_wait(wait));
tokio::time::advance(Duration::from_millis(200)).await;
assert!(!deadline.allows_wait(wait));
assert!(!deadline.allows_wait(Duration::from_millis(400)));
assert!(deadline.allows_wait(Duration::from_millis(399)));
tokio::time::advance(Duration::from_millis(400)).await;
assert!(!deadline.allows_wait(Duration::ZERO));
assert!(Deadline::new(None).unwrap().allows_wait(Duration::MAX));
}
fn stored(mode: &str) -> StoredRouter {
StoredRouter {
router: crate::params::Router {
mode: Some(mode.to_string()),
provider: Some("zyte".to_string()),
funding: Some("own".to_string()),
..crate::params::Router::default()
},
provider_options: Some(
[("zyte".to_string(), serde_json::json!({"geolocation": "US"}))].into(),
),
}
}
#[test]
fn a_stored_router_fills_in_what_the_caller_left_alone() {
let store = stored("fallback");
let caller = RequestParams::url("https://example.com");
let mut params = caller.clone();
apply_stored_router(Some(&store), &mut params, &caller);
assert_eq!(params.router, Some(store.router.clone()));
assert_eq!(params.provider_options, store.provider_options);
let wire = serde_json::to_value(¶ms).unwrap();
assert_eq!(wire["router"]["mode"], "fallback");
assert_eq!(wire["provider_options"]["zyte"]["geolocation"], "US");
let mut params = caller.clone();
apply_stored_router(None, &mut params, &caller);
assert_eq!(params, caller);
}
#[test]
fn a_stored_router_never_argues_with_the_caller() {
let store = stored("fallback");
let mut caller = RequestParams::url("https://example.com");
caller.router = Some(crate::params::Router {
mode: Some("off".to_string()),
..crate::params::Router::default()
});
caller.provider_options =
Some([("zyte".to_string(), serde_json::json!({"geolocation": "DE"}))].into());
let mut params = caller.clone();
apply_stored_router(Some(&store), &mut params, &caller);
assert_eq!(params, caller);
caller.provider_options = None;
let mut params = caller.clone();
apply_stored_router(Some(&store), &mut params, &caller);
assert_eq!(params.router, caller.router);
assert_eq!(params.provider_options, store.provider_options);
let caller = RequestParams::url("https://example.com");
let mut params = caller.clone();
params.router = Some(crate::params::Router::default());
apply_stored_router(Some(&store), &mut params, &caller);
assert_eq!(params.router, Some(store.router));
}
}