use crate::response::content::Body;
use crate::response::costs::{Costs, Credits};
use crate::response::metadata::Metadata;
use crate::status::{PageClass, PageStatus};
use std::collections::BTreeMap;
use std::time::Duration;
use url::Url;
#[derive(Debug, Clone)]
pub(crate) struct PageParts {
pub url: Url,
pub status: PageStatus,
pub body: Body,
pub duration: Option<Duration>,
pub call_elapsed: Duration,
pub json_data: Option<serde_json::Value>,
pub request_map: Option<serde_json::Value>,
pub response_map: Option<serde_json::Value>,
pub trace: Option<serde_json::Value>,
pub costs: Costs,
pub metadata: Option<Metadata>,
pub links: Option<Vec<Url>>,
pub headers: Option<BTreeMap<String, String>>,
pub cookies: Option<BTreeMap<String, String>>,
pub error: Option<String>,
}
#[cfg(test)]
#[allow(clippy::expect_used)]
impl PageParts {
pub(crate) fn for_test(url: &str, code: u16) -> Self {
PageParts {
url: Url::parse(url).expect("a test url"),
status: PageStatus::new(code),
body: Body::Text("hello".into()),
duration: Some(Duration::from_millis(80)),
call_elapsed: Duration::from_millis(120),
json_data: None,
request_map: None,
response_map: None,
trace: None,
costs: Costs {
total_cost: Credits::new(4.0).into(),
..Costs::default()
},
metadata: None,
links: None,
headers: None,
cookies: None,
error: None,
}
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct Page {
pub url: Url,
pub status: PageStatus,
pub body: Body,
pub duration: Option<Duration>,
pub call_elapsed: Duration,
pub json_data: Option<serde_json::Value>,
pub request_map: Option<serde_json::Value>,
pub response_map: Option<serde_json::Value>,
pub trace: Option<serde_json::Value>,
pub error: Option<String>,
pub costs: Costs,
pub metadata: Option<Metadata>,
pub links: Option<Vec<Url>>,
pub headers: Option<BTreeMap<String, String>>,
pub cookies: Option<BTreeMap<String, String>>,
}
impl Page {
#[allow(clippy::result_large_err)]
pub(crate) fn try_from_parts(parts: PageParts) -> Result<Page, FailedPage> {
if !parts.status.is_ok() {
return Err(FailedPage::from_parts(parts));
}
Ok(Self::document_from_parts(parts))
}
pub(crate) fn document_from_parts(parts: PageParts) -> Page {
Page {
url: parts.url,
status: parts.status,
body: parts.body,
duration: parts.duration,
call_elapsed: parts.call_elapsed,
json_data: parts.json_data,
request_map: parts.request_map,
response_map: parts.response_map,
trace: parts.trace,
error: parts.error,
costs: parts.costs,
metadata: parts.metadata,
links: parts.links,
headers: parts.headers,
cookies: parts.cookies,
}
}
pub fn cost(&self) -> Credits {
self.costs.total()
}
pub fn text(&self) -> Option<&str> {
self.body.as_str()
}
pub fn is_blank(&self) -> bool {
self.body.is_empty()
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct FailedPage {
pub url: Url,
pub status: PageStatus,
pub error: Option<String>,
pub costs: Costs,
pub hint: Hint,
pub body: Body,
pub duration: Option<Duration>,
pub call_elapsed: Duration,
pub json_data: Option<serde_json::Value>,
pub request_map: Option<serde_json::Value>,
pub response_map: Option<serde_json::Value>,
pub trace: Option<serde_json::Value>,
pub metadata: Option<Metadata>,
pub links: Option<Vec<Url>>,
pub headers: Option<BTreeMap<String, String>>,
pub cookies: Option<BTreeMap<String, String>>,
}
impl FailedPage {
fn from_parts(parts: PageParts) -> FailedPage {
FailedPage {
url: parts.url,
hint: Hint::for_class(parts.status.class()),
status: parts.status,
error: parts.error,
costs: parts.costs,
body: parts.body,
duration: parts.duration,
call_elapsed: parts.call_elapsed,
json_data: parts.json_data,
request_map: parts.request_map,
response_map: parts.response_map,
trace: parts.trace,
metadata: parts.metadata,
links: parts.links,
headers: parts.headers,
cookies: parts.cookies,
}
}
pub fn text(&self) -> Option<&str> {
self.body.as_str()
}
pub fn cost(&self) -> Credits {
self.costs.total()
}
pub fn was_billed(&self) -> bool {
self.status.consumes_credits()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum Hint {
TryBrowser,
TryResidentialProxy,
TryDifferentCountry,
NeedsSession,
Permanent,
SlowDown {
after: Duration,
},
}
impl Hint {
pub fn for_class(class: PageClass) -> Hint {
match class {
PageClass::Ok => Hint::TryBrowser,
PageClass::Blocked => Hint::TryResidentialProxy,
PageClass::NeedsLogin => Hint::NeedsSession,
PageClass::TargetRateLimited => Hint::SlowDown {
after: DEFAULT_SLOW_DOWN,
},
PageClass::BadRequest | PageClass::NotFound => Hint::Permanent,
PageClass::ServerError => Hint::TryBrowser,
_ => Hint::TryBrowser,
}
}
pub fn is_actionable(self) -> bool {
!matches!(self, Hint::Permanent)
}
}
const DEFAULT_SLOW_DOWN: Duration = Duration::from_secs(5);
#[allow(clippy::large_enum_variant)]
#[derive(Debug, Clone)]
pub enum PageResult {
Ok(Page),
Failed(FailedPage),
}
impl PageResult {
pub(crate) fn from_parts(parts: PageParts) -> PageResult {
match Page::try_from_parts(parts) {
Ok(page) => PageResult::Ok(page),
Err(failed) => PageResult::Failed(failed),
}
}
pub fn is_ok(&self) -> bool {
matches!(self, PageResult::Ok(_))
}
pub fn ok(&self) -> Option<&Page> {
match self {
PageResult::Ok(page) => Some(page),
PageResult::Failed(_) => None,
}
}
pub fn failed(&self) -> Option<&FailedPage> {
match self {
PageResult::Failed(failed) => Some(failed),
PageResult::Ok(_) => None,
}
}
pub fn into_ok(self) -> Option<Page> {
match self {
PageResult::Ok(page) => Some(page),
PageResult::Failed(_) => None,
}
}
pub fn status(&self) -> PageStatus {
match self {
PageResult::Ok(page) => page.status,
PageResult::Failed(failed) => failed.status,
}
}
pub fn url(&self) -> &Url {
match self {
PageResult::Ok(page) => &page.url,
PageResult::Failed(failed) => &failed.url,
}
}
pub fn cost(&self) -> Credits {
match self {
PageResult::Ok(page) => page.cost(),
PageResult::Failed(failed) => failed.cost(),
}
}
pub fn hint(&self) -> Option<Hint> {
self.failed().map(|f| f.hint)
}
}
#[derive(Debug, Clone, Default)]
pub struct Pages(pub Vec<PageResult>);
impl Pages {
pub(crate) fn retain_requested(&mut self, params: &crate::params::RequestParams) {
macro_rules! retain {
($page:expr) => {{
let page = $page;
if params.metadata == Some(false) {
page.metadata = None;
}
if params.return_headers == Some(false) {
page.headers = None;
}
if params.return_cookies == Some(false) {
page.cookies = None;
}
if params.return_page_links == Some(false) {
page.links = None;
}
if params.return_json_data == Some(false) {
page.json_data = None;
}
if params.return_embeddings == Some(false) {
if let Some(meta) = &mut page.metadata {
meta.embedding = None;
for value in meta.extra.values_mut() {
if let Some(object) = value.as_object_mut() {
object.remove("embedding");
}
}
}
}
let tracker = params.event_tracker.unwrap_or_default();
if tracker.requests != Some(true) {
page.request_map = None;
}
if tracker.responses != Some(true) {
page.response_map = None;
}
if tracker.automation != Some(true) {
page.trace = None;
}
let no_content = params.return_format.as_ref().is_some_and(|f| {
f.formats()
.iter()
.all(|f| *f == crate::params::ReturnFormat::Empty)
});
let fields = params.css_extraction_map.is_some();
page.body = match std::mem::take(&mut page.body) {
Body::WithFields {
content,
fields: values,
} => match (no_content, fields) {
(true, true) => Body::Fields(values),
(true, false) => Body::Empty,
(false, true) => Body::WithFields {
content,
fields: values,
},
(false, false) => *content,
},
Body::Fields(_) if !fields => Body::Empty,
body @ Body::Fields(_) => body,
_ if no_content => Body::Empty,
body => body,
};
}};
}
for result in &mut self.0 {
match result {
PageResult::Ok(page) => retain!(page),
PageResult::Failed(page) => retain!(page),
}
}
}
pub fn ok(&self) -> impl Iterator<Item = &Page> {
self.0.iter().filter_map(PageResult::ok)
}
pub fn failed(&self) -> impl Iterator<Item = &FailedPage> {
self.0.iter().filter_map(PageResult::failed)
}
pub fn into_ok(self) -> Vec<Page> {
self.0.into_iter().filter_map(PageResult::into_ok).collect()
}
pub fn first_ok(&self) -> Option<&Page> {
self.ok().next()
}
pub fn links(&self) -> Vec<Url> {
let mut out: Vec<Url> = Vec::new();
for page in self.ok() {
for link in page.links.iter().flatten() {
if !out.contains(link) {
out.push(link.clone());
}
}
}
out
}
pub fn total_cost(&self) -> Credits {
self.0.iter().map(PageResult::cost).sum()
}
pub fn len(&self) -> usize {
self.0.len()
}
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
}
impl From<Vec<PageResult>> for Pages {
fn from(results: Vec<PageResult>) -> Pages {
Pages(results)
}
}
impl IntoIterator for Pages {
type Item = PageResult;
type IntoIter = std::vec::IntoIter<PageResult>;
fn into_iter(self) -> Self::IntoIter {
self.0.into_iter()
}
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::indexing_slicing,
clippy::string_slice
)]
use super::*;
#[test]
fn effective_request_filters_both_successes_and_refusals() {
use crate::params::RequestParams;
use crate::thrift::{Need, Plan};
for code in [200, 403] {
let mut parts = PageParts::for_test("https://example.com", code);
parts.json_data = Some(serde_json::json!({"name": "Example"}));
parts.request_map = Some(serde_json::json!({}));
parts.response_map = Some(serde_json::json!({}));
parts.trace = Some(serde_json::json!([]));
parts.metadata = Some(Metadata {
embedding: Some(vec![0.1]),
..Metadata::default()
});
parts.headers = Some(BTreeMap::new());
parts.cookies = Some(BTreeMap::new());
parts.links = Some(Vec::new());
parts.body = Body::WithFields {
content: Box::new(Body::Markdown("# refused".into())),
fields: BTreeMap::from([("title".into(), serde_json::json!("refused"))]),
};
let mut pages = Pages(vec![PageResult::from_parts(parts)]);
let mut params = RequestParams::default();
Plan::for_need(&Need::Markdown).apply_over(&mut params, &RequestParams::default());
pages.retain_requested(¶ms);
macro_rules! check {
($p:expr) => {{
let p = $p;
assert_eq!(p.text(), Some("# refused"));
assert!(p.body.fields().is_none());
assert_eq!(p.duration, Some(Duration::from_millis(80)));
assert_eq!(p.call_elapsed, Duration::from_millis(120));
assert!(
p.metadata.is_none()
&& p.headers.is_none()
&& p.cookies.is_none()
&& p.links.is_none()
);
assert!(
p.json_data.is_none()
&& p.request_map.is_none()
&& p.response_map.is_none()
&& p.trace.is_none()
);
}};
}
match &pages.0[0] {
PageResult::Ok(p) => check!(p),
PageResult::Failed(p) => check!(p),
}
}
}
#[test]
fn a_non_2xx_status_cannot_produce_a_page() {
for code in [400u16, 401, 403, 404, 429, 500, 502, 503, 526, 301, 999] {
let parts = PageParts::for_test("https://example.com/a", code);
let built = Page::try_from_parts(parts.clone());
assert!(built.is_err(), "code {code} produced a page");
let result = PageResult::from_parts(parts);
assert!(!result.is_ok(), "code {code} produced a page");
assert!(result.ok().is_none());
assert!(result.into_ok().is_none());
}
}
#[test]
fn a_2xx_status_produces_a_page() {
for code in [200u16, 201, 204, 299] {
let result = PageResult::from_parts(PageParts::for_test("https://example.com/a", code));
let page = result.ok().expect("a page");
assert_eq!(page.status.code(), code);
assert!(page.status.is_ok());
}
}
#[test]
fn a_failure_keeps_the_cost_it_burned() {
let result = PageResult::from_parts(PageParts::for_test("https://example.com/a", 403));
let failed = result.failed().expect("a failure");
assert_eq!(failed.cost(), Credits(4.0));
assert!(failed.was_billed());
let result = PageResult::from_parts(PageParts::for_test("https://example.com/a", 503));
assert!(!result.failed().expect("a failure").was_billed());
}
#[test]
fn hints_follow_the_class() {
let cases = [
(403u16, Hint::TryResidentialProxy),
(401, Hint::NeedsSession),
(404, Hint::Permanent),
(400, Hint::Permanent),
(
429,
Hint::SlowDown {
after: DEFAULT_SLOW_DOWN,
},
),
(500, Hint::TryBrowser),
(526, Hint::TryBrowser),
];
for (code, hint) in cases {
let result = PageResult::from_parts(PageParts::for_test("https://example.com/a", code));
assert_eq!(result.hint(), Some(hint), "code {code}");
}
assert!(!Hint::Permanent.is_actionable());
assert!(Hint::TryResidentialProxy.is_actionable());
assert_eq!(Hint::for_class(PageClass::Ok), Hint::TryBrowser);
}
#[test]
fn a_set_reports_what_worked_and_what_it_all_cost() {
let pages = Pages(vec![
PageResult::from_parts(PageParts::for_test("https://example.com/a", 200)),
PageResult::from_parts(PageParts::for_test("https://example.com/b", 403)),
PageResult::from_parts(PageParts::for_test("https://example.com/c", 200)),
]);
assert_eq!(pages.len(), 3);
assert_eq!(pages.ok().count(), 2);
assert_eq!(pages.failed().count(), 1);
assert_eq!(
pages.first_ok().expect("a page").url.as_str(),
"https://example.com/a"
);
assert_eq!(pages.total_cost(), Credits(12.0));
assert_eq!(pages.into_ok().len(), 2);
}
#[test]
fn a_page_carries_what_the_request_asked_for() {
let mut parts = PageParts::for_test("https://example.com/a", 200);
parts.links = Some(vec![Url::parse("https://example.com/b").expect("a url")]);
parts.metadata = Some(Metadata {
title: Some("Example".into()),
..Metadata::default()
});
parts.headers = Some(BTreeMap::from([(
"content-type".to_string(),
"text/html".to_string(),
)]));
parts.cookies = Some(BTreeMap::from([("a".into(), "1".into())]));
let page = PageResult::from_parts(parts).into_ok().expect("a page");
assert_eq!(page.links.as_ref().expect("links").len(), 1);
assert_eq!(page.headers.as_ref().expect("headers").len(), 1);
assert_eq!(
page.cookies
.as_ref()
.and_then(|cookies| cookies.get("a"))
.map(String::as_str),
Some("1")
);
assert_eq!(page.text(), Some("hello"));
assert!(!page.is_blank());
assert_eq!(page.cost(), Credits(4.0));
}
#[test]
fn a_2xx_with_nothing_in_it_is_still_a_page_but_reads_as_blank() {
let mut parts = PageParts::for_test("https://example.com/a", 200);
parts.body = Body::Empty;
let page = PageResult::from_parts(parts).into_ok().expect("a page");
assert!(page.is_blank());
}
}