use crate::params::{CssExtractionMap, RequestParams, ReturnFormat};
use crate::thrift::need::Need;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum Endpoint {
Scrape,
Links,
Screenshot,
}
pub const OPTIONAL_RETURNS: &[&str] = &[
"metadata",
"return_headers",
"return_cookies",
"return_page_links",
"return_json_data",
"return_embeddings",
];
#[derive(Debug, Clone, Default, PartialEq)]
#[non_exhaustive]
pub struct Plan {
pub endpoint: Option<Endpoint>,
pub return_format: Option<ReturnFormat>,
pub readability: Option<bool>,
pub clean_html: Option<bool>,
pub metadata: Option<bool>,
pub return_headers: Option<bool>,
pub return_cookies: Option<bool>,
pub return_page_links: Option<bool>,
pub return_json_data: Option<bool>,
pub return_embeddings: Option<bool>,
pub css_extraction_map: Option<CssExtractionMap>,
}
macro_rules! settle {
($plan:expr, $params:expr, $caller:expr, $($field:ident),+ $(,)?) => {
$(
if $caller.$field.is_none() {
if let Some(value) = $plan.$field.clone() {
$params.$field = Some(value);
}
}
)+
};
}
impl Plan {
pub fn none() -> Plan {
Plan::default()
}
pub fn frugal() -> Plan {
Plan {
metadata: Some(false),
return_headers: Some(false),
return_cookies: Some(false),
return_page_links: Some(false),
return_json_data: Some(false),
return_embeddings: Some(false),
..Plan::default()
}
}
pub fn for_need(need: &Need) -> Plan {
match need {
Need::Raw => Plan::none(),
Need::Text => Plan {
return_format: Some(ReturnFormat::Text),
readability: Some(true),
clean_html: Some(true),
..Plan::frugal()
},
Need::Markdown => Plan {
return_format: Some(ReturnFormat::Markdown),
readability: Some(true),
clean_html: Some(true),
..Plan::frugal()
},
Need::Html => Plan {
return_format: Some(ReturnFormat::Raw),
clean_html: Some(true),
..Plan::frugal()
},
Need::Links => Plan {
endpoint: Some(Endpoint::Links),
return_format: Some(ReturnFormat::Empty),
return_page_links: Some(true),
..Plan::frugal()
},
Need::Metadata => Plan {
return_format: Some(ReturnFormat::Empty),
metadata: Some(true),
..Plan::frugal()
},
Need::Fields(spec) => {
if spec.is_empty() {
Plan::frugal()
} else {
Plan {
return_format: Some(ReturnFormat::Empty),
css_extraction_map: Some(spec.to_map()),
..Plan::frugal()
}
}
}
Need::Screenshot(shot) => Plan {
endpoint: Some(Endpoint::Screenshot),
return_format: Some(ReturnFormat::Bytes),
metadata: Some(shot.wants_metadata()),
..Plan::frugal()
},
}
}
pub fn apply_over(&self, params: &mut RequestParams, caller: &RequestParams) {
if caller.return_format.is_none() {
if let Some(format) = self.return_format {
set_format(params, format);
}
}
if caller.css_extraction_map.is_none() {
if let Some(map) = &self.css_extraction_map {
params.css_extraction_map = Some(map.clone());
}
}
settle!(
self,
params,
caller,
readability,
clean_html,
metadata,
return_headers,
return_cookies,
return_page_links,
return_json_data,
return_embeddings,
);
}
pub fn returns_no_page(&self) -> bool {
self.return_format == Some(ReturnFormat::Empty)
}
pub fn is_empty(&self) -> bool {
self == &Plan::none()
}
pub fn format(&self) -> Option<ReturnFormat> {
self.return_format
}
}
pub(crate) fn set_format(params: &mut RequestParams, format: ReturnFormat) {
params.return_format = Some(format.into());
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::indexing_slicing,
clippy::string_slice
)]
use super::*;
use crate::params::ReturnFormatHandling;
use crate::thrift::need::{FieldSpec, ShotSpec};
fn optional_returns(plan: &Plan) -> Vec<(&'static str, Option<bool>)> {
vec![
("metadata", plan.metadata),
("return_headers", plan.return_headers),
("return_cookies", plan.return_cookies),
("return_page_links", plan.return_page_links),
("return_json_data", plan.return_json_data),
("return_embeddings", plan.return_embeddings),
]
}
#[test]
fn the_optional_return_list_matches_the_fields_a_plan_carries() {
let named: Vec<&str> = optional_returns(&Plan::frugal())
.into_iter()
.map(|(name, _)| name)
.collect();
assert_eq!(named, OPTIONAL_RETURNS);
}
#[test]
fn raw_has_no_opinion_and_changes_nothing() {
let plan = Plan::for_need(&Need::Raw);
assert!(plan.is_empty());
let caller = RequestParams::url("https://example.com");
let mut params = caller.clone();
plan.apply_over(&mut params, &caller);
assert_eq!(params, caller);
}
#[test]
fn every_need_but_raw_turns_the_optional_returns_off() {
let needs = [
(Need::Text, None),
(Need::Markdown, None),
(Need::Html, None),
(Need::Links, Some("return_page_links")),
(Need::Metadata, Some("metadata")),
(Need::Fields(FieldSpec::new([("price", ".price")])), None),
(Need::Screenshot(ShotSpec::new()), None),
(
Need::Screenshot(ShotSpec::with_metadata()),
Some("metadata"),
),
];
for (need, asked_for) in needs {
let plan = Plan::for_need(&need);
for (name, value) in optional_returns(&plan) {
let expected = Some(Some(name) == asked_for);
assert_eq!(value, expected, "{} left {name} at {value:?}", need.label());
}
}
}
#[test]
fn fields_ask_for_the_extractions_and_none_of_the_page() {
let plan = Plan::for_need(&Need::fields([("price", ".price"), ("title", "h1")]));
assert_eq!(plan.return_format, Some(ReturnFormat::Empty));
assert!(plan.returns_no_page());
let map = plan.css_extraction_map.expect("an extraction map");
assert_eq!(map["/"].len(), 2);
}
#[test]
fn an_empty_field_spec_does_not_ask_for_a_response_with_nothing_in_it() {
let plan = Plan::for_need(&Need::Fields(FieldSpec::default()));
assert_eq!(plan.return_format, None);
assert!(plan.css_extraction_map.is_none());
}
#[test]
fn a_setting_the_caller_made_survives_the_plan() {
let mut caller = RequestParams::url("https://example.com");
caller.return_format = Some(ReturnFormatHandling::Single(ReturnFormat::Raw));
caller.metadata = Some(true);
let mut params = caller.clone();
let plan = Plan::for_need(&Need::Markdown);
plan.apply_over(&mut params, &caller);
assert_eq!(
params.return_format,
Some(ReturnFormatHandling::Single(ReturnFormat::Raw))
);
assert_eq!(params.metadata, Some(true));
assert_eq!(params.readability, Some(true));
assert_eq!(params.return_cookies, Some(false));
}
#[test]
fn applying_twice_lands_in_the_same_place() {
let caller = RequestParams::url("https://example.com");
let plan = Plan::for_need(&Need::Text);
let mut once = caller.clone();
plan.apply_over(&mut once, &caller);
let mut twice = once.clone();
plan.apply_over(&mut twice, &caller);
assert_eq!(once, twice);
}
#[test]
fn a_plan_reapplied_over_a_changed_request_puts_the_format_back() {
let caller = RequestParams::url("https://example.com");
let mut params = caller.clone();
let plan = Plan::for_need(&Need::Markdown);
plan.apply_over(&mut params, &caller);
params.return_format = Some(ReturnFormatHandling::Single(ReturnFormat::Raw));
plan.apply_over(&mut params, &caller);
assert_eq!(
params.return_format,
Some(ReturnFormatHandling::Single(ReturnFormat::Markdown))
);
}
#[test]
fn set_format_writes_one_format_not_a_list() {
let mut params = RequestParams::default();
set_format(&mut params, ReturnFormat::Text);
assert_eq!(
params.return_format,
Some(ReturnFormatHandling::Single(ReturnFormat::Text))
);
}
#[test]
fn endpoints_follow_the_need() {
assert_eq!(Plan::for_need(&Need::Links).endpoint, Some(Endpoint::Links));
assert_eq!(
Plan::for_need(&Need::screenshot()).endpoint,
Some(Endpoint::Screenshot)
);
assert_eq!(Plan::for_need(&Need::Markdown).endpoint, None);
assert_eq!(Plan::for_need(&Need::Raw).endpoint, None);
}
}