pub mod extraction;
pub mod js;
pub mod scope;
pub mod screenshot;
pub mod search;
pub mod transport;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
pub use crate::credits::{Credits, WholeCredits, CREDITS_PER_USD};
pub use extraction::{
ChunkingAlg, ChunkingKind, CssExtractionMap, ExtractSelector, ReturnFormat,
ReturnFormatHandling, SelectorGroup,
};
pub use js::{
Delay, EventTracker, ExecutionScriptsMap, IdleNetwork, SelectorWait, Timeout, WaitFor,
WebAutomation, WebAutomationMap,
};
pub use scope::{CrawlBudget, LinkRewriteRule};
pub use screenshot::ScreenshotParams;
pub use search::{SearchEngine, SearchParams, TimeWindow};
pub use transport::{
Country, InvalidCountry, Profile, ProxyPool, RedirectPolicy, RequestMode, Viewport,
WebhookSettings, BOT_USER_AGENT, PROXY_POOLS,
};
pub type HeaderMap = BTreeMap<String, String>;
#[derive(Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct RequestParams {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub url: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub request: Option<RequestMode>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub proxy: Option<ProxyPool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub country_code: Option<Country>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub remote_proxy: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub stealth: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub fingerprint: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub user_agent: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub viewport: Option<Viewport>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub locale: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cookies: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub headers: Option<HeaderMap>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub encoding: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub session: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub redirect_policy: Option<RedirectPolicy>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub request_timeout: Option<u8>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub service_worker_enabled: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub delay: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub concurrency_limit: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub wayback: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub limit: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub depth: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub budget: Option<CrawlBudget>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub blacklist: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub whitelist: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub subdomains: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tld: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub external_domains: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sitemap: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sitemap_only: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sitemap_path: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub respect_robots: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub link_rewrite: Option<LinkRewriteRule>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub crawl_timeout: Option<Timeout>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub run_in_background: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub wait_for: Option<WaitFor>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub scroll: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub execution_scripts: Option<ExecutionScriptsMap>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub automation_scripts: Option<WebAutomationMap>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub evaluate_on_new_document: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub disable_intercept: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub full_resources: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub block_ads: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub block_analytics: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub block_stylesheets: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub disable_first_party_stylesheets: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub disable_first_party_javascript: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub disable_first_party_visuals: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub network_whitelist: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub network_blacklist: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub event_tracker: Option<EventTracker>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub return_format: Option<ReturnFormatHandling>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub root_selector: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub exclude_selector: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub filter_output_images: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub filter_output_svg: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub filter_output_main_only: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_size: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub readability: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub clean_html: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub css_extraction_map: Option<CssExtractionMap>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub chunking_alg: Option<ChunkingAlg>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metadata: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub return_embeddings: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub return_headers: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub return_cookies: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub return_page_links: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub return_json_data: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache: Option<Cache>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub webhooks: Option<WebhookSettings>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub data_connectors: Option<serde_json::Map<String, serde_json::Value>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub router: Option<Router>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_options: Option<BTreeMap<String, serde_json::Value>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_credits_allowed: Option<WholeCredits>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_credits_per_page: Option<Credits>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub disable_hints: Option<bool>,
}
impl std::fmt::Debug for RequestParams {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RequestParams")
.field("url", &self.url.as_ref().map(|_| "<redacted>"))
.field("request", &self.request)
.field("proxy", &self.proxy)
.field("country_code", &self.country_code)
.field(
"remote_proxy",
&self.remote_proxy.as_deref().map(RedactedProxy),
)
.field("stealth", &self.stealth)
.field("fingerprint", &self.fingerprint)
.field(
"user_agent",
&self.user_agent.as_ref().map(|_| "<redacted>"),
)
.field("viewport", &self.viewport)
.field("locale", &self.locale.as_ref().map(|_| "<redacted>"))
.field("cookies", &self.cookies.as_ref().map(|_| "<redacted>"))
.field("headers", &self.headers.as_ref().map(RedactedHeaders))
.field("encoding", &self.encoding.as_ref().map(|_| "<redacted>"))
.field("session", &self.session)
.field("redirect_policy", &self.redirect_policy)
.field("request_timeout", &self.request_timeout)
.field("service_worker_enabled", &self.service_worker_enabled)
.field("delay", &self.delay)
.field("concurrency_limit", &self.concurrency_limit)
.field("wayback", &self.wayback)
.field("limit", &self.limit)
.field("depth", &self.depth)
.field("budget", &self.budget.as_ref().map(|_| "<redacted>"))
.field("blacklist", &self.blacklist.as_ref().map(|_| "<redacted>"))
.field("whitelist", &self.whitelist.as_ref().map(|_| "<redacted>"))
.field("subdomains", &self.subdomains)
.field("tld", &self.tld)
.field(
"external_domains",
&self.external_domains.as_ref().map(|_| "<redacted>"),
)
.field("sitemap", &self.sitemap)
.field("sitemap_only", &self.sitemap_only)
.field(
"sitemap_path",
&self.sitemap_path.as_ref().map(|_| "<redacted>"),
)
.field("respect_robots", &self.respect_robots)
.field(
"link_rewrite",
&self.link_rewrite.as_ref().map(|_| "<redacted>"),
)
.field("crawl_timeout", &self.crawl_timeout)
.field("run_in_background", &self.run_in_background)
.field("wait_for", &self.wait_for.as_ref().map(|_| "<redacted>"))
.field("scroll", &self.scroll)
.field(
"execution_scripts",
&self.execution_scripts.as_ref().map(|_| "<redacted>"),
)
.field(
"automation_scripts",
&self.automation_scripts.as_ref().map(|_| "<redacted>"),
)
.field(
"evaluate_on_new_document",
&self.evaluate_on_new_document.as_ref().map(|_| "<redacted>"),
)
.field("disable_intercept", &self.disable_intercept)
.field("full_resources", &self.full_resources)
.field("block_ads", &self.block_ads)
.field("block_analytics", &self.block_analytics)
.field("block_stylesheets", &self.block_stylesheets)
.field(
"disable_first_party_stylesheets",
&self.disable_first_party_stylesheets,
)
.field(
"disable_first_party_javascript",
&self.disable_first_party_javascript,
)
.field(
"disable_first_party_visuals",
&self.disable_first_party_visuals,
)
.field(
"network_whitelist",
&self.network_whitelist.as_ref().map(|_| "<redacted>"),
)
.field(
"network_blacklist",
&self.network_blacklist.as_ref().map(|_| "<redacted>"),
)
.field("event_tracker", &self.event_tracker)
.field("return_format", &self.return_format)
.field(
"root_selector",
&self.root_selector.as_ref().map(|_| "<redacted>"),
)
.field(
"exclude_selector",
&self.exclude_selector.as_ref().map(|_| "<redacted>"),
)
.field("filter_output_images", &self.filter_output_images)
.field("filter_output_svg", &self.filter_output_svg)
.field("filter_output_main_only", &self.filter_output_main_only)
.field("max_size", &self.max_size)
.field("readability", &self.readability)
.field("clean_html", &self.clean_html)
.field(
"css_extraction_map",
&self.css_extraction_map.as_ref().map(|_| "<redacted>"),
)
.field("chunking_alg", &self.chunking_alg)
.field("metadata", &self.metadata)
.field("return_embeddings", &self.return_embeddings)
.field("return_headers", &self.return_headers)
.field("return_cookies", &self.return_cookies)
.field("return_page_links", &self.return_page_links)
.field("return_json_data", &self.return_json_data)
.field("cache", &self.cache)
.field("webhooks", &self.webhooks)
.field(
"data_connectors",
&self.data_connectors.as_ref().map(|_| "<redacted>"),
)
.field("router", &self.router)
.field(
"provider_options",
&self.provider_options.as_ref().map(|_| "<redacted>"),
)
.field("max_credits_allowed", &self.max_credits_allowed)
.field("max_credits_per_page", &self.max_credits_per_page)
.field("disable_hints", &self.disable_hints)
.finish_non_exhaustive()
}
}
struct RedactedHeaders<'a>(&'a HeaderMap);
impl std::fmt::Debug for RedactedHeaders<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_map()
.entries(self.0.keys().map(|name| (name, "<redacted>")))
.finish()
}
}
struct RedactedProxy<'a>(&'a str);
impl std::fmt::Debug for RedactedProxy<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match url::Url::parse(self.0) {
Ok(url) if url.has_host() => f
.debug_struct("Proxy")
.field("scheme", &url.scheme())
.field("userinfo", &"<redacted>")
.field("host", &url.host_str())
.field("port", &url.port())
.finish_non_exhaustive(),
_ => f.write_str("<redacted>"),
}
}
}
#[derive(Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Router {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub mode: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub token: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub credentials: Option<BTreeMap<String, String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub funding: Option<String>,
}
impl std::fmt::Debug for Router {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Router")
.field("mode", &self.mode)
.field("provider", &self.provider)
.field("token", &self.token.as_ref().map(|_| "<redacted>"))
.field("credentials", &self.credentials.as_ref().map(|c| c.len()))
.field("funding", &self.funding)
.finish()
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum Cache {
Enabled(bool),
Control(CacheControl),
Other(serde_json::Value),
}
impl From<bool> for Cache {
fn from(enabled: bool) -> Self {
Self::Enabled(enabled)
}
}
impl From<CacheControl> for Cache {
fn from(control: CacheControl) -> Self {
Self::Control(control)
}
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct CacheControl {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_age: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub allow_stale: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub skip_browser: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub period: Option<String>,
#[serde(default, flatten)]
pub extra: serde_json::Map<String, serde_json::Value>,
}
impl RequestParams {
pub fn url(url: impl Into<String>) -> RequestParams {
RequestParams {
url: Some(url.into()),
..RequestParams::default()
}
}
pub fn apply_profile(&mut self, profile: Profile) {
if self.user_agent.is_none() {
if let Some(user_agent) = profile.user_agent() {
self.user_agent = Some(user_agent.to_string());
}
}
if self.viewport.is_none() {
self.viewport = Some(profile.viewport());
}
}
pub fn is_empty(&self) -> bool {
self == &RequestParams::default()
}
}
#[cfg(test)]
mod tests {
#![allow(
clippy::field_reassign_with_default,
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::indexing_slicing,
clippy::string_slice
)]
use super::*;
#[test]
fn credential_payloads_are_redacted_but_still_serialize() {
let secret = "synthetic-private-value";
let automation = [
WebAutomation::Evaluate(secret.into()),
WebAutomation::Type {
value: secret.into(),
modifier: None,
},
WebAutomation::Fill {
selector: "input".into(),
value: secret.into(),
},
];
let webhook = WebhookSettings::new(format!("https://example.com/{secret}"));
let rewrite = LinkRewriteRule::replace("old", secret);
let mut params = RequestParams::url(format!("https://user:{secret}@example.com"));
params.execution_scripts = Some([("/".into(), secret.into())].into());
params.evaluate_on_new_document = Some(secret.into());
params.automation_scripts = Some([("/".into(), automation.to_vec())].into());
params.webhooks = Some(webhook.clone());
params.link_rewrite = Some(rewrite.clone());
for value in [
format!("{params:?}"),
format!("{automation:?}"),
format!("{webhook:?}"),
format!("{rewrite:?}"),
] {
assert!(!value.contains(secret), "credential in Debug");
assert!(value.contains("<redacted>"));
}
let wire = serde_json::to_value(¶ms).expect("serialize");
assert!(wire["execution_scripts"]["/"] == secret);
assert!(wire["automation_scripts"]["/"][1]["Type"]["value"] == secret);
assert!(wire["evaluate_on_new_document"] == secret);
assert!(wire["webhooks"]["destination"] == webhook.destination);
}
#[test]
fn proxy_debug_redacts_userinfo_and_rejects_malformed_input() {
for proxy in [
"http://synthetic-user:synthetic-password@example.com:8080",
"http://synthetic-user:synthetic-password@example.com:8080/private?token=synthetic-token",
"not a proxy synthetic-password",
] {
let params = RequestParams { remote_proxy: Some(proxy.into()), ..RequestParams::default() };
let printed = format!("{params:#?}");
assert!(!printed.contains("synthetic-"), "credential in Debug");
assert!(printed.contains("<redacted>"));
if proxy.starts_with("http://") {
assert!(printed.contains("example.com"));
assert!(printed.contains("8080"));
}
assert!(serde_json::to_value(params).expect("serialize")["remote_proxy"] == proxy);
}
}
#[test]
fn default_params_serialize_to_an_empty_object() {
let json = serde_json::to_string(&RequestParams::default()).expect("serialize");
assert_eq!(json, "{}");
assert!(RequestParams::default().is_empty());
}
#[test]
fn set_fields_are_the_only_ones_sent() {
let mut params = RequestParams::url("https://example.com");
params.request = Some(RequestMode::Browser);
let json = serde_json::to_value(¶ms).expect("serialize");
let object = json.as_object().expect("object");
assert_eq!(object.len(), 2);
assert_eq!(object["url"], "https://example.com");
assert_eq!(object["request"], "browser");
}
#[test]
fn request_mode_wire_values_and_aliases() {
assert_eq!(
serde_json::to_string(&RequestMode::Browser).expect("serialize"),
"\"browser\""
);
assert_eq!(
serde_json::to_string(&RequestMode::Smart).expect("serialize"),
"\"smart\""
);
assert_eq!(
serde_json::to_string(&RequestMode::Http).expect("serialize"),
"\"http\""
);
for value in ["\"chrome\"", "\"headless\"", "\"BROWSER\"", "\"Chrome\""] {
let mode: RequestMode = serde_json::from_str(value).expect("deserialize");
assert_eq!(
mode,
RequestMode::Browser,
"{value} should be a browser mode"
);
}
for value in ["\"smart\"", "\"smart_mode\"", "\"SmartMode\""] {
let mode: RequestMode = serde_json::from_str(value).expect("deserialize");
assert_eq!(mode, RequestMode::Smart, "{value} should be smart");
}
assert!(serde_json::from_str::<RequestMode>("\"teapot\"").is_err());
assert_eq!(RequestMode::default(), RequestMode::Smart);
}
#[test]
fn proxy_pool_has_two_members_and_an_alias() {
assert_eq!(PROXY_POOLS.len(), 2);
assert_eq!(ProxyPool::default(), ProxyPool::Isp);
assert_eq!(
serde_json::to_string(&ProxyPool::Residential).expect("serialize"),
"\"residential\""
);
assert_eq!(
serde_json::from_str::<ProxyPool>("\"datacenter\"").expect("deserialize"),
ProxyPool::Isp
);
assert!(serde_json::from_str::<ProxyPool>("\"mobile\"").is_err());
for pool in PROXY_POOLS {
assert!(matches!(pool, ProxyPool::Isp | ProxyPool::Residential));
assert_ne!(pool.as_str(), "mobile");
}
}
#[test]
fn return_format_handling_round_trips_both_shapes() {
let single = ReturnFormatHandling::Single(ReturnFormat::Markdown);
let json = serde_json::to_string(&single).expect("serialize");
assert_eq!(json, "\"markdown\"");
assert_eq!(
serde_json::from_str::<ReturnFormatHandling>(&json).expect("deserialize"),
single
);
let multi = ReturnFormatHandling::Multi(vec![ReturnFormat::Markdown, ReturnFormat::Empty]);
let json = serde_json::to_string(&multi).expect("serialize");
assert_eq!(json, "[\"markdown\",\"empty\"]");
assert_eq!(
serde_json::from_str::<ReturnFormatHandling>(&json).expect("deserialize"),
multi
);
assert!(multi.contains(ReturnFormat::Empty));
assert_eq!(single.formats(), &[ReturnFormat::Markdown]);
let err = serde_json::from_str::<ReturnFormatHandling>("\"pdf\"")
.expect_err("pdf is not a return format");
assert!(err.to_string().contains("pdf"), "{err}");
}
#[test]
fn country_takes_two_letters_and_sends_them_lowercase() {
let country = Country::new("DE").expect("de is a country");
assert_eq!(country.as_str(), "de");
assert_eq!(
serde_json::to_string(&country).expect("serialize"),
"\"de\""
);
assert_eq!(
serde_json::from_str::<Country>("\"US\"").expect("deserialize"),
Country::new("us").expect("us is a country")
);
assert!(Country::new("usa").is_none());
assert!(Country::new("u").is_none());
assert!(Country::new("u1").is_none());
assert!(serde_json::from_str::<Country>("\"usa\"").is_err());
}
#[test]
fn wait_for_sends_only_what_was_asked_for() {
let mut params = RequestParams::default();
params.wait_for = Some(WaitFor::idle_network(Timeout::from_millis(1_500)));
let json = serde_json::to_value(¶ms).expect("serialize");
assert_eq!(json["wait_for"]["idle_network"]["timeout"]["secs"], 1);
assert_eq!(
json["wait_for"]["idle_network"]["timeout"]["nanos"],
500_000_000
);
assert_eq!(
json["wait_for"].as_object().expect("object").len(),
1,
"an unset wait is not sent"
);
assert!(WaitFor::default().is_empty());
}
#[test]
fn extraction_map_carries_css_and_xpath() {
let mut params = RequestParams::default();
let mut map = CssExtractionMap::new();
map.insert(
"/product".into(),
vec![
SelectorGroup::css("price", [".price", "[data-price]"]),
SelectorGroup::xpath("title", ["//h1"]),
],
);
params.css_extraction_map = Some(map);
params.return_format = Some(ReturnFormat::Empty.into());
let json = serde_json::to_value(¶ms).expect("serialize");
assert_eq!(json["css_extraction_map"]["/product"][0]["name"], "price");
assert_eq!(
json["css_extraction_map"]["/product"][0]["selectors"][0],
".price"
);
assert_eq!(
json["css_extraction_map"]["/product"][1]["selectors"][0],
"//h1"
);
assert_eq!(json["return_format"], "empty");
let back: RequestParams = serde_json::from_value(json).expect("deserialize");
let groups = &back.css_extraction_map.expect("map")["/product"];
assert_eq!(groups[0].selectors[0], ExtractSelector::css(".price"));
assert_eq!(groups[1].selectors[0], ExtractSelector::xpath("//h1"));
}
#[test]
fn profile_fills_presentation_without_overwriting() {
let mut params = RequestParams::default();
params.apply_profile(Profile::MobileDevice);
assert_eq!(params.user_agent, None, "the service picks this one");
assert_eq!(params.viewport.as_ref().expect("viewport").width, 390);
let mut bot = RequestParams::default();
bot.apply_profile(Profile::Bot);
assert_eq!(bot.user_agent.as_deref(), Some(BOT_USER_AGENT));
let mut mine = RequestParams::default();
mine.user_agent = Some("mine".into());
mine.apply_profile(Profile::Bot);
assert_eq!(mine.user_agent.as_deref(), Some("mine"));
}
#[test]
fn the_whole_operation_cap_goes_out_as_an_integer_and_the_page_cap_as_a_float() {
let mut params = RequestParams::default();
params.max_credits_per_page = Some(Credits::from_usd(0.01));
params.max_credits_allowed = Some(WholeCredits::floor(Credits::new(30.9)));
let json = serde_json::to_string(¶ms).expect("serialize");
assert!(
json.contains(r#""max_credits_allowed":30"#),
"a decimal point here is a 400: {json}"
);
assert!(json.contains(r#""max_credits_per_page":100.0"#), "{json}");
let value = serde_json::to_value(¶ms).expect("serialize");
assert!(value["max_credits_allowed"].is_u64());
assert_eq!(value["max_credits_per_page"], 100.0);
assert_eq!(Credits::new(10_000.0).to_usd(), 1.0);
}
#[test]
fn search_params_flatten_the_fetch_settings() {
let mut params = SearchParams::new("rust web crawler");
params.base.request = Some(RequestMode::Http);
params.search_limit = Some(5);
params.tbs = Some(TimeWindow::PastWeek);
let json = serde_json::to_value(¶ms).expect("serialize");
let object = json.as_object().expect("object");
assert_eq!(object["search"], "rust web crawler");
assert_eq!(object["request"], "http");
assert_eq!(object["search_limit"], 5);
assert_eq!(object["tbs"], "qdr:w");
assert_eq!(object.len(), 4, "nothing unset is sent");
}
#[test]
fn link_rewrite_and_automation_keep_their_wire_shapes() {
let rule =
LinkRewriteRule::replace("staging.example.com", "example.com").on_host("example.com");
let json = serde_json::to_value(&rule).expect("serialize");
assert_eq!(json["type"], "replace");
assert_eq!(json["host"], "example.com");
assert_eq!(
serde_json::from_value::<LinkRewriteRule>(json).expect("deserialize"),
rule
);
let step = WebAutomation::Fill {
selector: "#q".into(),
value: "spider".into(),
};
let json = serde_json::to_value(&step).expect("serialize");
assert_eq!(json["Fill"]["selector"], "#q");
assert_eq!(
serde_json::from_value::<WebAutomation>(json).expect("deserialize"),
step
);
}
#[test]
fn documented_fields_go_out_under_the_service_names() {
let mut params = RequestParams::default();
params.service_worker_enabled = Some(false);
params.delay = Some(1_000);
params.concurrency_limit = Some(8);
params.wayback = Some(true);
params.sitemap_only = Some(true);
params.sitemap_path = Some("sitemap-1.xml".into());
params.block_ads = Some(false);
params.block_analytics = Some(false);
params.block_stylesheets = Some(true);
params.disable_first_party_stylesheets = Some(true);
params.disable_first_party_javascript = Some(true);
params.disable_first_party_visuals = Some(true);
params.network_whitelist = Some(vec!["example.com".into()]);
params.network_blacklist = Some(vec!["doubleclick.net".into()]);
params.exclude_selector = Some(".ad".into());
params.filter_output_images = Some(true);
params.filter_output_svg = Some(true);
params.filter_output_main_only = Some(true);
params.max_size = Some(200_000);
params.data_connectors = Some(serde_json::Map::from_iter([(
"on_find".to_string(),
serde_json::Value::Bool(true),
)]));
params.router = Some(Router {
mode: Some("fallback".into()),
..Router::default()
});
params.provider_options = Some(BTreeMap::from([(
"zyte".to_string(),
serde_json::json!({"geolocation": "US"}),
)]));
let json = serde_json::to_value(¶ms).expect("serialize");
let object = json.as_object().expect("object");
for name in [
"service_worker_enabled",
"delay",
"concurrency_limit",
"wayback",
"sitemap_only",
"sitemap_path",
"block_ads",
"block_analytics",
"block_stylesheets",
"disable_first_party_stylesheets",
"disable_first_party_javascript",
"disable_first_party_visuals",
"network_whitelist",
"network_blacklist",
"exclude_selector",
"filter_output_images",
"filter_output_svg",
"filter_output_main_only",
"max_size",
"data_connectors",
"router",
"provider_options",
] {
assert!(object.contains_key(name), "{name} missing");
}
assert_eq!(object.len(), 22);
assert_eq!(json["router"], serde_json::json!({"mode": "fallback"}));
let back: RequestParams = serde_json::from_value(json).expect("deserialize");
assert_eq!(back, params);
}
#[test]
fn retired_fields_are_not_part_of_the_body() {
let back: RequestParams = serde_json::from_str(
r#"{"gpt_config":{},"proxy_enabled":true,"smart_mode":true,"custom_prompt":"x","vendor_credentials":{"A":"b"}}"#,
)
.expect("unknown fields are ignored");
assert!(back.is_empty());
}
#[test]
fn router_and_connector_secrets_stay_out_of_debug() {
let secret = "synthetic-router-token";
let mut params = RequestParams::default();
params.router = Some(Router {
provider: Some("zyte".into()),
token: Some(secret.into()),
credentials: Some(BTreeMap::from([("NAME".to_string(), secret.to_string())])),
..Router::default()
});
params.data_connectors = Some(serde_json::Map::from_iter([(
"s3".to_string(),
serde_json::json!({"secret_access_key": secret}),
)]));
let printed = format!("{params:?}");
assert!(!printed.contains(secret), "credential in Debug");
assert!(printed.contains("zyte"));
assert_eq!(
serde_json::to_value(¶ms).expect("serialize")["router"]["token"],
secret
);
}
#[test]
fn search_engine_uses_the_service_names() {
for (engine, wire) in [
(SearchEngine::Google, r#""google""#),
(SearchEngine::Brave, r#""brave""#),
(SearchEngine::All, r#""generic""#),
] {
assert_eq!(serde_json::to_string(&engine).expect("serialize"), wire);
}
assert_eq!(
serde_json::from_str::<SearchEngine>(r#""all""#).expect("alias"),
SearchEngine::All
);
}
#[test]
fn screenshot_params_flatten_the_fetch_settings() {
let mut shot = ScreenshotParams::default();
shot.base = RequestParams::url("https://example.com");
shot.full_page = Some(true);
shot.fast = Some(false);
shot.cdp_params = Some(serde_json::Map::from_iter([(
"format".to_string(),
serde_json::Value::from("jpeg"),
)]));
let json = serde_json::to_value(&shot).expect("serialize");
assert_eq!(
json,
serde_json::json!({
"url": "https://example.com",
"full_page": true,
"fast": false,
"cdp_params": {"format": "jpeg"}
})
);
}
#[test]
fn a_full_body_round_trips() {
let mut params = RequestParams::url("https://example.com");
params.request = Some(RequestMode::Browser);
params.proxy = Some(ProxyPool::Residential);
params.country_code = Country::new("gb");
params.return_format = Some(ReturnFormatHandling::Multi(vec![
ReturnFormat::Markdown,
ReturnFormat::Text,
]));
params.wait_for = Some(WaitFor::selector("main", Timeout::from_secs(5)));
params.redirect_policy = Some(RedirectPolicy::Loose);
params.chunking_alg = Some(ChunkingAlg::new(ChunkingKind::BySentence, 20));
params.budget = Some(CrawlBudget::from([("*".to_string(), 25)]));
params.webhooks = Some(WebhookSettings::new("https://example.com/hook"));
params.max_credits_allowed = Some(WholeCredits::floor(Credits::new(5_000.0)));
let json = serde_json::to_string(¶ms).expect("serialize");
let back: RequestParams = serde_json::from_str(&json).expect("deserialize");
assert_eq!(back, params);
assert!(!json.contains("proxy_enabled"), "dropped from the surface");
assert!(!json.contains("pipeline"));
}
#[test]
fn an_event_tracker_always_sends_all_three_switches() {
let mut params = RequestParams::default();
params.event_tracker = Some(EventTracker {
responses: Some(true),
..EventTracker::default()
});
let json = serde_json::to_value(¶ms).expect("serialize");
assert_eq!(
json["event_tracker"],
serde_json::json!({"responses": true, "requests": false, "automation": false})
);
params.event_tracker = Some(EventTracker::default());
let json = serde_json::to_value(¶ms).expect("serialize");
assert_eq!(
json["event_tracker"],
serde_json::json!({"responses": false, "requests": false, "automation": false})
);
let partial: EventTracker = serde_json::from_str(r#"{"automation": true}"#).expect("parse");
assert_eq!(partial.automation, Some(true));
assert_eq!(partial.responses, None);
}
}