use crate::credits::Credits;
use crate::params::js::Timeout;
use crate::params::{Country, Profile, ProxyPool, RequestMode, RequestParams, WaitFor};
use crate::policy::budget::Budget;
use std::time::Duration;
use url::Url;
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub enum Rung {
Mode(RequestMode),
Proxy(ProxyPool),
Country(Country),
Wait(WaitFor),
Profile(Profile),
Timeout(Duration),
Session(bool),
}
impl Rung {
pub fn apply(&self, params: &mut RequestParams) {
match self {
Rung::Mode(mode) => params.request = Some(*mode),
Rung::Proxy(pool) => params.proxy = Some(*pool),
Rung::Country(country) => params.country_code = Some(country.clone()),
Rung::Wait(wait) => params.wait_for = Some(wait.clone()),
Rung::Profile(profile) => {
if let Some(agent) = profile.user_agent() {
params.user_agent = Some(agent.to_string());
}
params.viewport = Some(profile.viewport());
}
Rung::Timeout(span) => {
let secs = span.as_secs().clamp(1, u64::from(u8::MAX)) as u8;
params.request_timeout = Some(secs);
}
Rung::Session(on) => params.session = Some(*on),
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct Step {
pub rungs: Vec<Rung>,
pub label: &'static str,
pub est_multiplier: f32,
}
impl Step {
pub fn new(label: &'static str, est_multiplier: f32, rungs: Vec<Rung>) -> Step {
Step {
rungs,
label,
est_multiplier,
}
}
pub fn apply(&self, params: &mut RequestParams) {
for rung in &self.rungs {
rung.apply(params);
}
}
pub fn country(&self) -> Option<&Country> {
self.rungs.iter().find_map(|rung| match rung {
Rung::Country(country) => Some(country),
_ => None,
})
}
pub fn moves_country(&self) -> bool {
self.country().is_some()
}
pub fn estimate(&self, last_cost: Credits) -> Credits {
Credits(Budget::floor(last_cost).get() * f64::from(self.est_multiplier))
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct CountryPool(Vec<Country>);
impl CountryPool {
pub fn new(countries: impl IntoIterator<Item = Country>) -> CountryPool {
let mut out: Vec<Country> = Vec::new();
for country in countries {
if !out.contains(&country) {
out.push(country);
}
}
CountryPool(out)
}
pub fn for_url(url: &str) -> CountryPool {
let mut countries = Vec::new();
if let Some(country) = Url::parse(url)
.ok()
.and_then(|parsed| parsed.host_str().map(str::to_owned))
.and_then(|host| host.rsplit('.').next().and_then(Country::new))
{
countries.push(country);
}
if let Some(fallback) = Country::new("us") {
countries.push(fallback);
}
CountryPool::new(countries)
}
pub fn countries(&self) -> &[Country] {
&self.0
}
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
}
impl Default for Ladder {
fn default() -> Ladder {
Ladder::standard()
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct Ladder(pub Vec<Step>);
const SETTLE: Timeout = Timeout::from_secs(10);
impl Ladder {
pub fn standard() -> Ladder {
Ladder::with_countries(&CountryPool::new(Country::new("us")))
}
pub fn for_target(url: &str) -> Ladder {
Ladder::with_countries(&CountryPool::for_url(url))
}
pub fn with_countries(pool: &CountryPool) -> Ladder {
let rendered = vec![Rung::Mode(RequestMode::Browser)];
let settled = vec![
Rung::Mode(RequestMode::Browser),
Rung::Wait(WaitFor::idle_network(SETTLE)),
];
let mut residential = settled.clone();
residential.push(Rung::Proxy(ProxyPool::Residential));
let mut steps = vec![
Step::new("browser", 4.0, rendered),
Step::new("browser+wait", 5.0, settled),
Step::new("residential", 8.0, residential.clone()),
];
for (offset, country) in pool.countries().iter().enumerate() {
let mut rungs = residential.clone();
rungs.push(Rung::Country(country.clone()));
steps.push(Step::new("residential+geo", 9.0 + offset as f32, rungs));
}
Ladder(steps)
}
pub fn get(&self, index: usize) -> Option<&Step> {
self.0.get(index)
}
pub fn len(&self) -> usize {
self.0.len()
}
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
pub fn first_geo_from(&self, from: usize) -> Option<usize> {
(from..self.0.len()).find(|index| self.0.get(*index).is_some_and(Step::moves_country))
}
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::indexing_slicing,
clippy::string_slice
)]
use super::*;
use crate::policy::budget::ASSUMED_MINIMUM_COST;
#[test]
fn the_standard_ladder_is_four_steps_ending_in_a_country_move() {
let ladder = Ladder::standard();
let labels: Vec<_> = ladder.0.iter().map(|step| step.label).collect();
assert_eq!(
labels,
["browser", "browser+wait", "residential", "residential+geo"]
);
assert_eq!(
ladder.get(3).and_then(Step::country).map(Country::as_str),
Some("us")
);
}
#[test]
fn each_step_is_a_superset_of_the_one_before() {
let ladder = Ladder::standard();
for pair in ladder.0.windows(2) {
for rung in &pair[0].rungs {
assert!(
pair[1].rungs.contains(rung),
"{} dropped {rung:?} that {} had",
pair[1].label,
pair[0].label
);
}
}
}
#[test]
fn the_estimates_only_go_up() {
for ladder in [
Ladder::standard(),
Ladder::for_target("https://example.de/thing"),
] {
for pair in ladder.0.windows(2) {
assert!(
pair[1].est_multiplier > pair[0].est_multiplier,
"{} does not cost more than {}",
pair[1].label,
pair[0].label
);
}
}
}
#[test]
fn a_national_domain_rotates_to_its_own_country_first() {
let pool = CountryPool::for_url("https://shop.example.de/a");
let codes: Vec<_> = pool.countries().iter().map(Country::as_str).collect();
assert_eq!(codes, ["de", "us"]);
let ladder = Ladder::for_target("https://shop.example.de/a");
assert_eq!(ladder.len(), 5);
assert_eq!(
ladder.get(3).and_then(Step::country).map(Country::as_str),
Some("de")
);
assert_eq!(
ladder.get(4).and_then(Step::country).map(Country::as_str),
Some("us")
);
}
#[test]
fn a_generic_domain_rotates_to_us_only() {
let pool = CountryPool::for_url("https://example.com/a");
let codes: Vec<_> = pool.countries().iter().map(Country::as_str).collect();
assert_eq!(codes, ["us"]);
}
#[test]
fn a_us_domain_is_not_listed_twice() {
let pool = CountryPool::for_url("https://example.us/a");
let codes: Vec<_> = pool.countries().iter().map(Country::as_str).collect();
assert_eq!(codes, ["us"]);
}
#[test]
fn the_geo_jump_finds_the_first_country_step() {
let ladder = Ladder::standard();
assert_eq!(ladder.first_geo_from(0), Some(3));
assert_eq!(ladder.first_geo_from(3), Some(3));
assert_eq!(ladder.first_geo_from(4), None);
}
#[test]
fn every_rung_and_standard_step_changes_exactly_its_documented_keys() {
use std::collections::BTreeSet;
let cases: Vec<(Rung, Vec<&str>)> = vec![
(Rung::Mode(RequestMode::Browser), vec!["request"]),
(Rung::Proxy(ProxyPool::Residential), vec!["proxy"]),
(
Rung::Country(Country::new("us").unwrap()),
vec!["country_code"],
),
(Rung::Wait(WaitFor::idle_network(SETTLE)), vec!["wait_for"]),
(Rung::Profile(Profile::Desktop), vec!["viewport"]),
(Rung::Profile(Profile::MobileDevice), vec!["viewport"]),
(Rung::Profile(Profile::Bot), vec!["user_agent", "viewport"]),
(
Rung::Timeout(Duration::from_secs(45)),
vec!["request_timeout"],
),
(Rung::Session(true), vec!["session"]),
];
let mut steps: Vec<(Step, Vec<&str>)> = cases
.into_iter()
.map(|(rung, keys)| (Step::new("single rung", 1.0, vec![rung]), keys))
.collect();
let standard_keys = [
vec!["request"],
vec!["request", "wait_for"],
vec!["request", "wait_for", "proxy"],
vec!["request", "wait_for", "proxy", "country_code"],
];
let standard = Ladder::standard();
assert_eq!(standard.len(), standard_keys.len());
steps.extend(standard.0.into_iter().zip(standard_keys));
let populated: RequestParams = serde_json::from_value(serde_json::json!({
"url": "https://example.org/page", "request": "http", "proxy": "isp",
"country_code": "de", "request_timeout": 12, "session": false,
"wait_for": {"delay": {"timeout": {"secs": 1, "nanos": 0}}},
"user_agent": "caller agent",
"viewport": {"width": 777, "height": 555},
"stealth": true, "fingerprint": true, "limit": 3,
"metadata": true, "headers": {"accept": "text/html"}, "cookies": "a=b"
}))
.unwrap();
for (step, expected) in steps {
for initial in [RequestParams::default(), populated.clone()] {
let before = serde_json::to_value(&initial).unwrap();
let mut after = initial;
step.apply(&mut after);
let after = serde_json::to_value(after).unwrap();
let before = before.as_object().unwrap();
let after = after.as_object().unwrap();
let changed: BTreeSet<&str> = before
.keys()
.chain(after.keys())
.filter(|key| before.get(*key) != after.get(*key))
.map(String::as_str)
.collect();
assert_eq!(changed, expected.iter().copied().collect(), "{step:?}");
}
}
}
#[test]
fn applying_a_step_writes_the_parameters_it_names() {
let mut params = RequestParams::url("https://example.com");
Ladder::standard()
.get(3)
.expect("the geographic step")
.apply(&mut params);
assert_eq!(params.request, Some(RequestMode::Browser));
assert_eq!(params.proxy, Some(ProxyPool::Residential));
assert_eq!(
params.country_code.as_ref().map(Country::as_str),
Some("us")
);
assert!(params
.wait_for
.as_ref()
.is_some_and(|wait| wait.idle_network.is_some()));
assert_eq!(params.stealth, None);
assert_eq!(params.fingerprint, None);
}
#[test]
fn applying_a_step_replaces_what_the_last_attempt_sent() {
let mut params = RequestParams::url("https://example.com");
params.proxy = Some(ProxyPool::Isp);
Ladder::standard()
.get(2)
.expect("the residential step")
.apply(&mut params);
assert_eq!(params.proxy, Some(ProxyPool::Residential));
}
#[test]
fn a_timeout_rung_is_written_in_whole_seconds() {
let mut params = RequestParams::default();
Rung::Timeout(Duration::from_millis(45_500)).apply(&mut params);
assert_eq!(params.request_timeout, Some(45));
Rung::Timeout(Duration::from_secs(9_000)).apply(&mut params);
assert_eq!(params.request_timeout, Some(255));
Rung::Timeout(Duration::from_millis(10)).apply(&mut params);
assert_eq!(params.request_timeout, Some(1));
}
#[test]
fn a_profile_rung_sets_the_viewport_and_only_pins_an_agent_when_the_profile_does() {
let mut params = RequestParams::default();
Rung::Profile(Profile::Desktop).apply(&mut params);
assert_eq!(params.user_agent, None);
assert!(params.viewport.is_some());
Rung::Profile(Profile::Bot).apply(&mut params);
assert!(params.user_agent.is_some());
}
#[test]
fn a_session_rung_sets_the_session_flag() {
let mut params = RequestParams::default();
Rung::Session(true).apply(&mut params);
assert_eq!(params.session, Some(true));
}
#[test]
fn the_ladder_ends_on_the_geographic_step() {
for ladder in [
Ladder::standard(),
Ladder::for_target("https://shop.example.de/a"),
] {
let last = ladder.0.last().expect("a last step");
assert_eq!(last.label, "residential+geo");
assert!(last.moves_country(), "the last step changes the country");
}
}
#[test]
fn an_estimate_scales_the_last_observed_cost() {
let step = Ladder::standard().get(2).expect("residential").clone();
assert_eq!(step.estimate(Credits(3.0)), Credits(24.0));
}
#[test]
fn an_estimate_after_a_free_attempt_still_names_a_figure() {
let ladder = Ladder::standard();
let cheapest = ladder.get(0).expect("browser");
let dearest = ladder.get(3).expect("the geographic step");
let floor = ASSUMED_MINIMUM_COST.get();
assert_eq!(cheapest.estimate(Credits::ZERO), Credits(floor * 4.0));
assert_eq!(dearest.estimate(Credits::ZERO), Credits(floor * 9.0));
assert!(dearest.estimate(Credits::ZERO) > cheapest.estimate(Credits::ZERO));
}
#[test]
fn an_empty_pool_leaves_the_ladder_without_a_country_step() {
let ladder = Ladder::with_countries(&CountryPool::default());
assert_eq!(ladder.len(), 3);
assert_eq!(ladder.first_geo_from(0), None);
}
}