use crate::edit::{Edit, EditSet, Op, Value};
use crate::observe::Observation;
use crate::params::Params;
use crate::schema::{service_default, Key, Schema, MODES, PROXIES, WAIT_BUCKETS};
use crate::validate::validate;
use spider_route::{DeclaredNeed, ProxyPool, RequestMode, RouteDecision};
use url::Url;
pub const MAX_CANDIDATES: usize = 24;
pub const MULTIPLIERS: [(RequestMode, ProxyPool, u32, f32); 6] = [
(RequestMode::Http, ProxyPool::Isp, 0, 1.0),
(RequestMode::Smart, ProxyPool::Isp, 0, 1.5),
(RequestMode::Browser, ProxyPool::Isp, 0, 4.0),
(RequestMode::Browser, ProxyPool::Isp, 10_000, 5.0),
(RequestMode::Smart, ProxyPool::Residential, 0, 3.0),
(RequestMode::Browser, ProxyPool::Residential, 10_000, 8.0),
];
pub const PAIRS: [(Key, Key); 4] = [
(Key::Request, Key::WaitIdleMillis),
(Key::Request, Key::Proxy),
(Key::BlockStylesheets, Key::FullResources),
(Key::NetworkBlacklist, Key::Request),
];
#[derive(Clone, Copy)]
pub struct Context<'a> {
pub url: &'a Url,
pub need: DeclaredNeed,
pub current: &'a dyn Params,
pub caller: &'a dyn Params,
pub routed: &'a RouteDecision,
pub observation: &'a Observation,
pub multiplier_cap: f32,
}
#[derive(Debug, Clone, PartialEq)]
pub struct Candidate {
pub edits: EditSet,
pub multiplier: f32,
}
impl Candidate {
pub fn keep(ctx: &Context<'_>) -> Candidate {
Candidate {
edits: EditSet::keep(),
multiplier: multiplier(ctx, &EditSet::keep()),
}
}
}
pub fn generate(schema: &Schema, ctx: &Context<'_>) -> Vec<Candidate> {
let mut list = List {
schema,
ctx,
out: Vec::with_capacity(MAX_CANDIDATES),
};
list.out.push(Candidate::keep(ctx));
for spec in schema.learnable() {
if spec.key != Key::NetworkBlacklist && left_to_us(ctx, spec.key) {
for edit in alternatives(ctx, spec.key) {
list.offer(vec![edit]);
}
}
}
for (first, second) in PAIRS {
if list.is_full() || !left_to_us(ctx, first) || !left_to_us(ctx, second) {
continue;
}
let firsts = alternatives(ctx, first);
let seconds = alternatives(ctx, second);
for a in &firsts {
for b in &seconds {
if !list.is_full() {
list.offer(vec![a.clone(), b.clone()]);
}
}
}
}
if let Some(resources) = &ctx.observation.resources {
for ident in &resources.candidates {
if !list.is_full() {
list.offer(vec![append(&ident.pattern)]);
}
}
}
list.out
}
struct List<'s, 'c, 'a> {
schema: &'s Schema,
ctx: &'c Context<'a>,
out: Vec<Candidate>,
}
impl List<'_, '_, '_> {
fn is_full(&self) -> bool {
self.out.len() >= MAX_CANDIDATES
}
fn offer(&mut self, edits: Vec<Edit>) {
if self.is_full() {
return;
}
let Ok(edits) = EditSet::new(edits) else {
return;
};
let candidate = Candidate {
multiplier: multiplier(self.ctx, &edits),
edits,
};
if validate(self.schema, self.ctx, &candidate).is_ok()
&& !self.out.iter().any(|seen| seen.edits == candidate.edits)
{
self.out.push(candidate);
}
}
}
fn left_to_us(ctx: &Context<'_>, key: Key) -> bool {
!ctx.caller.is_set(key)
&& !(key == Key::NetworkBlacklist && ctx.caller.is_set(Key::NetworkWhitelist))
}
fn alternatives(ctx: &Context<'_>, key: Key) -> Vec<Edit> {
let current = ctx.current;
match key {
Key::Request => MODES
.iter()
.filter(|mode| **mode != current_mode(current))
.map(|mode| Edit::set(key, Value::Mode(*mode)))
.collect(),
Key::Proxy => PROXIES
.iter()
.filter(|pool| **pool != current_proxy(current))
.map(|pool| Edit::set(key, Value::Proxy(*pool)))
.collect(),
Key::WaitIdleMillis => WAIT_BUCKETS
.iter()
.filter(|millis| **millis > 0 && **millis != current_wait(current))
.map(|millis| Edit::set(key, Value::Millis(*millis)))
.collect(),
Key::NetworkBlacklist => ctx
.observation
.resources
.as_ref()
.map(|resources| {
resources
.candidates
.iter()
.map(|ident| append(&ident.pattern))
.collect()
})
.unwrap_or_default(),
Key::DisableIntercept
| Key::FullResources
| Key::BlockAds
| Key::BlockAnalytics
| Key::BlockStylesheets => {
vec![Edit::set(key, Value::Bool(!effective_flag(current, key)))]
}
_ => Vec::new(),
}
}
fn append(pattern: &str) -> Edit {
Edit {
key: Key::NetworkBlacklist,
op: Op::Append(vec![pattern.to_string()]),
}
}
pub(crate) fn current_mode(params: &dyn Params) -> RequestMode {
params.request().unwrap_or_default()
}
pub(crate) fn current_proxy(params: &dyn Params) -> ProxyPool {
params.proxy().unwrap_or_default()
}
pub(crate) fn current_wait(params: &dyn Params) -> u32 {
params.idle_wait_millis().unwrap_or(0)
}
pub(crate) fn effective_flag(params: &dyn Params, key: Key) -> bool {
params.flag(key).unwrap_or(service_default(key))
}
pub(crate) fn resulting(ctx: &Context<'_>, edits: &EditSet) -> (RequestMode, ProxyPool, u32) {
let mut mode = current_mode(ctx.current);
let mut proxy = current_proxy(ctx.current);
let mut wait = current_wait(ctx.current);
for edit in edits.edits() {
match (edit.key, &edit.op) {
(Key::Request, Op::Set(Value::Mode(m))) => mode = *m,
(Key::Proxy, Op::Set(Value::Proxy(p))) => proxy = *p,
(Key::WaitIdleMillis, Op::Set(Value::Millis(n))) if *n > 0 => wait = *n,
_ => {}
}
}
(mode, proxy, wait)
}
pub(crate) const fn mode_rank(mode: RequestMode) -> u8 {
match mode {
RequestMode::Http => 0,
RequestMode::Smart => 1,
_ => 2,
}
}
pub(crate) const fn proxy_rank(pool: ProxyPool) -> u8 {
match pool {
ProxyPool::Isp => 0,
_ => 1,
}
}
pub fn price(mode: RequestMode, proxy: ProxyPool, wait: u32) -> f32 {
let mut best: Option<f32> = None;
let mut dearest = 0.0f32;
for (arm_mode, arm_proxy, arm_wait, cost) in MULTIPLIERS {
dearest = dearest.max(cost);
if mode_rank(arm_mode) >= mode_rank(mode)
&& proxy_rank(arm_proxy) >= proxy_rank(proxy)
&& arm_wait >= wait
{
best = Some(best.map_or(cost, |seen| seen.min(cost)));
}
}
best.unwrap_or(dearest)
}
pub(crate) fn multiplier(ctx: &Context<'_>, edits: &EditSet) -> f32 {
let (mode, proxy, wait) = resulting(ctx, edits);
price(mode, proxy, wait)
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::indexing_slicing,
clippy::string_slice
)]
use super::*;
use crate::testing::Fixture;
use spider_cloud_agent::params::{Timeout, WaitFor};
use spider_route::StatusClass;
#[test]
fn generation_is_deterministic() {
let fixture = Fixture::observed();
let schema = Schema::v1();
let first = generate(&schema, &fixture.ctx());
assert!(first.len() > 10, "only {} candidates", first.len());
for _ in 0..5 {
assert_eq!(generate(&schema, &fixture.ctx()), first);
}
assert_eq!(generate(&schema, &Fixture::observed().ctx()), first);
}
#[test]
fn keep_is_always_first() {
let schema = Schema::v1();
let mut fixtures = vec![Fixture::cold(), Fixture::observed(), Fixture::pinned()];
let mut limited = Fixture::observed();
limited.observation.last_status = StatusClass::RateLimited;
fixtures.push(limited);
let mut broke = Fixture::observed();
broke.cap = 0.0;
fixtures.push(broke);
for fixture in &fixtures {
let list = generate(&schema, &fixture.ctx());
assert!(list[0].edits.is_keep());
assert_eq!(list.iter().filter(|c| c.edits.is_keep()).count(), 1);
}
}
#[test]
fn no_candidate_survives_validation_with_a_pinned_key() {
let schema = Schema::v1();
let fixture = Fixture::pinned();
let ctx = fixture.ctx();
let list = generate(&schema, &ctx);
for cand in &list {
for edit in cand.edits.edits() {
assert!(
!ctx.caller.is_set(edit.key),
"{:?} edits a key the caller set",
edit.key
);
}
}
assert!(list.len() > 1);
let mut all = Fixture::observed();
all.caller = crate::schema::tests::populated();
assert_eq!(generate(&schema, &all.ctx()).len(), 1);
}
#[test]
fn never_more_than_max_candidates() {
let schema = Schema::v1();
let fixture = Fixture::observed();
let list = generate(&schema, &fixture.ctx());
assert_eq!(list.len(), MAX_CANDIDATES, "the cap was not reached");
let appends = list
.iter()
.filter(|c| c.edits.edits().len() == 1 && c.edits.get(Key::NetworkBlacklist).is_some())
.count();
assert!(appends < 4, "{appends} single appends survived the cap");
}
#[test]
fn a_candidate_is_priced_by_what_it_sends() {
assert_eq!(price(RequestMode::Http, ProxyPool::Isp, 0), 1.0);
assert_eq!(price(RequestMode::Browser, ProxyPool::Isp, 10_000), 5.0);
assert_eq!(price(RequestMode::Browser, ProxyPool::Isp, 2_000), 5.0);
assert_eq!(price(RequestMode::Http, ProxyPool::Residential, 0), 3.0);
assert_eq!(price(RequestMode::Browser, ProxyPool::Residential, 0), 8.0);
assert_eq!(price(RequestMode::Smart, ProxyPool::Isp, 60_000), 8.0);
let mut fixture = Fixture::cold();
fixture.current.request = Some(RequestMode::Smart);
fixture.current.wait_for = Some(WaitFor::idle_network(Timeout::from_millis(0)));
let ctx = fixture.ctx();
let wait = EditSet::new(vec![
Edit::set(Key::Request, Value::Mode(RequestMode::Browser)),
Edit::set(Key::WaitIdleMillis, Value::Millis(10_000)),
])
.unwrap();
assert_eq!(multiplier(&ctx, &wait), 5.0);
assert_eq!(Candidate::keep(&ctx).multiplier, 1.5);
}
}