use crate::params::Params;
use crate::schema::Key;
use spider_route::{ProxyPool, RequestMode};
use std::fmt;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Value {
Bool(bool),
Mode(RequestMode),
Proxy(ProxyPool),
Millis(u32),
Int(i64),
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum Op {
Set(Value),
Append(Vec<String>),
Remove(Vec<String>),
Replace(Vec<String>),
Clear,
}
impl Op {
pub const fn code(&self) -> u8 {
match self {
Op::Set(_) => 0,
Op::Append(_) => 1,
Op::Remove(_) => 2,
Op::Replace(_) => 3,
Op::Clear => 4,
}
}
pub const NAMES: [&'static str; 5] = ["set", "append", "remove", "replace", "clear"];
const fn is_list(&self) -> bool {
!matches!(self, Op::Set(_))
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Edit {
pub key: Key,
pub op: Op,
}
impl Edit {
pub const fn set(key: Key, value: Value) -> Edit {
Edit {
key,
op: Op::Set(value),
}
}
}
pub const MAX_EDITS: usize = 3;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EditError {
Duplicate(Key),
TooMany,
Unsupported(Key),
}
impl fmt::Display for EditError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
EditError::Duplicate(key) => write!(f, "two edits name {}", key.wire()),
EditError::TooMany => write!(f, "an edit set holds at most {MAX_EDITS} edits"),
EditError::Unsupported(key) => {
write!(f, "{} does not take a list operation", key.wire())
}
}
}
}
impl std::error::Error for EditError {}
#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
pub struct EditSet {
edits: Vec<Edit>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct Applied {
pub written: u8,
pub skipped_pinned: u8,
}
impl EditSet {
pub const fn keep() -> EditSet {
EditSet { edits: Vec::new() }
}
pub fn is_keep(&self) -> bool {
self.edits.is_empty()
}
pub fn new(mut edits: Vec<Edit>) -> Result<EditSet, EditError> {
if edits.len() > MAX_EDITS {
return Err(EditError::TooMany);
}
for edit in &edits {
if edit.op.is_list() && edit.key != Key::NetworkBlacklist {
return Err(EditError::Unsupported(edit.key));
}
}
edits.sort_by_key(|edit| edit.key.index());
for pair in edits.windows(2) {
if let [first, second] = pair {
if first.key == second.key {
return Err(EditError::Duplicate(first.key));
}
}
}
Ok(EditSet { edits })
}
pub fn edits(&self) -> &[Edit] {
&self.edits
}
pub fn get(&self, key: Key) -> Option<&Edit> {
self.edits.iter().find(|edit| edit.key == key)
}
pub fn apply(&self, params: &mut dyn Params, caller: &dyn Params) -> Applied {
let mut applied = Applied::default();
for edit in &self.edits {
let pinned = match edit.key {
Key::NetworkBlacklist => {
caller.is_set(Key::NetworkBlacklist) || caller.is_set(Key::NetworkWhitelist)
}
key => caller.is_set(key),
};
if pinned {
applied.skipped_pinned = applied.skipped_pinned.saturating_add(1);
continue;
}
let wrote = match (&edit.op, edit.key) {
(Op::Set(Value::Mode(mode)), Key::Request) => {
params.set_request(*mode);
true
}
(Op::Set(Value::Proxy(pool)), Key::Proxy) => {
params.set_proxy(*pool);
true
}
(Op::Set(Value::Millis(millis)), Key::WaitIdleMillis) => {
if *millis > 0 {
params.set_idle_wait(*millis);
}
*millis > 0
}
(Op::Set(Value::Bool(value)), key) => {
params.set_flag(key, *value);
true
}
(Op::Append(entries), Key::NetworkBlacklist) => {
let mut list: Vec<String> = params
.list(Key::NetworkBlacklist)
.map(<[String]>::to_vec)
.unwrap_or_default();
for entry in entries {
if !list.contains(entry) {
list.push(entry.clone());
}
}
params.set_list(Key::NetworkBlacklist, Some(list));
true
}
(Op::Remove(entries), Key::NetworkBlacklist) => {
let list: Vec<String> = params
.list(Key::NetworkBlacklist)
.unwrap_or_default()
.iter()
.filter(|entry| !entries.contains(entry))
.cloned()
.collect();
params.set_list(Key::NetworkBlacklist, non_empty(list));
true
}
(Op::Replace(entries), Key::NetworkBlacklist) => {
let mut list: Vec<String> = Vec::with_capacity(entries.len());
for entry in entries {
if !list.contains(entry) {
list.push(entry.clone());
}
}
params.set_list(Key::NetworkBlacklist, non_empty(list));
true
}
(Op::Clear, Key::NetworkBlacklist) => {
params.set_list(Key::NetworkBlacklist, None);
true
}
_ => false,
};
if wrote {
applied.written = applied.written.saturating_add(1);
}
}
applied
}
}
fn non_empty(list: Vec<String>) -> Option<Vec<String>> {
if list.is_empty() {
None
} else {
Some(list)
}
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::indexing_slicing,
clippy::string_slice
)]
use super::*;
use crate::schema::{Schema, MODES, PROXIES, WAIT_BUCKETS};
use spider_cloud_agent::params::{RequestParams, Timeout, WaitFor};
fn append(entries: &[&str]) -> EditSet {
EditSet::new(vec![Edit {
key: Key::NetworkBlacklist,
op: Op::Append(entries.iter().map(|e| e.to_string()).collect()),
}])
.unwrap()
}
#[test]
fn append_keeps_existing_entries_and_dedupes() {
let caller = RequestParams::default();
let mut params = RequestParams {
network_blacklist: Some(vec!["a.example".into(), "b.example".into()]),
..RequestParams::default()
};
let applied = append(&["b.example", "c.example", "c.example", "a.example"])
.apply(&mut params, &caller);
assert_eq!(applied.written, 1);
assert_eq!(
params.network_blacklist.as_deref().unwrap(),
["a.example", "b.example", "c.example"]
);
}
#[test]
fn a_whitelist_on_the_caller_blocks_a_blacklist_edit() {
let caller = RequestParams {
network_whitelist: Some(vec!["example.com".into()]),
..RequestParams::default()
};
let mut params = caller.clone();
let applied = append(&["c.example"]).apply(&mut params, &caller);
assert_eq!(applied.skipped_pinned, 1);
assert_eq!(params, caller);
}
fn every_learnable_edit() -> Vec<EditSet> {
let mut sets = Vec::new();
for spec in Schema::v1().learnable() {
let edits: Vec<Edit> = match spec.key {
Key::Request => MODES
.iter()
.map(|m| Edit::set(spec.key, Value::Mode(*m)))
.collect(),
Key::Proxy => PROXIES
.iter()
.map(|p| Edit::set(spec.key, Value::Proxy(*p)))
.collect(),
Key::WaitIdleMillis => WAIT_BUCKETS
.iter()
.map(|n| Edit::set(spec.key, Value::Millis(*n)))
.collect(),
Key::NetworkBlacklist => vec![
Edit {
key: spec.key,
op: Op::Append(vec!["c.example".into()]),
},
Edit {
key: spec.key,
op: Op::Remove(vec!["example.net".into()]),
},
Edit {
key: spec.key,
op: Op::Replace(vec!["c.example".into()]),
},
Edit {
key: spec.key,
op: Op::Clear,
},
],
key => vec![
Edit::set(key, Value::Bool(true)),
Edit::set(key, Value::Bool(false)),
],
};
for edit in edits {
sets.push(EditSet::new(vec![edit]).unwrap());
}
}
sets
}
#[test]
fn an_edit_never_touches_a_field_the_caller_set() {
let caller = crate::schema::tests::populated();
let sets = every_learnable_edit();
assert!(sets.len() >= 20, "only {} edits were tried", sets.len());
for set in sets {
let mut params = caller.clone();
let applied = set.apply(&mut params, &caller);
assert_eq!(params, caller, "{set:?} wrote over the caller");
assert_eq!(applied.written, 0, "{set:?}");
assert_eq!(applied.skipped_pinned, 1, "{set:?}");
}
}
#[test]
fn an_edit_writes_where_the_caller_left_a_gap() {
let caller = RequestParams::default();
let mut params = RequestParams::default();
let set = EditSet::new(vec![
Edit::set(Key::Request, Value::Mode(RequestMode::Browser)),
Edit::set(Key::WaitIdleMillis, Value::Millis(5_000)),
Edit::set(Key::BlockAds, Value::Bool(false)),
])
.unwrap();
let applied = set.apply(&mut params, &caller);
assert_eq!(applied.written, 3);
assert_eq!(params.request, Some(RequestMode::Browser));
assert_eq!(
params.wait_for,
Some(WaitFor::idle_network(Timeout::from_millis(5_000)))
);
assert_eq!(params.block_ads, Some(false));
}
#[test]
fn an_edit_set_rejects_duplicates_and_too_many() {
let mode = Edit::set(Key::Request, Value::Mode(RequestMode::Browser));
assert_eq!(
EditSet::new(vec![mode.clone(), mode.clone()]),
Err(EditError::Duplicate(Key::Request))
);
assert_eq!(
EditSet::new(vec![
mode.clone(),
Edit::set(Key::Proxy, Value::Proxy(ProxyPool::Residential)),
Edit::set(Key::BlockAds, Value::Bool(false)),
Edit::set(Key::FullResources, Value::Bool(true)),
]),
Err(EditError::TooMany)
);
assert_eq!(
EditSet::new(vec![Edit {
key: Key::Blacklist,
op: Op::Append(vec!["/a".into()]),
}]),
Err(EditError::Unsupported(Key::Blacklist))
);
let a = EditSet::new(vec![
Edit::set(Key::BlockAds, Value::Bool(false)),
mode.clone(),
])
.unwrap();
let b = EditSet::new(vec![mode, Edit::set(Key::BlockAds, Value::Bool(false))]).unwrap();
assert_eq!(a, b);
assert_eq!(a.edits()[0].key, Key::Request);
assert!(EditSet::new(Vec::new()).unwrap().is_keep());
}
#[test]
fn a_zero_wait_writes_nothing() {
let caller = RequestParams::default();
let mut params = RequestParams::default();
let set = EditSet::new(vec![Edit::set(Key::WaitIdleMillis, Value::Millis(0))]).unwrap();
let applied = set.apply(&mut params, &caller);
assert_eq!(applied, Applied::default());
assert_eq!(params.wait_for, None);
}
}