Skip to main content

spider_optimize/
edit.rs

1//! A change to a request, and writing it.
2//!
3//! An [`EditSet`] is up to [`MAX_EDITS`] edits on distinct keys, kept in key
4//! order so two sets holding the same edits compare equal. The empty set is
5//! [`EditSet::keep`], which leaves the request exactly as it is.
6//!
7//! [`EditSet::apply`] holds the precedence rule the client's router already
8//! follows: the caller beats everything. A field set on the caller's own
9//! snapshot of the request is never written, whatever the router or the plan
10//! wrote into the request since.
11
12use crate::params::Params;
13use crate::schema::Key;
14use spider_route::{ProxyPool, RequestMode};
15use std::fmt;
16
17/// A value an edit sets.
18#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
19pub enum Value {
20    /// For a `Bool` field.
21    Bool(bool),
22    /// For `request`.
23    Mode(RequestMode),
24    /// For `proxy`.
25    Proxy(ProxyPool),
26    /// For an idle network wait, in milliseconds.
27    Millis(u32),
28    /// For an `IntRange` field. No learnable key takes one in version one, so
29    /// [`EditSet::apply`] writes nothing for it.
30    Int(i64),
31}
32
33/// What an edit does to its field.
34#[derive(Debug, Clone, PartialEq, Eq, Hash)]
35pub enum Op {
36    /// Set a single value.
37    Set(Value),
38    /// Add these entries to the end of a list, skipping any already there.
39    Append(Vec<String>),
40    /// Take these entries out of a list.
41    Remove(Vec<String>),
42    /// Replace the list with these entries.
43    Replace(Vec<String>),
44    /// Unset the list.
45    Clear,
46}
47
48impl Op {
49    /// The code a row records for this op: set 0, append 1, remove 2,
50    /// replace 3, clear 4.
51    pub const fn code(&self) -> u8 {
52        match self {
53            Op::Set(_) => 0,
54            Op::Append(_) => 1,
55            Op::Remove(_) => 2,
56            Op::Replace(_) => 3,
57            Op::Clear => 4,
58        }
59    }
60
61    /// The name of each op, in code order.
62    pub const NAMES: [&'static str; 5] = ["set", "append", "remove", "replace", "clear"];
63
64    const fn is_list(&self) -> bool {
65        !matches!(self, Op::Set(_))
66    }
67}
68
69/// One change to one field.
70#[derive(Debug, Clone, PartialEq, Eq, Hash)]
71pub struct Edit {
72    /// The field.
73    pub key: Key,
74    /// The change.
75    pub op: Op,
76}
77
78impl Edit {
79    /// Set a field to a value.
80    pub const fn set(key: Key, value: Value) -> Edit {
81        Edit {
82            key,
83            op: Op::Set(value),
84        }
85    }
86}
87
88/// The most edits one set holds.
89pub const MAX_EDITS: usize = 3;
90
91/// Why a set of edits could not be built.
92#[derive(Debug, Clone, Copy, PartialEq, Eq)]
93pub enum EditError {
94    /// Two edits name the same key.
95    Duplicate(Key),
96    /// More than [`MAX_EDITS`] edits.
97    TooMany,
98    /// A list operation on a field that is not the network blacklist.
99    Unsupported(Key),
100}
101
102impl fmt::Display for EditError {
103    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
104        match self {
105            EditError::Duplicate(key) => write!(f, "two edits name {}", key.wire()),
106            EditError::TooMany => write!(f, "an edit set holds at most {MAX_EDITS} edits"),
107            EditError::Unsupported(key) => {
108                write!(f, "{} does not take a list operation", key.wire())
109            }
110        }
111    }
112}
113
114impl std::error::Error for EditError {}
115
116/// Up to [`MAX_EDITS`] edits on distinct keys, in key order.
117#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
118pub struct EditSet {
119    edits: Vec<Edit>,
120}
121
122/// What [`EditSet::apply`] did.
123#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
124pub struct Applied {
125    /// Edits written onto the request.
126    pub written: u8,
127    /// Edits left out because the caller set the field.
128    pub skipped_pinned: u8,
129}
130
131impl EditSet {
132    /// No edits: the request goes out as it is.
133    pub const fn keep() -> EditSet {
134        EditSet { edits: Vec::new() }
135    }
136
137    /// Whether this is the set that changes nothing.
138    pub fn is_keep(&self) -> bool {
139        self.edits.is_empty()
140    }
141
142    /// Build a set, sorted by key.
143    ///
144    /// Refuses two edits on one key, more than [`MAX_EDITS`] edits, and a
145    /// list operation on anything but [`Key::NetworkBlacklist`].
146    pub fn new(mut edits: Vec<Edit>) -> Result<EditSet, EditError> {
147        if edits.len() > MAX_EDITS {
148            return Err(EditError::TooMany);
149        }
150        for edit in &edits {
151            if edit.op.is_list() && edit.key != Key::NetworkBlacklist {
152                return Err(EditError::Unsupported(edit.key));
153            }
154        }
155        edits.sort_by_key(|edit| edit.key.index());
156        for pair in edits.windows(2) {
157            if let [first, second] = pair {
158                if first.key == second.key {
159                    return Err(EditError::Duplicate(first.key));
160                }
161            }
162        }
163        Ok(EditSet { edits })
164    }
165
166    /// The edits, in key order.
167    pub fn edits(&self) -> &[Edit] {
168        &self.edits
169    }
170
171    /// The edit on this key, if there is one.
172    pub fn get(&self, key: Key) -> Option<&Edit> {
173        self.edits.iter().find(|edit| edit.key == key)
174    }
175
176    /// Write the edits onto `params`, leaving every field the caller set.
177    ///
178    /// `caller` is the request as the caller left it, before the router or
179    /// the plan wrote anything. A `Set` is written only where the caller's
180    /// field is unset. An idle wait of zero writes nothing, because there is
181    /// no wait to send. A list operation on the blacklist is left out when
182    /// the caller set either network list, since the caller's whitelist wins
183    /// over any blacklist entry and an edit must not argue with it.
184    pub fn apply(&self, params: &mut dyn Params, caller: &dyn Params) -> Applied {
185        let mut applied = Applied::default();
186
187        for edit in &self.edits {
188            let pinned = match edit.key {
189                Key::NetworkBlacklist => {
190                    caller.is_set(Key::NetworkBlacklist) || caller.is_set(Key::NetworkWhitelist)
191                }
192                key => caller.is_set(key),
193            };
194            if pinned {
195                applied.skipped_pinned = applied.skipped_pinned.saturating_add(1);
196                continue;
197            }
198
199            let wrote = match (&edit.op, edit.key) {
200                (Op::Set(Value::Mode(mode)), Key::Request) => {
201                    params.set_request(*mode);
202                    true
203                }
204                (Op::Set(Value::Proxy(pool)), Key::Proxy) => {
205                    params.set_proxy(*pool);
206                    true
207                }
208                (Op::Set(Value::Millis(millis)), Key::WaitIdleMillis) => {
209                    if *millis > 0 {
210                        params.set_idle_wait(*millis);
211                    }
212                    *millis > 0
213                }
214                (Op::Set(Value::Bool(value)), key) => {
215                    params.set_flag(key, *value);
216                    true
217                }
218                (Op::Append(entries), Key::NetworkBlacklist) => {
219                    let mut list: Vec<String> = params
220                        .list(Key::NetworkBlacklist)
221                        .map(<[String]>::to_vec)
222                        .unwrap_or_default();
223                    for entry in entries {
224                        if !list.contains(entry) {
225                            list.push(entry.clone());
226                        }
227                    }
228                    params.set_list(Key::NetworkBlacklist, Some(list));
229                    true
230                }
231                (Op::Remove(entries), Key::NetworkBlacklist) => {
232                    let list: Vec<String> = params
233                        .list(Key::NetworkBlacklist)
234                        .unwrap_or_default()
235                        .iter()
236                        .filter(|entry| !entries.contains(entry))
237                        .cloned()
238                        .collect();
239                    params.set_list(Key::NetworkBlacklist, non_empty(list));
240                    true
241                }
242                (Op::Replace(entries), Key::NetworkBlacklist) => {
243                    let mut list: Vec<String> = Vec::with_capacity(entries.len());
244                    for entry in entries {
245                        if !list.contains(entry) {
246                            list.push(entry.clone());
247                        }
248                    }
249                    params.set_list(Key::NetworkBlacklist, non_empty(list));
250                    true
251                }
252                (Op::Clear, Key::NetworkBlacklist) => {
253                    params.set_list(Key::NetworkBlacklist, None);
254                    true
255                }
256                // A value of the wrong type for its key, or an integer, which
257                // no learnable key takes. Validation refuses both before an
258                // edit gets here, so nothing is written.
259                _ => false,
260            };
261
262            if wrote {
263                applied.written = applied.written.saturating_add(1);
264            }
265        }
266
267        applied
268    }
269}
270
271/// An empty list is sent as no list at all.
272fn non_empty(list: Vec<String>) -> Option<Vec<String>> {
273    if list.is_empty() {
274        None
275    } else {
276        Some(list)
277    }
278}
279
280#[cfg(test)]
281mod tests {
282    #![allow(
283        clippy::unwrap_used,
284        clippy::expect_used,
285        clippy::panic,
286        clippy::unreachable,
287        clippy::indexing_slicing,
288        clippy::string_slice
289    )]
290    use super::*;
291    use crate::schema::{Schema, MODES, PROXIES, WAIT_BUCKETS};
292    use spider_cloud_agent::params::{RequestParams, Timeout, WaitFor};
293
294    fn append(entries: &[&str]) -> EditSet {
295        EditSet::new(vec![Edit {
296            key: Key::NetworkBlacklist,
297            op: Op::Append(entries.iter().map(|e| e.to_string()).collect()),
298        }])
299        .unwrap()
300    }
301
302    #[test]
303    fn append_keeps_existing_entries_and_dedupes() {
304        let caller = RequestParams::default();
305        // Written by the router or the plan, not the caller.
306        let mut params = RequestParams {
307            network_blacklist: Some(vec!["a.example".into(), "b.example".into()]),
308            ..RequestParams::default()
309        };
310
311        let applied = append(&["b.example", "c.example", "c.example", "a.example"])
312            .apply(&mut params, &caller);
313
314        assert_eq!(applied.written, 1);
315        assert_eq!(
316            params.network_blacklist.as_deref().unwrap(),
317            ["a.example", "b.example", "c.example"]
318        );
319    }
320
321    #[test]
322    fn a_whitelist_on_the_caller_blocks_a_blacklist_edit() {
323        let caller = RequestParams {
324            network_whitelist: Some(vec!["example.com".into()]),
325            ..RequestParams::default()
326        };
327        let mut params = caller.clone();
328
329        let applied = append(&["c.example"]).apply(&mut params, &caller);
330
331        assert_eq!(applied.skipped_pinned, 1);
332        assert_eq!(params, caller);
333    }
334
335    /// Every value of every learnable key, one edit per set.
336    fn every_learnable_edit() -> Vec<EditSet> {
337        let mut sets = Vec::new();
338        for spec in Schema::v1().learnable() {
339            let edits: Vec<Edit> = match spec.key {
340                Key::Request => MODES
341                    .iter()
342                    .map(|m| Edit::set(spec.key, Value::Mode(*m)))
343                    .collect(),
344                Key::Proxy => PROXIES
345                    .iter()
346                    .map(|p| Edit::set(spec.key, Value::Proxy(*p)))
347                    .collect(),
348                Key::WaitIdleMillis => WAIT_BUCKETS
349                    .iter()
350                    .map(|n| Edit::set(spec.key, Value::Millis(*n)))
351                    .collect(),
352                Key::NetworkBlacklist => vec![
353                    Edit {
354                        key: spec.key,
355                        op: Op::Append(vec!["c.example".into()]),
356                    },
357                    Edit {
358                        key: spec.key,
359                        op: Op::Remove(vec!["example.net".into()]),
360                    },
361                    Edit {
362                        key: spec.key,
363                        op: Op::Replace(vec!["c.example".into()]),
364                    },
365                    Edit {
366                        key: spec.key,
367                        op: Op::Clear,
368                    },
369                ],
370                key => vec![
371                    Edit::set(key, Value::Bool(true)),
372                    Edit::set(key, Value::Bool(false)),
373                ],
374            };
375            for edit in edits {
376                sets.push(EditSet::new(vec![edit]).unwrap());
377            }
378        }
379        sets
380    }
381
382    #[test]
383    fn an_edit_never_touches_a_field_the_caller_set() {
384        let caller = crate::schema::tests::populated();
385
386        let sets = every_learnable_edit();
387        assert!(sets.len() >= 20, "only {} edits were tried", sets.len());
388
389        for set in sets {
390            let mut params = caller.clone();
391            let applied = set.apply(&mut params, &caller);
392            assert_eq!(params, caller, "{set:?} wrote over the caller");
393            assert_eq!(applied.written, 0, "{set:?}");
394            assert_eq!(applied.skipped_pinned, 1, "{set:?}");
395        }
396    }
397
398    #[test]
399    fn an_edit_writes_where_the_caller_left_a_gap() {
400        // The counterpart to the test above, so it cannot pass by writing
401        // nothing anywhere.
402        let caller = RequestParams::default();
403        let mut params = RequestParams::default();
404        let set = EditSet::new(vec![
405            Edit::set(Key::Request, Value::Mode(RequestMode::Browser)),
406            Edit::set(Key::WaitIdleMillis, Value::Millis(5_000)),
407            Edit::set(Key::BlockAds, Value::Bool(false)),
408        ])
409        .unwrap();
410
411        let applied = set.apply(&mut params, &caller);
412
413        assert_eq!(applied.written, 3);
414        assert_eq!(params.request, Some(RequestMode::Browser));
415        assert_eq!(
416            params.wait_for,
417            Some(WaitFor::idle_network(Timeout::from_millis(5_000)))
418        );
419        assert_eq!(params.block_ads, Some(false));
420    }
421
422    #[test]
423    fn an_edit_set_rejects_duplicates_and_too_many() {
424        let mode = Edit::set(Key::Request, Value::Mode(RequestMode::Browser));
425        assert_eq!(
426            EditSet::new(vec![mode.clone(), mode.clone()]),
427            Err(EditError::Duplicate(Key::Request))
428        );
429        assert_eq!(
430            EditSet::new(vec![
431                mode.clone(),
432                Edit::set(Key::Proxy, Value::Proxy(ProxyPool::Residential)),
433                Edit::set(Key::BlockAds, Value::Bool(false)),
434                Edit::set(Key::FullResources, Value::Bool(true)),
435            ]),
436            Err(EditError::TooMany)
437        );
438        assert_eq!(
439            EditSet::new(vec![Edit {
440                key: Key::Blacklist,
441                op: Op::Append(vec!["/a".into()]),
442            }]),
443            Err(EditError::Unsupported(Key::Blacklist))
444        );
445
446        // Order does not matter to what the set is.
447        let a = EditSet::new(vec![
448            Edit::set(Key::BlockAds, Value::Bool(false)),
449            mode.clone(),
450        ])
451        .unwrap();
452        let b = EditSet::new(vec![mode, Edit::set(Key::BlockAds, Value::Bool(false))]).unwrap();
453        assert_eq!(a, b);
454        assert_eq!(a.edits()[0].key, Key::Request);
455        assert!(EditSet::new(Vec::new()).unwrap().is_keep());
456    }
457
458    #[test]
459    fn a_zero_wait_writes_nothing() {
460        let caller = RequestParams::default();
461        let mut params = RequestParams::default();
462        let set = EditSet::new(vec![Edit::set(Key::WaitIdleMillis, Value::Millis(0))]).unwrap();
463
464        let applied = set.apply(&mut params, &caller);
465
466        assert_eq!(applied, Applied::default());
467        assert_eq!(params.wait_for, None);
468    }
469}