Skip to main content

spider_optimize/
validate.rs

1//! Whether a candidate may be applied at all, before anything scores it.
2//!
3//! The checks run in a fixed order and the first failure is the answer, so a
4//! rejection names the most basic thing wrong with a candidate: a key the
5//! caller set, then a key that changes content, then a key or operation this
6//! version does not learn, then a value outside its kind, then a setting that
7//! does nothing on the request it lands on, then the service's own hints,
8//! then a rate limit, then the budget. Keep is never rejected.
9//!
10//! # The content exception
11//!
12//! Version one refuses every edit on a `content_changing` key except two:
13//! `block_stylesheets` and `network_blacklist`. Both are learnable, and both
14//! can remove what a page shows, which is exactly why the label pipeline
15//! compares a candidate's content against its baseline with
16//! [`crate::labels`] before a row counts as a success. Any other content
17//! changing key would change what the caller asked for rather than how it is
18//! fetched, and no label can say that was fine.
19
20use crate::candidates::{
21    current_mode, current_proxy, current_wait, effective_flag, mode_rank, multiplier, proxy_rank,
22    resulting, Candidate, Context,
23};
24use crate::edit::{EditSet, Op, Value};
25use crate::observe::MAX_IDENTIFIERS;
26use crate::schema::{Dependency, Key, Kind, Schema, MODES, PROXIES};
27use spider_route::RequestMode;
28use std::fmt;
29
30/// Why a candidate was refused.
31#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
32pub enum Rejection {
33    /// The caller set this field.
34    PinnedByCaller(Key),
35    /// This version does not edit this field, or not with this operation.
36    Unsupported(Key),
37    /// The value is outside what this field's kind allows.
38    OutOfRange(Key),
39    /// The field does nothing on the request as edited.
40    Dependency(Key),
41    /// The site asked for a slower pace, and this candidate is heavier than
42    /// the request as it stands.
43    RateLimited,
44    /// The candidate costs more than the caller's budget allows.
45    OverBudget,
46    /// Changing this field can change what comes back.
47    ContentChanging(Key),
48    /// A blacklist edit while the service applies its own blocking hints, so
49    /// what the edit did could not be told apart from what the service did.
50    HintsActive,
51}
52
53impl fmt::Display for Rejection {
54    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
55        match self {
56            Rejection::PinnedByCaller(key) => write!(f, "the caller set {}", key.wire()),
57            Rejection::Unsupported(key) => write!(f, "{} is not edited this way", key.wire()),
58            Rejection::OutOfRange(key) => write!(f, "the value for {} is out of range", key.wire()),
59            Rejection::Dependency(key) => {
60                write!(f, "{} does nothing on this request", key.wire())
61            }
62            Rejection::RateLimited => f.write_str("heavier settings under a rate limit"),
63            Rejection::OverBudget => f.write_str("dearer than the budget allows"),
64            Rejection::ContentChanging(key) => {
65                write!(f, "{} can change the content", key.wire())
66            }
67            Rejection::HintsActive => {
68                f.write_str("a blacklist edit needs the service's hints turned off")
69            }
70        }
71    }
72}
73
74/// The content changing keys version one still edits. See the module docs.
75const CONTENT_EXCEPTIONS: [Key; 2] = [Key::BlockStylesheets, Key::NetworkBlacklist];
76
77/// Check one candidate against the schema and the context.
78///
79/// Allocates nothing.
80pub fn validate(schema: &Schema, ctx: &Context<'_>, cand: &Candidate) -> Result<(), Rejection> {
81    let edits = &cand.edits;
82    if edits.is_keep() {
83        return Ok(());
84    }
85
86    for edit in edits.edits() {
87        let key = edit.key;
88        let spec = schema.spec(key);
89
90        if ctx.caller.is_set(key) {
91            return Err(Rejection::PinnedByCaller(key));
92        }
93        if key == Key::NetworkBlacklist && ctx.caller.is_set(Key::NetworkWhitelist) {
94            return Err(Rejection::PinnedByCaller(Key::NetworkWhitelist));
95        }
96        if spec.content_changing && !CONTENT_EXCEPTIONS.contains(&key) {
97            return Err(Rejection::ContentChanging(key));
98        }
99        if !spec.learnable {
100            return Err(Rejection::Unsupported(key));
101        }
102
103        match (spec.kind, &edit.op) {
104            (Kind::Enum(_), Op::Set(Value::Mode(mode))) if key == Key::Request => {
105                if !MODES.contains(mode) {
106                    return Err(Rejection::OutOfRange(key));
107                }
108            }
109            (Kind::Enum(_), Op::Set(Value::Proxy(pool))) if key == Key::Proxy => {
110                if !PROXIES.contains(pool) {
111                    return Err(Rejection::OutOfRange(key));
112                }
113            }
114            (Kind::MillisBucket(buckets), Op::Set(Value::Millis(millis))) => {
115                if !buckets.contains(millis) {
116                    return Err(Rejection::OutOfRange(key));
117                }
118            }
119            (Kind::Bool, Op::Set(Value::Bool(_))) => {}
120            // Appending is the one list edit in version one.
121            (Kind::StringList, Op::Append(entries)) => {
122                if entries.is_empty()
123                    || entries.len() > MAX_IDENTIFIERS
124                    || !entries.iter().all(|entry| is_pattern(entry))
125                {
126                    return Err(Rejection::OutOfRange(key));
127                }
128            }
129            _ => return Err(Rejection::Unsupported(key)),
130        }
131    }
132
133    let (mode, _, _) = resulting(ctx, edits);
134    for edit in edits.edits() {
135        for dependency in schema.spec(edit.key).requires {
136            let met = match dependency {
137                Dependency::Rendered => {
138                    matches!(mode, RequestMode::Smart | RequestMode::Browser)
139                }
140                // An edit that switches its own field off cannot conflict
141                // with the other one, so only an edit that leaves its field
142                // on is held to this.
143                Dependency::NotWith(other) => {
144                    !active(ctx, edits, edit.key) || !active(ctx, edits, *other)
145                }
146            };
147            if !met {
148                return Err(Rejection::Dependency(edit.key));
149            }
150        }
151    }
152
153    if edits.get(Key::NetworkBlacklist).is_some()
154        && ctx.current.flag(Key::DisableHints) != Some(true)
155    {
156        return Err(Rejection::HintsActive);
157    }
158
159    if ctx.observation.rate_limited() && heavier(ctx, edits) {
160        return Err(Rejection::RateLimited);
161    }
162
163    // A cap that is not a number refuses rather than allows.
164    if ctx.multiplier_cap.is_nan() || multiplier(ctx, edits) > ctx.multiplier_cap {
165        return Err(Rejection::OverBudget);
166    }
167
168    Ok(())
169}
170
171/// Whether text can go into the blacklist as one pattern: not empty, no
172/// whitespace or control characters, and no longer than a host name.
173fn is_pattern(entry: &str) -> bool {
174    !entry.is_empty()
175        && entry.len() <= 253
176        && !entry
177            .chars()
178            .any(|ch| ch.is_whitespace() || ch.is_control())
179}
180
181/// Whether a field is switched on in the request once these edits apply.
182fn active(ctx: &Context<'_>, edits: &EditSet, key: Key) -> bool {
183    match edits.get(key).map(|edit| &edit.op) {
184        Some(Op::Set(Value::Bool(value))) => *value,
185        Some(Op::Append(entries)) => !entries.is_empty(),
186        Some(_) => true,
187        None => match key {
188            Key::NetworkBlacklist | Key::NetworkWhitelist => {
189                ctx.current.list(key).is_some_and(|list| !list.is_empty())
190            }
191            _ => effective_flag(ctx.current, key),
192        },
193    }
194}
195
196/// Whether these edits ask the site for more than the request as it stands.
197///
198/// A dearer mode, pool or price, a longer wait, or letting the page load
199/// something it would not have loaded. Blocking more is never heavier.
200pub(crate) fn heavier(ctx: &Context<'_>, edits: &EditSet) -> bool {
201    let (mode, proxy, wait) = resulting(ctx, edits);
202    if mode_rank(mode) > mode_rank(current_mode(ctx.current))
203        || proxy_rank(proxy) > proxy_rank(current_proxy(ctx.current))
204        || wait > current_wait(ctx.current)
205        || multiplier(ctx, edits) > multiplier(ctx, &EditSet::keep())
206    {
207        return true;
208    }
209
210    edits.edits().iter().any(|edit| match (&edit.op, edit.key) {
211        (Op::Set(Value::Bool(value)), Key::FullResources | Key::DisableIntercept) => {
212            *value && !effective_flag(ctx.current, edit.key)
213        }
214        (
215            Op::Set(Value::Bool(value)),
216            Key::BlockAds | Key::BlockAnalytics | Key::BlockStylesheets,
217        ) => !*value && effective_flag(ctx.current, edit.key),
218        _ => false,
219    })
220}
221
222#[cfg(test)]
223mod tests {
224    #![allow(
225        clippy::unwrap_used,
226        clippy::expect_used,
227        clippy::panic,
228        clippy::unreachable,
229        clippy::indexing_slicing,
230        clippy::string_slice
231    )]
232    use super::*;
233    use crate::candidates::generate;
234    use crate::edit::Edit;
235    use crate::testing::Fixture;
236    use spider_route::{ProxyPool, StatusClass};
237
238    fn cand(ctx: &Context<'_>, edits: Vec<Edit>) -> Candidate {
239        let edits = EditSet::new(edits).unwrap();
240        Candidate {
241            multiplier: multiplier(ctx, &edits),
242            edits,
243        }
244    }
245
246    fn check(fixture: &Fixture, edits: Vec<Edit>) -> Result<(), Rejection> {
247        let ctx = fixture.ctx();
248        validate(&Schema::v1(), &ctx, &cand(&ctx, edits))
249    }
250
251    fn append(pattern: &str) -> Edit {
252        Edit {
253            key: Key::NetworkBlacklist,
254            op: Op::Append(vec![pattern.into()]),
255        }
256    }
257
258    #[test]
259    fn a_pinned_key_is_rejected_by_name() {
260        let fixture = Fixture::pinned();
261        assert_eq!(
262            check(
263                &fixture,
264                vec![Edit::set(Key::Request, Value::Mode(RequestMode::Http))]
265            ),
266            Err(Rejection::PinnedByCaller(Key::Request))
267        );
268        let mut whitelisted = Fixture::observed();
269        whitelisted.caller.network_whitelist = Some(vec!["example.com".into()]);
270        assert_eq!(
271            check(&whitelisted, vec![append("cdn-beta.example")]),
272            Err(Rejection::PinnedByCaller(Key::NetworkWhitelist))
273        );
274    }
275
276    #[test]
277    fn rate_limit_rejects_every_heavier_candidate() {
278        let schema = Schema::v1();
279        let mut limited = Fixture::observed();
280        limited.observation.last_status = StatusClass::RateLimited;
281        let calm = Fixture::observed();
282
283        // Every candidate the calm request would get, checked under a limit.
284        let everything = generate(&schema, &calm.ctx());
285        let ctx = limited.ctx();
286        let mut heavier_seen = 0;
287        let mut lighter_seen = 0;
288        for candidate in &everything {
289            if heavier(&ctx, &candidate.edits) {
290                heavier_seen += 1;
291                assert_eq!(
292                    validate(&schema, &ctx, candidate),
293                    Err(Rejection::RateLimited),
294                    "{:?}",
295                    candidate.edits
296                );
297            } else {
298                lighter_seen += 1;
299                assert_eq!(validate(&schema, &ctx, candidate), Ok(()));
300            }
301        }
302        assert!(heavier_seen >= 8, "only {heavier_seen} heavier candidates");
303        // Keep, a plain fetch, and blocking more survive.
304        assert!(lighter_seen >= 3, "only {lighter_seen} lighter candidates");
305
306        for candidate in generate(&schema, &ctx) {
307            assert!(!heavier(&ctx, &candidate.edits), "{:?}", candidate.edits);
308        }
309
310        // The memory's last status counts the same as the observation's.
311        let mut remembered = Fixture::observed();
312        remembered.observation.memory = Some(spider_route::SiteMemory {
313            last_status: StatusClass::RateLimited,
314            ..spider_route::SiteMemory::cold()
315        });
316        assert_eq!(
317            check(
318                &remembered,
319                vec![Edit::set(Key::Proxy, Value::Proxy(ProxyPool::Residential))]
320            ),
321            Err(Rejection::RateLimited)
322        );
323    }
324
325    #[test]
326    fn a_dependency_violation_is_rejected() {
327        let mut fixture = Fixture::cold();
328        fixture.caller.request = Some(RequestMode::Http);
329        fixture.current.request = Some(RequestMode::Http);
330
331        assert_eq!(
332            check(
333                &fixture,
334                vec![Edit::set(Key::WaitIdleMillis, Value::Millis(5_000))]
335            ),
336            Err(Rejection::Dependency(Key::WaitIdleMillis))
337        );
338        assert_eq!(
339            check(&fixture, vec![Edit::set(Key::BlockAds, Value::Bool(false))]),
340            Err(Rejection::Dependency(Key::BlockAds))
341        );
342
343        // A blacklist append beside `disable_intercept` is allowed in both
344        // directions.
345        let mut observed = Fixture::observed();
346        observed.current.disable_intercept = Some(true);
347        assert_eq!(check(&observed, vec![append("cdn-beta.example")]), Ok(()));
348        let mut listed = Fixture::observed();
349        listed.current.network_blacklist = Some(vec!["cdn-beta.example".into()]);
350        assert_eq!(
351            check(
352                &listed,
353                vec![Edit::set(Key::DisableIntercept, Value::Bool(true))]
354            ),
355            Ok(())
356        );
357        let rendered = Fixture::cold();
358        assert_eq!(
359            check(
360                &rendered,
361                vec![Edit::set(Key::WaitIdleMillis, Value::Millis(5_000))]
362            ),
363            Ok(())
364        );
365    }
366
367    #[test]
368    fn blacklist_needs_disable_hints() {
369        let mut fixture = Fixture::observed();
370        assert_eq!(check(&fixture, vec![append("cdn-beta.example")]), Ok(()));
371
372        fixture.current.disable_hints = None;
373        assert_eq!(
374            check(&fixture, vec![append("cdn-beta.example")]),
375            Err(Rejection::HintsActive)
376        );
377        fixture.current.disable_hints = Some(false);
378        assert_eq!(
379            check(&fixture, vec![append("cdn-beta.example")]),
380            Err(Rejection::HintsActive)
381        );
382        assert!(generate(&Schema::v1(), &fixture.ctx())
383            .iter()
384            .all(|c| c.edits.get(Key::NetworkBlacklist).is_none()));
385    }
386
387    #[test]
388    fn over_budget_is_rejected() {
389        let mut fixture = Fixture::cold();
390        fixture.cap = 3.0;
391        let browser = vec![Edit::set(Key::Request, Value::Mode(RequestMode::Browser))];
392        assert_eq!(check(&fixture, browser.clone()), Err(Rejection::OverBudget));
393
394        fixture.cap = 4.0;
395        assert_eq!(check(&fixture, browser.clone()), Ok(()));
396
397        fixture.cap = f32::NAN;
398        assert_eq!(check(&fixture, browser), Err(Rejection::OverBudget));
399        assert!(generate(&Schema::v1(), &fixture.ctx()).len() == 1);
400    }
401
402    #[test]
403    fn content_values_and_operations_outside_version_one_are_refused() {
404        let fixture = Fixture::observed();
405        assert_eq!(
406            check(
407                &fixture,
408                vec![Edit::set(Key::Readability, Value::Bool(true))]
409            ),
410            Err(Rejection::ContentChanging(Key::Readability))
411        );
412        assert_eq!(
413            check(&fixture, vec![Edit::set(Key::Stealth, Value::Bool(true))]),
414            Err(Rejection::Unsupported(Key::Stealth))
415        );
416        assert_eq!(
417            check(
418                &fixture,
419                vec![Edit::set(Key::WaitIdleMillis, Value::Millis(3_000))]
420            ),
421            Err(Rejection::OutOfRange(Key::WaitIdleMillis))
422        );
423        assert_eq!(
424            check(&fixture, vec![Edit::set(Key::Request, Value::Bool(true))]),
425            Err(Rejection::Unsupported(Key::Request))
426        );
427        assert_eq!(
428            check(
429                &fixture,
430                vec![Edit {
431                    key: Key::NetworkBlacklist,
432                    op: Op::Clear,
433                }]
434            ),
435            Err(Rejection::Unsupported(Key::NetworkBlacklist))
436        );
437        assert_eq!(
438            check(&fixture, vec![append("two words")]),
439            Err(Rejection::OutOfRange(Key::NetworkBlacklist))
440        );
441        assert_eq!(
442            check(
443                &fixture,
444                vec![Edit::set(Key::BlockStylesheets, Value::Bool(false))]
445            ),
446            Ok(())
447        );
448    }
449}