Skip to main content

spider_optimize/
lib.rs

1//! A second decision layer for Spider Cloud requests.
2//!
3//! `spider-route` picks the settings of a first attempt from rules. This
4//! crate runs after the router and the request plan, when the request is
5//! about to go out. It generates valid candidate edits to the request
6//! parameters ([`generate`]), scores each for success, latency and credits
7//! through a [`Scorer`], and applies at most one edit set, only when it
8//! passes [`validate()`] and the evidence [`Gate`]. Otherwise the request goes
9//! out exactly as it was. Leaving it unchanged is always candidate zero and
10//! the baseline every other candidate is measured against.
11//!
12//! Like the router, it is sync, does no input or output, holds no clock,
13//! reads no environment, and allocates nothing while it scores. With no model
14//! compiled in, [`NoModel`] abstains and every request is kept.
15//!
16//! # What never reaches a number
17//!
18//! No host, domain, address or page body reaches a feature slot, a recorded
19//! row or a public numeric type. The one place a resource identifier lives is
20//! [`Identifier::pattern`], which exists to be written into
21//! `network_blacklist` and nowhere else.
22//!
23//! # The request type
24//!
25//! The request body is defined by the client crate, which depends on this one,
26//! so this crate reads and writes it through the [`Params`] trait. The client
27//! implements it for its own request type.
28
29#![cfg_attr(docsrs, feature(doc_cfg))]
30#![forbid(unsafe_code)]
31#![warn(missing_docs)]
32
33pub mod artifact;
34pub mod candidates;
35pub mod edit;
36pub mod features;
37pub mod gate;
38pub mod labels;
39pub mod model;
40pub mod monitor;
41pub mod observe;
42pub mod params;
43pub mod row;
44pub mod schema;
45pub mod validate;
46
47#[cfg(feature = "embedded-model")]
48pub use crate::artifact::embedded;
49pub use crate::artifact::{cell_id, ArtifactError, Compact, ModelKind, MAX_WIDTH};
50pub use candidates::{generate, Candidate, Context, MAX_CANDIDATES, MULTIPLIERS};
51pub use edit::{Applied, Edit, EditError, EditSet, Op, Value, MAX_EDITS};
52pub use features::{featurize_edit, EditFeatures, Input, EDIT_DIM, EDIT_FEATURE_VERSION};
53pub use gate::{choose, Choice, Gate, Reason};
54pub use labels::{byte_ratio, fields_ok, shingle_jaccard};
55pub use model::{ModelVersion, NoModel, Score, Scorer};
56pub use monitor::{Estimate, Monitor, MonitorConfig, Verdict};
57pub use observe::{summarize, Identifier, Observation, ResourceSummary, MAX_IDENTIFIERS};
58pub use params::Params;
59pub use row::{
60    comparison_row, Arm, ComparisonRow, EditDescriptor, IdentDescriptor, MemoryState,
61    CMP_ROW_VERSION,
62};
63pub use schema::{
64    edit_code, Dependency, Group, Key, Kind, ParamSpec, Schema, KEEP_CODE, SCHEMA_VERSION,
65};
66pub use validate::{validate, Rejection};
67
68#[cfg(test)]
69mod testing {
70    //! The client's request type, read and written for the tests.
71    //!
72    //! The client crate implements [`Params`] for its own type when it takes
73    //! this crate on. Until then, and because a test build of this crate is a
74    //! different crate from the one the client links, the tests carry their
75    //! own implementation. Presence and flags are read off the serialized
76    //! body, so this cannot drift from the struct.
77    #![allow(
78        clippy::unwrap_used,
79        clippy::expect_used,
80        clippy::panic,
81        clippy::unreachable,
82        clippy::indexing_slicing,
83        clippy::string_slice
84    )]
85    use crate::candidates::Context;
86    use crate::observe::{summarize, Observation};
87    use crate::params::Params;
88    use crate::schema::Key;
89    use spider_cloud_agent::params::{RequestParams, Timeout, WaitFor};
90    use spider_route::{DeclaredNeed, ProxyPool, RequestMode, RouteDecision};
91    use url::Url;
92
93    /// Everything a [`Context`] borrows, owned, so a test can change one
94    /// piece and borrow the rest.
95    pub(crate) struct Fixture {
96        pub url: Url,
97        pub need: DeclaredNeed,
98        pub current: RequestParams,
99        pub caller: RequestParams,
100        pub routed: RouteDecision,
101        pub observation: Observation,
102        pub cap: f32,
103    }
104
105    impl Fixture {
106        /// A smart fetch, nothing pinned, nothing known.
107        pub fn cold() -> Fixture {
108            Fixture {
109                url: Url::parse("https://example.com/articles/one").unwrap(),
110                need: DeclaredNeed::Markdown,
111                current: RequestParams {
112                    request: Some(RequestMode::Smart),
113                    ..RequestParams::default()
114                },
115                caller: RequestParams::default(),
116                routed: RouteDecision::default(),
117                observation: Observation::cold(),
118                cap: 8.0,
119            }
120        }
121
122        /// A smart fetch with hints off and four third party groups seen.
123        pub fn observed() -> Fixture {
124            let mut fixture = Fixture::cold();
125            fixture.current.disable_hints = Some(true);
126            let resources: Vec<(Url, u64)> = [
127                ("https://example.com/app.js", 40_000),
128                ("https://tracker-alpha.example/t.js", 90_000),
129                ("https://cdn-beta.example/big.js", 300_000),
130                ("https://ads-gamma.example/a.gif", 20_000),
131                ("https://fonts-delta.example/f.woff2", 5_000),
132                ("https://widget-epsilon.example/w.js", 1_000),
133            ]
134            .iter()
135            .map(|(url, bytes)| (Url::parse(url).unwrap(), *bytes))
136            .collect();
137            fixture.observation.resources = Some(summarize(&fixture.url, &resources));
138            fixture
139        }
140
141        /// The caller fixed the mode, the pool and a selector wait.
142        pub fn pinned() -> Fixture {
143            let mut fixture = Fixture::observed();
144            fixture.caller.request = Some(RequestMode::Browser);
145            fixture.caller.proxy = Some(ProxyPool::Isp);
146            fixture.caller.wait_for = Some(WaitFor::selector("main", Timeout::from_secs(2)));
147            fixture.current.request = fixture.caller.request;
148            fixture.current.proxy = fixture.caller.proxy;
149            fixture.current.wait_for = fixture.caller.wait_for.clone();
150            fixture
151        }
152
153        pub fn ctx(&self) -> Context<'_> {
154            Context {
155                url: &self.url,
156                need: self.need,
157                current: &self.current,
158                caller: &self.caller,
159                routed: &self.routed,
160                observation: &self.observation,
161                multiplier_cap: self.cap,
162            }
163        }
164    }
165
166    fn body(params: &RequestParams) -> serde_json::Map<String, serde_json::Value> {
167        match serde_json::to_value(params).expect("a request serializes") {
168            serde_json::Value::Object(map) => map,
169            other => panic!("a request is an object, got {other}"),
170        }
171    }
172
173    impl Params for RequestParams {
174        fn is_set(&self, key: Key) -> bool {
175            body(self).contains_key(key.wire())
176        }
177
178        fn request(&self) -> Option<RequestMode> {
179            self.request
180        }
181
182        fn proxy(&self) -> Option<ProxyPool> {
183            self.proxy
184        }
185
186        fn idle_wait_millis(&self) -> Option<u32> {
187            let idle = self.wait_for.as_ref()?.idle_network?;
188            let millis = idle.timeout.secs * 1_000 + u64::from(idle.timeout.nanos / 1_000_000);
189            Some(millis.min(u64::from(u32::MAX)) as u32)
190        }
191
192        fn flag(&self, key: Key) -> Option<bool> {
193            body(self)
194                .get(key.wire())
195                .and_then(serde_json::Value::as_bool)
196        }
197
198        fn list(&self, key: Key) -> Option<&[String]> {
199            match key {
200                Key::NetworkBlacklist => self.network_blacklist.as_deref(),
201                Key::NetworkWhitelist => self.network_whitelist.as_deref(),
202                Key::Blacklist => self.blacklist.as_deref(),
203                Key::Whitelist => self.whitelist.as_deref(),
204                Key::ExternalDomains => self.external_domains.as_deref(),
205                _ => None,
206            }
207        }
208
209        fn set_request(&mut self, mode: RequestMode) {
210            self.request = Some(mode);
211        }
212
213        fn set_proxy(&mut self, pool: ProxyPool) {
214            self.proxy = Some(pool);
215        }
216
217        fn set_idle_wait(&mut self, millis: u32) {
218            self.wait_for = Some(WaitFor::idle_network(Timeout::from_millis(u64::from(
219                millis,
220            ))));
221        }
222
223        fn set_flag(&mut self, key: Key, value: bool) {
224            let slot = match key {
225                Key::BlockStylesheets => &mut self.block_stylesheets,
226                Key::BlockAds => &mut self.block_ads,
227                Key::BlockAnalytics => &mut self.block_analytics,
228                Key::FullResources => &mut self.full_resources,
229                Key::DisableIntercept => &mut self.disable_intercept,
230                Key::DisableHints => &mut self.disable_hints,
231                _ => return,
232            };
233            *slot = Some(value);
234        }
235
236        fn set_list(&mut self, key: Key, list: Option<Vec<String>>) {
237            if key == Key::NetworkBlacklist {
238                self.network_blacklist = list;
239            }
240        }
241    }
242}