1#![cfg_attr(docsrs, feature(doc_cfg))]
30#![forbid(unsafe_code)]
31#![warn(missing_docs)]
32
33pub mod artifact;
34pub mod candidates;
35pub mod edit;
36pub mod features;
37pub mod gate;
38pub mod labels;
39pub mod model;
40pub mod monitor;
41pub mod observe;
42pub mod params;
43pub mod row;
44pub mod schema;
45pub mod validate;
46
47#[cfg(feature = "embedded-model")]
48pub use crate::artifact::embedded;
49pub use crate::artifact::{cell_id, ArtifactError, Compact, ModelKind, MAX_WIDTH};
50pub use candidates::{generate, Candidate, Context, MAX_CANDIDATES, MULTIPLIERS};
51pub use edit::{Applied, Edit, EditError, EditSet, Op, Value, MAX_EDITS};
52pub use features::{featurize_edit, EditFeatures, Input, EDIT_DIM, EDIT_FEATURE_VERSION};
53pub use gate::{choose, Choice, Gate, Reason};
54pub use labels::{byte_ratio, fields_ok, shingle_jaccard};
55pub use model::{ModelVersion, NoModel, Score, Scorer};
56pub use monitor::{Estimate, Monitor, MonitorConfig, Verdict};
57pub use observe::{summarize, Identifier, Observation, ResourceSummary, MAX_IDENTIFIERS};
58pub use params::Params;
59pub use row::{
60 comparison_row, Arm, ComparisonRow, EditDescriptor, IdentDescriptor, MemoryState,
61 CMP_ROW_VERSION,
62};
63pub use schema::{
64 edit_code, Dependency, Group, Key, Kind, ParamSpec, Schema, KEEP_CODE, SCHEMA_VERSION,
65};
66pub use validate::{validate, Rejection};
67
68#[cfg(test)]
69mod testing {
70 #![allow(
78 clippy::unwrap_used,
79 clippy::expect_used,
80 clippy::panic,
81 clippy::unreachable,
82 clippy::indexing_slicing,
83 clippy::string_slice
84 )]
85 use crate::candidates::Context;
86 use crate::observe::{summarize, Observation};
87 use crate::params::Params;
88 use crate::schema::Key;
89 use spider_cloud_agent::params::{RequestParams, Timeout, WaitFor};
90 use spider_route::{DeclaredNeed, ProxyPool, RequestMode, RouteDecision};
91 use url::Url;
92
93 pub(crate) struct Fixture {
96 pub url: Url,
97 pub need: DeclaredNeed,
98 pub current: RequestParams,
99 pub caller: RequestParams,
100 pub routed: RouteDecision,
101 pub observation: Observation,
102 pub cap: f32,
103 }
104
105 impl Fixture {
106 pub fn cold() -> Fixture {
108 Fixture {
109 url: Url::parse("https://example.com/articles/one").unwrap(),
110 need: DeclaredNeed::Markdown,
111 current: RequestParams {
112 request: Some(RequestMode::Smart),
113 ..RequestParams::default()
114 },
115 caller: RequestParams::default(),
116 routed: RouteDecision::default(),
117 observation: Observation::cold(),
118 cap: 8.0,
119 }
120 }
121
122 pub fn observed() -> Fixture {
124 let mut fixture = Fixture::cold();
125 fixture.current.disable_hints = Some(true);
126 let resources: Vec<(Url, u64)> = [
127 ("https://example.com/app.js", 40_000),
128 ("https://tracker-alpha.example/t.js", 90_000),
129 ("https://cdn-beta.example/big.js", 300_000),
130 ("https://ads-gamma.example/a.gif", 20_000),
131 ("https://fonts-delta.example/f.woff2", 5_000),
132 ("https://widget-epsilon.example/w.js", 1_000),
133 ]
134 .iter()
135 .map(|(url, bytes)| (Url::parse(url).unwrap(), *bytes))
136 .collect();
137 fixture.observation.resources = Some(summarize(&fixture.url, &resources));
138 fixture
139 }
140
141 pub fn pinned() -> Fixture {
143 let mut fixture = Fixture::observed();
144 fixture.caller.request = Some(RequestMode::Browser);
145 fixture.caller.proxy = Some(ProxyPool::Isp);
146 fixture.caller.wait_for = Some(WaitFor::selector("main", Timeout::from_secs(2)));
147 fixture.current.request = fixture.caller.request;
148 fixture.current.proxy = fixture.caller.proxy;
149 fixture.current.wait_for = fixture.caller.wait_for.clone();
150 fixture
151 }
152
153 pub fn ctx(&self) -> Context<'_> {
154 Context {
155 url: &self.url,
156 need: self.need,
157 current: &self.current,
158 caller: &self.caller,
159 routed: &self.routed,
160 observation: &self.observation,
161 multiplier_cap: self.cap,
162 }
163 }
164 }
165
166 fn body(params: &RequestParams) -> serde_json::Map<String, serde_json::Value> {
167 match serde_json::to_value(params).expect("a request serializes") {
168 serde_json::Value::Object(map) => map,
169 other => panic!("a request is an object, got {other}"),
170 }
171 }
172
173 impl Params for RequestParams {
174 fn is_set(&self, key: Key) -> bool {
175 body(self).contains_key(key.wire())
176 }
177
178 fn request(&self) -> Option<RequestMode> {
179 self.request
180 }
181
182 fn proxy(&self) -> Option<ProxyPool> {
183 self.proxy
184 }
185
186 fn idle_wait_millis(&self) -> Option<u32> {
187 let idle = self.wait_for.as_ref()?.idle_network?;
188 let millis = idle.timeout.secs * 1_000 + u64::from(idle.timeout.nanos / 1_000_000);
189 Some(millis.min(u64::from(u32::MAX)) as u32)
190 }
191
192 fn flag(&self, key: Key) -> Option<bool> {
193 body(self)
194 .get(key.wire())
195 .and_then(serde_json::Value::as_bool)
196 }
197
198 fn list(&self, key: Key) -> Option<&[String]> {
199 match key {
200 Key::NetworkBlacklist => self.network_blacklist.as_deref(),
201 Key::NetworkWhitelist => self.network_whitelist.as_deref(),
202 Key::Blacklist => self.blacklist.as_deref(),
203 Key::Whitelist => self.whitelist.as_deref(),
204 Key::ExternalDomains => self.external_domains.as_deref(),
205 _ => None,
206 }
207 }
208
209 fn set_request(&mut self, mode: RequestMode) {
210 self.request = Some(mode);
211 }
212
213 fn set_proxy(&mut self, pool: ProxyPool) {
214 self.proxy = Some(pool);
215 }
216
217 fn set_idle_wait(&mut self, millis: u32) {
218 self.wait_for = Some(WaitFor::idle_network(Timeout::from_millis(u64::from(
219 millis,
220 ))));
221 }
222
223 fn set_flag(&mut self, key: Key, value: bool) {
224 let slot = match key {
225 Key::BlockStylesheets => &mut self.block_stylesheets,
226 Key::BlockAds => &mut self.block_ads,
227 Key::BlockAnalytics => &mut self.block_analytics,
228 Key::FullResources => &mut self.full_resources,
229 Key::DisableIntercept => &mut self.disable_intercept,
230 Key::DisableHints => &mut self.disable_hints,
231 _ => return,
232 };
233 *slot = Some(value);
234 }
235
236 fn set_list(&mut self, key: Key, list: Option<Vec<String>>) {
237 if key == Key::NetworkBlacklist {
238 self.network_blacklist = list;
239 }
240 }
241 }
242}