Skip to main content

spider_optimize/
schema.rs

1//! Every request parameter, what it holds, and which ones may be learned.
2//!
3//! [`Key`] names each field of the client's `RequestParams` in the order they
4//! are declared there, and [`Schema::v1`] says for each one what kind of value
5//! it takes, what it needs to take effect, whether an edit may touch it, and
6//! whether changing it can change the content that comes back. A test
7//! serializes a fully populated request and fails when a field exists on one
8//! side and not the other.
9//!
10//! The learnable set in version one is small on purpose: the fetch mode, the
11//! proxy pool, an idle network wait, five resource switches and the network
12//! blacklist. Each is a label a model has to learn from rows that do not exist
13//! yet, so the set grows when the rows justify it.
14
15use spider_route::{ProxyPool, RequestMode, PROXY_POOLS};
16
17/// The version of the key table and its specs. Recorded on every row.
18///
19/// Version 2 has no `storageless`, `preserve_host`, `text` or
20/// `skip_config_checks`. Nothing in the agent read them, so every key after
21/// `encoding` moved down and an artifact stamped with version 1 no longer loads.
22pub const SCHEMA_VERSION: u16 = 2;
23
24/// How many keys there are.
25pub const KEY_COUNT: usize = 76;
26
27/// One field of the request body.
28///
29/// Declared in the order `RequestParams` declares its fields, so
30/// [`Key::index`] is stable for a given schema version. `WaitIdleMillis`
31/// stands for `wait_for`, because the only wait an edit writes is an idle
32/// network wait.
33#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
34#[non_exhaustive]
35#[repr(u8)]
36pub enum Key {
37    /// `url`
38    Url,
39    /// `request`
40    Request,
41    /// `proxy`
42    Proxy,
43    /// `country_code`
44    CountryCode,
45    /// `remote_proxy`
46    RemoteProxy,
47    /// `stealth`
48    Stealth,
49    /// `fingerprint`
50    Fingerprint,
51    /// `user_agent`
52    UserAgent,
53    /// `viewport`
54    Viewport,
55    /// `locale`
56    Locale,
57    /// `cookies`
58    Cookies,
59    /// `headers`
60    Headers,
61    /// `encoding`
62    Encoding,
63    /// `session`
64    Session,
65    /// `redirect_policy`
66    RedirectPolicy,
67    /// `request_timeout`
68    RequestTimeout,
69    /// `service_worker_enabled`
70    ServiceWorkerEnabled,
71    /// `delay`
72    Delay,
73    /// `concurrency_limit`
74    ConcurrencyLimit,
75    /// `wayback`
76    Wayback,
77    /// `limit`
78    Limit,
79    /// `depth`
80    Depth,
81    /// `budget`
82    Budget,
83    /// `blacklist`
84    Blacklist,
85    /// `whitelist`
86    Whitelist,
87    /// `subdomains`
88    Subdomains,
89    /// `tld`
90    Tld,
91    /// `external_domains`
92    ExternalDomains,
93    /// `sitemap`
94    Sitemap,
95    /// `sitemap_only`
96    SitemapOnly,
97    /// `sitemap_path`
98    SitemapPath,
99    /// `respect_robots`
100    RespectRobots,
101    /// `link_rewrite`
102    LinkRewrite,
103    /// `crawl_timeout`
104    CrawlTimeout,
105    /// `run_in_background`
106    RunInBackground,
107    /// `wait_for`, as an idle network wait in milliseconds.
108    WaitIdleMillis,
109    /// `scroll`
110    Scroll,
111    /// `execution_scripts`
112    ExecutionScripts,
113    /// `automation_scripts`
114    AutomationScripts,
115    /// `evaluate_on_new_document`
116    EvaluateOnNewDocument,
117    /// `disable_intercept`
118    DisableIntercept,
119    /// `full_resources`
120    FullResources,
121    /// `block_ads`
122    BlockAds,
123    /// `block_analytics`
124    BlockAnalytics,
125    /// `block_stylesheets`
126    BlockStylesheets,
127    /// `disable_first_party_stylesheets`
128    DisableFirstPartyStylesheets,
129    /// `disable_first_party_javascript`
130    DisableFirstPartyJavascript,
131    /// `disable_first_party_visuals`
132    DisableFirstPartyVisuals,
133    /// `network_whitelist`
134    NetworkWhitelist,
135    /// `network_blacklist`
136    NetworkBlacklist,
137    /// `event_tracker`
138    EventTracker,
139    /// `return_format`
140    ReturnFormat,
141    /// `root_selector`
142    RootSelector,
143    /// `exclude_selector`
144    ExcludeSelector,
145    /// `filter_output_images`
146    FilterOutputImages,
147    /// `filter_output_svg`
148    FilterOutputSvg,
149    /// `filter_output_main_only`
150    FilterOutputMainOnly,
151    /// `max_size`
152    MaxSize,
153    /// `readability`
154    Readability,
155    /// `clean_html`
156    CleanHtml,
157    /// `css_extraction_map`
158    CssExtractionMap,
159    /// `chunking_alg`
160    ChunkingAlg,
161    /// `metadata`
162    Metadata,
163    /// `return_embeddings`
164    ReturnEmbeddings,
165    /// `return_headers`
166    ReturnHeaders,
167    /// `return_cookies`
168    ReturnCookies,
169    /// `return_page_links`
170    ReturnPageLinks,
171    /// `return_json_data`
172    ReturnJsonData,
173    /// `cache`
174    Cache,
175    /// `webhooks`
176    Webhooks,
177    /// `data_connectors`
178    DataConnectors,
179    /// `router`
180    Router,
181    /// `provider_options`
182    ProviderOptions,
183    /// `max_credits_allowed`
184    MaxCreditsAllowed,
185    /// `max_credits_per_page`
186    MaxCreditsPerPage,
187    /// `disable_hints`
188    DisableHints,
189}
190
191/// Every key, in index order.
192const ALL_KEYS: [Key; KEY_COUNT] = [
193    Key::Url,
194    Key::Request,
195    Key::Proxy,
196    Key::CountryCode,
197    Key::RemoteProxy,
198    Key::Stealth,
199    Key::Fingerprint,
200    Key::UserAgent,
201    Key::Viewport,
202    Key::Locale,
203    Key::Cookies,
204    Key::Headers,
205    Key::Encoding,
206    Key::Session,
207    Key::RedirectPolicy,
208    Key::RequestTimeout,
209    Key::ServiceWorkerEnabled,
210    Key::Delay,
211    Key::ConcurrencyLimit,
212    Key::Wayback,
213    Key::Limit,
214    Key::Depth,
215    Key::Budget,
216    Key::Blacklist,
217    Key::Whitelist,
218    Key::Subdomains,
219    Key::Tld,
220    Key::ExternalDomains,
221    Key::Sitemap,
222    Key::SitemapOnly,
223    Key::SitemapPath,
224    Key::RespectRobots,
225    Key::LinkRewrite,
226    Key::CrawlTimeout,
227    Key::RunInBackground,
228    Key::WaitIdleMillis,
229    Key::Scroll,
230    Key::ExecutionScripts,
231    Key::AutomationScripts,
232    Key::EvaluateOnNewDocument,
233    Key::DisableIntercept,
234    Key::FullResources,
235    Key::BlockAds,
236    Key::BlockAnalytics,
237    Key::BlockStylesheets,
238    Key::DisableFirstPartyStylesheets,
239    Key::DisableFirstPartyJavascript,
240    Key::DisableFirstPartyVisuals,
241    Key::NetworkWhitelist,
242    Key::NetworkBlacklist,
243    Key::EventTracker,
244    Key::ReturnFormat,
245    Key::RootSelector,
246    Key::ExcludeSelector,
247    Key::FilterOutputImages,
248    Key::FilterOutputSvg,
249    Key::FilterOutputMainOnly,
250    Key::MaxSize,
251    Key::Readability,
252    Key::CleanHtml,
253    Key::CssExtractionMap,
254    Key::ChunkingAlg,
255    Key::Metadata,
256    Key::ReturnEmbeddings,
257    Key::ReturnHeaders,
258    Key::ReturnCookies,
259    Key::ReturnPageLinks,
260    Key::ReturnJsonData,
261    Key::Cache,
262    Key::Webhooks,
263    Key::DataConnectors,
264    Key::Router,
265    Key::ProviderOptions,
266    Key::MaxCreditsAllowed,
267    Key::MaxCreditsPerPage,
268    Key::DisableHints,
269];
270
271impl Key {
272    /// Every key, in index order.
273    pub const ALL: &'static [Key] = &ALL_KEYS;
274
275    /// Where this key sits in [`Key::ALL`] and in the spec table.
276    pub const fn index(self) -> usize {
277        self as usize
278    }
279
280    /// The field name on the wire.
281    pub const fn wire(self) -> &'static str {
282        match self {
283            Key::Url => "url",
284            Key::Request => "request",
285            Key::Proxy => "proxy",
286            Key::CountryCode => "country_code",
287            Key::RemoteProxy => "remote_proxy",
288            Key::Stealth => "stealth",
289            Key::Fingerprint => "fingerprint",
290            Key::UserAgent => "user_agent",
291            Key::Viewport => "viewport",
292            Key::Locale => "locale",
293            Key::Cookies => "cookies",
294            Key::Headers => "headers",
295            Key::Encoding => "encoding",
296            Key::Session => "session",
297            Key::RedirectPolicy => "redirect_policy",
298            Key::RequestTimeout => "request_timeout",
299            Key::ServiceWorkerEnabled => "service_worker_enabled",
300            Key::Delay => "delay",
301            Key::ConcurrencyLimit => "concurrency_limit",
302            Key::Wayback => "wayback",
303            Key::Limit => "limit",
304            Key::Depth => "depth",
305            Key::Budget => "budget",
306            Key::Blacklist => "blacklist",
307            Key::Whitelist => "whitelist",
308            Key::Subdomains => "subdomains",
309            Key::Tld => "tld",
310            Key::ExternalDomains => "external_domains",
311            Key::Sitemap => "sitemap",
312            Key::SitemapOnly => "sitemap_only",
313            Key::SitemapPath => "sitemap_path",
314            Key::RespectRobots => "respect_robots",
315            Key::LinkRewrite => "link_rewrite",
316            Key::CrawlTimeout => "crawl_timeout",
317            Key::RunInBackground => "run_in_background",
318            Key::WaitIdleMillis => "wait_for",
319            Key::Scroll => "scroll",
320            Key::ExecutionScripts => "execution_scripts",
321            Key::AutomationScripts => "automation_scripts",
322            Key::EvaluateOnNewDocument => "evaluate_on_new_document",
323            Key::DisableIntercept => "disable_intercept",
324            Key::FullResources => "full_resources",
325            Key::BlockAds => "block_ads",
326            Key::BlockAnalytics => "block_analytics",
327            Key::BlockStylesheets => "block_stylesheets",
328            Key::DisableFirstPartyStylesheets => "disable_first_party_stylesheets",
329            Key::DisableFirstPartyJavascript => "disable_first_party_javascript",
330            Key::DisableFirstPartyVisuals => "disable_first_party_visuals",
331            Key::NetworkWhitelist => "network_whitelist",
332            Key::NetworkBlacklist => "network_blacklist",
333            Key::EventTracker => "event_tracker",
334            Key::ReturnFormat => "return_format",
335            Key::RootSelector => "root_selector",
336            Key::ExcludeSelector => "exclude_selector",
337            Key::FilterOutputImages => "filter_output_images",
338            Key::FilterOutputSvg => "filter_output_svg",
339            Key::FilterOutputMainOnly => "filter_output_main_only",
340            Key::MaxSize => "max_size",
341            Key::Readability => "readability",
342            Key::CleanHtml => "clean_html",
343            Key::CssExtractionMap => "css_extraction_map",
344            Key::ChunkingAlg => "chunking_alg",
345            Key::Metadata => "metadata",
346            Key::ReturnEmbeddings => "return_embeddings",
347            Key::ReturnHeaders => "return_headers",
348            Key::ReturnCookies => "return_cookies",
349            Key::ReturnPageLinks => "return_page_links",
350            Key::ReturnJsonData => "return_json_data",
351            Key::Cache => "cache",
352            Key::Webhooks => "webhooks",
353            Key::DataConnectors => "data_connectors",
354            Key::Router => "router",
355            Key::ProviderOptions => "provider_options",
356            Key::MaxCreditsAllowed => "max_credits_allowed",
357            Key::MaxCreditsPerPage => "max_credits_per_page",
358            Key::DisableHints => "disable_hints",
359        }
360    }
361}
362
363/// What a field holds.
364#[derive(Debug, Clone, Copy, PartialEq, Eq)]
365pub enum Kind {
366    /// True or false.
367    Bool,
368    /// One of these wire values, in index order.
369    Enum(&'static [&'static str]),
370    /// A whole number in this range, inclusive.
371    IntRange {
372        /// The smallest value the service accepts.
373        min: i64,
374        /// The largest value the service accepts.
375        max: i64,
376    },
377    /// A span in milliseconds, limited to these buckets.
378    MillisBucket(&'static [u32]),
379    /// A list of strings.
380    StringList,
381    /// Keyed values.
382    Map,
383    /// Anything this schema does not model: free text, nested settings,
384    /// floats. Never learnable.
385    Opaque,
386}
387
388/// What a field needs before setting it does anything.
389#[derive(Debug, Clone, Copy, PartialEq, Eq)]
390pub enum Dependency {
391    /// The request is `browser` or `smart`. Everything that acts on a page
392    /// the service renders is ignored by a plain HTTP fetch, which still costs
393    /// what it costs.
394    Rendered,
395    /// The other field is not active on the same request.
396    NotWith(Key),
397}
398
399/// A coarse grouping of the fields, for reading the table.
400#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
401pub enum Group {
402    /// How the request is sent and presented.
403    Transport,
404    /// Where it leaves from.
405    Proxy,
406    /// How far a crawl goes.
407    Scope,
408    /// What has to happen before the page is read.
409    Wait,
410    /// What runs on the page.
411    Scripting,
412    /// What the page is allowed to load.
413    Resources,
414    /// What comes back and in what shape.
415    Extraction,
416    /// Stored copies.
417    Cache,
418    /// Where results go besides the response.
419    Delivery,
420    /// What the request may spend.
421    Budget,
422    /// Everything else.
423    Other,
424}
425
426/// One field's entry in the schema.
427#[derive(Debug, Clone, Copy, PartialEq, Eq)]
428pub struct ParamSpec {
429    /// The field.
430    pub key: Key,
431    /// What it holds.
432    pub kind: Kind,
433    /// What it needs before it does anything.
434    pub requires: &'static [Dependency],
435    /// Whether an edit may write it.
436    pub learnable: bool,
437    /// Whether changing it can change the content that comes back, rather
438    /// than only how the fetch is carried out.
439    pub content_changing: bool,
440    /// Where it sits in the table.
441    pub group: Group,
442}
443
444impl ParamSpec {
445    /// A field no edit touches.
446    const fn fixed(key: Key, kind: Kind, group: Group) -> ParamSpec {
447        ParamSpec {
448            key,
449            kind,
450            requires: &[],
451            learnable: false,
452            content_changing: false,
453            group,
454        }
455    }
456
457    /// The same spec, open to edits.
458    const fn learnable(mut self) -> ParamSpec {
459        self.learnable = true;
460        self
461    }
462
463    /// The same spec, marked as able to change the content.
464    const fn content(mut self) -> ParamSpec {
465        self.content_changing = true;
466        self
467    }
468
469    /// The same spec, with these dependencies.
470    const fn needs(mut self, requires: &'static [Dependency]) -> ParamSpec {
471        self.requires = requires;
472        self
473    }
474}
475
476/// The fetch modes, in the order the `Request` kind lists them.
477pub const MODES: [RequestMode; 3] = [RequestMode::Http, RequestMode::Smart, RequestMode::Browser];
478
479/// The proxy pools, in the order the `Proxy` kind lists them.
480pub const PROXIES: [ProxyPool; 2] = PROXY_POOLS;
481
482/// The idle network waits an edit may set, in milliseconds.
483pub const WAIT_BUCKETS: [u32; 4] = [0, 2_000, 5_000, 10_000];
484
485const RENDERED: &[Dependency] = &[Dependency::Rendered];
486
487// `disable_intercept` lets first party scripts through. It does not turn the
488// network lists off, so a blacklist edit and an intercept edit do not
489// exclude each other.
490const BLACKLIST_NEEDS: &[Dependency] = RENDERED;
491const INTERCEPT_NEEDS: &[Dependency] = RENDERED;
492
493const U32_MAX: i64 = 4_294_967_295;
494const I64_MAX: i64 = 9_223_372_036_854_775_807;
495
496/// Never returned for a real key. See [`Schema::spec`].
497const UNKNOWN: ParamSpec = ParamSpec::fixed(Key::Url, Kind::Opaque, Group::Other);
498
499use self::{Group as G, Kind as K, ParamSpec as S};
500
501/// The version one table, in key order. The one names the learnable set, not
502/// [`SCHEMA_VERSION`].
503const SPECS: [ParamSpec; KEY_COUNT] = [
504    S::fixed(Key::Url, K::Opaque, G::Transport),
505    S::fixed(
506        Key::Request,
507        K::Enum(&["http", "smart", "browser"]),
508        G::Transport,
509    )
510    .learnable(),
511    S::fixed(Key::Proxy, K::Enum(&["isp", "residential"]), G::Proxy).learnable(),
512    S::fixed(Key::CountryCode, K::Opaque, G::Proxy),
513    S::fixed(Key::RemoteProxy, K::Opaque, G::Proxy),
514    S::fixed(Key::Stealth, K::Bool, G::Transport),
515    S::fixed(Key::Fingerprint, K::Bool, G::Transport),
516    S::fixed(Key::UserAgent, K::Opaque, G::Transport),
517    S::fixed(Key::Viewport, K::Opaque, G::Transport),
518    S::fixed(Key::Locale, K::Opaque, G::Transport),
519    S::fixed(Key::Cookies, K::Opaque, G::Transport),
520    S::fixed(Key::Headers, K::Map, G::Transport),
521    S::fixed(Key::Encoding, K::Opaque, G::Transport),
522    S::fixed(Key::Session, K::Bool, G::Transport),
523    S::fixed(Key::RedirectPolicy, K::Opaque, G::Transport),
524    S::fixed(
525        Key::RequestTimeout,
526        K::IntRange { min: 0, max: 255 },
527        G::Transport,
528    ),
529    S::fixed(Key::ServiceWorkerEnabled, K::Bool, G::Transport),
530    S::fixed(
531        Key::Delay,
532        K::IntRange {
533            min: 0,
534            max: 60_000,
535        },
536        G::Transport,
537    ),
538    S::fixed(
539        Key::ConcurrencyLimit,
540        K::IntRange {
541            min: 0,
542            max: U32_MAX,
543        },
544        G::Transport,
545    ),
546    S::fixed(Key::Wayback, K::Bool, G::Transport),
547    S::fixed(
548        Key::Limit,
549        K::IntRange {
550            min: 0,
551            max: U32_MAX,
552        },
553        G::Scope,
554    ),
555    S::fixed(
556        Key::Depth,
557        K::IntRange {
558            min: 0,
559            max: U32_MAX,
560        },
561        G::Scope,
562    ),
563    S::fixed(Key::Budget, K::Map, G::Scope),
564    S::fixed(Key::Blacklist, K::StringList, G::Scope),
565    S::fixed(Key::Whitelist, K::StringList, G::Scope),
566    S::fixed(Key::Subdomains, K::Bool, G::Scope),
567    S::fixed(Key::Tld, K::Bool, G::Scope),
568    S::fixed(Key::ExternalDomains, K::StringList, G::Scope),
569    S::fixed(Key::Sitemap, K::Bool, G::Scope),
570    S::fixed(Key::SitemapOnly, K::Bool, G::Scope),
571    S::fixed(Key::SitemapPath, K::Opaque, G::Scope),
572    S::fixed(Key::RespectRobots, K::Bool, G::Scope),
573    S::fixed(Key::LinkRewrite, K::Opaque, G::Scope),
574    S::fixed(Key::CrawlTimeout, K::Opaque, G::Scope),
575    S::fixed(Key::RunInBackground, K::Bool, G::Delivery),
576    S::fixed(Key::WaitIdleMillis, K::MillisBucket(&WAIT_BUCKETS), G::Wait)
577        .learnable()
578        .needs(RENDERED),
579    S::fixed(
580        Key::Scroll,
581        K::IntRange {
582            min: 0,
583            max: U32_MAX,
584        },
585        G::Wait,
586    ),
587    S::fixed(Key::ExecutionScripts, K::Map, G::Scripting),
588    S::fixed(Key::AutomationScripts, K::Map, G::Scripting),
589    S::fixed(Key::EvaluateOnNewDocument, K::Opaque, G::Scripting),
590    S::fixed(Key::DisableIntercept, K::Bool, G::Resources)
591        .learnable()
592        .needs(INTERCEPT_NEEDS),
593    S::fixed(Key::FullResources, K::Bool, G::Resources)
594        .learnable()
595        .needs(RENDERED),
596    S::fixed(Key::BlockAds, K::Bool, G::Resources)
597        .learnable()
598        .needs(RENDERED),
599    S::fixed(Key::BlockAnalytics, K::Bool, G::Resources)
600        .learnable()
601        .needs(RENDERED),
602    S::fixed(Key::BlockStylesheets, K::Bool, G::Resources)
603        .learnable()
604        .content()
605        .needs(RENDERED),
606    S::fixed(Key::DisableFirstPartyStylesheets, K::Bool, G::Resources).content(),
607    S::fixed(Key::DisableFirstPartyJavascript, K::Bool, G::Resources).content(),
608    S::fixed(Key::DisableFirstPartyVisuals, K::Bool, G::Resources).content(),
609    S::fixed(Key::NetworkWhitelist, K::StringList, G::Resources).content(),
610    S::fixed(Key::NetworkBlacklist, K::StringList, G::Resources)
611        .learnable()
612        .content()
613        .needs(BLACKLIST_NEEDS),
614    S::fixed(Key::EventTracker, K::Opaque, G::Other),
615    S::fixed(Key::ReturnFormat, K::Opaque, G::Extraction).content(),
616    S::fixed(Key::RootSelector, K::Opaque, G::Extraction).content(),
617    S::fixed(Key::ExcludeSelector, K::Opaque, G::Extraction).content(),
618    S::fixed(Key::FilterOutputImages, K::Bool, G::Extraction).content(),
619    S::fixed(Key::FilterOutputSvg, K::Bool, G::Extraction).content(),
620    S::fixed(Key::FilterOutputMainOnly, K::Bool, G::Extraction).content(),
621    S::fixed(
622        Key::MaxSize,
623        K::IntRange {
624            min: 0,
625            max: I64_MAX,
626        },
627        G::Extraction,
628    )
629    .content(),
630    S::fixed(Key::Readability, K::Bool, G::Extraction).content(),
631    S::fixed(Key::CleanHtml, K::Bool, G::Extraction).content(),
632    S::fixed(Key::CssExtractionMap, K::Map, G::Extraction).content(),
633    S::fixed(Key::ChunkingAlg, K::Opaque, G::Extraction).content(),
634    S::fixed(Key::Metadata, K::Bool, G::Extraction),
635    S::fixed(Key::ReturnEmbeddings, K::Bool, G::Extraction),
636    S::fixed(Key::ReturnHeaders, K::Bool, G::Extraction),
637    S::fixed(Key::ReturnCookies, K::Bool, G::Extraction),
638    S::fixed(Key::ReturnPageLinks, K::Bool, G::Extraction),
639    S::fixed(Key::ReturnJsonData, K::Bool, G::Extraction),
640    S::fixed(Key::Cache, K::Opaque, G::Cache),
641    S::fixed(Key::Webhooks, K::Opaque, G::Delivery),
642    S::fixed(Key::DataConnectors, K::Map, G::Delivery),
643    S::fixed(Key::Router, K::Opaque, G::Other),
644    S::fixed(Key::ProviderOptions, K::Map, G::Other),
645    S::fixed(
646        Key::MaxCreditsAllowed,
647        K::IntRange {
648            min: 0,
649            max: I64_MAX,
650        },
651        G::Budget,
652    ),
653    S::fixed(Key::MaxCreditsPerPage, K::Opaque, G::Budget),
654    S::fixed(Key::DisableHints, K::Bool, G::Other),
655];
656
657// The table and the key list line up slot for slot, and the last key is the
658// last slot, so `Key::index` is always inside the table. Checked when the
659// crate compiles rather than when a request is edited.
660const _: () = {
661    let mut at = 0;
662    let mut keys: &[Key] = &ALL_KEYS;
663    let mut specs: &[ParamSpec] = &SPECS;
664    while let ([key, keys_rest @ ..], [spec, specs_rest @ ..]) = (keys, specs) {
665        assert!(*key as usize == at);
666        assert!(spec.key as usize == at);
667        keys = keys_rest;
668        specs = specs_rest;
669        at += 1;
670    }
671    assert!(at == KEY_COUNT);
672    assert!(Key::DisableHints as usize + 1 == KEY_COUNT);
673};
674
675/// The parameter table.
676#[derive(Debug, Clone, Copy, PartialEq, Eq)]
677pub struct Schema {
678    specs: &'static [ParamSpec; KEY_COUNT],
679}
680
681impl Schema {
682    /// The version one table.
683    ///
684    /// The one here counts the learnable set, which is still on its first
685    /// version. [`SCHEMA_VERSION`] counts the key table, which is on its
686    /// second. The two numbers move for different reasons, so they are not
687    /// expected to agree.
688    pub const fn v1() -> Schema {
689        Schema { specs: &SPECS }
690    }
691
692    /// One field's spec.
693    pub fn spec(&self, key: Key) -> &ParamSpec {
694        // The compile time block above proves every key indexes inside the
695        // table. Were that ever wrong, an opaque spec refuses every edit
696        // rather than panicking.
697        match self.specs.get(key.index()) {
698            Some(spec) => spec,
699            None => &UNKNOWN,
700        }
701    }
702
703    /// The fields an edit may write, in key order.
704    pub fn learnable(&self) -> impl Iterator<Item = &ParamSpec> {
705        self.specs.iter().filter(|spec| spec.learnable)
706    }
707
708    /// Every field, in key order.
709    pub fn specs(&self) -> &[ParamSpec] {
710        self.specs
711    }
712}
713
714impl Default for Schema {
715    fn default() -> Schema {
716        Schema::v1()
717    }
718}
719
720/// The position of a learnable key among the learnable keys, in key order.
721pub const fn learnable_slot(key: Key) -> Option<usize> {
722    match key {
723        Key::Request => Some(0),
724        Key::Proxy => Some(1),
725        Key::WaitIdleMillis => Some(2),
726        Key::DisableIntercept => Some(3),
727        Key::FullResources => Some(4),
728        Key::BlockAds => Some(5),
729        Key::BlockAnalytics => Some(6),
730        Key::BlockStylesheets => Some(7),
731        Key::NetworkBlacklist => Some(8),
732        _ => None,
733    }
734}
735
736/// What the service does when a `Bool` field is left unset.
737///
738/// The three blocking switches default to on at the service, so `None` and
739/// `Some(true)` send the same request. Everything else is off until set.
740pub const fn service_default(key: Key) -> bool {
741    matches!(
742        key,
743        Key::BlockAds | Key::BlockAnalytics | Key::BlockStylesheets
744    )
745}
746
747/// The compact edit code for leaving a request unchanged.
748pub const KEEP_CODE: u8 = 0;
749
750/// One plus the rank among `Schema::v1().learnable()` in `Key::ALL` order.
751/// The trainer derives the same codes from learnable keys in schema-v2.json.
752pub fn edit_code(key: Key) -> Option<u8> {
753    Schema::v1()
754        .learnable()
755        .position(|spec| spec.key == key)
756        .map(|rank| 1 + rank as u8)
757}
758
759#[cfg(test)]
760pub(crate) mod tests {
761    #![allow(
762        clippy::unwrap_used,
763        clippy::expect_used,
764        clippy::panic,
765        clippy::unreachable,
766        clippy::indexing_slicing,
767        clippy::string_slice
768    )]
769    use super::*;
770    use spider_cloud_agent::params::{
771        Cache, ChunkingAlg, ChunkingKind, CrawlBudget, CssExtractionMap, EventTracker,
772        LinkRewriteRule, RedirectPolicy, RequestParams, ReturnFormat, Router, SelectorGroup,
773        Timeout, Viewport, WaitFor, WebAutomation, WebhookSettings,
774    };
775    use spider_cloud_agent::{Credits, WholeCredits};
776    use std::collections::{BTreeMap, BTreeSet};
777
778    /// Every field set by hand. A field added to `RequestParams` and not set
779    /// here is caught too, because `RequestParams` has no `..Default` below.
780    pub(crate) fn populated() -> RequestParams {
781        let mut css = CssExtractionMap::new();
782        css.insert("/".into(), vec![SelectorGroup::css("title", ["h1"])]);
783        RequestParams {
784            url: Some("https://example.com/".into()),
785            request: Some(RequestMode::Browser),
786            proxy: Some(ProxyPool::Residential),
787            country_code: spider_route::Country::new("de"),
788            remote_proxy: Some("http://example.com:8080".into()),
789            stealth: Some(true),
790            fingerprint: Some(true),
791            user_agent: Some("agent".into()),
792            viewport: Some(Viewport::desktop()),
793            locale: Some("en-GB".into()),
794            cookies: Some("a=b".into()),
795            headers: Some(BTreeMap::from([("x".to_string(), "y".to_string())])),
796            encoding: Some("utf-8".into()),
797            session: Some(true),
798            redirect_policy: Some(RedirectPolicy::Loose),
799            request_timeout: Some(30),
800            service_worker_enabled: Some(false),
801            delay: Some(10),
802            concurrency_limit: Some(2),
803            wayback: Some(true),
804            limit: Some(5),
805            depth: Some(2),
806            budget: Some(CrawlBudget::from([("*".to_string(), 3)])),
807            blacklist: Some(vec!["/a".into()]),
808            whitelist: Some(vec!["/b".into()]),
809            subdomains: Some(true),
810            tld: Some(true),
811            external_domains: Some(vec!["example.org".into()]),
812            sitemap: Some(true),
813            sitemap_only: Some(true),
814            sitemap_path: Some("sitemap.xml".into()),
815            respect_robots: Some(true),
816            link_rewrite: Some(LinkRewriteRule::replace("a", "b")),
817            crawl_timeout: Some(Timeout::from_secs(5)),
818            run_in_background: Some(false),
819            wait_for: Some(WaitFor::idle_network(Timeout::from_millis(2_000))),
820            scroll: Some(1),
821            execution_scripts: Some(BTreeMap::from([("/".to_string(), "1".to_string())])),
822            automation_scripts: Some(BTreeMap::from([(
823                "/".to_string(),
824                vec![WebAutomation::Evaluate("1".into())],
825            )])),
826            evaluate_on_new_document: Some("1".into()),
827            disable_intercept: Some(false),
828            full_resources: Some(false),
829            block_ads: Some(true),
830            block_analytics: Some(true),
831            block_stylesheets: Some(true),
832            disable_first_party_stylesheets: Some(false),
833            disable_first_party_javascript: Some(false),
834            disable_first_party_visuals: Some(false),
835            network_whitelist: Some(vec!["example.com".into()]),
836            network_blacklist: Some(vec!["example.net".into()]),
837            event_tracker: Some(EventTracker::default()),
838            return_format: Some(ReturnFormat::Markdown.into()),
839            root_selector: Some("main".into()),
840            exclude_selector: Some("nav".into()),
841            filter_output_images: Some(true),
842            filter_output_svg: Some(true),
843            filter_output_main_only: Some(true),
844            max_size: Some(1_000),
845            readability: Some(true),
846            clean_html: Some(true),
847            css_extraction_map: Some(css),
848            chunking_alg: Some(ChunkingAlg::new(ChunkingKind::BySentence, 2)),
849            metadata: Some(true),
850            return_embeddings: Some(true),
851            return_headers: Some(true),
852            return_cookies: Some(true),
853            return_page_links: Some(true),
854            return_json_data: Some(true),
855            cache: Some(Cache::Enabled(true)),
856            webhooks: Some(WebhookSettings::new("https://example.com/hook")),
857            data_connectors: Some(serde_json::Map::from_iter([(
858                "s3".to_string(),
859                serde_json::Value::Bool(true),
860            )])),
861            router: Some(Router {
862                mode: Some("off".into()),
863                ..Router::default()
864            }),
865            provider_options: Some(BTreeMap::from([(
866                "p".to_string(),
867                serde_json::Value::Bool(true),
868            )])),
869            max_credits_allowed: Some(WholeCredits::floor(Credits::new(10.0))),
870            max_credits_per_page: Some(Credits::new(1.0)),
871            disable_hints: Some(true),
872        }
873    }
874
875    #[test]
876    fn every_request_params_field_has_a_spec() {
877        let json = serde_json::to_value(populated()).unwrap();
878        let wire: BTreeSet<String> = json.as_object().unwrap().keys().cloned().collect();
879        let keys: BTreeSet<String> = Key::ALL.iter().map(|k| k.wire().to_string()).collect();
880
881        let missing_spec: Vec<_> = wire.difference(&keys).collect();
882        let missing_field: Vec<_> = keys.difference(&wire).collect();
883        assert!(
884            missing_spec.is_empty(),
885            "fields with no Key: {missing_spec:?}"
886        );
887        assert!(
888            missing_field.is_empty(),
889            "Keys with no field: {missing_field:?}"
890        );
891        assert_eq!(Key::ALL.len(), wire.len());
892        assert_eq!(keys.len(), Key::ALL.len(), "two keys share a wire name");
893    }
894
895    #[test]
896    fn keys_follow_the_declaration_order_of_the_struct() {
897        // serde writes fields in declaration order, so the serialized order is
898        // the struct's order.
899        let text = serde_json::to_string(&populated()).unwrap();
900        let mut last = 0;
901        for key in Key::ALL {
902            let at = text
903                .find(&format!("\"{}\":", key.wire()))
904                .unwrap_or_else(|| panic!("{} is not on the wire", key.wire()));
905            assert!(at >= last, "{:?} is out of order", key);
906            last = at;
907        }
908    }
909
910    #[test]
911    fn the_learnable_set_is_exactly_version_one() {
912        let schema = Schema::v1();
913        let learnable: Vec<Key> = schema.learnable().map(|spec| spec.key).collect();
914        assert_eq!(
915            learnable,
916            [
917                Key::Request,
918                Key::Proxy,
919                Key::WaitIdleMillis,
920                Key::DisableIntercept,
921                Key::FullResources,
922                Key::BlockAds,
923                Key::BlockAnalytics,
924                Key::BlockStylesheets,
925                Key::NetworkBlacklist,
926            ]
927        );
928        for (slot, key) in learnable.iter().enumerate() {
929            assert_eq!(learnable_slot(*key), Some(slot));
930        }
931        for spec in schema.specs() {
932            if !spec.learnable {
933                assert_eq!(learnable_slot(spec.key), None, "{:?}", spec.key);
934            }
935        }
936
937        let content: BTreeSet<Key> = schema
938            .specs()
939            .iter()
940            .filter(|spec| spec.content_changing)
941            .map(|spec| spec.key)
942            .collect();
943        let expected = BTreeSet::from([
944            Key::Readability,
945            Key::CleanHtml,
946            Key::RootSelector,
947            Key::ExcludeSelector,
948            Key::ReturnFormat,
949            Key::FilterOutputMainOnly,
950            Key::FilterOutputImages,
951            Key::FilterOutputSvg,
952            Key::MaxSize,
953            Key::ChunkingAlg,
954            Key::CssExtractionMap,
955            Key::BlockStylesheets,
956            Key::DisableFirstPartyStylesheets,
957            Key::DisableFirstPartyJavascript,
958            Key::DisableFirstPartyVisuals,
959            Key::NetworkBlacklist,
960            Key::NetworkWhitelist,
961        ]);
962        assert_eq!(content, expected);
963    }
964
965    #[test]
966    fn the_spec_for_a_key_is_that_keys_spec() {
967        let schema = Schema::v1();
968        for key in Key::ALL {
969            assert_eq!(schema.spec(*key).key, *key);
970        }
971        assert_eq!(
972            schema.spec(Key::WaitIdleMillis).kind,
973            Kind::MillisBucket(&[0, 2_000, 5_000, 10_000])
974        );
975    }
976}