Skip to main content

spider_optimize/
schema.rs

1//! Every request parameter, what it holds, and which ones may be learned.
2//!
3//! [`Key`] names each field of the client's `RequestParams` in the order they
4//! are declared there, and [`Schema::v1`] says for each one what kind of value
5//! it takes, what it needs to take effect, whether an edit may touch it, and
6//! whether changing it can change the content that comes back. A test
7//! serializes a fully populated request and fails when a field exists on one
8//! side and not the other.
9//!
10//! The learnable set in version one is small on purpose: the fetch mode, the
11//! proxy pool, an idle network wait, five resource switches and the network
12//! blacklist. Each is a label a model has to learn from rows that do not exist
13//! yet, so the set grows when the rows justify it.
14
15use spider_route::{ProxyPool, RequestMode, PROXY_POOLS};
16
17/// The version of the key table and its specs. Recorded on every row.
18pub const SCHEMA_VERSION: u16 = 1;
19
20/// How many keys there are.
21pub const KEY_COUNT: usize = 80;
22
23/// One field of the request body.
24///
25/// Declared in the order `RequestParams` declares its fields, so
26/// [`Key::index`] is stable for a given schema version. `WaitIdleMillis`
27/// stands for `wait_for`, because the only wait an edit writes is an idle
28/// network wait.
29#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
30#[non_exhaustive]
31#[repr(u8)]
32pub enum Key {
33    /// `url`
34    Url,
35    /// `request`
36    Request,
37    /// `proxy`
38    Proxy,
39    /// `country_code`
40    CountryCode,
41    /// `remote_proxy`
42    RemoteProxy,
43    /// `stealth`
44    Stealth,
45    /// `fingerprint`
46    Fingerprint,
47    /// `user_agent`
48    UserAgent,
49    /// `viewport`
50    Viewport,
51    /// `locale`
52    Locale,
53    /// `cookies`
54    Cookies,
55    /// `headers`
56    Headers,
57    /// `encoding`
58    Encoding,
59    /// `storageless`
60    Storageless,
61    /// `session`
62    Session,
63    /// `redirect_policy`
64    RedirectPolicy,
65    /// `request_timeout`
66    RequestTimeout,
67    /// `service_worker_enabled`
68    ServiceWorkerEnabled,
69    /// `preserve_host`
70    PreserveHost,
71    /// `delay`
72    Delay,
73    /// `concurrency_limit`
74    ConcurrencyLimit,
75    /// `wayback`
76    Wayback,
77    /// `limit`
78    Limit,
79    /// `depth`
80    Depth,
81    /// `budget`
82    Budget,
83    /// `blacklist`
84    Blacklist,
85    /// `whitelist`
86    Whitelist,
87    /// `subdomains`
88    Subdomains,
89    /// `tld`
90    Tld,
91    /// `external_domains`
92    ExternalDomains,
93    /// `sitemap`
94    Sitemap,
95    /// `sitemap_only`
96    SitemapOnly,
97    /// `sitemap_path`
98    SitemapPath,
99    /// `respect_robots`
100    RespectRobots,
101    /// `link_rewrite`
102    LinkRewrite,
103    /// `crawl_timeout`
104    CrawlTimeout,
105    /// `run_in_background`
106    RunInBackground,
107    /// `wait_for`, as an idle network wait in milliseconds.
108    WaitIdleMillis,
109    /// `scroll`
110    Scroll,
111    /// `execution_scripts`
112    ExecutionScripts,
113    /// `automation_scripts`
114    AutomationScripts,
115    /// `evaluate_on_new_document`
116    EvaluateOnNewDocument,
117    /// `disable_intercept`
118    DisableIntercept,
119    /// `full_resources`
120    FullResources,
121    /// `block_ads`
122    BlockAds,
123    /// `block_analytics`
124    BlockAnalytics,
125    /// `block_stylesheets`
126    BlockStylesheets,
127    /// `disable_first_party_stylesheets`
128    DisableFirstPartyStylesheets,
129    /// `disable_first_party_javascript`
130    DisableFirstPartyJavascript,
131    /// `disable_first_party_visuals`
132    DisableFirstPartyVisuals,
133    /// `network_whitelist`
134    NetworkWhitelist,
135    /// `network_blacklist`
136    NetworkBlacklist,
137    /// `event_tracker`
138    EventTracker,
139    /// `return_format`
140    ReturnFormat,
141    /// `root_selector`
142    RootSelector,
143    /// `exclude_selector`
144    ExcludeSelector,
145    /// `filter_output_images`
146    FilterOutputImages,
147    /// `filter_output_svg`
148    FilterOutputSvg,
149    /// `filter_output_main_only`
150    FilterOutputMainOnly,
151    /// `max_size`
152    MaxSize,
153    /// `readability`
154    Readability,
155    /// `clean_html`
156    CleanHtml,
157    /// `css_extraction_map`
158    CssExtractionMap,
159    /// `chunking_alg`
160    ChunkingAlg,
161    /// `metadata`
162    Metadata,
163    /// `return_embeddings`
164    ReturnEmbeddings,
165    /// `return_headers`
166    ReturnHeaders,
167    /// `return_cookies`
168    ReturnCookies,
169    /// `return_page_links`
170    ReturnPageLinks,
171    /// `return_json_data`
172    ReturnJsonData,
173    /// `text`
174    Text,
175    /// `cache`
176    Cache,
177    /// `webhooks`
178    Webhooks,
179    /// `data_connectors`
180    DataConnectors,
181    /// `router`
182    Router,
183    /// `provider_options`
184    ProviderOptions,
185    /// `max_credits_allowed`
186    MaxCreditsAllowed,
187    /// `max_credits_per_page`
188    MaxCreditsPerPage,
189    /// `disable_hints`
190    DisableHints,
191    /// `skip_config_checks`
192    SkipConfigChecks,
193}
194
195/// Every key, in index order.
196const ALL_KEYS: [Key; KEY_COUNT] = [
197    Key::Url,
198    Key::Request,
199    Key::Proxy,
200    Key::CountryCode,
201    Key::RemoteProxy,
202    Key::Stealth,
203    Key::Fingerprint,
204    Key::UserAgent,
205    Key::Viewport,
206    Key::Locale,
207    Key::Cookies,
208    Key::Headers,
209    Key::Encoding,
210    Key::Storageless,
211    Key::Session,
212    Key::RedirectPolicy,
213    Key::RequestTimeout,
214    Key::ServiceWorkerEnabled,
215    Key::PreserveHost,
216    Key::Delay,
217    Key::ConcurrencyLimit,
218    Key::Wayback,
219    Key::Limit,
220    Key::Depth,
221    Key::Budget,
222    Key::Blacklist,
223    Key::Whitelist,
224    Key::Subdomains,
225    Key::Tld,
226    Key::ExternalDomains,
227    Key::Sitemap,
228    Key::SitemapOnly,
229    Key::SitemapPath,
230    Key::RespectRobots,
231    Key::LinkRewrite,
232    Key::CrawlTimeout,
233    Key::RunInBackground,
234    Key::WaitIdleMillis,
235    Key::Scroll,
236    Key::ExecutionScripts,
237    Key::AutomationScripts,
238    Key::EvaluateOnNewDocument,
239    Key::DisableIntercept,
240    Key::FullResources,
241    Key::BlockAds,
242    Key::BlockAnalytics,
243    Key::BlockStylesheets,
244    Key::DisableFirstPartyStylesheets,
245    Key::DisableFirstPartyJavascript,
246    Key::DisableFirstPartyVisuals,
247    Key::NetworkWhitelist,
248    Key::NetworkBlacklist,
249    Key::EventTracker,
250    Key::ReturnFormat,
251    Key::RootSelector,
252    Key::ExcludeSelector,
253    Key::FilterOutputImages,
254    Key::FilterOutputSvg,
255    Key::FilterOutputMainOnly,
256    Key::MaxSize,
257    Key::Readability,
258    Key::CleanHtml,
259    Key::CssExtractionMap,
260    Key::ChunkingAlg,
261    Key::Metadata,
262    Key::ReturnEmbeddings,
263    Key::ReturnHeaders,
264    Key::ReturnCookies,
265    Key::ReturnPageLinks,
266    Key::ReturnJsonData,
267    Key::Text,
268    Key::Cache,
269    Key::Webhooks,
270    Key::DataConnectors,
271    Key::Router,
272    Key::ProviderOptions,
273    Key::MaxCreditsAllowed,
274    Key::MaxCreditsPerPage,
275    Key::DisableHints,
276    Key::SkipConfigChecks,
277];
278
279impl Key {
280    /// Every key, in index order.
281    pub const ALL: &'static [Key] = &ALL_KEYS;
282
283    /// Where this key sits in [`Key::ALL`] and in the spec table.
284    pub const fn index(self) -> usize {
285        self as usize
286    }
287
288    /// The field name on the wire.
289    pub const fn wire(self) -> &'static str {
290        match self {
291            Key::Url => "url",
292            Key::Request => "request",
293            Key::Proxy => "proxy",
294            Key::CountryCode => "country_code",
295            Key::RemoteProxy => "remote_proxy",
296            Key::Stealth => "stealth",
297            Key::Fingerprint => "fingerprint",
298            Key::UserAgent => "user_agent",
299            Key::Viewport => "viewport",
300            Key::Locale => "locale",
301            Key::Cookies => "cookies",
302            Key::Headers => "headers",
303            Key::Encoding => "encoding",
304            Key::Storageless => "storageless",
305            Key::Session => "session",
306            Key::RedirectPolicy => "redirect_policy",
307            Key::RequestTimeout => "request_timeout",
308            Key::ServiceWorkerEnabled => "service_worker_enabled",
309            Key::PreserveHost => "preserve_host",
310            Key::Delay => "delay",
311            Key::ConcurrencyLimit => "concurrency_limit",
312            Key::Wayback => "wayback",
313            Key::Limit => "limit",
314            Key::Depth => "depth",
315            Key::Budget => "budget",
316            Key::Blacklist => "blacklist",
317            Key::Whitelist => "whitelist",
318            Key::Subdomains => "subdomains",
319            Key::Tld => "tld",
320            Key::ExternalDomains => "external_domains",
321            Key::Sitemap => "sitemap",
322            Key::SitemapOnly => "sitemap_only",
323            Key::SitemapPath => "sitemap_path",
324            Key::RespectRobots => "respect_robots",
325            Key::LinkRewrite => "link_rewrite",
326            Key::CrawlTimeout => "crawl_timeout",
327            Key::RunInBackground => "run_in_background",
328            Key::WaitIdleMillis => "wait_for",
329            Key::Scroll => "scroll",
330            Key::ExecutionScripts => "execution_scripts",
331            Key::AutomationScripts => "automation_scripts",
332            Key::EvaluateOnNewDocument => "evaluate_on_new_document",
333            Key::DisableIntercept => "disable_intercept",
334            Key::FullResources => "full_resources",
335            Key::BlockAds => "block_ads",
336            Key::BlockAnalytics => "block_analytics",
337            Key::BlockStylesheets => "block_stylesheets",
338            Key::DisableFirstPartyStylesheets => "disable_first_party_stylesheets",
339            Key::DisableFirstPartyJavascript => "disable_first_party_javascript",
340            Key::DisableFirstPartyVisuals => "disable_first_party_visuals",
341            Key::NetworkWhitelist => "network_whitelist",
342            Key::NetworkBlacklist => "network_blacklist",
343            Key::EventTracker => "event_tracker",
344            Key::ReturnFormat => "return_format",
345            Key::RootSelector => "root_selector",
346            Key::ExcludeSelector => "exclude_selector",
347            Key::FilterOutputImages => "filter_output_images",
348            Key::FilterOutputSvg => "filter_output_svg",
349            Key::FilterOutputMainOnly => "filter_output_main_only",
350            Key::MaxSize => "max_size",
351            Key::Readability => "readability",
352            Key::CleanHtml => "clean_html",
353            Key::CssExtractionMap => "css_extraction_map",
354            Key::ChunkingAlg => "chunking_alg",
355            Key::Metadata => "metadata",
356            Key::ReturnEmbeddings => "return_embeddings",
357            Key::ReturnHeaders => "return_headers",
358            Key::ReturnCookies => "return_cookies",
359            Key::ReturnPageLinks => "return_page_links",
360            Key::ReturnJsonData => "return_json_data",
361            Key::Text => "text",
362            Key::Cache => "cache",
363            Key::Webhooks => "webhooks",
364            Key::DataConnectors => "data_connectors",
365            Key::Router => "router",
366            Key::ProviderOptions => "provider_options",
367            Key::MaxCreditsAllowed => "max_credits_allowed",
368            Key::MaxCreditsPerPage => "max_credits_per_page",
369            Key::DisableHints => "disable_hints",
370            Key::SkipConfigChecks => "skip_config_checks",
371        }
372    }
373}
374
375/// What a field holds.
376#[derive(Debug, Clone, Copy, PartialEq, Eq)]
377pub enum Kind {
378    /// True or false.
379    Bool,
380    /// One of these wire values, in index order.
381    Enum(&'static [&'static str]),
382    /// A whole number in this range, inclusive.
383    IntRange {
384        /// The smallest value the service accepts.
385        min: i64,
386        /// The largest value the service accepts.
387        max: i64,
388    },
389    /// A span in milliseconds, limited to these buckets.
390    MillisBucket(&'static [u32]),
391    /// A list of strings.
392    StringList,
393    /// Keyed values.
394    Map,
395    /// Anything this schema does not model: free text, nested settings,
396    /// floats. Never learnable.
397    Opaque,
398}
399
400/// What a field needs before setting it does anything.
401#[derive(Debug, Clone, Copy, PartialEq, Eq)]
402pub enum Dependency {
403    /// The request is `browser` or `smart`. Everything that acts on a page
404    /// the service renders is ignored by a plain HTTP fetch, which still costs
405    /// what it costs.
406    Rendered,
407    /// The other field is not active on the same request.
408    NotWith(Key),
409}
410
411/// A coarse grouping of the fields, for reading the table.
412#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
413pub enum Group {
414    /// How the request is sent and presented.
415    Transport,
416    /// Where it leaves from.
417    Proxy,
418    /// How far a crawl goes.
419    Scope,
420    /// What has to happen before the page is read.
421    Wait,
422    /// What runs on the page.
423    Scripting,
424    /// What the page is allowed to load.
425    Resources,
426    /// What comes back and in what shape.
427    Extraction,
428    /// Stored copies.
429    Cache,
430    /// Where results go besides the response.
431    Delivery,
432    /// What the request may spend.
433    Budget,
434    /// Everything else.
435    Other,
436}
437
438/// One field's entry in the schema.
439#[derive(Debug, Clone, Copy, PartialEq, Eq)]
440pub struct ParamSpec {
441    /// The field.
442    pub key: Key,
443    /// What it holds.
444    pub kind: Kind,
445    /// What it needs before it does anything.
446    pub requires: &'static [Dependency],
447    /// Whether an edit may write it.
448    pub learnable: bool,
449    /// Whether changing it can change the content that comes back, rather
450    /// than only how the fetch is carried out.
451    pub content_changing: bool,
452    /// Where it sits in the table.
453    pub group: Group,
454}
455
456impl ParamSpec {
457    /// A field no edit touches.
458    const fn fixed(key: Key, kind: Kind, group: Group) -> ParamSpec {
459        ParamSpec {
460            key,
461            kind,
462            requires: &[],
463            learnable: false,
464            content_changing: false,
465            group,
466        }
467    }
468
469    /// The same spec, open to edits.
470    const fn learnable(mut self) -> ParamSpec {
471        self.learnable = true;
472        self
473    }
474
475    /// The same spec, marked as able to change the content.
476    const fn content(mut self) -> ParamSpec {
477        self.content_changing = true;
478        self
479    }
480
481    /// The same spec, with these dependencies.
482    const fn needs(mut self, requires: &'static [Dependency]) -> ParamSpec {
483        self.requires = requires;
484        self
485    }
486}
487
488/// The fetch modes, in the order the `Request` kind lists them.
489pub const MODES: [RequestMode; 3] = [RequestMode::Http, RequestMode::Smart, RequestMode::Browser];
490
491/// The proxy pools, in the order the `Proxy` kind lists them.
492pub const PROXIES: [ProxyPool; 2] = PROXY_POOLS;
493
494/// The idle network waits an edit may set, in milliseconds.
495pub const WAIT_BUCKETS: [u32; 4] = [0, 2_000, 5_000, 10_000];
496
497const RENDERED: &[Dependency] = &[Dependency::Rendered];
498
499// `disable_intercept` lets first party scripts through. It does not turn the
500// network lists off, so a blacklist edit and an intercept edit do not
501// exclude each other.
502const BLACKLIST_NEEDS: &[Dependency] = RENDERED;
503const INTERCEPT_NEEDS: &[Dependency] = RENDERED;
504
505const U32_MAX: i64 = 4_294_967_295;
506const I64_MAX: i64 = 9_223_372_036_854_775_807;
507
508/// Never returned for a real key. See [`Schema::spec`].
509const UNKNOWN: ParamSpec = ParamSpec::fixed(Key::Url, Kind::Opaque, Group::Other);
510
511use self::{Group as G, Kind as K, ParamSpec as S};
512
513/// The version one table, in key order.
514const SPECS: [ParamSpec; KEY_COUNT] = [
515    S::fixed(Key::Url, K::Opaque, G::Transport),
516    S::fixed(
517        Key::Request,
518        K::Enum(&["http", "smart", "browser"]),
519        G::Transport,
520    )
521    .learnable(),
522    S::fixed(Key::Proxy, K::Enum(&["isp", "residential"]), G::Proxy).learnable(),
523    S::fixed(Key::CountryCode, K::Opaque, G::Proxy),
524    S::fixed(Key::RemoteProxy, K::Opaque, G::Proxy),
525    S::fixed(Key::Stealth, K::Bool, G::Transport),
526    S::fixed(Key::Fingerprint, K::Bool, G::Transport),
527    S::fixed(Key::UserAgent, K::Opaque, G::Transport),
528    S::fixed(Key::Viewport, K::Opaque, G::Transport),
529    S::fixed(Key::Locale, K::Opaque, G::Transport),
530    S::fixed(Key::Cookies, K::Opaque, G::Transport),
531    S::fixed(Key::Headers, K::Map, G::Transport),
532    S::fixed(Key::Encoding, K::Opaque, G::Transport),
533    S::fixed(Key::Storageless, K::Bool, G::Transport),
534    S::fixed(Key::Session, K::Bool, G::Transport),
535    S::fixed(Key::RedirectPolicy, K::Opaque, G::Transport),
536    S::fixed(
537        Key::RequestTimeout,
538        K::IntRange { min: 0, max: 255 },
539        G::Transport,
540    ),
541    S::fixed(Key::ServiceWorkerEnabled, K::Bool, G::Transport),
542    S::fixed(Key::PreserveHost, K::Bool, G::Transport),
543    S::fixed(
544        Key::Delay,
545        K::IntRange {
546            min: 0,
547            max: 60_000,
548        },
549        G::Transport,
550    ),
551    S::fixed(
552        Key::ConcurrencyLimit,
553        K::IntRange {
554            min: 0,
555            max: U32_MAX,
556        },
557        G::Transport,
558    ),
559    S::fixed(Key::Wayback, K::Bool, G::Transport),
560    S::fixed(
561        Key::Limit,
562        K::IntRange {
563            min: 0,
564            max: U32_MAX,
565        },
566        G::Scope,
567    ),
568    S::fixed(
569        Key::Depth,
570        K::IntRange {
571            min: 0,
572            max: U32_MAX,
573        },
574        G::Scope,
575    ),
576    S::fixed(Key::Budget, K::Map, G::Scope),
577    S::fixed(Key::Blacklist, K::StringList, G::Scope),
578    S::fixed(Key::Whitelist, K::StringList, G::Scope),
579    S::fixed(Key::Subdomains, K::Bool, G::Scope),
580    S::fixed(Key::Tld, K::Bool, G::Scope),
581    S::fixed(Key::ExternalDomains, K::StringList, G::Scope),
582    S::fixed(Key::Sitemap, K::Bool, G::Scope),
583    S::fixed(Key::SitemapOnly, K::Bool, G::Scope),
584    S::fixed(Key::SitemapPath, K::Opaque, G::Scope),
585    S::fixed(Key::RespectRobots, K::Bool, G::Scope),
586    S::fixed(Key::LinkRewrite, K::Opaque, G::Scope),
587    S::fixed(Key::CrawlTimeout, K::Opaque, G::Scope),
588    S::fixed(Key::RunInBackground, K::Bool, G::Delivery),
589    S::fixed(Key::WaitIdleMillis, K::MillisBucket(&WAIT_BUCKETS), G::Wait)
590        .learnable()
591        .needs(RENDERED),
592    S::fixed(
593        Key::Scroll,
594        K::IntRange {
595            min: 0,
596            max: U32_MAX,
597        },
598        G::Wait,
599    ),
600    S::fixed(Key::ExecutionScripts, K::Map, G::Scripting),
601    S::fixed(Key::AutomationScripts, K::Map, G::Scripting),
602    S::fixed(Key::EvaluateOnNewDocument, K::Opaque, G::Scripting),
603    S::fixed(Key::DisableIntercept, K::Bool, G::Resources)
604        .learnable()
605        .needs(INTERCEPT_NEEDS),
606    S::fixed(Key::FullResources, K::Bool, G::Resources)
607        .learnable()
608        .needs(RENDERED),
609    S::fixed(Key::BlockAds, K::Bool, G::Resources)
610        .learnable()
611        .needs(RENDERED),
612    S::fixed(Key::BlockAnalytics, K::Bool, G::Resources)
613        .learnable()
614        .needs(RENDERED),
615    S::fixed(Key::BlockStylesheets, K::Bool, G::Resources)
616        .learnable()
617        .content()
618        .needs(RENDERED),
619    S::fixed(Key::DisableFirstPartyStylesheets, K::Bool, G::Resources).content(),
620    S::fixed(Key::DisableFirstPartyJavascript, K::Bool, G::Resources).content(),
621    S::fixed(Key::DisableFirstPartyVisuals, K::Bool, G::Resources).content(),
622    S::fixed(Key::NetworkWhitelist, K::StringList, G::Resources).content(),
623    S::fixed(Key::NetworkBlacklist, K::StringList, G::Resources)
624        .learnable()
625        .content()
626        .needs(BLACKLIST_NEEDS),
627    S::fixed(Key::EventTracker, K::Opaque, G::Other),
628    S::fixed(Key::ReturnFormat, K::Opaque, G::Extraction).content(),
629    S::fixed(Key::RootSelector, K::Opaque, G::Extraction).content(),
630    S::fixed(Key::ExcludeSelector, K::Opaque, G::Extraction).content(),
631    S::fixed(Key::FilterOutputImages, K::Bool, G::Extraction).content(),
632    S::fixed(Key::FilterOutputSvg, K::Bool, G::Extraction).content(),
633    S::fixed(Key::FilterOutputMainOnly, K::Bool, G::Extraction).content(),
634    S::fixed(
635        Key::MaxSize,
636        K::IntRange {
637            min: 0,
638            max: I64_MAX,
639        },
640        G::Extraction,
641    )
642    .content(),
643    S::fixed(Key::Readability, K::Bool, G::Extraction).content(),
644    S::fixed(Key::CleanHtml, K::Bool, G::Extraction).content(),
645    S::fixed(Key::CssExtractionMap, K::Map, G::Extraction).content(),
646    S::fixed(Key::ChunkingAlg, K::Opaque, G::Extraction).content(),
647    S::fixed(Key::Metadata, K::Bool, G::Extraction),
648    S::fixed(Key::ReturnEmbeddings, K::Bool, G::Extraction),
649    S::fixed(Key::ReturnHeaders, K::Bool, G::Extraction),
650    S::fixed(Key::ReturnCookies, K::Bool, G::Extraction),
651    S::fixed(Key::ReturnPageLinks, K::Bool, G::Extraction),
652    S::fixed(Key::ReturnJsonData, K::Bool, G::Extraction),
653    S::fixed(Key::Text, K::Opaque, G::Extraction).content(),
654    S::fixed(Key::Cache, K::Opaque, G::Cache),
655    S::fixed(Key::Webhooks, K::Opaque, G::Delivery),
656    S::fixed(Key::DataConnectors, K::Map, G::Delivery),
657    S::fixed(Key::Router, K::Opaque, G::Other),
658    S::fixed(Key::ProviderOptions, K::Map, G::Other),
659    S::fixed(
660        Key::MaxCreditsAllowed,
661        K::IntRange {
662            min: 0,
663            max: I64_MAX,
664        },
665        G::Budget,
666    ),
667    S::fixed(Key::MaxCreditsPerPage, K::Opaque, G::Budget),
668    S::fixed(Key::DisableHints, K::Bool, G::Other),
669    S::fixed(Key::SkipConfigChecks, K::Bool, G::Other),
670];
671
672// The table and the key list line up slot for slot, and the last key is the
673// last slot, so `Key::index` is always inside the table. Checked when the
674// crate compiles rather than when a request is edited.
675const _: () = {
676    let mut at = 0;
677    let mut keys: &[Key] = &ALL_KEYS;
678    let mut specs: &[ParamSpec] = &SPECS;
679    while let ([key, keys_rest @ ..], [spec, specs_rest @ ..]) = (keys, specs) {
680        assert!(*key as usize == at);
681        assert!(spec.key as usize == at);
682        keys = keys_rest;
683        specs = specs_rest;
684        at += 1;
685    }
686    assert!(at == KEY_COUNT);
687    assert!(Key::SkipConfigChecks as usize + 1 == KEY_COUNT);
688};
689
690/// The parameter table.
691#[derive(Debug, Clone, Copy, PartialEq, Eq)]
692pub struct Schema {
693    specs: &'static [ParamSpec; KEY_COUNT],
694}
695
696impl Schema {
697    /// The version one table.
698    pub const fn v1() -> Schema {
699        Schema { specs: &SPECS }
700    }
701
702    /// One field's spec.
703    pub fn spec(&self, key: Key) -> &ParamSpec {
704        // The compile time block above proves every key indexes inside the
705        // table. Were that ever wrong, an opaque spec refuses every edit
706        // rather than panicking.
707        match self.specs.get(key.index()) {
708            Some(spec) => spec,
709            None => &UNKNOWN,
710        }
711    }
712
713    /// The fields an edit may write, in key order.
714    pub fn learnable(&self) -> impl Iterator<Item = &ParamSpec> {
715        self.specs.iter().filter(|spec| spec.learnable)
716    }
717
718    /// Every field, in key order.
719    pub fn specs(&self) -> &[ParamSpec] {
720        self.specs
721    }
722}
723
724impl Default for Schema {
725    fn default() -> Schema {
726        Schema::v1()
727    }
728}
729
730/// The position of a learnable key among the learnable keys, in key order.
731pub const fn learnable_slot(key: Key) -> Option<usize> {
732    match key {
733        Key::Request => Some(0),
734        Key::Proxy => Some(1),
735        Key::WaitIdleMillis => Some(2),
736        Key::DisableIntercept => Some(3),
737        Key::FullResources => Some(4),
738        Key::BlockAds => Some(5),
739        Key::BlockAnalytics => Some(6),
740        Key::BlockStylesheets => Some(7),
741        Key::NetworkBlacklist => Some(8),
742        _ => None,
743    }
744}
745
746/// What the service does when a `Bool` field is left unset.
747///
748/// The three blocking switches default to on at the service, so `None` and
749/// `Some(true)` send the same request. Everything else is off until set.
750pub const fn service_default(key: Key) -> bool {
751    matches!(
752        key,
753        Key::BlockAds | Key::BlockAnalytics | Key::BlockStylesheets
754    )
755}
756
757/// The compact edit code for leaving a request unchanged.
758pub const KEEP_CODE: u8 = 0;
759
760/// One plus the rank among `Schema::v1().learnable()` in `Key::ALL` order.
761/// The trainer derives the same codes from learnable keys in schema-v1.json.
762pub fn edit_code(key: Key) -> Option<u8> {
763    Schema::v1()
764        .learnable()
765        .position(|spec| spec.key == key)
766        .map(|rank| 1 + rank as u8)
767}
768
769#[cfg(test)]
770pub(crate) mod tests {
771    #![allow(
772        clippy::unwrap_used,
773        clippy::expect_used,
774        clippy::panic,
775        clippy::unreachable,
776        clippy::indexing_slicing,
777        clippy::string_slice
778    )]
779    use super::*;
780    use spider_cloud_agent::params::{
781        Cache, ChunkingAlg, ChunkingKind, CrawlBudget, CssExtractionMap, EventTracker,
782        LinkRewriteRule, RedirectPolicy, RequestParams, ReturnFormat, Router, SelectorGroup,
783        Timeout, Viewport, WaitFor, WebAutomation, WebhookSettings,
784    };
785    use spider_cloud_agent::{Credits, WholeCredits};
786    use std::collections::{BTreeMap, BTreeSet};
787
788    /// Every field set by hand. A field added to `RequestParams` and not set
789    /// here is caught too, because `RequestParams` has no `..Default` below.
790    pub(crate) fn populated() -> RequestParams {
791        let mut css = CssExtractionMap::new();
792        css.insert("/".into(), vec![SelectorGroup::css("title", ["h1"])]);
793        RequestParams {
794            url: Some("https://example.com/".into()),
795            request: Some(RequestMode::Browser),
796            proxy: Some(ProxyPool::Residential),
797            country_code: spider_route::Country::new("de"),
798            remote_proxy: Some("http://example.com:8080".into()),
799            stealth: Some(true),
800            fingerprint: Some(true),
801            user_agent: Some("agent".into()),
802            viewport: Some(Viewport::desktop()),
803            locale: Some("en-GB".into()),
804            cookies: Some("a=b".into()),
805            headers: Some(BTreeMap::from([("x".to_string(), "y".to_string())])),
806            encoding: Some("utf-8".into()),
807            storageless: Some(true),
808            session: Some(true),
809            redirect_policy: Some(RedirectPolicy::Loose),
810            request_timeout: Some(30),
811            service_worker_enabled: Some(false),
812            preserve_host: Some(true),
813            delay: Some(10),
814            concurrency_limit: Some(2),
815            wayback: Some(true),
816            limit: Some(5),
817            depth: Some(2),
818            budget: Some(CrawlBudget::from([("*".to_string(), 3)])),
819            blacklist: Some(vec!["/a".into()]),
820            whitelist: Some(vec!["/b".into()]),
821            subdomains: Some(true),
822            tld: Some(true),
823            external_domains: Some(vec!["example.org".into()]),
824            sitemap: Some(true),
825            sitemap_only: Some(true),
826            sitemap_path: Some("sitemap.xml".into()),
827            respect_robots: Some(true),
828            link_rewrite: Some(LinkRewriteRule::replace("a", "b")),
829            crawl_timeout: Some(Timeout::from_secs(5)),
830            run_in_background: Some(false),
831            wait_for: Some(WaitFor::idle_network(Timeout::from_millis(2_000))),
832            scroll: Some(1),
833            execution_scripts: Some(BTreeMap::from([("/".to_string(), "1".to_string())])),
834            automation_scripts: Some(BTreeMap::from([(
835                "/".to_string(),
836                vec![WebAutomation::Evaluate("1".into())],
837            )])),
838            evaluate_on_new_document: Some("1".into()),
839            disable_intercept: Some(false),
840            full_resources: Some(false),
841            block_ads: Some(true),
842            block_analytics: Some(true),
843            block_stylesheets: Some(true),
844            disable_first_party_stylesheets: Some(false),
845            disable_first_party_javascript: Some(false),
846            disable_first_party_visuals: Some(false),
847            network_whitelist: Some(vec!["example.com".into()]),
848            network_blacklist: Some(vec!["example.net".into()]),
849            event_tracker: Some(EventTracker::default()),
850            return_format: Some(ReturnFormat::Markdown.into()),
851            root_selector: Some("main".into()),
852            exclude_selector: Some("nav".into()),
853            filter_output_images: Some(true),
854            filter_output_svg: Some(true),
855            filter_output_main_only: Some(true),
856            max_size: Some(1_000),
857            readability: Some(true),
858            clean_html: Some(true),
859            css_extraction_map: Some(css),
860            chunking_alg: Some(ChunkingAlg::new(ChunkingKind::BySentence, 2)),
861            metadata: Some(true),
862            return_embeddings: Some(true),
863            return_headers: Some(true),
864            return_cookies: Some(true),
865            return_page_links: Some(true),
866            return_json_data: Some(true),
867            text: Some("text".into()),
868            cache: Some(Cache::Enabled(true)),
869            webhooks: Some(WebhookSettings::new("https://example.com/hook")),
870            data_connectors: Some(serde_json::Map::from_iter([(
871                "s3".to_string(),
872                serde_json::Value::Bool(true),
873            )])),
874            router: Some(Router {
875                mode: Some("off".into()),
876                ..Router::default()
877            }),
878            provider_options: Some(BTreeMap::from([(
879                "p".to_string(),
880                serde_json::Value::Bool(true),
881            )])),
882            max_credits_allowed: Some(WholeCredits::floor(Credits::new(10.0))),
883            max_credits_per_page: Some(Credits::new(1.0)),
884            disable_hints: Some(true),
885            skip_config_checks: Some(false),
886        }
887    }
888
889    #[test]
890    fn every_request_params_field_has_a_spec() {
891        let json = serde_json::to_value(populated()).unwrap();
892        let wire: BTreeSet<String> = json.as_object().unwrap().keys().cloned().collect();
893        let keys: BTreeSet<String> = Key::ALL.iter().map(|k| k.wire().to_string()).collect();
894
895        let missing_spec: Vec<_> = wire.difference(&keys).collect();
896        let missing_field: Vec<_> = keys.difference(&wire).collect();
897        assert!(
898            missing_spec.is_empty(),
899            "fields with no Key: {missing_spec:?}"
900        );
901        assert!(
902            missing_field.is_empty(),
903            "Keys with no field: {missing_field:?}"
904        );
905        assert_eq!(Key::ALL.len(), wire.len());
906        assert_eq!(keys.len(), Key::ALL.len(), "two keys share a wire name");
907    }
908
909    #[test]
910    fn keys_follow_the_declaration_order_of_the_struct() {
911        // serde writes fields in declaration order, so the serialized order is
912        // the struct's order.
913        let text = serde_json::to_string(&populated()).unwrap();
914        let mut last = 0;
915        for key in Key::ALL {
916            let at = text
917                .find(&format!("\"{}\":", key.wire()))
918                .unwrap_or_else(|| panic!("{} is not on the wire", key.wire()));
919            assert!(at >= last, "{:?} is out of order", key);
920            last = at;
921        }
922    }
923
924    #[test]
925    fn the_learnable_set_is_exactly_version_one() {
926        let schema = Schema::v1();
927        let learnable: Vec<Key> = schema.learnable().map(|spec| spec.key).collect();
928        assert_eq!(
929            learnable,
930            [
931                Key::Request,
932                Key::Proxy,
933                Key::WaitIdleMillis,
934                Key::DisableIntercept,
935                Key::FullResources,
936                Key::BlockAds,
937                Key::BlockAnalytics,
938                Key::BlockStylesheets,
939                Key::NetworkBlacklist,
940            ]
941        );
942        for (slot, key) in learnable.iter().enumerate() {
943            assert_eq!(learnable_slot(*key), Some(slot));
944        }
945        for spec in schema.specs() {
946            if !spec.learnable {
947                assert_eq!(learnable_slot(spec.key), None, "{:?}", spec.key);
948            }
949        }
950
951        let content: BTreeSet<Key> = schema
952            .specs()
953            .iter()
954            .filter(|spec| spec.content_changing)
955            .map(|spec| spec.key)
956            .collect();
957        let expected = BTreeSet::from([
958            Key::Readability,
959            Key::CleanHtml,
960            Key::RootSelector,
961            Key::ExcludeSelector,
962            Key::ReturnFormat,
963            Key::FilterOutputMainOnly,
964            Key::FilterOutputImages,
965            Key::FilterOutputSvg,
966            Key::MaxSize,
967            Key::ChunkingAlg,
968            Key::CssExtractionMap,
969            Key::Text,
970            Key::BlockStylesheets,
971            Key::DisableFirstPartyStylesheets,
972            Key::DisableFirstPartyJavascript,
973            Key::DisableFirstPartyVisuals,
974            Key::NetworkBlacklist,
975            Key::NetworkWhitelist,
976        ]);
977        assert_eq!(content, expected);
978    }
979
980    #[test]
981    fn the_spec_for_a_key_is_that_keys_spec() {
982        let schema = Schema::v1();
983        for key in Key::ALL {
984            assert_eq!(schema.spec(*key).key, *key);
985        }
986        assert_eq!(
987            schema.spec(Key::WaitIdleMillis).kind,
988            Kind::MillisBucket(&[0, 2_000, 5_000, 10_000])
989        );
990    }
991}