1use spider_route::{ProxyPool, RequestMode, PROXY_POOLS};
16
17pub const SCHEMA_VERSION: u16 = 1;
19
20pub const KEY_COUNT: usize = 80;
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
30#[non_exhaustive]
31#[repr(u8)]
32pub enum Key {
33 Url,
35 Request,
37 Proxy,
39 CountryCode,
41 RemoteProxy,
43 Stealth,
45 Fingerprint,
47 UserAgent,
49 Viewport,
51 Locale,
53 Cookies,
55 Headers,
57 Encoding,
59 Storageless,
61 Session,
63 RedirectPolicy,
65 RequestTimeout,
67 ServiceWorkerEnabled,
69 PreserveHost,
71 Delay,
73 ConcurrencyLimit,
75 Wayback,
77 Limit,
79 Depth,
81 Budget,
83 Blacklist,
85 Whitelist,
87 Subdomains,
89 Tld,
91 ExternalDomains,
93 Sitemap,
95 SitemapOnly,
97 SitemapPath,
99 RespectRobots,
101 LinkRewrite,
103 CrawlTimeout,
105 RunInBackground,
107 WaitIdleMillis,
109 Scroll,
111 ExecutionScripts,
113 AutomationScripts,
115 EvaluateOnNewDocument,
117 DisableIntercept,
119 FullResources,
121 BlockAds,
123 BlockAnalytics,
125 BlockStylesheets,
127 DisableFirstPartyStylesheets,
129 DisableFirstPartyJavascript,
131 DisableFirstPartyVisuals,
133 NetworkWhitelist,
135 NetworkBlacklist,
137 EventTracker,
139 ReturnFormat,
141 RootSelector,
143 ExcludeSelector,
145 FilterOutputImages,
147 FilterOutputSvg,
149 FilterOutputMainOnly,
151 MaxSize,
153 Readability,
155 CleanHtml,
157 CssExtractionMap,
159 ChunkingAlg,
161 Metadata,
163 ReturnEmbeddings,
165 ReturnHeaders,
167 ReturnCookies,
169 ReturnPageLinks,
171 ReturnJsonData,
173 Text,
175 Cache,
177 Webhooks,
179 DataConnectors,
181 Router,
183 ProviderOptions,
185 MaxCreditsAllowed,
187 MaxCreditsPerPage,
189 DisableHints,
191 SkipConfigChecks,
193}
194
195const ALL_KEYS: [Key; KEY_COUNT] = [
197 Key::Url,
198 Key::Request,
199 Key::Proxy,
200 Key::CountryCode,
201 Key::RemoteProxy,
202 Key::Stealth,
203 Key::Fingerprint,
204 Key::UserAgent,
205 Key::Viewport,
206 Key::Locale,
207 Key::Cookies,
208 Key::Headers,
209 Key::Encoding,
210 Key::Storageless,
211 Key::Session,
212 Key::RedirectPolicy,
213 Key::RequestTimeout,
214 Key::ServiceWorkerEnabled,
215 Key::PreserveHost,
216 Key::Delay,
217 Key::ConcurrencyLimit,
218 Key::Wayback,
219 Key::Limit,
220 Key::Depth,
221 Key::Budget,
222 Key::Blacklist,
223 Key::Whitelist,
224 Key::Subdomains,
225 Key::Tld,
226 Key::ExternalDomains,
227 Key::Sitemap,
228 Key::SitemapOnly,
229 Key::SitemapPath,
230 Key::RespectRobots,
231 Key::LinkRewrite,
232 Key::CrawlTimeout,
233 Key::RunInBackground,
234 Key::WaitIdleMillis,
235 Key::Scroll,
236 Key::ExecutionScripts,
237 Key::AutomationScripts,
238 Key::EvaluateOnNewDocument,
239 Key::DisableIntercept,
240 Key::FullResources,
241 Key::BlockAds,
242 Key::BlockAnalytics,
243 Key::BlockStylesheets,
244 Key::DisableFirstPartyStylesheets,
245 Key::DisableFirstPartyJavascript,
246 Key::DisableFirstPartyVisuals,
247 Key::NetworkWhitelist,
248 Key::NetworkBlacklist,
249 Key::EventTracker,
250 Key::ReturnFormat,
251 Key::RootSelector,
252 Key::ExcludeSelector,
253 Key::FilterOutputImages,
254 Key::FilterOutputSvg,
255 Key::FilterOutputMainOnly,
256 Key::MaxSize,
257 Key::Readability,
258 Key::CleanHtml,
259 Key::CssExtractionMap,
260 Key::ChunkingAlg,
261 Key::Metadata,
262 Key::ReturnEmbeddings,
263 Key::ReturnHeaders,
264 Key::ReturnCookies,
265 Key::ReturnPageLinks,
266 Key::ReturnJsonData,
267 Key::Text,
268 Key::Cache,
269 Key::Webhooks,
270 Key::DataConnectors,
271 Key::Router,
272 Key::ProviderOptions,
273 Key::MaxCreditsAllowed,
274 Key::MaxCreditsPerPage,
275 Key::DisableHints,
276 Key::SkipConfigChecks,
277];
278
279impl Key {
280 pub const ALL: &'static [Key] = &ALL_KEYS;
282
283 pub const fn index(self) -> usize {
285 self as usize
286 }
287
288 pub const fn wire(self) -> &'static str {
290 match self {
291 Key::Url => "url",
292 Key::Request => "request",
293 Key::Proxy => "proxy",
294 Key::CountryCode => "country_code",
295 Key::RemoteProxy => "remote_proxy",
296 Key::Stealth => "stealth",
297 Key::Fingerprint => "fingerprint",
298 Key::UserAgent => "user_agent",
299 Key::Viewport => "viewport",
300 Key::Locale => "locale",
301 Key::Cookies => "cookies",
302 Key::Headers => "headers",
303 Key::Encoding => "encoding",
304 Key::Storageless => "storageless",
305 Key::Session => "session",
306 Key::RedirectPolicy => "redirect_policy",
307 Key::RequestTimeout => "request_timeout",
308 Key::ServiceWorkerEnabled => "service_worker_enabled",
309 Key::PreserveHost => "preserve_host",
310 Key::Delay => "delay",
311 Key::ConcurrencyLimit => "concurrency_limit",
312 Key::Wayback => "wayback",
313 Key::Limit => "limit",
314 Key::Depth => "depth",
315 Key::Budget => "budget",
316 Key::Blacklist => "blacklist",
317 Key::Whitelist => "whitelist",
318 Key::Subdomains => "subdomains",
319 Key::Tld => "tld",
320 Key::ExternalDomains => "external_domains",
321 Key::Sitemap => "sitemap",
322 Key::SitemapOnly => "sitemap_only",
323 Key::SitemapPath => "sitemap_path",
324 Key::RespectRobots => "respect_robots",
325 Key::LinkRewrite => "link_rewrite",
326 Key::CrawlTimeout => "crawl_timeout",
327 Key::RunInBackground => "run_in_background",
328 Key::WaitIdleMillis => "wait_for",
329 Key::Scroll => "scroll",
330 Key::ExecutionScripts => "execution_scripts",
331 Key::AutomationScripts => "automation_scripts",
332 Key::EvaluateOnNewDocument => "evaluate_on_new_document",
333 Key::DisableIntercept => "disable_intercept",
334 Key::FullResources => "full_resources",
335 Key::BlockAds => "block_ads",
336 Key::BlockAnalytics => "block_analytics",
337 Key::BlockStylesheets => "block_stylesheets",
338 Key::DisableFirstPartyStylesheets => "disable_first_party_stylesheets",
339 Key::DisableFirstPartyJavascript => "disable_first_party_javascript",
340 Key::DisableFirstPartyVisuals => "disable_first_party_visuals",
341 Key::NetworkWhitelist => "network_whitelist",
342 Key::NetworkBlacklist => "network_blacklist",
343 Key::EventTracker => "event_tracker",
344 Key::ReturnFormat => "return_format",
345 Key::RootSelector => "root_selector",
346 Key::ExcludeSelector => "exclude_selector",
347 Key::FilterOutputImages => "filter_output_images",
348 Key::FilterOutputSvg => "filter_output_svg",
349 Key::FilterOutputMainOnly => "filter_output_main_only",
350 Key::MaxSize => "max_size",
351 Key::Readability => "readability",
352 Key::CleanHtml => "clean_html",
353 Key::CssExtractionMap => "css_extraction_map",
354 Key::ChunkingAlg => "chunking_alg",
355 Key::Metadata => "metadata",
356 Key::ReturnEmbeddings => "return_embeddings",
357 Key::ReturnHeaders => "return_headers",
358 Key::ReturnCookies => "return_cookies",
359 Key::ReturnPageLinks => "return_page_links",
360 Key::ReturnJsonData => "return_json_data",
361 Key::Text => "text",
362 Key::Cache => "cache",
363 Key::Webhooks => "webhooks",
364 Key::DataConnectors => "data_connectors",
365 Key::Router => "router",
366 Key::ProviderOptions => "provider_options",
367 Key::MaxCreditsAllowed => "max_credits_allowed",
368 Key::MaxCreditsPerPage => "max_credits_per_page",
369 Key::DisableHints => "disable_hints",
370 Key::SkipConfigChecks => "skip_config_checks",
371 }
372 }
373}
374
375#[derive(Debug, Clone, Copy, PartialEq, Eq)]
377pub enum Kind {
378 Bool,
380 Enum(&'static [&'static str]),
382 IntRange {
384 min: i64,
386 max: i64,
388 },
389 MillisBucket(&'static [u32]),
391 StringList,
393 Map,
395 Opaque,
398}
399
400#[derive(Debug, Clone, Copy, PartialEq, Eq)]
402pub enum Dependency {
403 Rendered,
407 NotWith(Key),
409}
410
411#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
413pub enum Group {
414 Transport,
416 Proxy,
418 Scope,
420 Wait,
422 Scripting,
424 Resources,
426 Extraction,
428 Cache,
430 Delivery,
432 Budget,
434 Other,
436}
437
438#[derive(Debug, Clone, Copy, PartialEq, Eq)]
440pub struct ParamSpec {
441 pub key: Key,
443 pub kind: Kind,
445 pub requires: &'static [Dependency],
447 pub learnable: bool,
449 pub content_changing: bool,
452 pub group: Group,
454}
455
456impl ParamSpec {
457 const fn fixed(key: Key, kind: Kind, group: Group) -> ParamSpec {
459 ParamSpec {
460 key,
461 kind,
462 requires: &[],
463 learnable: false,
464 content_changing: false,
465 group,
466 }
467 }
468
469 const fn learnable(mut self) -> ParamSpec {
471 self.learnable = true;
472 self
473 }
474
475 const fn content(mut self) -> ParamSpec {
477 self.content_changing = true;
478 self
479 }
480
481 const fn needs(mut self, requires: &'static [Dependency]) -> ParamSpec {
483 self.requires = requires;
484 self
485 }
486}
487
488pub const MODES: [RequestMode; 3] = [RequestMode::Http, RequestMode::Smart, RequestMode::Browser];
490
491pub const PROXIES: [ProxyPool; 2] = PROXY_POOLS;
493
494pub const WAIT_BUCKETS: [u32; 4] = [0, 2_000, 5_000, 10_000];
496
497const RENDERED: &[Dependency] = &[Dependency::Rendered];
498
499const BLACKLIST_NEEDS: &[Dependency] = RENDERED;
503const INTERCEPT_NEEDS: &[Dependency] = RENDERED;
504
505const U32_MAX: i64 = 4_294_967_295;
506const I64_MAX: i64 = 9_223_372_036_854_775_807;
507
508const UNKNOWN: ParamSpec = ParamSpec::fixed(Key::Url, Kind::Opaque, Group::Other);
510
511use self::{Group as G, Kind as K, ParamSpec as S};
512
513const SPECS: [ParamSpec; KEY_COUNT] = [
515 S::fixed(Key::Url, K::Opaque, G::Transport),
516 S::fixed(
517 Key::Request,
518 K::Enum(&["http", "smart", "browser"]),
519 G::Transport,
520 )
521 .learnable(),
522 S::fixed(Key::Proxy, K::Enum(&["isp", "residential"]), G::Proxy).learnable(),
523 S::fixed(Key::CountryCode, K::Opaque, G::Proxy),
524 S::fixed(Key::RemoteProxy, K::Opaque, G::Proxy),
525 S::fixed(Key::Stealth, K::Bool, G::Transport),
526 S::fixed(Key::Fingerprint, K::Bool, G::Transport),
527 S::fixed(Key::UserAgent, K::Opaque, G::Transport),
528 S::fixed(Key::Viewport, K::Opaque, G::Transport),
529 S::fixed(Key::Locale, K::Opaque, G::Transport),
530 S::fixed(Key::Cookies, K::Opaque, G::Transport),
531 S::fixed(Key::Headers, K::Map, G::Transport),
532 S::fixed(Key::Encoding, K::Opaque, G::Transport),
533 S::fixed(Key::Storageless, K::Bool, G::Transport),
534 S::fixed(Key::Session, K::Bool, G::Transport),
535 S::fixed(Key::RedirectPolicy, K::Opaque, G::Transport),
536 S::fixed(
537 Key::RequestTimeout,
538 K::IntRange { min: 0, max: 255 },
539 G::Transport,
540 ),
541 S::fixed(Key::ServiceWorkerEnabled, K::Bool, G::Transport),
542 S::fixed(Key::PreserveHost, K::Bool, G::Transport),
543 S::fixed(
544 Key::Delay,
545 K::IntRange {
546 min: 0,
547 max: 60_000,
548 },
549 G::Transport,
550 ),
551 S::fixed(
552 Key::ConcurrencyLimit,
553 K::IntRange {
554 min: 0,
555 max: U32_MAX,
556 },
557 G::Transport,
558 ),
559 S::fixed(Key::Wayback, K::Bool, G::Transport),
560 S::fixed(
561 Key::Limit,
562 K::IntRange {
563 min: 0,
564 max: U32_MAX,
565 },
566 G::Scope,
567 ),
568 S::fixed(
569 Key::Depth,
570 K::IntRange {
571 min: 0,
572 max: U32_MAX,
573 },
574 G::Scope,
575 ),
576 S::fixed(Key::Budget, K::Map, G::Scope),
577 S::fixed(Key::Blacklist, K::StringList, G::Scope),
578 S::fixed(Key::Whitelist, K::StringList, G::Scope),
579 S::fixed(Key::Subdomains, K::Bool, G::Scope),
580 S::fixed(Key::Tld, K::Bool, G::Scope),
581 S::fixed(Key::ExternalDomains, K::StringList, G::Scope),
582 S::fixed(Key::Sitemap, K::Bool, G::Scope),
583 S::fixed(Key::SitemapOnly, K::Bool, G::Scope),
584 S::fixed(Key::SitemapPath, K::Opaque, G::Scope),
585 S::fixed(Key::RespectRobots, K::Bool, G::Scope),
586 S::fixed(Key::LinkRewrite, K::Opaque, G::Scope),
587 S::fixed(Key::CrawlTimeout, K::Opaque, G::Scope),
588 S::fixed(Key::RunInBackground, K::Bool, G::Delivery),
589 S::fixed(Key::WaitIdleMillis, K::MillisBucket(&WAIT_BUCKETS), G::Wait)
590 .learnable()
591 .needs(RENDERED),
592 S::fixed(
593 Key::Scroll,
594 K::IntRange {
595 min: 0,
596 max: U32_MAX,
597 },
598 G::Wait,
599 ),
600 S::fixed(Key::ExecutionScripts, K::Map, G::Scripting),
601 S::fixed(Key::AutomationScripts, K::Map, G::Scripting),
602 S::fixed(Key::EvaluateOnNewDocument, K::Opaque, G::Scripting),
603 S::fixed(Key::DisableIntercept, K::Bool, G::Resources)
604 .learnable()
605 .needs(INTERCEPT_NEEDS),
606 S::fixed(Key::FullResources, K::Bool, G::Resources)
607 .learnable()
608 .needs(RENDERED),
609 S::fixed(Key::BlockAds, K::Bool, G::Resources)
610 .learnable()
611 .needs(RENDERED),
612 S::fixed(Key::BlockAnalytics, K::Bool, G::Resources)
613 .learnable()
614 .needs(RENDERED),
615 S::fixed(Key::BlockStylesheets, K::Bool, G::Resources)
616 .learnable()
617 .content()
618 .needs(RENDERED),
619 S::fixed(Key::DisableFirstPartyStylesheets, K::Bool, G::Resources).content(),
620 S::fixed(Key::DisableFirstPartyJavascript, K::Bool, G::Resources).content(),
621 S::fixed(Key::DisableFirstPartyVisuals, K::Bool, G::Resources).content(),
622 S::fixed(Key::NetworkWhitelist, K::StringList, G::Resources).content(),
623 S::fixed(Key::NetworkBlacklist, K::StringList, G::Resources)
624 .learnable()
625 .content()
626 .needs(BLACKLIST_NEEDS),
627 S::fixed(Key::EventTracker, K::Opaque, G::Other),
628 S::fixed(Key::ReturnFormat, K::Opaque, G::Extraction).content(),
629 S::fixed(Key::RootSelector, K::Opaque, G::Extraction).content(),
630 S::fixed(Key::ExcludeSelector, K::Opaque, G::Extraction).content(),
631 S::fixed(Key::FilterOutputImages, K::Bool, G::Extraction).content(),
632 S::fixed(Key::FilterOutputSvg, K::Bool, G::Extraction).content(),
633 S::fixed(Key::FilterOutputMainOnly, K::Bool, G::Extraction).content(),
634 S::fixed(
635 Key::MaxSize,
636 K::IntRange {
637 min: 0,
638 max: I64_MAX,
639 },
640 G::Extraction,
641 )
642 .content(),
643 S::fixed(Key::Readability, K::Bool, G::Extraction).content(),
644 S::fixed(Key::CleanHtml, K::Bool, G::Extraction).content(),
645 S::fixed(Key::CssExtractionMap, K::Map, G::Extraction).content(),
646 S::fixed(Key::ChunkingAlg, K::Opaque, G::Extraction).content(),
647 S::fixed(Key::Metadata, K::Bool, G::Extraction),
648 S::fixed(Key::ReturnEmbeddings, K::Bool, G::Extraction),
649 S::fixed(Key::ReturnHeaders, K::Bool, G::Extraction),
650 S::fixed(Key::ReturnCookies, K::Bool, G::Extraction),
651 S::fixed(Key::ReturnPageLinks, K::Bool, G::Extraction),
652 S::fixed(Key::ReturnJsonData, K::Bool, G::Extraction),
653 S::fixed(Key::Text, K::Opaque, G::Extraction).content(),
654 S::fixed(Key::Cache, K::Opaque, G::Cache),
655 S::fixed(Key::Webhooks, K::Opaque, G::Delivery),
656 S::fixed(Key::DataConnectors, K::Map, G::Delivery),
657 S::fixed(Key::Router, K::Opaque, G::Other),
658 S::fixed(Key::ProviderOptions, K::Map, G::Other),
659 S::fixed(
660 Key::MaxCreditsAllowed,
661 K::IntRange {
662 min: 0,
663 max: I64_MAX,
664 },
665 G::Budget,
666 ),
667 S::fixed(Key::MaxCreditsPerPage, K::Opaque, G::Budget),
668 S::fixed(Key::DisableHints, K::Bool, G::Other),
669 S::fixed(Key::SkipConfigChecks, K::Bool, G::Other),
670];
671
672const _: () = {
676 let mut at = 0;
677 let mut keys: &[Key] = &ALL_KEYS;
678 let mut specs: &[ParamSpec] = &SPECS;
679 while let ([key, keys_rest @ ..], [spec, specs_rest @ ..]) = (keys, specs) {
680 assert!(*key as usize == at);
681 assert!(spec.key as usize == at);
682 keys = keys_rest;
683 specs = specs_rest;
684 at += 1;
685 }
686 assert!(at == KEY_COUNT);
687 assert!(Key::SkipConfigChecks as usize + 1 == KEY_COUNT);
688};
689
690#[derive(Debug, Clone, Copy, PartialEq, Eq)]
692pub struct Schema {
693 specs: &'static [ParamSpec; KEY_COUNT],
694}
695
696impl Schema {
697 pub const fn v1() -> Schema {
699 Schema { specs: &SPECS }
700 }
701
702 pub fn spec(&self, key: Key) -> &ParamSpec {
704 match self.specs.get(key.index()) {
708 Some(spec) => spec,
709 None => &UNKNOWN,
710 }
711 }
712
713 pub fn learnable(&self) -> impl Iterator<Item = &ParamSpec> {
715 self.specs.iter().filter(|spec| spec.learnable)
716 }
717
718 pub fn specs(&self) -> &[ParamSpec] {
720 self.specs
721 }
722}
723
724impl Default for Schema {
725 fn default() -> Schema {
726 Schema::v1()
727 }
728}
729
730pub const fn learnable_slot(key: Key) -> Option<usize> {
732 match key {
733 Key::Request => Some(0),
734 Key::Proxy => Some(1),
735 Key::WaitIdleMillis => Some(2),
736 Key::DisableIntercept => Some(3),
737 Key::FullResources => Some(4),
738 Key::BlockAds => Some(5),
739 Key::BlockAnalytics => Some(6),
740 Key::BlockStylesheets => Some(7),
741 Key::NetworkBlacklist => Some(8),
742 _ => None,
743 }
744}
745
746pub const fn service_default(key: Key) -> bool {
751 matches!(
752 key,
753 Key::BlockAds | Key::BlockAnalytics | Key::BlockStylesheets
754 )
755}
756
757pub const KEEP_CODE: u8 = 0;
759
760pub fn edit_code(key: Key) -> Option<u8> {
763 Schema::v1()
764 .learnable()
765 .position(|spec| spec.key == key)
766 .map(|rank| 1 + rank as u8)
767}
768
769#[cfg(test)]
770pub(crate) mod tests {
771 #![allow(
772 clippy::unwrap_used,
773 clippy::expect_used,
774 clippy::panic,
775 clippy::unreachable,
776 clippy::indexing_slicing,
777 clippy::string_slice
778 )]
779 use super::*;
780 use spider_cloud_agent::params::{
781 Cache, ChunkingAlg, ChunkingKind, CrawlBudget, CssExtractionMap, EventTracker,
782 LinkRewriteRule, RedirectPolicy, RequestParams, ReturnFormat, Router, SelectorGroup,
783 Timeout, Viewport, WaitFor, WebAutomation, WebhookSettings,
784 };
785 use spider_cloud_agent::{Credits, WholeCredits};
786 use std::collections::{BTreeMap, BTreeSet};
787
788 pub(crate) fn populated() -> RequestParams {
791 let mut css = CssExtractionMap::new();
792 css.insert("/".into(), vec![SelectorGroup::css("title", ["h1"])]);
793 RequestParams {
794 url: Some("https://example.com/".into()),
795 request: Some(RequestMode::Browser),
796 proxy: Some(ProxyPool::Residential),
797 country_code: spider_route::Country::new("de"),
798 remote_proxy: Some("http://example.com:8080".into()),
799 stealth: Some(true),
800 fingerprint: Some(true),
801 user_agent: Some("agent".into()),
802 viewport: Some(Viewport::desktop()),
803 locale: Some("en-GB".into()),
804 cookies: Some("a=b".into()),
805 headers: Some(BTreeMap::from([("x".to_string(), "y".to_string())])),
806 encoding: Some("utf-8".into()),
807 storageless: Some(true),
808 session: Some(true),
809 redirect_policy: Some(RedirectPolicy::Loose),
810 request_timeout: Some(30),
811 service_worker_enabled: Some(false),
812 preserve_host: Some(true),
813 delay: Some(10),
814 concurrency_limit: Some(2),
815 wayback: Some(true),
816 limit: Some(5),
817 depth: Some(2),
818 budget: Some(CrawlBudget::from([("*".to_string(), 3)])),
819 blacklist: Some(vec!["/a".into()]),
820 whitelist: Some(vec!["/b".into()]),
821 subdomains: Some(true),
822 tld: Some(true),
823 external_domains: Some(vec!["example.org".into()]),
824 sitemap: Some(true),
825 sitemap_only: Some(true),
826 sitemap_path: Some("sitemap.xml".into()),
827 respect_robots: Some(true),
828 link_rewrite: Some(LinkRewriteRule::replace("a", "b")),
829 crawl_timeout: Some(Timeout::from_secs(5)),
830 run_in_background: Some(false),
831 wait_for: Some(WaitFor::idle_network(Timeout::from_millis(2_000))),
832 scroll: Some(1),
833 execution_scripts: Some(BTreeMap::from([("/".to_string(), "1".to_string())])),
834 automation_scripts: Some(BTreeMap::from([(
835 "/".to_string(),
836 vec![WebAutomation::Evaluate("1".into())],
837 )])),
838 evaluate_on_new_document: Some("1".into()),
839 disable_intercept: Some(false),
840 full_resources: Some(false),
841 block_ads: Some(true),
842 block_analytics: Some(true),
843 block_stylesheets: Some(true),
844 disable_first_party_stylesheets: Some(false),
845 disable_first_party_javascript: Some(false),
846 disable_first_party_visuals: Some(false),
847 network_whitelist: Some(vec!["example.com".into()]),
848 network_blacklist: Some(vec!["example.net".into()]),
849 event_tracker: Some(EventTracker::default()),
850 return_format: Some(ReturnFormat::Markdown.into()),
851 root_selector: Some("main".into()),
852 exclude_selector: Some("nav".into()),
853 filter_output_images: Some(true),
854 filter_output_svg: Some(true),
855 filter_output_main_only: Some(true),
856 max_size: Some(1_000),
857 readability: Some(true),
858 clean_html: Some(true),
859 css_extraction_map: Some(css),
860 chunking_alg: Some(ChunkingAlg::new(ChunkingKind::BySentence, 2)),
861 metadata: Some(true),
862 return_embeddings: Some(true),
863 return_headers: Some(true),
864 return_cookies: Some(true),
865 return_page_links: Some(true),
866 return_json_data: Some(true),
867 text: Some("text".into()),
868 cache: Some(Cache::Enabled(true)),
869 webhooks: Some(WebhookSettings::new("https://example.com/hook")),
870 data_connectors: Some(serde_json::Map::from_iter([(
871 "s3".to_string(),
872 serde_json::Value::Bool(true),
873 )])),
874 router: Some(Router {
875 mode: Some("off".into()),
876 ..Router::default()
877 }),
878 provider_options: Some(BTreeMap::from([(
879 "p".to_string(),
880 serde_json::Value::Bool(true),
881 )])),
882 max_credits_allowed: Some(WholeCredits::floor(Credits::new(10.0))),
883 max_credits_per_page: Some(Credits::new(1.0)),
884 disable_hints: Some(true),
885 skip_config_checks: Some(false),
886 }
887 }
888
889 #[test]
890 fn every_request_params_field_has_a_spec() {
891 let json = serde_json::to_value(populated()).unwrap();
892 let wire: BTreeSet<String> = json.as_object().unwrap().keys().cloned().collect();
893 let keys: BTreeSet<String> = Key::ALL.iter().map(|k| k.wire().to_string()).collect();
894
895 let missing_spec: Vec<_> = wire.difference(&keys).collect();
896 let missing_field: Vec<_> = keys.difference(&wire).collect();
897 assert!(
898 missing_spec.is_empty(),
899 "fields with no Key: {missing_spec:?}"
900 );
901 assert!(
902 missing_field.is_empty(),
903 "Keys with no field: {missing_field:?}"
904 );
905 assert_eq!(Key::ALL.len(), wire.len());
906 assert_eq!(keys.len(), Key::ALL.len(), "two keys share a wire name");
907 }
908
909 #[test]
910 fn keys_follow_the_declaration_order_of_the_struct() {
911 let text = serde_json::to_string(&populated()).unwrap();
914 let mut last = 0;
915 for key in Key::ALL {
916 let at = text
917 .find(&format!("\"{}\":", key.wire()))
918 .unwrap_or_else(|| panic!("{} is not on the wire", key.wire()));
919 assert!(at >= last, "{:?} is out of order", key);
920 last = at;
921 }
922 }
923
924 #[test]
925 fn the_learnable_set_is_exactly_version_one() {
926 let schema = Schema::v1();
927 let learnable: Vec<Key> = schema.learnable().map(|spec| spec.key).collect();
928 assert_eq!(
929 learnable,
930 [
931 Key::Request,
932 Key::Proxy,
933 Key::WaitIdleMillis,
934 Key::DisableIntercept,
935 Key::FullResources,
936 Key::BlockAds,
937 Key::BlockAnalytics,
938 Key::BlockStylesheets,
939 Key::NetworkBlacklist,
940 ]
941 );
942 for (slot, key) in learnable.iter().enumerate() {
943 assert_eq!(learnable_slot(*key), Some(slot));
944 }
945 for spec in schema.specs() {
946 if !spec.learnable {
947 assert_eq!(learnable_slot(spec.key), None, "{:?}", spec.key);
948 }
949 }
950
951 let content: BTreeSet<Key> = schema
952 .specs()
953 .iter()
954 .filter(|spec| spec.content_changing)
955 .map(|spec| spec.key)
956 .collect();
957 let expected = BTreeSet::from([
958 Key::Readability,
959 Key::CleanHtml,
960 Key::RootSelector,
961 Key::ExcludeSelector,
962 Key::ReturnFormat,
963 Key::FilterOutputMainOnly,
964 Key::FilterOutputImages,
965 Key::FilterOutputSvg,
966 Key::MaxSize,
967 Key::ChunkingAlg,
968 Key::CssExtractionMap,
969 Key::Text,
970 Key::BlockStylesheets,
971 Key::DisableFirstPartyStylesheets,
972 Key::DisableFirstPartyJavascript,
973 Key::DisableFirstPartyVisuals,
974 Key::NetworkBlacklist,
975 Key::NetworkWhitelist,
976 ]);
977 assert_eq!(content, expected);
978 }
979
980 #[test]
981 fn the_spec_for_a_key_is_that_keys_spec() {
982 let schema = Schema::v1();
983 for key in Key::ALL {
984 assert_eq!(schema.spec(*key).key, *key);
985 }
986 assert_eq!(
987 schema.spec(Key::WaitIdleMillis).kind,
988 Kind::MillisBucket(&[0, 2_000, 5_000, 10_000])
989 );
990 }
991}