1use spider_route::{ProxyPool, RequestMode, PROXY_POOLS};
16
17pub const SCHEMA_VERSION: u16 = 2;
23
24pub const KEY_COUNT: usize = 76;
26
27#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
34#[non_exhaustive]
35#[repr(u8)]
36pub enum Key {
37 Url,
39 Request,
41 Proxy,
43 CountryCode,
45 RemoteProxy,
47 Stealth,
49 Fingerprint,
51 UserAgent,
53 Viewport,
55 Locale,
57 Cookies,
59 Headers,
61 Encoding,
63 Session,
65 RedirectPolicy,
67 RequestTimeout,
69 ServiceWorkerEnabled,
71 Delay,
73 ConcurrencyLimit,
75 Wayback,
77 Limit,
79 Depth,
81 Budget,
83 Blacklist,
85 Whitelist,
87 Subdomains,
89 Tld,
91 ExternalDomains,
93 Sitemap,
95 SitemapOnly,
97 SitemapPath,
99 RespectRobots,
101 LinkRewrite,
103 CrawlTimeout,
105 RunInBackground,
107 WaitIdleMillis,
109 Scroll,
111 ExecutionScripts,
113 AutomationScripts,
115 EvaluateOnNewDocument,
117 DisableIntercept,
119 FullResources,
121 BlockAds,
123 BlockAnalytics,
125 BlockStylesheets,
127 DisableFirstPartyStylesheets,
129 DisableFirstPartyJavascript,
131 DisableFirstPartyVisuals,
133 NetworkWhitelist,
135 NetworkBlacklist,
137 EventTracker,
139 ReturnFormat,
141 RootSelector,
143 ExcludeSelector,
145 FilterOutputImages,
147 FilterOutputSvg,
149 FilterOutputMainOnly,
151 MaxSize,
153 Readability,
155 CleanHtml,
157 CssExtractionMap,
159 ChunkingAlg,
161 Metadata,
163 ReturnEmbeddings,
165 ReturnHeaders,
167 ReturnCookies,
169 ReturnPageLinks,
171 ReturnJsonData,
173 Cache,
175 Webhooks,
177 DataConnectors,
179 Router,
181 ProviderOptions,
183 MaxCreditsAllowed,
185 MaxCreditsPerPage,
187 DisableHints,
189}
190
191const ALL_KEYS: [Key; KEY_COUNT] = [
193 Key::Url,
194 Key::Request,
195 Key::Proxy,
196 Key::CountryCode,
197 Key::RemoteProxy,
198 Key::Stealth,
199 Key::Fingerprint,
200 Key::UserAgent,
201 Key::Viewport,
202 Key::Locale,
203 Key::Cookies,
204 Key::Headers,
205 Key::Encoding,
206 Key::Session,
207 Key::RedirectPolicy,
208 Key::RequestTimeout,
209 Key::ServiceWorkerEnabled,
210 Key::Delay,
211 Key::ConcurrencyLimit,
212 Key::Wayback,
213 Key::Limit,
214 Key::Depth,
215 Key::Budget,
216 Key::Blacklist,
217 Key::Whitelist,
218 Key::Subdomains,
219 Key::Tld,
220 Key::ExternalDomains,
221 Key::Sitemap,
222 Key::SitemapOnly,
223 Key::SitemapPath,
224 Key::RespectRobots,
225 Key::LinkRewrite,
226 Key::CrawlTimeout,
227 Key::RunInBackground,
228 Key::WaitIdleMillis,
229 Key::Scroll,
230 Key::ExecutionScripts,
231 Key::AutomationScripts,
232 Key::EvaluateOnNewDocument,
233 Key::DisableIntercept,
234 Key::FullResources,
235 Key::BlockAds,
236 Key::BlockAnalytics,
237 Key::BlockStylesheets,
238 Key::DisableFirstPartyStylesheets,
239 Key::DisableFirstPartyJavascript,
240 Key::DisableFirstPartyVisuals,
241 Key::NetworkWhitelist,
242 Key::NetworkBlacklist,
243 Key::EventTracker,
244 Key::ReturnFormat,
245 Key::RootSelector,
246 Key::ExcludeSelector,
247 Key::FilterOutputImages,
248 Key::FilterOutputSvg,
249 Key::FilterOutputMainOnly,
250 Key::MaxSize,
251 Key::Readability,
252 Key::CleanHtml,
253 Key::CssExtractionMap,
254 Key::ChunkingAlg,
255 Key::Metadata,
256 Key::ReturnEmbeddings,
257 Key::ReturnHeaders,
258 Key::ReturnCookies,
259 Key::ReturnPageLinks,
260 Key::ReturnJsonData,
261 Key::Cache,
262 Key::Webhooks,
263 Key::DataConnectors,
264 Key::Router,
265 Key::ProviderOptions,
266 Key::MaxCreditsAllowed,
267 Key::MaxCreditsPerPage,
268 Key::DisableHints,
269];
270
271impl Key {
272 pub const ALL: &'static [Key] = &ALL_KEYS;
274
275 pub const fn index(self) -> usize {
277 self as usize
278 }
279
280 pub const fn wire(self) -> &'static str {
282 match self {
283 Key::Url => "url",
284 Key::Request => "request",
285 Key::Proxy => "proxy",
286 Key::CountryCode => "country_code",
287 Key::RemoteProxy => "remote_proxy",
288 Key::Stealth => "stealth",
289 Key::Fingerprint => "fingerprint",
290 Key::UserAgent => "user_agent",
291 Key::Viewport => "viewport",
292 Key::Locale => "locale",
293 Key::Cookies => "cookies",
294 Key::Headers => "headers",
295 Key::Encoding => "encoding",
296 Key::Session => "session",
297 Key::RedirectPolicy => "redirect_policy",
298 Key::RequestTimeout => "request_timeout",
299 Key::ServiceWorkerEnabled => "service_worker_enabled",
300 Key::Delay => "delay",
301 Key::ConcurrencyLimit => "concurrency_limit",
302 Key::Wayback => "wayback",
303 Key::Limit => "limit",
304 Key::Depth => "depth",
305 Key::Budget => "budget",
306 Key::Blacklist => "blacklist",
307 Key::Whitelist => "whitelist",
308 Key::Subdomains => "subdomains",
309 Key::Tld => "tld",
310 Key::ExternalDomains => "external_domains",
311 Key::Sitemap => "sitemap",
312 Key::SitemapOnly => "sitemap_only",
313 Key::SitemapPath => "sitemap_path",
314 Key::RespectRobots => "respect_robots",
315 Key::LinkRewrite => "link_rewrite",
316 Key::CrawlTimeout => "crawl_timeout",
317 Key::RunInBackground => "run_in_background",
318 Key::WaitIdleMillis => "wait_for",
319 Key::Scroll => "scroll",
320 Key::ExecutionScripts => "execution_scripts",
321 Key::AutomationScripts => "automation_scripts",
322 Key::EvaluateOnNewDocument => "evaluate_on_new_document",
323 Key::DisableIntercept => "disable_intercept",
324 Key::FullResources => "full_resources",
325 Key::BlockAds => "block_ads",
326 Key::BlockAnalytics => "block_analytics",
327 Key::BlockStylesheets => "block_stylesheets",
328 Key::DisableFirstPartyStylesheets => "disable_first_party_stylesheets",
329 Key::DisableFirstPartyJavascript => "disable_first_party_javascript",
330 Key::DisableFirstPartyVisuals => "disable_first_party_visuals",
331 Key::NetworkWhitelist => "network_whitelist",
332 Key::NetworkBlacklist => "network_blacklist",
333 Key::EventTracker => "event_tracker",
334 Key::ReturnFormat => "return_format",
335 Key::RootSelector => "root_selector",
336 Key::ExcludeSelector => "exclude_selector",
337 Key::FilterOutputImages => "filter_output_images",
338 Key::FilterOutputSvg => "filter_output_svg",
339 Key::FilterOutputMainOnly => "filter_output_main_only",
340 Key::MaxSize => "max_size",
341 Key::Readability => "readability",
342 Key::CleanHtml => "clean_html",
343 Key::CssExtractionMap => "css_extraction_map",
344 Key::ChunkingAlg => "chunking_alg",
345 Key::Metadata => "metadata",
346 Key::ReturnEmbeddings => "return_embeddings",
347 Key::ReturnHeaders => "return_headers",
348 Key::ReturnCookies => "return_cookies",
349 Key::ReturnPageLinks => "return_page_links",
350 Key::ReturnJsonData => "return_json_data",
351 Key::Cache => "cache",
352 Key::Webhooks => "webhooks",
353 Key::DataConnectors => "data_connectors",
354 Key::Router => "router",
355 Key::ProviderOptions => "provider_options",
356 Key::MaxCreditsAllowed => "max_credits_allowed",
357 Key::MaxCreditsPerPage => "max_credits_per_page",
358 Key::DisableHints => "disable_hints",
359 }
360 }
361}
362
363#[derive(Debug, Clone, Copy, PartialEq, Eq)]
365pub enum Kind {
366 Bool,
368 Enum(&'static [&'static str]),
370 IntRange {
372 min: i64,
374 max: i64,
376 },
377 MillisBucket(&'static [u32]),
379 StringList,
381 Map,
383 Opaque,
386}
387
388#[derive(Debug, Clone, Copy, PartialEq, Eq)]
390pub enum Dependency {
391 Rendered,
395 NotWith(Key),
397}
398
399#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
401pub enum Group {
402 Transport,
404 Proxy,
406 Scope,
408 Wait,
410 Scripting,
412 Resources,
414 Extraction,
416 Cache,
418 Delivery,
420 Budget,
422 Other,
424}
425
426#[derive(Debug, Clone, Copy, PartialEq, Eq)]
428pub struct ParamSpec {
429 pub key: Key,
431 pub kind: Kind,
433 pub requires: &'static [Dependency],
435 pub learnable: bool,
437 pub content_changing: bool,
440 pub group: Group,
442}
443
444impl ParamSpec {
445 const fn fixed(key: Key, kind: Kind, group: Group) -> ParamSpec {
447 ParamSpec {
448 key,
449 kind,
450 requires: &[],
451 learnable: false,
452 content_changing: false,
453 group,
454 }
455 }
456
457 const fn learnable(mut self) -> ParamSpec {
459 self.learnable = true;
460 self
461 }
462
463 const fn content(mut self) -> ParamSpec {
465 self.content_changing = true;
466 self
467 }
468
469 const fn needs(mut self, requires: &'static [Dependency]) -> ParamSpec {
471 self.requires = requires;
472 self
473 }
474}
475
476pub const MODES: [RequestMode; 3] = [RequestMode::Http, RequestMode::Smart, RequestMode::Browser];
478
479pub const PROXIES: [ProxyPool; 2] = PROXY_POOLS;
481
482pub const WAIT_BUCKETS: [u32; 4] = [0, 2_000, 5_000, 10_000];
484
485const RENDERED: &[Dependency] = &[Dependency::Rendered];
486
487const BLACKLIST_NEEDS: &[Dependency] = RENDERED;
491const INTERCEPT_NEEDS: &[Dependency] = RENDERED;
492
493const U32_MAX: i64 = 4_294_967_295;
494const I64_MAX: i64 = 9_223_372_036_854_775_807;
495
496const UNKNOWN: ParamSpec = ParamSpec::fixed(Key::Url, Kind::Opaque, Group::Other);
498
499use self::{Group as G, Kind as K, ParamSpec as S};
500
501const SPECS: [ParamSpec; KEY_COUNT] = [
504 S::fixed(Key::Url, K::Opaque, G::Transport),
505 S::fixed(
506 Key::Request,
507 K::Enum(&["http", "smart", "browser"]),
508 G::Transport,
509 )
510 .learnable(),
511 S::fixed(Key::Proxy, K::Enum(&["isp", "residential"]), G::Proxy).learnable(),
512 S::fixed(Key::CountryCode, K::Opaque, G::Proxy),
513 S::fixed(Key::RemoteProxy, K::Opaque, G::Proxy),
514 S::fixed(Key::Stealth, K::Bool, G::Transport),
515 S::fixed(Key::Fingerprint, K::Bool, G::Transport),
516 S::fixed(Key::UserAgent, K::Opaque, G::Transport),
517 S::fixed(Key::Viewport, K::Opaque, G::Transport),
518 S::fixed(Key::Locale, K::Opaque, G::Transport),
519 S::fixed(Key::Cookies, K::Opaque, G::Transport),
520 S::fixed(Key::Headers, K::Map, G::Transport),
521 S::fixed(Key::Encoding, K::Opaque, G::Transport),
522 S::fixed(Key::Session, K::Bool, G::Transport),
523 S::fixed(Key::RedirectPolicy, K::Opaque, G::Transport),
524 S::fixed(
525 Key::RequestTimeout,
526 K::IntRange { min: 0, max: 255 },
527 G::Transport,
528 ),
529 S::fixed(Key::ServiceWorkerEnabled, K::Bool, G::Transport),
530 S::fixed(
531 Key::Delay,
532 K::IntRange {
533 min: 0,
534 max: 60_000,
535 },
536 G::Transport,
537 ),
538 S::fixed(
539 Key::ConcurrencyLimit,
540 K::IntRange {
541 min: 0,
542 max: U32_MAX,
543 },
544 G::Transport,
545 ),
546 S::fixed(Key::Wayback, K::Bool, G::Transport),
547 S::fixed(
548 Key::Limit,
549 K::IntRange {
550 min: 0,
551 max: U32_MAX,
552 },
553 G::Scope,
554 ),
555 S::fixed(
556 Key::Depth,
557 K::IntRange {
558 min: 0,
559 max: U32_MAX,
560 },
561 G::Scope,
562 ),
563 S::fixed(Key::Budget, K::Map, G::Scope),
564 S::fixed(Key::Blacklist, K::StringList, G::Scope),
565 S::fixed(Key::Whitelist, K::StringList, G::Scope),
566 S::fixed(Key::Subdomains, K::Bool, G::Scope),
567 S::fixed(Key::Tld, K::Bool, G::Scope),
568 S::fixed(Key::ExternalDomains, K::StringList, G::Scope),
569 S::fixed(Key::Sitemap, K::Bool, G::Scope),
570 S::fixed(Key::SitemapOnly, K::Bool, G::Scope),
571 S::fixed(Key::SitemapPath, K::Opaque, G::Scope),
572 S::fixed(Key::RespectRobots, K::Bool, G::Scope),
573 S::fixed(Key::LinkRewrite, K::Opaque, G::Scope),
574 S::fixed(Key::CrawlTimeout, K::Opaque, G::Scope),
575 S::fixed(Key::RunInBackground, K::Bool, G::Delivery),
576 S::fixed(Key::WaitIdleMillis, K::MillisBucket(&WAIT_BUCKETS), G::Wait)
577 .learnable()
578 .needs(RENDERED),
579 S::fixed(
580 Key::Scroll,
581 K::IntRange {
582 min: 0,
583 max: U32_MAX,
584 },
585 G::Wait,
586 ),
587 S::fixed(Key::ExecutionScripts, K::Map, G::Scripting),
588 S::fixed(Key::AutomationScripts, K::Map, G::Scripting),
589 S::fixed(Key::EvaluateOnNewDocument, K::Opaque, G::Scripting),
590 S::fixed(Key::DisableIntercept, K::Bool, G::Resources)
591 .learnable()
592 .needs(INTERCEPT_NEEDS),
593 S::fixed(Key::FullResources, K::Bool, G::Resources)
594 .learnable()
595 .needs(RENDERED),
596 S::fixed(Key::BlockAds, K::Bool, G::Resources)
597 .learnable()
598 .needs(RENDERED),
599 S::fixed(Key::BlockAnalytics, K::Bool, G::Resources)
600 .learnable()
601 .needs(RENDERED),
602 S::fixed(Key::BlockStylesheets, K::Bool, G::Resources)
603 .learnable()
604 .content()
605 .needs(RENDERED),
606 S::fixed(Key::DisableFirstPartyStylesheets, K::Bool, G::Resources).content(),
607 S::fixed(Key::DisableFirstPartyJavascript, K::Bool, G::Resources).content(),
608 S::fixed(Key::DisableFirstPartyVisuals, K::Bool, G::Resources).content(),
609 S::fixed(Key::NetworkWhitelist, K::StringList, G::Resources).content(),
610 S::fixed(Key::NetworkBlacklist, K::StringList, G::Resources)
611 .learnable()
612 .content()
613 .needs(BLACKLIST_NEEDS),
614 S::fixed(Key::EventTracker, K::Opaque, G::Other),
615 S::fixed(Key::ReturnFormat, K::Opaque, G::Extraction).content(),
616 S::fixed(Key::RootSelector, K::Opaque, G::Extraction).content(),
617 S::fixed(Key::ExcludeSelector, K::Opaque, G::Extraction).content(),
618 S::fixed(Key::FilterOutputImages, K::Bool, G::Extraction).content(),
619 S::fixed(Key::FilterOutputSvg, K::Bool, G::Extraction).content(),
620 S::fixed(Key::FilterOutputMainOnly, K::Bool, G::Extraction).content(),
621 S::fixed(
622 Key::MaxSize,
623 K::IntRange {
624 min: 0,
625 max: I64_MAX,
626 },
627 G::Extraction,
628 )
629 .content(),
630 S::fixed(Key::Readability, K::Bool, G::Extraction).content(),
631 S::fixed(Key::CleanHtml, K::Bool, G::Extraction).content(),
632 S::fixed(Key::CssExtractionMap, K::Map, G::Extraction).content(),
633 S::fixed(Key::ChunkingAlg, K::Opaque, G::Extraction).content(),
634 S::fixed(Key::Metadata, K::Bool, G::Extraction),
635 S::fixed(Key::ReturnEmbeddings, K::Bool, G::Extraction),
636 S::fixed(Key::ReturnHeaders, K::Bool, G::Extraction),
637 S::fixed(Key::ReturnCookies, K::Bool, G::Extraction),
638 S::fixed(Key::ReturnPageLinks, K::Bool, G::Extraction),
639 S::fixed(Key::ReturnJsonData, K::Bool, G::Extraction),
640 S::fixed(Key::Cache, K::Opaque, G::Cache),
641 S::fixed(Key::Webhooks, K::Opaque, G::Delivery),
642 S::fixed(Key::DataConnectors, K::Map, G::Delivery),
643 S::fixed(Key::Router, K::Opaque, G::Other),
644 S::fixed(Key::ProviderOptions, K::Map, G::Other),
645 S::fixed(
646 Key::MaxCreditsAllowed,
647 K::IntRange {
648 min: 0,
649 max: I64_MAX,
650 },
651 G::Budget,
652 ),
653 S::fixed(Key::MaxCreditsPerPage, K::Opaque, G::Budget),
654 S::fixed(Key::DisableHints, K::Bool, G::Other),
655];
656
657const _: () = {
661 let mut at = 0;
662 let mut keys: &[Key] = &ALL_KEYS;
663 let mut specs: &[ParamSpec] = &SPECS;
664 while let ([key, keys_rest @ ..], [spec, specs_rest @ ..]) = (keys, specs) {
665 assert!(*key as usize == at);
666 assert!(spec.key as usize == at);
667 keys = keys_rest;
668 specs = specs_rest;
669 at += 1;
670 }
671 assert!(at == KEY_COUNT);
672 assert!(Key::DisableHints as usize + 1 == KEY_COUNT);
673};
674
675#[derive(Debug, Clone, Copy, PartialEq, Eq)]
677pub struct Schema {
678 specs: &'static [ParamSpec; KEY_COUNT],
679}
680
681impl Schema {
682 pub const fn v1() -> Schema {
689 Schema { specs: &SPECS }
690 }
691
692 pub fn spec(&self, key: Key) -> &ParamSpec {
694 match self.specs.get(key.index()) {
698 Some(spec) => spec,
699 None => &UNKNOWN,
700 }
701 }
702
703 pub fn learnable(&self) -> impl Iterator<Item = &ParamSpec> {
705 self.specs.iter().filter(|spec| spec.learnable)
706 }
707
708 pub fn specs(&self) -> &[ParamSpec] {
710 self.specs
711 }
712}
713
714impl Default for Schema {
715 fn default() -> Schema {
716 Schema::v1()
717 }
718}
719
720pub const fn learnable_slot(key: Key) -> Option<usize> {
722 match key {
723 Key::Request => Some(0),
724 Key::Proxy => Some(1),
725 Key::WaitIdleMillis => Some(2),
726 Key::DisableIntercept => Some(3),
727 Key::FullResources => Some(4),
728 Key::BlockAds => Some(5),
729 Key::BlockAnalytics => Some(6),
730 Key::BlockStylesheets => Some(7),
731 Key::NetworkBlacklist => Some(8),
732 _ => None,
733 }
734}
735
736pub const fn service_default(key: Key) -> bool {
741 matches!(
742 key,
743 Key::BlockAds | Key::BlockAnalytics | Key::BlockStylesheets
744 )
745}
746
747pub const KEEP_CODE: u8 = 0;
749
750pub fn edit_code(key: Key) -> Option<u8> {
753 Schema::v1()
754 .learnable()
755 .position(|spec| spec.key == key)
756 .map(|rank| 1 + rank as u8)
757}
758
759#[cfg(test)]
760pub(crate) mod tests {
761 #![allow(
762 clippy::unwrap_used,
763 clippy::expect_used,
764 clippy::panic,
765 clippy::unreachable,
766 clippy::indexing_slicing,
767 clippy::string_slice
768 )]
769 use super::*;
770 use spider_cloud_agent::params::{
771 Cache, ChunkingAlg, ChunkingKind, CrawlBudget, CssExtractionMap, EventTracker,
772 LinkRewriteRule, RedirectPolicy, RequestParams, ReturnFormat, Router, SelectorGroup,
773 Timeout, Viewport, WaitFor, WebAutomation, WebhookSettings,
774 };
775 use spider_cloud_agent::{Credits, WholeCredits};
776 use std::collections::{BTreeMap, BTreeSet};
777
778 pub(crate) fn populated() -> RequestParams {
781 let mut css = CssExtractionMap::new();
782 css.insert("/".into(), vec![SelectorGroup::css("title", ["h1"])]);
783 RequestParams {
784 url: Some("https://example.com/".into()),
785 request: Some(RequestMode::Browser),
786 proxy: Some(ProxyPool::Residential),
787 country_code: spider_route::Country::new("de"),
788 remote_proxy: Some("http://example.com:8080".into()),
789 stealth: Some(true),
790 fingerprint: Some(true),
791 user_agent: Some("agent".into()),
792 viewport: Some(Viewport::desktop()),
793 locale: Some("en-GB".into()),
794 cookies: Some("a=b".into()),
795 headers: Some(BTreeMap::from([("x".to_string(), "y".to_string())])),
796 encoding: Some("utf-8".into()),
797 session: Some(true),
798 redirect_policy: Some(RedirectPolicy::Loose),
799 request_timeout: Some(30),
800 service_worker_enabled: Some(false),
801 delay: Some(10),
802 concurrency_limit: Some(2),
803 wayback: Some(true),
804 limit: Some(5),
805 depth: Some(2),
806 budget: Some(CrawlBudget::from([("*".to_string(), 3)])),
807 blacklist: Some(vec!["/a".into()]),
808 whitelist: Some(vec!["/b".into()]),
809 subdomains: Some(true),
810 tld: Some(true),
811 external_domains: Some(vec!["example.org".into()]),
812 sitemap: Some(true),
813 sitemap_only: Some(true),
814 sitemap_path: Some("sitemap.xml".into()),
815 respect_robots: Some(true),
816 link_rewrite: Some(LinkRewriteRule::replace("a", "b")),
817 crawl_timeout: Some(Timeout::from_secs(5)),
818 run_in_background: Some(false),
819 wait_for: Some(WaitFor::idle_network(Timeout::from_millis(2_000))),
820 scroll: Some(1),
821 execution_scripts: Some(BTreeMap::from([("/".to_string(), "1".to_string())])),
822 automation_scripts: Some(BTreeMap::from([(
823 "/".to_string(),
824 vec![WebAutomation::Evaluate("1".into())],
825 )])),
826 evaluate_on_new_document: Some("1".into()),
827 disable_intercept: Some(false),
828 full_resources: Some(false),
829 block_ads: Some(true),
830 block_analytics: Some(true),
831 block_stylesheets: Some(true),
832 disable_first_party_stylesheets: Some(false),
833 disable_first_party_javascript: Some(false),
834 disable_first_party_visuals: Some(false),
835 network_whitelist: Some(vec!["example.com".into()]),
836 network_blacklist: Some(vec!["example.net".into()]),
837 event_tracker: Some(EventTracker::default()),
838 return_format: Some(ReturnFormat::Markdown.into()),
839 root_selector: Some("main".into()),
840 exclude_selector: Some("nav".into()),
841 filter_output_images: Some(true),
842 filter_output_svg: Some(true),
843 filter_output_main_only: Some(true),
844 max_size: Some(1_000),
845 readability: Some(true),
846 clean_html: Some(true),
847 css_extraction_map: Some(css),
848 chunking_alg: Some(ChunkingAlg::new(ChunkingKind::BySentence, 2)),
849 metadata: Some(true),
850 return_embeddings: Some(true),
851 return_headers: Some(true),
852 return_cookies: Some(true),
853 return_page_links: Some(true),
854 return_json_data: Some(true),
855 cache: Some(Cache::Enabled(true)),
856 webhooks: Some(WebhookSettings::new("https://example.com/hook")),
857 data_connectors: Some(serde_json::Map::from_iter([(
858 "s3".to_string(),
859 serde_json::Value::Bool(true),
860 )])),
861 router: Some(Router {
862 mode: Some("off".into()),
863 ..Router::default()
864 }),
865 provider_options: Some(BTreeMap::from([(
866 "p".to_string(),
867 serde_json::Value::Bool(true),
868 )])),
869 max_credits_allowed: Some(WholeCredits::floor(Credits::new(10.0))),
870 max_credits_per_page: Some(Credits::new(1.0)),
871 disable_hints: Some(true),
872 }
873 }
874
875 #[test]
876 fn every_request_params_field_has_a_spec() {
877 let json = serde_json::to_value(populated()).unwrap();
878 let wire: BTreeSet<String> = json.as_object().unwrap().keys().cloned().collect();
879 let keys: BTreeSet<String> = Key::ALL.iter().map(|k| k.wire().to_string()).collect();
880
881 let missing_spec: Vec<_> = wire.difference(&keys).collect();
882 let missing_field: Vec<_> = keys.difference(&wire).collect();
883 assert!(
884 missing_spec.is_empty(),
885 "fields with no Key: {missing_spec:?}"
886 );
887 assert!(
888 missing_field.is_empty(),
889 "Keys with no field: {missing_field:?}"
890 );
891 assert_eq!(Key::ALL.len(), wire.len());
892 assert_eq!(keys.len(), Key::ALL.len(), "two keys share a wire name");
893 }
894
895 #[test]
896 fn keys_follow_the_declaration_order_of_the_struct() {
897 let text = serde_json::to_string(&populated()).unwrap();
900 let mut last = 0;
901 for key in Key::ALL {
902 let at = text
903 .find(&format!("\"{}\":", key.wire()))
904 .unwrap_or_else(|| panic!("{} is not on the wire", key.wire()));
905 assert!(at >= last, "{:?} is out of order", key);
906 last = at;
907 }
908 }
909
910 #[test]
911 fn the_learnable_set_is_exactly_version_one() {
912 let schema = Schema::v1();
913 let learnable: Vec<Key> = schema.learnable().map(|spec| spec.key).collect();
914 assert_eq!(
915 learnable,
916 [
917 Key::Request,
918 Key::Proxy,
919 Key::WaitIdleMillis,
920 Key::DisableIntercept,
921 Key::FullResources,
922 Key::BlockAds,
923 Key::BlockAnalytics,
924 Key::BlockStylesheets,
925 Key::NetworkBlacklist,
926 ]
927 );
928 for (slot, key) in learnable.iter().enumerate() {
929 assert_eq!(learnable_slot(*key), Some(slot));
930 }
931 for spec in schema.specs() {
932 if !spec.learnable {
933 assert_eq!(learnable_slot(spec.key), None, "{:?}", spec.key);
934 }
935 }
936
937 let content: BTreeSet<Key> = schema
938 .specs()
939 .iter()
940 .filter(|spec| spec.content_changing)
941 .map(|spec| spec.key)
942 .collect();
943 let expected = BTreeSet::from([
944 Key::Readability,
945 Key::CleanHtml,
946 Key::RootSelector,
947 Key::ExcludeSelector,
948 Key::ReturnFormat,
949 Key::FilterOutputMainOnly,
950 Key::FilterOutputImages,
951 Key::FilterOutputSvg,
952 Key::MaxSize,
953 Key::ChunkingAlg,
954 Key::CssExtractionMap,
955 Key::BlockStylesheets,
956 Key::DisableFirstPartyStylesheets,
957 Key::DisableFirstPartyJavascript,
958 Key::DisableFirstPartyVisuals,
959 Key::NetworkBlacklist,
960 Key::NetworkWhitelist,
961 ]);
962 assert_eq!(content, expected);
963 }
964
965 #[test]
966 fn the_spec_for_a_key_is_that_keys_spec() {
967 let schema = Schema::v1();
968 for key in Key::ALL {
969 assert_eq!(schema.spec(*key).key, *key);
970 }
971 assert_eq!(
972 schema.spec(Key::WaitIdleMillis).kind,
973 Kind::MillisBucket(&[0, 2_000, 5_000, 10_000])
974 );
975 }
976}