crawlberg/types/
config.rs

1use std::collections::HashMap;
2use std::path::PathBuf;
3use std::time::Duration;
4
5use serde::{Deserialize, Serialize};
6
7use super::AssetCategory;
8use super::dispatch::DispatchProfile;
9use crate::net::SsrfPolicy;
10
11/// Metadata about an LLM extraction pass.
12#[derive(Debug, Clone, Default, Serialize, Deserialize)]
13#[serde(deny_unknown_fields)]
14pub struct ExtractionMeta {
15    /// Estimated cost of the LLM call in USD.
16    pub cost: Option<f64>,
17    /// Number of prompt (input) tokens consumed.
18    pub prompt_tokens: Option<u64>,
19    /// Number of completion (output) tokens generated.
20    pub completion_tokens: Option<u64>,
21    /// The model identifier used for extraction.
22    pub model: Option<String>,
23    /// Number of content chunks sent to the LLM.
24    pub chunks_processed: usize,
25}
26
27/// When to use the headless browser fallback.
28#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
29#[serde(rename_all = "snake_case")]
30pub enum BrowserMode {
31    /// Automatically detect when JS rendering is needed and fall back to browser.
32    #[default]
33    Auto,
34    /// Always use the browser for every request.
35    Always,
36    /// Never use the browser fallback.
37    Never,
38    /// Always use the browser with all stealth surfaces enabled.
39    ///
40    /// Behaves like [`Always`](BrowserMode::Always) for escalation purposes
41    /// (every request is routed through the browser tier), but additionally
42    /// enables:
43    ///
44    /// - browser JavaScript stealth patches
45    /// - native-backend TLS fingerprint spoofing
46    /// - stealth-aware default user-agent when no explicit UA is set
47    /// - 1920×1080 viewport override
48    ///
49    /// Use this instead of setting the now-removed `BrowserConfig.stealth`
50    /// boolean field.
51    Stealth,
52}
53
54/// Wait strategy for browser page rendering.
55#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
56#[serde(rename_all = "snake_case")]
57pub enum BrowserWait {
58    /// Wait until network activity is idle.
59    #[default]
60    NetworkIdle,
61    /// Wait for a specific CSS selector to appear in the DOM.
62    Selector,
63    /// Wait for a fixed duration after navigation.
64    Fixed,
65}
66
67/// Browser backend used for JavaScript rendering.
68#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
69#[serde(rename_all = "snake_case")]
70pub enum BrowserBackend {
71    /// Existing Chromium/CDP backend powered by chromiumoxide.
72    #[default]
73    Chromiumoxide,
74    /// Crawlberg-owned native browser backend derived from Obscura.
75    Native,
76}
77
78pub(crate) mod duration_ms {
79    use serde::{Deserialize, Deserializer, Serialize, Serializer};
80    use std::time::Duration;
81
82    pub fn serialize<S: Serializer>(d: &Duration, s: S) -> Result<S::Ok, S::Error> {
83        d.as_millis().serialize(s)
84    }
85
86    pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Duration, D::Error> {
87        let ms = u64::deserialize(d)?;
88        Ok(Duration::from_millis(ms))
89    }
90}
91
92pub(crate) mod option_duration_ms {
93    use serde::{Deserialize, Deserializer, Serialize, Serializer};
94    use std::time::Duration;
95
96    pub fn serialize<S: Serializer>(d: &Option<Duration>, s: S) -> Result<S::Ok, S::Error> {
97        d.map(|d| d.as_millis() as u64).serialize(s)
98    }
99
100    pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Option<Duration>, D::Error> {
101        let ms: Option<u64> = Option::deserialize(d)?;
102        Ok(ms.map(Duration::from_millis))
103    }
104}
105
106/// Proxy configuration for HTTP requests.
107#[derive(Debug, Clone, Default, Serialize, Deserialize)]
108#[serde(deny_unknown_fields)]
109pub struct ProxyConfig {
110    /// Proxy URL (e.g. "http://proxy:8080", "socks5://proxy:1080").
111    pub url: String,
112    /// Optional username for proxy authentication.
113    pub username: Option<String>,
114    /// Optional password for proxy authentication.
115    pub password: Option<String>,
116}
117
118/// Authentication configuration.
119#[derive(Debug, Clone, Serialize, Deserialize)]
120#[serde(deny_unknown_fields, tag = "type")]
121pub enum AuthConfig {
122    /// HTTP Basic authentication.
123    #[serde(rename = "basic")]
124    Basic {
125        /// Username sent in the `Authorization: Basic` header.
126        username: String,
127        /// Password sent in the `Authorization: Basic` header.
128        password: String,
129    },
130    /// Bearer token authentication.
131    #[serde(rename = "bearer")]
132    Bearer {
133        /// Token sent in the `Authorization: Bearer` header.
134        token: String,
135    },
136    /// Custom authentication header.
137    #[serde(rename = "header")]
138    Header {
139        /// HTTP header name to set on each request.
140        name: String,
141        /// HTTP header value to send.
142        value: String,
143    },
144}
145
146impl Default for AuthConfig {
147    fn default() -> Self {
148        Self::Basic {
149            username: String::new(),
150            password: String::new(),
151        }
152    }
153}
154
155/// Content extraction and conversion configuration.
156///
157/// Controls how HTML is converted to the output format. Uses
158/// html-to-markdown-rs as the conversion engine for all formats
159/// (markdown, plain text, djot).
160#[derive(Debug, Clone, Serialize, Deserialize)]
161#[serde(default)]
162pub struct ContentConfig {
163    /// Output format: `"markdown"` (default), `"plain"`, `"djot"`.
164    pub output_format: String,
165    /// Preprocessing aggressiveness: `"minimal"`, `"standard"` (default), `"aggressive"`.
166    ///
167    /// - Minimal: only scripts/styles removed.
168    /// - Standard: also removes nav, nav-hinted headers/footers/asides, forms.
169    /// - Aggressive: removes all footers/asides unconditionally.
170    pub preprocessing_preset: String,
171    /// Remove navigation elements (nav, breadcrumbs, menus). Default: `true`.
172    pub remove_navigation: bool,
173    /// Remove form elements. Default: `true`.
174    pub remove_forms: bool,
175    /// HTML tag names to strip (render children only, remove the tag wrapper).
176    /// Default: `["noscript"]`.
177    #[serde(default)]
178    pub strip_tags: Vec<String>,
179    /// HTML tag names to preserve as raw HTML in output.
180    #[serde(default)]
181    pub preserve_tags: Vec<String>,
182    /// CSS selectors for elements to exclude entirely (element + all content).
183    ///
184    /// Unlike `strip_tags` (which removes the wrapper but keeps children),
185    /// excluded elements and all descendants are dropped. Supports CSS selectors:
186    /// `.class`, `#id`, `[attribute]`, compound selectors.
187    ///
188    /// Example: `[".cookie-banner", "#ad-container", "[role='complementary']"]`
189    #[serde(default)]
190    pub exclude_selectors: Vec<String>,
191    /// Skip image elements in output. Default: `false`.
192    pub skip_images: bool,
193    /// Max DOM traversal depth. Prevents stack overflow on deeply nested HTML.
194    pub max_depth: Option<usize>,
195    /// Enable line wrapping. Default: `false`.
196    pub wrap: bool,
197    /// Wrap width when `wrap` is enabled. Default: `80`.
198    pub wrap_width: usize,
199    /// Include document structure tree in output. Default: `true`.
200    pub include_document_structure: bool,
201}
202
203impl Default for ContentConfig {
204    fn default() -> Self {
205        Self {
206            output_format: "markdown".to_owned(),
207            preprocessing_preset: "standard".to_owned(),
208            remove_navigation: true,
209            remove_forms: true,
210            strip_tags: vec!["noscript".to_owned()],
211            preserve_tags: Vec::new(),
212            exclude_selectors: Vec::new(),
213            skip_images: false,
214            max_depth: None,
215            wrap: false,
216            wrap_width: 80,
217            include_document_structure: true,
218        }
219    }
220}
221
222/// Browser fallback configuration.
223#[derive(Debug, Clone, Serialize, Deserialize)]
224#[serde(deny_unknown_fields, default)]
225pub struct BrowserConfig {
226    /// When to use the headless browser fallback.
227    pub mode: BrowserMode,
228    /// Browser backend used to render JavaScript-heavy pages.
229    pub backend: BrowserBackend,
230    /// CDP WebSocket endpoint for connecting to an external browser instance.
231    pub endpoint: Option<String>,
232    /// Timeout for browser page load and rendering (in milliseconds when serialized).
233    #[serde(with = "duration_ms")]
234    pub timeout: Duration,
235    /// Wait strategy after browser navigation.
236    pub wait: BrowserWait,
237    /// CSS selector to wait for when `wait` is `Selector`.
238    pub wait_selector: Option<String>,
239    /// Extra time to wait after the wait condition is met.
240    #[serde(default, with = "option_duration_ms")]
241    pub extra_wait: Option<Duration>,
242    /// Proxy for browser fetches. Overrides `CrawlConfig.proxy` when set.
243    /// Native backend supports http/https only (no SOCKS5).
244    #[serde(default)]
245    pub proxy: Option<ProxyConfig>,
246    /// URL patterns to block before the network request fires. Supports `*`
247    /// wildcards. Useful for skipping ads/analytics/large images. Honored by
248    /// `BrowserBackend::Native`; chromiumoxide ignores this field today.
249    #[serde(default)]
250    pub block_url_patterns: Vec<String>,
251    /// JavaScript snippet evaluated after navigation completes.
252    ///
253    /// Scraping captures the native backend result in `ScrapeResult.browser.eval_result`.
254    /// Interactions run this script before page actions on both browser backends but do
255    /// not include the script result in `InteractionResult`.
256    #[serde(default)]
257    pub eval_script: Option<String>,
258    /// User-agent used when fetching robots.txt. Defaults to `BrowserConfig.user_agent`
259    /// (or crawlberg's default) if unset. Native only.
260    #[serde(default)]
261    pub robots_user_agent: Option<String>,
262    /// Capture the full network event stream into the result. Default false
263    /// (only the document event is captured). Native only.
264    #[serde(default)]
265    pub capture_network_events: bool,
266    /// Enable session affinity: reuse chromiumoxide Pages for same-domain
267    /// requests so cookies + fingerprint + solved challenges persist.
268    /// Default: true. When false, each request gets a fresh Page.
269    #[serde(default)]
270    pub session_affinity: bool,
271}
272
273impl Default for BrowserConfig {
274    fn default() -> Self {
275        Self {
276            mode: BrowserMode::Auto,
277            backend: BrowserBackend::Chromiumoxide,
278            endpoint: None,
279            timeout: Duration::from_secs(30),
280            wait: BrowserWait::default(),
281            wait_selector: None,
282            extra_wait: None,
283            proxy: None,
284            block_url_patterns: Vec::new(),
285            eval_script: None,
286            robots_user_agent: None,
287            capture_network_events: false,
288            session_affinity: true,
289        }
290    }
291}
292
293/// Configuration for crawl, scrape, and map operations.
294#[derive(Debug, Clone, Serialize, Deserialize)]
295#[serde(deny_unknown_fields, default)]
296pub struct CrawlConfig {
297    /// Maximum crawl depth (number of link hops from the start URL).
298    pub max_depth: Option<usize>,
299    /// Maximum number of pages to crawl.
300    pub max_pages: Option<usize>,
301    /// Maximum number of concurrent requests.
302    pub max_concurrent: Option<usize>,
303    /// Whether to respect robots.txt directives.
304    pub respect_robots_txt: bool,
305    /// When true, HTTP-level error responses (404 NotFound, 403 Forbidden, WAF blocks)
306    /// are surfaced as `ScrapeResult` records with the matching `status_code` rather
307    /// than raised as `CrawlError`. Default `false` preserves the historical
308    /// throw-on-error contract for direct fetches. Independently of this flag,
309    /// 404s reached at the end of a redirect chain are *always* surfaced softly —
310    /// the user opted into redirect-following, so receiving a 404 there is part of
311    /// the normal flow rather than an unexpected error.
312    #[serde(default)]
313    pub soft_http_errors: bool,
314    /// Custom user-agent string.
315    pub user_agent: Option<String>,
316    /// Whether to restrict crawling to the same domain.
317    pub stay_on_domain: bool,
318    /// Whether to allow subdomains when `stay_on_domain` is true.
319    pub allow_subdomains: bool,
320    /// Regex patterns for paths to include during crawling.
321    #[serde(default)]
322    pub include_paths: Vec<String>,
323    /// Regex patterns for paths to exclude during crawling.
324    #[serde(default)]
325    pub exclude_paths: Vec<String>,
326    /// Custom HTTP headers to send with each request.
327    #[serde(default)]
328    pub custom_headers: HashMap<String, String>,
329    /// Timeout for individual HTTP requests (in milliseconds when serialized).
330    #[serde(with = "duration_ms")]
331    pub request_timeout: Duration,
332    /// Per-domain rate limit in milliseconds. When set, enforces a minimum delay
333    /// between requests to the same domain. Defaults to 200ms when `None`.
334    pub rate_limit_ms: Option<u64>,
335    /// Maximum number of redirects to follow.
336    pub max_redirects: usize,
337    /// Number of retry attempts for failed requests.
338    pub retry_count: usize,
339    /// HTTP status codes that should trigger a retry.
340    #[serde(default)]
341    pub retry_codes: Vec<u16>,
342    /// Whether to enable cookie handling.
343    pub cookies_enabled: bool,
344    /// Authentication configuration.
345    pub auth: Option<AuthConfig>,
346    /// Maximum response body size in bytes.
347    pub max_body_size: Option<usize>,
348    /// CSS selectors for tags to remove from HTML before processing.
349    #[serde(default)]
350    pub remove_tags: Vec<String>,
351    /// Content extraction and conversion configuration.
352    #[serde(default)]
353    pub content: ContentConfig,
354    /// Maximum number of URLs to return from a map operation.
355    pub map_limit: Option<usize>,
356    /// Search filter for map results (case-insensitive substring match on URLs).
357    pub map_search: Option<String>,
358    /// Whether to download assets (CSS, JS, images, etc.) from the page.
359    pub download_assets: bool,
360    /// Filter for asset categories to download.
361    #[serde(default)]
362    pub asset_types: Vec<AssetCategory>,
363    /// Maximum size in bytes for individual asset downloads.
364    pub max_asset_size: Option<usize>,
365    /// Browser configuration.
366    #[serde(default)]
367    pub browser: BrowserConfig,
368    /// Proxy configuration for HTTP requests.
369    pub proxy: Option<ProxyConfig>,
370    /// List of user-agent strings for rotation. If non-empty, overrides `user_agent`.
371    #[serde(default)]
372    pub user_agents: Vec<String>,
373    /// Whether to capture a screenshot when using the browser.
374    pub capture_screenshot: bool,
375    /// Re-enqueue discovered `LinkType::Document` URLs into the crawl frontier so
376    /// the crawl follows links *from* document pages (PDFs, etc.) as it would
377    /// from HTML pages. Default: `false` (documents terminate at materialisation).
378    #[serde(default)]
379    pub follow_document_urls: bool,
380    /// Maximum document-depth (from the seed URL through document links only)
381    /// when `follow_document_urls` is true. `None` means inherit `max_depth`.
382    /// Independent of `max_depth`: a document URL is enqueued only if BOTH the
383    /// outer `max_depth` and (if set) `document_url_depth` permit it.
384    #[serde(default)]
385    pub document_url_depth: Option<u32>,
386    /// Whether to download non-HTML documents (PDF, DOCX, images, code, etc.) instead of skipping them.
387    pub download_documents: bool,
388    /// Maximum size in bytes for document downloads. Defaults to 50 MB.
389    pub document_max_size: Option<usize>,
390    /// Allowlist of MIME types to download. If empty, uses built-in defaults.
391    #[serde(default)]
392    pub document_mime_types: Vec<String>,
393    /// Path to write WARC output. If `None`, WARC output is disabled.
394    pub warc_output: Option<PathBuf>,
395    /// Named browser profile for persistent sessions (cookies, localStorage).
396    pub browser_profile: Option<String>,
397    /// Whether to save changes back to the browser profile on exit.
398    pub save_browser_profile: bool,
399    /// SSRF policy for outbound network requests. Default: deny private networks,
400    /// allow http/https only, max 5 redirects.
401    ///
402    /// Phase 1: `deny_private` and `max_redirects` are exposed to all language
403    /// bindings. `allowlist` is skipped (see `SsrfPolicy` fields) and will be
404    /// added in a follow-up when `HostMatcher`'s tagged-enum FFI form is decided.
405    #[serde(default = "SsrfPolicy::from_env")]
406    pub ssrf: SsrfPolicy,
407    /// Pluggable dispatch components: bypass provider, escalation strategy,
408    /// retry policy, WAF classifier, domain state, escalation budget, and
409    /// max_total_attempts.
410    ///
411    /// When `None`, the engine uses its built-in defaults (no bypass, `BrowserOnly`
412    /// strategy, `SimpleRetryPolicy`, built-in WAF classifier, no domain state,
413    /// unlimited budget, 10 total attempt cap).
414    ///
415    /// Rust-only advanced field. Generated language bindings do not expose
416    /// pluggable dispatch components; language clients use the built-in
417    /// dispatch defaults configured by the Rust engine.
418    ///
419    /// Not serializable — Rust callers construct this at runtime and skip it
420    /// in TOML/JSON configs.
421    #[serde(skip)]
422    #[cfg_attr(alef, alef(skip))]
423    pub dispatch: Option<DispatchProfile>,
424    /// Shared browser pool for reusing Chrome across requests (not serializable).
425    #[cfg(feature = "browser")]
426    #[serde(skip)]
427    #[cfg_attr(alef, alef(skip))]
428    pub browser_pool: Option<std::sync::Arc<crate::browser_pool::BrowserPool>>,
429    /// Optional [`crate::ProxyProvider`] for per-request proxy rotation on the
430    /// reqwest HTTP path. Takes precedence over the static [`ProxyConfig`] in
431    /// `proxy` when set. Not serializable — Rust callers inject at runtime.
432    #[serde(skip)]
433    #[cfg_attr(alef, alef(skip))]
434    pub proxy_provider: Option<std::sync::Arc<dyn crate::ProxyProvider>>,
435    /// Shared browser session pool for session affinity (not serializable).
436    /// When set alongside `session_affinity: true` in BrowserConfig, the pool
437    /// is used to cache Pages by (domain, proxy) so cookies and fingerprint
438    /// persist across requests.
439    #[cfg(feature = "browser")]
440    #[serde(skip)]
441    #[cfg_attr(alef, alef(skip))]
442    pub browser_session_pool: Option<std::sync::Arc<crate::browser_session_pool::BrowserSessionPool>>,
443}
444
445impl Default for CrawlConfig {
446    fn default() -> Self {
447        Self {
448            max_depth: None,
449            max_pages: None,
450            max_concurrent: None,
451            respect_robots_txt: false,
452            soft_http_errors: false,
453            user_agent: None,
454            stay_on_domain: false,
455            allow_subdomains: false,
456            include_paths: Vec::new(),
457            exclude_paths: Vec::new(),
458            custom_headers: HashMap::new(),
459            request_timeout: Duration::from_secs(30),
460            rate_limit_ms: None,
461            max_redirects: 10,
462            retry_count: 0,
463            retry_codes: Vec::new(),
464            cookies_enabled: false,
465            auth: None,
466            max_body_size: None,
467            remove_tags: Vec::new(),
468            content: ContentConfig::default(),
469            map_limit: None,
470            map_search: None,
471            download_assets: false,
472            asset_types: Vec::new(),
473            max_asset_size: None,
474            browser: BrowserConfig::default(),
475            proxy: None,
476            user_agents: Vec::new(),
477            capture_screenshot: false,
478            follow_document_urls: false,
479            document_url_depth: None,
480            download_documents: true,
481            document_max_size: Some(50 * 1024 * 1024), // 50 MB
482            document_mime_types: Vec::new(),
483            warc_output: None,
484            browser_profile: None,
485            save_browser_profile: false,
486            ssrf: SsrfPolicy::from_env(),
487            dispatch: None,
488            #[cfg(feature = "browser")]
489            browser_pool: None,
490            #[cfg(feature = "browser")]
491            browser_session_pool: None,
492            proxy_provider: None,
493        }
494    }
495}
496
497impl CrawlConfig {
498    /// Start a fluent builder for `CrawlConfig`. See [`crate::CrawlConfigBuilder`].
499    #[cfg_attr(alef, alef(skip))]
500    pub fn builder() -> crate::types::builder::CrawlConfigBuilder {
501        crate::types::builder::CrawlConfigBuilder::default()
502    }
503
504    /// Validate the configuration, returning an error if any values are invalid.
505    pub fn validate(&self) -> Result<(), crate::error::CrawlError> {
506        use crate::error::CrawlError;
507
508        if let Some(0) = self.max_concurrent {
509            return Err(CrawlError::InvalidConfig("max_concurrent must be > 0".into()));
510        }
511        if self.browser.wait == BrowserWait::Selector && self.browser.wait_selector.is_none() {
512            return Err(CrawlError::InvalidConfig(
513                "browser.wait_selector required when browser.wait is Selector".into(),
514            ));
515        }
516        if let Some(max_depth) = self.max_depth
517            && max_depth > 100
518        {
519            return Err(CrawlError::InvalidConfig(format!(
520                "max_depth must be <= 100 (got {max_depth})"
521            )));
522        }
523        if let Some(max_pages) = self.max_pages
524            && max_pages == 0
525        {
526            return Err(CrawlError::InvalidConfig("max_pages must be > 0".into()));
527        }
528        if self.max_redirects > 100 {
529            return Err(CrawlError::InvalidConfig("max_redirects must be <= 100".into()));
530        }
531        if let Some(max_body_size) = self.max_body_size
532            && max_body_size == 0
533        {
534            return Err(CrawlError::InvalidConfig("max_body_size must be > 0".into()));
535        }
536        if let Some(ref proxy) = self.proxy {
537            let parsed = url::Url::parse(&proxy.url)
538                .map_err(|e| CrawlError::InvalidConfig(format!("invalid proxy URL '{}': {e}", proxy.url)))?;
539            let scheme = parsed.scheme();
540            if !matches!(scheme, "http" | "https" | "socks5" | "socks5h") {
541                return Err(CrawlError::InvalidConfig(format!(
542                    "invalid proxy URL scheme '{scheme}' (expected http, https, socks5, or socks5h)"
543                )));
544            }
545        }
546        if let Some(ref auth) = self.auth {
547            match auth {
548                AuthConfig::Basic { username, .. } if username.is_empty() => {
549                    return Err(CrawlError::InvalidConfig(
550                        "auth.basic.username must not be empty".into(),
551                    ));
552                }
553                AuthConfig::Bearer { token } if token.is_empty() => {
554                    return Err(CrawlError::InvalidConfig("auth.bearer.token must not be empty".into()));
555                }
556                AuthConfig::Header { name, value } if name.is_empty() || value.is_empty() => {
557                    return Err(CrawlError::InvalidConfig(
558                        "auth.header.name and auth.header.value must not be empty".into(),
559                    ));
560                }
561                _ => {}
562            }
563        }
564        for pattern in &self.include_paths {
565            regex::Regex::new(pattern)
566                .map_err(|e| CrawlError::InvalidConfig(format!("invalid include_path regex '{pattern}': {e}")))?;
567        }
568        for pattern in &self.exclude_paths {
569            regex::Regex::new(pattern)
570                .map_err(|e| CrawlError::InvalidConfig(format!("invalid exclude_path regex '{pattern}': {e}")))?;
571        }
572        for &code in &self.retry_codes {
573            if !(100..=599).contains(&code) {
574                return Err(CrawlError::InvalidConfig(format!("invalid retry code: {code}")));
575            }
576        }
577        if self.request_timeout.is_zero() {
578            return Err(CrawlError::InvalidConfig("request_timeout must be > 0".into()));
579        }
580        if let Some(ref endpoint) = self.browser.endpoint
581            && !endpoint.starts_with("ws://")
582            && !endpoint.starts_with("wss://")
583        {
584            return Err(CrawlError::InvalidConfig(format!(
585                "browser.endpoint must start with ws:// or wss://, got: {endpoint:?}"
586            )));
587        }
588        if self.browser.backend == BrowserBackend::Native && self.browser.endpoint.is_some() {
589            return Err(CrawlError::InvalidConfig(
590                "browser.endpoint is only supported by the chromiumoxide backend".into(),
591            ));
592        }
593        Ok(())
594    }
595}
596
597#[cfg(test)]
598mod tests {
599    use super::*;
600
601    #[test]
602    fn validate_rejects_http_browser_endpoint() {
603        let config = CrawlConfig {
604            browser: BrowserConfig {
605                endpoint: Some("http://not-websocket:3000".into()),
606                ..Default::default()
607            },
608            ..Default::default()
609        };
610        let err = config.validate().unwrap_err();
611        let msg = err.to_string();
612        assert!(msg.contains("endpoint"), "error should mention 'endpoint', got: {msg}");
613    }
614
615    #[test]
616    fn validate_accepts_ws_endpoint() {
617        let config = CrawlConfig {
618            browser: BrowserConfig {
619                endpoint: Some("ws://localhost:9222".into()),
620                ..Default::default()
621            },
622            ..Default::default()
623        };
624        assert!(config.validate().is_ok());
625    }
626
627    #[test]
628    fn validate_accepts_wss_endpoint() {
629        let config = CrawlConfig {
630            browser: BrowserConfig {
631                endpoint: Some("wss://remote-browser.example.com/devtools".into()),
632                ..Default::default()
633            },
634            ..Default::default()
635        };
636        assert!(config.validate().is_ok());
637    }
638
639    #[test]
640    fn validate_accepts_no_endpoint() {
641        let config = CrawlConfig {
642            browser: BrowserConfig {
643                endpoint: None,
644                ..Default::default()
645            },
646            ..Default::default()
647        };
648        assert!(config.validate().is_ok());
649    }
650
651    #[test]
652    fn browser_backend_defaults_to_chromiumoxide() {
653        assert_eq!(BrowserConfig::default().backend, BrowserBackend::Chromiumoxide);
654    }
655
656    #[test]
657    fn validate_rejects_native_endpoint() {
658        let config = CrawlConfig {
659            browser: BrowserConfig {
660                backend: BrowserBackend::Native,
661                endpoint: Some("ws://localhost:9222".into()),
662                ..Default::default()
663            },
664            ..Default::default()
665        };
666        let err = config.validate().unwrap_err();
667        let msg = err.to_string();
668        assert!(msg.contains("chromiumoxide"), "unexpected error: {msg}");
669    }
670}
crawlberg/types/config.rs

crawlberg/types/
config.rs
