crawlex 1.0.6

use serde::{Deserialize, Serialize};
use std::time::Duration;

use crate::impersonate::Profile;
use crate::proxy::RotationStrategy;
use crate::wait_strategy::WaitStrategy;

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Config {
    pub max_concurrent_render: usize,
    pub max_concurrent_http: usize,
    pub max_depth: Option<u32>,
    pub same_host_only: bool,
    pub include_subdomains: bool,
    /// Primary recon target. When set, the frontier accepts URLs whose
    /// registrable domain equals this value (and their subdomains) and
    /// rejects the rest; out-of-scope URLs are still recorded in
    /// `asset_refs` so the operator sees every cross-site reference
    /// without fetching the content. Coexists with `same_host_only` /
    /// `include_subdomains` — when `target_domain` is set it wins.
    #[serde(default)]
    pub target_domain: Option<String>,
    /// Per-stage toggles for `crawlex intel <target>` (Fase B+). A
    /// `crawlex crawl --target <domain>` run with `with_infra=true`
    /// consults this to decide which passive-intel stages to execute.
    #[serde(default)]
    pub infra_intel: InfraIntelConfig,
    /// Index into `identity::profiles::catalog()` picking the persona
    /// the render pool should project. `None` keeps the historical
    /// Intel-on-Linux default wired through `IdentityBundle::from_chromium`;
    /// `Some(i)` routes through `IdentityBundle::from_persona(catalog()[i], …)`
    /// so the operator can switch the OS/GPU face crawlex shows without
    /// rebuilding.
    #[serde(default)]
    pub identity_preset: Option<u8>,
    pub respect_robots_txt: bool,
    /// Purposes the operator declares for this crawl. Honored against the
    /// `Content-Signal:` directive in each host's robots.txt (Cloudflare
    /// extension). Default is the full set; a site whose Content-Signal
    /// denies every declared purpose aborts the run for that host before
    /// any non-robots fetch occurs.
    #[serde(default = "default_crawl_purposes")]
    pub crawl_purposes: Vec<crate::robots::Purpose>,
    pub user_agent_profile: Profile,
    pub chrome_path: Option<String>,
    pub chrome_flags: Vec<String>,
    pub block_resources: Vec<String>,
    /// Typed counterpart to `block_resources`: each variant expands to a
    /// canonical wildcard set fed into Chrome's `Network.setBlockedURLs`
    /// before navigation, so blocked types never hit the wire. Mirrors
    /// Cloudflare's accepted set (`image`, `media`, `font`, `stylesheet`).
    /// Auto-disabled (with a warn-level log) when the job requests a
    /// screenshot, so visual fidelity is preserved. Empty default preserves
    /// today's bandwidth behavior.
    #[serde(default)]
    pub reject_resource_types: Vec<RejectResourceType>,
    pub wait_strategy: WaitStrategy,
    pub rate_per_host_rps: Option<f64>,
    pub retry_max: u32,
    pub retry_backoff: Duration,
    pub queue_backend: QueueBackend,
    pub storage_backend: StorageBackend,
    pub output: OutputConfig,
    pub proxy: ProxyConfig,
    pub locale: Option<String>,
    pub timezone: Option<String>,
    pub metrics_prometheus_port: Option<u16>,
    pub hook_scripts: Vec<String>,
    pub discovery_filter_regex: Option<String>,
    /// When true, only follow URLs classified as Page/Document/Api.
    pub follow_pages_only: bool,
    /// Seed the frontier with crt.sh-discovered subdomains of each seed host.
    pub crtsh_enabled: bool,
    /// Expand robots.txt Disallow/Allow paths into seed URLs.
    pub robots_paths_enabled: bool,
    /// Probe /.well-known/* endpoints and harvest URLs from their bodies.
    pub well_known_enabled: bool,
    /// Probe PWA manifest and service worker paths; parse manifest for URLs.
    pub pwa_enabled: bool,
    /// Query the Internet Archive CDX API to seed historical URLs.
    pub wayback_enabled: bool,
    /// Fetch favicon.ico and compute its Shodan-style mmh3 hash.
    pub favicon_enabled: bool,
    /// Resolve DNS records per host; seed related_hosts as new roots.
    pub dns_enabled: bool,
    /// Opt-in: measure DNS/TCP/TLS/TTFB/download on the HTTP path and store.
    pub collect_net_timings: bool,
    /// Opt-in: run the Web Vitals JS after render and store (CLS, LCP, etc.).
    pub collect_web_vitals: bool,
    /// Opt-in: extract peer TLS certificate (CN, SANs, fingerprint) and seed
    /// SANs as candidate subdomains.
    pub collect_peer_cert: bool,
    /// Opt-in: RDAP lookup per registrable domain.
    pub rdap_enabled: bool,
    /// Persist cookies per registrable domain across requests.
    pub cookies_enabled: bool,
    /// Render-session reuse boundary. Controls how aggressively browser
    /// state is shared between rendered pages.
    #[serde(default)]
    pub render_session_scope: RenderSessionScope,
    /// Follow 3xx redirects inline.
    pub follow_redirects: bool,
    /// Max redirects before returning the redirect response as-is.
    pub max_redirects: u8,
    /// Action script executed on every rendered page after the wait strategy.
    /// Only present when the render backend is compiled in; mini builds
    /// skip the field entirely.
    #[cfg(feature = "cdp-backend")]
    #[serde(skip)]
    pub actions: Option<Vec<crate::render::actions::Action>>,
    /// Declarative ScriptSpec (v1) run on every rendered page in place of
    /// `actions`. When both are set, `script_spec` wins — the CLI wires
    /// `conflicts_with` so operators can't accidentally ship both. The
    /// runner slots in between wait-strategy settle and the Lua
    /// `on_after_load` hook.
    #[cfg(feature = "cdp-backend")]
    #[serde(skip)]
    pub script_spec: Option<crate::script::ScriptSpec>,
    /// When true, the render pool runs `<chrome> --version` once at startup
    /// and rewrites `user_agent_profile` to the closest known profile. This
    /// avoids the "spoof says Chrome/131, render is Chrome/149" mismatch.
    pub profile_autodetect: bool,
    /// When set, overrides the UA string both in spoof request headers and
    /// in the Chrome `--user-agent` launch flag. Takes precedence over the
    /// profile's canned UA.
    pub user_agent_override: Option<String>,
    /// When true, and no system Chrome is found on PATH (and no `chrome_path`
    /// was set), auto-download a pinned Chromium-for-Testing build into
    /// `$XDG_CACHE_HOME/crawlex/chromium/` via `the CDP fetcher`.
    pub auto_fetch_chromium: bool,
    /// Per-verb action policy applied to every ScriptSpec / `--actions-file`
    /// execution. Default is `permissive` (all verbs allowed) so legacy
    /// scripts authored by the operator keep working; callers running
    /// untrusted scripts should swap in `ActionPolicy::strict()` or a
    /// JSON-loaded variant via `--action-policy <path>`.
    #[serde(default)]
    pub action_policy: crate::policy::ActionPolicy,
    /// How the crawler should treat detected captcha/challenge flows.
    /// `avoidance` keeps the product strictly prevention-only; `solver_ready`
    /// enriches challenge telemetry with widget metadata so a future solver
    /// integration can slot in without changing the capture contract.
    #[serde(default)]
    pub challenge_mode: ChallengeMode,
    /// Inject the SPA/PWA JS observer (history + fetch + XHR wrappers)
    /// on every rendered page and emit `snapshot.runtime_routes`
    /// artifacts post-settle. Cheap; on by default. Disabling also
    /// stops routes from feeding the crawler frontier.
    #[serde(default = "default_true")]
    pub collect_runtime_routes: bool,
    /// Emit `snapshot.network_endpoints` artifacts post-settle and
    /// forward observed endpoints to the frontier. Shares the
    /// observer bundle with `collect_runtime_routes` — disabling
    /// only suppresses the artifact/frontier wire.
    #[serde(default = "default_true")]
    pub collect_network_endpoints: bool,
    /// Enumerate IndexedDB databases/object stores via CDP post-settle
    /// and emit `snapshot.indexeddb`. Heavy; off by default.
    #[serde(default)]
    pub collect_indexeddb: bool,
    /// Enumerate Cache Storage caches/keys via CDP post-settle and
    /// emit `snapshot.cache_storage`. Heavy; off by default.
    #[serde(default)]
    pub collect_cache_storage: bool,
    /// Fetch the Web App Manifest (when discovered via `<link rel=manifest>`)
    /// and emit `snapshot.manifest`. Default on, cheap (one HTTP fetch).
    #[serde(default = "default_true")]
    pub collect_manifest: bool,
    /// Emit `snapshot.service_workers` post-settle from the
    /// registrations already captured in origin state. Default on.
    #[serde(default = "default_true")]
    pub collect_service_workers: bool,
    /// Max Chrome instances kept alive simultaneously. When exceeded,
    /// the LRU browser is evicted — its tabs + contexts torn down. Keyed
    /// on `(proxy_url | "")` in the render pool.
    #[serde(default = "default_max_browsers")]
    pub max_browsers: usize,
    /// Max idle + in-flight pages per BrowserContext the render pool
    /// keeps reusable. Higher = more parallel tabs per session, lower
    /// memory reuse.
    #[serde(default = "default_max_pages_per_context")]
    pub max_pages_per_context: usize,
    /// Inflight budgets enforced per-host/origin/proxy/session before a
    /// render job starts. Jobs that exceed a budget are re-queued with
    /// a small delay.
    #[serde(default)]
    pub render_budgets: crate::scheduler::BudgetLimits,
    /// Render session time-to-live: the cleanup task drops BrowserContexts
    /// that haven't been touched in this many seconds. Default 3600 (1h).
    #[serde(default = "default_session_ttl_secs")]
    pub session_ttl_secs: u64,
    /// When `true`, a session transitioning to `Blocked` is evicted
    /// immediately (without waiting for the TTL cleanup sweep). Default on
    /// so hostile sites don't tie up a BrowserContext until TTL fires.
    #[serde(default = "default_true")]
    pub drop_session_on_block: bool,
    /// When `true`, policy can automatically demote `render_session_scope`
    /// based on page signals (login pages → Origin, hard blocks → Url).
    /// Turn off to pin the scope the CLI/config declared. Default off —
    /// operators opt in.
    #[serde(default)]
    pub session_scope_auto: bool,
    /// Human motion engine preset. Trades throughput for trajectory
    /// realism. `fast` keeps the legacy ~15 rps baseline (linear path,
    /// minimal delay); `balanced` (default) wires WindMouse + Fitts + OU
    /// jitter at ~8 rps; `human` and `paranoid` favour stealth over
    /// speed. See `crate::render::motion::MotionProfile` for the params.
    ///
    /// Gated on `cdp-backend` because the mini build ships no browser
    /// primitives to feed — a mini operator tuning stealth params would
    /// be a no-op.
    #[cfg(feature = "cdp-backend")]
    #[serde(default)]
    pub motion_profile: crate::render::motion::MotionProfile,
    /// Pre-navigation warm-up hit. Cloudflare scores the *first* request to
    /// an origin more harshly because `__cf_bm`/`cf_clearance` cookies
    /// aren't bound yet — visiting a cheap URL first lets the cookie store
    /// catch up before the scored request. Opt-in via config.
    #[serde(default)]
    pub warmup: WarmupPolicy,
    /// Post-settle "reading" dwell. Real humans linger on a page proportional
    /// to its text length — reCAPTCHA v3 / DataDome flag instant post-load
    /// extraction as bot-like. When `Some(cfg)` with `enabled=true`, the
    /// render pool sleeps for `(words / wpm) * 60_000 + jitter` ms after the
    /// wait strategy settles. Off by default so existing throughput stays put.
    #[serde(default)]
    pub reading_dwell: Option<ReadingDwellConfig>,
    /// Limits for spoofed HTTP fetches. These caps protect high-concurrency
    /// crawls from slow bodies and compressed bombs.
    #[serde(default)]
    pub http_limits: HttpLimits,
    /// Content-addressed body store. When enabled, full response/rendered
    /// bodies live in blobs and `pages` stores hashes + paths. Legacy inline
    /// columns stay opt-in for older direct SQL consumers.
    #[serde(default)]
    pub content_store: ContentStoreConfig,
    /// Lightweight cache freshness validation. When enabled, Crawlex can skip
    /// full processing for URLs whose cached page metadata still matches the
    /// current response validators or `<head>` fingerprint.
    #[serde(default)]
    pub cache_validation: CacheValidationConfig,
    /// Discovery-only mode. Fetches/serializes enough HTML to extract links,
    /// but skips heavier page processing such as rendered-page persistence,
    /// tech fingerprinting, artifacts, and post-processing.
    #[serde(default)]
    pub prefetch: bool,
    /// Optional best-first scoring for newly discovered URLs.
    #[serde(default)]
    pub crawl_scoring: CrawlScoringConfig,
    /// Optional last-resort fetch adapter. When set, Crawlex executes the
    /// configured command with a JSON request on stdin and expects a JSON
    /// response containing HTML/body data on stdout.
    #[serde(default)]
    pub fallback_fetch: Option<FallbackFetchConfig>,
    /// Connect to an already-running Chrome/Chromium CDP endpoint instead
    /// of launching a local browser. Example: `http://127.0.0.1:9222`.
    #[serde(default)]
    pub external_cdp_url: Option<String>,
    /// GPU launch posture for managed Chrome instances.
    #[serde(default)]
    pub gpu_policy: GpuPolicy,
    /// Optional DOM cleanup/capture transformations before serializing the
    /// rendered document.
    #[serde(default)]
    pub dom_capture: DomCaptureConfig,
    /// Operator-level render-path switch. `Auto` (default) keeps the
    /// historical behaviour: impersonate first, escalate to render via
    /// the policy engine when needed. `Always` forces every seeded job
    /// onto `FetchMethod::Render` and skips impersonation. `Never`
    /// pins every job to the impersonate path, refuses policy
    /// escalation to render, and prevents the render pool from being
    /// instantiated.
    #[serde(default)]
    pub render_mode: RenderMode,
    /// Slice 29 — vendor-neutral browser provider selector. `Stock`
    /// (default) preserves the historical local-launch Chromium flow.
    /// `Cdp` connects to an explicitly configured external CDP endpoint
    /// (`external_cdp_url` or the `CRAWLEX_EXTERNAL_CDP_URL` env var).
    /// `Auto` prefers the configured external endpoint when present and
    /// falls back to the stock local Chromium; it does NOT probe for
    /// local CDP endpoints.
    #[serde(default)]
    pub browser_provider: BrowserProvider,
    /// Slice 34 — what to do when the calibration probe surfaces a
    /// critical fingerprint mismatch the shim cannot reconcile.
    /// `Adapt` (default) records mismatches, emits a warning event,
    /// and continues with calibrated values where possible. `Strict`
    /// aborts the render before target navigation when at least one
    /// critical mismatch is unreconcilable (e.g. WebRTC IP leak past
    /// the proxy, mismatched browser engine, storage backing).
    #[serde(default)]
    pub mismatch_policy: MismatchPolicy,
    /// Slice 35 — how an external CDP session uses backend browser state.
    /// `Isolated` (default) creates a crawlex-owned `BrowserContext` per
    /// session so cookies/localStorage/cache/profile state are
    /// segregated. `Persistent` reuses the endpoint's default browser
    /// context, inheriting whatever cookies, localStorage, cache, and
    /// signed-in profile the backend already has. Ignored when
    /// `external_cdp_url` is unset.
    #[serde(default)]
    pub external_cdp_session_mode: ExternalCdpSessionMode,
    /// Slice 36 — opt-in provider fallback. Disabled by default. When
    /// `enabled = true` and `order` is non-empty, the render pool may
    /// retry preflight against the listed providers after the primary
    /// (`browser_provider`) fails. Fingerprint mismatch policy and
    /// session mode decisions ride along unchanged — fallback never
    /// silently relaxes them.
    #[serde(default)]
    pub provider_fallback: ProviderFallbackConfig,
    /// Slice 7 — wall-clock budget for the whole crawl. When `Some(n)`
    /// a watchdog auto-cancels the run after `n` seconds, writes
    /// `TerminalReason::CancelledDueToTimeout` to the `crawl_stats` row,
    /// and stamps `result_expires_at` on the per-URL pages so the
    /// reaper picks them up. `None` (default) keeps the historical
    /// behaviour — no watchdog, no auto-cancel.
    #[serde(default)]
    pub job_max_runtime_secs: Option<u64>,
    /// Slice 7 — TTL applied to a run's artifacts. When `Some(n)`,
    /// finishing the run sets `result_expires_at = now + n` on every
    /// `pages` and `crawl_stats` row written by it. The reaper task
    /// (`reap_expired_blocking`) GCs those rows after the window
    /// elapses. `None` (default) disables retention — rows live
    /// forever, matching legacy behaviour.
    #[serde(default)]
    pub result_retention_secs: Option<u64>,
    /// Slice 7 — hard cap on the number of per-URL pages successfully
    /// recorded in a single run. When the cap is hit the crawler stops
    /// dispatching new jobs and writes
    /// `TerminalReason::CancelledDueToLimits` to the `crawl_stats` row.
    /// `None` (default) preserves today's unbounded behaviour.
    #[serde(default)]
    pub max_pages: Option<u64>,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ContentStoreConfig {
    #[serde(default = "default_true")]
    pub enabled: bool,
    #[serde(default)]
    pub root: Option<String>,
    #[serde(default)]
    pub inline_legacy_columns: bool,
}

impl Default for ContentStoreConfig {
    fn default() -> Self {
        Self {
            enabled: true,
            root: None,
            inline_legacy_columns: false,
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct CacheValidationConfig {
    #[serde(default)]
    pub enabled: bool,
    /// If set, a cache row younger than this is accepted without a network
    /// validation probe. Leave unset to always compare validators/fingerprint.
    #[serde(default)]
    pub max_age_secs: Option<u64>,
    /// Unix-timestamp threshold: a stored `Last-Modified` at-or-before this
    /// value is treated as fresh and the page is skipped without a network
    /// probe. Leave unset to disable.
    #[serde(default)]
    pub modified_since: Option<u64>,
}

impl Default for CacheValidationConfig {
    fn default() -> Self {
        Self {
            enabled: false,
            max_age_secs: None,
            modified_since: None,
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct CrawlScoringConfig {
    #[serde(default)]
    pub enabled: bool,
    #[serde(default)]
    pub keywords: Vec<String>,
    #[serde(default = "default_score_same_host_bonus")]
    pub same_host_bonus: i32,
    #[serde(default = "default_score_keyword_bonus")]
    pub keyword_bonus: i32,
    #[serde(default = "default_score_depth_penalty")]
    pub depth_penalty: i32,
    #[serde(default = "default_score_path_depth_penalty")]
    pub path_depth_penalty: i32,
}

fn default_score_same_host_bonus() -> i32 {
    5
}

fn default_score_keyword_bonus() -> i32 {
    8
}

fn default_score_depth_penalty() -> i32 {
    2
}

fn default_score_path_depth_penalty() -> i32 {
    1
}

impl Default for CrawlScoringConfig {
    fn default() -> Self {
        Self {
            enabled: false,
            keywords: Vec::new(),
            same_host_bonus: default_score_same_host_bonus(),
            keyword_bonus: default_score_keyword_bonus(),
            depth_penalty: default_score_depth_penalty(),
            path_depth_penalty: default_score_path_depth_penalty(),
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct FallbackFetchConfig {
    pub command: Vec<String>,
    #[serde(default = "default_fallback_timeout_ms")]
    pub timeout_ms: u64,
    #[serde(default = "default_fallback_max_output_bytes")]
    pub max_output_bytes: u64,
}

fn default_fallback_timeout_ms() -> u64 {
    60_000
}

fn default_fallback_max_output_bytes() -> u64 {
    32 * 1024 * 1024
}

#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum GpuPolicy {
    #[default]
    Compat,
    Stealth,
}

#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct DomCaptureConfig {
    #[serde(default)]
    pub flatten_shadow_dom: bool,
    #[serde(default)]
    pub remove_overlays: bool,
    #[serde(default)]
    pub remove_consent_popups: bool,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct HttpLimits {
    #[serde(default = "default_http_request_timeout")]
    pub request_timeout: Duration,
    #[serde(default = "default_max_encoded_body_bytes")]
    pub max_encoded_body_bytes: Option<usize>,
    #[serde(default = "default_max_decoded_body_bytes")]
    pub max_decoded_body_bytes: Option<usize>,
    #[serde(default = "default_max_decompression_ratio")]
    pub max_decompression_ratio: usize,
    #[serde(default)]
    pub store_truncated_bodies: bool,
}

fn default_http_request_timeout() -> Duration {
    Duration::from_secs(30)
}

fn default_max_encoded_body_bytes() -> Option<usize> {
    Some(16 * 1024 * 1024)
}

fn default_max_decoded_body_bytes() -> Option<usize> {
    Some(32 * 1024 * 1024)
}

fn default_max_decompression_ratio() -> usize {
    100
}

impl Default for HttpLimits {
    fn default() -> Self {
        Self {
            request_timeout: default_http_request_timeout(),
            max_encoded_body_bytes: default_max_encoded_body_bytes(),
            max_decoded_body_bytes: default_max_decoded_body_bytes(),
            max_decompression_ratio: default_max_decompression_ratio(),
            store_truncated_bodies: false,
        }
    }
}

/// Reading dwell parameters. Gates a simulated "user reads the page"
/// delay between `settle_after_actions` and DOM serialization. WPM
/// defaults track typical adult prose reading (~250 wpm); jitter σ
/// keeps successive requests non-identical so fingerprint models can't
/// pin the exact cadence.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ReadingDwellConfig {
    #[serde(default)]
    pub enabled: bool,
    #[serde(default = "default_reading_dwell_wpm")]
    pub wpm: u32,
    #[serde(default = "default_reading_dwell_jitter_ms")]
    pub jitter_ms: u64,
    #[serde(default = "default_reading_dwell_min_ms")]
    pub min_ms: u64,
    #[serde(default = "default_reading_dwell_max_ms")]
    pub max_ms: u64,
}

fn default_reading_dwell_wpm() -> u32 {
    250
}
fn default_reading_dwell_jitter_ms() -> u64 {
    40
}
fn default_reading_dwell_min_ms() -> u64 {
    500
}
fn default_reading_dwell_max_ms() -> u64 {
    10_000
}

impl Default for ReadingDwellConfig {
    fn default() -> Self {
        Self {
            enabled: false,
            wpm: default_reading_dwell_wpm(),
            jitter_ms: default_reading_dwell_jitter_ms(),
            min_ms: default_reading_dwell_min_ms(),
            max_ms: default_reading_dwell_max_ms(),
        }
    }
}

/// Optional warm-up navigation performed before the crawl target.
///
/// The render core substitutes `{origin}` and `{host}` into `url_template`
/// against the target URL, navigates there, sleeps `dwell_ms`, then
/// continues to the real target. When the rendered template equals the
/// target we skip the warm-up (no point paying for a duplicate request).
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct WarmupPolicy {
    /// Master switch. Default `false` so the behaviour is strictly opt-in.
    #[serde(default)]
    pub enabled: bool,
    /// URL template with `{origin}` / `{host}` placeholders. Typical values:
    /// `"{origin}"` (origin root) or `"{origin}/search"`.
    #[serde(default = "default_warmup_template")]
    pub url_template: String,
    /// How long to idle on the warm-up URL before hitting the target. 1.5s
    /// is enough for Cloudflare's `__cf_bm` cookie to appear on the session
    /// without materially slowing a crawl.
    #[serde(default = "default_warmup_dwell_ms")]
    pub dwell_ms: u64,
}

fn default_warmup_template() -> String {
    "{origin}".to_string()
}

fn default_warmup_dwell_ms() -> u64 {
    1500
}

impl Default for WarmupPolicy {
    fn default() -> Self {
        Self {
            enabled: false,
            url_template: default_warmup_template(),
            dwell_ms: default_warmup_dwell_ms(),
        }
    }
}

impl WarmupPolicy {
    /// Substitute `{origin}` and `{host}` placeholders in `url_template`
    /// against `target`. Pure string op — no I/O, no URL parsing of the
    /// rendered output, so the caller decides what to do with garbage.
    ///
    /// `origin` is `scheme://host[:port]` (no trailing slash); `host` is
    /// the bare hostname. An empty `url_template` yields an empty string —
    /// the caller treats that as "skip warm-up".
    pub fn render_template(&self, target: &url::Url) -> String {
        let host = target.host_str().unwrap_or("");
        // Build `scheme://host[:port]` manually — `Url::origin()` stringifies
        // to `scheme://host:port` with the default-port elided which is what
        // we want, but it returns an `Origin` enum so we'd have to match on
        // `Tuple`. Hand-rolling is simpler and keeps the behaviour stable
        // across `url` crate versions.
        let mut origin = String::new();
        origin.push_str(target.scheme());
        origin.push_str("://");
        origin.push_str(host);
        if let Some(port) = target.port() {
            origin.push(':');
            origin.push_str(&port.to_string());
        }
        self.url_template
            .replace("{origin}", &origin)
            .replace("{host}", host)
    }
}

fn default_true() -> bool {
    true
}

fn default_max_browsers() -> usize {
    4
}

fn default_max_pages_per_context() -> usize {
    4
}

fn default_session_ttl_secs() -> u64 {
    crate::identity::DEFAULT_SESSION_TTL_SECS
}

#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct OutputConfig {
    pub html_dir: Option<String>,
    pub graph_path: Option<String>,
    pub metadata_path: Option<String>,
    pub screenshot_dir: Option<String>,
    pub screenshot: bool,
    /// Capture mode string: `viewport`, `fullpage`, or `element:<selector>`.
    /// `None` or an unrecognised value falls back to `fullpage` (the legacy
    /// default). Parsed by `parse_screenshot_mode`.
    #[serde(default)]
    pub screenshot_mode: Option<String>,
}

#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ProxyConfig {
    pub proxies: Vec<String>,
    pub proxy_file: Option<String>,
    pub strategy: RotationStrategy,
    pub sticky_per_host: bool,
    pub health_check_interval: Option<Duration>,
}

/// Normalize a RedDB connection target for Crawlex persistence.
///
/// Bare local paths become `file://...` embedded URIs. Documented RedDB
/// transports (`red://`, `reds://`, `red+wss://`, plus HTTP/gRPC and
/// embedded URI forms) are preserved verbatim so Crawlex can target local
/// embedded stores or remote RedDB services without SQLite fallback.
pub fn normalize_reddb_uri(target: impl AsRef<str>) -> String {
    let raw = target.as_ref().trim();
    if raw.is_empty() {
        return "file://crawlex.redb".to_string();
    }
    let lower = raw.to_ascii_lowercase();
    if lower.starts_with("red://")
        || lower.starts_with("reds://")
        || lower.starts_with("red+wss://")
        || lower.starts_with("grpc://")
        || lower.starts_with("grpcs://")
        || lower.starts_with("http://")
        || lower.starts_with("https://")
        || lower.starts_with("file://")
        || lower.starts_with("memory://")
    {
        raw.to_string()
    } else {
        format!("file://{raw}")
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum QueueBackend {
    InMemory,
    Reddb { uri: String },
    Sqlite { path: String },
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum StorageBackend {
    Memory,
    Reddb { uri: String },
    Sqlite { path: String },
    Filesystem { root: String },
}

/// Operator-level switch that decides whether the render (Chrome/CDP)
/// path is consulted for a job. Coexists with `FetchMethod` on the
/// individual job: `RenderMode::Always` upgrades every seeded job to
/// `FetchMethod::Render` regardless of what the operator passed; `Never`
/// keeps every job on the impersonate path and short-circuits any
/// policy escalation to render; `Auto` (default) preserves today's
/// behaviour where the policy engine may escalate `FetchMethod::Auto`
/// jobs to render after observing the response.
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum RenderMode {
    #[default]
    Auto,
    Always,
    Never,
}

/// Slice 29 — vendor-neutral selector for which browser implementation
/// the render path drives. Kept independent of `RenderMode` (which
/// decides *whether* to render): `BrowserProvider` decides *which*
/// browser. The default is `Stock` — Crawlex launches its bundled /
/// configured Chromium locally and preserves the historical behaviour.
/// `Cdp` requires an explicit external endpoint; the render pool will
/// refuse to fall back to a local browser. `Auto` prefers the
/// configured external endpoint when one is present and otherwise
/// behaves like `Stock`. Critically, `Auto` does NOT auto-discover
/// local CDP endpoints — operators must opt-in explicitly.
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq, Hash)]
#[serde(rename_all = "snake_case")]
pub enum BrowserProvider {
    #[default]
    Stock,
    Cdp,
    Auto,
}

/// Slice 34 — adapt vs strict policy for unreconciled critical
/// calibration mismatches. Defined at config scope so the crate
/// builds without the `cdp-backend` feature; the calibration module
/// re-exports this type for ergonomic callers.
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum MismatchPolicy {
    #[default]
    Adapt,
    Strict,
}

/// Slice 35 — external CDP session state mode. Defaults to `Isolated`:
/// crawlex creates and disposes its own `BrowserContext` so backend
/// browser state never leaks across sessions. `Persistent` opts in to
/// reusing the endpoint's default context (cookies, localStorage,
/// cache, signed-in profile) — explicit operator choice only.
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ExternalCdpSessionMode {
    #[default]
    Isolated,
    Persistent,
}

/// Slice 36 — explicit provider fallback configuration. `enabled` and a
/// non-empty `order` together arm fallback; otherwise crawlex never
/// switches providers. `order` is a vendor-neutral list of
/// `BrowserProvider` names tried sequentially after the primary fails.
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ProviderFallbackConfig {
    #[serde(default)]
    pub enabled: bool,
    #[serde(default)]
    pub order: Vec<BrowserProvider>,
}

impl ProviderFallbackConfig {
    /// True when both the toggle is on and a non-empty order is set.
    pub fn is_active(&self) -> bool {
        self.enabled && !self.order.is_empty()
    }

    /// Sequence of providers to try after `primary` fails. The primary
    /// itself is skipped (no point retrying the same one), duplicates
    /// are dropped (first occurrence wins), and `Auto` is dropped
    /// because `Auto` itself is a meta-selector — only concrete
    /// providers belong in a fallback chain.
    pub fn chain_after(&self, primary: BrowserProvider) -> Vec<BrowserProvider> {
        if !self.is_active() {
            return Vec::new();
        }
        let mut seen = std::collections::HashSet::new();
        let mut out = Vec::new();
        for p in &self.order {
            if *p == primary || *p == BrowserProvider::Auto {
                continue;
            }
            if seen.insert(*p) {
                out.push(*p);
            }
        }
        out
    }
}

impl ExternalCdpSessionMode {
    pub fn as_str(self) -> &'static str {
        match self {
            ExternalCdpSessionMode::Isolated => "isolated",
            ExternalCdpSessionMode::Persistent => "persistent",
        }
    }

    pub fn parse(raw: &str) -> Option<Self> {
        match raw.trim().to_ascii_lowercase().as_str() {
            "isolated" => Some(ExternalCdpSessionMode::Isolated),
            "persistent" => Some(ExternalCdpSessionMode::Persistent),
            _ => None,
        }
    }

    pub fn is_persistent(self) -> bool {
        matches!(self, ExternalCdpSessionMode::Persistent)
    }
}

impl MismatchPolicy {
    pub fn as_str(self) -> &'static str {
        match self {
            MismatchPolicy::Adapt => "adapt",
            MismatchPolicy::Strict => "strict",
        }
    }

    pub fn parse(raw: &str) -> Option<Self> {
        match raw.trim().to_ascii_lowercase().as_str() {
            "adapt" => Some(MismatchPolicy::Adapt),
            "strict" => Some(MismatchPolicy::Strict),
            _ => None,
        }
    }
}

impl BrowserProvider {
    pub fn as_str(self) -> &'static str {
        match self {
            BrowserProvider::Stock => "stock",
            BrowserProvider::Cdp => "cdp",
            BrowserProvider::Auto => "auto",
        }
    }

    pub fn parse(raw: &str) -> Option<Self> {
        match raw.trim().to_ascii_lowercase().as_str() {
            "stock" => Some(BrowserProvider::Stock),
            "cdp" => Some(BrowserProvider::Cdp),
            "auto" => Some(BrowserProvider::Auto),
            _ => None,
        }
    }
}

#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum RenderSessionScope {
    #[default]
    RegistrableDomain,
    Host,
    Origin,
    Url,
}

#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ChallengeMode {
    Avoidance,
    #[default]
    SolverReady,
}

/// Granular toggles for the infrastructure-intel orchestrator (Fase B+).
/// Every flag defaults to ON so `crawlex intel <target>` produces a full
/// recon report by default; operators strip stages down when they
/// already have data or hit rate limits.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct InfraIntelConfig {
    /// Enumerate subdomains via crt.sh + CertSpotter + HackerTarget +
    /// passive DNS aggregators. Produces rows in `domains`.
    pub subdomains: bool,
    /// Full DNS record set per domain: A/AAAA/CNAME/MX/TXT/NS/SOA/CAA
    /// + wildcard-detection probe. Produces rows in `dns_records`.
    pub dns: bool,
    /// WHOIS / RDAP for domain + parent TLD. Produces rows in
    /// `whois_records`.
    pub whois: bool,
    /// TLS handshake + X.509 deep parse (SAN, wildcard, sig algo,
    /// pubkey class, self-signed, validity window). Produces rows in
    /// `certs` + `cert_seen_on`.
    pub cert: bool,
    /// Certificate Transparency log query (crt.sh JSON API). Pulls
    /// ALL historical certs issued for target + its subdomains.
    pub ct_logs: bool,
    /// HTTP server fingerprint: ServerType / Framework / WAF / CDN /
    /// CloudProvider enums derived from headers/cookies/error pages.
    pub server_fp: bool,
    /// Reverse IP lookups (RDAP IP for ASN + HackerTarget reverseip)
    /// and cloud/CDN IP-range classification. Produces updates on
    /// `ip_addresses`.
    pub reverse_ip: bool,
    /// Fase D: active network probes — ICMP ping, traceroute, port
    /// scan. Requires CAP_NET_RAW/root for raw sockets; falls back to
    /// TCP-connect when unprivileged. Off by default because it is
    /// detectable.
    pub network_probe: bool,
}

impl Default for InfraIntelConfig {
    fn default() -> Self {
        Self {
            subdomains: true,
            dns: true,
            whois: true,
            cert: true,
            ct_logs: true,
            server_fp: true,
            reverse_ip: true,
            network_probe: false,
        }
    }
}

fn default_crawl_purposes() -> Vec<crate::robots::Purpose> {
    crate::robots::Purpose::all().to_vec()
}

impl Default for Config {
    fn default() -> Self {
        Self {
            max_concurrent_render: 0,
            max_concurrent_http: 500,
            max_depth: Some(5),
            same_host_only: false,
            include_subdomains: true,
            target_domain: None,
            infra_intel: InfraIntelConfig::default(),
            identity_preset: None,
            respect_robots_txt: true,
            crawl_purposes: default_crawl_purposes(),
            user_agent_profile: Profile::Chrome131Stable,
            chrome_path: None,
            chrome_flags: Vec::new(),
            block_resources: Vec::new(),
            reject_resource_types: Vec::new(),
            wait_strategy: WaitStrategy::NetworkIdle { idle_ms: 500 },
            rate_per_host_rps: None,
            retry_max: 3,
            retry_backoff: Duration::from_millis(500),
            queue_backend: QueueBackend::Reddb {
                uri: "file://crawlex-queue.rdb".into(),
            },
            storage_backend: StorageBackend::Reddb {
                uri: "file://crawlex-storage.rdb".into(),
            },
            output: OutputConfig::default(),
            proxy: ProxyConfig::default(),
            locale: None,
            timezone: None,
            metrics_prometheus_port: None,
            hook_scripts: Vec::new(),
            discovery_filter_regex: None,
            follow_pages_only: true,
            crtsh_enabled: false,
            robots_paths_enabled: true,
            well_known_enabled: true,
            pwa_enabled: true,
            wayback_enabled: false,
            favicon_enabled: true,
            dns_enabled: false,
            collect_net_timings: false,
            collect_web_vitals: false,
            collect_peer_cert: false,
            rdap_enabled: false,
            cookies_enabled: true,
            render_session_scope: RenderSessionScope::RegistrableDomain,
            follow_redirects: true,
            max_redirects: 10,
            #[cfg(feature = "cdp-backend")]
            actions: None,
            #[cfg(feature = "cdp-backend")]
            script_spec: None,
            profile_autodetect: true,
            user_agent_override: None,
            auto_fetch_chromium: true,
            action_policy: crate::policy::ActionPolicy::permissive(),
            challenge_mode: ChallengeMode::SolverReady,
            collect_runtime_routes: true,
            collect_network_endpoints: true,
            collect_indexeddb: false,
            collect_cache_storage: false,
            collect_manifest: true,
            collect_service_workers: true,
            max_browsers: default_max_browsers(),
            max_pages_per_context: default_max_pages_per_context(),
            render_budgets: crate::scheduler::BudgetLimits::default(),
            session_ttl_secs: default_session_ttl_secs(),
            drop_session_on_block: true,
            session_scope_auto: false,
            #[cfg(feature = "cdp-backend")]
            motion_profile: crate::render::motion::MotionProfile::default(),
            warmup: WarmupPolicy::default(),
            reading_dwell: None,
            http_limits: HttpLimits::default(),
            content_store: ContentStoreConfig::default(),
            cache_validation: CacheValidationConfig::default(),
            prefetch: false,
            crawl_scoring: CrawlScoringConfig::default(),
            fallback_fetch: None,
            external_cdp_url: None,
            gpu_policy: GpuPolicy::default(),
            dom_capture: DomCaptureConfig::default(),
            render_mode: RenderMode::default(),
            browser_provider: BrowserProvider::default(),
            mismatch_policy: MismatchPolicy::default(),
            external_cdp_session_mode: ExternalCdpSessionMode::default(),
            provider_fallback: ProviderFallbackConfig::default(),
            job_max_runtime_secs: None,
            result_retention_secs: None,
            max_pages: None,
        }
    }
}

impl Config {
    pub fn builder() -> ConfigBuilder {
        ConfigBuilder::default()
    }
}

/// CDP-level reject categories. Mirrors the canonical Cloudflare set so
/// operators can copy-paste a single value across edge config and crawler
/// config. Each variant expands to a wildcard pattern list via
/// [`RejectResourceType::url_patterns`]; pool wiring feeds those patterns
/// into `Network.setBlockedURLs` before the first navigation so blocked
/// types never reach the network stack.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum RejectResourceType {
    Image,
    Media,
    Font,
    Stylesheet,
}

impl RejectResourceType {
    /// Wildcard URL patterns matched by Chrome's `Network.setBlockedURLs`.
    /// The list is intentionally extension-based — `setBlockedURLs` does
    /// not understand MIME types — so it covers the file suffixes the
    /// edge would normally block for the same category.
    pub fn url_patterns(self) -> &'static [&'static str] {
        match self {
            RejectResourceType::Image => &[
                "*.png", "*.jpg", "*.jpeg", "*.gif", "*.webp", "*.avif", "*.svg", "*.ico",
            ],
            RejectResourceType::Media => &["*.mp3", "*.mp4", "*.webm", "*.ogg", "*.m3u8", "*.mpd"],
            RejectResourceType::Font => &["*.woff", "*.woff2", "*.ttf", "*.otf", "*.eot"],
            RejectResourceType::Stylesheet => &["*.css"],
        }
    }

    pub fn as_str(self) -> &'static str {
        match self {
            RejectResourceType::Image => "image",
            RejectResourceType::Media => "media",
            RejectResourceType::Font => "font",
            RejectResourceType::Stylesheet => "stylesheet",
        }
    }
}

impl std::str::FromStr for RejectResourceType {
    type Err = String;
    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
        match s.trim().to_ascii_lowercase().as_str() {
            "image" => Ok(RejectResourceType::Image),
            "media" => Ok(RejectResourceType::Media),
            "font" => Ok(RejectResourceType::Font),
            "stylesheet" => Ok(RejectResourceType::Stylesheet),
            other => Err(format!(
                "unknown reject_resource_type `{other}`; expected one of \
                 image, media, font, stylesheet"
            )),
        }
    }
}

#[derive(Default)]
pub struct ConfigBuilder {
    inner: Config,
}

impl ConfigBuilder {
    pub fn max_concurrent_render(mut self, n: usize) -> Self {
        self.inner.max_concurrent_render = n;
        self
    }
    pub fn max_concurrent_http(mut self, n: usize) -> Self {
        self.inner.max_concurrent_http = n;
        self
    }
    pub fn respect_robots_txt(mut self, v: bool) -> Self {
        self.inner.respect_robots_txt = v;
        self
    }
    pub fn crawl_purposes(mut self, p: Vec<crate::robots::Purpose>) -> Self {
        self.inner.crawl_purposes = p;
        self
    }
    pub fn user_agent_profile(mut self, p: Profile) -> Self {
        self.inner.user_agent_profile = p;
        self
    }
    pub fn wait_strategy(mut self, w: WaitStrategy) -> Self {
        self.inner.wait_strategy = w;
        self
    }
    pub fn queue(mut self, q: QueueBackend) -> Self {
        self.inner.queue_backend = q;
        self
    }
    pub fn storage(mut self, s: StorageBackend) -> Self {
        self.inner.storage_backend = s;
        self
    }
    pub fn proxy(mut self, p: ProxyConfig) -> Self {
        self.inner.proxy = p;
        self
    }
    pub fn build(self) -> crate::Result<Config> {
        Ok(self.inner)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn url(s: &str) -> url::Url {
        url::Url::parse(s).expect("test url parses")
    }

    #[test]
    fn warmup_default_is_disabled() {
        // The whole point of warmup is that it's opt-in: enabling it
        // silently would surprise operators with extra requests per target.
        let p = WarmupPolicy::default();
        assert!(!p.enabled);
        assert_eq!(p.url_template, "{origin}");
        assert_eq!(p.dwell_ms, 1500);
    }

    #[test]
    fn config_default_warmup_is_disabled() {
        let c = Config::default();
        assert!(!c.warmup.enabled);
    }

    #[test]
    fn render_template_empty_yields_empty_string() {
        // Empty template is the caller's "skip warmup" sentinel; we don't
        // substitute anything, so the result stays empty.
        let p = WarmupPolicy {
            enabled: true,
            url_template: String::new(),
            dwell_ms: 0,
        };
        assert_eq!(p.render_template(&url("https://example.com/foo")), "");
    }

    #[test]
    fn render_template_origin_placeholder() {
        let p = WarmupPolicy {
            enabled: true,
            url_template: "{origin}".to_string(),
            dwell_ms: 0,
        };
        // Default port elided — we want "https://example.com", not
        // "https://example.com:443", to match what the target actually uses.
        assert_eq!(
            p.render_template(&url("https://example.com/foo/bar")),
            "https://example.com"
        );
    }

    #[test]
    fn render_template_origin_preserves_non_default_port() {
        let p = WarmupPolicy {
            enabled: true,
            url_template: "{origin}".to_string(),
            dwell_ms: 0,
        };
        // Non-default port is load-bearing for cookie scoping — the warmup
        // and target need to agree on it.
        assert_eq!(
            p.render_template(&url("https://example.com:8443/x")),
            "https://example.com:8443"
        );
    }

    #[test]
    fn render_template_host_placeholder() {
        let p = WarmupPolicy {
            enabled: true,
            url_template: "{host}".to_string(),
            dwell_ms: 0,
        };
        assert_eq!(
            p.render_template(&url("https://sub.example.com/foo")),
            "sub.example.com"
        );
    }

    #[test]
    fn render_template_mixed_placeholders() {
        let p = WarmupPolicy {
            enabled: true,
            url_template: "{origin}/search?q={host}".to_string(),
            dwell_ms: 0,
        };
        assert_eq!(
            p.render_template(&url("https://example.com/x")),
            "https://example.com/search?q=example.com"
        );
    }

    #[test]
    fn render_template_literal_without_placeholders_passes_through() {
        // Operators can hard-code a third-party warmup (e.g. google.com)
        // if they want Referer-style cover — no placeholder substitution
        // required for that case.
        let p = WarmupPolicy {
            enabled: true,
            url_template: "https://www.google.com/".to_string(),
            dwell_ms: 0,
        };
        assert_eq!(
            p.render_template(&url("https://example.com/foo")),
            "https://www.google.com/"
        );
    }

    #[test]
    fn reject_resource_type_default_is_empty() {
        // Empty default is load-bearing: any non-empty list visibly changes
        // bandwidth for existing users, so the migration has to be opt-in.
        let c = Config::default();
        assert!(c.reject_resource_types.is_empty());
    }

    #[test]
    fn reject_resource_type_from_str_covers_canonical_set() {
        use std::str::FromStr;
        assert_eq!(
            RejectResourceType::from_str("image").unwrap(),
            RejectResourceType::Image
        );
        assert_eq!(
            RejectResourceType::from_str("MEDIA").unwrap(),
            RejectResourceType::Media
        );
        assert_eq!(
            RejectResourceType::from_str(" font ").unwrap(),
            RejectResourceType::Font
        );
        assert_eq!(
            RejectResourceType::from_str("stylesheet").unwrap(),
            RejectResourceType::Stylesheet
        );
        assert!(RejectResourceType::from_str("script").is_err());
    }

    #[test]
    fn reject_resource_type_url_patterns_are_nonempty_for_each_variant() {
        // The whole field is dead weight unless every variant expands to
        // at least one pattern. Guard against accidentally shipping a
        // variant with an empty pattern list.
        for rt in [
            RejectResourceType::Image,
            RejectResourceType::Media,
            RejectResourceType::Font,
            RejectResourceType::Stylesheet,
        ] {
            assert!(!rt.url_patterns().is_empty(), "{:?}", rt);
        }
    }

    #[test]
    fn browser_provider_default_is_stock() {
        // Slice 29: existing operators must see no behavioural change
        // until they opt into a different provider.
        let c = Config::default();
        assert_eq!(c.browser_provider, BrowserProvider::Stock);
    }

    #[test]
    fn browser_provider_parse_accepts_canonical_names() {
        assert_eq!(
            BrowserProvider::parse("stock"),
            Some(BrowserProvider::Stock)
        );
        assert_eq!(BrowserProvider::parse("CDP"), Some(BrowserProvider::Cdp));
        assert_eq!(
            BrowserProvider::parse(" auto "),
            Some(BrowserProvider::Auto)
        );
        assert_eq!(BrowserProvider::parse("bogus_provider"), None);
        assert_eq!(BrowserProvider::parse(""), None);
    }

    #[test]
    fn browser_provider_serde_roundtrip_snake_case() {
        let json = serde_json::to_string(&BrowserProvider::Cdp).unwrap();
        assert_eq!(json, "\"cdp\"");
        let back: BrowserProvider = serde_json::from_str("\"auto\"").unwrap();
        assert_eq!(back, BrowserProvider::Auto);
    }

    #[test]
    fn external_cdp_session_mode_default_is_isolated() {
        // Slice 35 acceptance: external CDP sessions default to
        // crawlex-owned isolated behavior. Persistent must be an
        // explicit operator opt-in.
        let c = Config::default();
        assert_eq!(
            c.external_cdp_session_mode,
            ExternalCdpSessionMode::Isolated
        );
        assert!(!c.external_cdp_session_mode.is_persistent());
    }

    #[test]
    fn external_cdp_session_mode_parse_accepts_canonical_names() {
        assert_eq!(
            ExternalCdpSessionMode::parse("isolated"),
            Some(ExternalCdpSessionMode::Isolated)
        );
        assert_eq!(
            ExternalCdpSessionMode::parse("PERSISTENT"),
            Some(ExternalCdpSessionMode::Persistent)
        );
        assert_eq!(
            ExternalCdpSessionMode::parse(" persistent "),
            Some(ExternalCdpSessionMode::Persistent)
        );
        assert_eq!(ExternalCdpSessionMode::parse("ephemeral"), None);
        assert_eq!(ExternalCdpSessionMode::parse(""), None);
    }

    #[test]
    fn external_cdp_session_mode_serde_roundtrip_snake_case() {
        let json = serde_json::to_string(&ExternalCdpSessionMode::Persistent).unwrap();
        assert_eq!(json, "\"persistent\"");
        let back: ExternalCdpSessionMode = serde_json::from_str("\"isolated\"").unwrap();
        assert_eq!(back, ExternalCdpSessionMode::Isolated);
    }

    #[test]
    fn external_cdp_session_mode_as_str_round_trips_via_parse() {
        for mode in [
            ExternalCdpSessionMode::Isolated,
            ExternalCdpSessionMode::Persistent,
        ] {
            let s = mode.as_str();
            assert_eq!(ExternalCdpSessionMode::parse(s), Some(mode));
        }
    }

    #[test]
    fn provider_fallback_default_is_disabled_and_empty() {
        // Slice 36 acceptance: fallback OFF by default. Existing configs
        // (including those serialized before slice 36) must deserialize
        // unchanged via `#[serde(default)]` on every new field.
        let c = Config::default();
        assert!(!c.provider_fallback.enabled);
        assert!(c.provider_fallback.order.is_empty());
        assert!(!c.provider_fallback.is_active());
        assert!(c
            .provider_fallback
            .chain_after(BrowserProvider::Cdp)
            .is_empty());
    }

    #[test]
    fn provider_fallback_inactive_unless_enabled_and_nonempty() {
        let mut fb = ProviderFallbackConfig::default();
        fb.order = vec![BrowserProvider::Stock];
        assert!(!fb.is_active(), "order without enable must stay inert");
        let mut fb = ProviderFallbackConfig::default();
        fb.enabled = true;
        assert!(!fb.is_active(), "enable without order must stay inert");
    }

    #[test]
    fn provider_fallback_chain_skips_primary_and_dedupes() {
        let fb = ProviderFallbackConfig {
            enabled: true,
            order: vec![
                BrowserProvider::Cdp,
                BrowserProvider::Stock,
                BrowserProvider::Cdp,
            ],
        };
        // primary == Cdp -> skip Cdp entries entirely, keep Stock once.
        assert_eq!(
            fb.chain_after(BrowserProvider::Cdp),
            vec![BrowserProvider::Stock]
        );
        // primary == Stock -> keep Cdp (first occurrence only).
        assert_eq!(
            fb.chain_after(BrowserProvider::Stock),
            vec![BrowserProvider::Cdp]
        );
    }

    #[test]
    fn provider_fallback_chain_drops_auto() {
        // Auto is a meta-selector — it cannot be a fallback target.
        let fb = ProviderFallbackConfig {
            enabled: true,
            order: vec![BrowserProvider::Auto, BrowserProvider::Stock],
        };
        assert_eq!(
            fb.chain_after(BrowserProvider::Cdp),
            vec![BrowserProvider::Stock]
        );
    }

    #[test]
    fn provider_fallback_serde_roundtrip_snake_case() {
        let fb = ProviderFallbackConfig {
            enabled: true,
            order: vec![BrowserProvider::Stock, BrowserProvider::Cdp],
        };
        let json = serde_json::to_string(&fb).unwrap();
        // Field names + provider variants are snake_case in YAML/JSON.
        assert!(json.contains("\"enabled\":true"), "{json}");
        assert!(json.contains("\"order\":[\"stock\",\"cdp\"]"), "{json}");
        let back: ProviderFallbackConfig = serde_json::from_str(&json).unwrap();
        assert_eq!(back, fb);
    }

    #[test]
    fn reject_resource_type_serde_roundtrip_lowercase() {
        // serde uses `rename_all = "lowercase"`, so YAML configs read
        // `image`/`media`/`font`/`stylesheet` rather than `Image` etc.
        let json = serde_json::to_string(&RejectResourceType::Stylesheet).unwrap();
        assert_eq!(json, "\"stylesheet\"");
        let back: RejectResourceType = serde_json::from_str("\"image\"").unwrap();
        assert_eq!(back, RejectResourceType::Image);
    }
}