hpx-browser 2.5.12

//! Browser page abstraction with challenge-aware navigation.

use std::{
    collections::HashSet,
    time::{Duration, Instant},
};

#[cfg(feature = "v8")]
use crate::js_runtime::runtime::BrowserJsRuntime;
use crate::{
    challenge::{ChallengeVerdict, EngineClass, engine_classify, engine_classify_lower},
    dom::{Dom, NodeId},
    net::{HttpClient, RedirectPolicy},
    resource_loader::{
        ResourceType, extract_resource_urls, fetch_resources, filter_by_block_types,
    },
    stealth::StealthProfile,
};

/// Default navigation budget.
const DEFAULT_NAV_BUDGET: Duration = Duration::from_secs(15);
/// Default max iterations for challenge retry loops.
const DEFAULT_MAX_ITERATIONS: u8 = 3;

/// Common interface for browser page types.
///
/// This trait provides synchronous accessors for page metadata.
/// For `CdpPage` (which is async-native), consider an async variant
/// or snapshot-based access in the future.
pub trait PageLike {
    /// The page title.
    fn title(&self) -> &str;
    /// The page HTML content.
    fn content(&self) -> &str;
    /// The current URL.
    fn url(&self) -> &str;
}

/// A browser page/tab.
pub struct Page {
    dom: Dom,
    url: String,
    title: String,
    html: String,
    challenge_class: EngineClass,
    profile: Option<StealthProfile>,
    stealth: bool,
    subresource_block_types: HashSet<ResourceType>,
    #[cfg(feature = "v8")]
    js_runtime: Option<BrowserJsRuntime>,
}

impl std::fmt::Debug for Page {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("Page")
            .field("url", &self.url)
            .field("title", &self.title)
            .field("stealth", &self.stealth)
            .field("challenge_class", &self.challenge_class)
            .field("profile", &self.profile.is_some())
            .finish()
    }
}

impl Page {
    pub fn new() -> Self {
        Self {
            dom: Dom::new(),
            url: "about:blank".to_string(),
            title: String::new(),
            html: String::new(),
            challenge_class: EngineClass {
                tag: "L3-RENDERED",
                verdict: ChallengeVerdict::Pass,
                len: 0,
            },
            profile: None,
            stealth: false,
            subresource_block_types: HashSet::new(),
            #[cfg(feature = "v8")]
            js_runtime: None,
        }
    }
}

impl Default for Page {
    fn default() -> Self {
        Self::new()
    }
}

impl Page {
    /// Create a page from raw HTML (no network).
    pub async fn from_html(html: &str, stealth: bool) -> Result<Self, PageError> {
        let dom = crate::html_parser::parse_html(html);
        let title = extract_title(html);
        let challenge_class = engine_classify(html);
        Ok(Self {
            dom,
            url: "about:blank".to_string(),
            title,
            html: html.to_string(),
            challenge_class,
            profile: None,
            stealth,
            subresource_block_types: HashSet::new(),
            #[cfg(feature = "v8")]
            js_runtime: None,
        })
    }

    /// Create a page with profile and URL (no network).
    pub async fn with_profile(
        html: &str,
        url: &str,
        profile: StealthProfile,
    ) -> Result<Self, PageError> {
        let dom = crate::html_parser::parse_html(html);
        let title = extract_title(html);
        let challenge_class = engine_classify(html);
        #[allow(unused_mut)] // mut needed for v8 feature path below
        let mut page = Self {
            dom,
            url: url.to_string(),
            title,
            html: html.to_string(),
            challenge_class,
            profile: Some(profile),
            stealth: true,
            subresource_block_types: HashSet::new(),
            #[cfg(feature = "v8")]
            js_runtime: None,
        };
        // Apply profile to V8 runtime when the feature is enabled.
        #[cfg(feature = "v8")]
        {
            if let Some(ref profile) = page.profile.clone() {
                page.set_profile(profile.clone());
            }
        }
        Ok(page)
    }

    /// Reload the page with new HTML (reuses V8 isolate in v8 mode).
    #[cfg_attr(feature = "hotpath", hotpath::measure)]
    pub fn reload_html(&mut self, html: &str, url: &str) {
        // Lowercase once and share between title extraction and challenge
        // classification; both only need case-insensitive matching on the body.
        let lowered_html = html.to_lowercase();
        self.dom = crate::html_parser::parse_html(html);
        self.url = url.to_string();
        self.html = html.to_string();
        self.title = extract_title_lower(&lowered_html, html);
        self.challenge_class = engine_classify_lower(&lowered_html);
        #[cfg(feature = "v8")]
        {
            if self.js_runtime.is_some() {
                // Reuse existing V8 isolate — just update the DOM reference.
                // ponytail: avoids re-bootstrapping V8 on every reload (~7 bootstrap scripts)
                let rt_dom = crate::html_parser::parse_html(html);
                if let Some(ref mut rt) = self.js_runtime {
                    rt.update_dom(rt_dom);
                }
            } else {
                let rt_dom = crate::html_parser::parse_html(html);
                match BrowserJsRuntime::new(rt_dom) {
                    Ok(rt) => self.js_runtime = Some(rt),
                    Err(e) => tracing::error!("BrowserJsRuntime init failed in reload_html: {e}"),
                }
            }
        }
    }

    /// Navigate to a URL with challenge-aware retry loop.
    ///
    /// Fetch → classify → if challenge detected, retry up to `max_iterations`.
    /// Uses 15s budget by default.
    pub async fn navigate(&mut self, url: &str) -> Result<(), PageError> {
        // ponytail: always Chrome profile; per-profile routing via tls_impersonate
        let client = HttpClient::new(hpx::BrowserProfile::Chrome).map_err(PageError::Net)?;
        self.navigate_inner(url, &client, DEFAULT_MAX_ITERATIONS, DEFAULT_NAV_BUDGET)
            .await
    }

    /// Navigate with a custom solver list.
    ///
    /// Same as `navigate()` but accepts external challenge solvers.
    pub async fn navigate_with_solvers(
        &mut self,
        url: &str,
        solvers: &[&dyn crate::challenge::ChallengeSolver],
    ) -> Result<(), PageError> {
        let client = HttpClient::new(hpx::BrowserProfile::Chrome).map_err(PageError::Net)?;
        self.navigate_with_solvers_inner(
            url,
            &client,
            solvers,
            DEFAULT_MAX_ITERATIONS,
            DEFAULT_NAV_BUDGET,
        )
        .await
    }

    /// Warm navigation — reuse existing page state, fetch new URL.
    ///
    /// Faster than cold `navigate()` because it skips profile setup.
    pub async fn navigate_warm(&mut self, url: &str) -> Result<(), PageError> {
        let client = HttpClient::new(hpx::BrowserProfile::Chrome).map_err(PageError::Net)?;
        let resp = client
            .request("GET", url, None, &[], RedirectPolicy::Follow(10))
            .await
            .map_err(PageError::Net)?;
        let html = resp.text();
        let resp_url = resp.url.clone();

        self.reload_html(&html, &resp_url);
        Ok(())
    }

    /// Core navigate loop with budget and cookie-diff retry.
    async fn navigate_inner(
        &mut self,
        url: &str,
        client: &HttpClient,
        max_iterations: u8,
        budget: Duration,
    ) -> Result<(), PageError> {
        self.navigate_with_solvers_inner(url, client, &[], max_iterations, budget)
            .await
    }

    /// Core navigate loop with solver support.
    async fn navigate_with_solvers_inner(
        &mut self,
        url: &str,
        client: &HttpClient,
        solvers: &[&dyn crate::challenge::ChallengeSolver],
        max_iterations: u8,
        budget: Duration,
    ) -> Result<(), PageError> {
        let t0 = Instant::now();
        let iterations = max_iterations.max(1);

        let resp = client
            .request("GET", url, None, &[], RedirectPolicy::Follow(10))
            .await
            .map_err(PageError::Net)?;
        let mut current_html = resp.text();
        let mut current_url = resp.url.clone();
        let mut cookies_before = cookie_snapshot(client, &current_url).await;

        for iter in 0..iterations {
            if t0.elapsed() >= budget {
                tracing::warn!(
                    iter,
                    elapsed_ms = t0.elapsed().as_millis(),
                    "navigate budget exhausted"
                );
                break;
            }

            self.reload_html(&current_html, &current_url);

            let challenge = engine_classify(&current_html);

            // Clean page — no challenge markers, load sub-resources and return.
            if !challenge.verdict.is_challenge() {
                self.load_subresources().await?;
                return Ok(());
            }

            // Try registered solvers.
            let kind = tag_to_kind(challenge.tag);
            let mut any_solved = false;
            for solver in solvers {
                if !solver.can_handle(&kind) {
                    continue;
                }
                if matches!(
                    solver.solve(&kind, self).await,
                    crate::challenge::SolveOutcome::Solved
                ) {
                    any_solved = true;
                }
            }

            if any_solved {
                // Re-fetch after solver ran.
                let resp = client
                    .request("GET", &current_url, None, &[], RedirectPolicy::Follow(10))
                    .await
                    .map_err(PageError::Net)?;
                current_html = resp.text();
                current_url = resp.url.clone();
                cookies_before = cookie_snapshot(client, &current_url).await;
                continue;
            }

            // Cookie-diff retry: if cookies changed during this iteration,
            // the challenge script may have self-solved.
            if iter + 1 < iterations {
                let cookies_after = cookie_snapshot(client, &current_url).await;
                if cookies_after != cookies_before && !cookies_after.is_empty() {
                    tracing::info!(iter, "cookie delta detected — retrying navigation");
                    let resp = client
                        .request("GET", &current_url, None, &[], RedirectPolicy::Follow(10))
                        .await
                        .map_err(PageError::Net)?;
                    current_html = resp.text();
                    current_url = resp.url.clone();
                    cookies_before = cookie_snapshot(client, &current_url).await;
                    continue;
                }
            }

            // Challenge still present, no solver helped, no cookie change.
            break;
        }

        // Load sub-resources (CSS, scripts) after HTML is settled.
        self.load_subresources().await?;

        Ok(())
    }

    pub async fn evaluate_async(&mut self, script: &str) -> Result<serde_json::Value, PageError> {
        #[cfg(feature = "v8")]
        {
            self.ensure_js_runtime();
            if let Some(ref mut rt) = self.js_runtime {
                let result = rt
                    .execute_script(script)
                    .map_err(|e| PageError::Evaluation(e.to_string()))?;
                return Ok(serde_json::Value::String(result));
            }
        }
        let _ = script;
        Err(PageError::Evaluation(
            "evaluate_async requires v8 feature".into(),
        ))
    }

    /// Evaluate JavaScript — uses V8 when available, stub otherwise.
    pub fn evaluate(&mut self, script: &str) -> Result<String, PageError> {
        #[cfg(feature = "v8")]
        {
            self.ensure_js_runtime();
            if let Some(ref mut rt) = self.js_runtime {
                return rt
                    .execute_script(script)
                    .map_err(|e| PageError::Evaluation(e.to_string()));
            }
        }
        let _ = script;
        Ok("undefined".to_string())
    }

    /// Lazily create the V8 runtime from current page HTML.
    #[cfg(feature = "v8")]
    fn ensure_js_runtime(&mut self) {
        if self.js_runtime.is_some() {
            return;
        }
        let rt_dom = crate::html_parser::parse_html(&self.html);
        match BrowserJsRuntime::new(rt_dom) {
            Ok(rt) => self.js_runtime = Some(rt),
            Err(e) => tracing::error!("BrowserJsRuntime init failed in ensure_js_runtime: {e}"),
        }
    }

    /// Execute all inline `<script>` tags (those without `src`) in document order.
    ///
    /// With the `v8` feature, scripts run in the page's persistent `BrowserJsRuntime`
    /// so globals set by inline scripts are accessible via `evaluate()`.
    pub fn execute_inline_scripts(&mut self) -> Result<(), PageError> {
        let scripts = self.collect_inline_scripts();
        if scripts.is_empty() {
            return Ok(());
        }

        #[cfg(feature = "v8")]
        {
            self.ensure_js_runtime();
            if let Some(ref mut rt) = self.js_runtime {
                for script in &scripts {
                    if let Err(e) = rt.execute_script(script) {
                        tracing::warn!(error = %e, "inline script execution failed");
                    }
                }
            }
        }

        #[cfg(not(feature = "v8"))]
        {
            for script in &scripts {
                tracing::warn!(len = script.len(), "inline script skipped (no v8)");
            }
        }

        Ok(())
    }

    /// Execute all scripts (inline + external) in document order.
    ///
    /// Inline scripts are executed first, then external scripts from the
    /// provided list (filtered to `ResourceType::Script`).
    /// Script errors are logged as warnings and do not stop execution.
    pub fn execute_scripts(
        &mut self,
        external_scripts: &[crate::resource_loader::LoadedResource],
    ) -> Result<(), PageError> {
        use crate::resource_loader::ResourceType;

        self.execute_inline_scripts()?;

        for script in external_scripts {
            if script.resource_type != ResourceType::Script {
                continue;
            }
            #[cfg(feature = "v8")]
            {
                self.ensure_js_runtime();
                if let Some(ref mut rt) = self.js_runtime {
                    if let Err(e) = rt.execute_script(&script.content) {
                        tracing::warn!(url = %script.url, error = %e, "external script execution failed");
                    }
                }
            }
            #[cfg(not(feature = "v8"))]
            {
                tracing::debug!(url = %script.url, "external script skipped (no v8)");
            }
        }

        Ok(())
    }

    /// Collect text content of inline `<script>` elements (no `src` attribute)
    /// in document order.
    pub fn collect_inline_scripts(&self) -> Vec<String> {
        use crate::dom::{DomElement, NodeId};

        let mut scripts = Vec::new();
        for script_id in self
            .dom
            .get_elements_by_tag_name(NodeId::DOCUMENT, "script")
        {
            if let Some(el) = DomElement::new(&self.dom, script_id) {
                if el.attr("src").is_none() {
                    let content = self.dom.text_content(script_id);
                    if !content.is_empty() {
                        scripts.push(content);
                    }
                }
            }
        }
        scripts
    }

    pub async fn title_async(&self) -> Result<String, PageError> {
        Ok(self.title.clone())
    }

    /// Synchronous title.
    pub fn title(&self) -> &str {
        &self.title
    }

    /// Current URL.
    pub fn url(&self) -> &str {
        &self.url
    }

    /// Whether stealth globals are enabled for this page.
    pub fn stealth(&self) -> bool {
        self.stealth
    }

    /// Apply a stealth profile's fields as JS globals and run page init.
    ///
    /// This sets `navigator.userAgent`, `navigator.platform`, screen
    /// dimensions, GPU info, and other fingerprint globals from the
    /// profile, then calls `__hpx_init()` to wire them into the
    /// JavaScript environment.
    #[cfg(feature = "v8")]
    pub fn set_profile(&mut self, profile: StealthProfile) {
        self.ensure_js_runtime();
        if let Some(ref mut rt) = self.js_runtime {
            rt.set_user_agent(&profile.user_agent);
            rt.set_platform(&profile.platform, &profile.os_name, &profile.os_version);
            rt.set_stealth(true);
            rt.run_page_init();
        }
        self.profile = Some(profile);
    }

    /// Page HTML content.
    pub fn content(&self) -> &str {
        &self.html
    }

    pub async fn text_content(&self) -> Result<String, PageError> {
        Ok(self.dom.text_content(crate::dom::NodeId::DOCUMENT))
    }

    pub async fn text_of(&self, selector: &str) -> Result<String, PageError> {
        if let Some(id) = query_selector(&self.dom, selector) {
            Ok(self.dom.text_content(id))
        } else {
            Err(PageError::ElementNotFound)
        }
    }

    /// Synchronous element check.
    pub fn has_element(&self, selector: &str) -> bool {
        query_selector(&self.dom, selector).is_some()
    }

    /// Challenge classification result.
    pub fn challenge_verdict(&self) -> ChallengeVerdict {
        self.challenge_class.verdict
    }

    /// Full challenge classification.
    pub fn engine_class(&self) -> &EngineClass {
        &self.challenge_class
    }

    pub fn dom(&self) -> &Dom {
        &self.dom
    }

    /// Apply external stylesheets by injecting them as `<style>` tags in `<head>`.
    pub fn apply_stylesheets(&mut self, styles: &[crate::resource_loader::LoadedResource]) {
        use crate::{dom::NodeId, resource_loader::ResourceType};

        let html_el = self
            .dom
            .child_elements(NodeId::DOCUMENT)
            .into_iter()
            .find(|&id| {
                self.dom
                    .get(id)
                    .map(|n| n.is_element_with_tag("html"))
                    .unwrap_or(false)
            });
        let head = html_el.and_then(|html| {
            self.dom
                .get_elements_by_tag_name(html, "head")
                .into_iter()
                .next()
        });

        if let Some(head) = head {
            for style in styles {
                if style.resource_type == ResourceType::Stylesheet {
                    let style_el = self
                        .dom
                        .create_element(crate::dom::QualName::new("style"), Vec::new());
                    let text = self.dom.create_text(style.content.clone());
                    self.dom.append_child(head, style_el);
                    self.dom.append_child(style_el, text);
                }
            }
        }
    }

    /// Set which resource types to block during subresource loading.
    pub fn set_subresource_block_types(&mut self, types: HashSet<ResourceType>) {
        self.subresource_block_types = types;
    }

    /// Orchestrate the full subresource loading pipeline:
    /// discover → filter → fetch → apply CSS → execute scripts.
    pub async fn load_subresources(&mut self) -> Result<(), PageError> {
        let resources = extract_resource_urls(&self.dom);
        let filtered = filter_by_block_types(resources, &self.subresource_block_types);
        if filtered.is_empty() {
            return Ok(());
        }
        let loaded = fetch_resources(filtered, &self.subresource_block_types, 6).await;

        let styles: Vec<_> = loaded
            .iter()
            .filter(|r| r.resource_type == ResourceType::Stylesheet)
            .cloned()
            .collect();
        let scripts: Vec<_> = loaded
            .iter()
            .filter(|r| r.resource_type == ResourceType::Script)
            .cloned()
            .collect();

        self.apply_stylesheets(&styles);
        self.execute_scripts(&scripts)?;

        // Sync html field with DOM state (stylesheets injected as <style> tags).
        self.html = self.dom.serialize_html(crate::dom::NodeId::DOCUMENT);

        Ok(())
    }
}

impl PageLike for Page {
    fn title(&self) -> &str {
        &self.title
    }

    fn content(&self) -> &str {
        &self.html
    }

    fn url(&self) -> &str {
        &self.url
    }
}

// TODO: Implement PageLike for CdpPage. CdpPage's accessors are async (return
// `Result<String>` via CDP evaluation), so an `AsyncPageLike` trait or a
// snapshot-based approach may be more appropriate.

/// Basic CSS selector query against the DOM.
///
/// Supports `#id`, `.class`, `tag`, `tag.class`, and `tag#id` selectors.
/// Returns the first matching `NodeId` in document order.
fn query_selector(dom: &Dom, selector: &str) -> Option<NodeId> {
    let sel = selector.trim();
    if sel.is_empty() {
        return None;
    }

    if let Some(id_val) = sel.strip_prefix('#') {
        return dom.get_element_by_id(id_val);
    }

    if let Some(class_val) = sel.strip_prefix('.') {
        return dom
            .get_elements_by_class_name(NodeId::DOCUMENT, class_val)
            .into_iter()
            .next();
    }

    // Compound selector: tag, tag#id, tag.class
    let (tag, rest) = match sel.find(|c: char| c == '#' || c == '.') {
        Some(pos) => (&sel[..pos], Some(&sel[pos..])),
        None => (sel, None),
    };

    if tag.is_empty() {
        return None;
    }

    let candidates = dom.get_elements_by_tag_name(NodeId::DOCUMENT, tag);
    match rest {
        Some(r) if r.starts_with('#') => {
            let id_val = &r[1..];
            candidates.into_iter().find(|&id| {
                crate::dom::DomElement::new(dom, id)
                    .and_then(|e| e.id().map(|v| v == id_val))
                    .unwrap_or(false)
            })
        }
        Some(r) if r.starts_with('.') => {
            let class_val = &r[1..];
            candidates.into_iter().find(|&id| {
                crate::dom::DomElement::new(dom, id)
                    .map(|e| e.has_class(class_val))
                    .unwrap_or(false)
            })
        }
        _ => candidates.into_iter().next(),
    }
}

/// Map an `engine_classify` tag to a `ChallengeKind` for solver dispatch.
fn tag_to_kind(tag: &'static str) -> crate::challenge::ChallengeKind {
    let (vendor, sub_kind): (&'static str, &'static str) = if tag.starts_with("cf-") {
        ("cloudflare", tag)
    } else if tag.starts_with("AWS-WAF") {
        ("aws-waf", tag)
    } else if tag.eq_ignore_ascii_case("datadome") {
        ("datadome", tag)
    } else if tag.starts_with("akamai") {
        ("akamai", tag)
    } else if tag.starts_with("px-") || tag.starts_with("PXC") {
        ("perimeterx", tag)
    } else if tag.starts_with("kasada") {
        ("kasada", tag)
    } else if tag.starts_with("sec-cpt") {
        ("sec-cpt", tag)
    } else if tag.starts_with("hcaptcha") {
        ("hcaptcha", tag)
    } else {
        ("unknown", tag)
    };
    crate::challenge::ChallengeKind::new(vendor, sub_kind)
}

/// Snapshot cookie jar for a URL (empty string if none).
async fn cookie_snapshot(client: &HttpClient, url: &str) -> String {
    if let Ok(parsed) = url::Url::parse(url) {
        client.cookies_for_url(&parsed).await.unwrap_or_default()
    } else {
        String::new()
    }
}

/// Extract <title> from HTML (cheap string scan, no full parse).
fn extract_title(html: &str) -> String {
    let lower = html.to_lowercase();
    extract_title_lower(&lower, html)
}

/// Extract <title> from HTML using a pre-lowercased body.
///
/// `lowered_html` is used only to locate the `<title` start tag and the
/// `</title>` end marker (case-insensitive matches); `original_html` is
/// indexed to preserve the title's original casing.
fn extract_title_lower(lowered_html: &str, original_html: &str) -> String {
    if let Some(start) = lowered_html.find("<title") {
        let after_tag = &original_html[start..];
        if let Some(gt) = after_tag.find('>') {
            let content = &after_tag[gt + 1..];
            if let Some(end) = content.to_lowercase().find("</title>") {
                return content[..end].trim().to_string();
            }
        }
    }
    String::new()
}

#[derive(Debug, thiserror::Error)]
pub enum PageError {
    #[error("navigation failed: {0}")]
    Navigation(String),
    #[error("evaluation failed: {0}")]
    Evaluation(String),
    #[error("element not found")]
    ElementNotFound,
    #[error("page not loaded")]
    NotLoaded,
    #[error("network error: {0}")]
    Net(#[from] crate::net::NetError),
}

#[cfg(test)]
mod tests {
    use super::*;

    // ── BDD Scenario 1: Navigate to clean page ──────────────────────────

    #[tokio::test]
    async fn bdd_navigate_to_clean_page() {
        let mut body = String::from("Hello World. ");
        // Push past THIN_BODY_MAX_BYTES (1000) and THIN_SHELL_MAX_BYTES (15KB)
        for _ in 0..500 {
            body.push_str("This is real rendered content for the test page. ");
        }
        let html = format!(
            r#"<!DOCTYPE html>
<html>
<head><title>Test Page</title></head>
<body>{body}</body>
</html>"#
        );
        let page = Page::from_html(&html, false).await.unwrap();

        assert_eq!(page.title(), "Test Page");
        assert!(page.content().contains("Hello World"));
        assert_eq!(page.challenge_verdict(), ChallengeVerdict::Pass);
    }

    // ── BDD Scenario 2: Navigate with challenge detection ────────────────

    #[tokio::test]
    async fn bdd_navigate_with_challenge_detection() {
        // Simulate a Cloudflare challenge response
        let html = r#"<!DOCTYPE html>
<html>
<head><title>Just a moment...</title></head>
<body>
<script>window._cf_chl_opt={cvId:'3',cType:'managed'};</script>
Checking your browser before accessing the site...
</body>
</html>"#;
        let page = Page::from_html(html, false).await.unwrap();

        assert_eq!(page.challenge_verdict(), ChallengeVerdict::EdgeBlock);
        assert!(page.challenge_verdict().is_challenge());
    }

    // ── BDD Scenario 3: ChallengeIncomplete for large managed shell ──────

    #[tokio::test]
    async fn bdd_challenge_incomplete_verdict() {
        let mut html = String::from(
            r#"<html><head><title>Just a moment...</title></head><body>
            <script>window._cf_chl_opt={cvId:'3',cType:'managed'};</script>"#,
        );
        for _ in 0..2000 {
            html.push_str("<div>cf challenge orchestrator shell padding</div>");
        }
        html.push_str("</body></html>");
        assert!(html.len() >= 50_000);

        let page = Page::from_html(&html, false).await.unwrap();
        assert_eq!(
            page.challenge_verdict(),
            ChallengeVerdict::ChallengeIncomplete
        );
        assert!(page.challenge_verdict().is_challenge());
    }

    // ── BDD: Clean page with substantial content passes ──────────────────

    #[tokio::test]
    async fn bdd_clean_page_passes() {
        let mut html = String::from("<html><body>");
        for _ in 0..400 {
            html.push_str("<p>Normal rendered content paragraph with enough text.</p>");
        }
        html.push_str("</body></html>");
        assert!(html.len() >= 15_000);

        let page = Page::from_html(&html, false).await.unwrap();
        assert_eq!(page.challenge_verdict(), ChallengeVerdict::Pass);
        assert!(!page.challenge_verdict().is_challenge());
    }

    // ── BDD: Warm reuse reloads HTML ─────────────────────────────────────

    #[tokio::test]
    async fn bdd_warm_reuse_reloads_html() {
        let html1 =
            r#"<!DOCTYPE html><html><head><title>First</title></head><body>Page One</body></html>"#;
        let html2 = r#"<!DOCTYPE html><html><head><title>Second</title></head><body>Page Two</body></html>"#;

        let mut page = Page::from_html(html1, false).await.unwrap();
        assert_eq!(page.title(), "First");
        assert!(page.content().contains("Page One"));

        // Warm reuse: reload with new HTML
        page.reload_html(html2, "https://example.com/second");
        assert_eq!(page.title(), "Second");
        assert!(page.content().contains("Page Two"));
        assert_eq!(page.url(), "https://example.com/second");
    }

    // ── BDD: Thin body is RenderIncomplete ───────────────────────────────

    #[tokio::test]
    async fn bdd_thin_body_render_incomplete() {
        let html = "<html><body>tiny</body></html>";
        let page = Page::from_html(html, false).await.unwrap();
        assert_eq!(page.challenge_verdict(), ChallengeVerdict::RenderIncomplete);
        assert!(!page.challenge_verdict().is_challenge());
    }

    // ── BDD: DataDome interstitial detected ──────────────────────────────

    #[tokio::test]
    async fn bdd_datadome_interstitial() {
        let html = r#"<script src="https://geo.captcha-delivery.com/captcha.js"></script>
<div id="ddcaptchaencoded">encoded_payload</div>"#;
        let page = Page::from_html(html, false).await.unwrap();
        assert!(page.challenge_verdict().is_challenge());
    }

    // ── BDD: AWS-WAF challenge detected ──────────────────────────────────

    #[tokio::test]
    async fn bdd_awswaf_challenge() {
        let html = r#"<html><body>
<script>window.gokuProps={key:'a',context:'b',iv:'c'};</script>
<script>window.awsWafCookieDomainList=["example.com"];</script>
<script src="https://x.token.awswaf.com/challenge.js"></script>
<script>AwsWafIntegration.checkForceRefresh();</script>
</body></html>"#;
        let page = Page::from_html(html, false).await.unwrap();
        assert!(page.challenge_verdict().is_challenge());
    }

    // ── extract_title tests ──────────────────────────────────────────────

    #[test]
    fn extract_title_basic() {
        assert_eq!(
            extract_title("<html><head><title>Hello</title></head></html>"),
            "Hello"
        );
    }

    #[test]
    fn extract_title_empty() {
        assert_eq!(extract_title("<html><body></body></html>"), "");
    }

    #[test]
    fn extract_title_case_insensitive() {
        assert_eq!(
            extract_title("<HTML><HEAD><TITLE>Test</TITLE></HEAD></HTML>"),
            "Test"
        );
    }

    #[tokio::test]
    async fn apply_stylesheets_injects_style_tags() {
        use crate::{
            dom::NodeId,
            resource_loader::{LoadedResource, ResourceType},
        };

        let mut page = Page::from_html("<html><head></head><body></body></html>", false)
            .await
            .unwrap();

        let styles = vec![LoadedResource {
            url: "http://example.com/style.css".to_string(),
            resource_type: ResourceType::Stylesheet,
            content: "body { color: red; }".to_string(),
            content_type: Some("text/css".to_string()),
        }];

        page.apply_stylesheets(&styles);

        let html = page.dom().child_elements(NodeId::DOCUMENT)[0];
        let head = page.dom().get_elements_by_tag_name(html, "head")[0];
        let style_tags = page.dom().get_elements_by_tag_name(head, "style");
        assert_eq!(style_tags.len(), 1);

        // Verify the text content of the injected <style>
        let text = page.dom().text_content(style_tags[0]);
        assert_eq!(text, "body { color: red; }");
    }

    #[tokio::test]
    async fn apply_stylesheets_skips_non_stylesheet_resources() {
        use crate::{
            dom::NodeId,
            resource_loader::{LoadedResource, ResourceType},
        };

        let mut page = Page::from_html("<html><head></head><body></body></html>", false)
            .await
            .unwrap();

        let styles = vec![
            LoadedResource {
                url: "http://example.com/script.js".to_string(),
                resource_type: ResourceType::Script,
                content: "alert(1)".to_string(),
                content_type: Some("application/javascript".to_string()),
            },
            LoadedResource {
                url: "http://example.com/style.css".to_string(),
                resource_type: ResourceType::Stylesheet,
                content: "h1 { font-size: 2em; }".to_string(),
                content_type: Some("text/css".to_string()),
            },
        ];

        page.apply_stylesheets(&styles);

        let html = page.dom().child_elements(NodeId::DOCUMENT)[0];
        let head = page.dom().get_elements_by_tag_name(html, "head")[0];
        let style_tags = page.dom().get_elements_by_tag_name(head, "style");
        assert_eq!(style_tags.len(), 1);
    }

    #[tokio::test]
    async fn collect_inline_scripts_excludes_external() {
        let html = r#"<!DOCTYPE html>
<html><head></head><body>
<script>window.a = 1;</script>
<script src="/app.js"></script>
<script>window.b = 2;</script>
</body></html>"#;
        let page = Page::from_html(html, false).await.unwrap();
        let scripts = page.collect_inline_scripts();
        assert_eq!(scripts.len(), 2);
        assert_eq!(scripts[0], "window.a = 1;");
        assert_eq!(scripts[1], "window.b = 2;");
    }

    #[tokio::test]
    async fn collect_inline_scripts_empty_when_none() {
        let html = r#"<!DOCTYPE html>
<html><head></head><body>
<script src="/app.js"></script>
<p>No inline scripts here</p>
</body></html>"#;
        let page = Page::from_html(html, false).await.unwrap();
        let scripts = page.collect_inline_scripts();
        assert!(scripts.is_empty());
    }

    #[cfg(feature = "v8")]
    #[tokio::test]
    async fn execute_inline_scripts_sets_globals() {
        let html = r#"<!DOCTYPE html>
<html><head></head><body>
<script>window.x = 42;</script>
</body></html>"#;
        let mut page = Page::from_html(html, false).await.unwrap();
        page.execute_inline_scripts().unwrap();

        let mut rt = BrowserJsRuntime::new(crate::dom::Dom::new()).unwrap();
        // The runtime is separate, so we verify via a fresh runtime that
        // our method returned Ok (scripts were executed without error).
        // The real integration test is that execute_inline_scripts doesn't panic.
        let result = rt.execute_script("1 + 1").unwrap();
        assert_eq!(result, "2");
    }

    #[cfg(feature = "v8")]
    #[tokio::test]
    async fn execute_inline_scripts_continues_on_error() {
        let html = r#"<!DOCTYPE html>
<html><head></head><body>
<script>throw new Error("boom");</script>
<script>window.ok = true;</script>
</body></html>"#;
        let mut page = Page::from_html(html, false).await.unwrap();
        // Should not return Err — logs warning and continues.
        page.execute_inline_scripts().unwrap();
    }

    #[tokio::test]
    async fn execute_scripts_processes_external_scripts() {
        use crate::resource_loader::{LoadedResource, ResourceType};

        let mut page = Page::from_html("<html><body></body></html>", false)
            .await
            .unwrap();

        let scripts = vec![LoadedResource {
            url: "http://example.com/app.js".to_string(),
            resource_type: ResourceType::Script,
            content: "var x = 1;".to_string(),
            content_type: Some("application/javascript".to_string()),
        }];
        page.execute_scripts(&scripts).unwrap();
    }

    #[tokio::test]
    async fn execute_scripts_mixed_inline_and_external() {
        use crate::resource_loader::{LoadedResource, ResourceType};

        let html = r#"<!DOCTYPE html>
<html><head></head><body>
<script>window.a = 1;</script>
<script src="/app.js"></script>
<script>window.b = 2;</script>
</body></html>"#;
        let mut page = Page::from_html(html, false).await.unwrap();

        let scripts = vec![LoadedResource {
            url: "http://example.com/app.js".to_string(),
            resource_type: ResourceType::Script,
            content: "window.c = 3;".to_string(),
            content_type: Some("application/javascript".to_string()),
        }];
        page.execute_scripts(&scripts).unwrap();
    }

    #[tokio::test]
    async fn execute_scripts_skips_non_script_resources() {
        use crate::resource_loader::{LoadedResource, ResourceType};

        let mut page = Page::from_html("<html><body></body></html>", false)
            .await
            .unwrap();

        let resources = vec![LoadedResource {
            url: "http://example.com/style.css".to_string(),
            resource_type: ResourceType::Stylesheet,
            content: "body { color: red; }".to_string(),
            content_type: Some("text/css".to_string()),
        }];
        page.execute_scripts(&resources).unwrap();
    }
}