crawlberg-browser 1.11.0

//! A single document navigation: robots policy, document fetch, sub-resources and lifecycle.

use url::Url;

use super::security::{escape_for_js_template_literal, subresource_allowed};
use super::{Page, PageError};
use crate::dom::{DomTree, parse_html};
use crate::lifecycle::LifecycleState;
use crate::net::Response;

const ROBOTS_OK_STATUS: u16 = 200;

/// Overall ceiling on the `networkidle` wait, independent of how quiet the page gets.
const NETWORK_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
/// How long the request count must stay at or below the threshold to count as idle.
const NETWORK_IDLE_SETTLE: std::time::Duration = std::time::Duration::from_millis(500);
/// Bound on one event-loop turn while waiting for network idle.
const NETWORK_IDLE_TICK: std::time::Duration = std::time::Duration::from_millis(50);
/// Poll interval while waiting for network idle on a page that has no JS realm to pump.
const NETWORK_IDLE_POLL_WITHOUT_JS: std::time::Duration = std::time::Duration::from_millis(100);
/// In-flight request count treated as idle by `WaitUntil::NetworkIdle0` / `NetworkIdle2`.
const NETWORK_IDLE0_THRESHOLD: u32 = 0;
const NETWORK_IDLE2_THRESHOLD: u32 = 2;
impl Page {
    /// Load one document, following at most `max_redirects` HTTP redirects to it (the client's
    /// cap when `None`), and return how many it followed.
    pub(super) async fn navigate_single(
        &mut self,
        url_str: &str,
        wait_until: crate::lifecycle::WaitUntil,
        method: &str,
        body: &str,
        max_redirects: Option<usize>,
    ) -> Result<usize, PageError> {
        let url = Url::parse(url_str).map_err(|e| PageError::InvalidUrl(e.to_string()))?;
        let initiator = self.url.clone();
        // ~keep Refused before robots.txt or the document is fetched, and before the URL is
        // ~keep recorded as the page's own, so nothing downstream sees the userinfo.
        crate::net::credential::refuse_userinfo(&url).map_err(|e| PageError::NetworkError(e.to_string()))?;

        self.lifecycle = LifecycleState::Loading;
        self.url = Some(url.clone());
        self.document_base_url = None;
        self.network_events.clear();

        self.enforce_robots(&url, initiator.as_ref()).await?;
        let response = self
            .fetch_document(&url, method, body, max_redirects, initiator.as_ref())
            .await?;
        let redirects = response.redirected_from.len();
        self.unfollowed_redirect = unfollowed_redirect(&response);

        self.record_network_event(
            url.as_str(),
            "GET",
            "Document",
            response.status,
            &response.headers,
            response.body.len(),
        );

        if !response.redirected_from.is_empty() {
            self.url = Some(response.url.clone());
        }

        let (body_text, charset) = match self.document_decoder {
            Some(decode) => decode(
                response.content_type().unwrap_or_default(),
                response.url.as_str(),
                &response.body,
            ),
            None => (String::from_utf8_lossy(&response.body).into_owned(), None),
        };
        self.document_charset = charset;
        let dom = parse_html(&body_text);
        self.document_base_url = self.base_url_from_dom(&dom);
        self.title = dom
            .query_selector("title")
            .ok()
            .flatten()
            .map(|title_id| dom.text_content(title_id))
            .unwrap_or_default();

        let css_sources = self.load_stylesheets(&dom).await;

        self.dom = Some(dom);
        self.lifecycle = LifecycleState::DomContentLoaded;

        if wait_until == crate::lifecycle::WaitUntil::DomContentLoaded {
            self.init_js();
            return Ok(redirects);
        }

        self.init_js();
        self.inject_stylesheets(&css_sources);
        self.load_iframe_sources();

        self.execute_scripts().await;

        if let Some(js) = &mut self.js
            && let Ok(new_title) = js.evaluate("document.title")
            && let Some(t) = new_title.as_str()
        {
            self.title = t.to_string();
        }

        self.lifecycle = LifecycleState::Loaded;
        self.wait_for_network_idle(wait_until).await;

        Ok(redirects)
    }

    fn base_url_from_dom(&self, dom: &DomTree) -> Option<Url> {
        let page_url = self.url.as_ref()?;
        let href = dom
            .query_selector_all("base")
            .unwrap_or_default()
            .into_iter()
            .find_map(|node_id| dom.get_node(node_id)?.get_attribute("href").map(str::to_owned));
        let Some(href) = href else {
            return Some(page_url.clone());
        };
        Url::options()
            .base_url(Some(page_url))
            .parse(&href)
            .ok()
            .or_else(|| Some(page_url.clone()))
    }

    /// Apply the context's `robots.txt` policy, fetching and caching the file on first use.
    ///
    /// Does nothing when `obey_robots` is off or the URL has no host.
    async fn enforce_robots(&mut self, url: &Url, initiator: Option<&Url>) -> Result<(), PageError> {
        if !self.context.obey_robots || url.host_str().is_none() {
            return Ok(());
        }

        // ~keep Keyed and fetched by origin, not by host: RFC 9309 section 2.3 scopes a
        // ~keep robots.txt file to a scheme, a host and a port, so two ports on one host are
        // ~keep two files. A host key asks the wrong port and shares one answer between them.
        let origin = url.origin().ascii_serialization();
        if self.context.robots_cache.is_allowed(&origin, "/robots.txt") {
            let robots_url = format!("{origin}/robots.txt");
            if let Ok(robots_url) = Url::parse(&robots_url)
                && let Ok(resp) = self.http_client.fetch_subresource(&robots_url, initiator).await
                && resp.status == ROBOTS_OK_STATUS
            {
                let body = String::from_utf8_lossy(&resp.body);
                self.context
                    .robots_cache
                    .parse_and_store(&origin, &body, &self.context.user_agent);
            }
        }

        if !self.context.robots_cache.is_allowed(&origin, url.path()) {
            self.lifecycle = LifecycleState::Failed;
            return Err(PageError::NetworkError(format!("Blocked by robots.txt: {}", url)));
        }
        Ok(())
    }

    async fn fetch_document(
        &mut self,
        url: &Url,
        method: &str,
        body: &str,
        max_redirects: Option<usize>,
        initiator: Option<&Url>,
    ) -> Result<Response, PageError> {
        let result = if method == "POST" {
            self.http_client
                .fetch_following_from(
                    reqwest::Method::POST,
                    url,
                    Some(body.as_bytes().to_vec()),
                    max_redirects,
                    initiator,
                )
                .await
        } else {
            self.do_fetch_from(url, max_redirects, initiator).await
        };
        result.map_err(|e| {
            self.lifecycle = LifecycleState::Failed;
            PageError::from(e)
        })
    }

    /// Resolved hrefs of the `<link rel=stylesheet>` elements policy and interception permit.
    fn allowed_stylesheet_urls(&self, dom: &DomTree) -> Vec<String> {
        let hrefs: Vec<String> = dom
            .query_selector_all("link")
            .unwrap_or_default()
            .iter()
            .filter_map(|&nid| {
                let node = dom.get_node(nid)?;
                let rel = node.get_attribute("rel")?;
                if rel.to_lowercase() != "stylesheet" {
                    return None;
                }
                node.get_attribute("href").map(|s| s.to_string())
            })
            .collect();

        let mut allowed = Vec::new();
        for href in &hrefs {
            let Some(full_url) = self.resolve_subresource_url(href) else {
                tracing::debug!(
                    "skipping unparseable <link rel=stylesheet href>: page={} href_len={}",
                    self.url_string(),
                    href.len(),
                );
                continue;
            };
            if !subresource_allowed(self.url.as_ref(), &full_url) {
                tracing::warn!(
                    "blocking cross-scheme <link rel=stylesheet href>: page={} href={}",
                    self.url_string(),
                    full_url,
                );
                continue;
            }
            if self.should_block_url(&full_url) {
                tracing::info!("Blocked stylesheet by interception: {}", full_url);
                continue;
            }
            allowed.push(full_url);
        }
        allowed
    }

    /// Fetch every permitted stylesheet concurrently and record a network event for each.
    async fn load_stylesheets(&mut self, dom: &DomTree) -> Vec<String> {
        let page = &*self;
        let css_futures: Vec<_> = self
            .allowed_stylesheet_urls(dom)
            .into_iter()
            .map(move |url_str| {
                let page = page;
                async move {
                    let parsed = Url::parse(&url_str).unwrap_or_else(|_| Url::parse("about:blank").unwrap());
                    match page.do_fetch_subresource(&parsed).await {
                        Ok(resp) => Some((url_str, resp)),
                        Err(e) => {
                            tracing::debug!("Failed to fetch stylesheet {}: {}", url_str, e);
                            None
                        }
                    }
                }
            })
            .collect();

        let mut css_sources = Vec::new();
        for (url_str, resp) in futures::future::join_all(css_futures).await.into_iter().flatten() {
            let css = String::from_utf8_lossy(&resp.body).to_string();
            self.record_network_event(
                &url_str,
                "GET",
                "Stylesheet",
                resp.status,
                &resp.headers,
                resp.body.len(),
            );
            css_sources.push(css);
        }
        css_sources
    }

    fn inject_stylesheets(&mut self, css_sources: &[String]) {
        if css_sources.is_empty() {
            return;
        }
        let Some(js) = &mut self.js else {
            return;
        };
        let combined_css = css_sources.join("\n");
        // ~keep Escape JS template literals fully so attacker-controlled CSS cannot break into executable JS.
        let escaped = escape_for_js_template_literal(&combined_css);
        let code = format!("globalThis.__crawlberg_css = `{}`;", escaped);
        let _ = js.execute_script("<css>", &code);
    }

    fn load_iframe_sources(&mut self) {
        if let Some(js) = &mut self.js {
            let _ = js.execute_script("<iframe-load>",
                "(function() { var iframes = document.querySelectorAll('iframe[src]'); for (var i = 0; i < iframes.length; i++) { var src = iframes[i].getAttribute('src'); if (src && src !== 'about:blank') iframes[i]._loadIframeSrc(src); } })()");
        }
    }

    /// Block until in-flight requests stay at or below the `wait_until` threshold for
    /// [`NETWORK_IDLE_SETTLE`], or until [`NETWORK_IDLE_TIMEOUT`] elapses. A no-op for the
    /// non-`networkidle` wait conditions.
    async fn wait_for_network_idle(&mut self, wait_until: crate::lifecycle::WaitUntil) {
        let threshold = match wait_until {
            crate::lifecycle::WaitUntil::NetworkIdle0 => NETWORK_IDLE0_THRESHOLD,
            crate::lifecycle::WaitUntil::NetworkIdle2 => NETWORK_IDLE2_THRESHOLD,
            _ => return,
        };

        let deadline = tokio::time::Instant::now() + NETWORK_IDLE_TIMEOUT;
        let mut idle_since: Option<tokio::time::Instant> = None;

        loop {
            let active = self.http_client.active_requests();
            let now = tokio::time::Instant::now();

            if active <= threshold {
                let since = *idle_since.get_or_insert(now);
                if now.duration_since(since) >= NETWORK_IDLE_SETTLE {
                    break;
                }
            } else {
                idle_since = None;
            }

            if now >= deadline {
                tracing::debug!("Network idle timeout reached with {} active requests", active);
                break;
            }

            if let Some(js) = &mut self.js {
                let _ = tokio::time::timeout(NETWORK_IDLE_TICK, js.run_event_loop()).await;
            } else {
                tokio::time::sleep(NETWORK_IDLE_POLL_WITHOUT_JS).await;
            }
        }

        self.lifecycle = LifecycleState::NetworkIdle;
    }
}

fn unfollowed_redirect(response: &Response) -> Option<PageError> {
    const REDIRECT_STATUSES: [u16; 5] = [301, 302, 303, 307, 308];
    if !REDIRECT_STATUSES.contains(&response.status) {
        return None;
    }
    let target = response.url.join(response.headers.get("location")?).ok()?;
    if !matches!(target.scheme(), "http" | "https") {
        return None;
    }
    let target = crate::net::credential::without_userinfo(&target);
    let visited = crate::net::credential::without_userinfo(&response.url) == target
        || response
            .redirected_from
            .iter()
            .any(|url| crate::net::credential::without_userinfo(url) == target);
    Some(PageError::UnfollowedRedirect { target, visited })
}

#[cfg(test)]
mod redirect_error_tests {
    use super::*;

    #[test]
    fn should_return_no_unfollowed_redirect_for_success_with_location() {
        let response = Response {
            url: Url::parse("http://example.test/a").expect("response URL"),
            status: 200,
            headers: std::collections::HashMap::from([("location".to_owned(), "/b".to_owned())]),
            body: Vec::new(),
            redirected_from: Vec::new(),
        };
        assert_eq!(unfollowed_redirect(&response).map(|error| error.to_string()), None);
    }

    #[test]
    fn should_return_no_unfollowed_redirect_for_non_http_location() {
        for location in [
            "mailto:help@example.test",
            "javascript:alert(1)",
            "data:text/plain,redirect",
        ] {
            let response = Response {
                url: Url::parse("http://example.test/a").expect("response URL"),
                status: 301,
                headers: std::collections::HashMap::from([("location".to_owned(), location.to_owned())]),
                body: Vec::new(),
                redirected_from: Vec::new(),
            };
            assert_eq!(
                unfollowed_redirect(&response).map(|error| error.to_string()),
                None,
                "location: {location}",
            );
        }
    }

    #[test]
    fn should_return_no_unfollowed_redirect_without_location() {
        let response = Response {
            url: Url::parse("http://example.test/a").expect("response URL"),
            status: 301,
            headers: std::collections::HashMap::new(),
            body: Vec::new(),
            redirected_from: Vec::new(),
        };
        assert_eq!(unfollowed_redirect(&response).map(|error| error.to_string()), None);
    }

    #[test]
    fn should_compare_credential_free_canonical_redirect_keys() {
        for (earlier, location, visited) in [
            (
                "http://old:secret@EXAMPLE.test/b#part",
                "http://user:pass@example.test/b#part",
                true,
            ),
            (
                "http://example.test/b#part",
                "http://user:pass@example.test/b#other",
                false,
            ),
            ("http://EXAMPLE.test", "http://user:pass@example.test/", true),
        ] {
            let response = Response {
                url: Url::parse("http://example.test/a").expect("response URL"),
                status: 301,
                headers: std::collections::HashMap::from([("location".to_owned(), location.to_owned())]),
                body: Vec::new(),
                redirected_from: vec![Url::parse(earlier).expect("earlier URL")],
            };
            let expected = crate::net::credential::without_userinfo(&Url::parse(location).expect("target"));
            match unfollowed_redirect(&response).expect("a pending redirect") {
                PageError::UnfollowedRedirect {
                    target,
                    visited: actual,
                } => {
                    assert_eq!(target, expected);
                    assert_eq!(actual, visited, "earlier: {earlier}, location: {location}");
                    assert_eq!(target.username(), "");
                    assert_eq!(target.password(), None);
                }
                error => panic!("unexpected redirect error: {error:?}"),
            }
        }
    }
}