chromiumoxide/

browser.rs

use hashbrown::HashMap;
use reqwest::header::{HeaderMap, HeaderValue, CONTENT_TYPE};
use std::future::Future;
use std::time::Duration;
use std::{
    io,
    path::{Path, PathBuf},
};

use tokio::sync::mpsc::{channel, unbounded_channel, Sender};
use tokio::sync::oneshot::channel as oneshot_channel;

use crate::async_process::{self, Child, ExitStatus, Stdio};
use crate::cmd::{to_command_response, CommandMessage};
use crate::conn::Connection;
use crate::detection::{self, DetectionOptions};
use crate::error::{BrowserStderr, CdpError, Result};
use crate::handler::browser::BrowserContext;
use crate::handler::viewport::Viewport;
use crate::handler::{Handler, HandlerConfig, HandlerMessage, REQUEST_TIMEOUT};
use crate::listeners::{EventListenerRequest, EventStream};
use crate::page::Page;
use crate::utils;
use chromiumoxide_cdp::cdp::browser_protocol::browser::{
    BrowserContextId, CloseReturns, GetVersionParams, GetVersionReturns,
};
use chromiumoxide_cdp::cdp::browser_protocol::browser::{
    PermissionDescriptor, PermissionSetting, SetPermissionParams,
};
use chromiumoxide_cdp::cdp::browser_protocol::network::{Cookie, CookieParam};
use chromiumoxide_cdp::cdp::browser_protocol::storage::{
    ClearCookiesParams, GetCookiesParams, SetCookiesParams,
};
use chromiumoxide_cdp::cdp::browser_protocol::target::{
    CreateBrowserContextParams, CreateTargetParams, DisposeBrowserContextParams,
    GetBrowserContextsParams, GetBrowserContextsReturns, TargetId, TargetInfo,
};

use chromiumoxide_cdp::cdp::{CdpEventMessage, IntoEventKind};
use chromiumoxide_types::*;
use spider_network_blocker::intercept_manager::NetworkInterceptManager;

/// Default `Browser::launch` timeout in MS
pub const LAUNCH_TIMEOUT: u64 = 20_000;

lazy_static::lazy_static! {
    /// The request client to get the web socket url.
    static ref REQUEST_CLIENT: reqwest::Client = reqwest::Client::builder()
        .timeout(Duration::from_secs(60))
        .default_headers({
            let mut m = HeaderMap::new();

            m.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));

            m
        })
        .tcp_keepalive(Some(Duration::from_secs(5)))
        .pool_idle_timeout(Some(Duration::from_secs(60)))
        .pool_max_idle_per_host(10)
        .build()
        .expect("client to build");
}

/// Returns chromey's global `reqwest::Client` for reuse by other subsystems
/// (e.g. remote cache uploads via `spider_remote_cache`).
pub fn request_client() -> &'static reqwest::Client {
    &REQUEST_CLIENT
}

/// A [`Browser`] is created when chromiumoxide connects to a Chromium instance.
#[derive(Debug)]
pub struct Browser {
    /// The `Sender` to send messages to the connection handler that drives the
    /// websocket
    pub(crate) sender: Sender<HandlerMessage>,
    /// How the spawned chromium instance was configured, if any
    config: Option<BrowserConfig>,
    /// The spawned chromium instance
    child: Option<Child>,
    /// The debug web socket url of the chromium instance
    debug_ws_url: String,
    /// The context of the browser
    pub browser_context: BrowserContext,
}

/// Browser connection information.
#[derive(serde::Deserialize, Debug, Default)]
pub struct BrowserConnection {
    #[serde(rename = "Browser")]
    /// The browser name
    pub browser: String,
    #[serde(rename = "Protocol-Version")]
    /// Browser version
    pub protocol_version: String,
    #[serde(rename = "User-Agent")]
    /// User Agent used by default.
    pub user_agent: String,
    #[serde(rename = "V8-Version")]
    /// The v8 engine version
    pub v8_version: String,
    #[serde(rename = "WebKit-Version")]
    /// Webkit version
    pub webkit_version: String,
    #[serde(rename = "webSocketDebuggerUrl")]
    /// Remote debugging address
    pub web_socket_debugger_url: String,
}

impl Browser {
    /// Connect to an already running chromium instance via the given URL.
    ///
    /// If the URL is a http(s) URL, it will first attempt to retrieve the Websocket URL from the `json/version` endpoint.
    pub async fn connect(url: impl Into<String>) -> Result<(Self, Handler)> {
        Self::connect_with_config(url, HandlerConfig::default()).await
    }

    // Connect to an already running chromium instance with a given `HandlerConfig`.
    ///
    /// If the URL is a http URL, it will first attempt to retrieve the Websocket URL from the `json/version` endpoint.
    pub async fn connect_with_config(
        url: impl Into<String>,
        config: HandlerConfig,
    ) -> Result<(Self, Handler)> {
        let mut debug_ws_url = url.into();
        let retries = config.connection_retries;

        if debug_ws_url.starts_with("http") {
            let version_url = if debug_ws_url.ends_with("/json/version")
                || debug_ws_url.ends_with("/json/version/")
            {
                debug_ws_url.to_owned()
            } else {
                format!(
                    "{}{}json/version",
                    &debug_ws_url,
                    if debug_ws_url.ends_with('/') { "" } else { "/" }
                )
            };

            let mut discovered = false;

            for attempt in 0..=retries {
                let retry = || async {
                    if attempt < retries {
                        let backoff_ms = 50u64 * 3u64.saturating_pow(attempt);
                        tokio::time::sleep(Duration::from_millis(backoff_ms)).await;
                    }
                };

                match REQUEST_CLIENT.get(&version_url).send().await {
                    Ok(req) => match req.bytes().await {
                        Ok(b) => {
                            match crate::serde_json::from_slice::<Box<BrowserConnection>>(&b) {
                                Ok(connection)
                                    if !connection.web_socket_debugger_url.is_empty() =>
                                {
                                    debug_ws_url = connection.web_socket_debugger_url;
                                    discovered = true;
                                    break;
                                }
                                _ => {
                                    // JSON parse failed or webSocketDebuggerUrl was empty — retry
                                    retry().await;
                                }
                            }
                        }
                        Err(_) => {
                            retry().await;
                        }
                    },
                    Err(_) => {
                        retry().await;
                    }
                }
            }

            if !discovered {
                return Err(CdpError::NoResponse);
            }
        }

        let conn =
            Connection::<CdpEventMessage>::connect_with_retries(&debug_ws_url, retries).await?;

        let (tx, rx) = channel(config.channel_capacity);

        let handler_config = BrowserConfig {
            ignore_https_errors: config.ignore_https_errors,
            viewport: config.viewport.clone(),
            request_timeout: config.request_timeout,
            request_intercept: config.request_intercept,
            cache_enabled: config.cache_enabled,
            ignore_visuals: config.ignore_visuals,
            ignore_stylesheets: config.ignore_stylesheets,
            ignore_javascript: config.ignore_javascript,
            ignore_analytics: config.ignore_analytics,
            ignore_prefetch: config.ignore_prefetch,
            ignore_ads: config.ignore_ads,
            extra_headers: config.extra_headers.clone(),
            only_html: config.only_html,
            service_worker_enabled: config.service_worker_enabled,
            intercept_manager: config.intercept_manager,
            max_bytes_allowed: config.max_bytes_allowed,
            whitelist_patterns: config.whitelist_patterns.clone(),
            blacklist_patterns: config.blacklist_patterns.clone(),
            ..Default::default()
        };

        let fut = Handler::new(conn, rx, config);
        let browser_context = fut.default_browser_context().clone();

        let browser = Self {
            sender: tx,
            config: Some(handler_config),
            child: None,
            debug_ws_url,
            browser_context,
        };

        Ok((browser, fut))
    }

    /// Launches a new instance of `chromium` in the background and attaches to
    /// its debug web socket.
    ///
    /// This fails when no chromium executable could be detected.
    ///
    /// This fails if no web socket url could be detected from the child
    /// processes stderr for more than the configured `launch_timeout`
    /// (20 seconds by default).
    pub async fn launch(mut config: BrowserConfig) -> Result<(Self, Handler)> {
        // Canonalize paths to reduce issues with sandboxing
        config.executable = utils::canonicalize_except_snap(config.executable).await?;

        // Launch a new chromium instance
        let mut child = config.launch()?;

        /// Faillible initialization to run once the child process is created.
        ///
        /// All faillible calls must be executed inside this function. This ensures that all
        /// errors are caught and that the child process is properly cleaned-up.
        async fn with_child(
            config: &BrowserConfig,
            child: &mut Child,
        ) -> Result<(String, Connection<CdpEventMessage>)> {
            let dur = config.launch_timeout;
            let timeout_fut = Box::pin(tokio::time::sleep(dur));

            // extract the ws:
            let debug_ws_url = ws_url_from_output(child, timeout_fut).await?;
            let conn = Connection::<CdpEventMessage>::connect_with_retries(
                &debug_ws_url,
                config.connection_retries,
            )
            .await?;
            Ok((debug_ws_url, conn))
        }

        let (debug_ws_url, conn) = match with_child(&config, &mut child).await {
            Ok(conn) => conn,
            Err(e) => {
                // An initialization error occurred, clean up the process
                if let Ok(Some(_)) = child.try_wait() {
                    // already exited, do nothing, may happen if the browser crashed
                } else {
                    // the process is still alive, kill it and wait for exit (avoid zombie processes)
                    let _ = child.kill().await;
                    let _ = child.wait().await;
                }
                return Err(e);
            }
        };

        // Only infaillible calls are allowed after this point to avoid clean-up issues with the
        // child process.

        let (tx, rx) = channel(config.channel_capacity);

        let handler_config = HandlerConfig {
            ignore_https_errors: config.ignore_https_errors,
            viewport: config.viewport.clone(),
            context_ids: Vec::new(),
            request_timeout: config.request_timeout,
            request_intercept: config.request_intercept,
            cache_enabled: config.cache_enabled,
            ignore_visuals: config.ignore_visuals,
            ignore_stylesheets: config.ignore_stylesheets,
            ignore_javascript: config.ignore_javascript,
            ignore_analytics: config.ignore_analytics,
            ignore_prefetch: config.ignore_prefetch,
            ignore_ads: config.ignore_ads,
            extra_headers: config.extra_headers.clone(),
            only_html: config.only_html,
            service_worker_enabled: config.service_worker_enabled,
            created_first_target: false,
            intercept_manager: config.intercept_manager,
            max_bytes_allowed: config.max_bytes_allowed,
            whitelist_patterns: config.whitelist_patterns.clone(),
            blacklist_patterns: config.blacklist_patterns.clone(),
            #[cfg(feature = "adblock")]
            adblock_filter_rules: config.adblock_filter_rules.clone(),
            channel_capacity: config.channel_capacity,
            connection_retries: config.connection_retries,
        };

        let fut = Handler::new(conn, rx, handler_config);
        let browser_context = fut.default_browser_context().clone();

        let browser = Self {
            sender: tx,
            config: Some(config),
            child: Some(child),
            debug_ws_url,
            browser_context,
        };

        Ok((browser, fut))
    }

    /// Request to fetch all existing browser targets.
    ///
    /// By default, only targets launched after the browser connection are tracked
    /// when connecting to a existing browser instance with the devtools websocket url
    /// This function fetches existing targets on the browser and adds them as pages internally
    ///
    /// The pages are not guaranteed to be ready as soon as the function returns
    /// You should wait a few millis if you need to use a page
    /// Returns [TargetInfo]
    pub async fn fetch_targets(&mut self) -> Result<Vec<TargetInfo>> {
        let (tx, rx) = oneshot_channel();

        self.sender.send(HandlerMessage::FetchTargets(tx)).await?;

        rx.await?
    }

    /// Request for the browser to close completely.
    ///
    /// If the browser was spawned by [`Browser::launch`], it is recommended to wait for the
    /// spawned instance exit, to avoid "zombie" processes ([`Browser::wait`],
    /// [`Browser::wait_sync`], [`Browser::try_wait`]).
    /// [`Browser::drop`] waits automatically if needed.
    pub async fn close(&self) -> Result<CloseReturns> {
        let (tx, rx) = oneshot_channel();

        self.sender.send(HandlerMessage::CloseBrowser(tx)).await?;

        rx.await?
    }

    /// Asynchronously wait for the spawned chromium instance to exit completely.
    ///
    /// The instance is spawned by [`Browser::launch`]. `wait` is usually called after
    /// [`Browser::close`]. You can call this explicitly to collect the process and avoid
    /// "zombie" processes.
    ///
    /// This call has no effect if this [`Browser`] did not spawn any chromium instance (e.g.
    /// connected to an existing browser through [`Browser::connect`])
    pub async fn wait(&mut self) -> io::Result<Option<ExitStatus>> {
        if let Some(child) = self.child.as_mut() {
            Ok(Some(child.wait().await?))
        } else {
            Ok(None)
        }
    }

    /// If the spawned chromium instance has completely exited, wait for it.
    ///
    /// The instance is spawned by [`Browser::launch`]. `try_wait` is usually called after
    /// [`Browser::close`]. You can call this explicitly to collect the process and avoid
    /// "zombie" processes.
    ///
    /// This call has no effect if this [`Browser`] did not spawn any chromium instance (e.g.
    /// connected to an existing browser through [`Browser::connect`])
    pub fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
        if let Some(child) = self.child.as_mut() {
            child.try_wait()
        } else {
            Ok(None)
        }
    }

    /// Get the spawned chromium instance
    ///
    /// The instance is spawned by [`Browser::launch`]. The result is a [`async_process::Child`]
    /// value. It acts as a compat wrapper for an `async-std` or `tokio` child process.
    ///
    /// You may use [`async_process::Child::as_mut_inner`] to retrieve the concrete implementation
    /// for the selected runtime.
    ///
    /// This call has no effect if this [`Browser`] did not spawn any chromium instance (e.g.
    /// connected to an existing browser through [`Browser::connect`])
    pub fn get_mut_child(&mut self) -> Option<&mut Child> {
        self.child.as_mut()
    }

    /// Has a browser instance launched on system.
    pub fn has_child(&self) -> bool {
        self.child.is_some()
    }

    /// Forcibly kill the spawned chromium instance
    ///
    /// The instance is spawned by [`Browser::launch`]. `kill` will automatically wait for the child
    /// process to exit to avoid "zombie" processes.
    ///
    /// This method is provided to help if the browser does not close by itself. You should prefer
    /// to use [`Browser::close`].
    ///
    /// This call has no effect if this [`Browser`] did not spawn any chromium instance (e.g.
    /// connected to an existing browser through [`Browser::connect`])
    pub async fn kill(&mut self) -> Option<io::Result<()>> {
        match self.child.as_mut() {
            Some(child) => Some(child.kill().await),
            None => None,
        }
    }

    /// If not launched as incognito this creates a new incognito browser
    /// context. After that this browser exists within the incognito session.
    /// New pages created while being in incognito mode will also run in the
    /// incognito context. Incognito contexts won't share cookies/cache with
    /// other browser contexts.
    pub async fn start_incognito_context(&mut self) -> Result<&mut Self> {
        if !self.is_incognito_configured() {
            let browser_context_id = self
                .create_browser_context(CreateBrowserContextParams::default())
                .await?;
            self.browser_context = BrowserContext::from(browser_context_id);
            self.sender
                .send(HandlerMessage::InsertContext(self.browser_context.clone()))
                .await?;
        }

        Ok(self)
    }

    /// If a incognito session was created with
    /// `Browser::start_incognito_context` this disposes this context.
    ///
    /// # Note This will also dispose all pages that were running within the
    /// incognito context.
    pub async fn quit_incognito_context_base(
        &self,
        browser_context_id: BrowserContextId,
    ) -> Result<&Self> {
        self.dispose_browser_context(browser_context_id.clone())
            .await?;
        self.sender
            .send(HandlerMessage::DisposeContext(BrowserContext::from(
                browser_context_id,
            )))
            .await?;
        Ok(self)
    }

    /// If a incognito session was created with
    /// `Browser::start_incognito_context` this disposes this context.
    ///
    /// # Note This will also dispose all pages that were running within the
    /// incognito context.
    pub async fn quit_incognito_context(&mut self) -> Result<&mut Self> {
        if let Some(id) = self.browser_context.take() {
            let _ = self.quit_incognito_context_base(id).await;
        }
        Ok(self)
    }

    /// Whether incognito mode was configured from the start
    fn is_incognito_configured(&self) -> bool {
        self.config
            .as_ref()
            .map(|c| c.incognito)
            .unwrap_or_default()
    }

    /// Returns the address of the websocket this browser is attached to
    pub fn websocket_address(&self) -> &String {
        &self.debug_ws_url
    }

    /// Whether the BrowserContext is incognito.
    pub fn is_incognito(&self) -> bool {
        self.is_incognito_configured() || self.browser_context.is_incognito()
    }

    /// The config of the spawned chromium instance if any.
    pub fn config(&self) -> Option<&BrowserConfig> {
        self.config.as_ref()
    }

    /// Create a new browser page
    pub async fn new_page(&self, params: impl Into<CreateTargetParams>) -> Result<Page> {
        let (tx, rx) = oneshot_channel();
        let mut params = params.into();

        if let Some(id) = self.browser_context.id() {
            if params.browser_context_id.is_none() {
                params.browser_context_id = Some(id.clone());
            }
        }

        let _ = self
            .sender
            .send(HandlerMessage::CreatePage(params, tx))
            .await;

        rx.await?
    }

    /// Version information about the browser
    pub async fn version(&self) -> Result<GetVersionReturns> {
        Ok(self.execute(GetVersionParams::default()).await?.result)
    }

    /// Returns the user agent of the browser
    pub async fn user_agent(&self) -> Result<String> {
        Ok(self.version().await?.user_agent)
    }

    /// Call a browser method.
    pub async fn execute<T: Command>(&self, cmd: T) -> Result<CommandResponse<T::Response>> {
        let (tx, rx) = oneshot_channel();
        let method = cmd.identifier();
        let msg = CommandMessage::new(cmd, tx)?;

        self.sender.send(HandlerMessage::Command(msg)).await?;
        let resp = rx.await??;
        to_command_response::<T>(resp, method)
    }

    /// Set permission settings for given embedding and embedded origins.
    /// [PermissionDescriptor](https://chromedevtools.github.io/devtools-protocol/tot/Browser/#type-PermissionDescriptor)
    /// [PermissionSetting](https://chromedevtools.github.io/devtools-protocol/tot/Browser/#type-PermissionSetting)
    pub async fn set_permission(
        &self,
        permission: PermissionDescriptor,
        setting: PermissionSetting,
        origin: Option<impl Into<String>>,
        embedded_origin: Option<impl Into<String>>,
        browser_context_id: Option<BrowserContextId>,
    ) -> Result<&Self> {
        self.execute(SetPermissionParams {
            permission,
            setting,
            origin: origin.map(Into::into),
            embedded_origin: embedded_origin.map(Into::into),
            browser_context_id: browser_context_id.or_else(|| self.browser_context.id.clone()),
        })
        .await?;
        Ok(self)
    }

    /// Convenience: set a permission for a single origin using the current browser context.
    pub async fn set_permission_for_origin(
        &self,
        origin: impl Into<String>,
        embedded_origin: Option<impl Into<String>>,
        permission: PermissionDescriptor,
        setting: PermissionSetting,
    ) -> Result<&Self> {
        self.set_permission(permission, setting, Some(origin), embedded_origin, None)
            .await
    }

    /// "Reset" a permission override by setting it back to Prompt.
    pub async fn reset_permission_for_origin(
        &self,
        origin: impl Into<String>,
        embedded_origin: Option<impl Into<String>>,
        permission: PermissionDescriptor,
    ) -> Result<&Self> {
        self.set_permission_for_origin(
            origin,
            embedded_origin,
            permission,
            PermissionSetting::Prompt,
        )
        .await
    }

    /// "Grant" all permissions.
    pub async fn grant_all_permission_for_origin(
        &self,
        origin: impl Into<String>,
        embedded_origin: Option<impl Into<String>>,
        permission: PermissionDescriptor,
    ) -> Result<&Self> {
        self.set_permission_for_origin(
            origin,
            embedded_origin,
            permission,
            PermissionSetting::Granted,
        )
        .await
    }

    /// "Deny" all permissions.
    pub async fn deny_all_permission_for_origin(
        &self,
        origin: impl Into<String>,
        embedded_origin: Option<impl Into<String>>,
        permission: PermissionDescriptor,
    ) -> Result<&Self> {
        self.set_permission_for_origin(
            origin,
            embedded_origin,
            permission,
            PermissionSetting::Denied,
        )
        .await
    }

    /// Return all of the pages of the browser
    pub async fn pages(&self) -> Result<Vec<Page>> {
        let (tx, rx) = oneshot_channel();
        self.sender.send(HandlerMessage::GetPages(tx)).await?;
        Ok(rx.await?)
    }

    /// Return page of given target_id
    pub async fn get_page(&self, target_id: TargetId) -> Result<Page> {
        let (tx, rx) = oneshot_channel();
        self.sender
            .send(HandlerMessage::GetPage(target_id, tx))
            .await?;
        rx.await?.ok_or(CdpError::NotFound)
    }

    /// Set listener for browser event
    pub async fn event_listener<T: IntoEventKind>(&self) -> Result<EventStream<T>> {
        let (tx, rx) = unbounded_channel();
        self.sender
            .send(HandlerMessage::AddEventListener(
                EventListenerRequest::new::<T>(tx),
            ))
            .await?;

        Ok(EventStream::new(rx))
    }

    /// Creates a new empty browser context.
    pub async fn create_browser_context(
        &mut self,
        params: CreateBrowserContextParams,
    ) -> Result<BrowserContextId> {
        let response = self.execute(params).await?;

        Ok(response.result.browser_context_id)
    }

    /// Returns all browser contexts created with Target.createBrowserContext method.
    pub async fn get_browser_contexts(
        &mut self,
        params: GetBrowserContextsParams,
    ) -> Result<GetBrowserContextsReturns> {
        let response = self.execute(params).await?;
        Ok(response.result)
    }

    /// Send a new empty browser context.
    pub async fn send_new_context(
        &mut self,
        browser_context_id: BrowserContextId,
    ) -> Result<&Self> {
        self.browser_context = BrowserContext::from(browser_context_id);
        self.sender
            .send(HandlerMessage::InsertContext(self.browser_context.clone()))
            .await?;
        Ok(self)
    }

    /// Deletes a browser context.
    pub async fn dispose_browser_context(
        &self,
        browser_context_id: impl Into<BrowserContextId>,
    ) -> Result<&Self> {
        self.execute(DisposeBrowserContextParams::new(browser_context_id))
            .await?;

        Ok(self)
    }

    /// Clears cookies.
    pub async fn clear_cookies(&self) -> Result<&Self> {
        self.execute(ClearCookiesParams::default()).await?;
        Ok(self)
    }

    /// Returns all browser cookies.
    pub async fn get_cookies(&self) -> Result<Vec<Cookie>> {
        let cmd = GetCookiesParams {
            browser_context_id: self.browser_context.id.clone(),
        };

        Ok(self.execute(cmd).await?.result.cookies)
    }

    /// Sets given cookies.
    pub async fn set_cookies(&self, mut cookies: Vec<CookieParam>) -> Result<&Self> {
        for cookie in &mut cookies {
            if let Some(url) = cookie.url.as_ref() {
                crate::page::validate_cookie_url(url)?;
            }
        }

        let mut cookies_param = SetCookiesParams::new(cookies);

        cookies_param.browser_context_id = self.browser_context.id.clone();

        self.execute(cookies_param).await?;
        Ok(self)
    }
}

impl Drop for Browser {
    fn drop(&mut self) {
        if let Some(child) = self.child.as_mut() {
            if let Ok(Some(_)) = child.try_wait() {
                // Already exited, do nothing. Usually occurs after using the method close or kill.
            } else {
                // We set the `kill_on_drop` property for the child process, so no need to explicitely
                // kill it here. It can't really be done anyway since the method is async.
                //
                // On Unix, the process will be reaped in the background by the runtime automatically
                // so it won't leave any resources locked. It is, however, a better practice for the user to
                // do it himself since the runtime doesn't provide garantees as to when the reap occurs, so we
                // warn him here.
                tracing::warn!("Browser was not closed manually, it will be killed automatically in the background");
            }
        }
    }
}

/// Resolve devtools WebSocket URL from the provided browser process
///
/// If an error occurs, it returns the browser's stderr output.
///
/// The URL resolution fails if:
/// - [`CdpError::LaunchTimeout`]: `timeout_fut` completes, this corresponds to a timeout
/// - [`CdpError::LaunchExit`]: the browser process exits (or is killed)
/// - [`CdpError::LaunchIo`]: an input/output error occurs when await the process exit or reading
///   the browser's stderr: end of stream, invalid UTF-8, other
async fn ws_url_from_output(
    child_process: &mut Child,
    timeout_fut: impl Future<Output = ()> + Unpin,
) -> Result<String> {
    use tokio::io::AsyncBufReadExt;
    let stderr = match child_process.stderr.take() {
        Some(stderr) => stderr,
        None => {
            return Err(CdpError::LaunchIo(
                io::Error::new(io::ErrorKind::NotFound, "browser process has no stderr"),
                BrowserStderr::new(Vec::new()),
            ));
        }
    };
    let mut stderr_bytes = Vec::<u8>::new();
    let mut buf = tokio::io::BufReader::new(stderr);
    let mut timeout_fut = timeout_fut;
    loop {
        tokio::select! {
            _ = &mut timeout_fut => return Err(CdpError::LaunchTimeout(BrowserStderr::new(stderr_bytes))),
            exit_status = child_process.wait() => {
                return Err(match exit_status {
                    Err(e) => CdpError::LaunchIo(e, BrowserStderr::new(stderr_bytes)),
                    Ok(exit_status) => CdpError::LaunchExit(exit_status, BrowserStderr::new(stderr_bytes)),
                })
            },
            read_res = buf.read_until(b'\n', &mut stderr_bytes) => {
                match read_res {
                    Err(e) => return Err(CdpError::LaunchIo(e, BrowserStderr::new(stderr_bytes))),
                    Ok(byte_count) => {
                        if byte_count == 0 {
                            let e = io::Error::new(io::ErrorKind::UnexpectedEof, "unexpected end of stream");
                            return Err(CdpError::LaunchIo(e, BrowserStderr::new(stderr_bytes)));
                        }
                        let start_offset = stderr_bytes.len() - byte_count;
                        let new_bytes = &stderr_bytes[start_offset..];
                        match std::str::from_utf8(new_bytes) {
                            Err(_) => {
                                let e = io::Error::new(io::ErrorKind::InvalidData, "stream did not contain valid UTF-8");
                                return Err(CdpError::LaunchIo(e, BrowserStderr::new(stderr_bytes)));
                            }
                            Ok(line) => {
                                if let Some((_, ws)) = line.rsplit_once("listening on ") {
                                    if ws.starts_with("ws") && ws.contains("devtools/browser") {
                                        return Ok(ws.trim().to_string());
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}

#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum HeadlessMode {
    /// The "headful" mode.
    False,
    /// The old headless mode.
    #[default]
    True,
    /// The new headless mode. See also: https://developer.chrome.com/docs/chromium/new-headless
    New,
}

#[derive(Debug, Clone, Default)]
pub struct BrowserConfig {
    /// Determines whether to run headless version of the browser. Defaults to
    /// true.
    headless: HeadlessMode,
    /// Determines whether to run the browser with a sandbox.
    sandbox: bool,
    /// Launch the browser with a specific window width and height.
    window_size: Option<(u32, u32)>,
    /// Launch the browser with a specific debugging port.
    port: u16,
    /// Path for Chrome or Chromium.
    ///
    /// If unspecified, the create will try to automatically detect a suitable
    /// binary.
    executable: std::path::PathBuf,

    /// A list of Chrome extensions to load.
    ///
    /// An extension should be a path to a folder containing the extension code.
    /// CRX files cannot be used directly and must be first extracted.
    ///
    /// Note that Chrome does not support loading extensions in headless-mode.
    /// See https://bugs.chromium.org/p/chromium/issues/detail?id=706008#c5
    extensions: Vec<String>,

    /// Environment variables to set for the Chromium process.
    /// Passes value through to std::process::Command::envs.
    pub process_envs: Option<HashMap<String, String>>,

    /// Data dir for user data
    pub user_data_dir: Option<PathBuf>,

    /// Whether to launch the `Browser` in incognito mode.
    incognito: bool,

    /// Timeout duration for `Browser::launch`.
    launch_timeout: Duration,

    /// Ignore https errors, default is true.
    ignore_https_errors: bool,
    pub viewport: Option<Viewport>,
    /// The duration after a request with no response should time out.
    request_timeout: Duration,

    /// Additional command line arguments to pass to the browser instance.
    args: Vec<String>,

    /// Whether to disable DEFAULT_ARGS or not, default is false.
    disable_default_args: bool,

    /// Whether to enable request interception.
    pub request_intercept: bool,

    /// Whether to enable cache.
    pub cache_enabled: bool,
    /// Whether to enable or disable Service Workers.
    /// Disabling may reduce background network activity and caching effects.
    pub service_worker_enabled: bool,
    /// Whether to ignore image/visual requests during interception.
    /// Can reduce bandwidth and speed up crawling when visuals are unnecessary.
    pub ignore_visuals: bool,
    /// Whether to ignore stylesheet (CSS) requests during interception.
    /// Useful for content-only crawls.
    pub ignore_stylesheets: bool,
    /// Whether to ignore JavaScript requests during interception.
    /// This still allows critical framework bundles to pass when applicable.
    pub ignore_javascript: bool,
    /// Whether to ignore analytics/telemetry requests during interception.
    pub ignore_analytics: bool,
    /// Ignore prefetch request.
    pub ignore_prefetch: bool,
    /// Whether to ignore ad network requests during interception.
    pub ignore_ads: bool,
    /// Extra headers.
    pub extra_headers: Option<std::collections::HashMap<String, String>>,
    /// Only html
    pub only_html: bool,
    /// The interception intercept manager.
    pub intercept_manager: NetworkInterceptManager,
    /// The max bytes to receive.
    pub max_bytes_allowed: Option<u64>,
    /// Whitelist patterns to allow through the network.
    pub whitelist_patterns: Option<Vec<String>>,
    /// Blacklist patterns to block through the network.
    pub blacklist_patterns: Option<Vec<String>>,
    /// Extra ABP/uBO filter rules to load into the adblock engine (requires `adblock` feature).
    /// These are merged with the built-in `ADBLOCK_PATTERNS` for richer blocking
    /// (e.g. EasyList / EasyPrivacy content).
    #[cfg(feature = "adblock")]
    pub adblock_filter_rules: Option<Vec<String>>,
    /// Capacity of the channel between browser handle and handler.
    /// Defaults to 1000.
    pub channel_capacity: usize,
    /// Number of WebSocket connection retry attempts with exponential backoff.
    /// Defaults to 4.
    pub connection_retries: u32,
}

#[derive(Debug, Clone)]
pub struct BrowserConfigBuilder {
    /// Headless mode configuration for the browser.
    headless: HeadlessMode,
    /// Whether to run the browser with a sandbox.
    sandbox: bool,
    /// Optional initial browser window size `(width, height)`.
    window_size: Option<(u32, u32)>,
    /// DevTools debugging port to bind to.
    port: u16,
    /// Optional explicit path to the Chrome/Chromium executable.
    /// If `None`, auto-detection may be attempted based on `executation_detection`.
    executable: Option<PathBuf>,
    /// Controls auto-detection behavior for finding a Chrome/Chromium binary.
    executation_detection: DetectionOptions,
    /// List of unpacked extensions (directories) to load at startup.
    extensions: Vec<String>,
    /// Environment variables to set on the spawned Chromium process.
    process_envs: Option<HashMap<String, String>>,
    /// User data directory to persist browser state, or `None` for ephemeral.
    user_data_dir: Option<PathBuf>,
    /// Whether to start the browser in incognito (off-the-record) mode.
    incognito: bool,
    /// Maximum time to wait for the browser to launch and become ready.
    launch_timeout: Duration,
    /// Whether to ignore HTTPS/TLS errors during navigation and requests.
    ignore_https_errors: bool,
    /// Default page viewport configuration applied on startup.
    viewport: Option<Viewport>,
    /// Timeout for individual network requests without response progress.
    request_timeout: Duration,
    /// Additional command-line flags passed directly to the browser process.
    args: Vec<String>,
    /// Disable the default argument set and use only the provided `args`.
    disable_default_args: bool,
    /// Enable Network.requestInterception for request filtering/handling.
    request_intercept: bool,
    /// Enable the browser cache for navigations and subresources.
    cache_enabled: bool,
    /// Enable/disable Service Workers.
    service_worker_enabled: bool,
    /// Drop image/visual requests when interception is enabled.
    ignore_visuals: bool,
    /// Drop ad network requests when interception is enabled.
    ignore_ads: bool,
    /// Drop JavaScript requests when interception is enabled.
    ignore_javascript: bool,
    /// Drop stylesheet (CSS) requests when interception is enabled.
    ignore_stylesheets: bool,
    /// Ignore prefetch domains.
    ignore_prefetch: bool,
    /// Drop analytics/telemetry requests when interception is enabled.
    ignore_analytics: bool,
    /// If `true`, limit fetching to HTML documents.
    only_html: bool,
    /// Extra HTTP headers to include with every request.
    extra_headers: Option<std::collections::HashMap<String, String>>,
    /// Network interception manager used to configure filtering behavior.
    intercept_manager: NetworkInterceptManager,
    /// Optional upper bound on bytes that may be received (per session/run).
    max_bytes_allowed: Option<u64>,
    /// Whitelist patterns to allow through the network.
    whitelist_patterns: Option<Vec<String>>,
    /// Blacklist patterns to block through the network.
    blacklist_patterns: Option<Vec<String>>,
    /// Extra ABP/uBO filter rules for the adblock engine.
    #[cfg(feature = "adblock")]
    adblock_filter_rules: Option<Vec<String>>,
    /// Capacity of the channel between browser handle and handler.
    channel_capacity: usize,
    /// Number of WebSocket connection retry attempts.
    connection_retries: u32,
}

impl BrowserConfig {
    /// Browser builder default config.
    pub fn builder() -> BrowserConfigBuilder {
        BrowserConfigBuilder::default()
    }

    /// Launch with the executable path.
    pub fn with_executable(path: impl AsRef<Path>) -> Self {
        // SAFETY: build() only fails when no executable is provided,
        // but we always provide one via chrome_executable().
        Self::builder().chrome_executable(path).build().unwrap()
    }
}

impl Default for BrowserConfigBuilder {
    fn default() -> Self {
        Self {
            headless: HeadlessMode::True,
            sandbox: true,
            window_size: None,
            port: 0,
            executable: None,
            executation_detection: DetectionOptions::default(),
            extensions: Vec::new(),
            process_envs: None,
            user_data_dir: None,
            incognito: false,
            launch_timeout: Duration::from_millis(LAUNCH_TIMEOUT),
            ignore_https_errors: true,
            viewport: Some(Default::default()),
            request_timeout: Duration::from_millis(REQUEST_TIMEOUT),
            args: Vec::new(),
            disable_default_args: false,
            request_intercept: false,
            cache_enabled: true,
            ignore_visuals: false,
            ignore_ads: false,
            ignore_javascript: false,
            ignore_analytics: false,
            ignore_stylesheets: false,
            ignore_prefetch: true,
            only_html: false,
            extra_headers: Default::default(),
            service_worker_enabled: true,
            intercept_manager: NetworkInterceptManager::Unknown,
            max_bytes_allowed: None,
            whitelist_patterns: None,
            blacklist_patterns: None,
            #[cfg(feature = "adblock")]
            adblock_filter_rules: None,
            channel_capacity: 4096,
            connection_retries: crate::conn::DEFAULT_CONNECTION_RETRIES,
        }
    }
}

impl BrowserConfigBuilder {
    /// Configure window size.
    pub fn window_size(mut self, width: u32, height: u32) -> Self {
        self.window_size = Some((width, height));
        self
    }
    /// Configure sandboxing.
    pub fn no_sandbox(mut self) -> Self {
        self.sandbox = false;
        self
    }
    /// Configure the launch to start non headless.
    pub fn with_head(mut self) -> Self {
        self.headless = HeadlessMode::False;
        self
    }
    /// Configure the launch with the new headless mode.
    pub fn new_headless_mode(mut self) -> Self {
        self.headless = HeadlessMode::New;
        self
    }
    /// Configure the launch with headless.
    pub fn headless_mode(mut self, mode: HeadlessMode) -> Self {
        self.headless = mode;
        self
    }
    /// Configure the launch in incognito.
    pub fn incognito(mut self) -> Self {
        self.incognito = true;
        self
    }

    pub fn respect_https_errors(mut self) -> Self {
        self.ignore_https_errors = false;
        self
    }

    pub fn port(mut self, port: u16) -> Self {
        self.port = port;
        self
    }

    pub fn with_max_bytes_allowed(mut self, max_bytes_allowed: Option<u64>) -> Self {
        self.max_bytes_allowed = max_bytes_allowed;
        self
    }

    pub fn launch_timeout(mut self, timeout: Duration) -> Self {
        self.launch_timeout = timeout;
        self
    }

    pub fn request_timeout(mut self, timeout: Duration) -> Self {
        self.request_timeout = timeout;
        self
    }

    /// Configures the viewport of the browser, which defaults to `800x600`.
    /// `None` disables viewport emulation (i.e., it uses the browsers default
    /// configuration, which fills the available space. This is similar to what
    /// Playwright does when you provide `null` as the value of its `viewport`
    /// option).
    pub fn viewport(mut self, viewport: impl Into<Option<Viewport>>) -> Self {
        self.viewport = viewport.into();
        self
    }

    pub fn user_data_dir(mut self, data_dir: impl AsRef<Path>) -> Self {
        self.user_data_dir = Some(data_dir.as_ref().to_path_buf());
        self
    }

    pub fn chrome_executable(mut self, path: impl AsRef<Path>) -> Self {
        self.executable = Some(path.as_ref().to_path_buf());
        self
    }

    pub fn chrome_detection(mut self, options: DetectionOptions) -> Self {
        self.executation_detection = options;
        self
    }

    pub fn extension(mut self, extension: impl Into<String>) -> Self {
        self.extensions.push(extension.into());
        self
    }

    pub fn extensions<I, S>(mut self, extensions: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: Into<String>,
    {
        for ext in extensions {
            self.extensions.push(ext.into());
        }
        self
    }

    pub fn env(mut self, key: impl Into<String>, val: impl Into<String>) -> Self {
        self.process_envs
            .get_or_insert(HashMap::new())
            .insert(key.into(), val.into());
        self
    }

    pub fn envs<I, K, V>(mut self, envs: I) -> Self
    where
        I: IntoIterator<Item = (K, V)>,
        K: Into<String>,
        V: Into<String>,
    {
        self.process_envs
            .get_or_insert(HashMap::new())
            .extend(envs.into_iter().map(|(k, v)| (k.into(), v.into())));
        self
    }

    pub fn arg(mut self, arg: impl Into<String>) -> Self {
        self.args.push(arg.into());
        self
    }

    pub fn args<I, S>(mut self, args: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: Into<String>,
    {
        for arg in args {
            self.args.push(arg.into());
        }
        self
    }

    pub fn disable_default_args(mut self) -> Self {
        self.disable_default_args = true;
        self
    }

    pub fn enable_request_intercept(mut self) -> Self {
        self.request_intercept = true;
        self
    }

    pub fn disable_request_intercept(mut self) -> Self {
        self.request_intercept = false;
        self
    }

    pub fn enable_cache(mut self) -> Self {
        self.cache_enabled = true;
        self
    }

    pub fn disable_cache(mut self) -> Self {
        self.cache_enabled = false;
        self
    }

    /// Set service worker enabled.
    pub fn set_service_worker_enabled(mut self, bypass: bool) -> Self {
        self.service_worker_enabled = bypass;
        self
    }

    /// Set extra request headers.
    pub fn set_extra_headers(
        mut self,
        headers: Option<std::collections::HashMap<String, String>>,
    ) -> Self {
        self.extra_headers = headers;
        self
    }

    /// Set whitelist patterns to allow through network interception allowing.
    pub fn set_whitelist_patterns(mut self, whitelist_patterns: Option<Vec<String>>) -> Self {
        self.whitelist_patterns = whitelist_patterns;
        self
    }

    /// Set blacklist patterns to block through network interception.
    pub fn set_blacklist_patterns(mut self, blacklist_patterns: Option<Vec<String>>) -> Self {
        self.blacklist_patterns = blacklist_patterns;
        self
    }

    /// Set extra ABP/uBO filter rules for the adblock engine.
    /// Pass EasyList/EasyPrivacy content lines for richer blocking coverage.
    #[cfg(feature = "adblock")]
    pub fn set_adblock_filter_rules(mut self, rules: Vec<String>) -> Self {
        self.adblock_filter_rules = Some(rules);
        self
    }

    /// Set the capacity of the channel between browser handle and handler.
    /// Defaults to 1000.
    pub fn channel_capacity(mut self, capacity: usize) -> Self {
        self.channel_capacity = capacity;
        self
    }

    /// Set the number of WebSocket connection retry attempts with exponential backoff.
    /// Defaults to 4. Set to 0 for a single attempt with no retries.
    pub fn connection_retries(mut self, retries: u32) -> Self {
        self.connection_retries = retries;
        self
    }

    /// Build the browser.
    pub fn build(self) -> std::result::Result<BrowserConfig, String> {
        let executable = if let Some(e) = self.executable {
            e
        } else {
            detection::default_executable(self.executation_detection)?
        };

        Ok(BrowserConfig {
            headless: self.headless,
            sandbox: self.sandbox,
            window_size: self.window_size,
            port: self.port,
            executable,
            extensions: self.extensions,
            process_envs: self.process_envs,
            user_data_dir: self.user_data_dir,
            incognito: self.incognito,
            launch_timeout: self.launch_timeout,
            ignore_https_errors: self.ignore_https_errors,
            viewport: self.viewport,
            request_timeout: self.request_timeout,
            args: self.args,
            disable_default_args: self.disable_default_args,
            request_intercept: self.request_intercept,
            cache_enabled: self.cache_enabled,
            ignore_visuals: self.ignore_visuals,
            ignore_ads: self.ignore_ads,
            ignore_javascript: self.ignore_javascript,
            ignore_analytics: self.ignore_analytics,
            ignore_stylesheets: self.ignore_stylesheets,
            ignore_prefetch: self.ignore_prefetch,
            extra_headers: self.extra_headers,
            only_html: self.only_html,
            intercept_manager: self.intercept_manager,
            service_worker_enabled: self.service_worker_enabled,
            max_bytes_allowed: self.max_bytes_allowed,
            whitelist_patterns: self.whitelist_patterns,
            blacklist_patterns: self.blacklist_patterns,
            #[cfg(feature = "adblock")]
            adblock_filter_rules: self.adblock_filter_rules,
            channel_capacity: self.channel_capacity,
            connection_retries: self.connection_retries,
        })
    }
}

impl BrowserConfig {
    pub fn launch(&self) -> io::Result<Child> {
        let mut cmd = async_process::Command::new(&self.executable);

        if self.disable_default_args {
            cmd.args(&self.args);
        } else {
            cmd.args(DEFAULT_ARGS).args(&self.args);
        }

        if !self
            .args
            .iter()
            .any(|arg| arg.contains("--remote-debugging-port="))
        {
            cmd.arg(format!("--remote-debugging-port={}", self.port));
        }

        cmd.args(
            self.extensions
                .iter()
                .map(|e| format!("--load-extension={e}")),
        );

        if let Some(ref user_data) = self.user_data_dir {
            cmd.arg(format!("--user-data-dir={}", user_data.display()));
        } else {
            // If the user did not specify a data directory, this would default to the systems default
            // data directory. In most cases, we would rather have a fresh instance of Chromium. Specify
            // a temp dir just for chromiumoxide instead.
            cmd.arg(format!(
                "--user-data-dir={}",
                std::env::temp_dir().join("chromiumoxide-runner").display()
            ));
        }

        if let Some((width, height)) = self.window_size {
            cmd.arg(format!("--window-size={width},{height}"));
        }

        if !self.sandbox {
            cmd.args(["--no-sandbox", "--disable-setuid-sandbox"]);
        }

        match self.headless {
            HeadlessMode::False => (),
            HeadlessMode::True => {
                cmd.args(["--headless", "--hide-scrollbars", "--mute-audio"]);
            }
            HeadlessMode::New => {
                cmd.args(["--headless=new", "--hide-scrollbars", "--mute-audio"]);
            }
        }

        if self.incognito {
            cmd.arg("--incognito");
        }

        if let Some(ref envs) = self.process_envs {
            cmd.envs(envs);
        }
        cmd.stderr(Stdio::piped()).spawn()
    }
}

/// Returns the path to Chrome's executable.
///
/// If the `CHROME` environment variable is set, `default_executable` will
/// use it as the default path. Otherwise, the filenames `google-chrome-stable`
/// `chromium`, `chromium-browser`, `chrome` and `chrome-browser` are
/// searched for in standard places. If that fails,
/// `/Applications/Google Chrome.app/...` (on MacOS) or the registry (on
/// Windows) is consulted. If all of the above fail, an error is returned.
#[deprecated(note = "Use detection::default_executable instead")]
pub fn default_executable() -> Result<std::path::PathBuf, String> {
    let options = DetectionOptions {
        msedge: false,
        unstable: false,
    };
    detection::default_executable(options)
}

/// These are passed to the Chrome binary by default.
/// Via https://github.com/puppeteer/puppeteer/blob/4846b8723cf20d3551c0d755df394cc5e0c82a94/src/node/Launcher.ts#L157
static DEFAULT_ARGS: [&str; 26] = [
    "--disable-background-networking",
    "--enable-features=NetworkService,NetworkServiceInProcess",
    "--disable-background-timer-throttling",
    "--disable-backgrounding-occluded-windows",
    "--disable-breakpad",
    "--disable-client-side-phishing-detection",
    "--disable-component-extensions-with-background-pages",
    "--disable-default-apps",
    "--disable-dev-shm-usage",
    "--disable-extensions",
    "--disable-features=TranslateUI",
    "--disable-hang-monitor",
    "--disable-ipc-flooding-protection",
    "--disable-popup-blocking",
    "--disable-prompt-on-repost",
    "--disable-renderer-backgrounding",
    "--disable-sync",
    "--force-color-profile=srgb",
    "--metrics-recording-only",
    "--no-first-run",
    "--enable-automation",
    "--password-store=basic",
    "--use-mock-keychain",
    "--enable-blink-features=IdleDetection",
    "--lang=en_US",
    "--disable-blink-features=AutomationControlled",
];