use crate::engines::browser_downloader::{BrowserDownloadConfig, BrowserDownloadManager};
use crate::engines::client::playwright_pool::{
get_global_pool, BrowserPool, BrowserPoolConfig, PooledPage,
};
use crate::engines::client::ua_metadata::{build_ua_fingerprint, cdp_accept_language};
use crate::engines::engine_client::{
EngineError, InternalCapturedResponse, InternalPageAction, InternalScrapeRequest,
InternalScrapeResponse, InternalScreenshotConfig, ScraperEngine,
};
use crate::infrastructure::services::config_service::BrowserConfigTrait;
use crate::utils::proxy::{redact_proxy_url, validate_proxy_url};
use crate::utils::ua_pool::{UaPool, UaProfile};
use async_trait::async_trait;
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
use chromiumoxide::cdp::browser_protocol::emulation::{
SetDeviceMetricsOverrideParams, SetUserAgentOverrideParams,
};
use chromiumoxide::cdp::browser_protocol::network::{
DisableParams as NetworkDisableParams, EnableParams as NetworkEnableParams,
EventLoadingFinished, EventResponseReceived, GetResponseBodyParams, ResourceType,
};
use chromiumoxide::cdp::browser_protocol::page::CaptureScreenshotFormat;
use chromiumoxide::{Browser, BrowserConfig};
use futures::StreamExt;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use crate::engines::xhr_capture::{
process_body, url_matches_capture_patterns, BodyOutcome, PendingXhr, RecordOutcome,
XhrCaptureBuffer, XhrCaptureConfig,
};
mod interception;
mod session;
const DEFAULT_PLAYWRIGHT_MRT_SECONDS: u64 = 30;
const WAIT_TIMEOUT_CAP: Duration = Duration::from_secs(10);
struct InterceptGuard(Option<tokio::task::JoinHandle<()>>);
impl InterceptGuard {
fn new() -> Self {
Self(None)
}
fn set(&mut self, handle: tokio::task::JoinHandle<()>) {
if let Some(old) = self.0.take() {
old.abort();
}
self.0 = Some(handle);
}
}
impl Drop for InterceptGuard {
fn drop(&mut self) {
if let Some(handle) = self.0.take() {
handle.abort();
}
}
}
#[derive(Clone, Debug, Default)]
pub struct PlaywrightContext {
pub remote_debugging_url: Option<String>,
pub proxy_url: Option<String>,
pub test_mode: bool,
}
impl PlaywrightContext {
pub fn new(
remote_debugging_url: Option<String>,
proxy_url: Option<String>,
test_mode: bool,
) -> Self {
Self {
remote_debugging_url,
proxy_url,
test_mode,
}
}
}
#[async_trait]
pub trait BrowserManagerTrait: Send + Sync {
async fn get_browser(&self) -> Result<Arc<Browser>, EngineError>;
async fn cleanup(&self);
fn reset(&self);
async fn check_health(&self, browser: &Browser) -> bool;
}
pub struct PlaywrightBrowserManagerComponent {
config: Arc<dyn BrowserConfigTrait>,
browser: Arc<Mutex<Option<Arc<Browser>>>>,
download_manager: Arc<BrowserDownloadManager>,
handler_handle: Arc<Mutex<Option<tokio::task::JoinHandle<()>>>>,
}
impl PlaywrightBrowserManagerComponent {
pub fn new(config: Arc<dyn BrowserConfigTrait>) -> Self {
Self::with_download_config(config, BrowserDownloadConfig::default())
}
pub fn with_download_config(
config: Arc<dyn BrowserConfigTrait>,
download_config: BrowserDownloadConfig,
) -> Self {
Self {
config,
browser: Arc::new(Mutex::new(None)),
download_manager: Arc::new(BrowserDownloadManager::new(download_config)),
handler_handle: Arc::new(Mutex::new(None)),
}
}
}
#[async_trait]
impl BrowserManagerTrait for PlaywrightBrowserManagerComponent {
async fn get_browser(&self) -> Result<Arc<Browser>, EngineError> {
self.get_browser_with_recovery(3).await
}
async fn cleanup(&self) {
let handler_to_abort = {
let mut handler_guard = self
.handler_handle
.lock()
.expect("handler_handle mutex poisoned");
handler_guard.take()
};
if let Some(handle) = handler_to_abort {
handle.abort();
let _ = tokio::time::timeout(std::time::Duration::from_secs(2), async {
let _ = handle.await;
})
.await;
log::debug!("CDP handler task aborted and awaited");
}
let mut guard = match self.browser.lock() {
Ok(g) => g,
Err(e) => {
log::error!("Browser mutex poisoned during cleanup: {}", e);
return;
}
};
if let Some(browser) = guard.take() {
log::info!("Closing browser instance");
drop(browser);
}
}
fn reset(&self) {
let mut guard = match self.browser.lock() {
Ok(g) => g,
Err(e) => {
log::error!("Browser mutex poisoned during reset: {}", e);
return;
}
};
*guard = None;
}
async fn check_health(&self, browser: &Browser) -> bool {
match browser.new_page("about:blank").await {
Ok(page) => {
let _ = page.close().await;
true
}
Err(_) => false,
}
}
}
impl PlaywrightBrowserManagerComponent {
async fn get_browser_with_recovery(
&self,
max_attempts: u32,
) -> Result<Arc<Browser>, EngineError> {
let mut attempts = 0;
loop {
attempts += 1;
match self.get_or_init_browser().await {
Ok(browser) => return Ok(browser),
Err(e) if attempts < max_attempts => {
log::warn!(
"Browser initialization attempt {} failed: {}, retrying...",
attempts,
e
);
self.cleanup().await;
tokio::time::sleep(Duration::from_secs(1)).await;
}
Err(e) => return Err(e),
}
}
}
async fn get_or_init_browser(&self) -> Result<Arc<Browser>, EngineError> {
if let Some(browser) = self.cached_browser_if_healthy().await {
return Ok(browser);
}
let (browser, handler) = self.connect_or_launch_browser().await?;
self.install_handler(handler);
let browser = Arc::new(browser);
self.store_browser(&browser);
Ok(browser)
}
async fn cached_browser_if_healthy(&self) -> Option<Arc<Browser>> {
let test_mode = self.config.is_test_mode();
let browser_to_check = {
let browser_guard = self.browser.lock().expect("Browser mutex poisoned");
browser_guard.as_ref().map(Arc::clone)
};
if let Some(browser) = browser_to_check {
if self.check_health(&browser).await && !test_mode {
return Some(browser);
}
}
None
}
fn store_browser(&self, browser: &Arc<Browser>) {
let mut browser_guard = self.browser.lock().expect("Browser mutex poisoned");
*browser_guard = Some(Arc::clone(browser));
}
async fn connect_or_launch_browser(
&self,
) -> Result<(Browser, chromiumoxide::Handler), EngineError> {
let remote_debugging_url = self.config.get_remote_debugging_url();
let proxy_url = self.config.get_proxy_url();
let (browser, handler) = if let Some(ref url) = remote_debugging_url {
log::info!("Connecting to remote Chrome instance at: {}", url);
Browser::connect(url).await.map_err(|e| {
EngineError::BrowserError(format!("Failed to connect to remote Chrome: {}", e))
})?
} else {
let browser_path = self.download_browser_if_needed().await?;
Self::launch_local_browser(browser_path, proxy_url).await?
};
Ok((browser, handler))
}
async fn launch_local_browser(
browser_path: Option<PathBuf>,
proxy_url: Option<String>,
) -> Result<(Browser, chromiumoxide::Handler), EngineError> {
let mut builder = BrowserConfig::builder()
.no_sandbox()
.request_timeout(Duration::from_secs(30));
if let Some(ref path) = browser_path {
log::info!("Using browser at: {:?}", path);
builder = builder.chrome_executable(path);
}
builder = builder.arg("--disable-gpu").arg("--disable-dev-shm-usage");
if let Some(ref proxy) = proxy_url {
const ALLOWED_PROXY_SCHEMES: &[&str] = &["http", "https", "socks5", "socks4"];
let validated = validate_proxy_url(proxy, ALLOWED_PROXY_SCHEMES)
.map_err(|e| EngineError::Other(format!("Invalid proxy URL: {}", e)))?;
log::info!(
"Using proxy for Playwright: {}",
redact_proxy_url(&validated)
);
builder = builder.arg("--proxy-server").arg(validated);
}
let (browser, handler) = Browser::launch(
builder
.build()
.map_err(|e| EngineError::Other(e.to_string()))?,
)
.await
.map_err(|e| EngineError::Other(e.to_string()))?;
Ok((browser, handler))
}
fn install_handler(&self, mut handler: chromiumoxide::Handler) {
let handler_task = tokio::spawn(async move {
while let Some(h) = handler.next().await {
if let Err(e) = h {
log::debug!("Browser handler event error (continuing): {:?}", e);
}
}
});
{
let mut guard = self
.handler_handle
.lock()
.expect("handler_handle mutex poisoned");
if let Some(old) = guard.take() {
old.abort();
}
*guard = Some(handler_task);
}
}
async fn download_browser_if_needed(&self) -> Result<Option<PathBuf>, EngineError> {
if let Some(path) = crate::engines::browser_downloader::browser_path_from_env() {
return Ok(Some(path));
}
if let Some(path) = crate::engines::browser_downloader::find_system_browser().await {
log::info!("Using system browser");
return Ok(Some(path));
}
if self.download_manager.is_browser_downloaded().await {
let path = crate::engines::browser_downloader::get_browser_executable_path(
self.download_manager.get_cache_dir(),
);
log::info!("Using downloaded browser: {:?}", path);
return Ok(Some(path));
}
log::info!("No browser detected, starting auto-download...");
match self.download_manager.download_browser().await {
Ok(path) => {
log::info!("Browser downloaded successfully: {:?}", path);
Ok(Some(path))
}
Err(e) => {
Err(EngineError::Unsupported(format!(
"playwright 引擎不可用:未找到系统 Chrome/Chromium,内置自动下载不可用({e})。 处置:安装 Chrome/Chromium,或设置环境变量 CRAWLRS__ENGINES__PLAYWRIGHT__BROWSER_PATH 指向浏览器可执行文件。"
)))
}
}
}
}
pub async fn check_browser_health(browser: &Browser) -> bool {
match browser.new_page("about:blank").await {
Ok(page) => {
let _ = page.close().await;
true
}
Err(_) => false,
}
}
pub struct PlaywrightEngine {
pool: Option<BrowserPool>,
ua_pool: UaPool,
mrt: Duration,
xhr_capture: Option<XhrCaptureConfig>,
}
impl PlaywrightEngine {
pub fn new() -> Self {
Self::with_mrt(Duration::from_secs(DEFAULT_PLAYWRIGHT_MRT_SECONDS))
}
#[must_use]
pub fn with_mrt(mrt: Duration) -> Self {
Self {
pool: None,
ua_pool: UaPool::default(),
mrt,
xhr_capture: None,
}
}
pub fn with_pool(pool: BrowserPool) -> Self {
Self::with_pool_and_mrt(pool, Duration::from_secs(DEFAULT_PLAYWRIGHT_MRT_SECONDS))
}
#[must_use]
pub fn with_pool_and_mrt(pool: BrowserPool, mrt: Duration) -> Self {
Self {
pool: Some(pool),
ua_pool: UaPool::default(),
mrt,
xhr_capture: None,
}
}
#[must_use]
pub fn with_xhr_capture(mut self, config: XhrCaptureConfig) -> Self {
self.xhr_capture = Some(config);
self
}
#[must_use]
pub fn xhr_capture_config(&self) -> Option<&XhrCaptureConfig> {
self.xhr_capture.as_ref()
}
#[must_use]
pub fn ua_pool(&self) -> &UaPool {
&self.ua_pool
}
#[must_use]
pub fn mrt(&self) -> Duration {
self.mrt
}
fn get_or_init_pool(&self) -> BrowserPool {
if let Some(pool) = &self.pool {
return pool.clone();
}
if let Some(pool) = get_global_pool() {
return pool.clone();
}
let config = BrowserPoolConfig::docker_safe_args();
let browser_config = Arc::new(
crate::infrastructure::services::config_service::BrowserConfigComponent::default(),
);
BrowserPool::new(config, browser_config)
}
}
impl Default for PlaywrightEngine {
fn default() -> Self {
Self::new()
}
}
pub(crate) async fn apply_browser_fingerprint(
page: &chromiumoxide::Page,
ua_pool: &UaPool,
mobile: bool,
) -> Result<(), EngineError> {
let profile = ua_pool.pick_chromium(mobile);
apply_ua_and_locale(page, profile).await?;
apply_viewport_and_stealth(page, profile).await?;
Ok(())
}
async fn apply_ua_and_locale(
page: &chromiumoxide::Page,
profile: &UaProfile,
) -> Result<(), EngineError> {
let fingerprint = build_ua_fingerprint(
profile.ua,
profile.sec_ch_ua,
profile.platform,
profile.mobile,
);
let mut ua_override = SetUserAgentOverrideParams::builder()
.user_agent(profile.ua)
.accept_language(cdp_accept_language(profile.accept_language));
if let Some(navigator_platform) = fingerprint.navigator_platform {
ua_override = ua_override.platform(navigator_platform);
}
if let Some(metadata) = fingerprint.metadata {
ua_override = ua_override.user_agent_metadata(metadata);
}
let ua_override = ua_override.build().map_err(|e| {
EngineError::BrowserError(format!("Failed to build user agent override: {}", e))
})?;
page.execute(ua_override).await.map_err(|e| {
EngineError::BrowserError(format!("Failed to set user agent override: {}", e))
})?;
if let Some(locale) = profile.accept_language.split(',').next() {
if let Err(e) = page
.execute(
chromiumoxide::cdp::browser_protocol::emulation::SetLocaleOverrideParams::builder()
.locale(locale.trim())
.build(),
)
.await
{
log::warn!("Failed to set locale override: {}", e);
}
}
Ok(())
}
async fn apply_viewport_and_stealth(
page: &chromiumoxide::Page,
profile: &UaProfile,
) -> Result<(), EngineError> {
let viewport_params = SetDeviceMetricsOverrideParams::new(
profile.viewport.0 as i64,
profile.viewport.1 as i64,
1.0,
profile.mobile,
);
page.execute(viewport_params)
.await
.map_err(|e| EngineError::BrowserError(format!("Failed to set viewport: {}", e)))?;
let stealth_injector = crate::engines::js_inject::JsInjector::stealth();
let already_installed: bool = match page
.evaluate("() => window.__crawlrs_stealth_installed === true")
.await
{
Ok(v) => v.into_value::<bool>().unwrap_or(false),
Err(_) => false,
};
if !already_installed {
if let Err(e) = stealth_injector.apply_before_document(page).await {
log::warn!(
"{}",
crate::i18n::tr_log_args(
"playwright-stealth-inject-failed",
&[("error", e.to_string())],
)
);
}
}
Ok(())
}
async fn acquire_fingerprinted_page(
pool: &BrowserPool,
ua_pool: &UaPool,
request: &InternalScrapeRequest,
) -> Result<(PooledPage, chromiumoxide::page::Page), EngineError> {
let pooled_page = pool.acquire_page().await?;
let page: chromiumoxide::page::Page = pooled_page.page().clone();
apply_browser_fingerprint(&page, ua_pool, request.mobile).await?;
Ok((pooled_page, page))
}
async fn navigate_and_wait(
page: &chromiumoxide::page::Page,
request: &InternalScrapeRequest,
) -> Result<(), EngineError> {
goto_and_inject_cleanup(page, &request.url).await?;
warn_if_bot_detection_page(page).await?;
run_page_actions(page, &request.actions).await?;
let wait_strategy = request.wait_for.clone().unwrap_or_default();
let wait_timeout = request.timeout.min(WAIT_TIMEOUT_CAP);
wait_strategy.wait(page, wait_timeout).await?;
Ok(())
}
async fn goto_and_inject_cleanup(
page: &chromiumoxide::page::Page,
url: &str,
) -> Result<(), EngineError> {
page.goto(url)
.await
.map_err(|e| EngineError::BrowserError(e.to_string()))?;
let cleanup_injector = crate::engines::js_inject::JsInjector::cleanup();
if let Err(e) = cleanup_injector
.apply(page, crate::engines::js_inject::InjectPhase::AfterLoad)
.await
{
log::warn!("Cleanup injection failed (best-effort, continue): {}", e);
}
Ok(())
}
async fn warn_if_bot_detection_page(page: &chromiumoxide::page::Page) -> Result<(), EngineError> {
let content: String = page
.content()
.await
.map_err(|e| EngineError::BrowserError(e.to_string()))?;
if is_bot_detection_page(&content) {
log::warn!("Detected bot detection page from Google");
}
Ok(())
}
async fn run_page_actions(
page: &chromiumoxide::page::Page,
actions: &[InternalPageAction],
) -> Result<(), EngineError> {
for action in actions {
execute_page_action(page, action).await?;
}
Ok(())
}
fn is_bot_detection_page(content: &str) -> bool {
content.contains("如果您在几秒钟内没有被重定向")
|| content.contains("Having trouble accessing Google")
|| content.contains("enablejs")
}
async fn execute_page_action(
page: &chromiumoxide::page::Page,
action: &InternalPageAction,
) -> Result<(), EngineError> {
match action {
InternalPageAction::Wait { milliseconds } => {
tokio::time::sleep(Duration::from_millis(*milliseconds)).await;
}
InternalPageAction::Click { selector } => {
let element: chromiumoxide::element::Element =
page.find_element(selector).await.map_err(|e| {
EngineError::BrowserError(format!("Click failed, element not found: {}", e))
})?;
element
.click()
.await
.map_err(|e| EngineError::BrowserError(format!("Click failed: {}", e)))?;
}
InternalPageAction::Scroll { direction } => {
let script = match direction.as_str() {
"down" => "window.scrollBy(0, window.innerHeight);",
"up" => "window.scrollBy(0, -window.innerHeight);",
"bottom" => "window.scrollTo(0, document.body.scrollHeight);",
"top" => "window.scrollTo(0, 0);",
_ => "window.scrollBy(0, window.innerHeight);",
};
let _: chromiumoxide::js::EvaluationResult = page
.evaluate(script)
.await
.map_err(|e| EngineError::BrowserError(format!("Scroll failed: {}", e)))?;
}
InternalPageAction::Screenshot { full_page: _ } => {
}
InternalPageAction::Input { selector, text } => {
let element: chromiumoxide::element::Element =
page.find_element(selector).await.map_err(|e| {
EngineError::BrowserError(format!("Input failed, element not found: {}", e))
})?;
element
.type_str(text)
.await
.map_err(|e| EngineError::BrowserError(format!("Input failed: {}", e)))?;
}
}
Ok(())
}
async fn capture_screenshot(
page: &chromiumoxide::page::Page,
request: &InternalScrapeRequest,
) -> Result<Option<String>, EngineError> {
if !request.needs_screenshot {
return Ok(None);
}
let config = request
.screenshot_config
.clone()
.unwrap_or(InternalScreenshotConfig {
full_page: true,
selector: None,
quality: Some(80),
format: Some("jpeg".to_string()),
});
let format = screenshot_format(config.format.as_deref());
let screenshot_bytes = if let Some(selector) = &config.selector {
capture_element_screenshot(page, selector, format).await?
} else {
capture_page_screenshot(page, &config, format).await?
};
Ok(Some(BASE64.encode(screenshot_bytes)))
}
fn screenshot_format(format: Option<&str>) -> CaptureScreenshotFormat {
match format {
Some("png") => CaptureScreenshotFormat::Png,
_ => CaptureScreenshotFormat::Jpeg,
}
}
async fn capture_element_screenshot(
page: &chromiumoxide::page::Page,
selector: &str,
format: CaptureScreenshotFormat,
) -> Result<Vec<u8>, EngineError> {
let element: chromiumoxide::element::Element = page
.find_element(selector)
.await
.map_err(|e| EngineError::BrowserError(format!("Element not found: {}", e)))?;
element
.screenshot(format)
.await
.map_err(|e| EngineError::BrowserError(format!("Element screenshot failed: {}", e)))
}
async fn capture_page_screenshot(
page: &chromiumoxide::page::Page,
config: &InternalScreenshotConfig,
format: CaptureScreenshotFormat,
) -> Result<Vec<u8>, EngineError> {
let params = chromiumoxide::page::ScreenshotParams::builder()
.format(format)
.quality(config.quality.unwrap_or(80) as i64)
.full_page(config.full_page)
.build();
page.screenshot(params)
.await
.map_err(|e| EngineError::BrowserError(format!("Page screenshot failed: {}", e)))
}
async fn collect_and_build_response(
page: &chromiumoxide::page::Page,
request: &InternalScrapeRequest,
xhr_session: Option<&XhrCaptureSession>,
pooled_page: PooledPage,
start: Instant,
) -> Result<InternalScrapeResponse, EngineError> {
let captured_responses = match xhr_session {
Some(session) => session.collect(page).await,
None => Vec::new(),
};
let final_url: String = page
.url()
.await
.ok()
.flatten()
.unwrap_or_else(|| request.url.clone());
let content_type = page
.evaluate(
r#"
() => document.contentType || document.querySelector('meta[http-equiv="content-type"]')?.getAttribute('content') || 'text/html'
"#,
)
.await
.map_err(|e| EngineError::BrowserError(e.to_string()))?
.into_value::<String>()
.unwrap_or_else(|_| "text/html".to_string())
.split(';')
.next()
.unwrap_or("text/html")
.trim()
.to_string();
let status_code = 200;
let content: String = page
.content()
.await
.map_err(|e| EngineError::BrowserError(e.to_string()))?;
let response_headers = {
let mut headers = std::collections::HashMap::with_capacity(2);
headers.insert("Content-Type".to_string(), content_type.clone());
headers
};
let screenshot = capture_screenshot(page, request).await?;
drop(pooled_page);
Ok(InternalScrapeResponse {
js_degraded: false,
status_code,
content,
screenshot,
content_type: "text/html".to_string(),
headers: response_headers,
response_time_ms: start.elapsed().as_millis() as u64,
final_url: Some(final_url),
captured_responses,
})
}
#[async_trait]
impl ScraperEngine for PlaywrightEngine {
async fn scrape(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
session::validate_scrape_request(request).await?;
let start = Instant::now();
let timeout_duration = request.timeout;
let pool = self.get_or_init_pool();
tokio::time::timeout(timeout_duration, self.scrape_session(request, &pool, start))
.await
.map_err(|_| EngineError::Timeout(timeout_duration))?
}
fn support_score(&self, request: &InternalScrapeRequest) -> u8 {
if request.method != crate::engines::engine_client::HttpMethod::Get {
return 0;
}
if request.needs_tls_fingerprint {
return 0;
}
if request.needs_js || request.needs_screenshot {
return 100;
}
0 }
fn name(&self) -> &'static str {
"playwright"
}
fn supports_tls_fingerprint(&self) -> bool {
false
}
fn max_response_time(&self) -> Duration {
self.mrt
}
}
struct XhrCaptureSession {
buffer: Arc<Mutex<XhrCaptureBuffer>>,
_guard: InterceptGuard,
}
impl XhrCaptureSession {
async fn start(page: &chromiumoxide::page::Page, config: &XhrCaptureConfig) -> Option<Self> {
let (responses, finished) = Self::subscribe_xhr_events(page).await?;
let buffer = Arc::new(Mutex::new(XhrCaptureBuffer::new(config.max_responses)));
let handle = spawn_xhr_capture_task(
responses,
finished,
Arc::clone(&buffer),
config.url_patterns.clone(),
config.max_bytes,
page,
);
let mut guard = InterceptGuard::new();
guard.set(handle);
Some(Self {
buffer,
_guard: guard,
})
}
async fn subscribe_xhr_events(
page: &chromiumoxide::page::Page,
) -> Option<(
chromiumoxide::listeners::EventStream<EventResponseReceived>,
chromiumoxide::listeners::EventStream<EventLoadingFinished>,
)> {
let setup = async {
page.execute(NetworkEnableParams::default()).await?;
let responses = page.event_listener::<EventResponseReceived>().await?;
let finished = page.event_listener::<EventLoadingFinished>().await?;
Ok::<_, chromiumoxide::error::CdpError>((responses, finished))
};
match setup.await {
Ok(streams) => Some(streams),
Err(e) => {
let _ = page.execute(NetworkDisableParams::default()).await;
log::warn!(
"{}",
crate::i18n::tr_log_args(
"playwright-xhr-setup-failed",
&[("error", e.to_string())],
)
);
None
}
}
}
async fn collect(&self, page: &chromiumoxide::page::Page) -> Vec<InternalCapturedResponse> {
let (captured, stats, pending_left) = {
let mut buf = self.buffer.lock().expect("xhr capture buffer poisoned");
(buf.drain_captured(), buf.stats(), buf.pending_len())
};
if pending_left > 0 {
log::debug!(
"{}",
crate::i18n::tr_log_args(
"playwright-xhr-unfinished-at-collect",
&[("count", (pending_left as i64).to_string())],
)
);
}
if stats.overflow_skipped > 0
|| stats.oversized_dropped > 0
|| stats.decode_failed > 0
|| stats.fetch_failed > 0
{
log::warn!(
"{}",
crate::i18n::tr_log_args(
"playwright-xhr-dropped",
&[
("oversize", (stats.oversized_dropped as i64).to_string(),),
("decode_failed", (stats.decode_failed as i64).to_string(),),
("fetch_failed", (stats.fetch_failed as i64).to_string(),),
("overflow", (stats.overflow_skipped as i64).to_string(),),
],
)
);
}
let _ = page.execute(NetworkDisableParams::default()).await;
captured
}
}
fn spawn_xhr_capture_task(
mut responses: chromiumoxide::listeners::EventStream<EventResponseReceived>,
mut finished: chromiumoxide::listeners::EventStream<EventLoadingFinished>,
buf_clone: Arc<Mutex<XhrCaptureBuffer>>,
patterns: Vec<String>,
max_bytes: u64,
page: &chromiumoxide::page::Page,
) -> tokio::task::JoinHandle<()> {
let page_clone = page.clone();
tokio::spawn(async move {
loop {
tokio::select! {
Some(event) = responses.next() => {
if !matches!(event.r#type, ResourceType::Xhr | ResourceType::Fetch) {
continue;
}
let url = event.response.url.clone();
if !url_matches_capture_patterns(&url, &patterns) {
continue;
}
if crate::engines::validators::is_internal_url(&url) {
log::debug!(
"{}",
crate::i18n::tr_log_args(
"playwright-xhr-internal-blocked",
&[("url", url.as_str().to_string())],
)
);
continue;
}
let entry = PendingXhr {
request_id: event.request_id.as_ref().to_string(),
url,
status: u16::try_from(event.response.status).unwrap_or(0),
mime_type: event.response.mime_type.clone(),
};
let early_entry = entry.clone();
let outcome = buf_clone
.lock()
.expect("xhr capture buffer poisoned")
.record(entry);
if outcome == RecordOutcome::AlreadyFinished {
fetch_and_store(&page_clone, &buf_clone, &early_entry, max_bytes)
.await;
}
}
Some(event) = finished.next() => {
let request_id = event.request_id.as_ref().to_string();
let pending = {
let mut buf = buf_clone.lock().expect("xhr capture buffer poisoned");
match buf.take_pending(&request_id) {
Some(entry) => Some(entry),
None => {
buf.mark_finished_early(&request_id);
None
}
}
};
if let Some(entry) = pending {
fetch_and_store(&page_clone, &buf_clone, &entry, max_bytes).await;
}
}
else => break,
}
}
})
}
async fn fetch_and_store(
page: &chromiumoxide::page::Page,
buffer: &Mutex<XhrCaptureBuffer>,
entry: &PendingXhr,
max_bytes: u64,
) {
let result = page
.execute(GetResponseBodyParams::new(entry.request_id.clone()))
.await;
let mut buf = buffer.lock().expect("xhr capture buffer poisoned");
match result {
Ok(resp) => {
let returns = resp.result;
match process_body(returns.body, returns.base64_encoded, max_bytes) {
BodyOutcome::Kept(body) => {
buf.push_captured(InternalCapturedResponse {
url: entry.url.clone(),
status: entry.status,
mime_type: entry.mime_type.clone(),
body,
});
}
BodyOutcome::Oversize => {
log::debug!(
"{}",
crate::i18n::tr_log_args(
"playwright-xhr-body-oversize",
&[("url", entry.url.as_str().to_string())],
)
);
buf.note_oversize();
}
BodyOutcome::DecodeFailed => {
log::debug!(
"{}",
crate::i18n::tr_log_args(
"playwright-xhr-body-decode-failed",
&[("url", entry.url.as_str().to_string())],
)
);
buf.note_decode_failed();
}
}
}
Err(e) => {
log::debug!(
"{}",
crate::i18n::tr_log_args(
"playwright-xhr-fetch-body-failed",
&[
("url", entry.url.as_str().to_string()),
("error", e.to_string()),
],
)
);
buf.note_fetch_failed();
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::engines::engine_client::{InternalScrapeRequest, WaitFor};
use crate::engines::intercept::BLOCK_REASON;
use chromiumoxide::cdp::browser_protocol::fetch::{
ContinueRequestParams, EnableParams, EventRequestPaused, FailRequestParams,
};
use std::collections::HashMap;
use std::time::Duration;
#[test]
fn test_support_score() {
let engine = PlaywrightEngine::new();
let request_js = InternalScrapeRequest {
url: "http://example.com".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: true,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: vec![],
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
assert_eq!(engine.support_score(&request_js), 100);
let request_screenshot = InternalScrapeRequest {
url: "http://example.com".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: true,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: vec![],
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
assert_eq!(engine.support_score(&request_screenshot), 100);
let request_basic = InternalScrapeRequest {
url: "http://example.com".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: vec![],
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
assert_eq!(engine.support_score(&request_basic), 0);
}
#[test]
fn test_playwright_engine_has_non_empty_ua_pool() {
let engine = PlaywrightEngine::new();
let pool = engine.ua_pool();
assert!(
pool.count(false) >= 20,
"desktop pool must have >= 20 profiles"
);
assert!(
pool.count(true) >= 20,
"mobile pool must have >= 20 profiles"
);
}
#[test]
fn test_playwright_engine_pick_ua_mobile_vs_desktop() {
let engine = PlaywrightEngine::new();
let pool = engine.ua_pool();
for _ in 0..10 {
let p = pool.pick(true);
assert!(p.mobile, "pick(true) must return mobile profile");
}
for _ in 0..10 {
let p = pool.pick(false);
assert!(!p.mobile, "pick(false) must return desktop profile");
}
}
#[test]
fn test_playwright_engine_pick_ua_returns_varied_profiles() {
let engine = PlaywrightEngine::new();
let pool = engine.ua_pool();
let mut uas = std::collections::HashSet::new();
for _ in 0..50 {
uas.insert(pool.pick(false).ua);
}
assert!(
uas.len() >= 2,
"multiple picks should return varied UAs, got only {} unique in 50 picks",
uas.len()
);
}
#[test]
fn test_playwright_engine_viewport_matches_platform() {
let engine = PlaywrightEngine::new();
let pool = engine.ua_pool();
for p in pool.desktop.iter().chain(pool.mobile.iter()) {
match p.platform {
"iOS" => {
assert!(
(375..=1366).contains(&p.viewport.0),
"iOS viewport width {} out of range [375, 1366]: {}",
p.viewport.0,
p.ua
);
}
"Android" => {
assert!(
(360..=1280).contains(&p.viewport.0),
"Android viewport width {} out of range [360, 1280]: {}",
p.viewport.0,
p.ua
);
}
"Windows" | "macOS" | "Linux" => {
assert!(
p.viewport.0 >= 1024,
"Desktop viewport width {} < 1024: {}",
p.viewport.0,
p.ua
);
}
_ => {
panic!("unknown platform: {}", p.platform);
}
}
}
}
#[test]
fn test_playwright_engine_ua_not_fixed_mobile_string() {
let engine = PlaywrightEngine::new();
let pool = engine.ua_pool();
let fixed_mobile_ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 14_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Mobile/15E148 Safari/604.1";
let non_fixed_count = pool
.mobile
.iter()
.filter(|p| p.ua != fixed_mobile_ua)
.count();
assert!(
non_fixed_count >= 20,
"most mobile UAs should differ from the old fixed UA, got {} non-fixed",
non_fixed_count
);
}
#[test]
fn test_playwright_engine_with_pool_has_ua_pool() {
let config = BrowserPoolConfig::default();
let browser_config = Arc::new(
crate::infrastructure::services::config_service::BrowserConfigComponent::default(),
);
let pool = BrowserPool::new(config, browser_config);
let engine = PlaywrightEngine::with_pool(pool);
assert!(engine.ua_pool().count(false) >= 20);
assert!(engine.ua_pool().count(true) >= 20);
}
#[test]
fn test_playwright_engine_default_has_ua_pool() {
let engine = PlaywrightEngine::default();
assert!(engine.ua_pool().count(false) >= 20);
assert!(engine.ua_pool().count(true) >= 20);
}
#[test]
fn test_playwright_engine_pick_seeded_stable() {
let engine = PlaywrightEngine::new();
let pool = engine.ua_pool();
let p1 = pool.pick_seeded(42, true);
let p2 = pool.pick_seeded(42, true);
assert_eq!(p1.ua, p2.ua, "same seed must return same mobile profile");
assert_eq!(
p1.viewport, p2.viewport,
"same seed must return same viewport"
);
}
#[test]
fn test_playwright_engine_xhr_capture_disabled_by_default() {
assert!(PlaywrightEngine::new().xhr_capture_config().is_none());
assert!(PlaywrightEngine::default().xhr_capture_config().is_none());
}
#[test]
fn test_playwright_engine_xhr_capture_builder() {
let engine = PlaywrightEngine::new().with_xhr_capture(XhrCaptureConfig {
url_patterns: vec!["*/api/*".to_string()],
max_bytes: 1024,
max_responses: 7,
});
let config = engine
.xhr_capture_config()
.expect("capture config must be set");
assert_eq!(config.url_patterns, vec!["*/api/*".to_string()]);
assert_eq!(config.max_bytes, 1024);
assert_eq!(config.max_responses, 7);
}
fn spawn_mock_xhr_server() -> String {
use std::io::{Read, Write};
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind mock server");
let addr = listener.local_addr().expect("local addr");
std::thread::spawn(move || {
for stream in listener.incoming() {
let mut stream = match stream {
Ok(s) => s,
Err(_) => continue,
};
let mut buf = [0u8; 2048];
let _ = stream.read(&mut buf);
let req = String::from_utf8_lossy(&buf);
let (content_type, body): (&str, String) = if req.starts_with("GET /page_multi") {
(
"text/html; charset=utf-8",
"<html><head><title>mock</title></head><body><script>\
[1,2,3].forEach(function (i) { fetch('/api/data?i=' + i); });\
</script></body></html>"
.to_string(),
)
} else if req.starts_with("GET /page_mixed") {
(
"text/html; charset=utf-8",
"<html><head><title>mock</title></head><body><script>\
fetch('/api/blocked'); fetch('/api/data');\
</script></body></html>"
.to_string(),
)
} else if req.starts_with("GET /page") {
(
"text/html; charset=utf-8",
"<html><head><title>mock</title></head><body><script>\
fetch('/api/data').then(function (r) { return r.json(); }).then(function (d) { document.title = 'loaded'; });\
</script></body></html>"
.to_string(),
)
} else if req.starts_with("GET /api/data") {
(
"application/json",
"{\"ok\":true,\"msg\":\"hello xhr\"}".to_string(),
)
} else {
("application/json", "{\"blocked\":true}".to_string())
};
let resp = format!(
"HTTP/1.1 200 OK\r\nContent-Type: {}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
content_type,
body.len(),
body
);
let _ = stream.write_all(resp.as_bytes());
let _ = stream.flush();
}
});
format!("http://127-0-0-1.sslip.io:{}", addr.port())
}
static CHROME_E2E_LOCK: std::sync::OnceLock<tokio::sync::Mutex<()>> =
std::sync::OnceLock::new();
async fn launch_test_page() -> (chromiumoxide::Browser, chromiumoxide::page::Page) {
static PROFILE_COUNTER: std::sync::OnceLock<std::sync::atomic::AtomicU64> =
std::sync::OnceLock::new();
let profile_dir = std::env::temp_dir().join(format!(
"crawlrs-chromium-e2e-{}-{}",
std::process::id(),
PROFILE_COUNTER
.get_or_init(std::sync::atomic::AtomicU64::default)
.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
));
std::fs::create_dir_all(&profile_dir).expect("create e2e profile dir");
let config = chromiumoxide::BrowserConfig::builder()
.no_sandbox()
.arg("--disable-gpu")
.arg("--disable-dev-shm-usage")
.user_data_dir(&profile_dir)
.request_timeout(Duration::from_secs(30))
.build()
.expect("browser config");
let (browser, mut handler) = chromiumoxide::Browser::launch(config)
.await
.expect("browser launch");
tokio::spawn(async move {
while let Some(h) = handler.next().await {
if let Err(e) = h {
log::debug!(
"{}",
crate::i18n::tr_log_args(
"playwright-handler-event-error",
&[("error", format!("{e:?}"))],
)
);
}
}
});
let page = browser.new_page("about:blank").await.expect("new page");
(browser, page)
}
#[tokio::test]
#[ignore = "需要本机 Chrome:cargo test --features full --lib xhr_capture_e2e -- --ignored --test-threads=1"]
async fn xhr_capture_e2e_captures_fetch_response() {
let base = spawn_mock_xhr_server();
let _chrome_guard = CHROME_E2E_LOCK
.get_or_init(|| tokio::sync::Mutex::new(()))
.lock()
.await;
let (mut browser, page) = launch_test_page().await;
let config = XhrCaptureConfig {
url_patterns: vec![],
max_bytes: 262_144,
max_responses: 50,
};
let session = XhrCaptureSession::start(&page, &config)
.await
.expect("capture session must start");
page.goto(&format!("{}/page", base)).await.expect("goto");
WaitFor::default()
.wait(&page, Duration::from_secs(10))
.await
.expect("network idle wait");
let captured = session.collect(&page).await;
assert!(
!captured.is_empty(),
"capture_xhr=true must produce captured responses"
);
let data = captured
.iter()
.find(|c| c.url.ends_with("/api/data"))
.expect("/api/data must be captured");
assert_eq!(data.status, 200);
assert_eq!(data.mime_type, "application/json");
assert_eq!(data.body, "{\"ok\":true,\"msg\":\"hello xhr\"}");
assert!(
captured.iter().all(|c| !c.url.ends_with("/page")),
"document must not be captured as XHR, got: {:?}",
captured
);
browser.close().await.ok();
}
#[tokio::test]
#[ignore = "需要本机 Chrome:cargo test --features full --lib xhr_capture_e2e -- --ignored --test-threads=1"]
async fn xhr_capture_e2e_url_glob_filter_excludes_non_matching() {
let base = spawn_mock_xhr_server();
let _chrome_guard = CHROME_E2E_LOCK
.get_or_init(|| tokio::sync::Mutex::new(()))
.lock()
.await;
let (mut browser, page) = launch_test_page().await;
let config = XhrCaptureConfig {
url_patterns: vec!["*/nomatch*".to_string()],
max_bytes: 262_144,
max_responses: 50,
};
let session = XhrCaptureSession::start(&page, &config)
.await
.expect("capture session must start");
page.goto(&format!("{}/page", base)).await.expect("goto");
WaitFor::default()
.wait(&page, Duration::from_secs(10))
.await
.expect("network idle wait");
let captured = session.collect(&page).await;
assert!(
captured.is_empty(),
"non-matching XHR must not be captured, got: {:?}",
captured
);
browser.close().await.ok();
}
#[tokio::test]
#[ignore = "需要本机 Chrome:cargo test --features full --lib xhr_capture_e2e -- --ignored --test-threads=1"]
async fn xhr_capture_e2e_max_responses_caps_collection() {
let base = spawn_mock_xhr_server();
let _chrome_guard = CHROME_E2E_LOCK
.get_or_init(|| tokio::sync::Mutex::new(()))
.lock()
.await;
let (mut browser, page) = launch_test_page().await;
let config = XhrCaptureConfig {
url_patterns: vec![],
max_bytes: 262_144,
max_responses: 2,
};
let session = XhrCaptureSession::start(&page, &config)
.await
.expect("capture session must start");
page.goto(&format!("{}/page_multi", base))
.await
.expect("goto");
WaitFor::default()
.wait(&page, Duration::from_secs(10))
.await
.expect("network idle wait");
let captured = session.collect(&page).await;
assert_eq!(
captured.len(),
2,
"max_responses=2 must cap captures at 2, got: {:?}",
captured
);
assert!(captured.iter().all(|c| c.url.contains("/api/data?i=")));
browser.close().await.ok();
}
#[tokio::test]
#[ignore = "需要本机 Chrome:cargo test --features full --lib xhr_capture_e2e -- --ignored --test-threads=1"]
async fn xhr_capture_e2e_max_bytes_drops_oversize_body() {
let base = spawn_mock_xhr_server();
let _chrome_guard = CHROME_E2E_LOCK
.get_or_init(|| tokio::sync::Mutex::new(()))
.lock()
.await;
let (mut browser, page) = launch_test_page().await;
let config = XhrCaptureConfig {
url_patterns: vec![],
max_bytes: 4,
max_responses: 50,
};
let session = XhrCaptureSession::start(&page, &config)
.await
.expect("capture session must start");
page.goto(&format!("{}/page", base)).await.expect("goto");
WaitFor::default()
.wait(&page, Duration::from_secs(10))
.await
.expect("network idle wait");
let captured = session.collect(&page).await;
assert!(
captured.is_empty(),
"oversize body must be dropped, got: {:?}",
captured
);
browser.close().await.ok();
}
#[tokio::test]
#[ignore = "需要本机 Chrome:cargo test --features full --lib xhr_capture_e2e -- --ignored --test-threads=1"]
async fn xhr_capture_e2e_coexists_with_fetch_interception() {
let base = spawn_mock_xhr_server();
let _chrome_guard = CHROME_E2E_LOCK
.get_or_init(|| tokio::sync::Mutex::new(()))
.lock()
.await;
let (mut browser, page) = launch_test_page().await;
page.execute(EnableParams::default())
.await
.expect("enable fetch");
let mut paused = page
.event_listener::<EventRequestPaused>()
.await
.expect("subscribe EventRequestPaused");
let intercept_page = page.clone();
let intercept_handle = tokio::spawn(async move {
while let Some(event) = paused.next().await {
let url = event.request.url.clone();
let request_id = event.request_id.clone();
if url.contains("/api/blocked") {
let _ = intercept_page
.execute(FailRequestParams::new(request_id, BLOCK_REASON))
.await;
} else {
let _ = intercept_page
.execute(ContinueRequestParams::new(request_id))
.await;
}
}
});
let config = XhrCaptureConfig {
url_patterns: vec![],
max_bytes: 262_144,
max_responses: 50,
};
let session = XhrCaptureSession::start(&page, &config)
.await
.expect("capture session must start");
page.goto(&format!("{}/page_mixed", base))
.await
.expect("goto");
WaitFor::default()
.wait(&page, Duration::from_secs(10))
.await
.expect("network idle wait");
let captured = session.collect(&page).await;
assert!(
captured.iter().any(|c| c.url.ends_with("/api/data")),
"continued XHR must be captured, got: {:?}",
captured
);
assert!(
captured.iter().all(|c| !c.url.contains("/api/blocked")),
"failed request must not appear in capture, got: {:?}",
captured
);
intercept_handle.abort();
browser.close().await.ok();
}
}