use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
use chromiumoxide::Browser;
use futures::StreamExt;
use std::time::Duration;
use crate::engines::engine_client::{
EngineError, HttpMethod, InternalScrapeRequest, InternalScrapeResponse, ScraperEngine, WaitFor,
};
const CDP_VERSION_PATH: &str = "/json/version";
const NETWORK_IDLE_SETTLE_MS: u64 = 500;
const WAIT_POLL_INTERVAL_MS: u64 = 100;
#[derive(Debug, Clone)]
pub struct MoliEngineConfig {
pub endpoint: String,
pub connect_timeout: Duration,
pub scrape_timeout: Duration,
}
impl MoliEngineConfig {
pub fn from_settings(settings: &crate::config::engines::MoliEngineSettings) -> Self {
Self {
endpoint: settings.endpoint.clone(),
connect_timeout: Duration::from_millis(settings.connect_timeout_ms),
scrape_timeout: Duration::from_millis(settings.scrape_timeout_ms),
}
}
}
pub struct MoliEngine {
config: MoliEngineConfig,
http_client: reqwest::Client,
}
impl MoliEngine {
pub fn new(config: MoliEngineConfig) -> Self {
Self {
config,
http_client: reqwest::Client::builder()
.timeout(Duration::from_secs(5))
.build()
.unwrap_or_default(),
}
}
async fn discover_ws_url(&self) -> Result<String, EngineError> {
let url = format!(
"{}{}",
self.config.endpoint.trim_end_matches('/'),
CDP_VERSION_PATH
);
let resp = tokio::time::timeout(
self.config.connect_timeout,
self.http_client.get(&url).send(),
)
.await
.map_err(|_| {
let e = format!(
"moli endpoint discovery timed out after {:?}",
self.config.connect_timeout
);
log::error!(target: "crawlrs::engine::moli", "{}", e);
EngineError::Other(e)
})?
.map_err(|e| {
log::error!(target: "crawlrs::engine::moli", "moli endpoint unreachable ({}): {}", url, e);
EngineError::Other("moli endpoint unreachable".to_string())
})?;
let text = resp.text().await.map_err(|e| {
let e = format!("moli endpoint discovery body read failed: {}", e);
log::error!(target: "crawlrs::engine::moli", "{}", e);
EngineError::Other(e)
})?;
serde_json::from_str::<serde_json::Value>(&text)
.ok()
.and_then(|v| {
v.get("webSocketDebuggerUrl")
.and_then(|s| s.as_str())
.map(str::to_string)
})
.ok_or_else(|| {
log::error!(target: "crawlrs::engine::moli", "moli endpoint {} returned no webSocketDebuggerUrl", url);
EngineError::Other("moli endpoint returned no webSocketDebuggerUrl".to_string())
})
}
async fn scrape_session(
config: MoliEngineConfig,
http_client: reqwest::Client,
request: InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
let engine = MoliEngine {
config: config.clone(),
http_client,
};
let ws_url = engine.discover_ws_url().await?;
let (browser, mut handler) = Self::connect_browser(&config, ws_url).await?;
let handler_task = tokio::spawn(async move {
while let Some(h) = handler.next().await {
if h.is_err() {
break;
}
}
});
let res = Self::scrape_page(&engine, &browser, &config, &request).await;
drop(browser);
handler_task.abort();
res
}
async fn connect_browser(
config: &MoliEngineConfig,
ws_url: String,
) -> Result<(Browser, chromiumoxide::Handler), EngineError> {
tokio::time::timeout(config.connect_timeout, Browser::connect(ws_url))
.await
.map_err(|_| {
log::error!(target: "crawlrs::engine::moli", "moli CDP connect timed out");
EngineError::Other("moli CDP connect timed out".to_string())
})?
.map_err(|e| {
log::error!(target: "crawlrs::engine::moli", "moli CDP connect failed: {}", e);
EngineError::Other("moli CDP connect failed".to_string())
})
}
async fn scrape_page(
engine: &MoliEngine,
browser: &Browser,
config: &MoliEngineConfig,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
let page = browser.new_page("about:blank").await.map_err(|e| {
log::error!(target: "crawlrs::engine::moli", "moli new_page failed: {}", e);
EngineError::Other("moli new_page failed".to_string())
})?;
let outcome = Self::scrape_page_inner(engine, &page, config, request).await;
let _ = page.close().await;
outcome
}
async fn scrape_page_inner(
engine: &MoliEngine,
page: &chromiumoxide::Page,
config: &MoliEngineConfig,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
Self::navigate_and_wait(engine, page, config, request).await?;
let html = Self::read_document_html(page).await?;
validate_html(&html)?;
let screenshot = Self::capture_screenshot(page, request.needs_screenshot).await?;
Ok::<InternalScrapeResponse, EngineError>(InternalScrapeResponse {
status_code: 200,
content: html,
screenshot,
content_type: "text/html".to_string(),
headers: std::collections::HashMap::new(),
response_time_ms: 0,
final_url: None,
captured_responses: Vec::new(),
js_degraded: false,
})
}
async fn navigate_and_wait(
engine: &MoliEngine,
page: &chromiumoxide::Page,
config: &MoliEngineConfig,
request: &InternalScrapeRequest,
) -> Result<(), EngineError> {
tokio::time::timeout(config.scrape_timeout, page.goto(request.url.as_str()))
.await
.map_err(|_| {
log::error!(
target: "crawlrs::engine::moli",
"moli navigation to {} timed out", request.url
);
EngineError::Timeout(config.scrape_timeout)
})?
.map_err(|e| {
log::error!(
target: "crawlrs::engine::moli",
"moli navigation to {} failed: {}", request.url, e
);
EngineError::BrowserError("moli navigation failed".to_string())
})?;
engine.wait_for_ready(page, &request.wait_for).await
}
async fn read_document_html(page: &chromiumoxide::Page) -> Result<String, EngineError> {
page.evaluate("() => document.documentElement.outerHTML")
.await
.map_err(|e| {
log::error!(target: "crawlrs::engine::moli", "moli outerHTML read failed: {}", e);
EngineError::Other("moli outerHTML read failed".to_string())
})?
.into_value::<String>()
.map_err(|e| {
log::error!(target: "crawlrs::engine::moli", "moli outerHTML decode failed: {}", e);
EngineError::Other("moli outerHTML decode failed".to_string())
})
}
async fn capture_screenshot(
page: &chromiumoxide::Page,
needs_screenshot: bool,
) -> Result<Option<String>, EngineError> {
let screenshot = if needs_screenshot {
let params = chromiumoxide::page::ScreenshotParams::builder()
.format(chromiumoxide::cdp::browser_protocol::page::CaptureScreenshotFormat::Png)
.build();
let bytes = page.screenshot(params).await.map_err(|e| {
log::error!(
target: "crawlrs::engine::moli",
"moli screenshot failed (endpoint must run `moli serve --layout`): {}", e
);
EngineError::BrowserError("moli screenshot failed".to_string())
})?;
Some(BASE64.encode(bytes))
} else {
None
};
Ok(screenshot)
}
async fn wait_for_ready(
&self,
page: &chromiumoxide::Page,
wait_for: &Option<WaitFor>,
) -> Result<(), EngineError> {
match wait_for {
None | Some(WaitFor::NetworkIdle) => self.wait_network_idle(page).await,
Some(WaitFor::Selector(selector)) => {
let expr = format!(r#"() => document.querySelector({:?}) !== null"#, selector);
self.poll_until(page, &expr, "selector never appeared")
.await
}
Some(WaitFor::DomStable(stable_for)) => self.wait_dom_stable(page, stable_for).await,
Some(WaitFor::ResponseUrl(_)) | Some(WaitFor::ResponseBody(_)) => {
Err(EngineError::Unsupported(
"response-level wait unsupported by moli".to_string(),
))
}
}
}
async fn wait_network_idle(&self, page: &chromiumoxide::Page) -> Result<(), EngineError> {
let deadline = tokio::time::Instant::now() + self.config.scrape_timeout;
loop {
let ready = page
.evaluate("() => document.readyState")
.await
.ok()
.and_then(|v| v.into_value::<String>().ok())
.is_some_and(|s| s == "complete");
if ready {
tokio::time::sleep(Duration::from_millis(NETWORK_IDLE_SETTLE_MS)).await;
return Ok(());
}
if tokio::time::Instant::now() >= deadline {
let e = "moli page never reached readyState=complete".to_string();
log::error!(target: "crawlrs::engine::moli", "{}", e);
return Err(EngineError::Timeout(self.config.scrape_timeout));
}
tokio::time::sleep(Duration::from_millis(WAIT_POLL_INTERVAL_MS)).await;
}
}
async fn wait_dom_stable(
&self,
page: &chromiumoxide::Page,
stable_for: &Duration,
) -> Result<(), EngineError> {
let mut last_len: Option<usize> = None;
let mut stable_since: Option<tokio::time::Instant> = None;
let deadline = tokio::time::Instant::now() + self.config.scrape_timeout;
loop {
let len = page
.evaluate("() => document.documentElement.outerHTML.length")
.await
.ok()
.and_then(|v| v.into_value::<f64>().ok())
.map(|f| f as usize);
if let (Some(prev), Some(cur)) = (last_len, len) {
if prev == cur {
let since = stable_since.get_or_insert(tokio::time::Instant::now());
if since.elapsed() >= *stable_for {
return Ok(());
}
} else {
stable_since = None;
}
}
last_len = len;
if tokio::time::Instant::now() >= deadline {
let e = "moli page DOM never stabilized".to_string();
log::error!(target: "crawlrs::engine::moli", "{}", e);
return Err(EngineError::Timeout(self.config.scrape_timeout));
}
tokio::time::sleep(Duration::from_millis(WAIT_POLL_INTERVAL_MS)).await;
}
}
async fn poll_until(
&self,
page: &chromiumoxide::Page,
expr: &str,
timeout_msg: &str,
) -> Result<(), EngineError> {
let deadline = tokio::time::Instant::now() + self.config.scrape_timeout;
loop {
let hit = page
.evaluate(expr)
.await
.ok()
.and_then(|v| v.into_value::<bool>().ok())
.unwrap_or(false);
if hit {
return Ok(());
}
if tokio::time::Instant::now() >= deadline {
log::error!(target: "crawlrs::engine::moli", "{}", timeout_msg);
return Err(EngineError::Timeout(self.config.scrape_timeout));
}
tokio::time::sleep(Duration::from_millis(WAIT_POLL_INTERVAL_MS)).await;
}
}
}
fn validate_html(html: &str) -> Result<(), EngineError> {
if html.trim().is_empty() {
let e = "moli returned empty HTML".to_string();
log::error!(target: "crawlrs::engine::moli", "{}", e);
Err(EngineError::Other(e))
} else {
Ok(())
}
}
#[async_trait::async_trait]
impl ScraperEngine for MoliEngine {
async fn scrape(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
if request.url.trim().is_empty() {
return Err(EngineError::InvalidUrl("empty url".to_string()));
}
if matches!(
request.wait_for,
Some(WaitFor::ResponseUrl(_)) | Some(WaitFor::ResponseBody(_))
) {
log::error!(
target: "crawlrs::engine::moli",
"response-level wait variants unsupported by moli CDP (Network events unavailable)"
);
return Err(EngineError::Unsupported(
"response-level wait (ResponseUrl/ResponseBody) unsupported by moli".to_string(),
));
}
let started = std::time::Instant::now();
let session = tokio::spawn(MoliEngine::scrape_session(
self.config.clone(),
self.http_client.clone(),
request.clone(),
));
match tokio::time::timeout(self.config.scrape_timeout, session).await {
Ok(Ok(Ok(mut resp))) => {
resp.response_time_ms = started.elapsed().as_millis() as u64;
Ok(resp)
}
Ok(Ok(Err(e))) => Err(e),
Ok(Err(join_err)) => Err(EngineError::Other(format!(
"moli session task failed: {}",
join_err
))),
Err(_) => {
log::error!(
target: "crawlrs::engine::moli",
"moli scrape timed out after {:?} (session will self-clean)", self.config.scrape_timeout
);
Err(EngineError::Timeout(self.config.scrape_timeout))
}
}
}
fn support_score(&self, request: &InternalScrapeRequest) -> u8 {
if request.method != HttpMethod::Get {
return 0;
}
if !request.actions.is_empty() {
return 0;
}
if request.needs_screenshot {
return 80;
}
if request.needs_js {
return 90;
}
0
}
fn name(&self) -> &'static str {
"moli"
}
fn supports_tls_fingerprint(&self) -> bool {
false
}
fn max_response_time(&self) -> Duration {
self.config.scrape_timeout
}
}
#[cfg(test)]
mod tests {
use super::*;
fn engine() -> MoliEngine {
MoliEngine::new(MoliEngineConfig {
endpoint: "http://127.0.0.1:9222".to_string(),
connect_timeout: Duration::from_millis(50),
scrape_timeout: Duration::from_millis(120),
})
}
fn request(needs_js: bool, needs_screenshot: bool) -> InternalScrapeRequest {
InternalScrapeRequest {
url: "https://example.com".to_string(),
method: HttpMethod::Get,
headers: Default::default(),
timeout: Duration::from_secs(30),
needs_js,
needs_screenshot,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
}
}
#[test]
fn trait_semantics_name_and_mrt() {
let e = engine();
assert_eq!(e.name(), "moli");
assert_eq!(e.max_response_time(), Duration::from_millis(120));
assert!(!e.supports_tls_fingerprint());
}
#[test]
fn support_score_static_and_js_branches() {
let e = engine();
assert_eq!(
e.support_score(&request(false, false)),
0,
"静态请求 score=0"
);
assert_eq!(
e.support_score(&request(true, false)),
90,
"needs_js 高分但低于 Playwright"
);
assert_eq!(
e.support_score(&request(true, true)),
80,
"截图场景低于 Playwright 基线"
);
assert_eq!(
e.support_score(&request(false, true)),
80,
"仅截图(无 needs_js)也给截图分"
);
let mut r = request(true, false);
r.method = HttpMethod::Post;
assert_eq!(e.support_score(&r), 0, "非 GET 不支持");
let mut r = request(true, false);
r.actions
.push(crate::engines::types::InternalPageAction::Click {
selector: "#x".to_string(),
});
assert_eq!(
e.support_score(&r),
0,
"交互动作 v1 未实现,score=0 诚实出局"
);
}
#[test]
fn validate_html_rejects_empty() {
assert!(validate_html("").is_err());
assert!(validate_html(" \n\t ").is_err());
assert!(validate_html("<html></html>").is_ok());
}
#[tokio::test]
async fn scrape_empty_url_rejected_before_any_io() {
let e = engine();
let mut r = request(true, false);
r.url = " ".to_string();
match e.scrape(&r).await {
Err(EngineError::InvalidUrl(_)) => {}
other => panic!("期望 InvalidUrl,实际: {:?}", other.err()),
}
}
#[tokio::test]
async fn response_wait_variants_unsupported_before_io() {
let e = MoliEngine::new(MoliEngineConfig {
endpoint: "http://127.0.0.1:1".to_string(),
connect_timeout: Duration::from_millis(200),
scrape_timeout: Duration::from_millis(400),
});
for wait in [
WaitFor::ResponseUrl("api".to_string()),
WaitFor::ResponseBody("token".to_string()),
] {
let mut r = request(true, false);
r.wait_for = Some(wait);
match e.scrape(&r).await {
Err(EngineError::Unsupported(msg)) => {
assert!(
msg.contains("response-level"),
"文案应指明响应级等待: {msg}"
);
}
other => panic!("期望 Unsupported,实际: {:?}", other.err()),
}
}
}
#[tokio::test]
async fn scrape_connection_refused_errors() {
let e = MoliEngine::new(MoliEngineConfig {
endpoint: "http://127.0.0.1:1".to_string(),
connect_timeout: Duration::from_millis(300),
scrape_timeout: Duration::from_millis(500),
});
let result = e.scrape(&request(true, false)).await;
assert!(result.is_err(), "端点不可达必须是错误而非空成功");
assert!(!matches!(result, Ok(_)));
}
#[tokio::test]
async fn scrape_non_cdp_endpoint_errors() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
loop {
let (mut sock, _) = match listener.accept().await {
Ok(v) => v,
Err(_) => break,
};
tokio::spawn(async move {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let mut buf = [0u8; 512];
let _ = sock.read(&mut buf).await;
let _ = sock
.write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n")
.await;
});
}
});
let e = MoliEngine::new(MoliEngineConfig {
endpoint: format!("http://{}", addr),
connect_timeout: Duration::from_millis(500),
scrape_timeout: Duration::from_millis(800),
});
let result = e.scrape(&request(true, false)).await;
server.abort();
assert!(result.is_err(), "非 CDP 端点必须报错而非空成功");
}
#[tokio::test]
async fn scrape_discovery_timeout_errors() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let mut held = Vec::new();
loop {
match listener.accept().await {
Ok((sock, _)) => held.push(sock),
Err(_) => break,
}
}
});
let e = MoliEngine::new(MoliEngineConfig {
endpoint: format!("http://{}", addr),
connect_timeout: Duration::from_millis(200),
scrape_timeout: Duration::from_millis(600),
});
let started = std::time::Instant::now();
let result = e.scrape(&request(true, false)).await;
server.abort();
assert!(result.is_err());
assert!(
started.elapsed() < Duration::from_secs(2),
"超时必须在 connect_timeout 量级返回,而非挂到外层超时"
);
}
#[tokio::test]
#[ignore = "requires local `moli serve --layout` on :9222(scripts/poc/moli/setup.sh)"]
async fn scrape_live_moli_serves_html() {
let e = MoliEngine::new(MoliEngineConfig {
endpoint: "http://127.0.0.1:9222".to_string(),
connect_timeout: Duration::from_secs(3),
scrape_timeout: Duration::from_secs(30),
});
let mut req = request(true, true);
req.url = std::env::var("MOLI_POC_LIVE_URL")
.unwrap_or_else(|_| "https://www.baidu.com".to_string());
let resp = e.scrape(&req).await.expect("live moli 应成功");
assert!(!resp.js_degraded);
assert!(resp.content.contains("<html"));
assert!(resp.response_time_ms > 0);
let shot = resp
.screenshot
.expect("needs_screenshot=true 不得静默省略截图");
let bytes = BASE64.decode(shot).expect("截图须为合法 base64");
assert!(bytes.starts_with(&[0x89, b'P', b'N', b'G']), "截图须为 PNG");
}
}