use crate::engines::engine_client::{EngineError, WaitFor};
use chromiumoxide::Page;
use std::time::{Duration, Instant};
const DEFAULT_NETWORK_IDLE_WINDOW: Duration = Duration::from_millis(500);
const NETWORK_EVENT_POLL_INTERVAL: Duration = Duration::from_millis(50);
const SELECTOR_POLL_INTERVAL: Duration = Duration::from_millis(100);
const DOM_STABLE_POLL_INTERVAL: Duration = Duration::from_millis(100);
const DOM_STABLE_MAX_DURATION: Duration = Duration::from_secs(60);
const WAIT_BODY_MAX_BYTES: u64 = 10 * 1024 * 1024;
type NetworkEventStreams = (
chromiumoxide::listeners::EventStream<
chromiumoxide::cdp::browser_protocol::network::EventRequestWillBeSent,
>,
chromiumoxide::listeners::EventStream<
chromiumoxide::cdp::browser_protocol::network::EventLoadingFinished,
>,
chromiumoxide::listeners::EventStream<
chromiumoxide::cdp::browser_protocol::network::EventLoadingFailed,
>,
);
type BodyPendingResponses =
std::collections::HashMap<chromiumoxide::cdp::browser_protocol::network::RequestId, u64>;
async fn subscribe_network_events(page: &Page) -> Result<NetworkEventStreams, String> {
use chromiumoxide::cdp::browser_protocol::network::{
EnableParams as NetworkEnableParams, EventLoadingFailed, EventLoadingFinished,
EventRequestWillBeSent,
};
page.execute(NetworkEnableParams::default())
.await
.map_err(|e| e.to_string())?;
let started = page
.event_listener::<EventRequestWillBeSent>()
.await
.map_err(|e| e.to_string())?;
let finished = page
.event_listener::<EventLoadingFinished>()
.await
.map_err(|e| e.to_string())?;
let failed = page
.event_listener::<EventLoadingFailed>()
.await
.map_err(|e| e.to_string())?;
Ok((started, finished, failed))
}
async fn disable_network(page: &Page) {
let _ = page
.execute(chromiumoxide::cdp::browser_protocol::network::DisableParams::default())
.await;
}
fn record_pending_response(
ev: std::sync::Arc<chromiumoxide::cdp::browser_protocol::network::EventResponseReceived>,
pending: &mut BodyPendingResponses,
) {
let len = (ev.response.encoded_data_length.max(0.0)) as u64;
if len <= WAIT_BODY_MAX_BYTES {
pending.insert(ev.request_id.clone(), len);
}
}
async fn finished_body_matches(
page: &Page,
fin: std::sync::Arc<chromiumoxide::cdp::browser_protocol::network::EventLoadingFinished>,
pending: &mut BodyPendingResponses,
needle: &str,
) -> bool {
use chromiumoxide::cdp::browser_protocol::network::GetResponseBodyParams;
let Some(len) = pending.remove(&fin.request_id) else {
return false;
};
let _ = len;
if let Ok(returns) = page
.execute(GetResponseBodyParams::new(fin.request_id.clone()))
.await
{
let text = if returns.base64_encoded {
use base64::Engine as _;
base64::engine::general_purpose::STANDARD
.decode(returns.body.as_bytes())
.ok()
.and_then(|b| String::from_utf8(b).ok())
} else {
Some(returns.body.clone())
};
if let Some(text) = text {
if text.contains(needle) {
return true;
}
}
}
false
}
impl WaitFor {
pub async fn wait(&self, page: &Page, timeout: Duration) -> Result<(), EngineError> {
match self {
Self::NetworkIdle => {
Self::wait_network_idle(page, DEFAULT_NETWORK_IDLE_WINDOW, timeout).await
}
Self::Selector(selector) => Self::wait_selector(page, selector, timeout).await,
Self::DomStable(stable_duration) => {
Self::wait_dom_stable(page, *stable_duration, timeout).await
}
Self::ResponseUrl(pattern) => Self::wait_response_url(page, pattern, timeout).await,
Self::ResponseBody(needle) => Self::wait_response_body(page, needle, timeout).await,
}
}
async fn wait_network_idle(
page: &Page,
quiet_window: Duration,
timeout: Duration,
) -> Result<(), EngineError> {
let (mut started, mut finished, mut failed) = match subscribe_network_events(page).await {
Ok(streams) => streams,
Err(e) => {
disable_network(page).await;
log::warn!(
"{}",
crate::i18n::tr_log_args(
"wait-network-idle-subscribe-failed",
&[("error", e.to_string())],
)
);
tokio::time::sleep(quiet_window.min(timeout)).await;
return Ok(());
}
};
let mut tracker = NetworkIdleTracker::default();
let deadline = Instant::now() + timeout;
let result = Self::wait_idle_events(
&mut started,
&mut finished,
&mut failed,
&mut tracker,
quiet_window,
deadline,
)
.await;
disable_network(page).await;
result
}
async fn wait_idle_events(
started: &mut chromiumoxide::listeners::EventStream<
chromiumoxide::cdp::browser_protocol::network::EventRequestWillBeSent,
>,
finished: &mut chromiumoxide::listeners::EventStream<
chromiumoxide::cdp::browser_protocol::network::EventLoadingFinished,
>,
failed: &mut chromiumoxide::listeners::EventStream<
chromiumoxide::cdp::browser_protocol::network::EventLoadingFailed,
>,
tracker: &mut NetworkIdleTracker,
quiet_window: Duration,
deadline: Instant,
) -> Result<(), EngineError> {
use futures::StreamExt;
let result: Result<(), EngineError> = loop {
tracker.tick(Instant::now());
if tracker.is_idle_for(Instant::now(), quiet_window) {
break Ok(());
}
if Instant::now() >= deadline {
break Ok(());
}
tokio::select! {
Some(ev) = started.next() => {
if ev.redirect_response.is_some() {
tracker.on_request_finished(Instant::now());
}
tracker.on_request_started(Instant::now());
}
Some(_) = finished.next() => tracker.on_request_finished(Instant::now()),
Some(_) = failed.next() => tracker.on_request_finished(Instant::now()),
_ = tokio::time::sleep(NETWORK_EVENT_POLL_INTERVAL) => {}
}
};
result
}
async fn wait_response_url(
page: &Page,
pattern: &str,
timeout: Duration,
) -> Result<(), EngineError> {
use chromiumoxide::cdp::browser_protocol::network::{
DisableParams as NetworkDisableParams, EnableParams as NetworkEnableParams,
EventResponseReceived,
};
use futures::StreamExt;
let regex = regex::Regex::new(pattern).map_err(|e| {
EngineError::BrowserError(format!("wait_response_url regex invalid: {e}"))
})?;
page.execute(NetworkEnableParams::default())
.await
.map_err(|e| EngineError::BrowserError(format!("network enable failed: {e}")))?;
let result: Result<(), EngineError> = async {
let mut events = page
.event_listener::<EventResponseReceived>()
.await
.map_err(|e| EngineError::BrowserError(format!("subscribe failed: {e}")))?;
let deadline = Instant::now() + timeout;
loop {
if Instant::now() >= deadline {
return Err(EngineError::BrowserError(format!(
"wait timeout: response_url pattern '{pattern}' not matched after {}ms",
timeout.as_millis()
)));
}
match tokio::time::timeout(NETWORK_EVENT_POLL_INTERVAL, events.next()).await {
Ok(Some(ev)) => {
if regex.is_match(&ev.response.url) {
return Ok(());
}
}
Ok(None) => {
return Err(EngineError::BrowserError(
"response event stream ended prematurely".to_string(),
));
}
Err(_) => {}
}
}
}
.await;
let _ = page.execute(NetworkDisableParams::default()).await;
result
}
async fn wait_response_body(
page: &Page,
needle: &str,
timeout: Duration,
) -> Result<(), EngineError> {
use chromiumoxide::cdp::browser_protocol::network::{
DisableParams as NetworkDisableParams, EnableParams as NetworkEnableParams,
EventResponseReceived,
};
use futures::StreamExt;
page.execute(NetworkEnableParams::default())
.await
.map_err(|e| EngineError::BrowserError(format!("network enable failed: {e}")))?;
let result: Result<(), EngineError> = async {
use chromiumoxide::cdp::browser_protocol::network::EventLoadingFinished;
let mut events = page
.event_listener::<EventResponseReceived>()
.await
.map_err(|e| EngineError::BrowserError(format!("subscribe failed: {e}")))?;
let mut finished = page
.event_listener::<EventLoadingFinished>()
.await
.map_err(|e| EngineError::BrowserError(format!("subscribe failed: {e}")))?;
let mut pending: BodyPendingResponses = BodyPendingResponses::new();
let deadline = Instant::now() + timeout;
loop {
if Instant::now() >= deadline {
return Err(EngineError::BrowserError(format!(
"wait timeout: response_body '{}' not found after {}ms",
needle,
timeout.as_millis()
)));
}
tokio::select! {
ev = events.next() => {
match ev {
Some(ev) => record_pending_response(ev, &mut pending),
None => {
return Err(EngineError::BrowserError(
"response event stream ended prematurely".to_string(),
));
}
}
}
fin = finished.next() => {
match fin {
Some(fin) => {
if finished_body_matches(page, fin, &mut pending, needle).await {
return Ok(());
}
}
None => {
return Err(EngineError::BrowserError(
"loading-finished stream ended prematurely".to_string(),
));
}
}
}
_ = tokio::time::sleep(NETWORK_EVENT_POLL_INTERVAL) => {}
}
}
}
.await;
let _ = page.execute(NetworkDisableParams::default()).await;
result
}
async fn wait_selector(
page: &Page,
selector: &str,
timeout: Duration,
) -> Result<(), EngineError> {
let escaped = escape_js_string(selector);
let check_expr = format!("() => document.querySelector({escaped}) !== null");
let deadline = Instant::now() + timeout;
loop {
if Instant::now() >= deadline {
return Err(EngineError::BrowserError(format!(
"wait timeout: selector '{selector}' not found after {}ms",
timeout.as_millis()
)));
}
let exists: bool = page
.evaluate(check_expr.clone())
.await
.map_err(|e| EngineError::BrowserError(format!("selector eval failed: {e}")))?
.into_value::<bool>()
.map_err(|e| {
EngineError::BrowserError(format!("selector eval decode failed: {e}"))
})?;
if exists {
return Ok(());
}
tokio::time::sleep(SELECTOR_POLL_INTERVAL).await;
}
}
async fn wait_dom_stable(
page: &Page,
stable_duration: Duration,
timeout: Duration,
) -> Result<(), EngineError> {
validate_dom_stable_duration(stable_duration)?;
let check_expr = "() => document.body ? document.body.innerHTML.length : -1";
let deadline = Instant::now() + timeout;
let mut last_len: i64 = -1;
let mut stable_since: Option<Instant> = None;
loop {
if Instant::now() >= deadline {
log_dom_stable_timeout(timeout);
return Ok(());
}
let len = read_body_inner_html_len(page, check_expr).await?;
if dom_length_stabilized(len, &mut last_len, &mut stable_since, stable_duration) {
return Ok(());
}
tokio::time::sleep(DOM_STABLE_POLL_INTERVAL).await;
}
}
}
fn validate_dom_stable_duration(stable_duration: Duration) -> Result<(), EngineError> {
if stable_duration > DOM_STABLE_MAX_DURATION {
return Err(EngineError::BrowserError(format!(
"dom_stable duration {}ms exceeds max {}ms",
stable_duration.as_millis(),
DOM_STABLE_MAX_DURATION.as_millis()
)));
}
Ok(())
}
fn log_dom_stable_timeout(timeout: Duration) {
log::warn!(
"{}",
crate::i18n::tr_log_args(
"wait-dom-unstable-degraded",
&[("timeout", (timeout.as_millis() as i64).to_string(),)],
)
);
}
async fn read_body_inner_html_len(page: &Page, check_expr: &str) -> Result<i64, EngineError> {
page.evaluate(check_expr)
.await
.map_err(|e| EngineError::BrowserError(format!("dom eval failed: {e}")))?
.into_value::<i64>()
.map_err(|e| EngineError::BrowserError(format!("dom eval decode failed: {e}")))
}
fn dom_length_stabilized(
len: i64,
last_len: &mut i64,
stable_since: &mut Option<Instant>,
stable_duration: Duration,
) -> bool {
if len == *last_len {
if stable_since.is_none() {
*stable_since = Some(Instant::now());
}
if let Some(since) = *stable_since {
if since.elapsed() >= stable_duration {
return true;
}
}
} else {
*last_len = len;
*stable_since = None;
}
false
}
#[derive(Debug, Default)]
struct NetworkIdleTracker {
in_flight: i64,
idle_since: Option<Instant>,
}
impl NetworkIdleTracker {
fn on_request_started(&mut self, _now: Instant) {
self.in_flight += 1;
self.idle_since = None;
}
fn on_request_finished(&mut self, _now: Instant) {
self.in_flight = (self.in_flight - 1).max(0);
}
fn tick(&mut self, now: Instant) {
if self.in_flight <= 0 && self.idle_since.is_none() {
self.idle_since = Some(now);
}
if self.in_flight > 0 {
self.idle_since = None;
}
}
fn is_idle_for(&self, now: Instant, quiet_window: Duration) -> bool {
match self.idle_since {
Some(since) => now.duration_since(since) >= quiet_window,
None => false,
}
}
}
fn escape_js_string(s: &str) -> String {
let mut out = String::with_capacity(s.len() + 2);
out.push('"');
for ch in s.chars() {
match ch {
'"' => out.push_str(r#"\""#),
'\\' => out.push_str(r"\\"),
'\n' => out.push_str(r"\n"),
'\r' => out.push_str(r"\r"),
'\t' => out.push_str(r"\t"),
c if c.is_control() => {
out.push_str(&format!(r"\u{:04x}", c as u32));
}
c => out.push(c),
}
}
out.push('"');
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn escape_plain_string() {
assert_eq!(escape_js_string("hello"), r#""hello""#);
}
#[test]
fn escape_double_quote() {
assert_eq!(escape_js_string(r#"a"b"#), r#""a\"b""#);
}
#[test]
fn escape_backslash() {
assert_eq!(escape_js_string(r"a\b"), r#""a\\b""#);
}
#[test]
fn escape_newline() {
assert_eq!(escape_js_string("a\nb"), r#""a\nb""#);
}
#[test]
fn escape_control_chars() {
let s = escape_js_string("a\x01b");
assert!(s.starts_with("\"a"));
assert!(s.contains(r"\u0001"));
}
#[test]
fn escape_empty_string() {
assert_eq!(escape_js_string(""), r#""""#);
}
#[test]
fn default_is_network_idle() {
let w = WaitFor::default();
assert_eq!(w, WaitFor::NetworkIdle);
}
#[test]
fn selector_equality() {
assert_eq!(
WaitFor::Selector("div".to_string()),
WaitFor::Selector("div".to_string())
);
assert_ne!(
WaitFor::Selector("div".to_string()),
WaitFor::Selector("span".to_string())
);
}
#[test]
fn dom_stable_equality() {
assert_eq!(
WaitFor::DomStable(Duration::from_millis(500)),
WaitFor::DomStable(Duration::from_millis(500))
);
}
#[test]
fn dom_stable_max_duration_constant() {
assert_eq!(DOM_STABLE_MAX_DURATION, Duration::from_secs(60));
}
#[test]
fn network_idle_window_constant() {
assert_eq!(DEFAULT_NETWORK_IDLE_WINDOW, Duration::from_millis(500));
}
#[test]
fn selector_poll_interval_constant() {
assert_eq!(SELECTOR_POLL_INTERVAL, Duration::from_millis(100));
}
#[test]
fn dom_stable_poll_interval_constant() {
assert_eq!(DOM_STABLE_POLL_INTERVAL, Duration::from_millis(100));
}
#[tokio::test]
async fn network_idle_returns_ok() {
let timeout = Duration::from_secs(1);
let wait = DEFAULT_NETWORK_IDLE_WINDOW.min(timeout);
assert_eq!(wait, Duration::from_millis(500));
}
#[tokio::test]
async fn network_idle_timeout_less_than_window() {
let timeout = Duration::from_millis(100);
let wait = DEFAULT_NETWORK_IDLE_WINDOW.min(timeout);
assert_eq!(wait, Duration::from_millis(100));
}
fn t(ms: u64) -> Instant {
let mut base = Instant::now();
base += Duration::from_millis(ms);
base
}
async fn spawn_wait_test_server() -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let handle = tokio::spawn(async move {
loop {
let (mut sock, _) = match listener.accept().await {
Ok(v) => v,
Err(_) => break,
};
tokio::spawn(async move {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let mut buf = [0u8; 1024];
let n = sock.read(&mut buf).await.unwrap_or(0);
let req = String::from_utf8_lossy(&buf[..n]).to_string();
let (status, body) = if req.starts_with("GET /page") {
(
"200 OK",
r#"<html><body>wait probe<script>setTimeout(()=>fetch('/api/payload'),300)</script></body></html>"#.to_string(),
)
} else if req.starts_with("GET /api/payload") {
("200 OK", r#"{"status":"READY_TOKEN"}"#.to_string())
} else {
("404 Not Found", String::new())
};
let resp = format!(
"HTTP/1.1 {status}\r\nContent-Type: text/html\r\nContent-Length: {}\r\n\r\n{}",
body.len(),
body
);
let _ = sock.write_all(resp.as_bytes()).await;
tokio::time::sleep(Duration::from_secs(5)).await;
});
}
});
(addr, handle)
}
fn find_playwright_chromium() -> Option<std::path::PathBuf> {
let base = std::path::PathBuf::from(std::env::var("HOME").ok()?)
.join("Library/Caches/ms-playwright");
let mut entries: Vec<_> = std::fs::read_dir(&base)
.ok()?
.filter_map(|e| e.ok().map(|e| e.path()))
.filter(|p| {
p.file_name()
.is_some_and(|n| n.to_string_lossy().starts_with("chromium-"))
})
.collect();
entries.sort();
let chrome = entries
.last()?
.join("chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing");
chrome.is_file().then_some(chrome)
}
async fn launch_page() -> (chromiumoxide::Browser, chromiumoxide::page::Page) {
let profile_dir = std::env::temp_dir().join(format!(
"crawlrs-wait-e2e-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&profile_dir).expect("profile dir");
let mut builder = chromiumoxide::BrowserConfig::builder()
.no_sandbox()
.arg("--disable-gpu")
.user_data_dir(&profile_dir);
if let Ok(bin) = std::env::var("CRAWLRS_TEST_CHROME") {
builder = builder.chrome_executable(bin);
} else if let Some(bin) = find_playwright_chromium() {
builder = builder.chrome_executable(bin);
}
let config = builder.build().expect("config");
let (browser, mut handler) = chromiumoxide::Browser::launch(config)
.await
.expect("launch");
tokio::spawn(async move {
use futures::StreamExt;
while let Some(h) = handler.next().await {
if h.is_err() {
break;
}
}
});
let page = browser.new_page("about:blank").await.expect("page");
(browser, page)
}
#[tokio::test]
#[ignore = "需要本机 Chrome:cargo test --features engine-playwright --lib response_wait -- --ignored --test-threads=1"]
async fn response_wait_url_hit_returns_early() {
let (addr, server) = spawn_wait_test_server().await;
let (_browser, page) = launch_page().await;
page.goto(format!("http://{addr}/page")).await.unwrap();
let started = Instant::now();
let result = WaitFor::ResponseUrl("api/payload".to_string())
.wait(&page, Duration::from_secs(10))
.await;
server.abort();
assert!(result.is_ok(), "命中必须提前返回: {:?}", result.err());
assert!(
started.elapsed() < Duration::from_secs(8),
"提前返回不得等到超时边界"
);
}
#[tokio::test]
#[ignore = "需要本机 Chrome:cargo test --features engine-playwright --lib response_wait -- --ignored --test-threads=1"]
async fn response_wait_body_substring_hit() {
let (addr, server) = spawn_wait_test_server().await;
let (_browser, page) = launch_page().await;
page.goto(format!("http://{addr}/page")).await.unwrap();
let result = WaitFor::ResponseBody("READY_TOKEN".to_string())
.wait(&page, Duration::from_secs(10))
.await;
server.abort();
assert!(
result.is_ok(),
"body 子串命中必须返回 Ok: {:?}",
result.err()
);
}
#[tokio::test]
#[ignore = "需要本机 Chrome:cargo test --features engine-playwright --lib response_wait -- --ignored --test-threads=1"]
async fn response_wait_url_timeout_errors() {
let (addr, server) = spawn_wait_test_server().await;
let (_browser, page) = launch_page().await;
page.goto(format!("http://{addr}/page")).await.unwrap();
let result = WaitFor::ResponseUrl("this-never-matches".to_string())
.wait(&page, Duration::from_secs(1))
.await;
server.abort();
match result {
Err(EngineError::BrowserError(msg)) => {
assert!(
msg.contains("wait timeout"),
"超时文案应指明 wait timeout: {msg}"
);
}
other => panic!("期望超时错误,实际: {:?}", other),
}
}
#[test]
fn tracker_idle_after_finish_and_quiet_window() {
let mut tr = NetworkIdleTracker::default();
tr.on_request_started(t(0));
assert!(!tr.is_idle_for(t(10), Duration::from_millis(500)));
tr.on_request_finished(t(20));
tr.tick(t(20));
assert!(!tr.is_idle_for(t(320), Duration::from_millis(500)));
assert!(tr.is_idle_for(t(620), Duration::from_millis(500)));
}
#[test]
fn tracker_stays_busy_while_in_flight() {
let mut tr = NetworkIdleTracker::default();
tr.on_request_started(t(0));
tr.tick(t(100));
tr.tick(t(2000));
assert!(
!tr.is_idle_for(t(2000), Duration::from_millis(500)),
"in-flight request must block idle detection"
);
}
#[test]
fn tracker_idle_interrupted_by_new_request() {
let mut tr = NetworkIdleTracker::default();
tr.on_request_started(t(0));
tr.on_request_finished(t(10));
tr.tick(t(10));
assert!(tr.is_idle_for(t(600), Duration::from_millis(500)));
tr.on_request_started(t(610));
tr.on_request_finished(t(620));
tr.tick(t(620));
assert!(
!tr.is_idle_for(t(1000), Duration::from_millis(500)),
"idle window must restart after new activity"
);
assert!(
tr.is_idle_for(t(1120), Duration::from_millis(500)),
"idle should be detected quiet_window after the last finish"
);
}
}