use super::*;
impl BrowserPoolState {
pub(super) async fn connect_remote_browser(
url: &str,
) -> Result<(Browser, chromiumoxide::Handler), EngineError> {
info!("Connecting to remote Chrome instance at: {}", url);
Browser::connect(url)
.await
.map_err(|e| EngineError::Other(format!("Failed to connect to remote Chrome: {}", e)))
}
pub(super) async fn launch_local_browser(
&self,
validated_proxy: Option<String>,
) -> Result<(Browser, chromiumoxide::Handler), EngineError> {
let browser_path = self.get_or_download_browser().await?;
let config = self.prepare_launch_config(browser_path.as_ref(), validated_proxy)?;
Browser::launch(config)
.await
.map_err(|e| EngineError::Other(format!("Failed to launch browser: {}", e)))
}
fn prepare_launch_config(
&self,
browser_path: Option<&PathBuf>,
validated_proxy: Option<String>,
) -> Result<BrowserConfig, EngineError> {
let mut builder = BrowserConfig::builder()
.no_sandbox()
.new_headless_mode()
.request_timeout(Duration::from_secs(30))
.arg("--disable-blink-features=AutomationControlled")
.arg("--lang=en-US,en");
let profile_dir = std::env::temp_dir().join(format!(
"crawlrs-chromium-{}-{}",
std::process::id(),
self.profile_dir_counter.fetch_add(1, Ordering::Relaxed)
));
std::fs::create_dir_all(&profile_dir).map_err(|e| {
EngineError::Other(format!(
"Failed to create browser profile dir {:?}: {}",
profile_dir, e
))
})?;
self.register_profile_dir(profile_dir.clone());
info!("Using per-instance browser profile dir: {:?}", profile_dir);
builder = builder.user_data_dir(&profile_dir);
if let Some(path) = browser_path {
info!("Using browser at: {:?}", path);
builder = builder.chrome_executable(path);
}
for arg in &self.config.browser_args {
builder = builder.arg(arg.as_str());
}
if let Some(validated) = validated_proxy {
info!(
"Using proxy for browser: {}",
crate::utils::proxy::redact_proxy_url(&validated)
);
builder = builder.arg("--proxy-server").arg(validated);
}
builder
.build()
.map_err(|e| EngineError::Other(e.to_string()))
}
async fn get_or_download_browser(&self) -> Result<Option<PathBuf>, EngineError> {
if let Some(path) = crate::engines::browser_downloader::browser_path_from_env() {
return Ok(Some(path));
}
match self
.browser_path
.get_or_try_init(|| async {
if let Some(path) = crate::engines::browser_downloader::find_system_browser().await
{
info!("Using system browser");
return Ok(path);
}
if self.download_manager.is_browser_downloaded().await {
let path = crate::engines::browser_downloader::get_browser_executable_path(
self.download_manager.get_cache_dir(),
);
info!("Using downloaded browser: {:?}", path);
return Ok(path);
}
info!("No browser found, starting automatic download...");
match self.download_manager.download_browser().await {
Ok(path) => {
info!("Browser downloaded successfully: {:?}", path);
Ok(path)
}
Err(e) => {
warn!("Browser download failed: {}", e);
Err(EngineError::Unsupported(format!(
"playwright 引擎不可用:未找到系统 Chrome/Chromium,内置自动下载不可用({e})。处置:安装 Chrome/Chromium,或设置环境变量 CRAWLRS__ENGINES__PLAYWRIGHT__BROWSER_PATH 指向浏览器可执行文件。"
)))
}
}
})
.await
{
Ok(path) => Ok(Some(path.clone())),
Err(e @ EngineError::Unsupported(_)) => Err(e),
Err(_) => Ok(None), }
}
pub(super) async fn check_browser_health(&self, browser: &Browser) -> bool {
match browser.new_page("about:blank").await {
Ok(page) => {
let _ = page.close().await;
true
}
Err(e) => {
warn!("Browser health check failed: {}", e);
false
}
}
}
#[cfg(feature = "engine-moli")]
pub(super) async fn reclaim_idle_instances_now(&self) {
let removed = {
let mut available = self.available.write().await;
available.drain().collect::<Vec<_>>()
};
for (_, pooled) in removed {
self.total_instances.fetch_sub(1, Ordering::Relaxed);
log::warn!(
"Reclaimed idle browser instance {} due to Critical memory watermark",
pooled.instance_id
);
drop(pooled);
}
}
pub(super) async fn cleanup_idle_instances(&self) {
let idle_ids = {
let available = self.available.read().await;
let now = Instant::now();
let idle_timeout = Duration::from_secs(self.config.idle_timeout_secs);
available
.iter()
.filter(|(_, p)| now.duration_since(p.last_used()) > idle_timeout)
.map(|(id, _)| *id)
.collect::<Vec<_>>()
};
let removed = {
let mut available = self.available.write().await;
idle_ids
.into_iter()
.filter_map(|id| available.remove(&id))
.collect::<Vec<_>>()
};
for pooled in removed {
self.total_instances.fetch_sub(1, Ordering::Relaxed);
info!("Cleaned up idle browser instance {}", pooled.instance_id);
drop(pooled);
}
}
pub(super) async fn health_check_all(&self) {
let candidates = {
let available = self.available.read().await;
available
.iter()
.map(|(id, p)| (*id, p.clone()))
.collect::<Vec<_>>()
};
let mut unhealthy_ids = Vec::new();
for (id, pooled) in &candidates {
if !self.check_browser_health(&pooled.browser).await {
pooled.mark_unhealthy();
unhealthy_ids.push(*id);
}
}
let removed = {
let mut available = self.available.write().await;
unhealthy_ids
.into_iter()
.filter_map(|id| available.remove(&id))
.collect::<Vec<_>>()
};
for pooled in removed {
self.total_instances.fetch_sub(1, Ordering::Relaxed);
warn!("Removed unhealthy browser instance {}", pooled.instance_id);
drop(pooled); }
}
pub(super) async fn shutdown_all(&self) {
self.shutdown.store(true, Ordering::Release);
let mut available = self.available.write().await;
available.clear();
let mut in_use = self.in_use.write().await;
in_use.clear();
self.total_instances.store(0, Ordering::Relaxed);
let dirs = self
.profile_dirs
.lock()
.expect("profile_dirs poisoned")
.drain(..)
.collect::<Vec<_>>();
for dir in dirs {
if let Err(e) = std::fs::remove_dir_all(&dir) {
debug!("Failed to remove browser profile dir {:?}: {}", dir, e);
}
}
info!("All browser instances shut down");
}
pub(super) fn clone_return_sender(&self) -> Arc<mpsc::Sender<ReturnMessage>> {
self.return_sender.clone()
}
pub(super) async fn reclaim_after_page_acquire_failure(
&self,
instance_id: u64,
browser: Arc<Browser>,
reason: &str,
) {
match self.clone_return_sender().try_send(ReturnMessage {
instance_id,
browser,
}) {
Ok(_) => {}
Err(mpsc::error::TrySendError::Full(msg))
| Err(mpsc::error::TrySendError::Closed(msg)) => {
error!(
"Return channel unavailable while reclaiming browser instance {} after page acquire failure ({}); removing in_use slot directly",
msg.instance_id, reason
);
self.return_instance(msg.instance_id, msg.browser).await;
}
}
}
}