use crate::domain::services::llm::vision_adapter::VisionAdapterTrait;
use crate::engines::client::playwright_pool::{get_global_pool, BrowserPool};
use crate::engines::engine_client::{
EngineError, InternalScrapeRequest, InternalScrapeResponse, ScraperEngine,
};
use crate::engines::mllm::action_executor::{execute_decision, ActionResult};
use crate::engines::mllm::ax_tree;
use crate::engines::mllm::config::MllmEngineConfig;
use crate::engines::mllm::decision::{parse_decision, MllmDecision};
use crate::engines::validators;
use async_trait::async_trait;
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
use chromiumoxide::cdp::browser_protocol::page::CaptureScreenshotFormat;
use log::{debug, error, info, warn};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
const DEFAULT_MLLM_MRT_SECONDS: u64 = 60;
pub struct MllmEngine {
pool: Option<BrowserPool>,
ua_pool: crate::utils::ua_pool::UaPool,
vision_adapter: Arc<dyn VisionAdapterTrait>,
config: MllmEngineConfig,
mrt: Duration,
}
impl MllmEngine {
pub fn new(vision_adapter: Arc<dyn VisionAdapterTrait>, config: MllmEngineConfig) -> Self {
let mrt = Duration::from_secs(config.mrt_seconds.max(DEFAULT_MLLM_MRT_SECONDS));
Self {
pool: None,
ua_pool: crate::utils::ua_pool::UaPool::new(),
vision_adapter,
config,
mrt,
}
}
#[allow(dead_code)]
pub fn with_pool(mut self, pool: BrowserPool) -> Self {
self.pool = Some(pool);
self
}
fn get_pool(&self) -> Result<&BrowserPool, EngineError> {
if let Some(pool) = &self.pool {
return Ok(pool);
}
get_global_pool().ok_or_else(|| {
EngineError::BrowserError("Global browser pool not initialized".to_string())
})
}
async fn take_screenshot_base64(
&self,
page: &chromiumoxide::Page,
) -> Result<String, EngineError> {
let format = CaptureScreenshotFormat::Jpeg;
let params = chromiumoxide::page::ScreenshotParams::builder()
.format(format)
.quality(self.config.screenshot_quality as i64)
.full_page(false)
.build();
let screenshot_bytes = page
.screenshot(params)
.await
.map_err(|e| EngineError::BrowserError(format!("Screenshot failed: {}", e)))?;
Ok(BASE64.encode(screenshot_bytes))
}
fn build_vision_prompt(
&self,
goal: &str,
iteration: u8,
last_result: Option<&ActionResult>,
outline: Option<&str>,
) -> String {
let mut prompt = format!(
"Analyze the current page screenshot and decide the next action to accomplish this goal: {}\n",
goal
);
if iteration > 0 {
prompt.push_str(&format!(
"\nThis is iteration {}/{}.\n",
iteration, self.config.max_iterations
));
}
if let Some(result) = last_result {
if !result.success {
prompt.push_str(&format!(
"\nLast action failed: {}. Please try a different approach.\n",
result.message
));
} else {
prompt.push_str(&format!("\nLast action result: {}.\n", result.message));
}
}
prompt.push_str("\nRespond with a JSON object representing your decision.");
if let Some(outline) = outline {
prompt.push_str(&format!(
"\nPage accessibility outline:\n{outline}\nUse [id] to reference elements"
));
}
prompt
}
async fn capture_ax_outline(&self, page: &chromiumoxide::Page) -> Option<String> {
if !self.config.ax_tree_enabled {
return None;
}
match ax_tree::capture(page).await {
Ok(nodes) => {
let outline = ax_tree::outline(nodes, self.config.max_token_budget as usize);
if outline.is_empty() {
None
} else {
Some(outline)
}
}
Err(e) => {
warn!(
"MLLM AX tree outline capture failed (visual channel unaffected): {}",
e
);
None
}
}
}
async fn scrape_with_pool(
&self,
pool: &BrowserPool,
request: &InternalScrapeRequest,
start: Instant,
) -> Result<InternalScrapeResponse, EngineError> {
let pooled_page = pool.acquire_page().await?;
let page: chromiumoxide::Page = pooled_page.page().clone();
crate::engines::client::playwright::apply_browser_fingerprint(
&page,
&self.ua_pool,
request.mobile,
)
.await?;
info!("MLLM engine navigating to: {}", request.url);
page.goto(&request.url)
.await
.map_err(|e| EngineError::BrowserError(format!("Navigation failed: {}", e)))?;
tokio::time::sleep(Duration::from_millis(1500)).await;
let goal = extract_goal_from_request(request);
let final_content = self.run_agentic_loop(&page, &goal, request, start).await?;
let mut screenshot: Option<String> = None;
if request.needs_screenshot {
screenshot = Some(self.take_screenshot_base64(&page).await?);
}
drop(pooled_page);
Ok(InternalScrapeResponse {
js_degraded: false,
status_code: 200,
content: final_content,
screenshot,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: start.elapsed().as_millis() as u64,
final_url: None,
captured_responses: Vec::new(),
})
}
async fn run_agentic_loop(
&self,
page: &chromiumoxide::Page,
goal: &str,
request: &InternalScrapeRequest,
start: Instant,
) -> Result<String, EngineError> {
let mut last_result: Option<ActionResult> = None;
let mut iteration: u8 = 0;
let final_content;
loop {
if let Some(content) = self.guard_fallback(page, request, start, iteration).await {
final_content = content;
break;
}
iteration += 1;
debug!(
"MLLM agentic loop iteration {}/{}",
iteration, self.config.max_iterations
);
let vision_response = self
.request_vision_response(page, goal, iteration, last_result.as_ref())
.await?;
let decision = match parse_decision(&vision_response) {
Ok(d) => d,
Err(e) => {
warn!("Failed to parse vision response: {}", e);
last_result = Some(ActionResult::err(format!("parse error: {}", e)));
continue;
}
};
debug!("MLLM decision at iteration {}: {:?}", iteration, decision);
if let Some(content) = terminal_decision_content(page, &decision, iteration).await {
final_content = content;
break;
}
let result = execute_decision(&decision, page).await;
last_result = Some(result);
tokio::time::sleep(Duration::from_millis(500)).await;
}
Ok(final_content)
}
async fn guard_fallback(
&self,
page: &chromiumoxide::Page,
request: &InternalScrapeRequest,
start: Instant,
iteration: u8,
) -> Option<String> {
if iteration >= self.config.max_iterations {
warn!(
"MLLM engine reached max_iterations ({}) for {}",
self.config.max_iterations, request.url
);
return Some(get_page_content(page).await);
}
let elapsed = start.elapsed();
if elapsed + Duration::from_secs(10) > request.timeout {
warn!("MLLM engine approaching timeout for {}", request.url);
return Some(get_page_content(page).await);
}
None
}
async fn request_vision_response(
&self,
page: &chromiumoxide::Page,
goal: &str,
iteration: u8,
last_result: Option<&ActionResult>,
) -> Result<String, EngineError> {
let screenshot_b64 = self.take_screenshot_base64(page).await?;
let ax_outline = self.capture_ax_outline(page).await;
let prompt = self.build_vision_prompt(goal, iteration, last_result, ax_outline.as_deref());
self.vision_adapter
.send_vision_request(
&screenshot_b64,
&prompt,
&self.config.system_prompt,
&self.config.vision_model,
)
.await
.map_err(|e| {
error!("Vision model call failed: {}", e);
EngineError::Other(format!("Vision model call failed: {}", e))
})
}
}
async fn terminal_decision_content(
page: &chromiumoxide::Page,
decision: &MllmDecision,
iteration: u8,
) -> Option<String> {
match decision {
MllmDecision::Extract { .. } => {
let content = get_page_content(page).await;
info!("MLLM engine extracted content at iteration {}", iteration);
Some(content)
}
MllmDecision::Done { .. } => {
let content = get_page_content(page).await;
info!(
"MLLM engine completed navigation at iteration {}",
iteration
);
Some(content)
}
_ => None,
}
}
#[async_trait]
impl ScraperEngine for MllmEngine {
async fn scrape(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
validators::validate_url(&request.url)
.await
.map_err(|e| EngineError::Other(format!("SSRF protection: {}", e)))?;
let start = Instant::now();
let timeout_duration = request.timeout;
let pool = self.get_pool()?;
tokio::time::timeout(
timeout_duration,
self.scrape_with_pool(pool, request, start),
)
.await
.map_err(|_| EngineError::Timeout(timeout_duration))?
}
fn support_score(&self, request: &InternalScrapeRequest) -> u8 {
if request.needs_mllm {
100 } else {
5 }
}
fn name(&self) -> &'static str {
"mllm"
}
fn max_response_time(&self) -> Duration {
self.mrt
}
}
fn extract_goal_from_request(request: &InternalScrapeRequest) -> String {
if let Some(body) = &request.body {
if !body.is_empty() {
return body.clone();
}
}
format!("Navigate to and extract content from: {}", request.url)
}
async fn get_page_content(page: &chromiumoxide::Page) -> String {
match page
.evaluate_expression("document.documentElement.outerHTML")
.await
{
Ok(result) => result.into_value::<String>().unwrap_or_default(),
Err(e) => {
error!("Failed to get page content: {}", e);
String::new()
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::services::llm::vision_adapter::VisionAdapterTrait;
struct UnusedVisionAdapter;
#[async_trait::async_trait]
impl VisionAdapterTrait for UnusedVisionAdapter {
async fn send_vision_request(
&self,
_screenshot_b64: &str,
_prompt: &str,
_system_prompt: &str,
_model: &str,
) -> anyhow::Result<String> {
panic!("vision adapter must not be called during prompt construction")
}
}
fn engine_with(ax_tree_enabled: bool) -> MllmEngine {
MllmEngine::new(
Arc::new(UnusedVisionAdapter),
MllmEngineConfig {
ax_tree_enabled,
..MllmEngineConfig::default()
},
)
}
#[test]
fn prompt_without_outline_is_byte_identical_to_legacy() {
let engine = engine_with(false);
let last = ActionResult::err("click failed for '#x': not found");
let prompt = engine.build_vision_prompt("buy milk", 2, Some(&last), None);
let mut legacy = format!(
"Analyze the current page screenshot and decide the next action to accomplish this goal: {}\n",
"buy milk"
);
legacy.push_str(&format!("\nThis is iteration {}/{}.\n", 2, 10));
legacy.push_str("\nLast action failed: click failed for '#x': not found. Please try a different approach.\n");
legacy.push_str("\nRespond with a JSON object representing your decision.");
assert_eq!(prompt, legacy);
}
#[test]
fn prompt_with_outline_appends_ax_outline_block() {
let engine = engine_with(true);
let base = engine.build_vision_prompt("goal", 1, None, None);
let with = engine.build_vision_prompt("goal", 1, None, Some("[12] heading: Welcome"));
assert!(
with.starts_with(&base),
"outline 只追加在尾部,不改动既有段落"
);
assert!(
with.ends_with("\nPage accessibility outline:\n[12] heading: Welcome\nUse [id] to reference elements"),
"追加块格式固定: {}",
with
);
}
}