crawlrs 0.2.0

crawlrs - Enterprise-grade web scraping and crawling platform built with Rust
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
// Copyright (c) 2025-2026 Kirky.X🌠
// SPDX-License-Identifier: Apache-2.0

//! MLLM 自主导航爬取引擎
//!
//! 实现 `ScraperEngine` trait,在 `scrape()` 内封装 agentic loop:
//! 导航 → 截图 → vision_analyze → execute_decision → 循环,
//! 直到 Extract/Done 决策或达到 max_iterations。

use crate::domain::services::llm::vision_adapter::VisionAdapterTrait;
use crate::engines::client::playwright_pool::{get_global_pool, BrowserPool};
use crate::engines::engine_client::{
    EngineError, InternalScrapeRequest, InternalScrapeResponse, ScraperEngine,
};
use crate::engines::mllm::action_executor::{execute_decision, ActionResult};
use crate::engines::mllm::ax_tree;
use crate::engines::mllm::config::MllmEngineConfig;
use crate::engines::mllm::decision::{parse_decision, MllmDecision};
use crate::engines::validators;
use async_trait::async_trait;
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
use chromiumoxide::cdp::browser_protocol::page::CaptureScreenshotFormat;
use log::{debug, error, info, warn};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};

/// MLLM 引擎默认 MRT(60 秒)
///
/// MLLM 引擎涉及多轮截图+视觉模型推理,比普通浏览器引擎需要更多时间。
const DEFAULT_MLLM_MRT_SECONDS: u64 = 60;

/// MLLM 自主导航爬取引擎
///
/// 使用视觉大模型分析页面截图,自主决策导航操作,实现 agentic loop 式智能爬取。
pub struct MllmEngine {
    /// 浏览器池引用
    pool: Option<BrowserPool>,
    /// UA profile 池(指纹一致性设置与 PlaywrightEngine 同源)
    ua_pool: crate::utils::ua_pool::UaPool,
    /// 视觉模型适配器
    vision_adapter: Arc<dyn VisionAdapterTrait>,
    /// 引擎配置
    config: MllmEngineConfig,
    /// 引擎级最大响应时间
    mrt: Duration,
}

impl MllmEngine {
    /// 创建 MLLM 引擎
    pub fn new(vision_adapter: Arc<dyn VisionAdapterTrait>, config: MllmEngineConfig) -> Self {
        let mrt = Duration::from_secs(config.mrt_seconds.max(DEFAULT_MLLM_MRT_SECONDS));
        Self {
            pool: None,
            ua_pool: crate::utils::ua_pool::UaPool::new(),
            vision_adapter,
            config,
            mrt,
        }
    }

    /// 设置浏览器池(用于测试注入)
    #[allow(dead_code)]
    pub fn with_pool(mut self, pool: BrowserPool) -> Self {
        self.pool = Some(pool);
        self
    }

    /// 获取或初始化浏览器池
    fn get_pool(&self) -> Result<&BrowserPool, EngineError> {
        if let Some(pool) = &self.pool {
            return Ok(pool);
        }
        get_global_pool().ok_or_else(|| {
            EngineError::BrowserError("Global browser pool not initialized".to_string())
        })
    }

    /// 截取当前页面的截图并返回 base64
    async fn take_screenshot_base64(
        &self,
        page: &chromiumoxide::Page,
    ) -> Result<String, EngineError> {
        let format = CaptureScreenshotFormat::Jpeg;
        let params = chromiumoxide::page::ScreenshotParams::builder()
            .format(format)
            .quality(self.config.screenshot_quality as i64)
            .full_page(false)
            .build();

        let screenshot_bytes = page
            .screenshot(params)
            .await
            .map_err(|e| EngineError::BrowserError(format!("Screenshot failed: {}", e)))?;

        Ok(BASE64.encode(screenshot_bytes))
    }

    /// 构造视觉分析的 prompt
    ///
    /// `outline` 为 AX 树文本大纲(F12 混合通道;`None` 时与旧版 prompt
    /// 逐字节一致,截图视觉通道不受影响)
    fn build_vision_prompt(
        &self,
        goal: &str,
        iteration: u8,
        last_result: Option<&ActionResult>,
        outline: Option<&str>,
    ) -> String {
        let mut prompt = format!(
            "Analyze the current page screenshot and decide the next action to accomplish this goal: {}\n",
            goal
        );

        if iteration > 0 {
            prompt.push_str(&format!(
                "\nThis is iteration {}/{}.\n",
                iteration, self.config.max_iterations
            ));
        }

        if let Some(result) = last_result {
            if !result.success {
                prompt.push_str(&format!(
                    "\nLast action failed: {}. Please try a different approach.\n",
                    result.message
                ));
            } else {
                prompt.push_str(&format!("\nLast action result: {}.\n", result.message));
            }
        }

        prompt.push_str("\nRespond with a JSON object representing your decision.");

        if let Some(outline) = outline {
            prompt.push_str(&format!(
                "\nPage accessibility outline:\n{outline}\nUse [id] to reference elements"
            ));
        }
        prompt
    }

    /// 采集 AX 树文本大纲(F12):`ax_tree_enabled = false` 时零开销返回 None;
    /// 采集失败仅 warn(差量通道不阻塞截图主通道)
    async fn capture_ax_outline(&self, page: &chromiumoxide::Page) -> Option<String> {
        if !self.config.ax_tree_enabled {
            return None;
        }
        match ax_tree::capture(page).await {
            Ok(nodes) => {
                let outline = ax_tree::outline(nodes, self.config.max_token_budget as usize);
                if outline.is_empty() {
                    None
                } else {
                    Some(outline)
                }
            }
            Err(e) => {
                warn!(
                    "MLLM AX tree outline capture failed (visual channel unaffected): {}",
                    e
                );
                None
            }
        }
    }

    /// 在浏览器池上执行一次 agentic 抓取(scrape 超时包裹的内部主体):
    /// 取页 → 指纹一致性设置 → 导航 → agentic loop → 截图 → 组装响应。
    async fn scrape_with_pool(
        &self,
        pool: &BrowserPool,
        request: &InternalScrapeRequest,
        start: Instant,
    ) -> Result<InternalScrapeResponse, EngineError> {
        // Acquire browser page
        let pooled_page = pool.acquire_page().await?;
        let page: chromiumoxide::Page = pooled_page.page().clone();

        // 指纹一致性设置与 PlaywrightEngine 同一套(UA/客户端提示/
        // locale/viewport/stealth)——本引擎此前裸 goto,是全引擎里
        // 唯一没化妆的检测点;共享函数见 playwright 模块文档。
        // UA/viewport 硬失败;运行时验证随引擎启用(视觉模型服务)
        crate::engines::client::playwright::apply_browser_fingerprint(
            &page,
            &self.ua_pool,
            request.mobile,
        )
        .await?;

        // Navigate to target URL
        info!("MLLM engine navigating to: {}", request.url);
        page.goto(&request.url)
            .await
            .map_err(|e| EngineError::BrowserError(format!("Navigation failed: {}", e)))?;

        tokio::time::sleep(Duration::from_millis(1500)).await;

        // Extract goal from URL or request context
        let goal = extract_goal_from_request(request);

        let final_content = self.run_agentic_loop(&page, &goal, request, start).await?;

        // Take final screenshot if requested
        let mut screenshot: Option<String> = None;
        if request.needs_screenshot {
            screenshot = Some(self.take_screenshot_base64(&page).await?);
        }

        drop(pooled_page);

        Ok(InternalScrapeResponse {
            js_degraded: false,
            status_code: 200,
            content: final_content,
            screenshot,
            content_type: "text/html".to_string(),
            headers: HashMap::new(),
            response_time_ms: start.elapsed().as_millis() as u64,
            final_url: None,
            captured_responses: Vec::new(),
        })
    }

    /// Agentic 主循环:max_iterations / 剩余时间双护栏,逐轮
    /// 截图 → AX 大纲 → 视觉模型 → 解析决策 → 终态判定 → 执行。
    /// 达到终态(max_iterations / 临近超时 / Extract / Done)返回当前页面内容;
    /// 中间步骤失败按原语义向上传播。
    async fn run_agentic_loop(
        &self,
        page: &chromiumoxide::Page,
        goal: &str,
        request: &InternalScrapeRequest,
        start: Instant,
    ) -> Result<String, EngineError> {
        // Agentic loop
        let mut last_result: Option<ActionResult> = None;
        let mut iteration: u8 = 0;
        let final_content;

        loop {
            if let Some(content) = self.guard_fallback(page, request, start, iteration).await {
                final_content = content;
                break;
            }

            iteration += 1;
            debug!(
                "MLLM agentic loop iteration {}/{}",
                iteration, self.config.max_iterations
            );

            let vision_response = self
                .request_vision_response(page, goal, iteration, last_result.as_ref())
                .await?;

            // Step 3: Parse decision
            let decision = match parse_decision(&vision_response) {
                Ok(d) => d,
                Err(e) => {
                    warn!("Failed to parse vision response: {}", e);
                    last_result = Some(ActionResult::err(format!("parse error: {}", e)));
                    continue;
                }
            };

            debug!("MLLM decision at iteration {}: {:?}", iteration, decision);

            // Step 4: Check for terminal decisions
            if let Some(content) = terminal_decision_content(page, &decision, iteration).await {
                final_content = content;
                break;
            }

            // Step 5: Execute the decision
            let result = execute_decision(&decision, page).await;
            last_result = Some(result);

            // Brief pause for page to settle after action
            tokio::time::sleep(Duration::from_millis(500)).await;
        }

        Ok(final_content)
    }

    /// 双护栏检查(max_iterations / 临近超时):命中即返回当前页面内容作为
    /// 兜底终态;未命中返回 None(调用方继续本轮迭代)。
    async fn guard_fallback(
        &self,
        page: &chromiumoxide::Page,
        request: &InternalScrapeRequest,
        start: Instant,
        iteration: u8,
    ) -> Option<String> {
        if iteration >= self.config.max_iterations {
            warn!(
                "MLLM engine reached max_iterations ({}) for {}",
                self.config.max_iterations, request.url
            );
            // Fallback: return current page content
            return Some(get_page_content(page).await);
        }

        let elapsed = start.elapsed();
        if elapsed + Duration::from_secs(10) > request.timeout {
            warn!("MLLM engine approaching timeout for {}", request.url);
            return Some(get_page_content(page).await);
        }

        None
    }

    /// 单轮视觉决策输入:截图 → AX 大纲 → 视觉模型调用。
    async fn request_vision_response(
        &self,
        page: &chromiumoxide::Page,
        goal: &str,
        iteration: u8,
        last_result: Option<&ActionResult>,
    ) -> Result<String, EngineError> {
        // Step 1: Take screenshot
        let screenshot_b64 = self.take_screenshot_base64(page).await?;

        // Step 1.5: AX tree outline (F12 hybrid channel; None when disabled)
        let ax_outline = self.capture_ax_outline(page).await;

        // Step 2: Send to vision model
        let prompt = self.build_vision_prompt(goal, iteration, last_result, ax_outline.as_deref());
        self.vision_adapter
            .send_vision_request(
                &screenshot_b64,
                &prompt,
                &self.config.system_prompt,
                &self.config.vision_model,
            )
            .await
            .map_err(|e| {
                error!("Vision model call failed: {}", e);
                EngineError::Other(format!("Vision model call failed: {}", e))
            })
    }
}

/// Extract/Done 终态判定:命中返回当前页面内容(含原日志),否则 None。
async fn terminal_decision_content(
    page: &chromiumoxide::Page,
    decision: &MllmDecision,
    iteration: u8,
) -> Option<String> {
    match decision {
        MllmDecision::Extract { .. } => {
            let content = get_page_content(page).await;
            info!("MLLM engine extracted content at iteration {}", iteration);
            Some(content)
        }
        MllmDecision::Done { .. } => {
            let content = get_page_content(page).await;
            info!(
                "MLLM engine completed navigation at iteration {}",
                iteration
            );
            Some(content)
        }
        _ => None,
    }
}

#[async_trait]
impl ScraperEngine for MllmEngine {
    async fn scrape(
        &self,
        request: &InternalScrapeRequest,
    ) -> Result<InternalScrapeResponse, EngineError> {
        // SSRF protection
        validators::validate_url(&request.url)
            .await
            .map_err(|e| EngineError::Other(format!("SSRF protection: {}", e)))?;

        let start = Instant::now();
        let timeout_duration = request.timeout;

        let pool = self.get_pool()?;

        tokio::time::timeout(
            timeout_duration,
            self.scrape_with_pool(pool, request, start),
        )
        .await
        .map_err(|_| EngineError::Timeout(timeout_duration))?
    }

    fn support_score(&self, request: &InternalScrapeRequest) -> u8 {
        if request.needs_mllm {
            100 // Highest priority for MLLM requests
        } else {
            5 // Low priority for non-MLLM requests
        }
    }

    fn name(&self) -> &'static str {
        "mllm"
    }

    fn max_response_time(&self) -> Duration {
        self.mrt
    }
}

/// 从请求中提取导航目标描述
fn extract_goal_from_request(request: &InternalScrapeRequest) -> String {
    // 如果有 extraction prompt,用作目标
    // 否则使用 URL 作为默认目标
    if let Some(body) = &request.body {
        if !body.is_empty() {
            return body.clone();
        }
    }
    format!("Navigate to and extract content from: {}", request.url)
}

/// 获取页面 HTML 内容
async fn get_page_content(page: &chromiumoxide::Page) -> String {
    match page
        .evaluate_expression("document.documentElement.outerHTML")
        .await
    {
        Ok(result) => result.into_value::<String>().unwrap_or_default(),
        Err(e) => {
            error!("Failed to get page content: {}", e);
            String::new()
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::domain::services::llm::vision_adapter::VisionAdapterTrait;

    /// 测试桩适配器:prompt 构造不触网,永不被调用(调用即 panic 显性失败)
    struct UnusedVisionAdapter;

    #[async_trait::async_trait]
    impl VisionAdapterTrait for UnusedVisionAdapter {
        async fn send_vision_request(
            &self,
            _screenshot_b64: &str,
            _prompt: &str,
            _system_prompt: &str,
            _model: &str,
        ) -> anyhow::Result<String> {
            panic!("vision adapter must not be called during prompt construction")
        }
    }

    fn engine_with(ax_tree_enabled: bool) -> MllmEngine {
        MllmEngine::new(
            Arc::new(UnusedVisionAdapter),
            MllmEngineConfig {
                ax_tree_enabled,
                ..MllmEngineConfig::default()
            },
        )
    }

    /// ax_tree_enabled=false:prompt 与旧版(无 outline 参数)逐字节一致(快照)
    #[test]
    fn prompt_without_outline_is_byte_identical_to_legacy() {
        let engine = engine_with(false);
        let last = ActionResult::err("click failed for '#x': not found");
        let prompt = engine.build_vision_prompt("buy milk", 2, Some(&last), None);
        let mut legacy = format!(
            "Analyze the current page screenshot and decide the next action to accomplish this goal: {}\n",
            "buy milk"
        );
        legacy.push_str(&format!("\nThis is iteration {}/{}.\n", 2, 10));
        legacy.push_str("\nLast action failed: click failed for '#x': not found. Please try a different approach.\n");
        legacy.push_str("\nRespond with a JSON object representing your decision.");
        assert_eq!(prompt, legacy);
    }

    /// outline 注入:追加「Page accessibility outline:\n{outline}\nUse [id] to reference elements」
    #[test]
    fn prompt_with_outline_appends_ax_outline_block() {
        let engine = engine_with(true);
        let base = engine.build_vision_prompt("goal", 1, None, None);
        let with = engine.build_vision_prompt("goal", 1, None, Some("[12] heading: Welcome"));
        assert!(
            with.starts_with(&base),
            "outline 只追加在尾部,不改动既有段落"
        );
        assert!(
            with.ends_with("\nPage accessibility outline:\n[12] heading: Welcome\nUse [id] to reference elements"),
            "追加块格式固定: {}",
            with
        );
    }
}