crawlrs 0.2.0

crawlrs - Enterprise-grade web scraping and crawling platform built with Rust
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
// Copyright (c) 2025-2026 Kirky.X🌠
// SPDX-License-Identifier: Apache-2.0

//! EngineClient - Unified public API for scraping operations
//!
//! This module provides the single entry point for all scraping operations.
//! All internal implementation details (UA rotation, circuit breaker, engine selection)
//! are encapsulated within EngineClient and not exposed to callers.

// ARC-002: DTO 类型已拆分至 `types.rs`,此处 re-export 保持向后兼容。
pub use crate::engines::types::{
    validate_session_id, CapturedResponse, HttpMethod, InternalCapturedResponse,
    InternalPageAction, InternalScrapeRequest, InternalScrapeResponse, InternalScreenshotConfig,
    PageAction, ScrapeOptions, ScrapeOptionsBuilder, ScrapeRequest, ScrapeResponse,
    ScreenshotConfig, ScrollDirection, WaitFor, MAX_SESSION_ID_LEN,
};

use crate::engines::health_monitor::{AggregateHealthStatus, EngineHealthMonitor};
use crate::engines::router::{EngineRouter, EngineRouterTrait};
use crate::engines::validators::validate_url;
use crate::utils::retry::RetryReason;
use std::sync::Arc;
use std::time::Duration;
use thiserror::Error;

/// Engine error types for EngineClient operations.
#[derive(Error, Debug, Clone)]
pub enum EngineError {
    /// Request failed with a specific error message
    #[error("Request failed: {0}")]
    RequestFailed(String),

    /// Request timed out
    #[error("Request timed out after {0:?}")]
    Timeout(Duration),

    /// All engines are unavailable
    #[error("All engines failed: {0}")]
    AllEnginesFailed(String),

    /// No engines available for the request
    #[error("No engines available")]
    NoEnginesAvailable,

    /// Invalid URL provided
    #[error("Invalid URL: {0}")]
    InvalidUrl(String),

    /// SSRF protection triggered
    #[error("SSRF protection: {0}")]
    SsrfProtection(String),

    /// 内存水位 Critical 且无可用 JS 引擎(moli 不可用)且配置禁用静态回退时的快速失败
    #[error("Memory constrained: {0}")]
    MemoryConstrained(String),

    /// 引擎不支持该操作(诚实拒绝,不静默降级——如 moli 对响应级等待变体)
    #[error("Unsupported operation: {0}")]
    Unsupported(String),

    /// Browser/Playwright error
    #[error("Browser error: {0}")]
    BrowserError(String),

    /// 反爬虫检测命中(1.7)
    ///
    /// 携带 `antibot::classifier::classify` 给出的 `Detection::reason` 字符串。
    /// 可重试:`is_retryable=true`,`retry_reason=AntiBot`,
    /// 路由层据此切身份(UA/代理/stealth)+ 强制浏览器/FlareSolverr 引擎改派。
    #[error("Anti-bot detected: {0}")]
    AntiBotDetected(String),

    /// 引擎特性切换
    ///
    /// 当引擎降级或特性切换(如 Chrome → HTTP、JS 渲染失败回退到静态抓取)时触发。
    /// 携带描述信息(如 "chrome_degraded_to_http")。
    /// 可重试:`is_retryable=true`,`retry_reason=FeatureToggle`,
    /// 路由层据此换引擎重试,并按 `RetryDirective` 立即轮换 UA(attempt=0 特例)。
    #[error("Feature toggle: {0}")]
    FeatureToggle(String),

    /// Request expired (circuit breaker open)
    #[error("Request expired")]
    Expired,

    /// 引擎级 MRT 超时
    ///
    /// 区别于 `Timeout`(请求整体超时),此变体表示单引擎在 MRT(Maximum Response Time)
    /// 内未完成,router 触发瀑布式 fallback 切换到下一引擎。
    ///
    /// 可重试:`is_retryable=true`,`retry_reason=Transient`(同引擎可重试,
    /// 但 router 会优先切下一引擎而非重试同引擎)。
    #[error("Engine {engine} exceeded MRT of {mrt:?}")]
    EngineMrtExceeded {
        /// 引擎名称
        engine: String,
        /// 实际使用的 MRT(即 `effective_timeout = min(remaining, engine_mrt)`)
        mrt: Duration,
    },

    /// Other error
    #[error("Other error: {0}")]
    Other(String),

    /// Internal error
    #[error("Internal error: {0}")]
    Internal(String),
}

// From implementations for EngineError
impl From<String> for EngineError {
    fn from(msg: String) -> Self {
        EngineError::RequestFailed(msg)
    }
}

impl From<&str> for EngineError {
    fn from(msg: &str) -> Self {
        EngineError::RequestFailed(msg.to_string())
    }
}

impl From<anyhow::Error> for EngineError {
    fn from(err: anyhow::Error) -> Self {
        EngineError::Internal(err.to_string())
    }
}

impl EngineError {
    /// Check if the error is retryable.
    pub fn is_retryable(&self) -> bool {
        match self {
            Self::RequestFailed(_) => true,
            Self::Timeout(_) => true,
            Self::NoEnginesAvailable => false,
            Self::InvalidUrl(_) => false,
            Self::SsrfProtection(_) => false,
            Self::BrowserError(_) => true,
            Self::AntiBotDetected(_) => true,
            Self::FeatureToggle(_) => true,
            Self::Internal(_) => false,
            Self::AllEnginesFailed(_) => false,
            Self::Expired => false,
            Self::Other(_) => false,
            // 内存水位 Critical 快速失败:重试不解除内存压力,等水位回升再接受请求
            Self::MemoryConstrained(_) => false,
            Self::Unsupported(_) => false,
            // 引擎级 MRT 超时:可重试(router 优先切下一引擎,而非重试同引擎)
            Self::EngineMrtExceeded { .. } => true,
        }
    }

    /// 将错误归类为重试原因。
    ///
    /// 调用方应先查 [`is_retryable()`](Self::is_retryable):
    /// 不可重试的错误虽返回 [`RetryReason::Transient`],
    /// 但不会被重试系统消费,仅作占位以保证返回值完备。
    ///
    /// 映射:
    /// - `RequestFailed` / `Timeout` / `BrowserError` → `Transient`(同引擎可重试)
    /// - `AntiBotDetected` → `AntiBot`(需切身份 + 浏览器引擎改派)
    /// - `FeatureToggle` → `FeatureToggle`(需换引擎重试)
    /// - 其余不可重试变体 → `Transient`(占位)
    pub fn retry_reason(&self) -> RetryReason {
        match self {
            Self::AntiBotDetected(_) => RetryReason::AntiBot,
            Self::FeatureToggle(_) => RetryReason::FeatureToggle,
            Self::RequestFailed(_)
            | Self::Timeout(_)
            | Self::BrowserError(_)
            | Self::NoEnginesAvailable
            | Self::InvalidUrl(_)
            | Self::SsrfProtection(_)
            | Self::Internal(_)
            | Self::AllEnginesFailed(_)
            | Self::Expired
            | Self::Other(_)
            | Self::MemoryConstrained(_)
            | Self::Unsupported(_)
            | Self::EngineMrtExceeded { .. } => RetryReason::Transient,
        }
    }
}

use async_trait::async_trait;

/// ScraperEngine trait - abstraction for different scraping engines
///
/// This trait defines the interface that all scraping engines must implement.
/// Each engine provides different capabilities (JS rendering, TLS fingerprinting, etc.)
/// and is scored based on how well it matches the request requirements.
#[async_trait]
pub trait ScraperEngine: Send + Sync {
    /// Perform a scraping request
    async fn scrape(
        &self,
        request: &InternalScrapeRequest,
    ) -> Result<InternalScrapeResponse, EngineError>;

    /// Calculate a support score for the given request
    ///
    /// Returns a score from 0-100 indicating how well this engine
    /// supports the request. Higher scores indicate better support.
    fn support_score(&self, request: &InternalScrapeRequest) -> u8;

    /// Get the engine name
    fn name(&self) -> &'static str;

    /// Check if the engine supports TLS fingerprinting
    ///
    /// Returns true if the engine can perform TLS fingerprinting
    /// for anti-fingerprinting purposes.
    fn supports_tls_fingerprint(&self) -> bool {
        false
    }

    /// 引擎级最大响应时间(MRT, Maximum Response Time)。
    ///
    /// 用于 router 顺序 fallback 路径瀑布式超时:单引擎调用以
    /// `min(remaining_timeout, engine.max_response_time)` 包裹,
    /// 超 MRT 即切下一引擎(不切整体失败)。race 模式不受影响。
    ///
    /// # 默认实现
    ///
    /// 返回 30 秒(与 `EngineTimeoutSettings::default_timeout_seconds` 默认值一致)。
    /// 各引擎按类型覆写为更精确的 MRT:
    /// - HTTP fetch 引擎(reqwest):5 秒(`fetch_seconds`)
    /// - CDP/浏览器引擎(playwright / flaresolverr_cdp / flaresolverr_full):30 秒(`cdp_seconds`)
    /// - TLS 指纹引擎(flaresolverr_tls):15 秒(`tls_seconds`)
    ///
    /// # 注入
    ///
    /// 引擎构造时应从 `EngineTimeoutSettings` 注入对应字段,避免硬编码
    /// (参考 `ReqwestEngine::new_with_timeout` 模式)。
    ///
    /// 删除 `DEFAULT_ENGINE_MRT` 常量,默认实现直接返回
    /// `Duration::from_secs(30)`,避免与 `EngineTimeoutSettings::default_timeout_seconds`
    /// 形成隐式耦合(注释承诺"保持一致"但代码无引用关系)。
    fn max_response_time(&self) -> Duration {
        Duration::from_secs(30)
    }
}

/// Health status of the engine system.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub enum EngineHealthStatus {
    /// All engines are operational
    #[default]
    Healthy,
    /// Some engines are degraded or unavailable
    Degraded {
        /// List of engines that are unhealthy
        unhealthy_engines: Vec<String>,
        /// Message describing the degradation
        message: String,
    },
    /// No engines are available
    Unavailable {
        /// Message describing the unavailability
        message: String,
    },
}

/// Trait for EngineClient - enables dependency injection
#[async_trait]
pub trait EngineClientTrait: Send + Sync {
    /// Perform a scraping request
    async fn scrape(&self, request: &ScrapeRequest) -> Result<ScrapeResponse, EngineError>;

    /// Perform health check on all registered engines
    async fn health_check(&self) -> EngineHealthStatus;

    /// Get the number of registered engines
    fn engine_count(&self) -> usize;

    /// Get list of registered engine names
    fn registered_engines(&self) -> Vec<String>;
}

/// Engine client - the single entry point for all scraping operations.
///
/// This struct encapsulates all internal implementation details:
/// - User-Agent rotation
/// - Circuit breaker state
/// - Engine selection algorithm
/// - Retry logic and backoff
/// - Connection pooling
///
/// Callers should use this struct for all scraping operations instead of
/// interacting with engines directly.
#[derive(Clone)]
pub struct EngineClient {
    /// Internal router for engine selection and request routing
    router: Arc<dyn EngineRouterTrait>,
    /// Internal health monitor for tracking engine health(auth-off 回退路径持有;
    /// auth-on 面的探活经 garrison registry,monitor 由 registry 检查项持有)
    #[cfg(not(feature = "auth"))]
    health_monitor: Arc<EngineHealthMonitor>,
    /// garrison 健康注册表(S2 吸收,auth 面):状态登记/聚合/check_timeout
    /// 护栏与 `health_routes` 路由暴露由库承载;auth-off 构建不编译此字段,
    /// 健康查询走监控器自有聚合回退。
    #[cfg(feature = "auth")]
    health_registry: Arc<garrison::health::HealthRegistry>,
}

impl EngineClient {
    /// Create a new EngineClient with default configuration.
    pub fn new() -> Self {
        let router: Arc<dyn EngineRouterTrait> = Arc::new(EngineRouter::new(Vec::new()));
        Self::with_router(router)
    }

    /// Create a new EngineClient with a custom router.
    pub fn with_router(router: Arc<dyn EngineRouterTrait>) -> Self {
        let health_monitor = Arc::new(EngineHealthMonitor::new(Vec::new()));
        #[cfg(feature = "auth")]
        let health_registry = Arc::new(
            crate::engines::health_monitor::garrison_health::build_health_registry(&health_monitor),
        );
        Self {
            router,
            #[cfg(not(feature = "auth"))]
            health_monitor,
            #[cfg(feature = "auth")]
            health_registry,
        }
    }

    /// Create an EngineClient with engines pre-registered.
    pub fn with_engines(engines: Vec<Arc<dyn ScraperEngine>>) -> Self {
        let router: Arc<dyn EngineRouterTrait> = Arc::new(EngineRouter::new(engines.clone()));
        let engines_for_health = Vec::new(); // Will need to get engines from router
        let health_monitor = Arc::new(EngineHealthMonitor::new(engines_for_health));
        #[cfg(feature = "auth")]
        let health_registry = Arc::new(
            crate::engines::health_monitor::garrison_health::build_health_registry(&health_monitor),
        );
        Self {
            router,
            #[cfg(not(feature = "auth"))]
            health_monitor,
            #[cfg(feature = "auth")]
            health_registry,
        }
    }

    /// Perform a scraping request.
    ///
    /// This method automatically:
    /// - Validates the URL
    /// - Selects the optimal engine based on request requirements
    /// - Handles retries and circuit breaking
    /// - Rotates user agents
    /// - Returns a unified response
    ///
    /// # Arguments
    ///
    /// * `request` - The scrape request containing URL and options
    ///
    /// # Returns
    ///
    /// * `Ok(ScrapeResponse)` on success
    /// * `Err(EngineError)` on failure
    pub async fn scrape(&self, request: &ScrapeRequest) -> Result<ScrapeResponse, EngineError> {
        if let Err(e) = validate_url(&request.url).await {
            return Err(EngineError::SsrfProtection(e.to_string()));
        }

        let internal_request = request.to_internal()?;

        // Route to appropriate engine
        match self.router.route(&internal_request).await {
            Ok(response) => Ok(response.to_public(&request.url)),
            Err(e) => Err(convert_error(e)),
        }
    }

    /// Perform health check on all registered engines.
    ///
    /// # Returns
    ///
    /// * `EngineHealthStatus` indicating the health of all engines
    ///
    /// auth-on:探活经 garrison `HealthRegistry::check_all`(单项 check_timeout
    /// 护栏 + 并发执行),聚合口径按报告明细重算,与 auth-off 回退路径一致。
    /// auth-off:监控器自有顺序探活 + `get_aggregate_status` 聚合(精简回退)。
    pub async fn health_check(&self) -> EngineHealthStatus {
        #[cfg(feature = "auth")]
        let status = {
            use crate::engines::health_monitor::garrison_health;
            let report = self.health_registry.check_all().await;
            match garrison_health::map_report(&report) {
                AggregateHealthStatus::Healthy => EngineHealthStatus::Healthy,
                AggregateHealthStatus::Degraded(unhealthy_engines) => {
                    let count = unhealthy_engines.len();
                    EngineHealthStatus::Degraded {
                        unhealthy_engines,
                        message: format!("{} engines degraded", count),
                    }
                }
                AggregateHealthStatus::Unavailable => EngineHealthStatus::Unavailable {
                    message: "All engines unavailable".to_string(),
                },
            }
        };

        #[cfg(not(feature = "auth"))]
        let status = {
            // Perform health check on all engines
            self.health_monitor.perform_all_health_checks().await;

            // Get aggregate status and convert to public type
            let status = self.health_monitor.get_aggregate_status().await;

            match status {
                AggregateHealthStatus::Healthy => EngineHealthStatus::Healthy,
                AggregateHealthStatus::Degraded(unhealthy_engines) => {
                    let count = unhealthy_engines.len();
                    EngineHealthStatus::Degraded {
                        unhealthy_engines,
                        message: format!("{} engines degraded", count),
                    }
                }
                AggregateHealthStatus::Unavailable => EngineHealthStatus::Unavailable {
                    message: "All engines unavailable".to_string(),
                },
            }
        };

        status
    }

    /// garrison 健康注册表(auth 面):供 `health_routes` 探针路由装配使用。
    #[cfg(feature = "auth")]
    pub fn health_registry(&self) -> Arc<garrison::health::HealthRegistry> {
        Arc::clone(&self.health_registry)
    }

    /// Get the number of registered engines.
    pub fn engine_count(&self) -> usize {
        self.router.registered_engines().len()
    }

    /// Get list of registered engine names.
    pub fn registered_engines(&self) -> Vec<String> {
        self.router.registered_engines()
    }

    /// 注入 robots.txt 解析出的 Crawl-delay(F5:联动 per-host 限速时钟)。
    ///
    /// 首值生效不回退;对未经过纪律门的 host 惰性建档。
    /// 委托给 [`EngineRouterTrait::set_crawl_delay`],与 `scrape` 同走
    /// 唯一公共入口模式。
    pub fn set_crawl_delay(&self, host: &str, delay: Duration) {
        self.router.set_crawl_delay(host, delay);
    }
}

impl Default for EngineClient {
    fn default() -> Self {
        Self::new()
    }
}

#[async_trait]
impl EngineClientTrait for EngineClient {
    async fn scrape(&self, request: &ScrapeRequest) -> Result<ScrapeResponse, EngineError> {
        self.scrape(request).await
    }

    async fn health_check(&self) -> EngineHealthStatus {
        self.health_check().await
    }

    fn engine_count(&self) -> usize {
        self.engine_count()
    }

    fn registered_engines(&self) -> Vec<String> {
        self.registered_engines()
    }
}

/// Convert internal errors to public EngineError
fn convert_error(e: EngineError) -> EngineError {
    match e {
        EngineError::RequestFailed(msg) => EngineError::RequestFailed(msg),
        EngineError::Timeout(duration) => EngineError::Timeout(duration),
        EngineError::AllEnginesFailed(msg) => {
            if msg.contains("No suitable engines") {
                EngineError::NoEnginesAvailable
            } else {
                EngineError::RequestFailed(msg)
            }
        }
        EngineError::SsrfProtection(msg) => EngineError::SsrfProtection(msg),
        EngineError::BrowserError(msg) => EngineError::BrowserError(msg),
        EngineError::AntiBotDetected(msg) => EngineError::AntiBotDetected(msg),
        EngineError::FeatureToggle(msg) => EngineError::FeatureToggle(msg),
        EngineError::Expired => EngineError::Internal("Request expired".to_string()),
        EngineError::Other(msg) => EngineError::Internal(msg),
        EngineError::MemoryConstrained(msg) => EngineError::MemoryConstrained(msg),
        EngineError::Unsupported(msg) => EngineError::Unsupported(msg),
        EngineError::NoEnginesAvailable => EngineError::NoEnginesAvailable,
        EngineError::InvalidUrl(msg) => EngineError::InvalidUrl(msg),
        EngineError::Internal(msg) => EngineError::Internal(msg),
        EngineError::EngineMrtExceeded { engine, mrt } => {
            EngineError::EngineMrtExceeded { engine, mrt }
        }
    }
}

#[cfg(test)]
#[path = "tests/engine_client_test.rs"]
mod tests;