crawlrs 0.2.0

crawlrs - Enterprise-grade web scraping and crawling platform built with Rust
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
// Copyright (c) 2025-2026 Kirky.X🌠
// SPDX-License-Identifier: Apache-2.0

//! EngineRouter - Internal routing logic for engine selection
//!
//! This module handles the internal routing logic for selecting appropriate
//! scraping engines based on request requirements.
//! This is an internal implementation detail.

use crate::engines::circuit_breaker::CircuitBreaker;
use crate::engines::engine_client::{
    EngineError, InternalScrapeRequest, InternalScrapeResponse, ScraperEngine,
};
use crate::engines::session::SessionManager;
use crate::engines::validators::validate_url;
use crate::utils::hedge::HedgeController;
use crate::utils::host_discipline::HostDiscipline;
use crate::utils::retry::{RetryReason, RetryTracker};
use crate::utils::ua_pool::UaPool;
use dashmap::DashMap;

/// host 反爬记忆 TTL:期内同 host 请求直接按 needs_js 路由
const HOST_BROWSER_HINT_TTL: std::time::Duration = std::time::Duration::from_secs(600);
use log::{info, warn};

use crate::i18n::tr_log_args;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::{Duration, Instant};

pub use crate::engines::router_metrics::{EngineStats, RouterMetrics};

// Section: EngineRouterTrait Definition

/// Trait for EngineRouter - enables dependency injection
#[async_trait::async_trait]
pub trait EngineRouterTrait: Send + Sync {
    /// Route a request to the optimal engine
    async fn route(
        &self,
        request: &InternalScrapeRequest,
    ) -> Result<InternalScrapeResponse, EngineError>;

    /// Aggregate results from multiple engines
    async fn aggregate(
        &self,
        request: &InternalScrapeRequest,
    ) -> Result<InternalScrapeResponse, EngineError>;

    /// Get engine statistics
    fn get_engine_stats(&self) -> std::collections::HashMap<String, EngineStats>;

    /// Reset statistics for a specific engine
    fn reset_engine_stats(&self, engine_name: &str);

    /// Get list of registered engine names
    fn registered_engines(&self) -> Vec<String>;

    /// 注入 robots.txt 解析出的 Crawl-delay(F5:联动 per-host 限速时钟)。
    ///
    /// 默认 no-op:测试替身不持有 host 纪律状态,无需实现。
    fn set_crawl_delay(&self, _host: &str, _delay: Duration) {}
}

/// 负载均衡策略
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum LoadBalancingStrategy {
    /// 轮询
    RoundRobin,
    /// 加权轮询 (基于成功率)
    WeightedRoundRobin,
    /// 最少连接/最少使用
    LeastConnections,
    /// 最快响应时间
    FastestResponse,
    /// 随机
    Random,
    /// 智能混合 (默认)
    SmartHybrid,
}

/// 引擎路由器
///
/// 负责根据请求特征和负载均衡策略选择合适的抓取引擎
pub struct EngineRouter {
    /// 引擎列表
    engines: Vec<Arc<dyn ScraperEngine>>,
    /// 熔断器
    circuit_breaker: Arc<CircuitBreaker>,
    /// 引擎性能统计(DashMap 优化并发读写,避免 RwLock 借用)
    engine_stats: Arc<DashMap<String, EngineStats>>,
    /// host 反爬记忆(F7):host → 检出 needs_browser 的时刻。
    /// TTL 内同 host 请求直接按 needs_js 路由,避免对已知挑战站点
    /// 每次都从 HTTP 引擎撞墙;读取时惰性清除过期条目。
    host_browser_hint: DashMap<String, Instant>,
    /// per-host 纪律:礼貌限速 + 429 冷却 + 裸 403 连击 → TLS 指纹升级。
    /// route_internal 发请求前 `gate`,反爬检出/成功后 `observe`。
    host_discipline: HostDiscipline,
    /// 当前轮询索引
    round_robin_index: Arc<parking_lot::Mutex<usize>>,
    /// 负载均衡策略
    strategy: LoadBalancingStrategy,
    /// 路由层指标
    metrics: Arc<RouterMetrics>,
    /// 最大引擎尝试次数
    max_engine_attempts: usize,
    /// 最大重试次数 (总请求时间限制)
    max_retries: usize,
    /// 是否启用特征检测过滤
    feature_filter_enabled: bool,
    /// 是否启用并发竞速模式
    race_mode_enabled: bool,
    /// 动态阈值因子 (根据历史数据调整)
    dynamic_threshold_factor: f64,
    /// UA 池(原在 route 内每请求 UaPool::new 分配 44 个 profile)
    ///
    /// UaPool 内部全 `&'static str`,构造后只读,可安全跨线程共享(Send+Sync 自动派生)。
    /// 用 `Arc` 是因为 `EngineRouter` 本身可能被 Clone 共享。
    ua_pool: Arc<UaPool>,
    /// Session 身份管理器(F2):代理 + UA/TLS 档 + Cookie 三件套绑定与
    /// 记分退休。`enabled=false`(默认)时 route_sequential 不经 session
    /// 路径,维持 pick_seeded 盲轮换现状。
    session: SessionManager,
    /// 失败现场快照器(F4):antibot/限速检出时按签名键存档响应现场。
    /// `None`(默认,`failure_snapshot_dir` 未配置)时调用点零成本短路。
    failure_snapshot:
        Option<Arc<crate::infrastructure::observability::failure_snapshot::FailureSnapshotter>>,
    /// Hedge 控制器
    ///
    /// 记录 race 胜出引擎延迟,估算 P84 阈值,供未来顺序路径决策是否发送副本请求。
    /// `HedgeController` 内部全 `Atomic*`,无锁线程安全,可直接共享无需 `Arc`。
    hedge_controller: HedgeController,
    /// 内存水位镜像(engine-moli 特性):评分后按水位修正引擎分数;
    /// `None`(未装配)时路由行为与无水位调度完全一致。
    #[cfg(feature = "engine-moli")]
    memory: Option<Arc<crate::engines::memory_watermark::MemoryWatermark>>,
    /// 内存水位调度参数:Critical 且无可用 JS 引擎时是否回退静态抓取
    /// (false = 快速失败 `EngineError::MemoryConstrained`)。
    #[cfg(feature = "engine-moli")]
    memory_fallback_to_static: bool,
}

impl EngineRouter {
    /// 创建新的引擎路由器
    ///
    /// # 参数
    ///
    /// * `engines` - 引擎列表
    ///
    /// # 返回值
    ///
    /// 返回新的引擎路由器实例
    pub fn new(engines: Vec<Arc<dyn ScraperEngine>>) -> Self {
        Self::with_host_discipline(engines, HostDiscipline::with_defaults())
    }

    /// 使用给定 per-host 纪律组件创建路由器(F1/F5:平台侧从
    /// `PolitenessSettings` 构造 `HostDiscipline` 后注入)。
    ///
    /// Session 走 disabled 默认(现状盲轮换);需启用请用
    /// [`with_discipline_and_session`](Self::with_discipline_and_session)。
    ///
    /// 库内自建 router(robots/geolocation/search 等)不走此构造器,
    /// 维持 `new` 的 `with_defaults` 默认。
    pub fn with_host_discipline(
        engines: Vec<Arc<dyn ScraperEngine>>,
        host_discipline: HostDiscipline,
    ) -> Self {
        Self::with_discipline_and_session(engines, host_discipline, SessionManager::disabled())
    }

    /// 使用给定 per-host 纪律组件与 Session 身份管理器创建路由器(F2)
    ///
    /// `session` 持有的 UA 池与路由层共享同一实例(身份 profile 下标同源);
    /// `session` 为 `SessionManager::disabled()` 时行为与
    /// [`with_host_discipline`](Self::with_host_discipline) 完全一致。
    pub fn with_discipline_and_session(
        engines: Vec<Arc<dyn ScraperEngine>>,
        host_discipline: HostDiscipline,
        session: SessionManager,
    ) -> Self {
        let engine_stats = DashMap::with_capacity(8);
        for engine in &engines {
            engine_stats.insert(engine.name().to_string(), EngineStats::default());
        }

        Self {
            engines,
            circuit_breaker: Arc::new(CircuitBreaker::new()),
            engine_stats: Arc::new(engine_stats),
            host_browser_hint: DashMap::new(),
            host_discipline,
            round_robin_index: Arc::new(parking_lot::Mutex::new(0)),
            strategy: LoadBalancingStrategy::SmartHybrid,
            metrics: Arc::new(RouterMetrics::new()),
            max_engine_attempts: 3,
            max_retries: 5,                // 默认最大重试次数
            feature_filter_enabled: true,  // 默认启用特征检测过滤
            race_mode_enabled: false,      // 默认禁用并发竞速模式
            dynamic_threshold_factor: 1.0, // 默认动态阈值因子
            // Session 与路由共用同一 UA 池实例(身份 profile 下标同源)
            ua_pool: session.ua_pool().clone(),
            session,
            failure_snapshot: None, // F4:默认未配置(failure_snapshot_dir 空串)
            hedge_controller: HedgeController::with_defaults(), // 默认参数
            #[cfg(feature = "engine-moli")]
            memory: None,
            #[cfg(feature = "engine-moli")]
            memory_fallback_to_static: true,
        }
    }

    /// 使用指定熔断器和策略创建引擎路由器
    ///
    /// # 参数
    ///
    /// * `engines` - 引擎列表
    /// * `circuit_breaker` - 熔断器
    /// * `strategy` - 负载均衡策略
    ///
    /// # 返回值
    ///
    /// 返回新的引擎路由器实例
    pub fn with_circuit_breaker_and_strategy(
        engines: Vec<Arc<dyn ScraperEngine>>,
        circuit_breaker: Arc<CircuitBreaker>,
        strategy: LoadBalancingStrategy,
    ) -> Self {
        let engine_stats = DashMap::with_capacity(8);
        for engine in &engines {
            engine_stats.insert(engine.name().to_string(), EngineStats::default());
        }

        Self {
            engines,
            circuit_breaker,
            engine_stats: Arc::new(engine_stats),
            host_browser_hint: DashMap::new(),
            host_discipline: HostDiscipline::with_defaults(),
            round_robin_index: Arc::new(parking_lot::Mutex::new(0)),
            strategy,
            metrics: Arc::new(RouterMetrics::new()),
            max_engine_attempts: 3,
            max_retries: 5,
            feature_filter_enabled: true,
            race_mode_enabled: false,
            dynamic_threshold_factor: 1.0,
            ua_pool: Arc::new(UaPool::new()),    // 构造一次共享
            session: SessionManager::disabled(), // F2:默认不启用(现状盲轮换)
            failure_snapshot: None,              // F4:默认未配置(failure_snapshot_dir 空串)
            hedge_controller: HedgeController::with_defaults(), // 默认参数
            #[cfg(feature = "engine-moli")]
            memory: None,
            #[cfg(feature = "engine-moli")]
            memory_fallback_to_static: true,
        }
    }

    /// 注入内存水位调度(engine-moli 特性)
    ///
    /// 由 DI 装配在构造后、`Arc::new` 共享前调用;`watermark` 为 `None`
    /// (调度器关闭)时路由行为与无水位调度完全一致。`fallback_to_static`
    /// 取自 `[engines.memory_scheduler]` 配置(Critical 且无可用 JS 引擎时
    /// 回退静态抓取并标记 `js_degraded`,否则快速失败)。
    #[cfg(feature = "engine-moli")]
    pub fn set_memory_scheduler(
        &mut self,
        watermark: Option<Arc<crate::engines::memory_watermark::MemoryWatermark>>,
        fallback_to_static: bool,
    ) {
        self.memory = watermark;
        self.memory_fallback_to_static = fallback_to_static;
    }

    pub fn set_max_engine_attempts(&mut self, attempts: usize) {
        self.max_engine_attempts = attempts.max(1);
    }

    /// 注入失败现场快照器(F4)
    ///
    /// 由 bootstrap 侧在 `Arc::new` 之前从 `[observability]` 配置构造注入;
    /// `None`(默认未配置)时 route_sequential 调用点零成本短路。
    pub fn set_failure_snapshotter(
        &mut self,
        snapshotter: Option<
            Arc<crate::infrastructure::observability::failure_snapshot::FailureSnapshotter>,
        >,
    ) {
        self.failure_snapshot = snapshotter;
    }

    /// 设置最大重试次数 (用于限制总请求时间)
    pub fn set_max_retries(&mut self, retries: usize) {
        self.max_retries = retries.max(1);
    }

    /// 启用/禁用特征检测过滤
    pub fn set_feature_filter_enabled(&mut self, enabled: bool) {
        self.feature_filter_enabled = enabled;
    }

    /// 启用/禁用并发竞速模式
    pub fn set_race_mode_enabled(&mut self, enabled: bool) {
        self.race_mode_enabled = enabled;
    }

    /// 设置动态阈值因子
    pub fn set_dynamic_threshold_factor(&mut self, factor: f64) {
        self.dynamic_threshold_factor = factor.clamp(0.1, 2.0);
    }

    /// 设置负载均衡策略
    pub fn set_strategy(&mut self, strategy: LoadBalancingStrategy) {
        self.strategy = strategy;
    }

    /// 获取路由层指标
    pub fn metrics(&self) -> &Arc<RouterMetrics> {
        &self.metrics
    }

    /// 获取 Hedge 控制器引用
    ///
    /// 返回 `&HedgeController`,外部可读取 P84 阈值、样本数等观测值,
    /// 也可调用 `should_hedge` 决策是否发起副本(未来顺序路径用)。
    /// `record_latency` / `reset` 已限定为 `pub(crate)`,外部无法篡改状态
    /// (接口隔离修复)。
    pub fn hedge_controller(&self) -> &HedgeController {
        &self.hedge_controller
    }

    // select_optimal_engines, should_filter_by_feature, calculate_engine_score,
    // sort_candidates_by_strategy 已拆分到 engine_selector.rs (partial impl block)

    /// 更新引擎统计信息
    fn update_engine_stats(&self, engine_name: &str, success: bool, response_time: Duration) {
        // DashMap::get_mut 返回 RefMut guard,作用域结束自动释放
        if let Some(mut stat) = self.engine_stats.get_mut(engine_name) {
            let alpha = 0.1; // 平滑因子
            let current_success = if success { 1.0 } else { 0.0 };
            stat.success_rate = stat.success_rate * (1.0 - alpha) + current_success * alpha;

            let current_avg_ns = stat.avg_response_time.as_nanos() as f64;
            let response_ns = response_time.as_nanos() as f64;
            let new_avg_ns = current_avg_ns * (1.0 - alpha) + response_ns * alpha;
            stat.avg_response_time = Duration::from_nanos(new_avg_ns as u64);

            stat.last_used = Some(Instant::now());
            stat.usage_count += 1;
        }
    }

    /// 获取下一个轮询索引
    fn get_next_round_robin_index(&self, max_index: usize) -> usize {
        let mut index = self.round_robin_index.lock();
        let current = *index;
        *index = (*index + 1) % max_index;
        current
    }

    /// 路由请求到合适的引擎
    ///
    /// # 参数
    ///
    /// * `request` - 抓取请求
    ///
    /// # 返回值
    ///
    /// * `Ok(ScrapeResponse)` - 抓取响应
    /// * `Err(EngineError)` - 抓取过程中出现的错误
    pub async fn _route_impl(
        &self,
        request: &InternalScrapeRequest,
    ) -> Result<InternalScrapeResponse, EngineError> {
        if let Err(e) = validate_url(&request.url).await {
            return Err(EngineError::SsrfProtection(e.to_string()));
        }

        let timeout = request.timeout;

        // Wrap the entire operation with timeout
        tokio::time::timeout(timeout, self.route_internal(request))
            .await
            .map_err(|_| EngineError::Timeout(timeout))
            .and_then(|result| result)
    }

    /// 提取 RetryTracker 上限检查的公共逻辑(DRY)
    ///
    /// 在 AntiBot 和 retryable error 两个分支中,原代码有以下重复:
    /// 1. `if !tracker.should_retry(reason)` → warn + metrics + return
    /// 2. `if total_attempts >= max_retries` → warn + metrics + return
    ///
    /// 本方法封装两个检查,返回 `true` 表示应停止重试。
    /// 调用方在调用前已 `tracker.record(reason)`,调用后根据返回值决定 `return`。
    ///
    /// # 参数
    ///
    /// - `tracker`: 重试跟踪器(已 record 过 reason)
    /// - `reason`: 本次失败的 RetryReason(用于日志)
    /// - `total_attempts`: 当前总尝试次数
    /// - `max_retries`: 最大重试次数
    ///
    /// # 返回值
    ///
    /// - `true`: 应停止重试(已记录指标 + warn 日志)
    /// - `false`: 可继续重试
    fn should_stop_after_retry_check(
        &self,
        tracker: &RetryTracker,
        reason: RetryReason,
        total_attempts: usize,
        max_retries: usize,
    ) -> bool {
        if !tracker.should_retry(reason) {
            warn!(
                "RetryTracker blocked {:?} after total={} (anti_bot={}, feature_toggle={}), stopping",
                reason,
                tracker.total(),
                tracker.anti_bot(),
                tracker.feature_toggle()
            );
            self.metrics.total_requests.fetch_add(1, Ordering::Relaxed);
            self.metrics.failed_requests.fetch_add(1, Ordering::Relaxed);
            return true;
        }
        if total_attempts >= max_retries {
            warn!(
                "Max retries {} reached after {:?}, stopping",
                max_retries, reason
            );
            self.metrics.total_requests.fetch_add(1, Ordering::Relaxed);
            self.metrics.failed_requests.fetch_add(1, Ordering::Relaxed);
            return true;
        }
        false
    }

    // route_internal 已拆分到 route_sequential/ (partial impl block)
    // route_race_mode 已拆分到 route_race.rs (partial impl block)

    /// 记录 host 检出反爬挑战(F7)
    ///
    /// 与唯一生产调用点(`route_sequential/antibot.rs` 反爬检测分支)同门控 `content`
    /// feature:`content` 关闭时该分支不编译,方法随之不定义,避免 dead_code
    /// 告警。`host_needs_browser`(读取本 hint 表)在 `content` 关闭时仍被路由
    /// 前置查询调用(此时表恒空 → 恒 false),故保持 ungated。
    #[cfg(feature = "content")]
    pub(super) fn record_host_browser_hint(&self, host: &str) {
        self.host_browser_hint
            .insert(host.to_string(), Instant::now());
    }

    /// 查询 host 是否处于"需要浏览器"记忆期内(TTL 内有效,过期惰性清除)
    pub(super) fn host_needs_browser(&self, host: &str) -> bool {
        // 先 drop get 守卫再 remove——DashMap 同 shard 的 get 守卫存活期间
        // remove 会死锁(过期清除路径必触发)
        let fresh = match self.host_browser_hint.get(host) {
            Some(ts) => ts.elapsed() < HOST_BROWSER_HINT_TTL,
            None => return false,
        };
        if !fresh {
            self.host_browser_hint.remove(host);
        }
        fresh
    }

    /// 聚合多个引擎的搜索结果
    ///
    /// # 参数
    ///
    /// * `request` - 抓取请求
    ///
    /// # 返回值
    ///
    /// * `Ok(ScrapeResponse)` - 聚合后的抓取响应
    /// * `Err(EngineError)` - 如果所有引擎都失败
    pub async fn _aggregate_impl(
        &self,
        request: &InternalScrapeRequest,
    ) -> Result<InternalScrapeResponse, EngineError> {
        // SSRF 防护:与 _route_impl 保持一致,避免 aggregate 绕过内网地址校验
        if let Err(e) = validate_url(&request.url).await {
            return Err(EngineError::SsrfProtection(e.to_string()));
        }

        let candidates = self.select_optimal_engines(request);
        if candidates.is_empty() {
            // 错误消息与 _route_impl 保持一致,避免 EngineRouterTrait::aggregate
            // 与 route 行为不一致(LSP)
            return Err(EngineError::AllEnginesFailed(
                "No suitable engines available".to_string(),
            ));
        }

        let mut results = Vec::new();
        let mut errors = Vec::new();

        for (_, engine) in candidates {
            match engine.scrape(request).await {
                Ok(response) => results.push((engine.name().to_string(), response)),
                Err(e) => errors.push((engine.name().to_string(), e)),
            }
        }

        if results.is_empty() {
            return Err(EngineError::AllEnginesFailed(
                "All engines failed in aggregate".to_string(),
            ));
        }

        // 简单的结果聚合:取第一个成功的结果,但在实际应用中可以合并多个结果
        // 这里我们选择第一个成功的结果作为基础,并记录其他成功的结果数量
        let (primary_name, primary_response) = results.remove(0);
        info!(
            "Aggregation: Primary result from {}, {} other successes",
            primary_name,
            results.len()
        );

        self.circuit_breaker.record_success(&primary_name).await;

        for (name, _) in results {
            self.circuit_breaker.record_success(&name).await;
        }

        for (name, error) in errors {
            if error.is_retryable() {
                self.circuit_breaker.record_failure(&name).await;
            }
        }

        Ok(primary_response)
    }

    /// 获取引擎统计信息
    pub fn _get_engine_stats_impl(&self) -> std::collections::HashMap<String, EngineStats> {
        // DashMap → HashMap 一次性收集(trait 契约要求返回 HashMap)
        self.engine_stats
            .iter()
            .map(|r| (r.key().clone(), r.value().clone()))
            .collect()
    }

    /// 重置引擎统计信息
    pub fn _reset_engine_stats_impl(&self, engine_name: &str) {
        // DashMap::get_mut 返回 RefMut guard,作用域结束自动释放
        if let Some(mut stat) = self.engine_stats.get_mut(engine_name) {
            *stat = EngineStats::default();
        }
    }

    /// 注册引擎
    pub fn register_engine(&mut self, engine: Arc<dyn ScraperEngine>) {
        let name = engine.name().to_string();
        self.engines.push(engine);
        // DashMap::insert 直接替换/插入,无需获取写锁
        self.engine_stats
            .insert(name.clone(), EngineStats::default());
        info!(
            "{}",
            tr_log_args("engine-registered", &[("name", name.as_str().to_string())],)
        );
    }

    /// 获取所有已注册的引擎名称
    pub fn _registered_engines_impl(&self) -> Vec<String> {
        self.engines.iter().map(|e| e.name().to_string()).collect()
    }

    /// Get all registered engines (internal use only)
    #[doc(hidden)]
    pub fn get_engines(&self) -> &Vec<Arc<dyn ScraperEngine>> {
        &self.engines
    }
}

#[async_trait::async_trait]
impl EngineRouterTrait for EngineRouter {
    async fn route(
        &self,
        request: &InternalScrapeRequest,
    ) -> Result<InternalScrapeResponse, EngineError> {
        self._route_impl(request).await
    }

    async fn aggregate(
        &self,
        request: &InternalScrapeRequest,
    ) -> Result<InternalScrapeResponse, EngineError> {
        self._aggregate_impl(request).await
    }

    fn get_engine_stats(&self) -> std::collections::HashMap<String, EngineStats> {
        self._get_engine_stats_impl()
    }

    fn reset_engine_stats(&self, engine_name: &str) {
        self._reset_engine_stats_impl(engine_name)
    }

    fn registered_engines(&self) -> Vec<String> {
        self._registered_engines_impl()
    }

    fn set_crawl_delay(&self, host: &str, delay: Duration) {
        self.host_discipline.set_crawl_delay(host, delay);
    }
}

/// 检查引擎"成功"响应是否为反爬挑战页。
///
/// 将 `InternalScrapeResponse` 的 `HashMap<String,String>` headers 转为
/// `reqwest::header::HeaderMap` 后调用 `antibot::classify`。仅在 `antibot`
/// feature 启用时编译;关闭时 route_internal 的检测块被 cfg 移除,此函数也不存在。
#[cfg(feature = "content")]
pub(super) fn check_antibot_response(
    response: &InternalScrapeResponse,
    url: &str,
) -> Option<crate::engines::antibot::Detection> {
    use reqwest::header::{HeaderMap, HeaderName, HeaderValue};

    let mut header_map = HeaderMap::new();
    for (k, v) in &response.headers {
        if let Ok(name) = HeaderName::from_bytes(k.as_bytes()) {
            if let Ok(value) = HeaderValue::from_str(v) {
                header_map.append(name, value);
            }
        }
    }
    crate::engines::antibot::classify(response.status_code, &response.content, &header_map, url)
}

/// 对引擎"成功"响应运行 JS 升级探测。
///
/// 将 `InternalScrapeResponse` 的 headers 转为 `reqwest::header::HeaderMap` 后
/// 调用 [`crate::engines::upgrade_probe::JsUpgradeProbe::evaluate`]。返回
/// [`crate::engines::upgrade_probe::ProbeVerdict`],由 `route_internal` 消费
/// `upgrade=true` 时以 `needs_js=true` 重新改派浏览器引擎。
///
/// 与 [`check_antibot_response`] 不同,此函数**不** feature-gate:`upgrade_probe`
/// 模块是纯 Rust 无外部依赖,始终编译;SPA 空壳探测是通用能力,不应受 `antibot` 特性开关影响。
pub(super) fn check_js_upgrade_probe(
    response: &InternalScrapeResponse,
) -> crate::engines::upgrade_probe::ProbeVerdict {
    use reqwest::header::{HeaderMap, HeaderName, HeaderValue};

    let probe = crate::engines::upgrade_probe::JsUpgradeProbe::default();
    let mut header_map = HeaderMap::new();
    for (k, v) in &response.headers {
        if let Ok(name) = HeaderName::from_bytes(k.as_bytes()) {
            if let Ok(value) = HeaderValue::from_str(v) {
                header_map.append(name, value);
            }
        }
    }
    // evaluate docstring 明确「body_prefix」语义,
    // 传入完整 body 会让多次 `contains`/`find` 退化为 O(body_len)。
    // 截取前 PROBE_PREFIX_LEN 字节(char 边界安全,中文页面不 panic),
    // 覆盖典型 SPA 空壳的 head+顶层 body。
    let body_prefix = crate::common::text_slice::safe_prefix(
        &response.content,
        crate::engines::upgrade_probe::PROBE_PREFIX_LEN,
    );
    probe.evaluate(&header_map, body_prefix)
}

#[cfg(test)]
#[path = "../tests/router_test.rs"]
mod tests;

#[cfg(test)]
#[path = "../tests/router_tests_impl.rs"]
mod tests_impl;

// 引擎选择逻辑拆分
mod engine_selector;
// 路由模式拆分
mod route_race;
mod route_sequential;