use super::*;
use crate::engines::client::reqwest::ReqwestEngine;
use async_trait::async_trait;
use std::collections::HashMap;
use std::sync::atomic::AtomicU64;
#[tokio::test]
async fn test_engine_router_creation() {
let http_client = Arc::new(
reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(30))
.build()
.unwrap(),
);
let engines: Vec<Arc<dyn ScraperEngine>> = vec![Arc::new(ReqwestEngine::new(http_client))];
let router = EngineRouter::new(engines);
assert_eq!(router.strategy, LoadBalancingStrategy::SmartHybrid);
}
#[tokio::test]
async fn test_route_respects_max_engine_attempts() {
struct CountingEngine {
name: &'static str,
calls: Arc<std::sync::atomic::AtomicU32>,
ok: bool,
}
#[async_trait]
impl ScraperEngine for CountingEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
self.calls.fetch_add(1, Ordering::SeqCst);
if self.ok {
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "ok".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
})
} else {
Err(EngineError::Timeout(Duration::from_millis(10)))
}
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
self.name
}
}
let c1 = Arc::new(std::sync::atomic::AtomicU32::new(0));
let c2 = Arc::new(std::sync::atomic::AtomicU32::new(0));
let c3 = Arc::new(std::sync::atomic::AtomicU32::new(0));
let e1: Arc<dyn ScraperEngine> = Arc::new(CountingEngine {
name: "e1",
calls: c1.clone(),
ok: false,
});
let e2: Arc<dyn ScraperEngine> = Arc::new(CountingEngine {
name: "e2",
calls: c2.clone(),
ok: false,
});
let e3: Arc<dyn ScraperEngine> = Arc::new(CountingEngine {
name: "e3",
calls: c3.clone(),
ok: true,
});
let mut router = EngineRouter::new(vec![e1, e2, e3]);
router.set_strategy(LoadBalancingStrategy::RoundRobin);
router.set_max_engine_attempts(2);
let request = InternalScrapeRequest {
url: "http://1.1.1.1".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
let result = router.route(&request).await;
assert!(result.is_err());
assert_eq!(c1.load(Ordering::SeqCst), 1);
assert_eq!(c2.load(Ordering::SeqCst), 1);
assert_eq!(c3.load(Ordering::SeqCst), 0);
}
#[test]
fn test_engine_score_calculation() {
let engines: Vec<Arc<dyn ScraperEngine>> = vec![];
let router = EngineRouter::new(engines);
let stats = EngineStats {
success_rate: 0.9,
avg_response_time: Duration::from_millis(200),
usage_count: 10,
last_used: None,
};
let score = router.calculate_engine_score(1.0, &stats);
assert!(score > 0.8 && score <= 1.0);
}
struct MockEngine {
engine_name: &'static str,
score: u8,
}
#[async_trait]
impl ScraperEngine for MockEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
Ok(InternalScrapeResponse { js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "<html><body><h1>Mock Response</h1><p>This is a mock response for testing router logic. It contains enough visible text to avoid being flagged as a near-empty shell by the antibot classifier.</p></body></html>".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
})
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
self.score
}
fn name(&self) -> &'static str {
self.engine_name
}
}
fn make_request() -> InternalScrapeRequest {
InternalScrapeRequest {
url: "http://example.com".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
}
}
#[test]
fn test_should_filter_by_feature_screenshot_low_score() {
let engine: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "low-score",
score: 30,
});
let router = EngineRouter::new(vec![]);
let mut request = make_request();
request.needs_screenshot = true;
let result = router.should_filter_by_feature(&request, &engine);
assert!(result.is_some());
assert!(result.unwrap().contains("screenshots"));
}
#[test]
fn test_should_filter_by_feature_screenshot_high_score() {
let engine: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "high-score",
score: 80,
});
let router = EngineRouter::new(vec![]);
let mut request = make_request();
request.needs_screenshot = true;
let result = router.should_filter_by_feature(&request, &engine);
assert!(result.is_none());
}
#[test]
fn test_should_filter_by_feature_js_low_score() {
let engine: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "low-score",
score: 20,
});
let router = EngineRouter::new(vec![]);
let mut request = make_request();
request.needs_js = true;
let result = router.should_filter_by_feature(&request, &engine);
assert!(result.is_some());
assert!(result.unwrap().contains("JavaScript"));
}
#[test]
fn test_should_filter_by_feature_actions_low_score() {
let engine: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "low-score",
score: 10,
});
let router = EngineRouter::new(vec![]);
let mut request = make_request();
request.actions = vec![crate::engines::engine_client::InternalPageAction::Click {
selector: "#btn".to_string(),
}];
let result = router.should_filter_by_feature(&request, &engine);
assert!(result.is_some());
assert!(result.unwrap().contains("JavaScript"));
}
#[test]
fn test_should_filter_by_feature_tls_fingerprint_low_score() {
let engine: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "low-score",
score: 40,
});
let router = EngineRouter::new(vec![]);
let mut request = make_request();
request.needs_tls_fingerprint = true;
let result = router.should_filter_by_feature(&request, &engine);
assert!(result.is_some());
assert!(result.unwrap().contains("TLS fingerprinting"));
}
#[test]
fn test_should_filter_by_feature_tls_fingerprint_high_score() {
let engine: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "high-score",
score: 60,
});
let router = EngineRouter::new(vec![]);
let mut request = make_request();
request.needs_tls_fingerprint = true;
let result = router.should_filter_by_feature(&request, &engine);
assert!(result.is_none());
}
#[test]
fn test_should_filter_by_feature_no_special_needs() {
let engine: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "any",
score: 10,
});
let router = EngineRouter::new(vec![]);
let request = make_request();
let result = router.should_filter_by_feature(&request, &engine);
assert!(result.is_none());
}
#[test]
fn test_sort_round_robin_preserves_order() {
let e1: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "e1",
score: 100,
});
let e2: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "e2",
score: 100,
});
let mut router = EngineRouter::new(vec![e1, e2]);
router.set_strategy(LoadBalancingStrategy::RoundRobin);
let stats = std::collections::HashMap::new();
let mut candidates: Vec<(f64, Arc<dyn ScraperEngine>)> = vec![
(1.0, router.engines[0].clone()),
(2.0, router.engines[1].clone()),
];
let original_names: Vec<_> = candidates.iter().map(|(_, e)| e.name()).collect();
router.sort_candidates_by_strategy(&mut candidates, &stats);
let sorted_names: Vec<_> = candidates.iter().map(|(_, e)| e.name()).collect();
assert_eq!(original_names, sorted_names);
}
#[test]
fn test_sort_weighted_round_robin_by_score() {
let e1: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "e1",
score: 100,
});
let e2: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "e2",
score: 100,
});
let mut router = EngineRouter::new(vec![e1, e2]);
router.set_strategy(LoadBalancingStrategy::WeightedRoundRobin);
let stats = std::collections::HashMap::new();
let mut candidates: Vec<(f64, Arc<dyn ScraperEngine>)> = vec![
(0.5, router.engines[0].clone()),
(0.9, router.engines[1].clone()),
];
router.sort_candidates_by_strategy(&mut candidates, &stats);
assert_eq!(candidates[0].1.name(), "e2");
assert_eq!(candidates[1].1.name(), "e1");
}
#[test]
fn test_sort_least_connections_by_usage() {
let e1: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "e1",
score: 100,
});
let e2: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "e2",
score: 100,
});
let mut router = EngineRouter::new(vec![e1, e2]);
router.set_strategy(LoadBalancingStrategy::LeastConnections);
let mut stats = std::collections::HashMap::new();
stats.insert(
"e1".to_string(),
EngineStats {
usage_count: 100,
..Default::default()
},
);
stats.insert(
"e2".to_string(),
EngineStats {
usage_count: 5,
..Default::default()
},
);
let mut candidates: Vec<(f64, Arc<dyn ScraperEngine>)> = vec![
(1.0, router.engines[0].clone()),
(1.0, router.engines[1].clone()),
];
router.sort_candidates_by_strategy(&mut candidates, &stats);
assert_eq!(candidates[0].1.name(), "e2");
assert_eq!(candidates[1].1.name(), "e1");
}
#[test]
fn test_sort_fastest_response_by_time() {
let e1: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "e1",
score: 100,
});
let e2: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "e2",
score: 100,
});
let mut router = EngineRouter::new(vec![e1, e2]);
router.set_strategy(LoadBalancingStrategy::FastestResponse);
let mut stats = std::collections::HashMap::new();
stats.insert(
"e1".to_string(),
EngineStats {
avg_response_time: Duration::from_millis(500),
..Default::default()
},
);
stats.insert(
"e2".to_string(),
EngineStats {
avg_response_time: Duration::from_millis(100),
..Default::default()
},
);
let mut candidates: Vec<(f64, Arc<dyn ScraperEngine>)> = vec![
(1.0, router.engines[0].clone()),
(1.0, router.engines[1].clone()),
];
router.sort_candidates_by_strategy(&mut candidates, &stats);
assert_eq!(candidates[0].1.name(), "e2");
}
#[test]
fn test_sort_random_shuffles() {
let engines: Vec<Arc<dyn ScraperEngine>> = vec![
Arc::new(MockEngine {
engine_name: "e1",
score: 100,
}),
Arc::new(MockEngine {
engine_name: "e2",
score: 100,
}),
Arc::new(MockEngine {
engine_name: "e3",
score: 100,
}),
];
let mut router = EngineRouter::new(engines);
router.set_strategy(LoadBalancingStrategy::Random);
let stats = std::collections::HashMap::new();
let mut candidates: Vec<(f64, Arc<dyn ScraperEngine>)> =
router.engines.iter().map(|e| (1.0, e.clone())).collect();
router.sort_candidates_by_strategy(&mut candidates, &stats);
assert_eq!(candidates.len(), 3);
}
#[test]
fn test_sort_smart_hybrid_combined() {
let e1: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "e1",
score: 100,
});
let e2: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "e2",
score: 100,
});
let mut router = EngineRouter::new(vec![e1, e2]);
router.set_strategy(LoadBalancingStrategy::SmartHybrid);
let mut stats = std::collections::HashMap::new();
stats.insert(
"e1".to_string(),
EngineStats {
success_rate: 0.5,
avg_response_time: Duration::from_millis(800),
usage_count: 50,
last_used: None,
},
);
stats.insert(
"e2".to_string(),
EngineStats {
success_rate: 0.95,
avg_response_time: Duration::from_millis(100),
usage_count: 5,
last_used: None,
},
);
let mut candidates: Vec<(f64, Arc<dyn ScraperEngine>)> = vec![
(0.6, router.engines[0].clone()),
(0.9, router.engines[1].clone()),
];
router.sort_candidates_by_strategy(&mut candidates, &stats);
assert_eq!(candidates[0].1.name(), "e2");
}
#[test]
fn test_update_engine_stats_success() {
let engine: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "test",
score: 100,
});
let router = EngineRouter::new(vec![engine]);
router.update_engine_stats("test", true, Duration::from_millis(100));
let stats = router.get_engine_stats();
let stat = stats.get("test").unwrap();
assert!(stat.success_rate > 0.9);
assert_eq!(stat.usage_count, 1);
assert!(stat.last_used.is_some());
}
#[test]
fn test_update_engine_stats_failure() {
let engine: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "test",
score: 100,
});
let router = EngineRouter::new(vec![engine]);
router.update_engine_stats("test", false, Duration::from_millis(500));
let stats = router.get_engine_stats();
let stat = stats.get("test").unwrap();
assert!(stat.success_rate < 1.0);
assert_eq!(stat.usage_count, 1);
}
#[test]
fn test_update_engine_stats_nonexistent() {
let router = EngineRouter::new(vec![]);
router.update_engine_stats("nonexistent", true, Duration::from_millis(50));
}
#[test]
fn test_get_next_round_robin_index_wraps() {
let router = EngineRouter::new(vec![]);
let idx1 = router.get_next_round_robin_index(3);
let idx2 = router.get_next_round_robin_index(3);
let idx3 = router.get_next_round_robin_index(3);
let idx4 = router.get_next_round_robin_index(3);
assert_eq!(idx1, 0);
assert_eq!(idx2, 1);
assert_eq!(idx3, 2);
assert_eq!(idx4, 0);
}
#[test]
fn test_get_next_round_robin_index_single() {
let router = EngineRouter::new(vec![]);
let idx = router.get_next_round_robin_index(1);
assert_eq!(idx, 0);
}
#[test]
fn test_reset_engine_stats() {
let engine: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "test",
score: 100,
});
let router = EngineRouter::new(vec![engine]);
router.update_engine_stats("test", false, Duration::from_millis(500));
let stats_before = router.get_engine_stats();
assert_eq!(stats_before.get("test").unwrap().usage_count, 1);
router.reset_engine_stats("test");
let stats_after = router.get_engine_stats();
let stat = stats_after.get("test").unwrap();
assert_eq!(stat.usage_count, 0);
assert_eq!(stat.success_rate, 1.0);
}
#[test]
fn test_reset_engine_stats_nonexistent() {
let router = EngineRouter::new(vec![]);
router.reset_engine_stats("nonexistent");
}
#[test]
fn test_register_engine() {
let mut router = EngineRouter::new(vec![]);
assert!(router.get_engine_stats().is_empty());
let engine: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "new-engine",
score: 100,
});
router.register_engine(engine);
assert!(router.get_engine_stats().contains_key("new-engine"));
assert_eq!(router.registered_engines(), vec!["new-engine".to_string()]);
}
#[test]
fn test_router_metrics_new() {
let metrics = RouterMetrics::new();
assert_eq!(metrics.total_requests.load(Ordering::Relaxed), 0);
assert_eq!(metrics.successful_requests.load(Ordering::Relaxed), 0);
assert_eq!(metrics.failed_requests.load(Ordering::Relaxed), 0);
}
#[test]
fn test_router_metrics_record_candidates() {
let metrics = RouterMetrics::new();
metrics.record_candidates(5);
metrics.record_candidates(3);
assert_eq!(metrics.candidate_count_total.load(Ordering::Relaxed), 8);
}
#[test]
fn test_router_metrics_record_attempt() {
let metrics = RouterMetrics::new();
metrics.record_attempt();
metrics.record_attempt();
metrics.record_attempt();
assert_eq!(metrics.attempt_count_total.load(Ordering::Relaxed), 3);
}
#[test]
fn test_router_metrics_record_engine_selection() {
let metrics = RouterMetrics::new();
metrics.record_engine_selection("engine1");
assert_eq!(metrics.engine_selection_total.load(Ordering::Relaxed), 1);
}
#[test]
fn test_router_metrics_record_engine_latency() {
let metrics = RouterMetrics::new();
metrics.record_engine_latency("engine1", Duration::from_millis(100));
metrics.record_engine_latency("engine1", Duration::from_millis(200));
let total_ref = metrics.engine_latencies.get("engine1").unwrap();
assert_eq!(total_ref.load(Ordering::Relaxed), 300_000_000);
let avg = metrics.get_avg_latency_ns("engine1");
assert!(avg.is_none());
}
#[test]
fn test_router_metrics_record_engine_success() {
let metrics = RouterMetrics::new();
metrics.record_engine_success("engine1");
metrics.record_engine_success("engine1");
let count_ref = metrics.engine_success_count.get("engine1").unwrap();
assert_eq!(count_ref.load(Ordering::Relaxed), 2);
}
#[test]
fn test_router_metrics_record_engine_failure() {
let metrics = RouterMetrics::new();
metrics.record_engine_failure("engine1", "timeout error");
metrics.record_engine_failure("engine1", "network error");
let count_ref = metrics.engine_failure_count.get("engine1").unwrap();
assert_eq!(count_ref.load(Ordering::Relaxed), 2);
let timeout_count = metrics.failure_classification.get("timeout").unwrap();
assert_eq!(*timeout_count, 1);
let network_count = metrics.failure_classification.get("network_error").unwrap();
assert_eq!(*network_count, 1);
}
#[test]
fn test_router_metrics_classify_error() {
assert_eq!(RouterMetrics::classify_error("request timeout"), "timeout");
assert_eq!(
RouterMetrics::classify_error("SSRF protection triggered"),
"ssrf_protection"
);
assert_eq!(
RouterMetrics::classify_error("network unreachable"),
"network_error"
);
assert_eq!(
RouterMetrics::classify_error("circuit breaker open"),
"circuit_breaker"
);
assert_eq!(
RouterMetrics::classify_error("browser crashed"),
"browser_error"
);
assert_eq!(RouterMetrics::classify_error("unknown issue"), "other");
}
#[test]
fn test_router_metrics_get_success_rate() {
let metrics = RouterMetrics::new();
assert_eq!(metrics.get_success_rate(), 1.0);
metrics.total_requests.store(10, Ordering::Relaxed);
metrics.successful_requests.store(7, Ordering::Relaxed);
assert_eq!(metrics.get_success_rate(), 0.7);
}
#[test]
fn test_router_metrics_get_avg_latency_ns_no_data() {
let metrics = RouterMetrics::new();
assert!(metrics.get_avg_latency_ns("nonexistent").is_none());
}
#[test]
fn test_router_metrics_get_avg_latency_ns_with_data() {
let metrics = RouterMetrics::new();
metrics
.engine_latencies
.insert("engine1".to_string(), AtomicU64::new(1_000_000));
metrics
.engine_success_count
.insert("engine1".to_string(), AtomicU64::new(10));
let avg = metrics.get_avg_latency_ns("engine1");
assert_eq!(avg, Some(100_000));
}
#[test]
fn test_router_metrics_record_engine_success_initializes_to_one() {
let metrics = RouterMetrics::new();
metrics.record_engine_success("engine1");
assert_eq!(
metrics
.engine_success_count
.get("engine1")
.unwrap()
.load(Ordering::Relaxed),
1u64
);
metrics.record_engine_success("engine1");
assert_eq!(
metrics
.engine_success_count
.get("engine1")
.unwrap()
.load(Ordering::Relaxed),
2u64
);
}
#[test]
fn test_calculate_engine_score_zero_success_rate() {
let router = EngineRouter::new(vec![]);
let stats = EngineStats {
success_rate: 0.0,
avg_response_time: Duration::from_secs(5),
usage_count: 500,
last_used: None,
};
let score = router.calculate_engine_score(1.0, &stats);
assert!(score < 0.5);
}
#[test]
fn test_calculate_engine_score_perfect_stats() {
let router = EngineRouter::new(vec![]);
let stats = EngineStats {
success_rate: 1.0,
avg_response_time: Duration::from_millis(10),
usage_count: 0,
last_used: None,
};
let score = router.calculate_engine_score(1.0, &stats);
assert!(score > 0.95);
}
#[test]
fn test_calculate_engine_score_high_usage_penalty() {
let router = EngineRouter::new(vec![]);
let stats = EngineStats {
success_rate: 1.0,
avg_response_time: Duration::from_millis(10),
usage_count: 2000,
last_used: None,
};
let score = router.calculate_engine_score(1.0, &stats);
let perfect_stats = EngineStats {
success_rate: 1.0,
avg_response_time: Duration::from_millis(10),
usage_count: 0,
last_used: None,
};
let perfect_score = router.calculate_engine_score(1.0, &perfect_stats);
assert!(score < perfect_score);
}
#[test]
fn test_set_max_engine_attempts() {
let mut router = EngineRouter::new(vec![]);
router.set_max_engine_attempts(5);
assert_eq!(router.max_engine_attempts, 5);
}
#[test]
fn test_set_max_engine_attempts_min_one() {
let mut router = EngineRouter::new(vec![]);
router.set_max_engine_attempts(0);
assert_eq!(router.max_engine_attempts, 1);
}
#[test]
fn test_set_max_retries() {
let mut router = EngineRouter::new(vec![]);
router.set_max_retries(10);
assert_eq!(router.max_retries, 10);
}
#[test]
fn test_set_max_retries_min_one() {
let mut router = EngineRouter::new(vec![]);
router.set_max_retries(0);
assert_eq!(router.max_retries, 1);
}
#[test]
fn test_set_feature_filter_enabled() {
let mut router = EngineRouter::new(vec![]);
router.set_feature_filter_enabled(false);
assert!(!router.feature_filter_enabled);
router.set_feature_filter_enabled(true);
assert!(router.feature_filter_enabled);
}
#[test]
fn test_set_race_mode_enabled() {
let mut router = EngineRouter::new(vec![]);
router.set_race_mode_enabled(true);
assert!(router.race_mode_enabled);
}
#[test]
fn test_set_dynamic_threshold_factor() {
let mut router = EngineRouter::new(vec![]);
router.set_dynamic_threshold_factor(1.5);
assert_eq!(router.dynamic_threshold_factor, 1.5);
}
#[test]
fn test_set_dynamic_threshold_factor_clamped() {
let mut router = EngineRouter::new(vec![]);
router.set_dynamic_threshold_factor(0.01);
assert_eq!(router.dynamic_threshold_factor, 0.1);
router.set_dynamic_threshold_factor(3.0);
assert_eq!(router.dynamic_threshold_factor, 2.0);
}
#[test]
fn test_set_strategy() {
let mut router = EngineRouter::new(vec![]);
router.set_strategy(LoadBalancingStrategy::RoundRobin);
assert_eq!(router.strategy, LoadBalancingStrategy::RoundRobin);
router.set_strategy(LoadBalancingStrategy::Random);
assert_eq!(router.strategy, LoadBalancingStrategy::Random);
}
#[test]
fn test_metrics_accessor() {
let router = EngineRouter::new(vec![]);
let _metrics = router.metrics();
}
#[test]
fn test_with_circuit_breaker_and_strategy() {
let engine: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "test",
score: 100,
});
let cb = Arc::new(CircuitBreaker::new());
let router = EngineRouter::with_circuit_breaker_and_strategy(
vec![engine],
cb,
LoadBalancingStrategy::LeastConnections,
);
assert_eq!(router.strategy, LoadBalancingStrategy::LeastConnections);
assert!(router.get_engine_stats().contains_key("test"));
}
#[test]
fn test_get_engines() {
let e1: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "e1",
score: 100,
});
let e2: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "e2",
score: 100,
});
let router = EngineRouter::new(vec![e1, e2]);
let engines = router.get_engines();
assert_eq!(engines.len(), 2);
assert_eq!(engines[0].name(), "e1");
assert_eq!(engines[1].name(), "e2");
}
#[test]
fn test_engine_stats_default() {
let stats = EngineStats::default();
assert_eq!(stats.success_rate, 1.0);
assert_eq!(stats.avg_response_time, Duration::from_millis(500));
assert!(stats.last_used.is_none());
assert_eq!(stats.usage_count, 0);
}
#[tokio::test]
async fn test_route_success_path() {
let engine: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "success-engine",
score: 100,
});
let router = EngineRouter::new(vec![engine]);
let request = make_request();
let result = router.route(&request).await;
assert!(result.is_ok());
let response = result.unwrap();
assert_eq!(response.status_code, 200);
assert!(response.content.contains("Mock Response"));
}
#[tokio::test]
async fn test_route_ssrf_protection() {
let engine: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "test",
score: 100,
});
let router = EngineRouter::new(vec![engine]);
let mut request = make_request();
request.url = "http://127.0.0.1".to_string();
let result = router.route(&request).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert!(matches!(err, EngineError::SsrfProtection(_)));
}
#[tokio::test]
async fn test_route_non_retryable_error() {
struct NonRetryableEngine;
#[async_trait]
impl ScraperEngine for NonRetryableEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
Err(EngineError::InvalidUrl("bad url".to_string()))
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
"non-retryable"
}
}
let engine: Arc<dyn ScraperEngine> = Arc::new(NonRetryableEngine);
let router = EngineRouter::new(vec![engine]);
let request = make_request();
let result = router.route(&request).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert!(matches!(err, EngineError::InvalidUrl(_)));
}
#[tokio::test]
async fn test_route_max_retries_reached() {
struct AlwaysTimeoutEngine;
#[async_trait]
impl ScraperEngine for AlwaysTimeoutEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
Err(EngineError::Timeout(Duration::from_secs(10)))
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
"always-timeout"
}
}
let engine: Arc<dyn ScraperEngine> = Arc::new(AlwaysTimeoutEngine);
let mut router = EngineRouter::new(vec![engine]);
router.set_max_retries(1);
let request = make_request();
let result = router.route(&request).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert!(matches!(err, EngineError::Timeout(_)));
}
#[tokio::test]
async fn test_route_race_mode_success() {
struct FastEngine {
name: &'static str,
delay_ms: u64,
}
#[async_trait]
impl ScraperEngine for FastEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
tokio::time::sleep(Duration::from_millis(self.delay_ms)).await;
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: format!(
"<html><body><p>from-{} with plenty of realistic rendered \
content to pass the structural checks of the antibot \
classifier in race mode.</p></body></html>",
self.name
),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: self.delay_ms,
final_url: None,
})
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
self.name
}
}
let e1: Arc<dyn ScraperEngine> = Arc::new(FastEngine {
name: "slow",
delay_ms: 500,
});
let e2: Arc<dyn ScraperEngine> = Arc::new(FastEngine {
name: "fast",
delay_ms: 10,
});
let mut router = EngineRouter::new(vec![e1, e2]);
router.set_race_mode_enabled(true);
let request = make_request();
let result = router.route(&request).await;
assert!(result.is_ok());
let response = result.unwrap();
assert!(
response.content.contains("from-"),
"fastest clean engine must win the race, got: {}",
response.content
);
}
#[tokio::test]
async fn test_route_race_mode_records_hedge_latency() {
struct FastEngine {
name: &'static str,
delay_ms: u64,
}
#[async_trait]
impl ScraperEngine for FastEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
tokio::time::sleep(Duration::from_millis(self.delay_ms)).await;
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: format!(
"<html><body><p>from-{} with plenty of realistic rendered \
content to pass the structural checks of the antibot \
classifier in race mode.</p></body></html>",
self.name
),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: self.delay_ms,
final_url: None,
})
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
self.name
}
}
let e1: Arc<dyn ScraperEngine> = Arc::new(FastEngine {
name: "slow",
delay_ms: 100,
});
let e2: Arc<dyn ScraperEngine> = Arc::new(FastEngine {
name: "fast",
delay_ms: 5,
});
let mut router = EngineRouter::new(vec![e1, e2]);
router.set_race_mode_enabled(true);
assert_eq!(router.hedge_controller().sample_count(), 0);
let request = make_request();
for _ in 0..12 {
let _ = router.route(&request).await.unwrap();
}
let controller = router.hedge_controller();
assert!(
controller.sample_count() >= 10,
"hedge should have >= 10 samples, got {}",
controller.sample_count()
);
let threshold = controller
.p84_threshold()
.expect("P84 threshold should be available");
let threshold_ms = threshold.as_secs_f64() * 1000.0;
assert!(
threshold_ms < 50.0,
"P84 should be near fast engine latency, got {threshold_ms}ms"
);
assert!(router
.hedge_controller()
.should_hedge(Duration::from_millis(100)));
assert!(!router
.hedge_controller()
.should_hedge(Duration::from_micros(1)));
}
#[tokio::test]
async fn test_route_race_mode_all_fail() {
struct FailingEngine;
#[async_trait]
impl ScraperEngine for FailingEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
Err(EngineError::RequestFailed("connection refused".to_string()))
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
"failing"
}
}
let e1: Arc<dyn ScraperEngine> = Arc::new(FailingEngine);
let e2: Arc<dyn ScraperEngine> = Arc::new(FailingEngine);
let mut router = EngineRouter::new(vec![e1, e2]);
router.set_race_mode_enabled(true);
let request = make_request();
let result = router.route(&request).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_aggregate_no_candidates() {
struct ZeroScoreEngine;
#[async_trait]
impl ScraperEngine for ZeroScoreEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "ok".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
})
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
0
}
fn name(&self) -> &'static str {
"zero-score"
}
}
let engine: Arc<dyn ScraperEngine> = Arc::new(ZeroScoreEngine);
let router = EngineRouter::new(vec![engine]);
let request = make_request();
let result = router.aggregate(&request).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert!(matches!(err, EngineError::AllEnginesFailed(_)));
}
#[tokio::test]
async fn test_aggregate_all_engines_fail() {
struct FailingEngine;
#[async_trait]
impl ScraperEngine for FailingEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
Err(EngineError::RequestFailed("failed".to_string()))
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
"failing"
}
}
let e1: Arc<dyn ScraperEngine> = Arc::new(FailingEngine);
let e2: Arc<dyn ScraperEngine> = Arc::new(FailingEngine);
let router = EngineRouter::new(vec![e1, e2]);
let request = make_request();
let result = router.aggregate(&request).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert!(matches!(err, EngineError::AllEnginesFailed(_)));
}
#[tokio::test]
async fn test_engine_router_trait_methods() {
let engine: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "trait-test",
score: 100,
});
let router = EngineRouter::new(vec![engine]);
let trait_ref: &dyn EngineRouterTrait = &router;
let engines = trait_ref.registered_engines();
assert_eq!(engines, vec!["trait-test".to_string()]);
let stats = trait_ref.get_engine_stats();
assert!(stats.contains_key("trait-test"));
trait_ref.reset_engine_stats("trait-test");
let stats_after = trait_ref.get_engine_stats();
assert_eq!(stats_after.get("trait-test").unwrap().usage_count, 0);
let request = make_request();
let result = trait_ref.route(&request).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_route_no_engines_available() {
let router = EngineRouter::new(vec![]);
let request = make_request();
let result = router.route(&request).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert!(matches!(err, EngineError::AllEnginesFailed(_)));
}
#[tokio::test]
async fn test_route_support_score_zero_filtered() {
struct ZeroScoreEngine;
#[async_trait]
impl ScraperEngine for ZeroScoreEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "ok".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
})
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
0
}
fn name(&self) -> &'static str {
"zero-score"
}
}
let engine: Arc<dyn ScraperEngine> = Arc::new(ZeroScoreEngine);
let router = EngineRouter::new(vec![engine]);
let request = make_request();
let result = router.route(&request).await;
assert!(result.is_err());
assert!(matches!(
result.unwrap_err(),
EngineError::AllEnginesFailed(_)
));
}
#[tokio::test]
async fn test_trait_aggregate_delegates_to_impl() {
struct SucceedingEngine;
#[async_trait]
impl ScraperEngine for SucceedingEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
Ok(InternalScrapeResponse { js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "<html><body><h1>OK</h1><p>Succeeding engine response for testing trait delegation. It has enough visible text to pass the antibot classifier near-empty check.</p></body></html>".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
})
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
"succeeding"
}
}
let engine: Arc<dyn ScraperEngine> = Arc::new(SucceedingEngine);
let router = EngineRouter::new(vec![engine]);
let request = make_request();
let result = EngineRouterTrait::aggregate(&router, &request).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_trait_route_delegates_to_impl() {
struct SucceedingEngine;
#[async_trait]
impl ScraperEngine for SucceedingEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
Ok(InternalScrapeResponse { js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "<html><body><h1>OK</h1><p>Succeeding engine response for testing trait delegation. It has enough visible text to pass the antibot classifier near-empty check.</p></body></html>".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
})
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
"succeeding"
}
}
let engine: Arc<dyn ScraperEngine> = Arc::new(SucceedingEngine);
let router = EngineRouter::new(vec![engine]);
let request = make_request();
let result = EngineRouterTrait::route(&router, &request).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_feature_filter_excludes_low_score_engine_for_screenshot() {
struct LowScoreEngine;
#[async_trait]
impl ScraperEngine for LowScoreEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "ok".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
})
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
10 }
fn name(&self) -> &'static str {
"low-score"
}
}
let engine: Arc<dyn ScraperEngine> = Arc::new(LowScoreEngine);
let mut router = EngineRouter::new(vec![engine]);
router.set_feature_filter_enabled(true);
let request = InternalScrapeRequest {
url: "http://example.com".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: true, screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
let result = router.route(&request).await;
assert!(result.is_err());
assert!(matches!(
result.unwrap_err(),
EngineError::AllEnginesFailed(_)
));
}
#[tokio::test]
async fn test_route_skips_engine_when_circuit_breaker_open() {
use crate::engines::circuit_breaker::CircuitConfig;
struct CountingEngine {
name: &'static str,
calls: Arc<std::sync::atomic::AtomicU32>,
}
#[async_trait]
impl ScraperEngine for CountingEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
self.calls.fetch_add(1, Ordering::SeqCst);
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "ok".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 1,
final_url: None,
})
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
self.name
}
}
let calls = Arc::new(std::sync::atomic::AtomicU32::new(0));
let engine: Arc<dyn ScraperEngine> = Arc::new(CountingEngine {
name: "guarded",
calls: calls.clone(),
});
let router = EngineRouter::new(vec![engine]);
router.circuit_breaker.set_config(
"guarded",
CircuitConfig {
failure_threshold: 1,
recovery_timeout: Duration::from_secs(60),
failure_window: Duration::from_secs(60),
},
);
router.circuit_breaker.record_failure("guarded").await;
assert!(
router.circuit_breaker.is_open("guarded"),
"circuit breaker should be open after 1 failure"
);
let request = make_request();
let result = router.route(&request).await;
assert!(
result.is_err(),
"route should fail when the only engine is open"
);
assert!(
matches!(result.unwrap_err(), EngineError::AllEnginesFailed(_)),
"expected AllEnginesFailed when circuit breaker blocks the only engine"
);
assert_eq!(
calls.load(Ordering::SeqCst),
0,
"engine must not be invoked when its circuit breaker is open"
);
}
#[tokio::test]
async fn test_route_returns_timeout_when_remaining_time_zero() {
struct SlowTimeoutEngine;
#[async_trait]
impl ScraperEngine for SlowTimeoutEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
tokio::time::sleep(Duration::from_millis(120)).await;
Err(EngineError::Timeout(Duration::from_millis(120)))
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
"slow-timeout"
}
}
let e1: Arc<dyn ScraperEngine> = Arc::new(SlowTimeoutEngine);
let e2: Arc<dyn ScraperEngine> = Arc::new(SlowTimeoutEngine);
let mut router = EngineRouter::new(vec![e1, e2]);
router.set_max_engine_attempts(2);
router.set_max_retries(5);
let mut request = make_request();
request.timeout = Duration::from_millis(10);
let result = router.route(&request).await;
assert!(result.is_err(), "route should fail with Timeout");
match result.unwrap_err() {
EngineError::Timeout(d) => {
assert_eq!(
d,
Duration::from_millis(10),
"should report the original request timeout"
);
}
other => panic!("Expected Timeout, got {:?}", other),
}
}
#[tokio::test]
async fn test_route_mrt_waterfall_first_engine_exceeds_mrt_falls_to_second() {
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Instant;
let engine1_called = Arc::new(AtomicBool::new(false));
let engine2_called = Arc::new(AtomicBool::new(false));
struct MrtSlowEngine {
mrt: Duration,
sleep_dur: Duration,
called: Arc<AtomicBool>,
}
#[async_trait]
impl ScraperEngine for MrtSlowEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
self.called.store(true, Ordering::SeqCst);
tokio::time::sleep(self.sleep_dur).await;
Ok(InternalScrapeResponse { js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "<html><body><h1>Slow Engine Response</h1><p>This is the slow engine response with sufficient visible text to pass the anti-bot detection threshold of fifty bytes required by the tier3 visible text minimum check.</p></body></html>".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: self.sleep_dur.as_millis() as u64,
final_url: None,
})
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100 }
fn name(&self) -> &'static str {
"mrt-slow"
}
fn max_response_time(&self) -> Duration {
self.mrt
}
}
struct FastEngine {
called: Arc<AtomicBool>,
}
#[async_trait]
impl ScraperEngine for FastEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
self.called.store(true, Ordering::SeqCst);
Ok(InternalScrapeResponse { js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "<html><body><h1>Fast Engine Response</h1><p>This is the fast engine response with sufficient visible text to pass the anti-bot detection threshold of fifty bytes required by the tier3 visible text minimum check.</p></body></html>".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 1,
final_url: None,
})
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
90 }
fn name(&self) -> &'static str {
"fast"
}
}
let e1: Arc<dyn ScraperEngine> = Arc::new(MrtSlowEngine {
mrt: Duration::from_millis(50),
sleep_dur: Duration::from_millis(500),
called: engine1_called.clone(),
});
let e2: Arc<dyn ScraperEngine> = Arc::new(FastEngine {
called: engine2_called.clone(),
});
let mut router = EngineRouter::new(vec![e1, e2]);
router.set_max_engine_attempts(2);
router.set_max_retries(5);
router.set_race_mode_enabled(false);
let request = make_request();
let start = Instant::now();
let result = router.route(&request).await;
let elapsed = start.elapsed();
assert!(result.is_ok(), "route should succeed via engine2 fallback");
let resp = result.unwrap();
assert_eq!(
resp.content, "<html><body><h1>Fast Engine Response</h1><p>This is the fast engine response with sufficient visible text to pass the anti-bot detection threshold of fifty bytes required by the tier3 visible text minimum check.</p></body></html>",
"response should come from fast engine (waterfall fallback)"
);
assert!(
engine1_called.load(Ordering::SeqCst),
"engine1 should have been called (first attempt)"
);
assert!(
engine2_called.load(Ordering::SeqCst),
"engine2 should have been called after engine1 exceeded MRT (waterfall)"
);
assert!(
elapsed < Duration::from_millis(400),
"should not wait for engine1's full 500ms sleep; elapsed={:?}",
elapsed
);
}
#[tokio::test]
async fn test_route_mrt_engine_within_mrt_succeeds_normally() {
use std::sync::atomic::{AtomicBool, Ordering};
let engine1_called = Arc::new(AtomicBool::new(false));
struct MrtOkEngine {
mrt: Duration,
sleep_dur: Duration,
called: Arc<AtomicBool>,
}
#[async_trait]
impl ScraperEngine for MrtOkEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
self.called.store(true, Ordering::SeqCst);
tokio::time::sleep(self.sleep_dur).await;
Ok(InternalScrapeResponse { js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "<html><body><h1>Real Content Page</h1><p>This is a real page with sufficient visible text to pass the anti-bot detection threshold of fifty bytes required by the tier3 visible text minimum check.</p></body></html>".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: self.sleep_dur.as_millis() as u64,
final_url: None,
})
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
"mrt-ok"
}
fn max_response_time(&self) -> Duration {
self.mrt
}
}
let e1: Arc<dyn ScraperEngine> = Arc::new(MrtOkEngine {
mrt: Duration::from_secs(1),
sleep_dur: Duration::from_millis(50),
called: engine1_called.clone(),
});
let mut router = EngineRouter::new(vec![e1]);
router.set_race_mode_enabled(false);
let request = make_request();
let result = router.route(&request).await;
assert!(result.is_ok(), "engine within MRT should succeed");
let resp = result.unwrap();
assert_eq!(resp.content, "<html><body><h1>Real Content Page</h1><p>This is a real page with sufficient visible text to pass the anti-bot detection threshold of fifty bytes required by the tier3 visible text minimum check.</p></body></html>");
assert!(
engine1_called.load(Ordering::SeqCst),
"engine1 should have been called"
);
}
#[tokio::test]
async fn test_route_mrt_uses_min_remaining_when_remaining_less_than_mrt() {
use std::sync::atomic::{AtomicU32, Ordering};
let engine1_calls = Arc::new(AtomicU32::new(0));
struct LongMrtSlowEngine {
mrt: Duration,
sleep_dur: Duration,
calls: Arc<AtomicU32>,
}
#[async_trait]
impl ScraperEngine for LongMrtSlowEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
self.calls.fetch_add(1, Ordering::SeqCst);
tokio::time::sleep(self.sleep_dur).await;
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "should-not-reach-here".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 0,
final_url: None,
})
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
"long-mrt-slow"
}
fn max_response_time(&self) -> Duration {
self.mrt
}
}
let e1: Arc<dyn ScraperEngine> = Arc::new(LongMrtSlowEngine {
mrt: Duration::from_secs(10),
sleep_dur: Duration::from_millis(200),
calls: engine1_calls.clone(),
});
let mut router = EngineRouter::new(vec![e1]);
router.set_max_engine_attempts(1);
router.set_max_retries(1);
router.set_race_mode_enabled(false);
let mut request = make_request();
request.timeout = Duration::from_millis(80);
let start = std::time::Instant::now();
let result = router.route(&request).await;
let elapsed = start.elapsed();
assert!(result.is_err(), "should fail with Timeout");
match result.unwrap_err() {
EngineError::Timeout(_) => {}
other => panic!("Expected Timeout, got {:?}", other),
}
assert!(
elapsed < Duration::from_millis(2000),
"should timeout at ~80ms (remaining); elapsed={:?}",
elapsed
);
assert_eq!(
engine1_calls.load(Ordering::SeqCst),
1,
"engine1 should be called exactly once"
);
}
#[tokio::test]
async fn test_route_race_mode_returns_timeout_when_remaining_zero() {
struct NeverCalledEngine;
#[async_trait]
impl ScraperEngine for NeverCalledEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
panic!("engine must not be called when remaining time is zero");
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
"never-called"
}
}
let e1: Arc<dyn ScraperEngine> = Arc::new(NeverCalledEngine);
let e2: Arc<dyn ScraperEngine> = Arc::new(NeverCalledEngine);
let mut router = EngineRouter::new(vec![e1, e2]);
router.set_race_mode_enabled(true);
let mut request = make_request();
request.timeout = Duration::from_millis(0);
let result = router.route(&request).await;
assert!(result.is_err(), "race_mode with zero remaining should fail");
match result.unwrap_err() {
EngineError::Timeout(_) => {}
other => panic!("Expected Timeout, got {:?}", other),
}
}
#[tokio::test]
async fn test_route_race_mode_non_retryable_error_returns_err() {
struct InvalidUrlEngine;
#[async_trait]
impl ScraperEngine for InvalidUrlEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
Err(EngineError::InvalidUrl("malformed url".to_string()))
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
"invalid-url"
}
}
let e1: Arc<dyn ScraperEngine> = Arc::new(InvalidUrlEngine);
let e2: Arc<dyn ScraperEngine> = Arc::new(InvalidUrlEngine);
let mut router = EngineRouter::new(vec![e1, e2]);
router.set_race_mode_enabled(true);
let request = make_request();
let result = router.route(&request).await;
assert!(
result.is_err(),
"race_mode with non-retryable error should fail"
);
match result.unwrap_err() {
EngineError::InvalidUrl(msg) => {
assert_eq!(msg, "malformed url");
}
other => panic!("Expected InvalidUrl, got {:?}", other),
}
}
#[tokio::test]
async fn test_route_race_mode_returns_timeout_on_select_all_timeout() {
struct SlowOkEngine;
#[async_trait]
impl ScraperEngine for SlowOkEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
tokio::time::sleep(Duration::from_secs(5)).await;
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "ok".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 5000,
final_url: None,
})
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
"slow-ok"
}
}
let e1: Arc<dyn ScraperEngine> = Arc::new(SlowOkEngine);
let e2: Arc<dyn ScraperEngine> = Arc::new(SlowOkEngine);
let mut router = EngineRouter::new(vec![e1, e2]);
router.set_race_mode_enabled(true);
let mut request = make_request();
request.timeout = Duration::from_secs(1);
let result = router.route(&request).await;
assert!(
result.is_err(),
"race_mode with all-slow engines should time out"
);
match result.unwrap_err() {
EngineError::Timeout(d) => {
assert!(
d >= Duration::from_millis(100),
"timeout duration should be at least 100ms, got {:?}",
d
);
}
other => panic!("Expected Timeout, got {:?}", other),
}
}
#[test]
fn test_sort_candidates_fastest_response() {
let e1: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "slow",
score: 100,
});
let e2: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "fast",
score: 100,
});
let mut router = EngineRouter::new(vec![e1.clone(), e2.clone()]);
router.set_strategy(LoadBalancingStrategy::FastestResponse);
router
.engine_stats
.get_mut("slow")
.unwrap()
.avg_response_time = Duration::from_secs(2);
router
.engine_stats
.get_mut("fast")
.unwrap()
.avg_response_time = Duration::from_millis(50);
let stats: std::collections::HashMap<String, EngineStats> = router
.engine_stats
.iter()
.map(|r| (r.key().clone(), r.value().clone()))
.collect();
let mut candidates = vec![(1.0, e1), (1.0, e2)];
router.sort_candidates_by_strategy(&mut candidates, &stats);
assert_eq!(
candidates[0].1.name(),
"fast",
"FastestResponse should put fastest engine first"
);
assert_eq!(candidates[1].1.name(), "slow");
}
#[test]
fn test_sort_candidates_least_connections() {
let e1: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "busy",
score: 100,
});
let e2: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "idle",
score: 100,
});
let mut router = EngineRouter::new(vec![e1.clone(), e2.clone()]);
router.set_strategy(LoadBalancingStrategy::LeastConnections);
router.engine_stats.get_mut("busy").unwrap().usage_count = 500;
router.engine_stats.get_mut("idle").unwrap().usage_count = 5;
let stats: std::collections::HashMap<String, EngineStats> = router
.engine_stats
.iter()
.map(|r| (r.key().clone(), r.value().clone()))
.collect();
let mut candidates = vec![(1.0, e1), (1.0, e2)];
router.sort_candidates_by_strategy(&mut candidates, &stats);
assert_eq!(
candidates[0].1.name(),
"idle",
"LeastConnections should put least-used engine first"
);
assert_eq!(candidates[1].1.name(), "busy");
}
#[test]
fn test_sort_candidates_smart_hybrid_score_priority() {
let e1: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "low-score",
score: 50,
});
let e2: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "high-score",
score: 100,
});
let router = EngineRouter::new(vec![e1.clone(), e2.clone()]);
let stats: std::collections::HashMap<String, EngineStats> = router
.engine_stats
.iter()
.map(|r| (r.key().clone(), r.value().clone()))
.collect();
let mut candidates = vec![(0.5, e1), (1.0, e2)];
router.sort_candidates_by_strategy(&mut candidates, &stats);
assert_eq!(
candidates[0].1.name(),
"high-score",
"SmartHybrid should put higher-scored engine first"
);
}
#[tokio::test]
async fn test_select_optimal_engines_circuit_breaker_skips_open() {
let e1: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "open-cb",
score: 100,
});
let e2: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "closed-cb",
score: 100,
});
let router = EngineRouter::new(vec![e1.clone(), e2.clone()]);
for _ in 0..10 {
router.circuit_breaker.record_failure("open-cb").await;
}
let request = make_request();
let candidates = router.select_optimal_engines(&request);
let names: Vec<&str> = candidates.iter().map(|(_, e)| e.name()).collect();
assert!(
!names.contains(&"open-cb"),
"circuit breaker open engine should be filtered out"
);
assert!(
names.contains(&"closed-cb"),
"closed circuit breaker engine should remain"
);
}
#[test]
fn test_select_optimal_engines_feature_filter_tls_fingerprint() {
let e1: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "no-tls",
score: 10,
});
let e2: Arc<dyn ScraperEngine> = Arc::new(MockEngine {
engine_name: "with-tls",
score: 100,
});
let router = EngineRouter::new(vec![e1.clone(), e2.clone()]);
let mut request = make_request();
request.needs_tls_fingerprint = true;
let candidates = router.select_optimal_engines(&request);
let names: Vec<&str> = candidates.iter().map(|(_, e)| e.name()).collect();
assert!(
!names.contains(&"no-tls"),
"low-score engine should be filtered when needs_tls_fingerprint=true"
);
assert!(
names.contains(&"with-tls"),
"high-score engine should remain"
);
}
struct CallCountEngine {
name: &'static str,
content: String,
calls: std::sync::atomic::AtomicU32,
}
#[async_trait]
impl ScraperEngine for CallCountEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: self.content.clone(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 5,
final_url: None,
})
}
fn support_score(&self, request: &InternalScrapeRequest) -> u8 {
if request.needs_js {
if self.name == "browser" {
100
} else {
10
}
} else if self.name == "browser" {
10
} else {
100
}
}
fn name(&self) -> &'static str {
self.name
}
}
fn hint_test_request() -> InternalScrapeRequest {
InternalScrapeRequest {
url: "https://example.com/page".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
}
}
fn browser_content() -> String {
"<html><body><p>browser rendered content with enough visible text to pass structural checks of the antibot classifier.</p></body></html>".to_string()
}
#[cfg(feature = "content")]
#[tokio::test]
async fn test_host_browser_hint_skips_http_engine() {
let http = Arc::new(CallCountEngine {
name: "http-reqwest",
content: "<html><body><p>http engine content with enough visible text for structural checks.</p></body></html>".to_string(),
calls: std::sync::atomic::AtomicU32::new(0),
});
let browser = Arc::new(CallCountEngine {
name: "browser",
content: browser_content(),
calls: std::sync::atomic::AtomicU32::new(0),
});
let mut router = EngineRouter::new(vec![http.clone(), browser.clone()]);
router.set_feature_filter_enabled(false);
router.record_host_browser_hint("example.com");
let resp = router.route(&hint_test_request()).await.unwrap();
assert!(
resp.content.contains("browser rendered"),
"hint must route to browser engine, got: {}",
resp.content
);
assert_eq!(http.calls.load(std::sync::atomic::Ordering::SeqCst), 0);
assert_eq!(browser.calls.load(std::sync::atomic::Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_host_browser_hint_expiry_restores_http_candidates() {
let http = Arc::new(CallCountEngine {
name: "http-reqwest",
content: "<html><body><p>http engine content with enough visible text for structural checks.</p></body></html>".to_string(),
calls: std::sync::atomic::AtomicU32::new(0),
});
let browser = Arc::new(CallCountEngine {
name: "browser",
content: browser_content(),
calls: std::sync::atomic::AtomicU32::new(0),
});
let mut router = EngineRouter::new(vec![http.clone(), browser.clone()]);
router.set_feature_filter_enabled(false);
router.host_browser_hint.insert(
"example.com".to_string(),
Instant::now() - crate::engines::router::HOST_BROWSER_HINT_TTL - Duration::from_secs(60),
);
let resp = router.route(&hint_test_request()).await.unwrap();
assert!(
resp.content.contains("http engine content"),
"expired hint must restore http candidate, got: {}",
resp.content
);
assert_eq!(http.calls.load(std::sync::atomic::Ordering::SeqCst), 1);
}
use crate::config::settings::SessionSettings;
use crate::engines::provider::ProxyProvider;
use crate::engines::proxy_pool::ProxyPool;
use crate::engines::session::SessionManager;
use crate::utils::host_discipline::{HostDiscipline, ObserveOutcome, PolitenessOptions};
use crate::utils::ua_pool::UaPool;
#[derive(Debug, Clone)]
struct SessionObservedAttempt {
user_agent: Option<String>,
accept_language: Option<String>,
cookie: Option<String>,
proxy: Option<String>,
}
struct SessionProbeEngine {
calls: std::sync::Mutex<Vec<SessionObservedAttempt>>,
status: u16,
set_cookie: Option<String>,
}
impl SessionProbeEngine {
fn new(status: u16, set_cookie: Option<&str>) -> Self {
Self {
calls: std::sync::Mutex::new(Vec::new()),
status,
set_cookie: set_cookie.map(str::to_string),
}
}
fn observations(&self) -> Vec<SessionObservedAttempt> {
self.calls.lock().expect("probe calls lock").clone()
}
}
#[async_trait]
impl ScraperEngine for SessionProbeEngine {
async fn scrape(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
self.calls
.lock()
.expect("probe calls lock")
.push(SessionObservedAttempt {
user_agent: request.headers.get("User-Agent").cloned(),
accept_language: request.headers.get("Accept-Language").cloned(),
cookie: request.headers.get("Cookie").cloned(),
proxy: request.proxy.clone(),
});
let mut headers = HashMap::new();
if let Some(sc) = &self.set_cookie {
headers.insert("set-cookie".to_string(), sc.clone());
}
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: self.status,
content: "<html><head><title>probe</title></head><body><p>session probe page \
with enough visible text for structural checks.</p></body></html>"
.to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers,
response_time_ms: 1,
final_url: None,
})
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
"session-probe"
}
}
fn session_probe_request(session_id: &str) -> InternalScrapeRequest {
InternalScrapeRequest {
url: "http://1.1.1.1/probe".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(10),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: Some(session_id.to_string()),
wait_for: None,
needs_mllm: false,
}
}
fn make_two_proxy_provider() -> Arc<dyn ProxyProvider> {
Arc::new(ProxyPool::from_urls(
vec![
"http://proxy-a:8080".to_string(),
"http://proxy-b:8080".to_string(),
],
Duration::from_secs(60),
Duration::from_secs(30),
))
}
fn make_enabled_session(
max_error_score: f64,
provider: Option<Arc<dyn ProxyProvider>>,
) -> SessionManager {
SessionManager::from_settings(
&SessionSettings {
enabled: true,
max_sessions: 16,
max_error_score,
max_uses: 1000,
max_age_secs: 1800,
cookie_cache_max_bytes: 4096,
},
Arc::new(UaPool::new()),
provider,
)
}
#[tokio::test]
async fn test_session_identity_rotates_after_error_score_threshold() {
let engine = Arc::new(SessionProbeEngine::new(401, None));
let session = make_enabled_session(1.0, Some(make_two_proxy_provider()));
let router = EngineRouter::with_discipline_and_session(
vec![engine.clone() as Arc<dyn ScraperEngine>],
HostDiscipline::new(Duration::from_millis(1), Duration::from_secs(60), 3),
session,
);
let req = session_probe_request("sess-401");
router.route(&req).await.expect("第一次 route");
router.route(&req).await.expect("第二次 route");
let obs = engine.observations();
assert_eq!(obs.len(), 2, "各一次 attempt");
assert_ne!(
obs[0].user_agent, obs[1].user_agent,
"身份退休后 UA 必须更换"
);
assert_ne!(obs[0].proxy, obs[1].proxy, "身份退休后代理必须更换");
}
#[tokio::test]
async fn test_stalled_host_forces_session_identity_rotation() {
let engine = Arc::new(SessionProbeEngine::new(200, None));
let session = make_enabled_session(100.0, None); let router = EngineRouter::with_discipline_and_session(
vec![engine.clone() as Arc<dyn ScraperEngine>],
HostDiscipline::with_options(PolitenessOptions {
min_interval: Duration::from_millis(1),
cooldown_default: Duration::ZERO,
backoff_base: Duration::ZERO,
stalled_threshold: 3,
..PolitenessOptions::default()
}),
session,
);
let req = session_probe_request("sess-stall");
router.route(&req).await.expect("停滞前 route");
for _ in 0..3 {
router
.host_discipline
.observe("1.1.1.1", ObserveOutcome::RateLimited { retry_after: None });
}
router.route(&req).await.expect("停滞中 route");
let obs = engine.observations();
assert_eq!(obs.len(), 2);
assert_ne!(
obs[0].user_agent, obs[1].user_agent,
"stalled host 必须强制换身份(UA 更换)"
);
let pool = UaPool::new();
for (i, o) in obs.iter().enumerate() {
let ua = o.user_agent.as_deref().unwrap_or_default();
let profile = pool
.desktop
.iter()
.find(|p| p.ua == ua)
.unwrap_or_else(|| panic!("attempt #{i} 的 UA 必须来自内置 profile:{ua}"));
assert_eq!(
o.accept_language.as_deref(),
Some(profile.accept_language),
"attempt #{i} 的 Accept-Language 必须与 UA 同 profile 绑定"
);
}
}
#[tokio::test]
async fn test_session_disabled_keeps_request_passthrough() {
let engine = Arc::new(SessionProbeEngine::new(200, Some("sid=server1")));
let router = EngineRouter::with_host_discipline(
vec![engine.clone() as Arc<dyn ScraperEngine>],
HostDiscipline::new(Duration::from_millis(1), Duration::from_secs(60), 3),
);
let mut req = session_probe_request("sess-off");
req.proxy = Some("http://request-proxy:9090".to_string());
router.route(&req).await.expect("第一次 route");
router.route(&req).await.expect("第二次 route");
let obs = engine.observations();
assert_eq!(obs.len(), 2);
for o in &obs {
assert_eq!(
o.proxy.as_deref(),
Some("http://request-proxy:9090"),
"禁用时请求代理必须原样透传"
);
assert!(o.cookie.is_none(), "禁用时不得注入 Cookie 通道");
assert!(
o.user_agent.is_none(),
"禁用时首次 attempt 无 rotate_ua 指令,不得注入 UA"
);
}
}
#[tokio::test]
async fn test_session_cookie_captured_then_injected() {
let engine = Arc::new(SessionProbeEngine::new(200, Some("sid=abc123; Path=/")));
let session = make_enabled_session(3.0, Some(make_two_proxy_provider()));
let router = EngineRouter::with_discipline_and_session(
vec![engine.clone() as Arc<dyn ScraperEngine>],
HostDiscipline::new(Duration::from_millis(1), Duration::from_secs(60), 3),
session,
);
let req = session_probe_request("sess-cookie");
router.route(&req).await.expect("第一次 route");
router.route(&req).await.expect("第二次 route");
let obs = engine.observations();
assert_eq!(obs.len(), 2);
assert_eq!(obs[0].cookie, None, "首次请求无 Cookie 缓存");
assert_eq!(
obs[1].cookie.as_deref(),
Some("sid=abc123"),
"同身份第二请求必须注入捕获的 Cookie"
);
assert_eq!(
obs[0].user_agent, obs[1].user_agent,
"2xx 成功后同 session 保持同一身份(不轮换)"
);
assert_eq!(obs[0].proxy, obs[1].proxy, "2xx 成功后代理粘性保持");
}
fn spa_shell_fixture(prefix: &str) -> String {
let mut body =
String::from("<!DOCTYPE html><html><head><title>Ticket Search</title></head><body>");
for i in 0..10 {
body.push_str(&format!(
r#"<script src="/assets/{prefix}-{i:02}.js"></script>"#
));
}
body.push_str(
"<p>Legacy multi-page document that embeds many external scripts but still renders \
its content server-side; the original HTTP response already carries the payload \
that callers need when no JavaScript execution is required.</p>",
);
body.push_str("</body></html>");
body
}
struct JsUpgradeMockEngine {
name: &'static str,
content: String,
http_score: u8,
js_score: u8,
calls: std::sync::atomic::AtomicU32,
}
#[async_trait]
impl ScraperEngine for JsUpgradeMockEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let mut headers = HashMap::new();
headers.insert(
"content-type".to_string(),
"text/html; charset=utf-8".to_string(),
);
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: self.content.clone(),
screenshot: None,
content_type: "text/html".to_string(),
headers,
response_time_ms: 5,
final_url: None,
})
}
fn support_score(&self, request: &InternalScrapeRequest) -> u8 {
if request.needs_js {
self.js_score
} else {
self.http_score
}
}
fn name(&self) -> &'static str {
self.name
}
}
fn js_upgrade_request() -> InternalScrapeRequest {
InternalScrapeRequest {
url: "http://1.1.1.1/otn/leftTicket/init".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(10),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
}
}
#[tokio::test]
async fn test_js_upgrade_dead_end_degrades_to_original_response() {
let engine = Arc::new(JsUpgradeMockEngine {
name: "http-only",
content: spa_shell_fixture("app"),
http_score: 100,
js_score: 10,
calls: std::sync::atomic::AtomicU32::new(0),
});
let router = EngineRouter::new(vec![engine.clone() as Arc<dyn ScraperEngine>]);
let response = router
.route(&js_upgrade_request())
.await
.expect("dead-end js upgrade must degrade instead of failing");
assert_eq!(response.content, engine.content, "原始响应体必须逐字节返回");
assert_eq!(response.status_code, 200);
assert_eq!(
engine.calls.load(std::sync::atomic::Ordering::SeqCst),
1,
"死端降级不得递归重派(scrape 恰 1 次)"
);
let stats = router.get_engine_stats();
let stat = stats.get("http-only").expect("engine stats 必须存在");
assert_eq!(stat.usage_count, 1, "降级只记一次成功口径");
assert!(
stat.success_rate > 0.99,
"probe 命中的失败口径不得残留,success_rate={}",
stat.success_rate
);
assert!(stat.last_used.is_some());
assert!(
!router.host_needs_browser("1.1.1.1"),
"降级不得写入 needs_js host 提示"
);
}
#[tokio::test]
async fn test_js_upgrade_reroute_dispatches_browser_when_available() {
let http = Arc::new(JsUpgradeMockEngine {
name: "http-reqwest",
content: spa_shell_fixture("legacy"),
http_score: 100,
js_score: 10,
calls: std::sync::atomic::AtomicU32::new(0),
});
let browser = Arc::new(JsUpgradeMockEngine {
name: "browser-playwright",
content: "<html><body>This is the fully rendered content from the browser engine \
after JavaScript execution completed successfully.</body></html>"
.to_string(),
http_score: 10,
js_score: 100,
calls: std::sync::atomic::AtomicU32::new(0),
});
let router = EngineRouter::with_host_discipline(
vec![
http.clone() as Arc<dyn ScraperEngine>,
browser.clone() as Arc<dyn ScraperEngine>,
],
HostDiscipline::new(Duration::from_millis(1), Duration::from_secs(60), 3),
);
let response = router
.route(&js_upgrade_request())
.await
.expect("probe 命中且存在 JS 引擎时必须升级成功");
assert!(
response.content.contains("fully rendered"),
"最终响应必须来自 JS 引擎,got: {}",
response.content
);
assert_eq!(
http.calls.load(std::sync::atomic::Ordering::SeqCst),
1,
"HTTP 引擎恰被调用 1 次"
);
assert_eq!(
browser.calls.load(std::sync::atomic::Ordering::SeqCst),
1,
"JS 引擎恰被调用 1 次(升级改派)"
);
let stats = router.get_engine_stats();
let http_stat = stats.get("http-reqwest").expect("http stats 必须存在");
assert_eq!(http_stat.usage_count, 1, "HTTP 引擎该次按失败计 1 次 usage");
assert!(
http_stat.success_rate < 0.95,
"HTTP 引擎失败记账必须保留,success_rate={}",
http_stat.success_rate
);
let browser_stat = stats
.get("browser-playwright")
.expect("browser stats 必须存在");
assert_eq!(browser_stat.usage_count, 1);
assert!(browser_stat.success_rate > 0.99);
}
#[cfg(feature = "engine-moli")]
mod memory_critical {
use super::*;
use crate::engines::memory_watermark::{MemoryLevel, MemoryWatermark};
struct ScoredEngine {
engine_name: &'static str,
js_score: u8,
static_score: u8,
calls: std::sync::atomic::AtomicU32,
}
impl ScoredEngine {
fn new(engine_name: &'static str, js_score: u8, static_score: u8) -> Arc<Self> {
Arc::new(Self {
engine_name,
js_score,
static_score,
calls: std::sync::atomic::AtomicU32::new(0),
})
}
fn calls(&self) -> u32 {
self.calls.load(std::sync::atomic::Ordering::SeqCst)
}
}
#[async_trait]
impl ScraperEngine for ScoredEngine {
async fn scrape(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
let _ = request;
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: format!(
"<html><body><h1>{}</h1><p>engine-served page body with enough visible text for the antibot near-empty heuristic.</p></body></html>",
self.engine_name
),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
})
}
fn support_score(&self, request: &InternalScrapeRequest) -> u8 {
if request.needs_js {
self.js_score
} else {
self.static_score
}
}
fn name(&self) -> &'static str {
self.engine_name
}
}
fn needs_js_request() -> InternalScrapeRequest {
let mut req = make_request();
req.needs_js = true;
req
}
fn critical_watermark() -> Arc<MemoryWatermark> {
let wm = MemoryWatermark::new(512, 2048);
assert_eq!(wm.apply_sample(Some(100)), MemoryLevel::Critical);
wm
}
#[tokio::test]
async fn critical_fallback_to_static_marks_js_degraded() {
let playwright = ScoredEngine::new("playwright", 200, 200);
let reqwest = ScoredEngine::new("reqwest", 10, 200);
let mut router = EngineRouter::new(vec![
Arc::clone(&playwright) as Arc<dyn ScraperEngine>,
Arc::clone(&reqwest) as Arc<dyn ScraperEngine>,
]);
router.set_memory_scheduler(Some(critical_watermark()), true);
let response = router
.route(&needs_js_request())
.await
.expect("fallback_to_static=true 时静态回退应成功");
assert!(response.js_degraded, "静态回退必须标记 js_degraded=true");
assert!(
response.content.contains("reqwest"),
"回退应由 reqwest 引擎服务,实际内容: {}",
response.content
);
assert_eq!(playwright.calls(), 0, "Critical 下 Playwright 不得被尝试");
}
#[tokio::test]
async fn critical_fast_fail_when_fallback_disabled() {
let playwright = ScoredEngine::new("playwright", 200, 200);
let reqwest = ScoredEngine::new("reqwest", 10, 200);
let mut router = EngineRouter::new(vec![
Arc::clone(&playwright) as Arc<dyn ScraperEngine>,
Arc::clone(&reqwest) as Arc<dyn ScraperEngine>,
]);
router.set_memory_scheduler(Some(critical_watermark()), false);
let result = router.route(&needs_js_request()).await;
match result {
Err(EngineError::MemoryConstrained(_)) => {}
other => panic!(
"期望 MemoryConstrained 快速失败,实际: {:?}",
other.map(|r| r.js_degraded)
),
}
assert_eq!(playwright.calls(), 0);
assert_eq!(reqwest.calls(), 0, "快速失败路径不得触发静态引擎");
}
#[tokio::test]
async fn abundant_routes_browser_normally_without_degradation() {
let playwright = ScoredEngine::new("playwright", 200, 200);
let reqwest = ScoredEngine::new("reqwest", 10, 200);
let mut router = EngineRouter::new(vec![
Arc::clone(&playwright) as Arc<dyn ScraperEngine>,
Arc::clone(&reqwest) as Arc<dyn ScraperEngine>,
]);
let wm = MemoryWatermark::new(512, 2048);
assert_eq!(wm.apply_sample(Some(4096)), MemoryLevel::Abundant);
router.set_memory_scheduler(Some(wm), true);
let response = router
.route(&needs_js_request())
.await
.expect("Abundant 正常路由");
assert!(!response.js_degraded, "Abundant 路由不得标记降级");
assert!(response.content.contains("playwright"));
}
#[tokio::test]
async fn critical_without_watermark_keeps_legacy_behavior() {
let playwright = ScoredEngine::new("playwright", 200, 200);
let router = EngineRouter::new(vec![Arc::clone(&playwright) as Arc<dyn ScraperEngine>]);
let response = router
.route(&needs_js_request())
.await
.expect("无水位时正常路由");
assert!(!response.js_degraded);
assert_eq!(playwright.calls(), 1);
}
}