use super::*;
use crate::engines::engine_client::{
EngineError, InternalScrapeRequest, InternalScrapeResponse, ScraperEngine,
};
use async_trait::async_trait;
use std::collections::HashMap;
use std::sync::atomic::{AtomicU32, Ordering};
use std::time::Duration;
struct TestScraperEngineImpl {
name: &'static str,
_supported_domains: Vec<String>,
_weight: u8,
response_content: String,
is_error: bool,
call_count: AtomicU32,
max_calls: u32,
}
impl TestScraperEngineImpl {
fn new(
name: &'static str,
supported_domains: Vec<String>,
weight: u8,
result: Result<InternalScrapeResponse, EngineError>,
max_calls: u32,
) -> Self {
match result {
Ok(resp) => Self {
name,
_supported_domains: supported_domains,
_weight: weight,
response_content: resp.content,
is_error: false,
call_count: AtomicU32::new(0),
max_calls,
},
Err(_) => Self {
name,
_supported_domains: supported_domains,
_weight: weight,
response_content: String::new(),
is_error: true,
call_count: AtomicU32::new(0),
max_calls,
},
}
}
}
#[async_trait]
impl ScraperEngine for TestScraperEngineImpl {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
let call_count = self.call_count.fetch_add(1, Ordering::SeqCst) + 1;
if call_count <= self.max_calls {
if self.is_error {
return Err(EngineError::Timeout(Duration::from_secs(30)));
}
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: self.response_content.clone(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
})
} else {
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "Default Result".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
})
}
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
self.name
}
}
#[tokio::test]
async fn test_aggregate_concurrent_search() {
let engine1 = TestScraperEngineImpl::new(
"engine1",
vec!["example.com".to_string()],
1,
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "Result 1".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
}),
10, );
let engine2 = TestScraperEngineImpl::new(
"engine2",
vec!["example.com".to_string()],
1,
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "Result 2".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
}),
10, );
let router = EngineRouter::new(vec![Arc::new(engine1), Arc::new(engine2)]);
let request = InternalScrapeRequest {
url: "http://example.com".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
let result = router.aggregate(&request).await;
assert!(result.is_ok());
let response = result.unwrap();
assert!(response.content.contains("Result"));
}
#[tokio::test]
async fn test_aggregate_partial_failure() {
let engine1 = TestScraperEngineImpl::new(
"engine1",
vec!["example.com".to_string()],
1,
Err(EngineError::Timeout(Duration::from_secs(30))),
10, );
let engine2 = TestScraperEngineImpl::new(
"engine2",
vec!["example.com".to_string()],
1,
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "Result 2".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
}),
10, );
let router = EngineRouter::new(vec![Arc::new(engine1), Arc::new(engine2)]);
let request = InternalScrapeRequest {
url: "http://example.com".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
let result = router.aggregate(&request).await;
assert!(result.is_ok());
let response = result.unwrap();
assert_eq!(response.content, "Result 2");
}
#[cfg(feature = "content")]
#[tokio::test]
async fn test_t013_antibot_cloudflare_forces_needs_js_for_next_attempt() {
use std::sync::Mutex;
struct NeedsJsRecordingEngine {
name: &'static str,
recorded_needs_js: Arc<Mutex<Vec<bool>>>,
response: InternalScrapeResponse,
}
#[async_trait]
impl ScraperEngine for NeedsJsRecordingEngine {
async fn scrape(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
self.recorded_needs_js
.lock()
.expect("lock recorded_needs_js")
.push(request.needs_js);
Ok(self.response.clone())
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
self.name
}
}
let cloudflare_body = concat!(
"<html><head><title>Just a moment...</title></head>",
"<body>",
"<script src=\"/cdn-cgi/challenge-platform/h/g/orchestrate/jsch/v1\"></script>",
"</body></html>"
);
let http_record = Arc::new(Mutex::new(Vec::new()));
let http_engine: Arc<dyn ScraperEngine> = Arc::new(NeedsJsRecordingEngine {
name: "http-reqwest",
recorded_needs_js: http_record.clone(),
response: InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: cloudflare_body.to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 50,
final_url: None,
},
});
let browser_record = Arc::new(Mutex::new(Vec::new()));
let browser_engine: Arc<dyn ScraperEngine> = Arc::new(NeedsJsRecordingEngine {
name: "browser-playwright",
recorded_needs_js: browser_record.clone(),
response: InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "<html><body>This is the real rendered content from the browser \
engine after JavaScript execution completed successfully.</body></html>"
.to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 200,
final_url: None,
},
});
let mut router = EngineRouter::new(vec![http_engine, browser_engine]);
router.set_feature_filter_enabled(false);
router.set_race_mode_enabled(false);
router.set_strategy(LoadBalancingStrategy::WeightedRoundRobin);
router.set_max_engine_attempts(2);
router.set_max_retries(2);
let request = InternalScrapeRequest {
url: "https://example.com/protected".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
let result = router.route(&request).await;
assert!(
result.is_ok(),
"route should succeed via browser engine after antibot block, got: {:?}",
result.err()
);
let resp = result.unwrap();
assert!(resp
.content
.contains("real rendered content from the browser"));
let http_calls = http_record.lock().unwrap().clone();
assert_eq!(
http_calls.len(),
1,
"http engine should be called exactly once"
);
assert!(
!http_calls[0],
"first attempt must have needs_js=false (original request)"
);
let browser_calls = browser_record.lock().unwrap().clone();
assert_eq!(
browser_calls.len(),
1,
"browser engine should be called exactly once"
);
assert!(
browser_calls[0],
"second attempt must have needs_js=true (force_needs_js after antibot block)"
);
}
#[cfg(feature = "content")]
#[tokio::test]
async fn test_t013_normal_response_does_not_trigger_force_needs_js() {
use std::sync::Mutex;
struct SingleCallEngine {
name: &'static str,
recorded_needs_js: Arc<Mutex<Vec<bool>>>,
response: InternalScrapeResponse,
}
#[async_trait]
impl ScraperEngine for SingleCallEngine {
async fn scrape(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
self.recorded_needs_js
.lock()
.expect("lock recorded_needs_js")
.push(request.needs_js);
Ok(self.response.clone())
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
self.name
}
}
let http_record = Arc::new(Mutex::new(Vec::new()));
let http_engine: Arc<dyn ScraperEngine> = Arc::new(SingleCallEngine {
name: "http-reqwest",
recorded_needs_js: http_record.clone(),
response: InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "<html><body>Normal page with sufficient visible text content \
to pass tier3 structural checks.</body></html>"
.to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 30,
final_url: None,
},
});
let browser_record = Arc::new(Mutex::new(Vec::new()));
let browser_engine: Arc<dyn ScraperEngine> = Arc::new(SingleCallEngine {
name: "browser-playwright",
recorded_needs_js: browser_record.clone(),
response: InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "browser content".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 0,
final_url: None,
},
});
let mut router = EngineRouter::new(vec![http_engine, browser_engine]);
router.set_feature_filter_enabled(false);
router.set_race_mode_enabled(false);
router.set_strategy(LoadBalancingStrategy::WeightedRoundRobin);
router.set_max_engine_attempts(2);
router.set_max_retries(2);
let request = InternalScrapeRequest {
url: "https://example.com/normal".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
let result = router.route(&request).await;
assert!(result.is_ok());
assert_eq!(result.unwrap().content, "<html><body>Normal page with sufficient visible text content to pass tier3 structural checks.</body></html>");
let http_calls = http_record.lock().unwrap().clone();
assert_eq!(http_calls.len(), 1);
assert!(!http_calls[0]);
let browser_calls = browser_record.lock().unwrap().clone();
assert!(
browser_calls.is_empty(),
"browser engine should NOT be called for normal response"
);
}
#[tokio::test]
async fn test_t015_spa_shell_triggers_js_upgrade_re_dispatch() {
use std::sync::Mutex;
struct NeedsJsRecordingEngine {
name: &'static str,
recorded_needs_js: Arc<Mutex<Vec<bool>>>,
response: InternalScrapeResponse,
}
#[async_trait]
impl ScraperEngine for NeedsJsRecordingEngine {
async fn scrape(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
self.recorded_needs_js
.lock()
.expect("lock recorded_needs_js")
.push(request.needs_js);
Ok(self.response.clone())
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
self.name
}
}
let spa_shell = concat!(
r#"<html><head>"#,
r#"<script id="__NEXT_DATA__" type="application/json">{"props":{}}</script>"#,
r#"</head><body>"#,
r#"Loading... please wait while we render the content for you. "#,
r#"This page requires JavaScript to function properly."#,
r#"</body></html>"#
);
let http_record = Arc::new(Mutex::new(Vec::new()));
let http_engine: Arc<dyn ScraperEngine> = Arc::new(NeedsJsRecordingEngine {
name: "http-reqwest",
recorded_needs_js: http_record.clone(),
response: InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: spa_shell.to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 30,
final_url: None,
},
});
let browser_record = Arc::new(Mutex::new(Vec::new()));
let browser_engine: Arc<dyn ScraperEngine> = Arc::new(NeedsJsRecordingEngine {
name: "browser-playwright",
recorded_needs_js: browser_record.clone(),
response: InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "<html><body>This is the fully rendered content from the browser \
engine after JavaScript execution completed successfully.</body></html>"
.to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 200,
final_url: None,
},
});
let mut router = EngineRouter::new(vec![http_engine, browser_engine]);
router.set_feature_filter_enabled(false);
router.set_race_mode_enabled(false);
router.set_strategy(LoadBalancingStrategy::WeightedRoundRobin);
router.set_max_engine_attempts(2);
router.set_max_retries(2);
let request = InternalScrapeRequest {
url: "https://example.com/spa-page".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
let result = router.route(&request).await;
assert!(
result.is_ok(),
"route should succeed via browser engine after SPA shell probe, got: {:?}",
result.err()
);
let resp = result.unwrap();
assert!(
resp.content
.contains("fully rendered content from the browser"),
"should return browser engine's rendered content, got: {}",
resp.content
);
let http_calls = http_record.lock().unwrap().clone();
assert_eq!(
http_calls.len(),
1,
"http engine should be called exactly once"
);
assert!(
!http_calls[0],
"http engine attempt must have needs_js=false (original request)"
);
let browser_calls = browser_record.lock().unwrap().clone();
assert_eq!(
browser_calls.len(),
1,
"browser engine should be called exactly once"
);
assert!(
browser_calls[0],
"browser engine attempt must have needs_js=true (probe-triggered re-route)"
);
}
#[tokio::test]
async fn test_t015_non_spa_response_does_not_trigger_re_dispatch() {
use std::sync::Mutex;
struct SingleCallEngine {
name: &'static str,
recorded_needs_js: Arc<Mutex<Vec<bool>>>,
response: InternalScrapeResponse,
}
#[async_trait]
impl ScraperEngine for SingleCallEngine {
async fn scrape(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
self.recorded_needs_js
.lock()
.expect("lock recorded_needs_js")
.push(request.needs_js);
Ok(self.response.clone())
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
self.name
}
}
let http_record = Arc::new(Mutex::new(Vec::new()));
let http_engine: Arc<dyn ScraperEngine> = Arc::new(SingleCallEngine {
name: "http-reqwest",
recorded_needs_js: http_record.clone(),
response: InternalScrapeResponse { js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "<html><body>This is a static page with sufficient visible text content \
to pass all antibot and probe checks. No SPA framework signals here.</body></html>"
.to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 30,
final_url: None,
},
});
let browser_record = Arc::new(Mutex::new(Vec::new()));
let browser_engine: Arc<dyn ScraperEngine> = Arc::new(SingleCallEngine {
name: "browser-playwright",
recorded_needs_js: browser_record.clone(),
response: InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "browser content".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 0,
final_url: None,
},
});
let mut router = EngineRouter::new(vec![http_engine, browser_engine]);
router.set_feature_filter_enabled(false);
router.set_race_mode_enabled(false);
router.set_strategy(LoadBalancingStrategy::WeightedRoundRobin);
router.set_max_engine_attempts(2);
router.set_max_retries(2);
let request = InternalScrapeRequest {
url: "https://example.com/static-page".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
let result = router.route(&request).await;
assert!(result.is_ok());
assert!(result
.unwrap()
.content
.contains("static page with sufficient visible text"));
let http_calls = http_record.lock().unwrap().clone();
assert_eq!(http_calls.len(), 1);
assert!(!http_calls[0]);
let browser_calls = browser_record.lock().unwrap().clone();
assert!(
browser_calls.is_empty(),
"browser engine should NOT be called for non-SPA response"
);
}
#[tokio::test]
async fn test_t028_ua_rotated_across_transient_retries() {
use std::sync::Mutex;
struct UaRecordingEngine {
name: &'static str,
recorded_ua: Arc<Mutex<Vec<Option<String>>>>,
error_msg: Option<String>,
response: Option<InternalScrapeResponse>,
}
#[async_trait]
impl ScraperEngine for UaRecordingEngine {
async fn scrape(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
let ua = request.headers.get("User-Agent").map(|v| v.to_string());
self.recorded_ua.lock().expect("lock recorded_ua").push(ua);
if let Some(ref msg) = self.error_msg {
return Err(EngineError::RequestFailed(msg.clone()));
}
Ok(self.response.clone().expect("success response must be set"))
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
self.name
}
}
fn make_failing_engine(
name: &'static str,
record: Arc<Mutex<Vec<Option<String>>>>,
error_msg: &str,
) -> Arc<dyn ScraperEngine> {
Arc::new(UaRecordingEngine {
name,
recorded_ua: record,
error_msg: Some(error_msg.to_string()),
response: None,
})
}
fn make_success_engine(
name: &'static str,
record: Arc<Mutex<Vec<Option<String>>>>,
) -> Arc<dyn ScraperEngine> {
Arc::new(UaRecordingEngine {
name,
recorded_ua: record,
error_msg: None,
response: Some(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "<html><body>success content with enough visible text to pass \
any antibot or probe checks along the retry path</body></html>"
.to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
}),
})
}
let rec1 = Arc::new(Mutex::new(Vec::new()));
let rec2 = Arc::new(Mutex::new(Vec::new()));
let rec3 = Arc::new(Mutex::new(Vec::new()));
let rec4 = Arc::new(Mutex::new(Vec::new()));
let engines: Vec<Arc<dyn ScraperEngine>> = vec![
make_failing_engine("fail-1", rec1.clone(), "transient-1"),
make_failing_engine("fail-2", rec2.clone(), "transient-2"),
make_failing_engine("fail-3", rec3.clone(), "transient-3"),
make_success_engine("success-4", rec4.clone()),
];
let mut router = EngineRouter::new(engines);
router.set_feature_filter_enabled(false);
router.set_race_mode_enabled(false);
router.set_strategy(LoadBalancingStrategy::WeightedRoundRobin);
router.set_max_engine_attempts(4);
router.set_max_retries(4);
let request = InternalScrapeRequest {
url: "https://example.com/test".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
let result = router.route(&request).await;
assert!(
result.is_ok(),
"route should succeed via 4th engine after 3 transient failures, got: {:?}",
result.err()
);
let r1 = rec1.lock().unwrap().clone();
assert_eq!(r1.len(), 1, "engine 1 should be called exactly once");
assert!(
r1[0].is_none(),
"attempt 1 must not rotate UA (default directive)"
);
let r2 = rec2.lock().unwrap().clone();
assert_eq!(r2.len(), 1, "engine 2 should be called exactly once");
assert!(
r2[0].is_none(),
"attempt 2 must not rotate UA (Transient attempt=0 → default directive)"
);
let r3 = rec3.lock().unwrap().clone();
assert_eq!(r3.len(), 1, "engine 3 should be called exactly once");
assert!(
r3[0].is_some(),
"attempt 3 must rotate UA (Transient attempt=1 → rotate_ua=true)"
);
let ua3 = r3[0].clone().unwrap();
let r4 = rec4.lock().unwrap().clone();
assert_eq!(r4.len(), 1, "engine 4 should be called exactly once");
assert!(
r4[0].is_some(),
"attempt 4 must rotate UA (Transient attempt=2 → rotate_ua=true)"
);
let ua4 = r4[0].clone().unwrap();
assert_ne!(
ua3, ua4,
"UA must differ across retry attempts (seed=2 vs seed=3)"
);
}
#[tokio::test]
async fn test_c1_fingerprint_headers_rotated_together() {
use crate::utils::ua_pool::UaPool;
use std::sync::Mutex;
type RecordedHeaders = Vec<(Option<String>, Option<String>, Option<String>)>;
struct FingerprintRecordingEngine {
name: &'static str,
recorded: Arc<Mutex<RecordedHeaders>>,
error_msg: Option<String>,
response: Option<InternalScrapeResponse>,
}
#[async_trait]
impl ScraperEngine for FingerprintRecordingEngine {
async fn scrape(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
let ua = request.headers.get("User-Agent").map(|v| v.to_string());
let al = request
.headers
.get("Accept-Language")
.map(|v| v.to_string());
let ch = request.headers.get("sec-ch-ua").map(|v| v.to_string());
self.recorded
.lock()
.expect("lock recorded")
.push((ua, al, ch));
if let Some(ref msg) = self.error_msg {
return Err(EngineError::RequestFailed(msg.clone()));
}
Ok(self.response.clone().expect("success response must be set"))
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
self.name
}
}
fn make_failing(
name: &'static str,
rec: Arc<Mutex<RecordedHeaders>>,
) -> Arc<dyn ScraperEngine> {
Arc::new(FingerprintRecordingEngine {
name,
recorded: rec,
error_msg: Some("transient".to_string()),
response: None,
})
}
fn make_success(
name: &'static str,
rec: Arc<Mutex<RecordedHeaders>>,
) -> Arc<dyn ScraperEngine> {
Arc::new(FingerprintRecordingEngine {
name,
recorded: rec,
error_msg: None,
response: Some(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "<html><body>success content with enough visible text to pass \
any antibot or probe checks along the retry path</body></html>"
.to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
}),
})
}
let rec1 = Arc::new(Mutex::new(Vec::new()));
let rec2 = Arc::new(Mutex::new(Vec::new()));
let rec3 = Arc::new(Mutex::new(Vec::new()));
let rec4 = Arc::new(Mutex::new(Vec::new()));
let engines: Vec<Arc<dyn ScraperEngine>> = vec![
make_failing("fail-1", rec1.clone()),
make_failing("fail-2", rec2.clone()),
make_failing("fail-3", rec3.clone()),
make_success("success-4", rec4.clone()),
];
let mut router = EngineRouter::new(engines);
router.set_feature_filter_enabled(false);
router.set_race_mode_enabled(false);
router.set_strategy(LoadBalancingStrategy::WeightedRoundRobin);
router.set_max_engine_attempts(4);
router.set_max_retries(4);
let request = InternalScrapeRequest {
url: "https://example.com/test".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
let result = router.route(&request).await;
assert!(
result.is_ok(),
"route should succeed via 4th engine after 3 transient failures, got: {:?}",
result.err()
);
let r3 = rec3.lock().unwrap().clone();
assert_eq!(r3.len(), 1, "engine 3 should be called exactly once");
let (ua3, al3, ch3) = r3[0].clone();
assert!(ua3.is_some(), "attempt 3 must rotate User-Agent");
let ua3 = ua3.expect("ua3 set");
let r4 = rec4.lock().unwrap().clone();
assert_eq!(r4.len(), 1, "engine 4 should be called exactly once");
let (ua4, al4, ch4) = r4[0].clone();
assert!(ua4.is_some(), "attempt 4 must rotate User-Agent");
let ua4 = ua4.expect("ua4 set");
let pool = UaPool::new();
let p3 = pool.pick_seeded(2, false);
let p4 = pool.pick_seeded(3, false);
assert_eq!(
ua3, p3.ua,
"attempt 3 User-Agent must match pick_seeded(2).ua"
);
assert_eq!(
al3.as_deref(),
Some(p3.accept_language),
"attempt 3 Accept-Language must match profile.accept_language (C-1: 同步轮换)"
);
assert_eq!(
ch3.as_deref(),
if p3.sec_ch_ua.is_empty() {
None
} else {
Some(p3.sec_ch_ua)
},
"attempt 3 sec-ch-ua must match profile.sec_ch_ua (C-1: 同步轮换,Firefox/Safari 为 None)"
);
assert_eq!(
ua4, p4.ua,
"attempt 4 User-Agent must match pick_seeded(3).ua"
);
assert_eq!(
al4.as_deref(),
Some(p4.accept_language),
"attempt 4 Accept-Language must match profile.accept_language (C-1: 同步轮换)"
);
assert_eq!(
ch4.as_deref(),
if p4.sec_ch_ua.is_empty() {
None
} else {
Some(p4.sec_ch_ua)
},
"attempt 4 sec-ch-ua must match profile.sec_ch_ua (C-1: 同步轮换)"
);
assert_ne!(
ua3, ua4,
"UA must differ across retry attempts (seed=2 vs seed=3)"
);
}
#[tokio::test]
async fn test_t028_retry_tracker_caps_feature_toggle() {
use std::sync::Mutex;
struct FtFailingEngine {
name: &'static str,
call_count: Arc<Mutex<u32>>,
}
#[async_trait]
impl ScraperEngine for FtFailingEngine {
async fn scrape(
&self,
_request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
let mut c = self.call_count.lock().unwrap();
*c += 1;
Err(EngineError::FeatureToggle(format!("toggle-fail-{}", *c)))
}
fn support_score(&self, _request: &InternalScrapeRequest) -> u8 {
100
}
fn name(&self) -> &'static str {
self.name
}
}
let counts: Vec<Arc<Mutex<u32>>> = (0..5).map(|_| Arc::new(Mutex::new(0u32))).collect();
let engines: Vec<Arc<dyn ScraperEngine>> = (0..5)
.map(|i| {
let e: Arc<dyn ScraperEngine> = Arc::new(FtFailingEngine {
name: Box::leak(format!("ft-fail-{}", i).into_boxed_str()),
call_count: counts[i].clone(),
});
e
})
.collect();
let mut router = EngineRouter::new(engines);
router.set_feature_filter_enabled(false);
router.set_race_mode_enabled(false);
router.set_strategy(LoadBalancingStrategy::WeightedRoundRobin);
router.set_max_engine_attempts(5);
router.set_max_retries(5);
let request = InternalScrapeRequest {
url: "https://example.com/ft-test".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
let result = router.route(&request).await;
assert!(
result.is_err(),
"route must fail after RetryTracker caps FeatureToggle"
);
let err = result.unwrap_err();
assert!(
matches!(err, EngineError::FeatureToggle(_)),
"error must be FeatureToggle, got: {:?}",
err
);
for (i, count) in counts.iter().enumerate().take(3) {
let c = count.lock().unwrap();
assert_eq!(
*c, 1,
"engine {} should be called exactly once (within cap)",
i
);
}
for (i, count) in counts.iter().enumerate().skip(3) {
let c = count.lock().unwrap();
assert_eq!(
*c, 0,
"engine {} should NOT be called (RetryTracker stopped after cap=3)",
i
);
}
}
#[cfg(feature = "content")]
#[tokio::test]
async fn test_race_mode_discards_antibot_challenge_winner() {
let cloudflare_body = concat!(
"<html><head><title>Just a moment...</title></head>",
"<body>",
"<script src=\"/cdn-cgi/challenge-platform/h/g/orchestrate/jsch/v1\"></script>",
"</body></html>"
);
let challenge_engine: Arc<dyn ScraperEngine> = Arc::new(TestScraperEngineImpl::new(
"challenge-fast",
vec!["example.com".to_string()],
100,
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: cloudflare_body.to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
}),
10,
));
let clean_engine: Arc<dyn ScraperEngine> = Arc::new(TestScraperEngineImpl::new(
"clean-slow",
vec!["example.com".to_string()],
100,
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "<html><body>This is the clean rendered content that must survive \
the race even when the challenge engine finishes first.</body></html>"
.to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 20,
final_url: None,
}),
10,
));
let mut router = EngineRouter::new(vec![challenge_engine, clean_engine]);
router.set_feature_filter_enabled(false);
router.set_race_mode_enabled(true);
let request = InternalScrapeRequest {
url: "https://example.com/protected".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
let result = router.route(&request).await;
let resp = result.expect("clean engine result must win after challenge is discarded");
assert!(
!resp.content.contains("Just a moment"),
"challenge page must never be returned in race mode"
);
assert!(
resp.content.contains("clean rendered content"),
"expected clean engine content, got: {}",
resp.content
);
}
#[cfg(feature = "content")]
#[tokio::test]
async fn test_race_mode_all_challenge_candidates_returns_error() {
let cloudflare_body = concat!(
"<html><head><title>Just a moment...</title></head>",
"<body>",
"<script src=\"/cdn-cgi/challenge-platform/h/g/orchestrate/jsch/v1\"></script>",
"</body></html>"
);
let engine_a: Arc<dyn ScraperEngine> = Arc::new(TestScraperEngineImpl::new(
"challenge-a",
vec!["example.com".to_string()],
100,
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: cloudflare_body.to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
}),
10,
));
let engine_b: Arc<dyn ScraperEngine> = Arc::new(TestScraperEngineImpl::new(
"challenge-b",
vec!["example.com".to_string()],
100,
Ok(InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: cloudflare_body.to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 20,
final_url: None,
}),
10,
));
let mut router = EngineRouter::new(vec![engine_a, engine_b]);
router.set_feature_filter_enabled(false);
router.set_race_mode_enabled(true);
let request = InternalScrapeRequest {
url: "https://example.com/protected".to_string(),
method: crate::engines::engine_client::HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
let result = router.route(&request).await;
assert!(
result.is_err(),
"all-challenge race must fail, not return a challenge page"
);
}