use crate::config::settings::ProxyStrategy;
use crate::engines::client::handle::ClientHandle;
use crate::engines::engine_client::{
EngineError, InternalScrapeRequest, InternalScrapeResponse, ScraperEngine,
};
use crate::engines::provider::{ProxyCategory, ProxyProvider};
use crate::engines::validators;
use crate::utils::http_client::{read_body_limited, BodyReadError};
use crate::utils::proxy::redact_proxy_url;
use crate::utils::ua_pool::UaPool;
use async_trait::async_trait;
use log::{error, warn};
use lru::LruCache;
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use std::num::NonZeroUsize;
use std::sync::Arc;
use std::time::{Duration, Instant};
mod request_client;
const DEFAULT_TIMEOUT_SECONDS: u64 = 30;
const DEFAULT_PROXY_STRATEGY: ProxyStrategy = ProxyStrategy::RoundRobin;
const DEFAULT_REQWEST_MRT_SECONDS: u64 = 5;
const DEFAULT_MAX_RESPONSE_BODY_BYTES: usize = 10 * 1024 * 1024;
pub(crate) fn build_all_proxy(url: &str) -> Result<reqwest::Proxy, reqwest::Error> {
reqwest::Proxy::all(url)
}
pub struct ReqwestEngine {
max_response_body_bytes: usize,
http_client: Arc<reqwest::Client>,
proxy_provider: Option<Arc<dyn ProxyProvider>>,
proxy_strategy: ProxyStrategy,
timeout_seconds: u64,
mrt: Duration,
ua_pool: UaPool,
proxy_client_cache: parking_lot::Mutex<LruCache<String, reqwest::Client>>,
}
impl ReqwestEngine {
pub fn new(http_client: Arc<reqwest::Client>) -> Self {
Self::new_with_timeout(http_client, DEFAULT_TIMEOUT_SECONDS)
}
pub fn new_with_timeout(http_client: Arc<reqwest::Client>, timeout_seconds: u64) -> Self {
Self::new_with_timeout_and_mrt(
http_client,
timeout_seconds,
Duration::from_secs(DEFAULT_REQWEST_MRT_SECONDS),
)
}
#[must_use]
pub fn new_with_timeout_and_mrt(
http_client: Arc<reqwest::Client>,
timeout_seconds: u64,
mrt: Duration,
) -> Self {
Self {
max_response_body_bytes: DEFAULT_MAX_RESPONSE_BODY_BYTES,
http_client,
proxy_provider: None,
proxy_strategy: DEFAULT_PROXY_STRATEGY,
timeout_seconds,
mrt,
ua_pool: UaPool::default(),
proxy_client_cache: parking_lot::Mutex::new(LruCache::new(
NonZeroUsize::new(64).unwrap(),
)),
}
}
#[must_use]
pub fn with_provider(
http_client: Arc<reqwest::Client>,
proxy_provider: Arc<dyn ProxyProvider>,
) -> Self {
Self::with_provider_strategy_and_timeout(
http_client,
proxy_provider,
DEFAULT_PROXY_STRATEGY,
DEFAULT_TIMEOUT_SECONDS,
)
}
#[must_use]
pub fn with_provider_strategy_and_timeout(
http_client: Arc<reqwest::Client>,
proxy_provider: Arc<dyn ProxyProvider>,
proxy_strategy: ProxyStrategy,
timeout_seconds: u64,
) -> Self {
Self::with_provider_strategy_timeout_and_mrt(
http_client,
proxy_provider,
proxy_strategy,
timeout_seconds,
Duration::from_secs(DEFAULT_REQWEST_MRT_SECONDS),
)
}
#[must_use]
pub fn with_provider_strategy_timeout_and_mrt(
http_client: Arc<reqwest::Client>,
proxy_provider: Arc<dyn ProxyProvider>,
proxy_strategy: ProxyStrategy,
timeout_seconds: u64,
mrt: Duration,
) -> Self {
Self {
max_response_body_bytes: DEFAULT_MAX_RESPONSE_BODY_BYTES,
http_client,
proxy_provider: Some(proxy_provider),
proxy_strategy,
timeout_seconds,
mrt,
ua_pool: UaPool::default(),
proxy_client_cache: parking_lot::Mutex::new(LruCache::new(
NonZeroUsize::new(64).unwrap(),
)),
}
}
#[must_use]
pub fn with_max_response_body_bytes(mut self, bytes: usize) -> Self {
self.max_response_body_bytes = bytes;
self
}
#[must_use]
pub fn ua_pool(&self) -> &UaPool {
&self.ua_pool
}
#[must_use]
pub fn has_proxy_provider(&self) -> bool {
self.proxy_provider.is_some()
}
#[must_use]
pub fn proxy_strategy(&self) -> ProxyStrategy {
self.proxy_strategy
}
#[must_use]
pub fn mrt(&self) -> Duration {
self.mrt
}
fn build_custom_client(
proxy_url: Option<&str>,
skip_tls: bool,
fallback: &Arc<reqwest::Client>,
timeout_seconds: u64,
) -> (reqwest::Client, bool) {
let mut builder = reqwest::Client::builder()
.timeout(Duration::from_secs(timeout_seconds))
.connect_timeout(crate::utils::http_client::connect_timeout_for(
Duration::from_secs(timeout_seconds),
))
.cookie_store(true)
.local_address(Some(std::net::Ipv4Addr::UNSPECIFIED.into()))
.dns_resolver(crate::infrastructure::dns::create_ipv4_only_resolver())
.redirect(crate::utils::http_client::create_ssrf_safe_redirect_policy(
5,
));
if skip_tls {
builder = builder.danger_accept_invalid_certs(true);
}
let effective_proxy = proxy_url.map(|s| s.trim()).filter(|s| !s.is_empty());
match effective_proxy {
Some(url) => {
Self::build_proxy_client(builder, url, skip_tls, fallback, timeout_seconds)
}
None => Self::build_direct_client(builder, skip_tls, fallback, timeout_seconds),
}
}
fn build_proxy_client(
builder: reqwest::ClientBuilder,
url: &str,
skip_tls: bool,
fallback: &Arc<reqwest::Client>,
timeout_seconds: u64,
) -> (reqwest::Client, bool) {
match build_all_proxy(url) {
Ok(proxy) => match builder.proxy(proxy).build() {
Ok(client) => {
log::debug!(
"Using HTTP proxy: {} (skip_tls={}, timeout={}s)",
redact_proxy_url(url),
skip_tls,
timeout_seconds
);
(client, false)
}
Err(e) => {
error!(
"Failed to build proxy client (url={}, skip_tls={}, timeout={}s): {}, \
falling back to injected http_client",
redact_proxy_url(url),
skip_tls,
timeout_seconds,
e
);
((**fallback).clone(), true)
}
},
Err(e) => {
error!(
"Failed to configure HTTP proxy (url={}): {}, falling back to injected http_client",
redact_proxy_url(url), e
);
((**fallback).clone(), true)
}
}
}
fn build_direct_client(
builder: reqwest::ClientBuilder,
skip_tls: bool,
fallback: &Arc<reqwest::Client>,
timeout_seconds: u64,
) -> (reqwest::Client, bool) {
match builder.build() {
Ok(client) => {
if skip_tls {
log::debug!(
"Using client with skip_tls=true (no proxy, timeout={}s)",
timeout_seconds
);
}
(client, false)
}
Err(e) => {
error!(
"Failed to build client (skip_tls={}, timeout={}s): {}, \
falling back to injected http_client",
skip_tls, timeout_seconds, e
);
((**fallback).clone(), true)
}
}
}
fn get_client(
&self,
proxy: &Option<String>,
skip_tls: bool,
session_id: Option<&str>,
) -> ClientHandle {
if skip_tls {
return self.build_skip_tls_handle(proxy);
}
let request_proxy = proxy.as_ref().map(|s| s.trim()).filter(|s| !s.is_empty());
if let Some(url) = request_proxy {
let (client, is_fallback) = Self::build_custom_client(
Some(url),
false,
&self.http_client,
self.timeout_seconds,
);
return ClientHandle::new(client, Some(url.to_string()), is_fallback);
}
let Some(url) = self.pick_proxy_url(session_id) else {
return ClientHandle::new(self.http_client.as_ref().clone(), None, false);
};
let (client, is_fallback) = self.cached_proxy_client(&url);
ClientHandle::new(client, Some(url), is_fallback)
}
fn build_skip_tls_handle(&self, proxy: &Option<String>) -> ClientHandle {
log::warn!(
"skip_tls_verification=true: TLS certificate validation disabled for this request"
);
let proxy_url = proxy.as_ref().map(|s| s.as_str());
let (client, is_fallback) =
Self::build_custom_client(proxy_url, true, &self.http_client, self.timeout_seconds);
let used = proxy.as_ref().and_then(|s| {
let t = s.trim();
if t.is_empty() {
None
} else {
Some(t.to_string())
}
});
ClientHandle::new(client, used, is_fallback)
}
fn pick_proxy_url(&self, session_id: Option<&str>) -> Option<String> {
let Some(provider) = &self.proxy_provider else {
return None;
};
let picked = match self.proxy_strategy {
ProxyStrategy::RoundRobin => provider.next(ProxyCategory::Html),
ProxyStrategy::Sticky => match session_id {
Some(sid) if !sid.is_empty() => provider.sticky(sid),
_ => {
log::warn!(
"proxy_strategy=Sticky but session_id is missing/empty; \
falling back to RoundRobin (next)"
);
provider.next(ProxyCategory::Html)
}
},
};
let Some(url) = picked else {
log::debug!(
"proxy_provider returned None (empty pool or all proxies in cooldown), \
using injected http_client"
);
return None;
};
Some(url)
}
fn cached_proxy_client(&self, url: &str) -> (reqwest::Client, bool) {
let mut cache = self.proxy_client_cache.lock();
if let Some(cached) = cache.get(url) {
(cached.clone(), false)
} else {
drop(cache); let (client, is_fallback) = Self::build_custom_client(
Some(url),
false,
&self.http_client,
self.timeout_seconds,
);
if !is_fallback {
self.proxy_client_cache
.lock()
.put(url.to_string(), client.clone());
}
(client, is_fallback)
}
}
fn build_request_builder(
&self,
request: &InternalScrapeRequest,
client: &reqwest::Client,
request_url: &str,
) -> reqwest::RequestBuilder {
let mut headers = HeaderMap::new();
for (k, v) in &request.headers {
if let (Ok(header_name), Ok(header_value)) = (
HeaderName::from_bytes(k.as_bytes()),
HeaderValue::from_str(v),
) {
headers.insert(header_name, header_value);
}
}
let profile = self.ua_pool.pick(request.mobile);
let mut request_builder = match request.method {
crate::engines::engine_client::HttpMethod::Get => client.get(request_url),
crate::engines::engine_client::HttpMethod::Post => client.post(request_url),
};
request_builder = request_builder
.header("User-Agent", profile.ua)
.header("Accept-Language", profile.accept_language);
if !profile.sec_ch_ua.is_empty() {
request_builder = request_builder.header("sec-ch-ua", profile.sec_ch_ua);
}
request_builder = request_builder.headers(headers);
if let Some(body) = &request.body {
request_builder = request_builder.body(body.clone());
}
request_builder = request_builder.timeout(request.timeout);
request_builder
}
async fn send_request(
&self,
request_builder: reqwest::RequestBuilder,
request: &InternalScrapeRequest,
handle: &ClientHandle,
) -> Result<reqwest::Response, EngineError> {
let response = match request_builder.send().await {
Ok(resp) => resp,
Err(e) if e.is_timeout() => {
if let Some(provider) = &self.proxy_provider {
handle.report_failure(provider.as_ref());
}
return Err(EngineError::Timeout(request.timeout));
}
Err(e) => {
if let Some(provider) = &self.proxy_provider {
handle.report_failure(provider.as_ref());
}
let mut chain = Vec::new();
let mut current: Option<&dyn std::error::Error> = Some(&e);
while let Some(err) = current {
chain.push(err.to_string());
current = err.source();
}
error!(
"reqwest send failed for {}: error chain: {}",
request.url,
chain.join(" -> ")
);
return Err(EngineError::RequestFailed(e.to_string()));
}
};
Ok(response)
}
fn collect_response_headers(
response: &reqwest::Response,
request_cookie_jar: &Arc<reqwest::cookie::Jar>,
) -> (String, std::collections::HashMap<String, String>) {
let content_type = response
.headers()
.get("content-type")
.and_then(|v| v.to_str().ok())
.unwrap_or("text/html")
.to_string();
let content_type = if content_type.trim().is_empty() {
"text/html".to_string()
} else {
content_type
};
let mut response_headers = std::collections::HashMap::with_capacity(32);
for (k, v) in response.headers() {
if let Ok(v_str) = v.to_str() {
response_headers.insert(k.as_str().to_string(), v_str.to_string());
}
}
let final_url = response.url().clone();
use reqwest::cookie::CookieStore as _;
if let Some(jar_cookies) = request_cookie_jar.cookies(&final_url) {
if let Ok(pairs) = jar_cookies.to_str() {
if !pairs.is_empty() {
response_headers.insert("set-cookie".to_string(), pairs.to_string());
}
}
}
(content_type, response_headers)
}
async fn resolve_pinned_target(
url: &str,
) -> Result<(String, reqwest::Url, Vec<std::net::SocketAddr>), EngineError> {
let validator = validators::SsrfValidator::new();
let validated_url = validator
.validate(url)
.await
.map_err(|e| EngineError::Other(format!("SSRF protection: {}", e)))?;
let host = validated_url
.parsed_url
.host_str()
.unwrap_or("")
.to_string();
let port = validated_url.port;
let resolve_addrs: Vec<std::net::SocketAddr> = validated_url
.resolved_ips
.iter()
.map(|ip| std::net::SocketAddr::new(*ip, port))
.collect();
if resolve_addrs.is_empty() {
return Err(EngineError::Other("SSRF: no resolved IPs".to_string()));
}
Ok((host, validated_url.parsed_url, resolve_addrs))
}
async fn read_body_with_limit(
response: reqwest::Response,
max_bytes: usize,
request_url: &str,
) -> Result<String, EngineError> {
read_body_limited(response, max_bytes)
.await
.map_err(|e| match e {
BodyReadError::LimitExceeded { max_bytes } => {
warn!(
"Response body from {} exceeds {} byte limit, rejecting",
request_url, max_bytes
);
EngineError::RequestFailed(format!(
"response body exceeds {} byte limit",
max_bytes
))
}
BodyReadError::Network(e) => EngineError::RequestFailed(e.to_string()),
})
}
}
#[async_trait]
impl ScraperEngine for ReqwestEngine {
async fn scrape(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
let (host, parsed_url, resolve_addrs) = Self::resolve_pinned_target(&request.url).await?;
let mut handle = self.get_client(
&request.proxy,
request.skip_tls_verification,
request.session_id.as_deref(),
);
let (request_cookie_jar, ssrf_client) =
self.build_request_client(request, &mut handle, &host, &resolve_addrs)?;
let effective_client = &ssrf_client;
let request_url = parsed_url.as_str();
let request_builder = self.build_request_builder(request, effective_client, request_url);
let start = Instant::now();
let response = self.send_request(request_builder, request, &handle).await?;
let status_code = response.status().as_u16();
let final_url = response.url().to_string();
if (200..300).contains(&status_code) {
if let Some(provider) = &self.proxy_provider {
handle.report_success(provider.as_ref());
}
}
let (content_type, response_headers) =
Self::collect_response_headers(&response, &request_cookie_jar);
let content =
Self::read_body_with_limit(response, self.max_response_body_bytes, &request.url)
.await?;
if request.sync_wait_ms > 0 {
tokio::time::sleep(Duration::from_millis(request.sync_wait_ms as u64)).await;
}
Ok(InternalScrapeResponse {
js_degraded: false,
status_code,
content,
screenshot: None,
content_type,
headers: response_headers,
response_time_ms: start.elapsed().as_millis() as u64,
final_url: Some(final_url),
captured_responses: Vec::new(),
})
}
fn support_score(&self, request: &InternalScrapeRequest) -> u8 {
if request.needs_js || request.needs_screenshot {
return 10; }
if request.needs_tls_fingerprint {
return 10;
}
100 }
fn name(&self) -> &'static str {
"reqwest"
}
fn max_response_time(&self) -> Duration {
self.mrt
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::engines::engine_client::{
HttpMethod, InternalScrapeRequest, InternalScreenshotConfig,
};
use crate::engines::proxy_pool::ProxyPool;
use std::collections::HashMap;
use std::time::Duration;
#[test]
fn test_build_all_proxy_accepts_valid_url_and_rejects_invalid() {
assert!(build_all_proxy("http://proxy.example:8080").is_ok());
assert!(build_all_proxy("socks5://user:pass@proxy.example:1080").is_ok()); assert!(build_all_proxy("::not a url::").is_err());
assert!(build_all_proxy("").is_err());
}
fn create_test_client() -> Arc<reqwest::Client> {
Arc::new(
reqwest::Client::builder()
.timeout(Duration::from_secs(30))
.build()
.unwrap(),
)
}
fn create_basic_request(url: &str) -> InternalScrapeRequest {
InternalScrapeRequest {
url: url.to_string(),
method: HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
}
}
fn create_request_with_js(url: &str) -> InternalScrapeRequest {
InternalScrapeRequest {
url: url.to_string(),
method: HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: true,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
}
}
fn create_request_with_screenshot(url: &str) -> InternalScrapeRequest {
InternalScrapeRequest {
url: url.to_string(),
method: HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: true,
screenshot_config: Some(InternalScreenshotConfig {
full_page: true,
selector: None,
quality: Some(80),
format: Some("jpeg".to_string()),
}),
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
}
}
#[test]
fn test_reqwest_engine_new() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
assert_eq!(engine.name(), "reqwest");
assert!(!engine.has_proxy_provider());
}
#[test]
fn test_reqwest_engine_with_provider() {
let client = create_test_client();
let pool: Arc<dyn ProxyProvider> = Arc::new(ProxyPool::from_urls(
vec!["http://a:8080".to_string(), "http://b:8080".to_string()],
Duration::from_secs(60),
Duration::from_secs(30),
));
let engine = ReqwestEngine::with_provider(client, pool);
assert_eq!(engine.name(), "reqwest");
assert!(engine.has_proxy_provider());
}
#[test]
fn test_reqwest_engine_with_provider_strategy_and_timeout() {
let client = create_test_client();
let pool: Arc<dyn ProxyProvider> = Arc::new(ProxyPool::from_urls(
vec!["http://a:8080".to_string()],
Duration::from_secs(60),
Duration::from_secs(30),
));
let engine = ReqwestEngine::with_provider_strategy_and_timeout(
client,
pool,
ProxyStrategy::RoundRobin,
45,
);
assert_eq!(engine.name(), "reqwest");
assert!(engine.has_proxy_provider());
assert_eq!(engine.proxy_strategy(), ProxyStrategy::RoundRobin);
assert_eq!(engine.timeout_seconds, 45);
}
#[test]
fn test_name_returns_reqwest() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
assert_eq!(engine.name(), "reqwest");
}
#[test]
fn test_name_returns_reqwest_with_provider() {
let client = create_test_client();
let pool: Arc<dyn ProxyProvider> = Arc::new(ProxyPool::from_urls(
vec!["http://proxy:8080".to_string()],
Duration::from_secs(60),
Duration::from_secs(30),
));
let engine = ReqwestEngine::with_provider(client, pool);
assert_eq!(engine.name(), "reqwest");
}
#[test]
fn test_support_score_basic_get_request() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = create_basic_request("https://example.com");
assert_eq!(engine.support_score(&request), 100);
}
#[test]
fn test_support_score_basic_post_request() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = InternalScrapeRequest {
url: "https://example.com".to_string(),
method: HttpMethod::Post,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: Some("data".to_string()),
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
assert_eq!(engine.support_score(&request), 100);
}
#[test]
fn test_support_score_needs_js_returns_low() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = create_request_with_js("https://example.com");
assert_eq!(engine.support_score(&request), 10);
}
#[test]
fn test_support_score_needs_screenshot_returns_low() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = create_request_with_screenshot("https://example.com");
assert_eq!(engine.support_score(&request), 10);
}
#[test]
fn test_support_score_needs_js_and_screenshot_returns_low() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = InternalScrapeRequest {
url: "https://example.com".to_string(),
method: HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: true,
needs_screenshot: true,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
assert_eq!(engine.support_score(&request), 10);
}
#[test]
fn test_support_score_mobile_without_js() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = InternalScrapeRequest {
url: "https://example.com".to_string(),
method: HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: true,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
assert_eq!(engine.support_score(&request), 100);
}
#[test]
fn test_support_score_with_provider() {
let client = create_test_client();
let pool: Arc<dyn ProxyProvider> = Arc::new(ProxyPool::from_urls(
vec!["http://proxy:8080".to_string()],
Duration::from_secs(60),
Duration::from_secs(30),
));
let engine = ReqwestEngine::with_provider(client, pool);
let request = create_basic_request("https://example.com");
assert_eq!(engine.support_score(&request), 100);
}
#[test]
fn test_support_score_needs_tls_fingerprint_returns_low() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = InternalScrapeRequest {
needs_tls_fingerprint: true,
..create_basic_request("https://example.com")
};
assert_eq!(engine.support_score(&request), 10);
}
#[test]
fn test_support_score_no_tls_fingerprint_returns_full() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = create_basic_request("https://example.com");
assert_eq!(engine.support_score(&request), 100);
}
#[tokio::test]
async fn test_scrape_rejects_localhost() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = create_basic_request("http://localhost");
let result = engine.scrape(&request).await;
assert!(result.is_err());
match result.unwrap_err() {
EngineError::Other(msg) => {
assert!(msg.contains("SSRF protection"));
}
other => panic!("Expected Other with SSRF, got {:?}", other),
}
}
#[tokio::test]
async fn test_scrape_rejects_127_0_0_1() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = create_basic_request("http://127.0.0.1");
let result = engine.scrape(&request).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_scrape_rejects_private_ip_192_168() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = create_basic_request("http://192.168.1.1");
let result = engine.scrape(&request).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_scrape_rejects_private_ip_10_0() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = create_basic_request("http://10.0.0.1");
let result = engine.scrape(&request).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_scrape_rejects_private_ip_172_16() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = create_basic_request("http://172.16.0.1");
let result = engine.scrape(&request).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_scrape_rejects_file_scheme() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = create_basic_request("file:///etc/passwd");
let result = engine.scrape(&request).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_scrape_rejects_ftp_scheme() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = create_basic_request("ftp://example.com");
let result = engine.scrape(&request).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_scrape_rejects_metadata_endpoint() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = create_basic_request("http://169.254.169.254");
let result = engine.scrape(&request).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_scrape_rejects_0000() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = create_basic_request("http://0.0.0.0");
let result = engine.scrape(&request).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_scrape_post_request_rejects_ssrf() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = InternalScrapeRequest {
url: "http://localhost".to_string(),
method: HttpMethod::Post,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: Some("data".to_string()),
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
let result = engine.scrape(&request).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_scrape_mobile_request_rejects_ssrf() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = InternalScrapeRequest {
url: "http://localhost".to_string(),
method: HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: true,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
};
let result = engine.scrape(&request).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_scrape_invalid_url_format() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = create_basic_request("not-a-valid-url");
let result = engine.scrape(&request).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_scrape_empty_url() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let request = create_basic_request("");
let result = engine.scrape(&request).await;
assert!(result.is_err());
}
#[test]
fn test_http_method_default_is_get() {
assert_eq!(HttpMethod::default(), HttpMethod::Get);
}
use crate::test_utils::ensure_debug_logger;
#[test]
fn test_get_client_with_no_proxy_returns_injected_client_and_no_url() {
ensure_debug_logger();
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let handle = engine.get_client(&None, false, None);
assert!(
handle.used_proxy_url().is_none(),
"no proxy + no provider → used_proxy_url must be None"
);
assert!(!handle.has_proxy());
assert!(!handle.is_fallback(), "no proxy path must not be fallback");
}
#[test]
fn test_get_client_with_request_level_proxy_returns_proxy_url() {
ensure_debug_logger();
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let handle = engine.get_client(
&Some("http://proxy.example.com:8080".to_string()),
false,
None,
);
assert_eq!(
handle.used_proxy_url(),
Some("http://proxy.example.com:8080"),
"request-level proxy must be returned as used_proxy_url"
);
assert!(handle.has_proxy());
assert!(
!handle.is_fallback(),
"valid proxy path must not be fallback"
);
}
#[test]
fn test_get_client_with_invalid_request_proxy_still_returns_url() {
ensure_debug_logger();
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let handle = engine.get_client(&Some("://invalid".to_string()), false, None);
assert_eq!(handle.used_proxy_url(), Some("://invalid"));
assert!(
handle.is_fallback(),
"invalid proxy must set is_fallback=true (M4: 失败显性化)"
);
}
#[test]
fn test_get_client_with_empty_request_proxy_returns_no_url() {
ensure_debug_logger();
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let handle = engine.get_client(&Some("".to_string()), false, None);
assert!(
handle.used_proxy_url().is_none(),
"empty proxy string → used_proxy_url must be None"
);
assert!(
!handle.is_fallback(),
"empty proxy → no proxy path, not fallback"
);
}
#[test]
fn test_get_client_with_provider_returns_pool_url() {
ensure_debug_logger();
let client = create_test_client();
let pool: Arc<dyn ProxyProvider> = Arc::new(ProxyPool::from_urls(
vec!["http://pool-proxy:8080".to_string()],
Duration::from_secs(60),
Duration::from_secs(30),
));
let engine = ReqwestEngine::with_provider(client, pool);
let handle = engine.get_client(&None, false, None);
assert_eq!(
handle.used_proxy_url(),
Some("http://pool-proxy:8080"),
"provider must provide used_proxy_url for report_failure / report_success"
);
assert!(
!handle.is_fallback(),
"valid pool proxy must not be fallback"
);
}
#[test]
fn test_get_client_request_proxy_overrides_provider() {
ensure_debug_logger();
let client = create_test_client();
let pool: Arc<dyn ProxyProvider> = Arc::new(ProxyPool::from_urls(
vec!["http://pool-proxy:8080".to_string()],
Duration::from_secs(60),
Duration::from_secs(30),
));
let engine = ReqwestEngine::with_provider(client, pool);
let handle = engine.get_client(&Some("http://request-proxy:9090".to_string()), false, None);
assert_eq!(
handle.used_proxy_url(),
Some("http://request-proxy:9090"),
"request-level proxy must override provider"
);
assert!(
!handle.is_fallback(),
"valid request proxy must not be fallback"
);
}
#[test]
fn test_get_client_empty_pool_returns_injected_client() {
ensure_debug_logger();
let client = create_test_client();
let pool: Arc<dyn ProxyProvider> = Arc::new(ProxyPool::from_urls(
Vec::<String>::new(),
Duration::from_secs(60),
Duration::from_secs(30),
));
let engine = ReqwestEngine::with_provider(client, pool);
let handle = engine.get_client(&None, false, None);
assert!(
handle.used_proxy_url().is_none(),
"empty pool → used_proxy_url must be None"
);
assert!(
!handle.is_fallback(),
"empty pool is normal path, not build failure"
);
}
#[test]
fn test_get_client_pool_all_cooldown_returns_injected_client() {
ensure_debug_logger();
let client = create_test_client();
let pool = Arc::new(ProxyPool::from_urls(
vec!["http://pool-proxy:8080".to_string()],
Duration::from_secs(60),
Duration::from_secs(30),
));
pool.mark_failure("http://pool-proxy:8080");
let pool_dyn: Arc<dyn ProxyProvider> = pool;
let engine = ReqwestEngine::with_provider(client, pool_dyn);
let handle = engine.get_client(&None, false, None);
assert!(
handle.used_proxy_url().is_none(),
"all cooldown → fallback to injected client, used_proxy_url must be None"
);
assert!(
!handle.is_fallback(),
"all cooldown is normal path, not build failure"
);
}
#[test]
fn test_get_client_rr_cycles_through_pool() {
ensure_debug_logger();
let client = create_test_client();
let pool: Arc<dyn ProxyProvider> = Arc::new(ProxyPool::from_urls(
vec![
"http://a:8080".to_string(),
"http://b:8080".to_string(),
"http://c:8080".to_string(),
],
Duration::from_secs(60),
Duration::from_secs(30),
));
let engine = ReqwestEngine::with_provider(client, pool);
let urls: Vec<String> = (0..3)
.map(|_| {
engine
.get_client(&None, false, None)
.used_proxy_url()
.unwrap()
.to_string()
})
.collect();
assert!(urls.contains(&"http://a:8080".to_string()));
assert!(urls.contains(&"http://b:8080".to_string()));
assert!(urls.contains(&"http://c:8080".to_string()));
}
#[test]
fn test_get_client_sticky_strategy_uses_sticky_when_session_id_present() {
ensure_debug_logger();
let client = create_test_client();
let pool: Arc<dyn ProxyProvider> = Arc::new(ProxyPool::from_urls(
vec!["http://a:8080".to_string(), "http://b:8080".to_string()],
Duration::from_secs(60),
Duration::from_secs(30),
));
let engine = ReqwestEngine::with_provider_strategy_and_timeout(
client,
pool,
ProxyStrategy::Sticky,
30,
);
let url1 = engine
.get_client(&None, false, Some("sess-1"))
.used_proxy_url()
.unwrap()
.to_string();
let url2 = engine
.get_client(&None, false, Some("sess-1"))
.used_proxy_url()
.unwrap()
.to_string();
assert_eq!(
url1, url2,
"sticky strategy must return same url within session"
);
}
#[test]
fn test_get_client_sticky_strategy_falls_back_to_rr_when_session_id_missing() {
ensure_debug_logger();
let client = create_test_client();
let pool: Arc<dyn ProxyProvider> = Arc::new(ProxyPool::from_urls(
vec!["http://a:8080".to_string(), "http://b:8080".to_string()],
Duration::from_secs(60),
Duration::from_secs(30),
));
let engine = ReqwestEngine::with_provider_strategy_and_timeout(
client,
pool,
ProxyStrategy::Sticky,
30,
);
let handle = engine.get_client(&None, false, None);
assert!(
handle.used_proxy_url().is_some(),
"sticky without session_id should fall back to next and return a proxy url"
);
}
#[test]
fn test_get_client_sticky_strategy_falls_back_to_rr_when_session_id_empty() {
ensure_debug_logger();
let client = create_test_client();
let pool: Arc<dyn ProxyProvider> = Arc::new(ProxyPool::from_urls(
vec!["http://a:8080".to_string()],
Duration::from_secs(60),
Duration::from_secs(30),
));
let engine = ReqwestEngine::with_provider_strategy_and_timeout(
client,
pool,
ProxyStrategy::Sticky,
30,
);
let handle = engine.get_client(&None, false, Some(""));
assert!(
handle.used_proxy_url().is_some(),
"sticky with empty session_id should fall back to next"
);
}
#[test]
fn test_get_client_with_valid_https_proxy() {
ensure_debug_logger();
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let handle = engine.get_client(
&Some("https://proxy.example.com:8443".to_string()),
false,
None,
);
assert_eq!(
handle.used_proxy_url(),
Some("https://proxy.example.com:8443")
);
}
#[test]
fn test_get_client_with_skip_tls_verification_builds_custom_client() {
ensure_debug_logger();
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let handle = engine.get_client(&None, true, None);
assert!(
handle.used_proxy_url().is_none(),
"skip_tls + no proxy → used_proxy_url None"
);
}
#[test]
fn test_get_client_with_skip_tls_and_proxy_builds_custom_client() {
ensure_debug_logger();
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let handle = engine.get_client(&Some("http://proxy:8080".to_string()), true, None);
assert_eq!(handle.used_proxy_url(), Some("http://proxy:8080"));
}
#[test]
fn test_new_with_timeout_sets_timeout_seconds() {
let client = create_test_client();
let engine = ReqwestEngine::new_with_timeout(client, 60);
let _ = engine.get_client(&None, false, None);
assert_eq!(engine.timeout_seconds, 60);
}
#[test]
fn test_with_provider_strategy_and_timeout_sets_timeout_seconds() {
let client = create_test_client();
let pool: Arc<dyn ProxyProvider> = Arc::new(ProxyPool::from_urls(
vec!["http://proxy.example.com:8080".to_string()],
Duration::from_secs(60),
Duration::from_secs(30),
));
let engine = ReqwestEngine::with_provider_strategy_and_timeout(
client,
pool,
ProxyStrategy::RoundRobin,
45,
);
assert_eq!(engine.timeout_seconds, 45);
let _ = engine.get_client(&None, false, None);
}
#[test]
fn test_new_defaults_to_30_seconds() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
assert_eq!(engine.timeout_seconds, 30);
}
#[test]
fn test_with_provider_defaults_to_30_seconds() {
let client = create_test_client();
let pool: Arc<dyn ProxyProvider> = Arc::new(ProxyPool::from_urls(
vec!["http://proxy:8080".to_string()],
Duration::from_secs(60),
Duration::from_secs(30),
));
let engine = ReqwestEngine::with_provider(client, pool);
assert_eq!(engine.timeout_seconds, 30);
}
#[test]
fn test_get_client_with_injected_timeout_and_skip_tls_no_panic() {
ensure_debug_logger();
let client = create_test_client();
let engine = ReqwestEngine::new_with_timeout(client, 120);
let _ = engine.get_client(&Some("http://proxy:8080".to_string()), true, None);
}
#[test]
fn test_reqwest_engine_has_non_empty_ua_pool() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let pool = engine.ua_pool();
assert!(
pool.count(false) >= 20,
"desktop pool must have >= 20 profiles"
);
assert!(
pool.count(true) >= 20,
"mobile pool must have >= 20 profiles"
);
}
#[test]
fn test_reqwest_engine_pick_ua_returns_varied_profiles() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let pool = engine.ua_pool();
let mut uas = std::collections::HashSet::new();
for _ in 0..50 {
uas.insert(pool.pick(false).ua);
}
assert!(
uas.len() >= 2,
"multiple picks should return varied UAs, got only {} unique in 50 picks",
uas.len()
);
}
#[test]
fn test_reqwest_engine_pick_ua_mobile_vs_desktop() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let pool = engine.ua_pool();
for _ in 0..10 {
let p = pool.pick(true);
assert!(p.mobile, "pick(true) must return mobile profile");
}
for _ in 0..10 {
let p = pool.pick(false);
assert!(!p.mobile, "pick(false) must return desktop profile");
}
}
#[test]
fn test_reqwest_engine_pick_seeded_is_stable() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let pool = engine.ua_pool();
let seed = 7_u64;
let p1 = pool.pick_seeded(seed, false);
let p2 = pool.pick_seeded(seed, false);
let p3 = pool.pick_seeded(seed, true);
let p4 = pool.pick_seeded(seed, true);
assert_eq!(p1.ua, p2.ua, "desktop: same seed must return same UA");
assert_eq!(p3.ua, p4.ua, "mobile: same seed must return same UA");
assert_ne!(p1.ua, p3.ua, "desktop and mobile must differ for same seed");
}
#[test]
fn test_reqwest_engine_ua_profile_header_consistency() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let pool = engine.ua_pool();
for p in pool.desktop.iter().chain(pool.mobile.iter()) {
assert!(!p.ua.is_empty(), "UA must be non-empty");
assert!(
!p.accept_language.is_empty(),
"Accept-Language must be non-empty"
);
let is_wkwebview =
p.ua.contains("CriOS") || p.ua.contains("EdgiOS") || p.ua.contains("FxiOS");
let is_chromium = !is_wkwebview
&& (p.ua.contains("Chrome")
|| p.ua.contains("Edg")
|| p.ua.contains("SamsungBrowser"));
if is_chromium {
assert!(
!p.sec_ch_ua.is_empty(),
"Chromium UA must have non-empty sec_ch_ua: {}",
p.ua
);
} else {
assert!(
p.sec_ch_ua.is_empty(),
"Non-Chromium (incl. WKWebView) UA must have empty sec_ch_ua: {}",
p.ua
);
}
}
}
#[test]
fn test_reqwest_engine_ua_not_default_user_agent() {
let client = create_test_client();
let engine = ReqwestEngine::new(client);
let pool = engine.ua_pool();
let default_ua =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
let non_default_count = pool.desktop.iter().filter(|p| p.ua != default_ua).count();
assert!(
non_default_count >= 20,
"most desktop UAs should differ from DEFAULT_USER_AGENT, got {} non-default",
non_default_count
);
}
#[test]
fn test_reqwest_engine_with_provider_has_ua_pool() {
let client = create_test_client();
let pool: Arc<dyn ProxyProvider> = Arc::new(ProxyPool::from_urls(
vec!["http://proxy:8080".to_string()],
Duration::from_secs(60),
Duration::from_secs(30),
));
let engine = ReqwestEngine::with_provider(client, pool);
assert!(engine.ua_pool().count(false) >= 20);
assert!(engine.ua_pool().count(true) >= 20);
}
#[test]
fn test_reqwest_engine_with_timeout_has_ua_pool() {
let client = create_test_client();
let engine = ReqwestEngine::new_with_timeout(client, 60);
assert!(engine.ua_pool().count(false) >= 20);
assert!(engine.ua_pool().count(true) >= 20);
}
#[test]
fn test_reqwest_engine_with_proxy_pool_and_timeout_has_ua_pool() {
let client = create_test_client();
let pool: Arc<dyn ProxyProvider> = Arc::new(ProxyPool::from_urls(
vec!["http://proxy:8080".to_string()],
Duration::from_secs(60),
Duration::from_secs(30),
));
let engine = ReqwestEngine::with_provider_strategy_and_timeout(
client,
pool,
ProxyStrategy::RoundRobin,
45,
);
assert!(engine.ua_pool().count(false) >= 20);
assert!(engine.ua_pool().count(true) >= 20);
}
}