use crate::config::settings::ProxyStrategy;
use crate::engines::engine_client::{
EngineError, InternalScrapeRequest, InternalScrapeResponse, ScraperEngine,
};
use crate::engines::provider::{ProxyCategory, ProxyProvider};
use crate::engines::validators;
use crate::utils::proxy::redact_proxy_url;
use crate::utils::ua_pool::{TlsEmulation, UaPool, UaProfile};
use async_trait::async_trait;
use log::error;
use std::sync::Arc;
use std::time::{Duration, Instant};
const DEFAULT_PROXY_STRATEGY: ProxyStrategy = ProxyStrategy::RoundRobin;
#[must_use]
pub fn emulation_provider(emulation: TlsEmulation) -> wreq::Emulation {
if !emulation.is_chromium_family() {
debug_assert!(
emulation.is_chromium_family(),
"non-Chromium TlsEmulation must never reach WreqEngine (UA pool filters them)"
);
log::warn!(
"{}",
crate::i18n::tr_log_args(
"wreq-non-chromium-emulation",
&[("emulation", format!("{emulation:?}"),)],
)
);
}
wreq::Emulation::builder()
.headers(chromium_default_headers())
.http2_options(chrome_http2_options())
.build(wreq::Group::default())
}
fn chromium_default_headers() -> wreq::header::HeaderMap {
use wreq::header::{HeaderMap, HeaderName, HeaderValue};
const DEFAULTS: &[(&str, &str)] = &[
(
"accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
),
("sec-fetch-dest", "document"),
("sec-fetch-mode", "navigate"),
("sec-fetch-site", "none"),
("sec-fetch-user", "?1"),
("upgrade-insecure-requests", "1"),
];
let mut headers = HeaderMap::new();
for (name, value) in DEFAULTS {
let name =
HeaderName::from_bytes(name.as_bytes()).expect("static header name must be valid");
let value = HeaderValue::from_str(value).expect("static header value must be valid");
headers.insert(name, value);
}
headers
}
fn chrome_http2_options() -> wreq::http2::Http2Options {
wreq::http2::Http2Options::builder()
.header_table_size(65_536)
.enable_push(false)
.initial_window_size(6_291_456)
.max_header_list_size(262_144)
.max_concurrent_streams(1_000)
.build()
}
async fn read_wreq_body_limited(
response: wreq::Response,
max_bytes: usize,
) -> Result<String, EngineError> {
use futures::StreamExt;
let charset = response
.headers()
.get(wreq::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.and_then(|ct| {
ct.split(';')
.map(|part| part.trim())
.find_map(|part| part.strip_prefix("charset="))
})
.map(|s| s.trim().trim_matches('"').to_string());
if let Some(len) = response.content_length() {
if len as usize > max_bytes {
return Err(EngineError::Other(format!(
"Response body size {} exceeds limit {}",
len, max_bytes
)));
}
}
let mut buffer: Vec<u8> = Vec::with_capacity(64 * 1024);
let mut stream = response.bytes_stream();
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(|e| EngineError::RequestFailed(e.to_string()))?;
if buffer.len() + chunk.len() > max_bytes {
return Err(EngineError::Other(format!(
"Response body exceeds limit {} (chunked transfer)",
max_bytes
)));
}
buffer.extend_from_slice(&chunk);
}
let decoded = match charset.as_deref() {
Some(label) => {
let encoding =
encoding_rs::Encoding::for_label(label.as_bytes()).unwrap_or(encoding_rs::UTF_8);
let (text, _, _) = encoding.decode(&buffer);
text.into_owned()
}
None => String::from_utf8_lossy(&buffer).into_owned(),
};
Ok(decoded)
}
pub struct WreqEngine {
#[allow(dead_code)]
client: wreq::Client,
ua_pool: Arc<UaPool>,
proxy_provider: Option<Arc<dyn ProxyProvider>>,
proxy_strategy: ProxyStrategy,
timeout_seconds: u64,
mrt: Duration,
max_response_body_bytes: usize,
}
const DEFAULT_WREQ_MAX_RESPONSE_BODY_BYTES: usize = 10 * 1024 * 1024;
fn create_ssrf_safe_wreq_redirect_policy(max_redirects: usize) -> wreq::redirect::Policy {
wreq::redirect::Policy::custom(move |attempt| {
if attempt.previous.len() >= max_redirects {
return attempt.error("too many redirects");
}
let uri = &*attempt.uri;
if !wreq_redirect_target_allowed(uri) {
return attempt.stop();
}
attempt.follow()
})
}
fn wreq_redirect_target_allowed(uri: &wreq::Uri) -> bool {
use crate::infrastructure::security::ssrf::is_internal_url;
if uri.scheme_str() != Some("http") && uri.scheme_str() != Some("https") {
log::warn!(
"SSRF protection: Blocking wreq redirect with non-HTTP scheme: {:?}",
uri.scheme_str()
);
return false;
}
let uri_str = uri.to_string();
if is_internal_url(&uri_str) {
log::warn!(
"SSRF protection: Blocking wreq redirect to internal URL: {}",
uri_str
);
return false;
}
wreq_redirect_dns_allowed(uri)
}
fn wreq_redirect_dns_allowed(uri: &wreq::Uri) -> bool {
use crate::engines::shared::is_private_ip;
use std::net::ToSocketAddrs;
let Some(host) = uri.host() else {
return true;
};
let port = uri
.port_u16()
.unwrap_or(if uri.scheme_str() == Some("https") {
443
} else {
80
});
match (host, port).to_socket_addrs() {
Ok(addrs) => {
let mut any = false;
for addr in addrs {
any = true;
if is_private_ip(addr.ip()) {
log::warn!(
"SSRF protection: wreq redirect host {} resolves to private IP {}, blocking",
host,
addr.ip()
);
return false;
}
}
any
}
Err(e) => {
log::warn!(
"SSRF protection: wreq redirect DNS failed for host {} ({}), blocking",
host,
e
);
false
}
}
}
impl WreqEngine {
pub fn new(
ua_pool: Arc<UaPool>,
mrt: Duration,
timeout_seconds: u64,
) -> Result<Self, EngineError> {
let client = Self::build_base_client(timeout_seconds)?;
Ok(Self {
client,
ua_pool,
proxy_provider: None,
proxy_strategy: DEFAULT_PROXY_STRATEGY,
timeout_seconds,
mrt,
max_response_body_bytes: DEFAULT_WREQ_MAX_RESPONSE_BODY_BYTES,
})
}
pub fn with_provider(
ua_pool: Arc<UaPool>,
proxy_provider: Arc<dyn ProxyProvider>,
proxy_strategy: ProxyStrategy,
mrt: Duration,
timeout_seconds: u64,
) -> Result<Self, EngineError> {
let client = Self::build_base_client(timeout_seconds)?;
Ok(Self {
client,
ua_pool,
proxy_provider: Some(proxy_provider),
proxy_strategy,
timeout_seconds,
mrt,
max_response_body_bytes: DEFAULT_WREQ_MAX_RESPONSE_BODY_BYTES,
})
}
pub fn with_max_response_body_bytes(mut self, bytes: usize) -> Self {
self.max_response_body_bytes = bytes;
self
}
fn build_base_client(timeout_seconds: u64) -> Result<wreq::Client, EngineError> {
wreq::Client::builder()
.emulation(emulation_provider(TlsEmulation::Chrome131))
.redirect(create_ssrf_safe_wreq_redirect_policy(5))
.timeout(Duration::from_secs(timeout_seconds))
.connect_timeout(crate::utils::http_client::connect_timeout_for(
Duration::from_secs(timeout_seconds),
))
.cookie_store(true)
.build()
.map_err(|e| EngineError::Internal(format!("wreq client build failed: {e}")))
}
#[cfg(test)]
fn build_proxy_client(proxy_url: &str, timeout_seconds: u64) -> Option<wreq::Client> {
let proxy = match wreq::Proxy::all(proxy_url) {
Ok(p) => p,
Err(e) => {
error!(
"Failed to configure wreq proxy (url={}): {}",
redact_proxy_url(proxy_url),
e
);
return None;
}
};
match wreq::Client::builder()
.redirect(create_ssrf_safe_wreq_redirect_policy(5))
.proxy(proxy)
.timeout(Duration::from_secs(timeout_seconds))
.connect_timeout(crate::utils::http_client::connect_timeout_for(
Duration::from_secs(timeout_seconds),
))
.cookie_store(true)
.build()
{
Ok(c) => {
log::debug!("Using wreq HTTP proxy: {}", redact_proxy_url(proxy_url));
Some(c)
}
Err(e) => {
error!(
"Failed to build wreq client (proxy={}): {}",
redact_proxy_url(proxy_url),
e
);
None
}
}
}
#[must_use]
pub fn ua_pool(&self) -> &UaPool {
&self.ua_pool
}
#[must_use]
pub fn has_proxy_provider(&self) -> bool {
self.proxy_provider.is_some()
}
#[must_use]
pub fn proxy_strategy(&self) -> ProxyStrategy {
self.proxy_strategy
}
#[must_use]
pub fn mrt(&self) -> Duration {
self.mrt
}
#[must_use]
pub fn timeout_seconds(&self) -> u64 {
self.timeout_seconds
}
#[cfg(test)]
fn get_client(
&self,
proxy: &Option<String>,
session_id: Option<&str>,
) -> (wreq::Client, Option<String>) {
let proxy_url = self.resolve_proxy_url(proxy, session_id);
let client = if let Some(ref url) = proxy_url {
Self::build_proxy_client(url, self.timeout_seconds)
.unwrap_or_else(|| self.client.clone())
} else {
self.client.clone()
};
(client, proxy_url)
}
fn resolve_proxy_url(
&self,
proxy: &Option<String>,
session_id: Option<&str>,
) -> Option<String> {
let request_proxy = proxy.as_ref().map(|s| s.trim()).filter(|s| !s.is_empty());
if let Some(url) = request_proxy {
return Some(url.to_string());
}
let provider = self.proxy_provider.as_ref()?;
let picked = match self.proxy_strategy {
ProxyStrategy::RoundRobin => provider.next(ProxyCategory::Html),
ProxyStrategy::Sticky => match session_id {
Some(sid) if !sid.is_empty() => provider.sticky(sid),
_ => {
log::warn!(
"proxy_strategy=Sticky but session_id is missing/empty; \
falling back to RoundRobin (next)"
);
provider.next(ProxyCategory::Html)
}
},
};
match picked {
Some(url) => Some(url),
None => {
log::debug!(
"wreq proxy_provider returned None (empty pool or all cooldown); \
no proxy for this request"
);
None
}
}
}
fn build_resolve_client(
host: &str,
resolve_addrs: &[std::net::SocketAddr],
proxy_url: Option<&str>,
skip_tls: bool,
timeout_seconds: u64,
emulation: TlsEmulation,
) -> Result<wreq::Client, EngineError> {
let mut builder = wreq::Client::builder()
.emulation(emulation_provider(emulation))
.redirect(create_ssrf_safe_wreq_redirect_policy(5))
.timeout(Duration::from_secs(timeout_seconds))
.connect_timeout(crate::utils::http_client::connect_timeout_for(
Duration::from_secs(timeout_seconds),
))
.cookie_store(true);
if !host.is_empty() && host.parse::<std::net::IpAddr>().is_err() {
builder = builder.resolve_to_addrs(host.to_string(), resolve_addrs.iter().copied());
}
if skip_tls {
log::warn!(
"wreq skip_tls_verification=true: TLS certificate validation disabled for this request"
);
builder = builder.tls_cert_verification(false);
}
if let Some(url) = proxy_url {
match wreq::Proxy::all(url) {
Ok(px) => {
builder = builder.proxy(px);
log::debug!("Using wreq proxy (all schemes): {}", redact_proxy_url(url));
}
Err(e) => {
error!(
"Failed to configure wreq proxy (url={}): {}",
redact_proxy_url(url),
e
);
}
}
}
builder
.build()
.map_err(|e| EngineError::Other(format!("Failed to build wreq resolve client: {e}")))
}
async fn validate_and_pin_ssrf(
request: &InternalScrapeRequest,
) -> Result<(validators::ValidatedUrl, Vec<std::net::SocketAddr>), EngineError> {
let validated_url = validators::SsrfValidator::new()
.validate(&request.url)
.await
.map_err(|e| EngineError::Other(format!("SSRF protection: {}", e)))?;
let resolve_addrs: Vec<std::net::SocketAddr> = validated_url
.resolved_ips
.iter()
.map(|ip| std::net::SocketAddr::new(*ip, validated_url.port))
.collect();
Ok((validated_url, resolve_addrs))
}
fn build_request_builder(
client: &wreq::Client,
request_url: &str,
request: &InternalScrapeRequest,
profile: &UaProfile,
) -> wreq::RequestBuilder {
let mut request_builder = match request.method {
crate::common::HttpMethod::Get => client.get(request_url),
crate::common::HttpMethod::Post => client.post(request_url),
};
request_builder = request_builder
.header("User-Agent", profile.ua)
.header("Accept-Language", profile.accept_language);
if !profile.sec_ch_ua.is_empty() {
request_builder = request_builder.header("sec-ch-ua", profile.sec_ch_ua);
}
request_builder = request_builder.headers(Self::to_wreq_header_map(&request.headers));
if let Some(body) = &request.body {
request_builder = request_builder.body(body.clone());
}
request_builder.timeout(request.timeout)
}
fn to_wreq_header_map(
headers: &std::collections::HashMap<String, String>,
) -> wreq::header::HeaderMap {
let mut map = wreq::header::HeaderMap::new();
for (k, v) in headers {
if let (Ok(name), Ok(value)) = (
wreq::header::HeaderName::from_bytes(k.as_bytes()),
wreq::header::HeaderValue::from_str(v),
) {
map.insert(name, value);
}
}
map
}
async fn send_and_handle_failure(
&self,
request_builder: wreq::RequestBuilder,
used_proxy_url: &Option<String>,
request: &InternalScrapeRequest,
) -> Result<wreq::Response, EngineError> {
match request_builder.send().await {
Ok(resp) => Ok(resp),
Err(e) => {
if let Some(url) = used_proxy_url {
if let Some(provider) = &self.proxy_provider {
provider.mark_failure(url);
}
}
if e.is_timeout() {
return Err(EngineError::Timeout(request.timeout));
}
error!("wreq send failed for {}: {e}", request.url);
Err(EngineError::RequestFailed(e.to_string()))
}
}
}
fn mark_proxy_success(&self, used_proxy_url: &Option<String>, status_code: u16) {
if (200..300).contains(&status_code) {
if let Some(url) = used_proxy_url {
if let Some(provider) = &self.proxy_provider {
provider.mark_success(url);
}
}
}
}
fn collect_response_headers(
response: &wreq::Response,
) -> std::collections::HashMap<String, String> {
let mut response_headers = std::collections::HashMap::with_capacity(32);
for (k, v) in response.headers() {
if let Ok(v_str) = v.to_str() {
response_headers.insert(k.as_str().to_string(), v_str.to_string());
}
}
response_headers
}
fn extract_content_type(response: &wreq::Response) -> String {
response
.headers()
.get("content-type")
.and_then(|v| v.to_str().ok())
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.unwrap_or_else(|| "text/html".to_string())
}
fn build_scrape_response(
status_code: u16,
content: String,
content_type: String,
headers: std::collections::HashMap<String, String>,
response_time_ms: u64,
final_url: String,
) -> InternalScrapeResponse {
InternalScrapeResponse {
js_degraded: false,
status_code,
content,
screenshot: None,
content_type,
headers,
response_time_ms,
final_url: Some(final_url),
captured_responses: Vec::new(),
}
}
}
#[async_trait]
impl ScraperEngine for WreqEngine {
async fn scrape(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
let (validated_url, resolve_addrs) = Self::validate_and_pin_ssrf(request).await?;
if resolve_addrs.is_empty() {
return Err(EngineError::Other("SSRF: no resolved IPs".to_string()));
}
let host = validated_url.parsed_url.host_str().unwrap_or("");
let profile = self.ua_pool.pick_chromium(request.mobile);
let used_proxy_url = self.resolve_proxy_url(&request.proxy, request.session_id.as_deref());
let effective_client = Self::build_resolve_client(
host,
&resolve_addrs,
used_proxy_url.as_deref(),
request.skip_tls_verification,
self.timeout_seconds,
profile.tls_emulation,
)?;
let request_builder = Self::build_request_builder(
&effective_client,
validated_url.parsed_url.as_str(),
request,
profile,
);
let start = Instant::now();
let response = self
.send_and_handle_failure(request_builder, &used_proxy_url, request)
.await?;
let status_code = response.status().as_u16();
self.mark_proxy_success(&used_proxy_url, status_code);
let content_type = Self::extract_content_type(&response);
let response_headers = Self::collect_response_headers(&response);
let final_url = {
let uri = response.uri().to_string();
if uri.starts_with("http://") || uri.starts_with("https://") {
uri
} else {
request.url.clone()
}
};
let content = read_wreq_body_limited(response, self.max_response_body_bytes).await?;
if request.sync_wait_ms > 0 {
tokio::time::sleep(Duration::from_millis(request.sync_wait_ms as u64)).await;
}
Ok(Self::build_scrape_response(
status_code,
content,
content_type,
response_headers,
start.elapsed().as_millis() as u64,
final_url,
))
}
fn support_score(&self, request: &InternalScrapeRequest) -> u8 {
if request.needs_js || request.needs_screenshot {
return 10;
}
if request.needs_tls_fingerprint {
return 100;
}
10
}
fn name(&self) -> &'static str {
"wreq"
}
fn supports_tls_fingerprint(&self) -> bool {
true
}
fn max_response_time(&self) -> Duration {
self.mrt
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::engines::engine_client::{
HttpMethod, InternalScrapeRequest, InternalScreenshotConfig,
};
use crate::engines::proxy_pool::ProxyPool;
use std::collections::HashMap;
use std::time::Duration;
fn create_engine() -> WreqEngine {
WreqEngine::new(Arc::new(UaPool::new()), Duration::from_secs(15), 30)
.expect("wreq client build should succeed")
}
fn create_request(url: &str) -> InternalScrapeRequest {
InternalScrapeRequest {
url: url.to_string(),
method: HttpMethod::Get,
headers: HashMap::new(),
timeout: Duration::from_secs(30),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
body: None,
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
wait_for: None,
needs_mllm: false,
}
}
fn create_proxy_pool() -> Arc<dyn ProxyProvider> {
Arc::new(ProxyPool::from_urls(
vec![
"http://proxy-a:8080".to_string(),
"http://proxy-b:8080".to_string(),
],
Duration::from_secs(60),
Duration::from_secs(30),
))
}
#[test]
fn test_wreq_engine_new() {
let engine = create_engine();
assert_eq!(engine.name(), "wreq");
assert!(!engine.has_proxy_provider());
assert!(engine.supports_tls_fingerprint());
}
#[test]
fn test_wreq_engine_with_provider() {
let engine = WreqEngine::with_provider(
Arc::new(UaPool::new()),
create_proxy_pool(),
ProxyStrategy::RoundRobin,
Duration::from_secs(15),
30,
)
.expect("wreq client build should succeed");
assert_eq!(engine.name(), "wreq");
assert!(engine.has_proxy_provider());
assert_eq!(engine.proxy_strategy(), ProxyStrategy::RoundRobin);
}
#[test]
fn test_wreq_engine_mrt_and_timeout_injected() {
let engine = WreqEngine::new(Arc::new(UaPool::new()), Duration::from_secs(15), 45)
.expect("wreq client build should succeed");
assert_eq!(engine.mrt(), Duration::from_secs(15));
assert_eq!(engine.timeout_seconds(), 45);
}
#[test]
fn test_name_returns_wreq() {
assert_eq!(create_engine().name(), "wreq");
}
#[test]
fn test_supports_tls_fingerprint_true() {
assert!(create_engine().supports_tls_fingerprint());
}
#[test]
fn test_ua_pool_getter() {
let pool = Arc::new(UaPool::new());
let engine = WreqEngine::new(pool.clone(), Duration::from_secs(15), 30)
.expect("wreq client build should succeed");
assert_eq!(engine.ua_pool().count(false), pool.count(false));
}
#[test]
fn test_support_score_tls_fingerprint_returns_100() {
let engine = create_engine();
let mut request = create_request("https://example.com");
request.needs_tls_fingerprint = true;
assert_eq!(engine.support_score(&request), 100);
}
#[test]
fn test_support_score_basic_returns_10() {
let engine = create_engine();
assert_eq!(
engine.support_score(&create_request("https://example.com")),
10
);
}
#[test]
fn test_support_score_needs_js_returns_10() {
let engine = create_engine();
let mut request = create_request("https://example.com");
request.needs_js = true;
request.needs_tls_fingerprint = true;
assert_eq!(engine.support_score(&request), 10);
}
#[test]
fn test_emulation_provider_sets_browser_defaults() {
let emulation = emulation_provider(TlsEmulation::Chrome131);
assert!(
!emulation.headers.is_empty(),
"emulation must carry browser default headers (accept/sec-fetch-*), got empty"
);
assert!(
emulation.headers.contains_key("accept"),
"accept header is the most basic browser fingerprint header"
);
assert!(
emulation.http2_options.is_some(),
"emulation must set explicit H2 SETTINGS (Chrome fingerprint)"
);
}
#[test]
fn test_emulation_provider_accepts_all_chromium_variants() {
for variant in [
TlsEmulation::Chrome131,
TlsEmulation::Chrome130,
TlsEmulation::Chrome120,
TlsEmulation::Edge131,
] {
let emulation = emulation_provider(variant);
assert!(
!emulation.headers.is_empty(),
"{variant:?} must get headers"
);
}
}
#[test]
fn test_support_score_needs_screenshot_returns_10() {
let engine = create_engine();
let mut request = create_request("https://example.com");
request.needs_screenshot = true;
request.screenshot_config = Some(InternalScreenshotConfig {
full_page: false,
selector: None,
quality: None,
format: None,
});
assert_eq!(engine.support_score(&request), 10);
}
#[test]
fn test_get_client_no_provider_returns_base_client() {
let engine = create_engine();
let (client, used_proxy) = engine.get_client(&None, None);
assert!(used_proxy.is_none());
let _ = client;
}
#[test]
fn test_get_client_round_robin_provides_proxy() {
let engine = WreqEngine::with_provider(
Arc::new(UaPool::new()),
create_proxy_pool(),
ProxyStrategy::RoundRobin,
Duration::from_secs(15),
30,
)
.expect("wreq client build should succeed");
let (_, used_proxy) = engine.get_client(&None, None);
assert!(used_proxy.is_some(), "RoundRobin 应从池中拿到代理");
}
#[test]
fn test_get_client_sticky_uses_session() {
let engine = WreqEngine::with_provider(
Arc::new(UaPool::new()),
create_proxy_pool(),
ProxyStrategy::Sticky,
Duration::from_secs(15),
30,
)
.expect("wreq client build should succeed");
let (_, first) = engine.get_client(&None, Some("session-1"));
let (_, second) = engine.get_client(&None, Some("session-1"));
assert!(first.is_some() && second.is_some());
assert_eq!(first, second);
}
#[test]
fn test_get_client_sticky_without_session_falls_back_round_robin() {
let engine = WreqEngine::with_provider(
Arc::new(UaPool::new()),
create_proxy_pool(),
ProxyStrategy::Sticky,
Duration::from_secs(15),
30,
)
.expect("wreq client build should succeed");
let (_, used_proxy) = engine.get_client(&None, None);
assert!(used_proxy.is_some());
}
#[test]
fn test_get_client_request_proxy_overrides_pool() {
let engine = WreqEngine::with_provider(
Arc::new(UaPool::new()),
create_proxy_pool(),
ProxyStrategy::RoundRobin,
Duration::from_secs(15),
30,
)
.expect("wreq client build should succeed");
let (_, used_proxy) = engine.get_client(&Some("http://override:9999".to_string()), None);
assert_eq!(used_proxy.as_deref(), Some("http://override:9999"));
}
#[test]
fn test_emulation_provider_roundtrips_all_variants() {
for variant in [
TlsEmulation::Chrome131,
TlsEmulation::Chrome130,
TlsEmulation::Chrome120,
TlsEmulation::Edge131,
] {
let _provider = emulation_provider(variant);
}
}
#[tokio::test]
async fn test_scrape_rejects_private_ip() {
let engine = create_engine();
let result = engine.scrape(&create_request("http://192.168.1.1/")).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_scrape_rejects_localhost() {
let engine = create_engine();
let result = engine.scrape(&create_request("http://localhost/")).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_scrape_rejects_metadata_endpoint() {
let engine = create_engine();
let result = engine
.scrape(&create_request("http://169.254.169.254/latest/meta-data/"))
.await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_scrape_rejects_ftp_scheme() {
let engine = create_engine();
let result = engine
.scrape(&create_request("ftp://example.com/file"))
.await;
assert!(result.is_err());
}
#[tokio::test]
#[ignore] async fn test_ja4_fingerprint_matches_chrome() {
let engine = create_engine();
let request = create_request("https://ja4er.com/");
let response = engine
.scrape(&request)
.await
.expect("ja4er.com request should succeed");
assert!(
(200..300).contains(&response.status_code),
"expected 2xx, got {}",
response.status_code
);
let body = &response.content;
assert!(
body.contains("ja4"),
"response should contain ja4 field, got: {}",
&body[..body.len().min(500)]
);
let has_chrome_prefix = body.contains("t13d") || body.contains("t13");
assert!(
has_chrome_prefix,
"JA4 fingerprint should contain Chrome TLS 1.3 prefix (t13/t13d), got: {}",
&body[..body.len().min(500)]
);
}
#[tokio::test]
#[ignore] async fn test_tls_peet_ws_returns_fingerprint_data() {
let engine = create_engine();
let request = create_request("https://tls.peet.ws/");
let response = engine
.scrape(&request)
.await
.expect("tls.peet.ws request should succeed");
assert!(
(200..300).contains(&response.status_code),
"expected 2xx, got {}",
response.status_code
);
let body = &response.content;
let has_fingerprint = body.contains("ja4") || body.contains("ja3") || body.contains("tls");
assert!(
has_fingerprint,
"response should contain TLS fingerprint data, got: {}",
&body[..body.len().min(500)]
);
}
}