use std::sync::Arc;
use std::time::Duration;
use crate::config::{Config, TCP_CONNECT_TIMEOUT_SECS};
use crate::security::safe_resolver::SafeResolver;
use crate::security::ssrf_safe_redirect_policy;
use anyhow::Context;
use hickory_resolver::TokioResolver;
use reqwest::ClientBuilder;
#[derive(Debug, Clone, Copy)]
pub(crate) enum RedirectMode {
None,
SsrfSafe,
}
#[derive(Debug, Clone)]
pub(crate) struct SsrfClientOptions {
pub timeout: Duration,
pub connect_timeout: Duration,
pub user_agent: Option<String>,
pub redirect: RedirectMode,
pub pool_idle_timeout: Option<Duration>,
pub pool_max_idle_per_host: Option<usize>,
pub tcp_nodelay: bool,
}
impl SsrfClientOptions {
#[must_use]
pub(crate) fn download(timeout: Duration) -> Self {
Self {
timeout,
connect_timeout: Duration::from_secs(TCP_CONNECT_TIMEOUT_SECS),
user_agent: None,
redirect: RedirectMode::SsrfSafe,
pool_idle_timeout: None,
pool_max_idle_per_host: None,
tcp_nodelay: false,
}
}
#[must_use]
pub(crate) fn scan(config: &Config) -> Self {
Self {
timeout: Duration::from_secs(config.timeout_seconds),
connect_timeout: Duration::from_secs(TCP_CONNECT_TIMEOUT_SECS),
user_agent: Some(config.user_agent.clone()),
redirect: RedirectMode::None,
pool_idle_timeout: Some(Duration::from_secs(30)),
pool_max_idle_per_host: Some(10),
tcp_nodelay: true,
}
}
}
pub(crate) fn build_ssrf_client(
resolver: Arc<TokioResolver>,
opts: SsrfClientOptions,
) -> Result<reqwest::Client, reqwest::Error> {
let redirect = match opts.redirect {
RedirectMode::None => reqwest::redirect::Policy::none(),
RedirectMode::SsrfSafe => ssrf_safe_redirect_policy(),
};
let mut builder = ClientBuilder::new()
.use_rustls_tls()
.dns_resolver(Arc::new(SafeResolver::new(resolver)))
.redirect(redirect)
.timeout(opts.timeout)
.connect_timeout(opts.connect_timeout);
if let Some(ua) = opts.user_agent {
builder = builder.user_agent(ua);
}
if let Some(idle) = opts.pool_idle_timeout {
builder = builder.pool_idle_timeout(idle);
}
if let Some(max_idle) = opts.pool_max_idle_per_host {
builder = builder.pool_max_idle_per_host(max_idle);
}
if opts.tcp_nodelay {
builder = builder.tcp_nodelay(true);
}
builder.build()
}
pub(crate) fn build_download_client(timeout: Duration) -> Result<reqwest::Client, anyhow::Error> {
let resolver = crate::initialization::init_resolver()
.context("Failed to initialize DNS resolver for download")?;
build_ssrf_client(resolver, SsrfClientOptions::download(timeout))
.context("Failed to build SSRF-safe download client")
}
pub async fn init_client(
config: &Config,
resolver: Arc<TokioResolver>,
) -> Result<Arc<reqwest::Client>, reqwest::Error> {
let client = build_ssrf_client(resolver, SsrfClientOptions::scan(config))?;
Ok(Arc::new(client))
}
pub async fn init_redirect_client(
config: &Config,
resolver: Arc<TokioResolver>,
) -> Result<Arc<reqwest::Client>, reqwest::Error> {
init_client(config, resolver).await
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::Config;
use crate::config::FailOn;
use crate::initialization::test_resolver;
use std::path::PathBuf;
fn create_test_config() -> Config {
Config {
file: PathBuf::from("test.txt"),
user_agent: "test-agent/1.0".to_string(),
timeout_seconds: 10,
db_path: PathBuf::from("./test.db"),
max_concurrency: 30,
fail_on: FailOn::Never,
fail_on_pct_threshold: 10,
rate_limit_rps: 15,
log_level: crate::config::LogLevel::Info,
log_level_filter_override: None,
log_format: crate::config::LogFormat::Plain,
status_port: None,
fingerprints: None,
geoip: None,
enable_whois: false,
cache_dir: None,
scan_external_scripts: false,
log_file: None,
progress_callback: None,
dependency_overrides: None,
allow_localhost_for_tests: false,
drain_timeout_secs: 10,
allow_degraded_fingerprints: false,
}
}
#[tokio::test]
async fn test_init_client_success() {
let config = create_test_config();
let resolver = test_resolver();
let result = init_client(&config, resolver).await;
assert!(result.is_ok());
let client = result.unwrap();
assert_eq!(Arc::strong_count(&client), 1);
}
#[tokio::test]
async fn test_init_client_with_custom_timeout() {
let mut config = create_test_config();
config.timeout_seconds = 30;
let result = init_client(&config, test_resolver()).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_init_client_with_custom_user_agent() {
let mut config = create_test_config();
config.user_agent = "Custom-Agent/2.0".to_string();
let result = init_client(&config, test_resolver()).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_init_redirect_client_success() {
let config = create_test_config();
let result = init_redirect_client(&config, test_resolver()).await;
assert!(result.is_ok());
let client = result.unwrap();
assert_eq!(Arc::strong_count(&client), 1);
}
#[tokio::test]
async fn test_init_client_and_redirect_client_different_instances() {
let config = create_test_config();
let resolver = test_resolver();
let client1 = init_client(&config, Arc::clone(&resolver)).await.unwrap();
let client2 = init_redirect_client(&config, resolver).await.unwrap();
assert!(!Arc::ptr_eq(&client1, &client2));
}
#[tokio::test]
async fn test_init_client_empty_user_agent() {
let mut config = create_test_config();
config.user_agent = String::new();
let result = init_client(&config, test_resolver()).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_init_client_zero_timeout() {
let mut config = create_test_config();
config.timeout_seconds = 0;
let result = init_client(&config, test_resolver()).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_init_client_very_large_timeout() {
let mut config = create_test_config();
config.timeout_seconds = u64::MAX / 1000; let result = init_client(&config, test_resolver()).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_init_client_does_not_follow_redirects() {
use httptest::{matchers::*, responders::*, Expectation, Server};
let server = Server::run();
let redirect_url = format!("http://{}/redirect", server.addr());
let target_url = format!("http://{}/target", server.addr());
server.expect(
Expectation::matching(request::method_path("GET", "/redirect"))
.respond_with(status_code(302).insert_header("Location", target_url.clone())),
);
let config = create_test_config();
let client = init_client(&config, test_resolver())
.await
.expect("Should create client");
let response = client
.get(&redirect_url)
.send()
.await
.expect("Should send request");
assert_eq!(
response.status().as_u16(),
302,
"Main client should NOT follow redirects - got status {} instead of 302",
response.status().as_u16()
);
assert_eq!(
response.url().path(),
"/redirect",
"Main client should NOT follow redirects"
);
}
#[tokio::test]
async fn test_both_clients_have_redirects_disabled() {
use httptest::{matchers::*, responders::*, Expectation, Server};
let server = Server::run();
let redirect_url = format!("http://{}/redirect", server.addr());
let target_url = format!("http://{}/target", server.addr());
server.expect(
Expectation::matching(request::method_path("GET", "/redirect"))
.times(2..) .respond_with(status_code(302).insert_header("Location", target_url.clone())),
);
let config = create_test_config();
let resolver = test_resolver();
let main_client = init_client(&config, Arc::clone(&resolver))
.await
.expect("Should create main client");
let redirect_client = init_redirect_client(&config, resolver)
.await
.expect("Should create redirect client");
let main_response = main_client
.get(&redirect_url)
.send()
.await
.expect("Main client should send request");
assert_eq!(
main_response.status().as_u16(),
302,
"Main client should not follow redirects"
);
let redirect_response = redirect_client
.get(&redirect_url)
.send()
.await
.expect("Redirect client should send request");
assert_eq!(
redirect_response.status().as_u16(),
302,
"Redirect client should not follow redirects"
);
}
#[tokio::test]
async fn test_init_client_rejects_invalid_tls_certificate() {
crate::initialization::init_crypto_provider();
let addr = spawn_self_signed_tls_listener().await;
let client = init_client(&create_test_config(), test_resolver())
.await
.expect("Should create client");
let result = client.get(format!("https://{addr}/")).send().await;
assert!(
result.is_err(),
"Page-fetch client must reject invalid (self-signed) certificates; got Ok"
);
}
fn localhost_self_signed_der() -> (Vec<u8>, Vec<u8>) {
use rcgen::{CertificateParams, DistinguishedName, DnType, IsCa, KeyPair};
let mut params =
CertificateParams::new(vec!["127.0.0.1".to_string()]).expect("certificate params");
let mut distinguished_name = DistinguishedName::new();
distinguished_name.push(DnType::CommonName, "localhost");
params.distinguished_name = distinguished_name;
params.is_ca = IsCa::ExplicitNoCa;
let key_pair = KeyPair::generate().expect("key pair");
let cert = params.self_signed(&key_pair).expect("certificate");
(cert.der().to_vec(), key_pair.serialize_der())
}
async fn spawn_self_signed_tls_listener() -> std::net::SocketAddr {
use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer};
use tokio::net::TcpListener;
use tokio_rustls::TlsAcceptor;
let (cert_der, key_der) = localhost_self_signed_der();
let server_config = rustls::ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(
vec![CertificateDer::from(cert_der)],
PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(key_der)),
)
.expect("server TLS config");
let acceptor = TlsAcceptor::from(Arc::new(server_config));
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind local TLS listener");
let addr = listener.local_addr().expect("listener addr");
tokio::spawn(async move {
if let Ok((tcp, _)) = listener.accept().await {
let _ = acceptor.accept(tcp).await;
}
});
addr
}
}