use super::{build_all_proxy, ReqwestEngine};
use crate::engines::client::handle::ClientHandle;
use crate::engines::engine_client::{EngineError, InternalScrapeRequest};
use crate::utils::proxy::redact_proxy_url;
use log::error;
use std::sync::Arc;
use std::time::Duration;
impl ReqwestEngine {
pub(super) fn build_request_client(
&self,
request: &InternalScrapeRequest,
handle: &mut ClientHandle,
host: &str,
resolve_addrs: &[std::net::SocketAddr],
) -> Result<(Arc<reqwest::cookie::Jar>, reqwest::Client), EngineError> {
let need_tls_bypass = request.skip_tls_verification;
let request_cookie_jar = Arc::new(reqwest::cookie::Jar::default());
let ssrf_client = if !host.is_empty() && !host.parse::<std::net::IpAddr>().is_ok() {
self.build_ssrf_resolve_client(
host,
resolve_addrs,
Arc::clone(&request_cookie_jar),
need_tls_bypass,
handle,
)?
} else {
let temp_client = self.build_temp_tls_bypass_client(
Arc::clone(&request_cookie_jar),
need_tls_bypass,
handle,
)?;
temp_client.unwrap_or_else(|| handle.client.clone())
};
Ok((request_cookie_jar, ssrf_client))
}
fn build_ssrf_resolve_client(
&self,
host: &str,
resolve_addrs: &[std::net::SocketAddr],
request_cookie_jar: Arc<reqwest::cookie::Jar>,
need_tls_bypass: bool,
handle: &mut ClientHandle,
) -> Result<reqwest::Client, EngineError> {
let mut b = reqwest::Client::builder()
.timeout(Duration::from_secs(self.timeout_seconds))
.connect_timeout(crate::utils::http_client::connect_timeout_for(
Duration::from_secs(self.timeout_seconds),
))
.cookie_provider(Arc::clone(&request_cookie_jar))
.local_address(Some(std::net::Ipv4Addr::UNSPECIFIED.into()))
.dns_resolver(crate::infrastructure::dns::create_ipv4_only_resolver())
.resolve_to_addrs(host, resolve_addrs)
.redirect(crate::utils::http_client::create_ssrf_safe_redirect_policy(
5,
));
if need_tls_bypass {
b = b.danger_accept_invalid_certs(true);
}
if let Some(proxy_url) = handle.used_proxy_url.take() {
match build_all_proxy(&proxy_url) {
Ok(px) => b = b.proxy(px),
Err(e) => {
error!(
"Failed to parse proxy URL for SSRF resolve client; \
request will NOT use the proxy: {}; proxy_url={}",
e,
redact_proxy_url(&proxy_url)
);
}
}
}
b.build()
.map_err(|e| EngineError::Other(format!("Failed to build SSRF resolve client: {}", e)))
}
fn build_temp_tls_bypass_client(
&self,
request_cookie_jar: Arc<reqwest::cookie::Jar>,
need_tls_bypass: bool,
handle: &mut ClientHandle,
) -> Result<Option<reqwest::Client>, EngineError> {
let temp_client: Option<reqwest::Client> = if need_tls_bypass {
let mut b = reqwest::Client::builder()
.timeout(Duration::from_secs(self.timeout_seconds))
.connect_timeout(crate::utils::http_client::connect_timeout_for(
Duration::from_secs(self.timeout_seconds),
))
.cookie_provider(Arc::clone(&request_cookie_jar))
.redirect(crate::utils::http_client::create_ssrf_safe_redirect_policy(
5,
))
.local_address(Some(std::net::Ipv4Addr::UNSPECIFIED.into()))
.danger_accept_invalid_certs(true);
if let Some(proxy_url) = handle.used_proxy_url.take() {
match build_all_proxy(&proxy_url) {
Ok(px) => b = b.proxy(px),
Err(e) => {
error!(
"Failed to parse proxy URL for temp TLS-bypass client; \
request will NOT use the proxy: {}; proxy_url={}",
e,
redact_proxy_url(&proxy_url)
);
}
}
}
Some(
b.build()
.map_err(|e| EngineError::Other(format!("Failed build temp client: {}", e)))?,
)
} else {
None
};
Ok(temp_client)
}
}