api_scanner/

http_client.rs

// src/http_client.rs
//
// Thin wrapper around `reqwest::Client` with WAF evasion, size capping,
// and convenience methods for the scanner modules.

use crate::{
    config::Config,
    error::{CapturedError, ScannerError, ScannerResult},
    waf::WafEvasion,
};
use dashmap::DashMap;
use reqwest::{
    header::{HeaderMap, HeaderName, HeaderValue, CONTENT_TYPE},
    Client, Method, Response,
};
use serde::{Deserialize, Serialize};
use std::{
    collections::HashMap,
    path::PathBuf,
    sync::{
        atomic::{AtomicBool, AtomicU64, Ordering},
        Arc,
    },
    time::Duration,
};
use tokio::sync::{Mutex, OwnedSemaphorePermit, Semaphore};
use tracing::{debug, info};
use url::Url;

// Optional auth credential support
pub use crate::auth::LiveCredential;

/// Parsed, size-capped HTTP response.
#[derive(Debug, Clone)]
#[allow(dead_code)]
pub struct HttpResponse {
    pub status: u16,
    pub headers: HashMap<String, String>,
    pub body: String,
    pub url: String,
}

/// Runtime transport counters captured during scan execution.
#[derive(Debug, Clone, Copy, Default, Serialize)]
pub struct HttpRuntimeMetrics {
    pub requests_sent: u64,
    pub retries_performed: u64,
}

#[allow(dead_code)]
impl HttpResponse {
    pub fn header(&self, key: &str) -> Option<&str> {
        self.headers.get(&key.to_lowercase()).map(|s| s.as_str())
    }

    pub fn is_success(&self) -> bool {
        self.status < 400
    }

    pub fn is_redirect(&self) -> bool {
        (300..400).contains(&self.status)
    }
}

/// Maximum response body size in bytes (512 KB).
const MAX_RESPONSE_BYTES: usize = 512 * 1024;

/// Default auth-like headers to strip for unauthenticated probes.
const DEFAULT_UNAUTH_STRIP_HEADERS: &[&str] = &[
    "authorization",
    "cookie",
    "x-api-key",
    "x-auth-token",
    "x-access-token",
    "x-authorization",
    "api-key",
    "x-session-token",
];

/// Thin wrapper around `reqwest::Client` with WAF evasion & size capping.
#[derive(Clone)]
pub struct HttpClient {
    inner: Client,
    no_redirect_inner: Client,
    unauth_inner: Client,
    client_config: ClientConfig,
    unauth_client_config: ClientConfig,
    per_host_clients: bool,
    clients: Arc<DashMap<String, Client>>,
    unauth_clients: Arc<DashMap<String, Client>>,
    spec_cache: Arc<DashMap<String, String>>,
    waf_enabled: bool,
    delay_ms: u64,
    retries: u32,
    host_last_request: Arc<DashMap<String, tokio::time::Instant>>,
    session_store: Option<Arc<Mutex<SessionStore>>>,
    session_path: Option<PathBuf>,
    adaptive: Option<Arc<AdaptiveLimiter>>,
    request_count: Arc<AtomicU64>,
    retry_count: Arc<AtomicU64>,
    /// Optional live credential for auth flow injection.
    live_credential: Option<Arc<LiveCredential>>,
    /// Header names to strip for unauthenticated probes.
    unauth_strip_headers: Vec<HeaderName>,
}

#[derive(Debug)]
struct AdaptiveLimiter {
    semaphore: Arc<Semaphore>,
    max: usize,
    min: usize,
    held: Arc<Mutex<Vec<OwnedSemaphorePermit>>>,
    decrease_scheduled: Arc<AtomicBool>,
    success_streak: std::sync::atomic::AtomicUsize,
}

impl AdaptiveLimiter {
    fn new(max: usize) -> Self {
        Self {
            semaphore: Arc::new(Semaphore::new(max)),
            max,
            min: 1,
            held: Arc::new(Mutex::new(Vec::new())),
            decrease_scheduled: Arc::new(AtomicBool::new(false)),
            success_streak: std::sync::atomic::AtomicUsize::new(0),
        }
    }

    async fn acquire(&self) -> Result<OwnedSemaphorePermit, &'static str> {
        self.semaphore
            .clone()
            .acquire_owned()
            .await
            .map_err(|_| "semaphore closed")
    }

    async fn on_success(&self) {
        let streak = self
            .success_streak
            .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
            + 1;
        if streak >= 10 {
            self.success_streak
                .store(0, std::sync::atomic::Ordering::Relaxed);
            self.increase().await;
        }
    }

    async fn on_backoff(&self) {
        self.success_streak
            .store(0, std::sync::atomic::Ordering::Relaxed);
        self.decrease().await;
    }

    async fn decrease(&self) {
        let mut held = self.held.lock().await;
        let current = self.max.saturating_sub(held.len());
        if current <= self.min {
            return;
        }
        if let Ok(permit) = self.semaphore.clone().try_acquire_owned() {
            held.push(permit);
            let new_limit = self.max.saturating_sub(held.len());
            info!(
                old_limit = current,
                new_limit, "Adaptive concurrency decreased"
            );
            return;
        }
        drop(held);

        // When the limiter is saturated, try_acquire fails even though we still
        // need to reduce future concurrency. Queue a single background reduction
        // that acquires and parks the next available permit.
        if self.decrease_scheduled.swap(true, Ordering::AcqRel) {
            return;
        }

        let semaphore = Arc::clone(&self.semaphore);
        let held = Arc::clone(&self.held);
        let decrease_scheduled = Arc::clone(&self.decrease_scheduled);
        let max = self.max;
        let min = self.min;
        tokio::spawn(async move {
            let permit = match semaphore.acquire_owned().await {
                Ok(p) => p,
                Err(_) => {
                    decrease_scheduled.store(false, Ordering::Release);
                    return;
                }
            };

            let mut held = held.lock().await;
            let current = max.saturating_sub(held.len());
            if current > min {
                held.push(permit);
                let new_limit = max.saturating_sub(held.len());
                info!(
                    old_limit = current,
                    new_limit, "Adaptive concurrency decreased"
                );
            } else {
                drop(permit);
            }
            decrease_scheduled.store(false, Ordering::Release);
        });
    }

    async fn increase(&self) {
        let mut held = self.held.lock().await;
        let current = self.max.saturating_sub(held.len());
        if let Some(permit) = held.pop() {
            drop(permit);
            let new_limit = self.max.saturating_sub(held.len());
            info!(
                old_limit = current,
                new_limit, "Adaptive concurrency increased"
            );
        }
    }
}

#[derive(Debug, Clone)]
struct ClientConfig {
    timeout_secs: u64,
    danger_accept_invalid_certs: bool,
    default_headers: HeaderMap,
    proxy: Option<String>,
}

#[derive(Debug, Default, Clone, Serialize, Deserialize)]
struct SessionFile {
    hosts: HashMap<String, HashMap<String, String>>,
}

type SessionStore = HashMap<String, HashMap<String, String>>;

impl HttpClient {
    pub fn new(config: &Config) -> ScannerResult<Self> {
        let mut default_headers = HeaderMap::new();
        for (k, v) in &config.default_headers {
            if let (Ok(name), Ok(value)) = (
                HeaderName::from_bytes(k.as_bytes()),
                HeaderValue::from_str(v),
            ) {
                default_headers.insert(name, value);
            }
        }

        if !config.cookies.is_empty() {
            let cookie_value = config
                .cookies
                .iter()
                .map(|(k, v)| format!("{k}={v}"))
                .collect::<Vec<_>>()
                .join("; ");

            let key = HeaderName::from_static("cookie");
            if let Some(existing) = default_headers.get(&key).cloned() {
                let mut combined = existing.to_str().unwrap_or("").to_string();
                if !combined.is_empty() {
                    combined.push_str("; ");
                }
                combined.push_str(&cookie_value);
                if let Ok(value) = HeaderValue::from_str(&combined) {
                    default_headers.insert(key, value);
                }
            } else if let Ok(value) = HeaderValue::from_str(&cookie_value) {
                default_headers.insert(key, value);
            }
        }

        let client_config = ClientConfig {
            timeout_secs: config.politeness.timeout_secs,
            danger_accept_invalid_certs: config.danger_accept_invalid_certs,
            default_headers,
            proxy: config.proxy.clone(),
        };

        let inner = build_client(&client_config)?;
        let no_redirect_inner = build_client_no_redirect(&client_config)?;
        let unauth_client_config = ClientConfig {
            default_headers: HeaderMap::new(),
            ..client_config.clone()
        };
        let unauth_inner = build_client(&unauth_client_config)?;
        let unauth_strip_headers = build_unauth_strip_headers(&config.unauth_strip_headers)?;

        let session_store = if let Some(path) = &config.session_file {
            if path.exists() {
                match load_session_file(path) {
                    Ok(store) => Some(Arc::new(Mutex::new(store))),
                    Err(e) => {
                        return Err(ScannerError::Config(format!(
                            "Failed to load session file: {e}"
                        )));
                    }
                }
            } else {
                Some(Arc::new(Mutex::new(HashMap::new())))
            }
        } else {
            None
        };

        Ok(Self {
            inner,
            no_redirect_inner,
            unauth_inner,
            client_config,
            unauth_client_config,
            per_host_clients: config.per_host_clients,
            clients: Arc::new(DashMap::new()),
            unauth_clients: Arc::new(DashMap::new()),
            spec_cache: Arc::new(DashMap::new()),
            waf_enabled: config.waf_evasion.enabled,
            delay_ms: config.politeness.delay_ms,
            retries: config.politeness.retries,
            host_last_request: Arc::new(DashMap::new()),
            session_store,
            session_path: config.session_file.clone(),
            adaptive: if config.adaptive_concurrency {
                Some(Arc::new(AdaptiveLimiter::new(config.concurrency.max(1))))
            } else {
                None
            },
            request_count: Arc::new(AtomicU64::new(0)),
            retry_count: Arc::new(AtomicU64::new(0)),
            live_credential: None,
            unauth_strip_headers,
        })
    }

    /// Attach a live credential to this client for auth flow injection.
    pub fn with_credential(mut self, cred: Arc<LiveCredential>) -> Self {
        self.live_credential = Some(cred);
        self
    }

    pub fn cache_spec(&self, url: &str, body: &str) {
        self.spec_cache.insert(url.to_string(), body.to_string());
    }

    pub fn get_cached_spec(&self, url: &str) -> Option<String> {
        self.spec_cache.get(url).map(|v| v.value().clone())
    }

    pub fn runtime_metrics(&self) -> HttpRuntimeMetrics {
        HttpRuntimeMetrics {
            requests_sent: self.request_count.load(Ordering::Relaxed),
            retries_performed: self.retry_count.load(Ordering::Relaxed),
        }
    }

    // ------------------------------------------------------------------ //
    //  Core request entry point
    // ------------------------------------------------------------------ //

    /// Send a request, applying WAF evasion delays + rotating headers.
    pub async fn request(
        &self,
        method: Method,
        url: &str,
        extra_headers: Option<HeaderMap>,
        body: Option<serde_json::Value>,
    ) -> Result<HttpResponse, CapturedError> {
        let _adaptive_permit = if let Some(adaptive) = &self.adaptive {
            match adaptive.acquire().await {
                Ok(permit) => Some(permit),
                Err(e) => {
                    debug!("[{method} {url}] adaptive limiter acquire failed: {e}");
                    None
                }
            }
        } else {
            None
        };

        self.enforce_host_delay(url).await;

        // Random inter-request delay based on configured delay_ms.
        if self.waf_enabled && self.delay_ms > 0 {
            let min_secs = self.delay_ms as f64 / 1000.0;
            let max_secs = min_secs * 3.0; // jitter up to 3x
            WafEvasion::random_delay(min_secs, max_secs).await;
        }

        let attempts = self.retries + 1;
        let mut last_err: Option<CapturedError> = None;

        for attempt in 0..attempts {
            if attempt > 0 {
                self.retry_count.fetch_add(1, Ordering::Relaxed);
                let backoff = retry_backoff(attempt);
                tokio::time::sleep(backoff).await;
            }

            self.request_count.fetch_add(1, Ordering::Relaxed);
            match self
                .send_once(
                    method.clone(),
                    url,
                    extra_headers.as_ref().cloned(),
                    body.as_ref().cloned(),
                )
                .await
            {
                Ok(resp) => {
                    if let Some(adaptive) = &self.adaptive {
                        if should_retry_status(resp.status) {
                            adaptive.on_backoff().await;
                        } else {
                            adaptive.on_success().await;
                        }
                    }
                    if should_retry_status(resp.status) && attempt + 1 < attempts {
                        debug!(
                            "[{method} {url}] retrying due to status {} (attempt {}/{})",
                            resp.status,
                            attempt + 1,
                            attempts
                        );
                        continue;
                    }
                    return Ok(resp);
                }
                Err(e) => {
                    debug!(
                        "[{method} {url}] attempt {}/{} failed: {}",
                        attempt + 1,
                        attempts,
                        e
                    );
                    if let Some(adaptive) = &self.adaptive {
                        adaptive.on_backoff().await;
                    }
                    last_err = Some(e);
                    if attempt + 1 == attempts {
                        break;
                    }
                }
            }
        }

        Err(last_err.unwrap_or_else(|| {
            CapturedError::from_str(
                "http::send",
                Some(url.to_string()),
                "request failed after retries",
            )
        }))
    }

    async fn send_once(
        &self,
        method: Method,
        url: &str,
        extra_headers: Option<HeaderMap>,
        body: Option<serde_json::Value>,
    ) -> Result<HttpResponse, CapturedError> {
        let client = self
            .client_for_url(url)
            .map_err(|e| CapturedError::from_str("http::client", Some(url.to_string()), e))?;
        let mut req = client.request(method.clone(), url);

        // Rotate UA + evasion headers on every request.
        if self.waf_enabled {
            req = req.headers(WafEvasion::evasion_headers());
        }

        let mut combined_headers = HeaderMap::new();

        if let Some(hdrs) = extra_headers {
            combined_headers.extend(hdrs);
        }

        if let Some(cookie) = self.cookie_header_for(url).await {
            let key = HeaderName::from_static("cookie");
            let merged = if let Some(existing) = combined_headers.get(&key) {
                let mut combined = existing.to_str().unwrap_or("").to_string();
                if !combined.is_empty() {
                    combined.push_str("; ");
                }
                combined.push_str(&cookie);
                combined
            } else {
                cookie
            };
            if let Ok(value) = HeaderValue::from_str(&merged) {
                combined_headers.insert(key, value);
            }
        }

        // Apply live credential from auth flow
        if let Some(ref cred) = self.live_credential {
            cred.apply_to(&mut combined_headers);
        }

        if !combined_headers.is_empty() {
            req = req.headers(combined_headers);
        }

        if let Some(json_body) = body {
            req = req
                .header(CONTENT_TYPE, HeaderValue::from_static("application/json"))
                .json(&json_body);
        }

        let response = req.send().await.map_err(|e| {
            debug!("[{method} {url}] send error: {e}");
            CapturedError::new("http::send", Some(url.to_string()), &e)
        })?;

        self.read_response(response, url).await
    }

    async fn send_once_no_redirect(
        &self,
        method: Method,
        url: &str,
        extra_headers: Option<HeaderMap>,
        body: Option<serde_json::Value>,
    ) -> Result<HttpResponse, CapturedError> {
        let client = self.no_redirect_inner.clone();
        let mut req = client.request(method.clone(), url);

        if self.waf_enabled {
            req = req.headers(WafEvasion::evasion_headers());
        }

        let mut combined_headers = HeaderMap::new();
        if let Some(hdrs) = extra_headers {
            combined_headers.extend(hdrs);
        }

        if let Some(cookie) = self.cookie_header_for(url).await {
            let key = HeaderName::from_static("cookie");
            let merged = if let Some(existing) = combined_headers.get(&key) {
                let mut combined = existing.to_str().unwrap_or("").to_string();
                if !combined.is_empty() {
                    combined.push_str("; ");
                }
                combined.push_str(&cookie);
                combined
            } else {
                cookie
            };
            if let Ok(value) = HeaderValue::from_str(&merged) {
                combined_headers.insert(key, value);
            }
        }

        if let Some(ref cred) = self.live_credential {
            cred.apply_to(&mut combined_headers);
        }

        if !combined_headers.is_empty() {
            req = req.headers(combined_headers);
        }

        if let Some(json_body) = body {
            req = req
                .header(CONTENT_TYPE, HeaderValue::from_static("application/json"))
                .json(&json_body);
        }

        let response = req.send().await.map_err(|e| {
            debug!("[{method} {url}] send error (no-redirect): {e}");
            CapturedError::new("http::send_no_redirect", Some(url.to_string()), &e)
        })?;
        self.read_response(response, url).await
    }

    async fn enforce_host_delay(&self, url: &str) {
        if self.delay_ms == 0 {
            return;
        }

        let parsed = match Url::parse(url) {
            Ok(u) => u,
            Err(_) => return,
        };

        let host = match parsed.host_str() {
            Some(h) => h,
            None => return,
        };

        let mut key = host.to_string();
        if let Some(port) = parsed.port() {
            key.push_str(&format!(":{port}"));
        }

        let min_gap = Duration::from_millis(self.delay_ms);
        let now = tokio::time::Instant::now();

        // Reserve the next allowed time per host to prevent concurrent TOCTOU races.
        // Keep lock scope to a single map entry update to avoid global
        // serialization across hosts under high concurrency.
        let next_allowed = {
            let mut entry = self.host_last_request.entry(key).or_insert(now);
            let candidate = *entry + min_gap;
            let next = if candidate > now { candidate } else { now };
            *entry = next;
            next
        };

        let sleep_for = if next_allowed > now {
            next_allowed - now
        } else {
            Duration::from_millis(0)
        };

        if !sleep_for.is_zero() {
            tokio::time::sleep(sleep_for).await;
        }
    }

    // ------------------------------------------------------------------ //
    //  Convenience wrappers
    // ------------------------------------------------------------------ //

    pub async fn get(&self, url: &str) -> Result<HttpResponse, CapturedError> {
        self.request(Method::GET, url, None, None).await
    }

    /// GET request for burst probes: bypasses host-delay and retry orchestration
    /// to exercise server-side rate limiting under near-simultaneous load.
    pub async fn get_burst(&self, url: &str) -> Result<HttpResponse, CapturedError> {
        self.request_count.fetch_add(1, Ordering::Relaxed);
        self.send_once(Method::GET, url, None, None).await
    }

    /// GET with extra request headers specified as `[(name, value)]` pairs.
    pub async fn get_with_headers(
        &self,
        url: &str,
        extra: &[(String, String)],
    ) -> Result<HttpResponse, CapturedError> {
        let mut map = HeaderMap::new();
        for (k, v) in extra {
            if let (Ok(name), Ok(value)) = (
                HeaderName::from_bytes(k.as_bytes()),
                HeaderValue::from_str(v),
            ) {
                map.insert(name, value);
            }
        }
        self.request(Method::GET, url, Some(map), None).await
    }

    /// Headered GET for burst probes: bypasses host-delay and retries.
    pub async fn get_with_headers_burst(
        &self,
        url: &str,
        extra: &[(String, String)],
    ) -> Result<HttpResponse, CapturedError> {
        let mut map = HeaderMap::new();
        for (k, v) in extra {
            if let (Ok(name), Ok(value)) = (
                HeaderName::from_bytes(k.as_bytes()),
                HeaderValue::from_str(v),
            ) {
                map.insert(name, value);
            }
        }
        self.request_count.fetch_add(1, Ordering::Relaxed);
        self.send_once(Method::GET, url, Some(map), None).await
    }

    /// GET with extra headers while forcing redirect policy to `none`.
    /// This keeps transport accounting/evasion behavior consistent with normal
    /// requests while allowing scanners to inspect 30x `Location` responses.
    pub async fn get_with_headers_no_redirect(
        &self,
        url: &str,
        extra: &[(String, String)],
    ) -> Result<HttpResponse, CapturedError> {
        let _adaptive_permit = if let Some(adaptive) = &self.adaptive {
            match adaptive.acquire().await {
                Ok(permit) => Some(permit),
                Err(e) => {
                    debug!("[GET {url}] adaptive limiter acquire failed: {e}");
                    None
                }
            }
        } else {
            None
        };

        self.enforce_host_delay(url).await;
        if self.waf_enabled && self.delay_ms > 0 {
            let min_secs = self.delay_ms as f64 / 1000.0;
            let max_secs = min_secs * 3.0;
            WafEvasion::random_delay(min_secs, max_secs).await;
        }

        let mut map = HeaderMap::new();
        for (k, v) in extra {
            if let (Ok(name), Ok(value)) = (
                HeaderName::from_bytes(k.as_bytes()),
                HeaderValue::from_str(v),
            ) {
                map.insert(name, value);
            }
        }

        let attempts = self.retries + 1;
        let mut last_err: Option<CapturedError> = None;
        for attempt in 0..attempts {
            if attempt > 0 {
                self.retry_count.fetch_add(1, Ordering::Relaxed);
                tokio::time::sleep(retry_backoff(attempt)).await;
            }

            self.request_count.fetch_add(1, Ordering::Relaxed);
            match self
                .send_once_no_redirect(Method::GET, url, Some(map.clone()), None)
                .await
            {
                Ok(resp) => {
                    if let Some(adaptive) = &self.adaptive {
                        if should_retry_status(resp.status) {
                            adaptive.on_backoff().await;
                        } else {
                            adaptive.on_success().await;
                        }
                    }
                    if should_retry_status(resp.status) && attempt + 1 < attempts {
                        continue;
                    }
                    return Ok(resp);
                }
                Err(e) => {
                    if let Some(adaptive) = &self.adaptive {
                        adaptive.on_backoff().await;
                    }
                    last_err = Some(e);
                    if attempt + 1 == attempts {
                        break;
                    }
                }
            }
        }

        Err(last_err.unwrap_or_else(|| {
            CapturedError::from_str(
                "http::send_no_redirect",
                Some(url.to_string()),
                "request failed after retries",
            )
        }))
    }

    #[allow(dead_code)]
    pub async fn head(&self, url: &str) -> Result<HttpResponse, CapturedError> {
        self.request(Method::HEAD, url, None, None).await
    }

    pub async fn options(
        &self,
        url: &str,
        extra: Option<HeaderMap>,
    ) -> Result<HttpResponse, CapturedError> {
        self.request(Method::OPTIONS, url, extra, None).await
    }

    pub async fn post_json(
        &self,
        url: &str,
        body: &serde_json::Value,
    ) -> Result<HttpResponse, CapturedError> {
        self.request(Method::POST, url, None, Some(body.clone()))
            .await
    }

    /// GET request without the live credential (used for unauthenticated comparison in IDOR checks).
    pub async fn get_without_auth(&self, url: &str) -> Result<HttpResponse, CapturedError> {
        let _adaptive_permit = if let Some(adaptive) = &self.adaptive {
            match adaptive.acquire().await {
                Ok(permit) => Some(permit),
                Err(e) => {
                    debug!("[GET {url}] adaptive limiter acquire failed: {e}");
                    None
                }
            }
        } else {
            None
        };

        self.enforce_host_delay(url).await;

        if self.waf_enabled && self.delay_ms > 0 {
            let min_secs = self.delay_ms as f64 / 1000.0;
            let max_secs = min_secs * 3.0;
            WafEvasion::random_delay(min_secs, max_secs).await;
        }

        let attempts = self.retries + 1;
        let mut last_err: Option<CapturedError> = None;
        for attempt in 0..attempts {
            if attempt > 0 {
                self.retry_count.fetch_add(1, Ordering::Relaxed);
                tokio::time::sleep(retry_backoff(attempt)).await;
            }

            self.request_count.fetch_add(1, Ordering::Relaxed);
            match self.send_once_without_auth(url).await {
                Ok(resp) => {
                    if let Some(adaptive) = &self.adaptive {
                        if should_retry_status(resp.status) {
                            adaptive.on_backoff().await;
                        } else {
                            adaptive.on_success().await;
                        }
                    }
                    if should_retry_status(resp.status) && attempt + 1 < attempts {
                        continue;
                    }
                    return Ok(resp);
                }
                Err(e) => {
                    if let Some(adaptive) = &self.adaptive {
                        adaptive.on_backoff().await;
                    }
                    last_err = Some(e);
                    if attempt + 1 == attempts {
                        break;
                    }
                }
            }
        }

        Err(last_err.unwrap_or_else(|| {
            CapturedError::from_str(
                "http::get_without_auth",
                Some(url.to_string()),
                "request failed after retries",
            )
        }))
    }

    async fn send_once_without_auth(&self, url: &str) -> Result<HttpResponse, CapturedError> {
        let client = self.client_for_url_unauth(url).map_err(|e| {
            CapturedError::from_str("http::get_without_auth", Some(url.to_string()), e)
        })?;

        let mut req = client.request(reqwest::Method::GET, url);

        if self.waf_enabled {
            req = req.headers(WafEvasion::evasion_headers());
        }

        let mut req = req
            .build()
            .map_err(|e| CapturedError::new("http::get_without_auth", Some(url.to_string()), &e))?;

        // Strip any default auth-like headers that could be set on the client.
        let headers = req.headers_mut();
        for name in &self.unauth_strip_headers {
            headers.remove(name);
        }

        let response = client.execute(req).await.map_err(|e| {
            debug!("[GET {url}] send error (no auth): {e}");
            CapturedError::new("http::get_without_auth", Some(url.to_string()), &e)
        })?;

        self.read_response(response, url).await
    }

    pub async fn method_probe(
        &self,
        method: &str,
        url: &str,
    ) -> Result<HttpResponse, CapturedError> {
        let m = Method::from_bytes(method.as_bytes()).map_err(|e| {
            CapturedError::from_str("http::method_probe", Some(url.to_string()), e.to_string())
        })?;
        self.request(m, url, None, None).await
    }

    // ------------------------------------------------------------------ //
    //  Response reading with size cap
    // ------------------------------------------------------------------ //

    async fn read_response(
        &self,
        response: Response,
        url: &str,
    ) -> Result<HttpResponse, CapturedError> {
        let status = response.status().as_u16();
        let final_url = response.url().to_string();

        let set_cookies: Vec<String> = response
            .headers()
            .get_all("set-cookie")
            .iter()
            .filter_map(|v| v.to_str().ok().map(|s| s.to_string()))
            .collect();

        // Flatten headers into lowercase map (last value wins for duplicates).
        let headers: HashMap<String, String> = response
            .headers()
            .iter()
            .map(|(k, v)| {
                (
                    k.as_str().to_lowercase(),
                    v.to_str().unwrap_or("").to_string(),
                )
            })
            .collect();

        if let Some(store) = &self.session_store {
            if let Err(e) = self
                .update_session_from_set_cookie(&set_cookies, &final_url, store)
                .await
            {
                debug!("[session] update error for {final_url}: {e}");
            }
        }

        // Read body with size cap.
        let raw_bytes = response
            .bytes()
            .await
            .map_err(|e| CapturedError::new("http::read_body", Some(url.to_string()), &e))?;

        let capped: &[u8] = if raw_bytes.len() > MAX_RESPONSE_BYTES {
            &raw_bytes[..MAX_RESPONSE_BYTES]
        } else {
            &raw_bytes
        };

        // Best-effort UTF-8 decode.
        let body = String::from_utf8_lossy(capped).into_owned();

        Ok(HttpResponse {
            status,
            headers,
            body,
            url: final_url,
        })
    }

    fn client_for_url(&self, url: &str) -> Result<Client, String> {
        if !self.per_host_clients {
            return Ok(self.inner.clone());
        }

        let host = parse_host_or_unknown(url);

        if let Some(client) = self.clients.get(&host) {
            return Ok(client.value().clone());
        }

        let client = build_client(&self.client_config)
            .map_err(|e| format!("per-host client build failed: {e}"))?;
        self.clients.insert(host, client.clone());
        Ok(client)
    }

    fn client_for_url_unauth(&self, url: &str) -> Result<Client, String> {
        if !self.per_host_clients {
            return Ok(self.unauth_inner.clone());
        }

        let host = parse_host_or_unknown(url);

        if let Some(client) = self.unauth_clients.get(&host) {
            return Ok(client.value().clone());
        }

        let client = build_client(&self.unauth_client_config)
            .map_err(|e| format!("per-host unauth client build failed: {e}"))?;
        self.unauth_clients.insert(host, client.clone());
        Ok(client)
    }

    async fn cookie_header_for(&self, url: &str) -> Option<String> {
        let store = self.session_store.as_ref()?;
        let host = Url::parse(url)
            .ok()
            .and_then(|u| u.host_str().map(|h| h.to_string()))?;
        let map = store.lock().await;
        let cookies = map.get(&host)?;
        if cookies.is_empty() {
            return None;
        }
        let value = cookies
            .iter()
            .map(|(k, v)| format!("{k}={v}"))
            .collect::<Vec<_>>()
            .join("; ");
        Some(value)
    }

    async fn update_session_from_set_cookie(
        &self,
        set_cookies: &[String],
        url: &str,
        store: &Arc<Mutex<SessionStore>>,
    ) -> Result<(), String> {
        let host = Url::parse(url)
            .ok()
            .and_then(|u| u.host_str().map(|h| h.to_string()))
            .ok_or_else(|| "invalid response url".to_string())?;

        if set_cookies.is_empty() {
            return Ok(());
        }

        let mut map = store.lock().await;
        let entry = map.entry(host).or_insert_with(HashMap::new);

        for raw in set_cookies {
            if let Some((name, value)) = parse_set_cookie_pair(raw) {
                entry.insert(name.to_string(), value.to_string());
            }
        }
        Ok(())
    }

    pub async fn save_session(&self) -> ScannerResult<()> {
        let Some(path) = &self.session_path else {
            return Ok(());
        };
        let Some(store) = &self.session_store else {
            return Ok(());
        };

        let map = store.lock().await;
        let doc = SessionFile { hosts: map.clone() };
        let json = serde_json::to_string_pretty(&doc)
            .map_err(|e| ScannerError::Config(format!("Session serialise failed: {e}")))?;
        std::fs::write(path, json)
            .map_err(|e| ScannerError::Config(format!("Session write failed: {e}")))?;
        Ok(())
    }
}

fn parse_host_or_unknown(url: &str) -> String {
    match Url::parse(url) {
        Ok(parsed) => parsed.host_str().map(|h| h.to_string()).unwrap_or_else(|| {
            debug!("Failed to extract host from URL: {url}");
            "unknown".to_string()
        }),
        Err(e) => {
            debug!("Failed to parse URL {url}: {e}");
            "unknown".to_string()
        }
    }
}

fn parse_set_cookie_pair(raw: &str) -> Option<(&str, &str)> {
    let first_part = raw.split(';').next()?.trim();
    let (name, value) = first_part.split_once('=')?;
    let name = name.trim();
    let value = value.trim();
    if name.is_empty() {
        return None;
    }
    Some((name, value))
}

fn build_unauth_strip_headers(raws: &[String]) -> ScannerResult<Vec<HeaderName>> {
    let mut out = Vec::new();
    let mut seen = std::collections::HashSet::new();

    let names = DEFAULT_UNAUTH_STRIP_HEADERS
        .iter()
        .copied()
        .chain(raws.iter().map(|s| s.as_str()));

    for name in names {
        let trimmed = name.trim();
        if trimmed.is_empty() {
            continue;
        }
        let key = trimmed.to_ascii_lowercase();
        if !seen.insert(key) {
            continue;
        }
        let header = HeaderName::from_bytes(trimmed.as_bytes()).map_err(|e| {
            ScannerError::Config(format!("Invalid unauth strip header '{trimmed}': {e}"))
        })?;
        out.push(header);
    }
    Ok(out)
}

fn should_retry_status(status: u16) -> bool {
    matches!(status, 429 | 500 | 502 | 503 | 504)
}

fn retry_backoff(attempt: u32) -> Duration {
    let shift = attempt.min(6);
    let exp = 1u64 << shift;
    Duration::from_millis(200 * exp)
}

fn build_client(cfg: &ClientConfig) -> ScannerResult<Client> {
    build_client_with_redirect(cfg, reqwest::redirect::Policy::limited(5))
}

fn build_client_no_redirect(cfg: &ClientConfig) -> ScannerResult<Client> {
    build_client_with_redirect(cfg, reqwest::redirect::Policy::none())
}

fn build_client_with_redirect(
    cfg: &ClientConfig,
    redirect: reqwest::redirect::Policy,
) -> ScannerResult<Client> {
    let mut builder = Client::builder()
        .timeout(Duration::from_secs(cfg.timeout_secs))
        .danger_accept_invalid_certs(cfg.danger_accept_invalid_certs)
        .gzip(true)
        .deflate(true)
        .redirect(redirect)
        .tcp_keepalive(Duration::from_secs(30));

    if !cfg.default_headers.is_empty() {
        builder = builder.default_headers(cfg.default_headers.clone());
    }

    if let Some(proxy_url) = &cfg.proxy {
        let proxy = reqwest::Proxy::all(proxy_url)
            .map_err(|e| ScannerError::Config(format!("Invalid proxy: {e}")))?;
        builder = builder.proxy(proxy);
    }

    builder
        .build()
        .map_err(|e| ScannerError::Config(format!("Client build failed: {e}")))
}

fn load_session_file(path: &PathBuf) -> Result<SessionStore, ScannerError> {
    let content = std::fs::read_to_string(path)
        .map_err(|e| ScannerError::Config(format!("Session read failed: {e}")))?;
    let doc: SessionFile = serde_json::from_str(&content)
        .map_err(|e| ScannerError::Config(format!("Session parse failed: {e}")))?;
    Ok(doc.hosts)
}