webpage_quality_analyzer 1.0.2

//! Web fetching module for URL-based analysis
//!
//! This module provides HTTP client functionality to fetch web pages
//! for quality analysis, supporting various configuration options including
//! caching, retry logic, and connection pooling.

use crate::models::models::{AnalyzeError, Result};

#[cfg(feature = "async")]
use once_cell::sync::Lazy;
#[cfg(feature = "async")]
use reqwest::Client;
#[cfg(feature = "async")]
use std::collections::HashMap;
#[cfg(feature = "async")]
use std::sync::{Arc, Mutex};
#[cfg(feature = "async")]
use std::time::Duration;
#[cfg(feature = "async")]
use tokio::time::sleep;

/// Shared HTTP client instance for better resource management
#[cfg(feature = "async")]
static HTTP_CLIENT: Lazy<Client> = Lazy::new(|| {
    Client::builder()
        .timeout(Duration::from_secs(30)) // Default timeout
        .pool_max_idle_per_host(10) // Connection pooling
        .pool_idle_timeout(Duration::from_secs(90))
        .build()
        .unwrap_or_else(|_| {
            // Fallback to default client if builder fails
            Client::new()
        })
});

/// Cache entry for fetched content
#[cfg(feature = "async")]
#[derive(Debug, Clone)]
struct CacheEntry {
    content: String,
    headers: reqwest::header::HeaderMap,
    size: usize,
    timestamp: std::time::Instant,
}

/// Shared cache for HTTP responses
#[cfg(feature = "async")]
static RESPONSE_CACHE: Lazy<Arc<Mutex<HashMap<String, CacheEntry>>>> =
    Lazy::new(|| Arc::new(Mutex::new(HashMap::new())));

/// Retry configuration for failed requests
#[derive(Debug, Clone, Copy)]
pub struct RetryConfig {
    /// Maximum number of retry attempts
    pub max_retries: u32,
    /// Initial delay between retries (exponential backoff)
    pub initial_delay_ms: u64,
    /// Maximum delay between retries
    pub max_delay_ms: u64,
}

impl Default for RetryConfig {
    fn default() -> Self {
        Self {
            max_retries: 3,
            initial_delay_ms: 100,
            max_delay_ms: 5000,
        }
    }
}

/// Configuration options for web fetching
#[derive(Debug, Clone)]
pub struct FetchOptions {
    /// Timeout in seconds for HTTP requests
    #[cfg(feature = "async")]
    pub timeout_seconds: u64,
    /// User agent string to use for requests
    #[cfg(feature = "async")]
    pub user_agent: String,
    /// Whether to follow HTTP redirects
    #[cfg(feature = "async")]
    pub follow_redirects: bool,
    /// Maximum number of redirects to follow
    #[cfg(feature = "async")]
    pub max_redirects: usize,
    /// Future feature: JavaScript rendering support
    #[cfg(feature = "async")]
    pub javascript_rendering: bool,
    /// Cache TTL in seconds (0 = no caching)
    #[cfg(feature = "async")]
    pub cache_ttl_seconds: u64,
    /// Retry configuration
    #[cfg(feature = "async")]
    pub retry_config: RetryConfig,
    /// Placeholder for non-async builds
    #[cfg(not(feature = "async"))]
    _phantom: (),
}

impl Default for FetchOptions {
    fn default() -> Self {
        Self {
            #[cfg(feature = "async")]
            timeout_seconds: 10,
            #[cfg(feature = "async")]
            user_agent: "Mozilla/5.0 (compatible; PageQualityAnalyzer/1.0; +https://github.com/NotGyashu/webpage-quality-analyser)".to_string(),
            #[cfg(feature = "async")]
            follow_redirects: true,
            #[cfg(feature = "async")]
            max_redirects: 5,
            #[cfg(feature = "async")]
            javascript_rendering: false,
            #[cfg(feature = "async")]
            cache_ttl_seconds: 300, // 5 minutes default cache
            #[cfg(feature = "async")]
            retry_config: RetryConfig::default(),
            #[cfg(not(feature = "async"))]
            _phantom: (),
        }
    }
}

/// Web fetcher for retrieving HTML content from URLs
#[derive(Debug)]
pub struct WebFetcher {
    #[cfg(feature = "async")]
    options: FetchOptions,
}

impl WebFetcher {
    /// Create a new web fetcher with the given options
    pub fn new(options: FetchOptions) -> Result<Self> {
        #[cfg(feature = "async")]
        {
            Ok(Self { options })
        }
        #[cfg(not(feature = "async"))]
        {
            Err(crate::models::models::AnalyzeError::InternalError(
                "Async functionality not available. Enable the 'async' feature to use URL fetching.".to_string()
            ))
        }
    }

    /// Get a configured request builder using the shared client
    #[cfg(feature = "async")]
    fn request_builder(&self, url: &str) -> reqwest::RequestBuilder {
        HTTP_CLIENT
            .get(url)
            .timeout(Duration::from_secs(self.options.timeout_seconds))
            .header("User-Agent", &self.options.user_agent)
    }

    /// Fetch HTML content from a URL
    #[cfg(feature = "async")]
    pub async fn fetch_html(&self, url: &str) -> Result<String> {
        let response = self
            .request_builder(url)
            .send()
            .await
            .map_err(|e| AnalyzeError::InternalError(e.to_string()))?;

        if !response.status().is_success() {
            return Err(AnalyzeError::InternalError(format!(
                "HTTP {}: Failed to fetch {}",
                response.status(),
                url
            )));
        }

        let html = response
            .text()
            .await
            .map_err(|e| AnalyzeError::InternalError(e.to_string()))?;

        Ok(html)
    }

    /// Fetch HTML content and HTTP headers from a URL
    #[cfg(feature = "async")]
    pub async fn fetch_with_headers(
        &self,
        url: &str,
    ) -> Result<(String, reqwest::header::HeaderMap)> {
        let response = self
            .request_builder(url)
            .send()
            .await
            .map_err(|e| AnalyzeError::InternalError(e.to_string()))?;

        if !response.status().is_success() {
            return Err(AnalyzeError::InternalError(format!(
                "HTTP {}: Failed to fetch {}",
                response.status(),
                url
            )));
        }

        let headers = response.headers().clone();
        let html = response
            .text()
            .await
            .map_err(|e| AnalyzeError::InternalError(e.to_string()))?;

        Ok((html, headers))
    }

    /// Fetch HTML content, headers, and response size from a URL
    #[cfg(feature = "async")]
    pub async fn fetch_with_metrics(
        &self,
        url: &str,
    ) -> Result<(String, reqwest::header::HeaderMap, usize)> {
        let response = self
            .request_builder(url)
            .send()
            .await
            .map_err(|e| AnalyzeError::InternalError(e.to_string()))?;

        if !response.status().is_success() {
            return Err(AnalyzeError::InternalError(format!(
                "HTTP {}: Failed to fetch {}",
                response.status(),
                url
            )));
        }

        let headers = response.headers().clone();
        let bytes = response
            .bytes()
            .await
            .map_err(|e| AnalyzeError::InternalError(e.to_string()))?;

        let response_size = bytes.len();
        let html = String::from_utf8(bytes.to_vec())
            .map_err(|e| AnalyzeError::InternalError(format!("Invalid UTF-8: {}", e)))?;

        Ok((html, headers, response_size))
    }

    /// Get the fetch options used by this fetcher
    #[cfg(feature = "async")]
    pub fn options(&self) -> &FetchOptions {
        &self.options
    }

    /// Get cached response if available and not expired
    #[cfg(feature = "async")]
    fn get_cached(&self, url: &str) -> Option<CacheEntry> {
        if let Ok(cache) = RESPONSE_CACHE.lock() {
            if let Some(entry) = cache.get(url) {
                let age = entry.timestamp.elapsed().as_secs();
                if age < self.options.cache_ttl_seconds {
                    return Some(entry.clone());
                }
            }
        }
        None
    }

    /// Store response in cache
    #[cfg(feature = "async")]
    fn cache_response(
        &self,
        url: &str,
        content: String,
        headers: reqwest::header::HeaderMap,
        size: usize,
    ) {
        if self.options.cache_ttl_seconds == 0 {
            return;
        }

        if let Ok(mut cache) = RESPONSE_CACHE.lock() {
            cache.insert(
                url.to_string(),
                CacheEntry {
                    content,
                    headers,
                    size,
                    timestamp: std::time::Instant::now(),
                },
            );

            // Simple cache cleanup: remove entries older than TTL
            let ttl = Duration::from_secs(self.options.cache_ttl_seconds);
            cache.retain(|_, entry| entry.timestamp.elapsed() < ttl);
        }
    }

    /// Fetch with retry logic and exponential backoff
    #[cfg(feature = "async")]
    async fn fetch_with_retry(&self, url: &str) -> Result<reqwest::Response> {
        let mut last_error: Option<AnalyzeError> = None;
        let mut delay = self.options.retry_config.initial_delay_ms;

        for attempt in 0..=self.options.retry_config.max_retries {
            match self.request_builder(url).send().await {
                Ok(response) => {
                    if response.status().is_success() {
                        return Ok(response);
                    } else if response.status().is_server_error()
                        && attempt < self.options.retry_config.max_retries
                    {
                        // Retry on server errors (5xx)
                        last_error = Some(AnalyzeError::InternalError(format!(
                            "HTTP {}: Server error (attempt {})",
                            response.status(),
                            attempt + 1
                        )));
                    } else {
                        // Don't retry client errors (4xx)
                        return Err(AnalyzeError::InternalError(format!(
                            "HTTP {}: Failed to fetch {}",
                            response.status(),
                            url
                        )));
                    }
                }
                Err(e) => {
                    last_error = Some(AnalyzeError::InternalError(e.to_string()));
                    if attempt < self.options.retry_config.max_retries {
                        // Only sleep if we're going to retry
                        sleep(Duration::from_millis(delay)).await;
                        delay = (delay * 2).min(self.options.retry_config.max_delay_ms);
                    }
                }
            }
        }

        Err(last_error.unwrap_or_else(|| AnalyzeError::InternalError("Unknown error".to_string())))
    }

    /// Fetch HTML content with caching and retry logic
    #[cfg(feature = "async")]
    pub async fn fetch_html_cached(&self, url: &str) -> Result<String> {
        // Check cache first
        if let Some(cached) = self.get_cached(url) {
            return Ok(cached.content);
        }

        // Fetch with retry logic
        let response = self.fetch_with_retry(url).await?;
        let headers = response.headers().clone();
        let html = response
            .text()
            .await
            .map_err(|e| AnalyzeError::InternalError(e.to_string()))?;

        // Cache the response
        let size = html.len();
        self.cache_response(url, html.clone(), headers, size);

        Ok(html)
    }

    /// Fetch HTML content, headers, and metrics with caching and retry
    #[cfg(feature = "async")]
    pub async fn fetch_with_metrics_cached(
        &self,
        url: &str,
    ) -> Result<(String, reqwest::header::HeaderMap, usize)> {
        // Check cache first
        if let Some(cached) = self.get_cached(url) {
            return Ok((cached.content, cached.headers, cached.size));
        }

        // Fetch with retry logic
        let response = self.fetch_with_retry(url).await?;
        let headers = response.headers().clone();
        let bytes = response
            .bytes()
            .await
            .map_err(|e| AnalyzeError::InternalError(e.to_string()))?;

        let response_size = bytes.len();
        let html = String::from_utf8(bytes.to_vec())
            .map_err(|e| AnalyzeError::InternalError(format!("Invalid UTF-8: {}", e)))?;

        // Cache the response
        self.cache_response(url, html.clone(), headers.clone(), response_size);

        Ok((html, headers, response_size))
    }

    /// Clear the response cache
    #[cfg(feature = "async")]
    pub fn clear_cache() {
        if let Ok(mut cache) = RESPONSE_CACHE.lock() {
            cache.clear();
        }
    }

    /// Get cache statistics
    #[cfg(feature = "async")]
    pub fn cache_stats() -> (usize, usize) {
        if let Ok(cache) = RESPONSE_CACHE.lock() {
            let total_entries = cache.len();
            let total_size: usize = cache.values().map(|entry| entry.size).sum();
            (total_entries, total_size)
        } else {
            (0, 0)
        }
    }
}