rust_scraper 1.0.0

//! Crawler domain entities
//!
//! Core business entities for web crawling functionality.
//! Following Clean Architecture: pure domain logic, no framework dependencies.
//!
//! # Rules Applied
//!
//! - **err-thiserror-for-libraries**: `CrawlError` uses thiserror, NO reqwest/anyhow
//! - **api-builder**: `CrawlerConfig` with builder pattern
//! - **api-must-use**: `#[must_use]` on result structs
//! - **api-non-exhaustive**: `#[non_exhaustive]` for future evolution
//! - **clean-architecture-dependency-rule**: Domain NO depende de Infra (reqwest/anyhow)
//! - **security-ssrf-prevention**: URL parsing antes de comparación en `matches_pattern`

use thiserror::Error;
use url::Url;

/// Content type discovered during crawling
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum ContentType {
    /// HTML page
    #[default]
    Html,
    /// XML document (including sitemaps)
    Xml,
    /// Plain text
    Text,
    /// Unknown or other content type
    Other,
}

/// A discovered URL during crawling
///
/// Note: Cannot derive `Copy` because `Url` is not `Copy`.
/// Following **own-borrow-over-clone**: We'll pass references where possible.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DiscoveredUrl {
    /// The discovered URL
    pub url: Url,
    /// Depth in the crawl tree (0 = seed)
    pub depth: u8,
    /// Parent URL that led to this discovery
    pub parent_url: Url,
    /// Content type if known
    pub content_type: ContentType,
}

impl DiscoveredUrl {
    /// Create a new discovered URL
    #[must_use]
    pub fn new(url: Url, depth: u8, parent_url: Url, content_type: ContentType) -> Self {
        Self {
            url,
            depth,
            parent_url,
            content_type,
        }
    }

    /// Create a new discovered URL with default HTML content type
    #[must_use]
    pub fn html(url: Url, depth: u8, parent_url: Url) -> Self {
        Self {
            url,
            depth,
            parent_url,
            content_type: ContentType::Html,
        }
    }
}

/// Crawler configuration with builder pattern
///
/// Following **api-builder**: Provides fluent builder API.
/// Following **api-non-exhaustive**: Can evolve without breaking changes.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct CrawlerConfig {
    /// Seed URL to start crawling from
    pub seed_url: Url,
    /// Maximum depth to crawl (0 = only seed)
    pub max_depth: u8,
    /// Maximum number of pages to crawl
    pub max_pages: usize,
    /// URL patterns to include (glob-style)
    pub include_patterns: Vec<String>,
    /// URL patterns to exclude (glob-style)
    pub exclude_patterns: Vec<String>,
    /// Concurrency level (number of parallel requests)
    pub concurrency: usize,
    /// Delay between requests in milliseconds (rate limiting)
    pub delay_ms: u64,
    /// User agent string
    pub user_agent: String,
    /// Timeout for each request in seconds
    pub timeout_secs: u64,
    /// Use sitemap for URL discovery (FASE 3)
    pub use_sitemap: bool,
    /// Explicit sitemap URL (auto-discovers if None)
    pub sitemap_url: Option<String>,
}

impl CrawlerConfig {
    /// Create a new config with seed URL
    ///
    /// Following **api-builder**: Returns builder for fluent configuration.
    pub fn builder(seed_url: Url) -> CrawlerConfigBuilder {
        CrawlerConfigBuilder::new(seed_url)
    }

    /// Create a new config with default values
    pub fn new(seed_url: Url) -> Self {
        Self {
            seed_url,
            max_depth: 3,
            max_pages: 100,
            include_patterns: Vec::new(),
            exclude_patterns: Vec::new(),
            concurrency: 3, // Hardware-aware: nproc - 1 for 4C CPU
            delay_ms: 500,  // Hardware-aware: 500ms for HDD
            user_agent: "rust-scraper/0.3.0 (Web Crawler)".to_string(),
            timeout_secs: 30,
            use_sitemap: false,
            sitemap_url: None,
        }
    }

    /// Check if a URL matches the include patterns
    #[inline]
    #[must_use]
    pub fn matches_include(&self, url: &str) -> bool {
        if self.include_patterns.is_empty() {
            return true;
        }
        self.include_patterns
            .iter()
            .any(|pattern| matches_pattern(url, pattern))
    }

    /// Check if a URL matches the exclude patterns
    #[inline]
    #[must_use]
    pub fn matches_exclude(&self, url: &str) -> bool {
        self.exclude_patterns
            .iter()
            .any(|pattern| matches_pattern(url, pattern))
    }
}

/// Builder for CrawlerConfig
///
/// Following **api-builder** and **api-must-use**.
#[derive(Debug)]
#[must_use]
pub struct CrawlerConfigBuilder {
    use_sitemap: bool,
    sitemap_url: Option<String>,
    seed_url: Url,
    max_depth: u8,
    max_pages: usize,
    include_patterns: Vec<String>,
    exclude_patterns: Vec<String>,
    concurrency: usize,
    delay_ms: u64,
    user_agent: String,
    timeout_secs: u64,
}

impl CrawlerConfigBuilder {
    /// Create a new builder with seed URL
    pub fn new(seed_url: Url) -> Self {
        Self {
            seed_url,
            max_depth: 3,
            max_pages: 100,
            include_patterns: Vec::new(),
            exclude_patterns: Vec::new(),
            concurrency: 3,
            delay_ms: 500,
            user_agent: "rust-scraper/0.3.0 (Web Crawler)".to_string(),
            timeout_secs: 30,
            use_sitemap: false,
            sitemap_url: None,
        }
    }

    /// Set maximum crawl depth
    pub fn max_depth(mut self, depth: u8) -> Self {
        self.max_depth = depth;
        self
    }

    /// Set maximum number of pages
    pub fn max_pages(mut self, pages: usize) -> Self {
        self.max_pages = pages;
        self
    }

    /// Add an include pattern
    pub fn include_pattern(mut self, pattern: impl Into<String>) -> Self {
        self.include_patterns.push(pattern.into());
        self
    }

    /// Add multiple include patterns
    pub fn include_patterns(mut self, patterns: Vec<String>) -> Self {
        self.include_patterns.extend(patterns);
        self
    }

    /// Add an exclude pattern
    pub fn exclude_pattern(mut self, pattern: impl Into<String>) -> Self {
        self.exclude_patterns.push(pattern.into());
        self
    }

    /// Add multiple exclude patterns
    pub fn exclude_patterns(mut self, patterns: Vec<String>) -> Self {
        self.exclude_patterns.extend(patterns);
        self
    }

    /// Set concurrency level
    pub fn concurrency(mut self, level: usize) -> Self {
        self.concurrency = level;
        self
    }

    /// Set delay between requests in milliseconds
    pub fn delay_ms(mut self, ms: u64) -> Self {
        self.delay_ms = ms;
        self
    }

    /// Set user agent string
    pub fn user_agent(mut self, ua: impl Into<String>) -> Self {
        self.user_agent = ua.into();
        self
    }

    /// Set request timeout in seconds
    pub fn timeout_secs(mut self, secs: u64) -> Self {
        self.timeout_secs = secs;
        self
    }

    /// Set use_sitemap flag (FASE 3)
    pub fn use_sitemap(mut self, use_sitemap: bool) -> Self {
        self.use_sitemap = use_sitemap;
        self
    }

    /// Set explicit sitemap URL (FASE 3)
    pub fn sitemap_url(mut self, url: impl Into<String>) -> Self {
        self.sitemap_url = Some(url.into());
        self
    }

    #[must_use]
    pub fn build(self) -> CrawlerConfig {
        CrawlerConfig {
            seed_url: self.seed_url,
            max_depth: self.max_depth,
            max_pages: self.max_pages,
            include_patterns: self.include_patterns,
            exclude_patterns: self.exclude_patterns,
            concurrency: self.concurrency,
            delay_ms: self.delay_ms,
            user_agent: self.user_agent,
            timeout_secs: self.timeout_secs,
            use_sitemap: self.use_sitemap,
            sitemap_url: self.sitemap_url,
        }
    }
}

/// Crawl result containing discovered URLs
///
/// Following **api-must-use** and **api-non-exhaustive**.
#[derive(Debug, Clone, Default)]
#[must_use]
#[non_exhaustive]
pub struct CrawlResult {
    /// All discovered URLs
    pub urls: Vec<DiscoveredUrl>,
    /// Total number of pages crawled
    pub total_pages: usize,
    /// Number of errors encountered
    pub errors: usize,
}

impl CrawlResult {
    /// Create a new crawl result
    pub fn new(urls: Vec<DiscoveredUrl>, total_pages: usize, errors: usize) -> Self {
        Self {
            urls,
            total_pages,
            errors,
        }
    }

    /// Create an empty crawl result
    pub fn empty() -> Self {
        Self::default()
    }

    /// Check if the result is empty
    #[inline]
    #[must_use]
    pub fn is_empty(&self) -> bool {
        self.urls.is_empty()
    }
}

/// Crawl errors
///
/// Following **err-thiserror-for-libraries**: Uses thiserror for library error types.
/// Following **api-non-exhaustive**: Can add variants without breaking changes.
/// Following **clean-architecture**: NO dependencies on reqwest/anyhow (Infra layer)
///
/// # Architecture Note
///
/// This error type does NOT contain `reqwest::Error` or `anyhow::Error`.
/// Those are infrastructure details. The Infrastructure layer converts
/// `reqwest::Error` → `CrawlError::Network` and `anyhow::Error` → specific variants.
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum CrawlError {
    /// Network error during HTTP request
    ///
    /// Note: Does NOT contain reqwest::Error (that's Infra detail).
    /// Infrastructure layer converts reqwest::Error → this variant.
    #[error("network error: {message} (status: {status_code:?})")]
    Network {
        message: String,
        status_code: Option<u16>,
    },

    /// HTTP error (status code or request failure)
    #[error("HTTP error: {0}")]
    Http(String),

    /// URL parsing error
    #[error("invalid URL: {0}")]
    InvalidUrl(String),

    /// HTML parsing error
    #[error("parse error: {0}")]
    Parse(String),

    /// Rate limit exceeded
    #[error("rate limit exceeded")]
    RateLimit,

    /// Maximum depth exceeded
    #[error("maximum depth {max} exceeded at depth {current}")]
    MaxDepthExceeded { current: u8, max: u8 },

    /// Maximum pages exceeded
    #[error("maximum pages {max} exceeded")]
    MaxPagesExceeded { max: usize },

    /// URL excluded by pattern
    #[error("URL excluded: {0}")]
    UrlExcluded(String),

    /// Invalid content type
    #[error("invalid content type: {0}")]
    InvalidContentType(String),

    /// I/O error
    #[error("I/O error: {0}")]
    Io(#[from] std::io::Error),

    /// Semaphore error (concurrency control)
    #[error("semaphore error: {0}")]
    Semaphore(String),

    /// Internal error (unspecified)
    #[error("internal error: {0}")]
    Internal(String),

    /// Sitemap parsing error (FASE 3)
    /// Note: Does NOT contain sitemap_parser::SitemapError (that's Infra detail).
    /// Infrastructure layer converts SitemapError → this variant.
    #[error("sitemap error: {0}")]
    Sitemap(String),
}

/// Pattern matching helper function
///
/// Following **own-borrow-over-clone**: Accepts &str not &String.
/// Following **opt-inline**: Inlined for hot path performance.
/// Following **security-ssrf-prevention**: Parses URL before comparison (no .contains() on raw string)
///
/// # Security
///
/// This function parses the URL using `url::Url` and compares HOSTS only,
/// NOT raw string substrings. This prevents SSRF attacks where malicious
/// URLs like `https://evil.com/?q=example.com/path` could bypass filters.
///
/// # Examples
///
/// ```
/// use rust_scraper::domain::crawler_entities::matches_pattern;
///
/// // Valid subdomain match
/// assert!(matches_pattern("https://blog.example.com/post", "*.example.com/*"));
///
/// // SSRF bypass attempt (should NOT match)
/// assert!(!matches_pattern("https://evil.com/?q=example.com/path", "*.example.com/*"));
/// ```
#[inline]
#[must_use]
pub fn matches_pattern(url_str: &str, pattern: &str) -> bool {
    // Parse URL FIRST (extract real host)
    let url = match Url::parse(url_str) {
        Ok(u) => u,
        Err(_) => return false, // Invalid URL → no match
    };

    let host = match url.host_str() {
        Some(h) => h,
        None => return false, // No host → no match
    };

    // Handle empty pattern
    if pattern.is_empty() {
        return true;
    }

    // Handle wildcard
    if pattern == "*" {
        return true;
    }

    // Compare HOSTS only (NOT raw URL strings)
    match pattern {
        // *.example.com/* → match subdomain ONLY (not root domain)
        p if p.starts_with("*.") && p.ends_with("*") => {
            let domain = &p[2..p.len() - 1]; // "example.com/"
            let domain = domain.trim_end_matches('/');
            // Must be a subdomain, NOT the root domain itself
            host.ends_with(&format!(".{}", domain))
        }
        // *.example.com → match subdomain ONLY (not root domain)
        p if p.starts_with("*.") => {
            let domain = &p[2..];
            // Must be a subdomain, NOT the root domain itself
            host.ends_with(&format!(".{}", domain))
        }
        // Exact host match (no wildcard)
        p => host == p,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_discovered_url_new() {
        let url = Url::parse("https://example.com/page").unwrap();
        let parent = Url::parse("https://example.com/").unwrap();
        let discovered = DiscoveredUrl::new(url, 1, parent, ContentType::Html);

        assert_eq!(discovered.depth, 1);
        assert_eq!(discovered.content_type, ContentType::Html);
    }

    #[test]
    fn test_discovered_url_html() {
        let url = Url::parse("https://example.com/page").unwrap();
        let parent = Url::parse("https://example.com/").unwrap();
        let discovered = DiscoveredUrl::html(url, 0, parent);

        assert_eq!(discovered.depth, 0);
        assert_eq!(discovered.content_type, ContentType::Html);
    }

    #[test]
    fn test_crawler_config_builder() {
        let seed = Url::parse("https://example.com").unwrap();
        let config = CrawlerConfig::builder(seed)
            .max_depth(5)
            .max_pages(500)
            .concurrency(5)
            .delay_ms(1000)
            .include_pattern("*.example.com/*".to_string())
            .exclude_pattern("*/admin/*".to_string())
            .build();

        assert_eq!(config.max_depth, 5);
        assert_eq!(config.max_pages, 500);
        assert_eq!(config.concurrency, 5);
        assert_eq!(config.delay_ms, 1000);
        assert_eq!(config.include_patterns.len(), 1);
        assert_eq!(config.exclude_patterns.len(), 1);
    }

    #[test]
    fn test_crawler_config_default() {
        let seed = Url::parse("https://example.com").unwrap();
        let config = CrawlerConfig::new(seed);

        assert_eq!(config.max_depth, 3);
        assert_eq!(config.max_pages, 100);
        assert_eq!(config.concurrency, 3);
        assert_eq!(config.delay_ms, 500);
    }

    #[test]
    fn test_crawl_result_empty() {
        let result = CrawlResult::empty();
        assert!(result.is_empty());
        assert_eq!(result.total_pages, 0);
        assert_eq!(result.errors, 0);
    }

    #[test]
    fn test_crawl_result_new() {
        let url = Url::parse("https://example.com").unwrap();
        let parent = Url::parse("https://example.com/").unwrap();
        let discovered = DiscoveredUrl::html(url, 0, parent);
        let result = CrawlResult::new(vec![discovered], 1, 0);

        assert!(!result.is_empty());
        assert_eq!(result.total_pages, 1);
        assert_eq!(result.errors, 0);
        assert_eq!(result.urls.len(), 1);
    }

    // ========== CRAWL ERROR TESTS (NO reqwest/anyhow) ==========

    #[test]
    fn test_crawl_error_network_no_reqwest() {
        // Network error NO depende de reqwest
        let error = CrawlError::Network {
            message: "timeout".to_string(),
            status_code: Some(408),
        };
        assert!(error.to_string().contains("timeout"));
        assert!(error.to_string().contains("408"));
    }

    #[test]
    fn test_crawl_error_network_no_status() {
        let error = CrawlError::Network {
            message: "connection refused".to_string(),
            status_code: None,
        };
        assert!(error.to_string().contains("connection refused"));
        assert!(error.to_string().contains("None"));
    }

    #[test]
    fn test_crawl_error_io() {
        // Io error con std::io::Error
        let io_error = std::io::Error::new(std::io::ErrorKind::NotFound, "file not found");
        let error = CrawlError::from(io_error);
        assert!(matches!(error, CrawlError::Io(_)));
        assert!(error.to_string().contains("file not found"));
    }

    #[test]
    fn test_crawl_error_semaphore() {
        let error = CrawlError::Semaphore("permit lost".to_string());
        assert!(error.to_string().contains("permit lost"));
    }

    #[test]
    fn test_crawl_error_internal() {
        let error = CrawlError::Internal("something went wrong".to_string());
        assert!(error.to_string().contains("something went wrong"));
    }

    #[test]
    fn test_crawl_error_display_all_variants() {
        // Test all error variants display correctly
        let error = CrawlError::InvalidUrl("bad-url".to_string());
        assert!(error.to_string().contains("bad-url"));

        let error = CrawlError::Parse("html parse failed".to_string());
        assert!(error.to_string().contains("html parse failed"));

        let error = CrawlError::RateLimit;
        assert_eq!(error.to_string(), "rate limit exceeded");

        let error = CrawlError::MaxDepthExceeded { current: 5, max: 3 };
        assert_eq!(error.to_string(), "maximum depth 3 exceeded at depth 5");

        let error = CrawlError::MaxPagesExceeded { max: 100 };
        assert_eq!(error.to_string(), "maximum pages 100 exceeded");

        let error = CrawlError::UrlExcluded("https://evil.com".to_string());
        assert!(error.to_string().contains("evil.com"));

        let error = CrawlError::InvalidContentType("image/png".to_string());
        assert!(error.to_string().contains("image/png"));
    }

    // ========== SSRF PREVENTION TESTS ==========

    #[test]
    fn test_matches_pattern_ssrf_bypass_attempt() {
        // ATAQUE: Evil URL con query params que contienen el dominio
        // Esto NO debe matchear
        assert!(!matches_pattern(
            "https://evil.com/?q=example.com/path",
            "*.example.com/*"
        ));

        assert!(!matches_pattern(
            "https://attacker.com/?redirect=example.com/admin",
            "*.example.com/*"
        ));

        // Another SSRF bypass attempt
        assert!(!matches_pattern(
            "https://malicious.com/redirect?url=example.com/secret",
            "*.example.com/*"
        ));
    }

    #[test]
    fn test_matches_pattern_real_subdomain() {
        // Subdominio real DEBE matchear
        assert!(matches_pattern(
            "https://blog.example.com/post",
            "*.example.com/*"
        ));

        assert!(matches_pattern(
            "https://sub.example.com/page",
            "*.example.com"
        ));

        // Multiple subdomain levels
        assert!(matches_pattern(
            "https://deep.sub.example.com/page",
            "*.example.com/*"
        ));
    }

    #[test]
    fn test_matches_pattern_with_port() {
        // URLs with ports should work
        // Note: example.com:8080 does NOT match *.example.com/* (needs subdomain)
        assert!(matches_pattern(
            "https://blog.example.com:8080/path",
            "*.example.com/*"
        ));

        assert!(matches_pattern(
            "https://blog.example.com:443/post",
            "*.example.com/*"
        ));
    }

    #[test]
    fn test_matches_pattern_ipv4() {
        // IPv4 debe funcionar
        assert!(matches_pattern(
            "http://192.168.1.1:8080/path",
            "192.168.1.1"
        ));
    }

    #[test]
    fn test_matches_pattern_ipv6() {
        // IPv6 debe funcionar
        assert!(matches_pattern("http://[::1]:8080/path", "[::1]"));
    }

    #[test]
    fn test_matches_pattern_invalid_url() {
        // URLs inválidas NO deben matchear
        assert!(!matches_pattern("not-a-url", "*.example.com/*"));
        assert!(!matches_pattern("://missing-scheme.com", "*"));
        assert!(!matches_pattern("", "*"));
    }

    #[test]
    fn test_matches_pattern_wildcard() {
        assert!(matches_pattern("https://example.com/page", "*"));
        assert!(matches_pattern("https://any.domain.com/page", "*"));
    }

    #[test]
    fn test_matches_pattern_empty() {
        assert!(matches_pattern("https://example.com", ""));
    }

    #[test]
    fn test_matches_pattern_no_match() {
        assert!(!matches_pattern("https://other.com/page", "example.com"));
        assert!(!matches_pattern("https://evil.com/page", "*.example.com/*"));
    }

    #[test]
    fn test_matches_pattern_exact_host() {
        assert!(matches_pattern("https://example.com/page", "example.com"));
        assert!(!matches_pattern(
            "https://sub.example.com/page",
            "example.com"
        ));
    }

    #[test]
    fn test_matches_pattern_prefix_wildcard() {
        // After SSRF fix, *.example.com/* matches SUBDOMAINS only, not root domain
        // Use "example.com/*" pattern to match the root domain itself
        assert!(matches_pattern(
            "https://blog.example.com/admin/users",
            "*.example.com/*"
        ));
        assert!(matches_pattern(
            "https://admin.example.com/users",
            "*.example.com/*"
        ));
        // Root domain does NOT match *.example.com/*
        assert!(!matches_pattern(
            "https://example.com/admin/users",
            "*.example.com/*"
        ));
    }

    #[test]
    fn test_matches_pattern_slash_wildcard() {
        // *.example.com/* matches any subdomain of example.com
        // Note: Compares HOSTS only, path is not considered
        assert!(matches_pattern(
            "https://blog.example.com/admin/users",
            "*.example.com/*"
        ));
        assert!(matches_pattern(
            "https://admin.example.com/users",
            "*.example.com/*"
        ));
        // Root domain does NOT match *.example.com/*
        assert!(!matches_pattern(
            "https://example.com/admin/users",
            "*.example.com/*"
        ));
    }
}