crawlrs 0.2.0

crawlrs - Enterprise-grade web scraping and crawling platform built with Rust
Documentation
// Copyright (c) 2025-2026 Kirky.X🌠
// SPDX-License-Identifier: Apache-2.0

//! SSRF protection and URL validation utilities
//!
//! This module provides security-focused validation for URLs to prevent
//! Server-Side Request Forgery (SSRF) attacks.
//!
//! ## Architecture
//!
//! This module now delegates to the unified SSRF protection module at
//! `crate::infrastructure::security::ssrf` which provides:
//! - Static URL validation (fast pre-check)
//! - DNS resolution validation (DNS rebinding protection)
//! - Redirect validation
//! - TOCTOU protection
//!
//! ## Usage
//!
//! For quick synchronous checks:
//! ```ignore
//! use crate::engines::validators::is_internal_url;
//!
//! if is_internal_url(&url) {
//!     return Err("Internal URLs not allowed");
//! }
//! ```
//!
//! For full async validation with DNS resolution:
//! ```ignore
//! use crate::engines::validators::validate_url;
//!
//! validate_url(&url).await?;
//! ```

// Re-export from unified SSRF module
pub use crate::infrastructure::security::ssrf::{
    is_internal_url, validate_domain_blacklist, validate_url, RedirectPolicy, RedirectValidator,
    SsrfConfig, SsrfError, SsrfValidationResult, SsrfValidator, ValidatedUrl,
};

#[cfg(test)]
mod tests {
    use super::*;
    use crate::engines::shared::is_blocked_hostname;

    #[tokio::test]
    async fn test_validate_url_ssrf() {
        // Localhost variants should be blocked
        assert!(validate_url("http://localhost").await.is_err());
        assert!(validate_url("http://localhost.localdomain").await.is_err());
        assert!(validate_url("http://127.0.0.1").await.is_err());
        assert!(validate_url("http://0.0.0.0").await.is_err());

        // Private IPs should be blocked
        assert!(validate_url("http://10.0.0.1").await.is_err());
        assert!(validate_url("http://192.168.1.1").await.is_err());
        assert!(validate_url("http://172.16.0.1").await.is_err());

        // Cloud metadata endpoints should be blocked
        assert!(validate_url("http://169.254.169.254").await.is_err());
        assert!(validate_url("http://metadata.google.internal")
            .await
            .is_err());

        // Invalid schemes should be blocked
        assert!(validate_url("file:///etc/passwd").await.is_err());
        assert!(validate_url("ftp://example.com").await.is_err());
        assert!(validate_url("gopher://localhost").await.is_err());

        // Valid public URLs - only run with network tests enabled
        if std::env::var("CRAWLRS_ENABLE_NETWORK_TESTS").is_ok() {
            assert!(validate_url("http://example.com").await.is_ok());
            assert!(validate_url("https://google.com").await.is_ok());
        }
    }

    #[test]
    fn test_is_internal_url() {
        // Localhost
        assert!(is_internal_url("http://localhost"));
        assert!(is_internal_url("http://127.0.0.1"));

        // Private IPs
        assert!(is_internal_url("http://10.0.0.1"));
        assert!(is_internal_url("http://192.168.1.1"));
        assert!(is_internal_url("http://172.16.0.1"));

        // Link-local
        assert!(is_internal_url("http://169.254.0.1"));

        // IPv6
        assert!(is_internal_url("http://[::1]"));
        assert!(is_internal_url("http://[fe80::1]"));
        assert!(is_internal_url("http://[fc00::1]"));

        // External
        assert!(!is_internal_url("http://8.8.8.8"));
        assert!(!is_internal_url("http://google.com"));
    }

    #[test]
    fn test_validate_domain_blacklist() {
        let blacklist = vec!["example.com".to_string(), "malicious.net".to_string()];

        // Blocked exact match
        assert!(validate_domain_blacklist("http://example.com", &blacklist).is_err());
        assert!(validate_domain_blacklist("http://malicious.net/path", &blacklist).is_err());

        // Blocked subdomain
        assert!(validate_domain_blacklist("http://sub.example.com", &blacklist).is_err());
        assert!(validate_domain_blacklist("http://api.malicious.net", &blacklist).is_err());

        // Allowed
        assert!(validate_domain_blacklist("http://google.com", &blacklist).is_ok());
        assert!(validate_domain_blacklist("http://example.org", &blacklist).is_ok());

        // Partial match should not block
        assert!(validate_domain_blacklist("http://myexample.com", &blacklist).is_ok());
    }

    #[test]
    fn test_is_blocked_hostname() {
        assert!(is_blocked_hostname("localhost"));
        assert!(is_blocked_hostname("127.0.0.1"));
        assert!(is_blocked_hostname("169.254.169.254"));
        assert!(is_blocked_hostname("metadata.google.internal"));

        // Case insensitive
        assert!(is_blocked_hostname("LOCALHOST"));
        assert!(is_blocked_hostname("LocalHost"));

        // Not blocked
        assert!(!is_blocked_hostname("google.com"));
        assert!(!is_blocked_hostname("example.com"));
        assert!(!is_blocked_hostname("8.8.8.8"));
    }

    // SSRF edge cases

    #[tokio::test]
    async fn test_validate_url_ssrf_double_encoded_blocked() {
        // Double-encoded localhost: %2531%2532%2537%252e%2530%252e%2530%252e%2531
        // These should be caught by scheme/host validation even if decoded
        assert!(validate_url("http://%31%32%37%2e%30%2e%30%2e%31")
            .await
            .is_err());
        assert!(validate_url("http://0x7f000001").await.is_err()); // hex IP for 127.0.0.1
        assert!(validate_url("http://2130706433").await.is_err()); // decimal IP for 127.0.0.1
    }

    #[tokio::test]
    async fn test_validate_url_ssrf_url_with_credentials_blocked() {
        // URLs with embedded credentials should be blocked (potential bypass)
        assert!(validate_url("http://user:pass@127.0.0.1/path")
            .await
            .is_err());
        assert!(validate_url("http://admin@localhost/admin").await.is_err());
    }

    #[tokio::test]
    async fn test_validate_url_ssrf_ipv6_shorthand_blocked() {
        // Various IPv6 loopback representations
        assert!(validate_url("http://[::1]").await.is_err());
        assert!(validate_url("http://[0:0:0:0:0:0:0:1]").await.is_err());
        assert!(validate_url("http://[::ffff:127.0.0.1]").await.is_err());
        // IPv4-compatible IPv6 (x.x.x.x, RFC 4291 deprecated) - bypass vector
        assert!(validate_url("http://[::127.0.0.1]").await.is_err());
        assert!(validate_url("http://[::169.254.169.254]").await.is_err());
        assert!(validate_url("http://[::10.0.0.1]").await.is_err());
        assert!(validate_url("http://[::192.168.1.1]").await.is_err());
    }

    #[tokio::test]
    async fn test_validate_url_ssrf_cloud_metadata_variants() {
        // AWS metadata
        assert!(validate_url("http://169.254.169.254/latest/meta-data/")
            .await
            .is_err());
        // GCP metadata
        assert!(
            validate_url("http://metadata.google.internal/computeMetadata/v1/")
                .await
                .is_err()
        );
        // Azure metadata
        assert!(validate_url("http://169.254.169.254/metadata/instance")
            .await
            .is_err());
    }

    #[tokio::test]
    async fn test_validate_url_empty_and_invalid() {
        assert!(validate_url("").await.is_err());
        assert!(validate_url("not-a-url").await.is_err());
        assert!(validate_url("javascript:alert(1)").await.is_err());
        assert!(validate_url("data:text/html,<h1>hi</h1>").await.is_err());
    }

    #[test]
    fn test_is_internal_url_edge_cases() {
        // 0.0.0.0 is internal
        assert!(is_internal_url("http://0.0.0.0"));
        // Mapped IPv4 in IPv6
        assert!(is_internal_url("http://[::ffff:10.0.0.1]"));
        // Empty host should not panic
        assert!(!is_internal_url("http://"));
    }
}