webfind 0.5.0

Self-hosted native search engine for AI agents — pure CLI, zero cost
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205

use std::net::{IpAddr, ToSocketAddrs};
use url::Url;

/// Security validation error when a candidate seed or URL violates network policies.
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
pub enum SecurityGateError {
    #[error("invalid URL format: {0}")]
    InvalidUrl(String),

    #[error("unsupported scheme '{0}': only http and https are permitted")]
    DisallowedScheme(String),

    #[error("missing host in URL: {0}")]
    MissingHost(String),

    #[error("SSRF blocked: target IP {0} is in a forbidden private/local network range")]
    ForbiddenIp(IpAddr),

    #[error("DNS resolution failed or yielded no IP addresses: {0}")]
    DnsResolutionFailed(String),
}

/// Checks whether an IP address belongs to a forbidden/private network.
///
/// Disallows:
/// - Loopback addresses (127.0.0.0/8, ::1)
/// - RFC 1918 private subnets (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16)
/// - Link-local addresses (169.254.0.0/16, fe80::/10)
/// - Cloud metadata service (169.254.169.254)
/// - Broadcast & Unspecified (0.0.0.0, 255.255.255.255, ::)
pub fn is_forbidden_ip(ip: IpAddr) -> bool {
    // When explicitly enabled via environment variable (e.g. integration tests),
    // allow loopback (127.0.0.1, ::1) so tests can crawl mock servers.
    let allow_local = std::env::var("WEBFIND_ALLOW_LOCAL_SEEDS").map(|v| v == "1" || v == "true").unwrap_or(false);

    match ip {
        IpAddr::V4(ipv4) => {
            let is_loop = ipv4.is_loopback();
            if is_loop && allow_local {
                return false;
            }
            is_loop
                || ipv4.is_private()
                || ipv4.is_link_local()
                || ipv4.is_broadcast()
                || ipv4.is_unspecified()
                || ipv4.octets() == [169, 254, 169, 254]
        }
        IpAddr::V6(ipv6) => {
            let is_loop = ipv6.is_loopback();
            if is_loop && allow_local {
                return false;
            }
            is_loop || ipv6.is_unspecified() || ipv6.segments()[0] & 0xffc0 == 0xfe80
        }
    }
}

/// Pre-flight security validation for a seed URL.
///
/// Verifies:
/// 1. URL parsing succeeds.
/// 2. Scheme is strictly HTTP or HTTPS.
/// 3. Host is present.
/// 4. If host is an IP literal or resolves via DNS, none of the addresses are in forbidden ranges.
pub fn validate_url_security(raw_url: &str) -> Result<Url, SecurityGateError> {
    let parsed = Url::parse(raw_url).map_err(|e| SecurityGateError::InvalidUrl(e.to_string()))?;

    let scheme = parsed.scheme().to_lowercase();
    if scheme != "http" && scheme != "https" {
        return Err(SecurityGateError::DisallowedScheme(scheme));
    }

    let host = parsed
        .host_str()
        .ok_or_else(|| SecurityGateError::MissingHost(raw_url.to_string()))?;

    // Check if host is direct IP literal.
    if let Ok(ip) = host.parse::<IpAddr>() {
        if is_forbidden_ip(ip) {
            return Err(SecurityGateError::ForbiddenIp(ip));
        }
    } else {
        // Resolve hostname to inspect resolved IPs for SSRF defense.
        let port = parsed.port_or_known_default().unwrap_or(80);
        let socket_addr_str = format!("{}:{}", host, port);
        if let Ok(addrs) = socket_addr_str.to_socket_addrs() {
            for addr in addrs {
                if is_forbidden_ip(addr.ip()) {
                    return Err(SecurityGateError::ForbiddenIp(addr.ip()));
                }
            }
        }
    }

    Ok(parsed)
}

/// Normalizes a URL by stripping tracking parameters, query clutter, and fragments.
pub fn sanitize_url(raw_url: &str) -> Result<String, SecurityGateError> {
    let mut parsed = validate_url_security(raw_url)?;

    // Remove fragments.
    parsed.set_fragment(None);

    // Strip common tracking and telemetry query parameters.
    if parsed.query().is_some() {
        let clean_pairs: Vec<(String, String)> = parsed
            .query_pairs()
            .filter(|(k, _)| {
                let key = k.to_lowercase();
                !key.starts_with("utm_")
                    && key != "fbclid"
                    && key != "gclid"
                    && key != "ref"
                    && key != "source"
            })
            .map(|(k, v)| (k.into_owned(), v.into_owned()))
            .collect();

        if clean_pairs.is_empty() {
            parsed.set_query(None);
        } else {
            let mut serializer = url::form_urlencoded::Serializer::new(String::new());
            for (k, v) in clean_pairs {
                serializer.append_pair(&k, &v);
            }
            parsed.set_query(Some(&serializer.finish()));
        }
    }

    Ok(parsed.to_string())
}

const ADULT_NSFW_TOKENS: &[&str] = &[
    "porn", "pornography", "xxx", "sex", "erotic", "adult content",
    "hentai", "nsfw", "escort", "nude", "nudity", "webcam", "onlyfans",
    "camgirl", "dating adult", "xvideos", "pornhub", "redtube", "xnxx",
    "milf", "bdsm", "fetish", "erotica", "fuck", "pussy", "penis",
];

const AD_TRACKING_DOMAINS: &[&str] = &[
    "doubleclick.net", "googlesyndication.com", "adnxs.com", "criteo.com",
    "taboola.com", "outbrain.com", "advertising.com", "popads.net",
    "propellerads.com", "adroll.com", "zedo.com", "adsterra.com",
];

/// Hard quarantine gate: Returns true if any text contains adult/NSFW signals.
pub fn is_adult_or_nsfw_text(text: &str) -> bool {
    let lower = text.to_lowercase();
    for token in ADULT_NSFW_TOKENS {
        // Match word boundaries or exact substrings
        if lower.contains(token) {
            return true;
        }
    }
    false
}

/// Hard quarantine gate: Returns true if domain is known ad-network or tracking hub.
pub fn is_ad_or_tracker_domain(domain: &str) -> bool {
    let d = domain.to_lowercase();
    for ad_domain in AD_TRACKING_DOMAINS {
        if d == *ad_domain || d.ends_with(&format!(".{ad_domain}")) {
            return true;
        }
    }
    false
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_disallow_non_http() {
        assert!(validate_url_security("file:///etc/passwd").is_err());
        assert!(validate_url_security("ftp://example.com").is_err());
        assert!(validate_url_security("gopher://example.com").is_err());
    }

    #[test]
    fn test_disallow_forbidden_ips() {
        assert!(validate_url_security("http://127.0.0.1:8080").is_err());
        assert!(validate_url_security("http://localhost:3000").is_err());
        assert!(validate_url_security("http://169.254.169.254/latest/meta-data").is_err());
        assert!(validate_url_security("http://10.0.0.1/admin").is_err());
        assert!(validate_url_security("http://192.168.1.1").is_err());
    }

    #[test]
    fn test_allow_public_http() {
        assert!(validate_url_security("https://example.com/test").is_ok());
        assert!(validate_url_security("https://docs.rs/tokio/latest/tokio/").is_ok());
    }

    #[test]
    fn test_sanitize_tracking_params() {
        let cleaned = sanitize_url(
            "https://example.com/page?utm_source=twitter&utm_medium=cpc&id=42&fbclid=xyz#section",
        )
        .unwrap();
        assert_eq!(cleaned, "https://example.com/page?id=42");
    }
}