codoseo_core/
url.rs

1//! URL normalisation, so the same page always maps to the same URL and hash.
2
3use ::url::{Host, Url};
4use xxhash_rust::xxh3::xxh3_64;
5
6/// Resolves `href` against `base` and normalises it. Returns `None` for links that
7/// aren't crawlable pages: non-HTTP schemes, fragment-only links and blanks.
8///
9/// On top of what the URL parser already does (lowercase scheme and host, punycode,
10/// default ports, dot segments), this drops the fragment and a trailing dot on the
11/// host, upper-cases percent-escapes and decodes escaped unreserved characters.
12/// An empty query (`/a?`) is dropped; other query strings are kept as written.
13pub fn normalize(base: &Url, href: &str) -> Option<Url> {
14    let href = href.trim();
15    if href.is_empty() || href.starts_with('#') {
16        return None;
17    }
18    let mut url = base.join(href).ok()?;
19    if !matches!(url.scheme(), "http" | "https") {
20        return None;
21    }
22    url.set_fragment(None);
23
24    if let Some(Host::Domain(domain)) = url.host() {
25        if domain.is_empty() {
26            return None;
27        }
28        if let Some(stripped) = domain.strip_suffix('.') {
29            let stripped = stripped.to_owned();
30            url.set_host(Some(&stripped)).ok()?;
31        }
32    } else if url.host().is_none() {
33        return None;
34    }
35
36    let path = normalize_escapes(url.path());
37    if path != url.path() {
38        url.set_path(&path);
39    }
40    if let Some(query) = url.query() {
41        let query = normalize_escapes(query);
42        if Some(query.as_str()) != url.query() {
43            url.set_query(Some(&query));
44        }
45    }
46    if url.query() == Some("") {
47        url.set_query(None);
48    }
49    Some(url)
50}
51
52/// Stable 64-bit hash of a normalised URL.
53pub fn url_hash(url: &Url) -> u64 {
54    xxh3_64(url.as_str().as_bytes())
55}
56
57fn normalize_escapes(s: &str) -> String {
58    let bytes = s.as_bytes();
59    let mut out = String::with_capacity(s.len());
60    let mut i = 0;
61    while i < bytes.len() {
62        if bytes[i] == b'%' && i + 2 < bytes.len() {
63            let (hi, lo) = (bytes[i + 1], bytes[i + 2]);
64            if let (Some(h), Some(l)) = (hex_value(hi), hex_value(lo)) {
65                let decoded = h * 16 + l;
66                if is_unreserved(decoded) {
67                    out.push(decoded as char);
68                } else {
69                    out.push('%');
70                    out.push(hi.to_ascii_uppercase() as char);
71                    out.push(lo.to_ascii_uppercase() as char);
72                }
73                i += 3;
74                continue;
75            }
76        }
77        // The input is a valid &str, so copying char by char keeps it valid.
78        let ch = s[i..].chars().next().expect("index is on a char boundary");
79        out.push(ch);
80        i += ch.len_utf8();
81    }
82    out
83}
84
85fn hex_value(b: u8) -> Option<u8> {
86    match b {
87        b'0'..=b'9' => Some(b - b'0'),
88        b'a'..=b'f' => Some(b - b'a' + 10),
89        b'A'..=b'F' => Some(b - b'A' + 10),
90        _ => None,
91    }
92}
93
94fn is_unreserved(b: u8) -> bool {
95    b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b'_' | b'~')
96}
codoseo_core/url.rs

codoseo_core/
url.rs
