codoseo_crawler/
scope.rs

1//! The internal/external rule: same host and port as the start address.
2
3use url::Url;
4
5/// Decides which URLs belong to the site being crawled.
6///
7/// A URL is internal when its host (lower-cased, one leading `www.` removed) and its port
8/// match the start address. The scheme is ignored, so `http://` and `https://` versions of
9/// a page are the same site. `Url::port` is `None` for a scheme's default port, so `:80` on
10/// `http` and `:443` on `https` both read as "no explicit port"; any other port is part of
11/// the site's identity.
12/// Subdomains other than `www.` are external.
13#[derive(Debug, Clone, PartialEq, Eq)]
14pub struct SiteScope {
15    host: String,
16    port: Option<u16>,
17}
18
19impl SiteScope {
20    pub fn new(origin: &Url) -> SiteScope {
21        SiteScope {
22            host: host_key(origin),
23            port: origin.port(),
24        }
25    }
26
27    pub fn is_internal(&self, url: &Url) -> bool {
28        host_key(url) == self.host && url.port() == self.port
29    }
30}
31
32fn host_key(url: &Url) -> String {
33    let host = url.host_str().unwrap_or("").to_ascii_lowercase();
34    match host.strip_prefix("www.") {
35        Some(rest) => rest.to_owned(),
36        None => host,
37    }
38}
39
40#[cfg(test)]
41mod tests {
42    use super::*;
43
44    fn u(s: &str) -> Url {
45        Url::parse(s).unwrap()
46    }
47
48    #[test]
49    fn ignores_scheme_and_leading_www() {
50        let s = SiteScope::new(&u("https://www.e.com/"));
51        assert!(s.is_internal(&u("https://e.com/a")));
52        assert!(s.is_internal(&u("http://www.e.com/b")));
53        assert!(s.is_internal(&u("HTTPS://WWW.E.COM/c")));
54    }
55
56    #[test]
57    fn subdomains_and_other_ports_are_external() {
58        let s = SiteScope::new(&u("https://www.e.com/"));
59        assert!(!s.is_internal(&u("https://blog.e.com/")));
60        assert!(!s.is_internal(&u("https://e.com:8443/")));
61        assert!(!s.is_internal(&u("https://other.org/")));
62        assert!(!s.is_internal(&u("https://www.www.e.com/")));
63    }
64
65    #[test]
66    fn explicit_ports_must_match_and_default_ports_are_equal() {
67        let s = SiteScope::new(&u("http://e.com/"));
68        assert!(s.is_internal(&u("https://e.com/")));
69        assert!(s.is_internal(&u("http://e.com:80/")));
70        assert!(s.is_internal(&u("https://e.com:443/")));
71        let s = SiteScope::new(&u("http://127.0.0.1:8080/"));
72        assert!(s.is_internal(&u("https://127.0.0.1:8080/x")));
73        assert!(!s.is_internal(&u("http://127.0.0.1:9090/x")));
74        assert!(!s.is_internal(&u("http://127.0.0.1/x")));
75    }
76}
codoseo_crawler/scope.rs

codoseo_crawler/
scope.rs
