iriq 0.30.2

IRI/URL extraction, normalization, and shape clustering.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

use once_cell::sync::Lazy;
use regex::Regex;
use std::collections::HashSet;

pub(crate) static IPV4_RE: Lazy<Regex> =
    Lazy::new(|| Regex::new(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$").unwrap());

static TWO_LABEL_SUFFIXES: Lazy<HashSet<&'static str>> = Lazy::new(|| {
    [
        "co.uk",
        "org.uk",
        "gov.uk",
        "ac.uk",
        "net.uk",
        "me.uk",
        "ltd.uk",
        "plc.uk",
        "sch.uk",
        "co.jp",
        "ac.jp",
        "or.jp",
        "ne.jp",
        "go.jp",
        "gr.jp",
        "ed.jp",
        "lg.jp",
        "com.au",
        "net.au",
        "org.au",
        "edu.au",
        "gov.au",
        "asn.au",
        "id.au",
        "co.nz",
        "net.nz",
        "org.nz",
        "govt.nz",
        "ac.nz",
        "school.nz",
        "com.br",
        "net.br",
        "org.br",
        "gov.br",
        "edu.br",
        "com.cn",
        "net.cn",
        "org.cn",
        "gov.cn",
        "edu.cn",
        "ac.cn",
        "co.za",
        "net.za",
        "org.za",
        "gov.za",
        "ac.za",
        "co.kr",
        "ne.kr",
        "or.kr",
        "re.kr",
        "go.kr",
        "ac.kr",
        "co.in",
        "net.in",
        "org.in",
        "gov.in",
        "ac.in",
        "co.il",
        "net.il",
        "org.il",
        "gov.il",
        "ac.il",
        "muni.il",
        "com.mx",
        "net.mx",
        "org.mx",
        "gob.mx",
        "edu.mx",
        "com.ar",
        "net.ar",
        "org.ar",
        "gov.ar",
        "com.hk",
        "net.hk",
        "org.hk",
        "gov.hk",
        "edu.hk",
        "com.tw",
        "net.tw",
        "org.tw",
        "gov.tw",
        "edu.tw",
        "com.sg",
        "net.sg",
        "org.sg",
        "gov.sg",
        "edu.sg",
        "per.sg",
        "com.tr",
        "net.tr",
        "org.tr",
        "gov.tr",
        "edu.tr",
        "k12.tr",
    ]
    .iter()
    .copied()
    .collect()
});

/// Returns the registrable apex for a hostname (port not included).
/// Returns the input unchanged for IPv4 literals, single-label hosts, and
/// hosts already at 2-label apex form.
pub fn registrable_domain(host: &str) -> String {
    if host.is_empty() || IPV4_RE.is_match(host) {
        return host.to_string();
    }
    let labels: Vec<&str> = host.split('.').collect();
    if labels.len() <= 2 {
        return host.to_string();
    }
    let tail2 = format!("{}.{}", labels[labels.len() - 2], labels[labels.len() - 1]);
    if TWO_LABEL_SUFFIXES.contains(tail2.as_str()) {
        return labels[labels.len() - 3..].join(".");
    }
    labels[labels.len() - 2..].join(".")
}