rust_web_server/traffic/
classify.rs

1use std::net::IpAddr;
2use std::sync::OnceLock;
3
4use super::data::{AI_AGENTS, CRAWLERS, RANGES};
5
6/// The three groups the Traffic page shows (and this server's own calls).
7#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
8pub enum Group {
9    /// Browsers that behave like a person's.
10    Person,
11    /// AI crawlers, AI search and AI assistants fetching for someone.
12    Ai,
13    /// Everything else automated.
14    Bot,
15    /// This machine (health checks, updaters, the admin UI's own calls).
16    Internal,
17}
18
19/// What a client is, more precisely.
20#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
21pub enum Kind {
22    /// A browser User-Agent; a person if it loads page assets.
23    Browser,
24    AiTraining,
25    AiSearch,
26    /// An assistant fetching a page because someone asked it to.
27    AiAssistant,
28    AiAgent,
29    SearchEngine,
30    Seo,
31    Monitoring,
32    SocialPreview,
33    Feed,
34    Archiver,
35    /// Probing for secrets and admin pages (`/.env`, `/wp-login.php`).
36    Scanner,
37    /// Claims to be a crawler whose published network it isn't from.
38    Spoofed,
39    /// A library or tool (`curl`, `python-requests`) or an unknown bot.
40    Tool,
41    NoUserAgent,
42    Local,
43}
44
45impl Kind {
46    pub fn group(self) -> Group {
47        match self {
48            Kind::Browser => Group::Person,
49            Kind::AiTraining | Kind::AiSearch | Kind::AiAssistant | Kind::AiAgent => Group::Ai,
50            Kind::Local => Group::Internal,
51            _ => Group::Bot,
52        }
53    }
54
55    pub fn label(self) -> &'static str {
56        match self {
57            Kind::Browser => "browser",
58            Kind::AiTraining => "ai-training",
59            Kind::AiSearch => "ai-search",
60            Kind::AiAssistant => "ai-assistant",
61            Kind::AiAgent => "ai-agent",
62            Kind::SearchEngine => "search-engine",
63            Kind::Seo => "seo",
64            Kind::Monitoring => "monitoring",
65            Kind::SocialPreview => "social-preview",
66            Kind::Feed => "feed",
67            Kind::Archiver => "archiver",
68            Kind::Scanner => "scanner",
69            Kind::Spoofed => "spoofed",
70            Kind::Tool => "tool",
71            Kind::NoUserAgent => "no-user-agent",
72            Kind::Local => "local",
73        }
74    }
75
76    fn from_label(label: &str) -> Kind {
77        match label {
78            "ai-training" => Kind::AiTraining,
79            "ai-search" => Kind::AiSearch,
80            "ai-assistant" => Kind::AiAssistant,
81            "ai-agent" => Kind::AiAgent,
82            "search-engine" | "fetcher" => Kind::SearchEngine,
83            "seo" => Kind::Seo,
84            "monitoring" => Kind::Monitoring,
85            "social-preview" => Kind::SocialPreview,
86            "feed" => Kind::Feed,
87            "archiver" => Kind::Archiver,
88            "scanner" => Kind::Scanner,
89            _ => Kind::Tool,
90        }
91    }
92}
93
94/// A classified client.
95#[derive(Clone, Debug, PartialEq, Eq)]
96pub struct Class {
97    pub kind: Kind,
98    /// The bot's or agent's name (`GPTBot`, `Googlebot`), empty for browsers.
99    pub name: &'static str,
100    /// Who runs it, when known (`OpenAI`).
101    pub operator: &'static str,
102    /// The client's address is in the network its operator publishes.
103    pub verified: bool,
104}
105
106impl Class {
107    pub fn group(&self) -> Group {
108        self.kind.group()
109    }
110
111    fn of(kind: Kind, name: &'static str, operator: &'static str, verified: bool) -> Class {
112        Class { kind, name, operator, verified }
113    }
114}
115
116/// One published crawler network, parsed.
117struct Network {
118    name: &'static str,
119    kind: Kind,
120    /// The User-Agent token it verifies (`googlebot`), may be empty.
121    token: &'static str,
122    v4: Vec<(u32, u32)>,
123    v6: Vec<(u128, u128)>,
124}
125
126impl Network {
127    fn contains(&self, ip: IpAddr) -> bool {
128        match ip {
129            IpAddr::V4(v4) => {
130                let ip = u32::from(v4);
131                self.v4.iter().any(|(net, mask)| ip & mask == *net)
132            }
133            IpAddr::V6(v6) => {
134                if let Some(v4) = v6.to_ipv4_mapped() {
135                    return self.contains(IpAddr::V4(v4));
136                }
137                let ip = u128::from(v6);
138                self.v6.iter().any(|(net, mask)| ip & mask == *net)
139            }
140        }
141    }
142}
143
144fn networks() -> &'static [Network] {
145    static NETWORKS: OnceLock<Vec<Network>> = OnceLock::new();
146    NETWORKS.get_or_init(|| {
147        RANGES
148            .iter()
149            .map(|(name, _group, kind, token, prefixes)| {
150                let mut network = Network { name, kind: Kind::from_label(kind), token, v4: Vec::new(), v6: Vec::new() };
151                for prefix in *prefixes {
152                    let Some((address, bits)) = prefix.split_once('/') else { continue };
153                    let Ok(bits) = bits.parse::<u32>() else { continue };
154                    match address.parse::<IpAddr>() {
155                        Ok(IpAddr::V4(a)) if bits <= 32 => {
156                            let mask = if bits == 0 { 0 } else { u32::MAX << (32 - bits) };
157                            network.v4.push((u32::from(a) & mask, mask));
158                        }
159                        Ok(IpAddr::V6(a)) if bits <= 128 => {
160                            let mask = if bits == 0 { 0 } else { u128::MAX << (128 - bits) };
161                            network.v6.push((u128::from(a) & mask, mask));
162                        }
163                        _ => {}
164                    }
165                }
166                network
167            })
168            .collect()
169    })
170}
171
172/// Requests only scanners make: secrets, VCS metadata, other servers'
173/// admin pages. Checked on the lowercase path.
174const SCANNER_PATHS: &[&str] = &[
175    "/.env", "/.git/", "/.aws/", "/.ssh/", "/.svn/", "/.hg/", "/.ds_store", "/wp-login.php", "/wp-admin", "/wp-content/",
176    "/wp-includes/", "/xmlrpc.php", "/phpmyadmin", "/pma/", "/cgi-bin/", "/boaform", "/hnap1", "/actuator/", "/server-status",
177    "/vendor/phpunit", "/.well-known/acme-challenge/../", "/config.json", "/config.php", "/owa/", "/ecp/", "/solr/", "/druid/",
178    "/telescope/", "/_ignition/", "/debug/default/view", "/console/", "/manager/html", "/geoserver/", "/autodiscover/",
179];
180
181pub(super) fn scanner_path(path: &str) -> bool {
182    let path = path.split('?').next().unwrap_or(path);
183    // Lowercase only when needed: most paths already are.
184    let path: std::borrow::Cow<'_, str> =
185        if path.bytes().any(|b| b.is_ascii_uppercase()) { path.to_ascii_lowercase().into() } else { path.into() };
186    SCANNER_PATHS.iter().any(|p| path.starts_with(p) || path.contains(p))
187        || path.ends_with(".sql")
188        || path.ends_with(".bak")
189}
190
191fn browser(ua: &str) -> bool {
192    ua.starts_with("mozilla/") && ["chrome/", "safari/", "firefox/", "edg/", "opr/"].iter().any(|t| ua.contains(t))
193}
194
195/// What the client at `ip`, sending `user_agent`, asking for `path`, is.
196pub fn classify(ip: IpAddr, user_agent: Option<&str>, path: &str) -> Class {
197    if ip.is_loopback() || ip.to_canonical().is_loopback() {
198        return Class::of(Kind::Local, "", "", true);
199    }
200    let ua = user_agent.map(|u| u.trim().to_ascii_lowercase()).unwrap_or_default();
201    // A published crawler network: verified, whatever the User-Agent says.
202    let network = networks().iter().find(|n| n.contains(ip));
203    if let Some(network) = network {
204        return Class::of(network.kind, network.name, operator_of(network.name), true);
205    }
206    // Claims a crawler that publishes its network, from outside it.
207    if let Some(claimed) = networks().iter().find(|n| !n.token.is_empty() && ua.contains(n.token)) {
208        return Class::of(Kind::Spoofed, claimed.name, operator_of(claimed.name), false);
209    }
210    if scanner_path(path) {
211        return Class::of(Kind::Scanner, "", "", false);
212    }
213    if ua.is_empty() {
214        return Class::of(Kind::NoUserAgent, "", "", false);
215    }
216    if let Some((_, name, operator, kind)) = AI_AGENTS.iter().find(|(token, ..)| ua.contains(token)) {
217        return Class::of(Kind::from_label(kind), name, operator, false);
218    }
219    if let Some((_, _, name, kind)) = CRAWLERS.iter().find(|(anchored, literal, ..)| if *anchored { ua.starts_with(literal) } else { ua.contains(literal) }) {
220        return Class::of(Kind::from_label(kind), name, "", false);
221    }
222    if browser(&ua) {
223        return Class::of(Kind::Browser, "", "", false);
224    }
225    Class::of(Kind::Tool, "", "", false)
226}
227
228fn operator_of(network: &str) -> &'static str {
229    match network {
230        "GPTBot" | "OAI-SearchBot" | "ChatGPT-User" => "OpenAI",
231        "PerplexityBot" | "Perplexity-User" => "Perplexity",
232        "CCBot" => "Common Crawl",
233        "Bingbot" => "Microsoft",
234        "Applebot" => "Apple",
235        "DuckDuckBot" => "DuckDuckGo",
236        _ => "Google",
237    }
238}
rust_web_server/traffic/classify.rs

rust_web_server/traffic/
classify.rs
