rws_proxy/traffic/
classify.rs

1use std::net::IpAddr;
2use std::sync::OnceLock;
3
4use super::data::{AI_AGENTS, CRAWLERS, RANGES};
5
6/// The three groups the Traffic page shows (and this server's own calls).
7#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
8pub enum Group {
9    /// Browsers that behave like a person's.
10    Person,
11    /// AI crawlers, AI search and AI assistants fetching for someone.
12    Ai,
13    /// Everything else automated.
14    Bot,
15    /// The site's own native apps and browser extensions (`[traffic] apps`,
16    /// or recognised by platform): counted as app users, not as bots.
17    App,
18    /// The site's owner (`traffic::owner`): counted apart, so the other
19    /// groups show everyone else.
20    Owner,
21    /// This machine (health checks, updaters, the admin UI's own calls).
22    Internal,
23}
24
25/// What a client is, more precisely.
26#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
27pub enum Kind {
28    /// A browser User-Agent; a person if it loads page assets.
29    Browser,
30    AiTraining,
31    AiSearch,
32    /// An assistant fetching a page because someone asked it to.
33    AiAssistant,
34    AiAgent,
35    SearchEngine,
36    Seo,
37    Monitoring,
38    SocialPreview,
39    Feed,
40    Archiver,
41    /// Probing for secrets and admin pages (`/.env`, `/wp-login.php`).
42    Scanner,
43    /// Claims to be a crawler whose published network it isn't from.
44    Spoofed,
45    /// A browser User-Agent from a cloud or server hosting network
46    /// (`traffic::hosting`): a program, not a person.
47    Hosting,
48    /// A library or tool (`curl`, `python-requests`) or an unknown bot.
49    Tool,
50    /// A native app or browser extension (`traffic::apps`).
51    App,
52    /// The site's owner: signed in to the admin API, from a configured
53    /// network, or a browser marked with the owner link (`traffic::owner`).
54    Owner,
55    NoUserAgent,
56    Local,
57}
58
59impl Kind {
60    pub fn group(self) -> Group {
61        match self {
62            Kind::Browser => Group::Person,
63            Kind::AiTraining | Kind::AiSearch | Kind::AiAssistant | Kind::AiAgent => Group::Ai,
64            Kind::Local => Group::Internal,
65            Kind::App => Group::App,
66            Kind::Owner => Group::Owner,
67            _ => Group::Bot,
68        }
69    }
70
71    pub fn label(self) -> &'static str {
72        match self {
73            Kind::Browser => "browser",
74            Kind::AiTraining => "ai-training",
75            Kind::AiSearch => "ai-search",
76            Kind::AiAssistant => "ai-assistant",
77            Kind::AiAgent => "ai-agent",
78            Kind::SearchEngine => "search-engine",
79            Kind::Seo => "seo",
80            Kind::Monitoring => "monitoring",
81            Kind::SocialPreview => "social-preview",
82            Kind::Feed => "feed",
83            Kind::Archiver => "archiver",
84            Kind::Scanner => "scanner",
85            Kind::Spoofed => "spoofed",
86            Kind::Hosting => "hosting",
87            Kind::Tool => "tool",
88            Kind::NoUserAgent => "no-user-agent",
89            Kind::Local => "local",
90            Kind::App => "app",
91            Kind::Owner => "owner",
92        }
93    }
94
95    fn from_label(label: &str) -> Kind {
96        match label {
97            "ai-training" => Kind::AiTraining,
98            "ai-search" => Kind::AiSearch,
99            "ai-assistant" => Kind::AiAssistant,
100            "ai-agent" => Kind::AiAgent,
101            "search-engine" | "fetcher" => Kind::SearchEngine,
102            "seo" => Kind::Seo,
103            "monitoring" => Kind::Monitoring,
104            "social-preview" => Kind::SocialPreview,
105            "feed" => Kind::Feed,
106            "archiver" => Kind::Archiver,
107            "scanner" => Kind::Scanner,
108            "hosting" => Kind::Hosting,
109            _ => Kind::Tool,
110        }
111    }
112}
113
114/// A classified client.
115#[derive(Clone, Debug, PartialEq, Eq)]
116pub struct Class {
117    pub kind: Kind,
118    /// The bot's or agent's name (`GPTBot`, `Googlebot`), empty for browsers.
119    pub name: &'static str,
120    /// Who runs it, when known (`OpenAI`).
121    pub operator: &'static str,
122    /// The client's address is in the network its operator publishes.
123    pub verified: bool,
124}
125
126impl Class {
127    pub fn group(&self) -> Group {
128        self.kind.group()
129    }
130
131    fn of(kind: Kind, name: &'static str, operator: &'static str, verified: bool) -> Class {
132        Class { kind, name, operator, verified }
133    }
134}
135
136/// One published crawler network, parsed.
137struct Network {
138    name: &'static str,
139    kind: Kind,
140    /// The User-Agent token it verifies (`googlebot`), may be empty.
141    token: &'static str,
142    v4: Vec<(u32, u32)>,
143    v6: Vec<(u128, u128)>,
144}
145
146impl Network {
147    fn contains(&self, ip: IpAddr) -> bool {
148        match ip {
149            IpAddr::V4(v4) => {
150                let ip = u32::from(v4);
151                self.v4.iter().any(|(net, mask)| ip & mask == *net)
152            }
153            IpAddr::V6(v6) => {
154                if let Some(v4) = v6.to_ipv4_mapped() {
155                    return self.contains(IpAddr::V4(v4));
156                }
157                let ip = u128::from(v6);
158                self.v6.iter().any(|(net, mask)| ip & mask == *net)
159            }
160        }
161    }
162}
163
164fn networks() -> &'static [Network] {
165    static NETWORKS: OnceLock<Vec<Network>> = OnceLock::new();
166    NETWORKS.get_or_init(|| {
167        RANGES
168            .iter()
169            .map(|(name, _group, kind, token, prefixes)| {
170                let mut network = Network { name, kind: Kind::from_label(kind), token, v4: Vec::new(), v6: Vec::new() };
171                for prefix in *prefixes {
172                    let Some((address, bits)) = prefix.split_once('/') else { continue };
173                    let Ok(bits) = bits.parse::<u32>() else { continue };
174                    match address.parse::<IpAddr>() {
175                        Ok(IpAddr::V4(a)) if bits <= 32 => {
176                            let mask = if bits == 0 { 0 } else { u32::MAX << (32 - bits) };
177                            network.v4.push((u32::from(a) & mask, mask));
178                        }
179                        Ok(IpAddr::V6(a)) if bits <= 128 => {
180                            let mask = if bits == 0 { 0 } else { u128::MAX << (128 - bits) };
181                            network.v6.push((u128::from(a) & mask, mask));
182                        }
183                        _ => {}
184                    }
185                }
186                network
187            })
188            .collect()
189    })
190}
191
192/// Requests only scanners make: secrets, VCS metadata, other servers'
193/// admin pages. Checked on the lowercase path.
194const SCANNER_PATHS: &[&str] = &[
195    "/.env", "/.git/", "/.aws/", "/.ssh/", "/.svn/", "/.hg/", "/.ds_store", "/wp-login.php", "/wp-admin", "/wp-content/",
196    "/wp-includes/", "/xmlrpc.php", "/phpmyadmin", "/pma/", "/cgi-bin/", "/boaform", "/hnap1", "/actuator/", "/server-status",
197    "/vendor/phpunit", "/.well-known/acme-challenge/../", "/config.json", "/config.php", "/owa/", "/ecp/", "/solr/", "/druid/",
198    "/telescope/", "/_ignition/", "/debug/default/view", "/console/", "/manager/html", "/geoserver/", "/autodiscover/",
199];
200
201pub(super) fn scanner_path(path: &str) -> bool {
202    let path = path.split('?').next().unwrap_or(path);
203    // Lowercase only when needed: most paths already are.
204    let path: std::borrow::Cow<'_, str> =
205        if path.bytes().any(|b| b.is_ascii_uppercase()) { path.to_ascii_lowercase().into() } else { path.into() };
206    SCANNER_PATHS.iter().any(|p| path.starts_with(p) || path.contains(p))
207        || path.ends_with(".sql")
208        || path.ends_with(".bak")
209}
210
211fn browser(ua: &str) -> bool {
212    ua.starts_with("mozilla/") && ["chrome/", "safari/", "firefox/", "edg/", "opr/"].iter().any(|t| ua.contains(t))
213}
214
215/// What the client at `ip`, sending `user_agent`, asking for `path`, is.
216pub fn classify(ip: IpAddr, user_agent: Option<&str>, path: &str) -> Class {
217    if ip.is_loopback() || ip.to_canonical().is_loopback() {
218        return Class::of(Kind::Local, "", "", true);
219    }
220    let ua = user_agent.map(|u| u.trim().to_ascii_lowercase()).unwrap_or_default();
221    // A published crawler network: verified, whatever the User-Agent says.
222    let network = networks().iter().find(|n| n.contains(ip));
223    if let Some(network) = network {
224        return Class::of(network.kind, network.name, operator_of(network.name), true);
225    }
226    // Claims a crawler that publishes its network, from outside it.
227    if let Some(claimed) = networks().iter().find(|n| !n.token.is_empty() && ua.contains(n.token)) {
228        return Class::of(Kind::Spoofed, claimed.name, operator_of(claimed.name), false);
229    }
230    if scanner_path(path) {
231        return Class::of(Kind::Scanner, "", "", false);
232    }
233    if ua.is_empty() {
234        return Class::of(Kind::NoUserAgent, "", "", false);
235    }
236    if let Some((_, name, operator, kind)) = AI_AGENTS.iter().find(|(token, ..)| ua.contains(token)) {
237        return Class::of(Kind::from_label(kind), name, operator, false);
238    }
239    if let Some((_, _, name, kind)) = CRAWLERS.iter().find(|(anchored, literal, ..)| if *anchored { ua.starts_with(literal) } else { ua.contains(literal) }) {
240        return Class::of(Kind::from_label(kind), name, "", false);
241    }
242    if browser(&ua) {
243        return Class::of(Kind::Browser, "", "", false);
244    }
245    Class::of(Kind::Tool, "", "", false)
246}
247
248fn operator_of(network: &str) -> &'static str {
249    match network {
250        "GPTBot" | "OAI-SearchBot" | "ChatGPT-User" => "OpenAI",
251        "PerplexityBot" | "Perplexity-User" => "Perplexity",
252        "CCBot" => "Common Crawl",
253        "Bingbot" => "Microsoft",
254        "Applebot" => "Apple",
255        "DuckDuckBot" => "DuckDuckGo",
256        _ => "Google",
257    }
258}
rws_proxy/traffic/classify.rs

rws_proxy/traffic/
classify.rs
