rws_proxy/traffic/
mod.rs

1//! Traffic insights (18.15.0): who visits — people, AI agents, bots —
2//! worked out by rws from each request, for the admin UI's Traffic page.
3//!
4//! This first part is the classifier, [`classify`]: from the client's
5//! address, User-Agent and path it names what kind of client this is, using
6//! - the IP ranges Google, Bing, Apple, DuckDuckGo, OpenAI, Perplexity and
7//!   Common Crawl publish for their crawlers (a match is *verified*; a
8//!   User-Agent that claims one of them from elsewhere is *spoofed*);
9//! - [ai.robots.txt](https://github.com/ai-robots-txt/ai.robots.txt)'s AI
10//!   agents and [crawler-user-agents](https://github.com/monperrus/crawler-user-agents)'
11//!   bot patterns (both MIT, bundled, regenerated by `ci/gen_bot_lists.py`);
12//! - requests only scanners make (`/.env`, `/.git/config`, `/wp-login.php`…).
13//!
14//! A browser User-Agent that matches none of them is a [`Kind::Browser`]:
15//! whether it's a person is decided later, by whether that client loads the
16//! page's assets as a browser does.
17
18mod classify;
19pub mod config;
20mod data;
21#[doc(hidden)]
22pub mod persist;
23pub mod record;
24pub mod summary;
25
26pub use classify::{classify, Class, Group, Kind};
27pub use record::{enable, enabled};
28
29/// Count one finished request (called with the access log line, whatever
30/// the log format).
31#[doc(hidden)]
32pub fn observe(request: &crate::request::Request, response: &crate::response::Response, ip: std::net::IpAddr) {
33    if !enabled() {
34        return;
35    }
36    let header = |name: &str| request.header(name).map(|h| h.value.as_ref()).filter(|v: &&str| !v.is_empty());
37    let content_type = response
38        .content_range_list
39        .first()
40        .map(|c| c.content_type.as_str())
41        .or_else(|| response.headers.iter().find(|h| h.name.eq_ignore_ascii_case("content-type")).map(|h| h.value.as_ref()));
42    record::record(&record::Hit {
43        ip,
44        host: header("host").unwrap_or(""),
45        user_agent: header("user-agent"),
46        referer: header("referer"),
47        method: &request.method,
48        path: &request.request_uri,
49        status: response.status_code,
50        content_type,
51        now: record::now(),
52    });
53}
54
55/// Start counting, with the saved counts loaded and saved again every five
56/// minutes (when `persist` gives a file). The `rws` binary calls this when
57/// the admin API is on and `[traffic] enabled` isn't `false`.
58pub fn start() {
59    if enabled() {
60        return;
61    }
62    enable();
63    let Some(path) = config::persist_path() else { return };
64    persist::load(&path);
65    let _ = std::thread::Builder::new().name("rws-traffic".to_string()).spawn(move || loop {
66        std::thread::sleep(std::time::Duration::from_secs(300));
67        if let Err(e) = persist::save(&path) {
68            eprintln!("traffic: can't save {}: {e}", path.display());
69        }
70    });
71}
72
73/// Save the counts now (at shutdown).
74pub fn save() {
75    if let (true, Some(path)) = (enabled(), config::persist_path()) {
76        if let Err(e) = persist::save(&path) {
77            eprintln!("traffic: can't save {}: {e}", path.display());
78        }
79    }
80}
81
82#[cfg(test)]
83mod tests;
rws_proxy/traffic/mod.rs

rws_proxy/traffic/
mod.rs
