rws_proxy/traffic/
mod.rs

1//! Traffic insights (18.15.0): who visits — people, AI agents, bots —
2//! worked out by rws from each request, for the admin UI's Traffic page.
3//!
4//! This first part is the classifier, [`classify`]: from the client's
5//! address, User-Agent and path it names what kind of client this is, using
6//! - the IP ranges Google, Bing, Apple, DuckDuckGo, OpenAI, Perplexity and
7//!   Common Crawl publish for their crawlers (a match is *verified*; a
8//!   User-Agent that claims one of them from elsewhere is *spoofed*);
9//! - [ai.robots.txt](https://github.com/ai-robots-txt/ai.robots.txt)'s AI
10//!   agents and [crawler-user-agents](https://github.com/monperrus/crawler-user-agents)'
11//!   bot patterns (both MIT, bundled, regenerated by `ci/gen_bot_lists.py`);
12//! - requests only scanners make (`/.env`, `/.git/config`, `/wp-login.php`…).
13//!
14//! A browser User-Agent that matches none of them is a [`Kind::Browser`]:
15//! whether it's a person is decided later, by whether that client loads the
16//! page's assets as a browser does.
17
18pub mod apps;
19mod classify;
20pub mod config;
21mod data;
22pub mod device;
23pub mod geo;
24pub mod hosting;
25pub mod owner;
26#[doc(hidden)]
27pub mod persist;
28pub mod record;
29pub mod script;
30#[cfg(feature = "model-postgres")]
31pub mod store;
32pub mod summary;
33
34pub use classify::{classify, Class, Group, Kind};
35pub use record::{enable, enabled};
36pub use script::TrafficScriptLayer;
37
38/// Count one finished request (called with the access log line, whatever
39/// the log format).
40#[doc(hidden)]
41pub fn observe(request: &crate::request::Request, response: &crate::response::Response, ip: std::net::IpAddr) {
42    if !enabled() {
43        return;
44    }
45    let header = |name: &str| request.header(name).map(|h| h.value.as_ref()).filter(|v: &&str| !v.is_empty());
46    let content_type = response
47        .content_range_list
48        .first()
49        .map(|c| c.content_type.as_str())
50        .or_else(|| response.headers.iter().find(|h| h.name.eq_ignore_ascii_case("content-type")).map(|h| h.value.as_ref()));
51    record::record(&record::Hit {
52        ip,
53        host: header("host").unwrap_or(""),
54        user_agent: header("user-agent"),
55        referer: header("referer"),
56        method: &request.method,
57        path: &request.request_uri,
58        status: response.status_code,
59        content_type,
60        headers: &request.headers,
61        now: record::now(),
62    });
63}
64
65/// Start counting, with the saved counts loaded and saved again every five
66/// minutes (when `persist` gives a file). The `rws` binary calls this when
67/// the admin API is on and `[traffic] enabled` isn't `false`.
68pub fn start() {
69    if enabled() {
70        return;
71    }
72    enable();
73    // Decode the country table now rather than on the first visitor's request.
74    let _ = std::thread::Builder::new().name("rws-traffic-geo".to_string()).spawn(|| {
75        geo::table();
76    });
77    start_store();
78    // Before loading the saved counts, so the configured sites keep slots.
79    #[cfg(not(target_arch = "wasm32"))]
80    configure_sites(&std::fs::read_to_string(crate::proxy_config::config_file_path()).unwrap_or_default());
81    let Some(path) = config::persist_path() else { return };
82    persist::load(&path);
83    let _ = std::thread::Builder::new().name("rws-traffic".to_string()).spawn(move || loop {
84        std::thread::sleep(std::time::Duration::from_secs(300));
85        if let Err(e) = persist::save(&path) {
86            eprintln!("traffic: can't save {}: {e}", path.display());
87        }
88    });
89}
90
91/// Count the sites `toml`'s routes match (`[[route]] match.host`) on
92/// their own and list them first ([`record::set_configured_hosts`]).
93#[cfg(not(target_arch = "wasm32"))]
94pub fn configure_sites(toml: &str) {
95    let config = crate::proxy_config::ProxyConfig::from_str(toml);
96    record::set_configured_hosts(config.routes.iter().filter_map(|r| r.match_.host.clone()));
97    // `[traffic] apps`, `owner_networks` and `ignore_referrers` follow the file too.
98    apps::set_apps(&config::apps());
99    record::set_ignored_referrers(&config::ignore_referrers());
100    owner::set_networks(&config::owner_networks());
101}
102
103/// Connect to `[traffic] database_url_env`'s PostgreSQL, when set.
104fn start_store() {
105    let Some(url) = config::database_url() else { return };
106    #[cfg(feature = "model-postgres")]
107    match store::start(&url) {
108        Ok(()) => println!("traffic: writing page views to PostgreSQL."),
109        Err(e) => eprintln!("traffic: can't use the traffic database: {e}; counting without it"),
110    }
111    #[cfg(not(feature = "model-postgres"))]
112    {
113        let _ = url;
114        eprintln!("traffic: [traffic] database_url_env is set, but this rws is built without the model-postgres feature; counting without it");
115    }
116}
117
118/// Save the counts now (at shutdown).
119pub fn save() {
120    #[cfg(feature = "model-postgres")]
121    store::stop();
122    if let (true, Some(path)) = (enabled(), config::persist_path()) {
123        if let Err(e) = persist::save(&path) {
124            eprintln!("traffic: can't save {}: {e}", path.display());
125        }
126    }
127}
128
129#[cfg(test)]
130mod tests;
rws_proxy/traffic/mod.rs

rws_proxy/traffic/
mod.rs
