rws-proxy 18.36.0

//! The Traffic page's data: totals for a period and the one before it, a
//! series (hourly up to 48 hours, daily beyond), and the top lists.

use std::collections::BTreeMap;

use super::record::{snapshot, Day, Sessions, Top, BEHAVIOUR, DIMS, EXTRA, GROUPS};

fn quote(s: &str) -> String {
    let mut out = String::with_capacity(s.len() + 2);
    out.push('"');
    for c in s.chars() {
        match c {
            '"' => out.push_str("\\\""),
            '\\' => out.push_str("\\\\"),
            c if (c as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", c as u32)),
            c => out.push(c),
        }
    }
    out.push('"');
    out
}

/// Whether a `host/path` key is `host`'s (any key when no host is given;
/// the shared `(other)` entry only then).
fn on_host(key: &str, host: Option<&str>) -> bool {
    match host {
        None => true,
        Some(h) => key.len() > h.len() && key.as_bytes()[h.len()] == b'/' && key[..h.len()].eq_ignore_ascii_case(h),
    }
}

fn pairs(top: &Top, n: usize, host: Option<&str>) -> String {
    let mut filtered = Top::default();
    for (key, count) in &top.0 {
        if on_host(key, host) {
            filtered.0.insert(key.clone(), *count);
        }
    }
    let rows: Vec<String> = filtered.sorted(n).iter().map(|(k, v)| format!("[{},{v}]", quote(k))).collect();
    format!("[{}]", rows.join(","))
}

/// An audience/source list (`host\tvalue` keys) as `[[value, n], …]`: one
/// host's, or all hosts' added up.
fn dim(top: &Top, n: usize, host: Option<&str>) -> String {
    let mut merged = Top::default();
    for (key, count) in &top.0 {
        let (h, value) = key.split_once('\t').unwrap_or(("", key.as_str()));
        if host.is_none_or(|wanted| h.eq_ignore_ascii_case(wanted)) || (host.is_none() && key == "(other)") {
            *merged.0.entry(value.to_string()).or_default() += count;
        }
    }
    let rows: Vec<String> = merged.sorted(n).iter().map(|(k, v)| format!("[{},{v}]", quote(k))).collect();
    format!("[{}]", rows.join(","))
}

/// A per-site list (`host\t…` keys) without the host: one site's, or all
/// sites' added up, together with `legacy` (the all-sites list days saved
/// before per-site lists have).
fn site_list(site: &Top, legacy: &Top, host: Option<&str>) -> Top {
    let mut out = Top::default();
    for (key, n) in &site.0 {
        let Some((h, rest)) = key.split_once('\t') else {
            // The overflow entry: no site of its own.
            if host.is_none() {
                *out.0.entry(key.clone()).or_default() += n;
            }
            continue;
        };
        if host.is_none_or(|wanted| h.eq_ignore_ascii_case(wanted)) {
            *out.0.entry(rest.to_string()).or_default() += n;
        }
    }
    if host.is_none() {
        out.merge(legacy);
    }
    out
}

/// A day's unique visitors per group: one site's, or all sites'.
fn day_visitors(day: &Day, host: Option<&str>) -> [u64; GROUPS] {
    let Some(wanted) = host else { return day.visitors };
    let mut out = [0u64; GROUPS];
    for (key, n) in &day.site_visitors.0 {
        if let Some((h, g)) = key.rsplit_once('\t') {
            if let (true, Ok(g @ 0..=4)) = (h.eq_ignore_ascii_case(wanted), g.parse::<usize>()) {
                out[g] += n;
            }
        }
    }
    out
}

/// The JSON for the last `hours` hours (1–720), optionally one host's.
pub fn json(hours: u64, host: Option<&str>, now: u64) -> String {
    let hours = hours.clamp(1, 24 * 90);
    let (buckets, days) = snapshot(now);
    let names = super::record::host_names();
    let configured = super::record::configured_hosts();
    let host_index = host.and_then(|h| names.iter().position(|n| n.eq_ignore_ascii_case(h))).map(|i| i as u8);
    let wanted = |h: u8| host.is_none() || Some(h) == host_index;
    let this_hour = now / 3600;
    let daily = hours > 48;
    // Daily views cover whole UTC days (today and the days before it), so
    // the per-day counts — visitors, sessions, lists — line up with them.
    let hours = if daily { hours.div_ceil(24) * 24 } else { hours };
    let start = if daily { (this_hour / 24 + 1).saturating_sub(hours / 24) * 24 } else { this_hour + 1 - hours };
    let previous_start = start.saturating_sub(hours);

    let mut totals = [[0u64; 2]; GROUPS];
    let mut previous = [[0u64; 2]; GROUPS];
    // Per bucket: requests per group, then people's page views.
    let mut series: BTreeMap<u64, [u64; GROUPS + 1]> = BTreeMap::new();
    // The period before, by the same bucket offsets (for comparing).
    let mut previous_series: BTreeMap<u64, [u64; GROUPS + 1]> = BTreeMap::new();
    let mut first_hour = None;
    for ((hour, h), counts) in &buckets {
        if !wanted(*h) {
            continue;
        }
        first_hour = Some(first_hour.map_or(*hour, |f: u64| f.min(*hour)));
        let target = if *hour >= start && *hour <= this_hour {
            Some(&mut totals)
        } else if *hour >= previous_start && *hour < start {
            Some(&mut previous)
        } else {
            None
        };
        if let Some(t) = target {
            for g in 0..GROUPS {
                t[g][0] += counts[g].requests;
                t[g][1] += counts[g].pages;
            }
        }
        let into = if *hour >= start && *hour <= this_hour {
            Some((&mut series, *hour))
        } else if *hour >= previous_start && *hour < start {
            Some((&mut previous_series, *hour + hours))
        } else {
            None
        };
        if let Some((map, at)) = into {
            let key = if daily { at / 24 * 24 } else { at };
            let point = map.entry(key).or_default();
            for g in 0..GROUPS {
                point[g] += counts[g].requests;
            }
            point[GROUPS] += counts[0].pages;
        }
    }
    // Every bucket of the period, empty ones included.
    let step = if daily { 24 } else { 1 };
    let mut t = if daily { start / 24 * 24 } else { start };
    while t <= this_hour {
        series.entry(t).or_default();
        previous_series.entry(t).or_default();
        t += step;
    }
    previous_series.retain(|k, _| series.contains_key(k));

    let start_day = start / 24;
    let previous_day = previous_start / 24;
    let mut lists = Day::default();
    let mut visitors = [0u64; GROUPS];
    let mut previous_visitors = [0u64; GROUPS];
    let mut sessions = Sessions::default();
    let mut previous_sessions = Sessions::default();
    let host_sessions = |d: &Day, into: &mut Sessions| {
        for (h, s) in &d.sessions {
            if host.is_none_or(|wanted| h.eq_ignore_ascii_case(wanted)) {
                into.add(s);
            }
        }
    };
    // People's visitors and sessions per day (current and previous period,
    // the previous keyed by the matching current day).
    let mut per_day: BTreeMap<u64, [u64; 6]> = BTreeMap::new();
    let period_days = hours.div_ceil(24).max(1);
    for day in start_day..=this_hour / 24 {
        per_day.entry(day).or_default();
    }
    for (day, d) in &days {
        let mut s = Sessions::default();
        host_sessions(d, &mut s);
        let (key, offset) = if *day >= start_day { (*day, 0) } else { (*day + period_days, 3) };
        if let Some(row) = per_day.get_mut(&key) {
            row[offset] += day_visitors(d, host)[0];
            row[offset + 1] += s.sessions;
            row[offset + 2] += s.bounces;
        }
        if *day >= start_day {
            host_sessions(d, &mut sessions);
        } else if *day >= previous_day {
            host_sessions(d, &mut previous_sessions);
        }
        let v = day_visitors(d, host);
        if *day >= start_day {
            lists.merge(d);
            for g in 0..GROUPS {
                visitors[g] += v[g];
            }
        } else if *day >= previous_day {
            for g in 0..GROUPS {
                previous_visitors[g] += v[g];
            }
        }
    }

    let group = |t: &[[u64; 2]; GROUPS], v: &[u64; GROUPS], g: usize| format!(r#"{{"requests":{},"pages":{},"visitors":{}}}"#, t[g][0], t[g][1], v[g]);
    let groups = |t: &[[u64; 2]; GROUPS], v: &[u64; GROUPS]| {
        format!(r#"{{"people":{},"ai":{},"bots":{},"apps":{},"owner":{}}}"#, group(t, v, 0), group(t, v, 1), group(t, v, 2), group(t, v, 3), group(t, v, 4))
    };
    let point = |(t, p): (&u64, &[u64; GROUPS + 1])| {
        format!(r#"{{"t":{},"people":{},"ai":{},"bots":{},"apps":{},"owner":{},"pages":{}}}"#, t * 3600, p[0], p[1], p[2], p[3], p[4], p[5])
    };
    let points: Vec<String> = series.iter().map(point).collect();
    let previous_points: Vec<String> = previous_series.iter().map(point).collect();
    let day_points: Vec<String> = per_day
        .iter()
        .map(|(d, r)| format!(r#"{{"t":{},"visitors":{},"sessions":{},"bounces":{},"previous_visitors":{},"previous_sessions":{},"previous_bounces":{}}}"#, d * 86_400, r[0], r[1], r[2], r[3], r[4], r[5]))
        .collect();
    let agents: Vec<String> = site_list(&lists.site_ai_agents, &lists.ai_agents, host)
        .sorted(25)
        .iter()
        .map(|(key, n)| {
            let mut parts = key.split('\t');
            let (name, operator, kind, verified) = (parts.next().unwrap_or(""), parts.next().unwrap_or(""), parts.next().unwrap_or(""), parts.next() == Some("true"));
            format!(r#"{{"name":{},"operator":{},"kind":{},"verified":{verified},"requests":{n}}}"#, quote(name), quote(operator), quote(kind))
        })
        .collect();
    let bots: Vec<String> = site_list(&lists.site_bots, &lists.bots, host)
        .sorted(25)
        .iter()
        .map(|(key, n)| {
            let (kind, name) = key.split_once('\t').unwrap_or((key, ""));
            format!(r#"{{"kind":{},"name":{},"requests":{n}}}"#, quote(kind), quote(name))
        })
        .collect();
    let dims: Vec<String> = DIMS.iter().enumerate().map(|(i, name)| format!("{}:{}", quote(name), dim(&lists.dims[i], if i == 0 { 50 } else { 25 }, host))).collect();
    // Time on page: seconds and reported views per page, as [page, avg, views].
    let mut on_page: Vec<(String, u64, u64)> = lists.behaviour[1]
        .0
        .iter()
        .filter(|(k, _)| on_host(k, host) && k.as_str() != "(other)")
        .map(|(k, views)| (k.clone(), lists.behaviour[0].0.get(k).copied().unwrap_or(0) / (*views).max(1), *views))
        .collect();
    on_page.sort_by(|a, b| b.2.cmp(&a.2).then_with(|| a.0.cmp(&b.0)));
    on_page.truncate(25);
    let on_page: Vec<String> = on_page.iter().map(|(k, avg, views)| format!("[{},{avg},{views}]", quote(k))).collect();
    let behaviour: Vec<String> = BEHAVIOUR
        .iter()
        .enumerate()
        .skip(2)
        .map(|(i, name)| format!("{}:{}", quote(name), dim(&lists.behaviour[i], 25, host)))
        .collect();
    let session_json = |s: &Sessions| format!(r#"{{"sessions":{},"bounces":{},"duration":{},"pages":{}}}"#, s.sessions, s.bounces, s.duration, s.pages);
    let geo = super::geo::table().map_or("null".to_string(), |t| quote(&t.source));
    let extra = |name: &str| &lists.extra[EXTRA.iter().position(|e| *e == name).expect("an EXTRA list")];
    // How sure rws is that the people are people.
    let tier = |name: &str| -> u64 { extra("site_tiers").0.iter().filter(|(k, _)| k.split_once('\t').is_some_and(|(h, t)| t == name && host.is_none_or(|w| h.eq_ignore_ascii_case(w)))).map(|(_, n)| *n).sum() };
    // The newest visits of this site (or all), newest first.
    let mut visits: Vec<&super::record::Visit> = days.values().flat_map(|d| d.visits.iter()).filter(|v| host.is_none_or(|w| v.host.eq_ignore_ascii_case(w))).collect();
    visits.sort_by(|a, b| b.end.cmp(&a.end).then(b.start.cmp(&a.start)));
    visits.truncate(100);
    let visits: Vec<String> = visits.iter().map(|v| super::persist::visit_json(v)).collect();
    let crawl = crawl_json(extra("site_crawl"), host);
    let new_lists = format!(
        r#""tiers":{{"verified":{},"page_only":{},"hosting":{}}},"apps":{{"users":{},"versions":{},"events":{}}},"probes":{{"paths":{},"sources":{}}},"crawl":[{}],"crawl_404":{},"entries":{},"downloads":{},"owner_pages":{},"visits":[{}]"#,
        tier("verified"),
        tier("page-only"),
        tier("hosting"),
        dim(extra("site_apps"), 25, host),
        dim(extra("app_versions"), 25, host),
        dim(extra("app_events"), 25, host),
        dim(extra("site_probes"), 25, host),
        dim(extra("probe_sources"), 25, host),
        crawl,
        pairs(extra("crawl_404"), 25, host),
        pairs(extra("site_entries"), 50, host),
        pairs(extra("downloads"), 25, host),
        pairs(extra("owner_pages"), 25, host),
        visits.join(","),
    );
    format!(
        r#"{{"hours":{hours},"bucket":"{}","since":{},"hosts":[{}],"configured":[{}],"host":{},"totals":{},"previous":{},"series":[{}],"people_pages":{},"ai_pages":{},"ai_agents":[{}],"bots":[{}],"referrers":{},"not_found":{},{},"geo_source":{geo},"sessions":{},"previous_sessions":{},"entry_pages":{},"exit_pages":{},"previous_series":[{}],"days":[{}],"script":{},"store":{},"time_on_page":[{}],{},{new_lists}}}"#,
        if daily { "day" } else { "hour" },
        first_hour.map_or("null".to_string(), |h| (h * 3600).to_string()),
        site_order(&names, &configured, &buckets).iter().map(|n| quote(n)).collect::<Vec<_>>().join(","),
        configured.iter().map(|n| quote(n)).collect::<Vec<_>>().join(","),
        host.map_or("null".to_string(), quote),
        groups(&totals, &visitors),
        groups(&previous, &previous_visitors),
        points.join(","),
        pairs(&lists.people_pages, 25, host),
        pairs(&lists.ai_pages, 25, host),
        agents.join(","),
        bots.join(","),
        pairs(&site_list(&lists.site_referrers, &lists.referrers, host), 25, None),
        pairs(&lists.not_found, 25, host),
        dims.join(","),
        session_json(&sessions),
        session_json(&previous_sessions),
        pairs(&lists.entry_pages, 25, host),
        pairs(&lists.exit_pages, 25, host),
        previous_points.join(","),
        day_points.join(","),
        super::config::script(),
        store_on(),
        on_page.join(","),
        behaviour.join(","),
    )
}

/// Named crawlers' page fetches and 404s (`host\tname\tpages|404`), as
/// `{"name", "pages", "not_found"}`, the busiest first.
fn crawl_json(top: &Top, host: Option<&str>) -> String {
    let mut by: BTreeMap<String, [u64; 2]> = BTreeMap::new();
    for (key, n) in &top.0 {
        let mut parts = key.split('\t');
        let (Some(h), Some(name), Some(what)) = (parts.next(), parts.next(), parts.next()) else { continue };
        if host.is_some_and(|w| !h.eq_ignore_ascii_case(w)) {
            continue;
        }
        by.entry(name.to_string()).or_default()[(what == "404") as usize] += n;
    }
    let mut rows: Vec<(String, [u64; 2])> = by.into_iter().collect();
    rows.sort_by(|a, b| (b.1[0] + b.1[1]).cmp(&(a.1[0] + a.1[1])).then(a.0.cmp(&b.0)));
    rows.truncate(25);
    rows.iter().map(|(name, c)| format!(r#"{{"name":{},"pages":{},"not_found":{}}}"#, quote(name), c[0], c[1])).collect::<Vec<_>>().join(",")
}

/// The sites to pick from: the configured ones first (all of them, the
/// busiest first), then other hosts with counted traffic (this machine's
/// own calls, which aren't counted, still register their Host), busiest
/// first, and `other` last.
fn site_order(names: &[String], configured: &[String], buckets: &BTreeMap<(u64, u8), [super::record::Counts; GROUPS]>) -> Vec<String> {
    let mut requests = vec![0u64; names.len()];
    for ((_, h), c) in buckets {
        if let Some(r) = requests.get_mut(*h as usize) {
            *r += c.iter().map(|g| g.requests).sum::<u64>();
        }
    }
    let traffic = |name: &str| names.iter().position(|n| n == name).map_or(0, |i| requests[i]);
    let mut sites: Vec<(bool, u64, &str)> = configured.iter().map(|c| (false, traffic(c), c.as_str())).collect();
    for name in names {
        if name != "other" && configured.binary_search(name).is_err() && traffic(name) > 0 {
            sites.push((true, traffic(name), name));
        }
    }
    sites.sort_by(|a, b| a.0.cmp(&b.0).then(b.1.cmp(&a.1)).then(a.2.cmp(b.2)));
    let mut out: Vec<String> = sites.into_iter().map(|(_, _, n)| n.to_string()).collect();
    if traffic("other") > 0 {
        out.push("other".to_string());
    }
    out
}

/// The live view: people on the site in the last 5 minutes, page views per
/// minute for the last 30, and what the active ones read and where they
/// are; optionally one host's.
pub fn live_json(host: Option<&str>, now: u64) -> String {
    use std::collections::{HashMap, HashSet};
    let prefix = host.map(|h| format!("{}/", h.to_ascii_lowercase()));
    let views: Vec<_> = super::record::recent(now).into_iter().filter(|r| prefix.as_ref().is_none_or(|p| r.page.starts_with(p.as_str()))).collect();
    let this_minute = now / 60;
    let mut minutes = [0u64; 30];
    let active_since = now.saturating_sub(5 * 60);
    let mut active: HashSet<u64> = HashSet::new();
    let mut pages = Top::default();
    let mut countries: HashMap<u64, String> = HashMap::new();
    for r in &views {
        let ago = this_minute.saturating_sub(r.at / 60) as usize;
        if ago < 30 {
            minutes[29 - ago] += 1;
        }
        if r.at >= active_since {
            active.insert(r.visitor);
            *pages.0.entry(r.page.clone()).or_default() += 1;
            let country = r.country.map_or("(unknown)".to_string(), |c| String::from_utf8_lossy(&c).into_owned());
            countries.insert(r.visitor, country);
        }
    }
    let mut by_country = Top::default();
    for c in countries.values() {
        *by_country.0.entry(c.clone()).or_default() += 1;
    }
    let series: Vec<String> = minutes.iter().enumerate().map(|(i, n)| format!(r#"{{"t":{},"views":{n}}}"#, (this_minute + 1 + i as u64 - 30) * 60)).collect();
    format!(
        r#"{{"now":{now},"host":{},"visitors":{},"views_5m":{},"series":[{}],"pages":{},"countries":{}}}"#,
        host.map_or("null".to_string(), quote),
        active.len(),
        views.iter().filter(|r| r.at >= active_since).count(),
        series.join(","),
        pairs(&pages, 10, None),
        pairs(&by_country, 10, None),
    )
}

fn store_on() -> bool {
    #[cfg(feature = "model-postgres")]
    return super::store::enabled();
    #[cfg(not(feature = "model-postgres"))]
    false
}

/// Days since the epoch of `YYYY-MM-DD`.
#[doc(hidden)]
pub fn parse_day(text: &str) -> Option<u64> {
    let mut parts = text.trim().splitn(3, '-');
    let (y, m, d): (i64, i64, i64) = (parts.next()?.parse().ok()?, parts.next()?.parse().ok()?, parts.next()?.parse().ok()?);
    if !(1970..=9999).contains(&y) || !(1..=12).contains(&m) || !(1..=31).contains(&d) {
        return None;
    }
    // Howard Hinnant's days_from_civil.
    let y = if m <= 2 { y - 1 } else { y };
    let era = y.div_euclid(400);
    let yoe = y - era * 400;
    let mp = (m + 9) % 12;
    let doy = (153 * mp + 2) / 5 + d - 1;
    let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
    let days = era * 146_097 + doe - 719_468;
    (days >= 0).then_some(days as u64)
}

/// The `hours` and `now` for the summary of days `from`..=`to` (UTC, as
/// `YYYY-MM-DD`): the period ends with `to`, or now if `to` is today.
pub fn day_range(from: &str, to: &str, now: u64) -> Result<(u64, u64), String> {
    let today = now / 86_400;
    let to = if to.is_empty() { today } else { parse_day(to).ok_or(format!("to: expected YYYY-MM-DD, got {to:?}"))? };
    let from = parse_day(from).ok_or(format!("from: expected YYYY-MM-DD, got {from:?}"))?;
    if from > to {
        return Err("from must be on or before to".to_string());
    }
    let end = if to >= today { now } else { (to + 1) * 86_400 - 1 };
    let hours = (end / 3600 + 1).saturating_sub(from * 24).clamp(1, 24 * 90);
    Ok((hours, end))
}