zc2 0.0.39

P2P compute broker with credit-based billing, WAL, and broker mesh support
//! Network probes: RTT to the hub (`/api/health`) and peers (`/health`), blob
//! throughput from the hub.
use super::budget::Budget;
use super::report::NetMetrics;
use std::io::Read;
use std::time::{Duration, Instant};

const BLOB_BYTES: u64 = 16 * 1024 * 1024;
/// The hub serves its health check under `/api`; a broker peer at the root.
const HUB_HEALTH: &str = "/api/health";
const PEER_HEALTH: &str = "/health";
/// At most this many peers are probed.
const MAX_PEERS: usize = 3;

fn agent_get(url: &str, timeout: Duration) -> Result<ureq::http::Response<ureq::Body>, String> {
    ureq::get(url)
        .config()
        .timeout_global(Some(timeout))
        .http_status_as_error(false)
        .build()
        .call()
        .map_err(|e| e.to_string())
}

fn median(mut v: Vec<f64>) -> Option<f64> {
    if v.is_empty() {
        return None;
    }
    v.sort_by(|a, b| a.total_cmp(b));
    let m = v.len() / 2;
    Some(if v.len() % 2 == 1 {
        v[m]
    } else {
        (v[m - 1] + v[m]) / 2.0
    })
}

/// Median RTT of `health_path` over up to `max` samples (5 when the budget is short).
fn rtt_ms(base: &str, health_path: &str, b: &Budget) -> Option<f64> {
    let max = if b.remaining() < Duration::from_secs(5) {
        5
    } else {
        10
    };
    let url = format!("{}{health_path}", base.trim_end_matches('/'));
    let mut samples = Vec::new();
    for _ in 0..max {
        if b.expired() {
            break;
        }
        let t = Instant::now();
        match agent_get(&url, Duration::from_secs(2)) {
            Ok(r) if r.status().as_u16() == 200 => samples.push(t.elapsed().as_secs_f64() * 1000.0),
            _ => break, // unreachable or unhealthy: don't burn the budget retrying
        }
    }
    median(samples)
}

/// Mbit/s downloading the hub's bench blob; `None` unless it answers 200.
fn throughput_mbps(base: &str, b: &Budget) -> Option<f64> {
    if b.expired() {
        return None;
    }
    let url = format!(
        "{}/api/broker/bench/blob?bytes={BLOB_BYTES}",
        base.trim_end_matches('/')
    );
    let timeout = b
        .remaining()
        .clamp(Duration::from_millis(500), Duration::from_secs(20));
    let t = Instant::now();
    let resp = agent_get(&url, timeout).ok()?;
    if resp.status().as_u16() != 200 {
        return None;
    }
    let mut body = resp.into_body();
    let n = std::io::copy(&mut body.as_reader().take(BLOB_BYTES), &mut std::io::sink()).ok()?;
    let secs = t.elapsed().as_secs_f64();
    (n == BLOB_BYTES && secs > 0.0).then(|| n as f64 * 8.0 / secs / 1e6)
}

pub fn probe(hub_url: Option<&str>, peers: &[String], b: &Budget) -> NetMetrics {
    let hub_rtt_ms = hub_url.and_then(|u| rtt_ms(u, HUB_HEALTH, &b.slice(0.4)));
    // Throughput does not depend on the health route answering.
    let mbps = hub_url.and_then(|u| throughput_mbps(u, &b.slice(0.6)));
    let peers = &peers[..peers.len().min(MAX_PEERS)];
    // Each peer gets an equal share of what is left now, not of what the
    // previous peer left over.
    let per_peer_time = b.remaining() / peers.len().max(1) as u32;
    let per_peer: Vec<f64> = peers
        .iter()
        .filter_map(|p| {
            rtt_ms(
                p,
                PEER_HEALTH,
                &Budget::new(per_peer_time.min(b.remaining()), b.cancel_flag()),
            )
        })
        .collect();
    NetMetrics {
        hub_rtt_ms,
        peer_rtt_ms_p50: median(per_peer),
        mbps,
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::sync::atomic::AtomicBool;
    use std::sync::Arc;

    #[test]
    fn unreachable_targets_give_none_not_errors() {
        let m = probe(
            Some("http://127.0.0.1:1"),
            &["http://127.0.0.1:2".into()],
            &Budget::new(Duration::from_secs(3), Arc::new(AtomicBool::new(false))),
        );
        assert_eq!(m.hub_rtt_ms, None);
        assert_eq!(m.peer_rtt_ms_p50, None);
    }

    #[test]
    fn a_short_blob_body_gives_no_throughput() {
        let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = l.local_addr().unwrap();
        std::thread::spawn(move || {
            for s in l.incoming().flatten() {
                use std::io::{Read, Write};
                let mut s = s;
                let mut buf = [0u8; 1024];
                let _ = s.read(&mut buf);
                let _ = s.write_all(
                    b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nhello",
                );
            }
        });
        let b = Budget::new(Duration::from_secs(5), Arc::new(AtomicBool::new(false)));
        assert_eq!(throughput_mbps(&format!("http://{addr}"), &b), None);
    }

    #[test]
    fn rtt_against_a_local_health_server() {
        let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = l.local_addr().unwrap();
        std::thread::spawn(move || {
            for s in l.incoming().flatten() {
                use std::io::{Read, Write};
                let mut s = s;
                let mut buf = [0u8; 1024];
                let _ = s.read(&mut buf);
                let _ = s.write_all(
                    b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
                );
            }
        });
        let m = probe(
            Some(&format!("http://{addr}")),
            &[],
            &Budget::new(Duration::from_secs(3), Arc::new(AtomicBool::new(false))),
        );
        assert!(m.hub_rtt_ms.unwrap() >= 0.0);
    }

    /// Serves the listed `(path, status)` pairs, recording every request path.
    /// A path "/api/broker/bench/blob" answers with 16 MiB.
    fn stub(
        routes: Vec<(&'static str, u16)>,
    ) -> (String, std::sync::Arc<std::sync::Mutex<Vec<String>>>) {
        let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = l.local_addr().unwrap();
        let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
        let seen2 = seen.clone();
        std::thread::spawn(move || {
            for s in l.incoming().flatten() {
                use std::io::{Read, Write};
                let mut s = s;
                let mut buf = [0u8; 2048];
                let n = s.read(&mut buf).unwrap_or(0);
                let head = String::from_utf8_lossy(&buf[..n]).to_string();
                let path = head.split_whitespace().nth(1).unwrap_or("").to_string();
                let bare = path.split('?').next().unwrap_or("").to_string();
                seen2.lock().unwrap().push(bare.clone());
                let status = routes
                    .iter()
                    .find(|(p, _)| *p == bare)
                    .map(|r| r.1)
                    .unwrap_or(404);
                let body = if status == 200 && bare.ends_with("/blob") {
                    vec![0u8; BLOB_BYTES as usize]
                } else {
                    vec![]
                };
                let _ = write!(
                    s,
                    "HTTP/1.1 {status} X\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
                    body.len()
                );
                let _ = s.write_all(&body);
            }
        });
        (format!("http://{addr}"), seen)
    }

    fn budget(secs: u64) -> Budget {
        Budget::new(Duration::from_secs(secs), Arc::new(AtomicBool::new(false)))
    }

    #[test]
    fn the_hub_is_probed_at_api_health_and_peers_at_health() {
        let (hub, hub_seen) = stub(vec![("/api/health", 200)]);
        let (peer, peer_seen) = stub(vec![("/health", 200)]);
        let m = probe(Some(&hub), &[peer], &budget(5));
        assert!(m.hub_rtt_ms.is_some(), "hub served /api/health only");
        assert!(m.peer_rtt_ms_p50.is_some());
        assert!(hub_seen.lock().unwrap().iter().any(|p| p == "/api/health"));
        assert!(!hub_seen.lock().unwrap().iter().any(|p| p == "/health"));
        assert!(peer_seen.lock().unwrap().iter().all(|p| p == "/health"));
    }

    #[test]
    fn throughput_runs_even_when_the_health_route_fails() {
        let (hub, _) = stub(vec![("/api/broker/bench/blob", 200)]);
        let m = probe(Some(&hub), &[], &budget(10));
        assert_eq!(m.hub_rtt_ms, None);
        assert!(m.mbps.unwrap() > 0.0);
    }

    #[test]
    fn at_most_three_peers_are_probed() {
        let (a, a_seen) = stub(vec![("/health", 200)]);
        let (b, b_seen) = stub(vec![("/health", 200)]);
        let (c, c_seen) = stub(vec![("/health", 200)]);
        let (d, d_seen) = stub(vec![("/health", 200)]);
        probe(None, &[a, b, c, d], &budget(5));
        assert!(!a_seen.lock().unwrap().is_empty());
        assert!(!b_seen.lock().unwrap().is_empty());
        assert!(!c_seen.lock().unwrap().is_empty());
        assert!(
            d_seen.lock().unwrap().is_empty(),
            "the 4th peer is not probed"
        );
    }
}