zc2 0.0.48

P2P compute broker with yen-denominated billing, WAL, and broker mesh support
//! Spaces lease preemption (ADR ZC-0025 amendment): when the owner comes
//! back, or an offer that pays more cannot fit because leases hold the
//! budget, the lowest-value running Space is stopped.
//!
//! Stopping is the hub's own lease lifecycle: the container is stopped and
//! removed and the session stops heartbeating, so the lease lapses at its
//! deadline and the hub re-leases the batch (the hub has no release/abandon
//! route for a lease, and `max_runtime_s` bounds the wait). Work that is about
//! to finish is left to finish, within a grace period, and the owner's own
//! work is never a Space lease at all.
//!
//! Pure: the leases, the clock and the ask are arguments.

use crate::broker::revenue::Need;
use std::time::Duration;

/// `ZAKURO_SPACES_PREEMPT_GRACE_SECS`: a Space whose leases all finish within
/// this long is left to finish (default 120).
pub fn grace() -> Duration {
    Duration::from_secs(
        std::env::var("ZAKURO_SPACES_PREEMPT_GRACE_SECS")
            .ok()
            .and_then(|v| v.trim().parse().ok())
            .unwrap_or(120),
    )
}

/// One lease of a running Space, as its session sees it.
#[derive(Debug, Clone, PartialEq)]
pub struct LeaseInfo {
    /// Unix seconds when the batch was handed to the container; `None` while
    /// it still waits in the queue.
    pub started_epoch: Option<i64>,
    pub est_runtime_s: u64,
    pub max_runtime_s: u64,
    /// What the hub pays per hour, when it said (an absent price is unknown,
    /// never zero).
    pub price_per_hour: Option<f64>,
    pub need: Need,
}

/// A Space (one container) and what stopping it would cost.
#[derive(Debug, Clone, PartialEq)]
pub struct Candidate {
    pub key: String,
    pub leases: usize,
    /// `None` when any of its leases has no known price.
    pub value_per_hour: Option<f64>,
    /// Until its last lease is expected to finish; `None` when unknown.
    pub remaining_secs: Option<u64>,
    pub need: Need,
    /// How long it has been running (work lost when stopped).
    pub age_secs: u64,
}

pub fn candidate(key: &str, infos: &[LeaseInfo], now_epoch: i64) -> Candidate {
    let expected = |i: &LeaseInfo| -> Option<u64> {
        let total = if i.est_runtime_s > 0 {
            i.est_runtime_s
        } else {
            i.max_runtime_s
        };
        if total == 0 {
            return None;
        }
        let elapsed = i.started_epoch.map_or(0, |s| (now_epoch - s).max(0) as u64);
        Some(total.saturating_sub(elapsed))
    };
    let mut remaining = Some(0u64);
    let mut value = Some(0f64);
    let mut need = Need {
        cpus: 0.0,
        memory_bytes: 0,
        gpus: 0,
    };
    let mut age = 0u64;
    for i in infos {
        remaining = match (remaining, expected(i)) {
            (Some(a), Some(b)) => Some(a.max(b)),
            _ => None,
        };
        value = match (value, i.price_per_hour) {
            (Some(a), Some(b)) => Some(a.max(b)),
            _ => None,
        };
        need.cpus = need.cpus.max(i.need.cpus);
        need.memory_bytes = need.memory_bytes.max(i.need.memory_bytes);
        need.gpus = need.gpus.max(i.need.gpus);
        if let Some(s) = i.started_epoch {
            age = age.max((now_epoch - s).max(0) as u64);
        }
    }
    Candidate {
        key: key.to_string(),
        leases: infos.len(),
        // An idle container holds budget and pays nothing: the first to go.
        value_per_hour: if infos.is_empty() { Some(0.0) } else { value },
        remaining_secs: if infos.is_empty() { None } else { remaining },
        need,
        age_secs: age,
    }
}

fn finishing(c: &Candidate, grace: Duration) -> bool {
    c.remaining_secs.is_some_and(|r| r <= grace.as_secs())
}

/// The owner is back: every Space that is not about to finish.
pub fn pick_owner_active(cands: &[Candidate], grace: Duration) -> Vec<String> {
    cands
        .iter()
        .filter(|c| !finishing(c, grace))
        .map(|c| c.key.clone())
        .collect()
}

fn fits(want: &Need, left: (u64, u64)) -> bool {
    ((want.cpus * 1000.0).round() as u64) <= left.0 && want.memory_bytes <= left.1
}

/// An offer paying `offered` yen/hour does not fit `left` (cpu millis, memory
/// bytes): the cheapest Spaces that pay less, youngest first (least work
/// lost), until it does. Nothing when even all of them would not make room, or
/// when it fits already.
pub fn pick_better_offer(
    cands: &[Candidate],
    grace: Duration,
    want: &Need,
    offered: f64,
    left: (u64, u64),
) -> Vec<String> {
    if fits(want, left) {
        return Vec::new();
    }
    let mut eligible: Vec<&Candidate> = cands
        .iter()
        .filter(|c| !finishing(c, grace))
        .filter(|c| c.value_per_hour.is_some_and(|v| v < offered))
        .collect();
    eligible.sort_by(|a, b| {
        a.value_per_hour
            .partial_cmp(&b.value_per_hour)
            .unwrap_or(std::cmp::Ordering::Equal)
            .then(a.age_secs.cmp(&b.age_secs))
            .then(a.key.cmp(&b.key))
    });
    let mut free = left;
    let mut chosen = Vec::new();
    for c in eligible {
        free.0 += (c.need.cpus * 1000.0).round() as u64;
        free.1 += c.need.memory_bytes;
        chosen.push(c.key.clone());
        if fits(want, free) {
            return chosen;
        }
    }
    Vec::new()
}

#[cfg(test)]
mod tests {
    use super::*;

    fn need(cpus: f64, gib: u64) -> Need {
        Need {
            cpus,
            memory_bytes: gib << 30,
            gpus: 0,
        }
    }

    fn lease(started: Option<i64>, est: u64, price: Option<f64>, n: Need) -> LeaseInfo {
        LeaseInfo {
            started_epoch: started,
            est_runtime_s: est,
            max_runtime_s: est * 2,
            price_per_hour: price,
            need: n,
        }
    }

    const NOW: i64 = 10_000;
    const G: Duration = Duration::from_secs(120);

    fn cand(key: &str, started: i64, est: u64, price: Option<f64>, n: Need) -> Candidate {
        candidate(key, &[lease(Some(started), est, price, n)], NOW)
    }

    #[test]
    fn a_space_about_to_finish_is_left_alone_and_the_rest_stop_when_the_owner_returns() {
        let c = [
            cand("nearly-done", NOW - 580, 600, Some(5.0), need(1.0, 1)), // 20 s left
            cand("fresh", NOW - 10, 600, Some(5.0), need(1.0, 1)),
            cand("exact-grace", NOW - 480, 600, Some(5.0), need(1.0, 1)), // 120 s left
        ];
        assert_eq!(pick_owner_active(&c, G), vec!["fresh".to_string()]);
        assert_eq!(pick_owner_active(&c, Duration::ZERO).len(), 3);
    }

    #[test]
    fn an_unknown_runtime_is_never_treated_as_about_to_finish() {
        let c = candidate("x", &[lease(Some(NOW - 5), 0, None, need(1.0, 1))], NOW);
        // est 0 and max 0 = unknown.
        let mut i = lease(Some(NOW - 5), 0, None, need(1.0, 1));
        i.max_runtime_s = 0;
        let unknown = candidate("u", &[i], NOW);
        assert_eq!(unknown.remaining_secs, None);
        assert_eq!(pick_owner_active(&[unknown], G), vec!["u".to_string()]);
        // est 0 falls back to max_runtime_s (0 * 2 = 0 here, so also unknown).
        assert_eq!(c.remaining_secs, None);
    }

    #[test]
    fn a_container_serving_several_leases_is_done_when_its_last_one_is() {
        let c = candidate(
            "s",
            &[
                lease(Some(NOW - 590), 600, Some(5.0), need(1.0, 1)),
                lease(None, 600, Some(5.0), need(1.0, 1)),
            ],
            NOW,
        );
        assert_eq!(
            c.remaining_secs,
            Some(600),
            "the queued one has not started"
        );
        assert_eq!(c.leases, 2);
        assert_eq!(pick_owner_active(&[c], G), vec!["s".to_string()]);
    }

    #[test]
    fn a_better_offer_stops_the_cheapest_youngest_space_that_makes_it_fit() {
        let c = [
            cand("pricey", NOW - 50, 3600, Some(90.0), need(2.0, 4)),
            cand("cheap-old", NOW - 900, 3600, Some(10.0), need(2.0, 4)),
            cand("cheap-young", NOW - 30, 3600, Some(10.0), need(2.0, 4)),
        ];
        let want = need(2.0, 4);
        // Nothing left; the offer pays 40: only the two cheap ones may go,
        // the younger first, and one is enough.
        let got = pick_better_offer(&c, G, &want, 40.0, (0, 0));
        assert_eq!(got, vec!["cheap-young".to_string()]);
        // Needing four cores takes both cheap ones, never the pricey one.
        let got = pick_better_offer(&c, G, &need(4.0, 4), 40.0, (0, 0));
        assert_eq!(
            got,
            vec!["cheap-young".to_string(), "cheap-old".to_string()]
        );
    }

    #[test]
    fn nothing_is_stopped_when_it_would_not_be_enough_or_the_offer_does_not_pay_more() {
        let c = [cand("a", NOW - 30, 3600, Some(10.0), need(1.0, 1))];
        // Wants 4 cores; one 1-core Space cannot make room.
        assert!(pick_better_offer(&c, G, &need(4.0, 4), 40.0, (0, 0)).is_empty());
        // Pays no more than the lease.
        assert!(pick_better_offer(&c, G, &need(1.0, 1), 10.0, (0, 0)).is_empty());
        // An unpriced lease has an unknown value: never stopped for an offer.
        let u = [cand("u", NOW - 30, 3600, None, need(1.0, 1))];
        assert!(pick_better_offer(&u, G, &need(1.0, 1), 1e6, (0, 0)).is_empty());
        // Fits already.
        assert!(pick_better_offer(&c, G, &need(1.0, 1), 40.0, (2000, 2 << 30)).is_empty());
        // Finishing soon: left alone.
        let f = [cand("f", NOW - 3550, 3600, Some(1.0), need(1.0, 1))];
        assert!(pick_better_offer(&f, G, &need(1.0, 1), 40.0, (0, 0)).is_empty());
    }

    #[test]
    fn free_room_counts_towards_the_fit() {
        let c = [cand("a", NOW - 30, 3600, Some(10.0), need(2.0, 4))];
        // Wants 3 cores, 1 is free: stopping the 2-core Space is enough.
        let got = pick_better_offer(&c, G, &need(3.0, 2), 40.0, (1000, 8 << 30));
        assert_eq!(got, vec!["a".to_string()]);
    }
}