hearth-core 0.3.2

Deterministic model residency: pin models warm, and tell the truth about which ones are.
Documentation
//! VRAM accounting. Refuse to overcommit instead of thrashing.
//!
//! The failure this prevents is quiet and expensive. Declare four 20GB models
//! on a 48GB card and nothing errors — the runtime simply loads, evicts,
//! loads, evicts, forever. Every request pays a cold start, and from the
//! outside it looks like the models got slow. Nobody sees an out-of-memory,
//! because technically nothing ever ran out of memory: it just never kept
//! anything.
//!
//! A card has a fixed size. That is arithmetic, and arithmetic can be checked
//! at declare time, before a single weight is read from disk. So hearth
//! refuses the fifth model and says exactly how much it was over by, which is
//! a sentence an operator can act on. "The models got slow" is not.
//!
//! ## Headroom is not pessimism
//!
//! Weights are not the whole cost. KV cache scales with context length and
//! parallel slots, the CUDA context itself costs hundreds of megabytes, and
//! fragmentation is real on a card that has been up for weeks. Planning to
//! 100% of VRAM means the last model loads and then the first long context
//! kills it. The reserve exists so a working fleet stays working.

use serde::{Deserialize, Serialize};

/// A model we intend to keep resident, and what we expect it to cost.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Declared {
    pub model: String,
    /// Weights, in bytes. Measured if we have ever loaded it, estimated from
    /// the file otherwise.
    pub weights_bytes: u64,
    /// Expected KV cache at the configured context and parallelism. Separate
    /// from weights because it is the part that grows with how you USE the
    /// model, and the part people forget.
    pub kv_bytes: u64,
}

impl Declared {
    pub fn total_bytes(&self) -> u64 {
        self.weights_bytes.saturating_add(self.kv_bytes)
    }
}

/// The VRAM available and what we hold back.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct Budget {
    pub total_bytes: u64,
    /// Never planned into. Covers CUDA context, fragmentation, and the honest
    /// error in every size estimate here.
    pub reserve_bytes: u64,
}

pub const GIB: u64 = 1024 * 1024 * 1024;

impl Budget {
    /// A budget with a proportional reserve, floored so small cards are not
    /// left with a reserve too thin to cover a CUDA context.
    pub fn with_reserve_pct(total_bytes: u64, pct: u8) -> Budget {
        let pct = pct.min(100) as u64;
        let proportional = total_bytes / 100 * pct;
        Budget {
            total_bytes,
            reserve_bytes: proportional.max(GIB).min(total_bytes),
        }
    }

    /// What is actually available to models.
    pub fn usable_bytes(&self) -> u64 {
        self.total_bytes.saturating_sub(self.reserve_bytes)
    }
}

/// The verdict on a declared set. Never a bare bool — the numbers are the
/// point, because "you are 14.2 GiB over" is actionable and "invalid
/// configuration" is not.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Plan {
    /// Models that fit, in the order given. Declaration order is priority
    /// order: the operator says what matters most, not a heuristic.
    pub admitted: Vec<String>,
    /// Models that do not fit, each with what it needed.
    pub rejected: Vec<Rejection>,
    pub committed_bytes: u64,
    pub usable_bytes: u64,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Rejection {
    pub model: String,
    pub needed_bytes: u64,
    /// How much room was left when we got to it.
    pub free_bytes: u64,
}

impl Rejection {
    pub fn short_bytes(&self) -> u64 {
        self.needed_bytes.saturating_sub(self.free_bytes)
    }
}

impl Plan {
    pub fn fits(&self) -> bool {
        self.rejected.is_empty()
    }

    pub fn headroom_bytes(&self) -> u64 {
        self.usable_bytes.saturating_sub(self.committed_bytes)
    }

    /// A report an operator can act on without reading the source.
    pub fn explain(&self) -> String {
        let mut out = format!(
            "{} of {} admitted, {:.1} GiB committed of {:.1} GiB usable",
            self.admitted.len(),
            self.admitted.len() + self.rejected.len(),
            gib(self.committed_bytes),
            gib(self.usable_bytes),
        );
        for r in &self.rejected {
            out.push_str(&format!(
                "\n  REJECTED {} — needs {:.1} GiB, {:.1} GiB free, short by {:.1} GiB",
                r.model,
                gib(r.needed_bytes),
                gib(r.free_bytes),
                gib(r.short_bytes()),
            ));
        }
        out
    }
}

pub fn gib(bytes: u64) -> f64 {
    bytes as f64 / GIB as f64
}

/// Decide what can be held resident, in declaration order.
///
/// First fit, not best fit, and deliberately so. Reordering to squeeze in one
/// more model would silently demote the model the operator listed first, and
/// on a serving box the first model is first because it matters most. A
/// planner that outsmarts the operator is a planner that surprises them at
/// 3am.
pub fn plan(budget: Budget, declared: &[Declared]) -> Plan {
    let usable = budget.usable_bytes();
    let mut committed: u64 = 0;
    let mut admitted = Vec::new();
    let mut rejected = Vec::new();

    for d in declared {
        let need = d.total_bytes();
        let free = usable.saturating_sub(committed);
        if need <= free {
            committed += need;
            admitted.push(d.model.clone());
        } else {
            rejected.push(Rejection {
                model: d.model.clone(),
                needed_bytes: need,
                free_bytes: free,
            });
        }
    }

    Plan {
        admitted,
        rejected,
        committed_bytes: committed,
        usable_bytes: usable,
    }
}

/// Read a declared roster out of JSON.
///
/// This lives in the core, not in the bindings, because parsing is a RULE and
/// rules belong where they can be tested. It got moved here after the Node
/// binding silently dropped an entire roster and then cheerfully reported
/// `fits: true` — JavaScript has no integers, so every size arrived as an f64
/// and `as_u64()` returned None for all of them. A `filter_map` swallowed the
/// lot. An empty roster trivially fits, so the answer was confident, instant
/// and completely wrong.
///
/// Two rules came out of that, and both are load-bearing:
///
///   1. Accept a float that is exactly an integer. A caller in a language
///      without integers is not making a mistake by sending 2.147e10.
///   2. NEVER skip an entry you could not read. Every failure is named and
///      returned, because a roster that silently shrinks produces a plan that
///      is right about the wrong question.
pub fn declared_from_json(v: &serde_json::Value) -> Result<Vec<Declared>, Vec<String>> {
    let Some(items) = v.as_array() else {
        return Err(vec!["expected an array of declared models".into()]);
    };

    let mut out = Vec::with_capacity(items.len());
    let mut errors = Vec::new();

    for (i, item) in items.iter().enumerate() {
        match one_declared(item) {
            Ok(d) => out.push(d),
            Err(why) => errors.push(format!("entry {i}: {why}")),
        }
    }

    if errors.is_empty() {
        Ok(out)
    } else {
        Err(errors)
    }
}

/// A size in bytes, from either an integer or an integer-valued float.
fn size_of(v: Option<&serde_json::Value>) -> Option<u64> {
    let v = v?;
    if let Some(n) = v.as_u64() {
        return Some(n);
    }
    // JS numbers are f64. Accept one only when it is genuinely a whole
    // non-negative count — silently truncating 1.5 bytes would be inventing
    // data, and a negative size is a bug worth surfacing rather than clamping.
    let f = v.as_f64()?;
    if f.is_finite() && f >= 0.0 && f.fract() == 0.0 {
        Some(f as u64)
    } else {
        None
    }
}

fn one_declared(v: &serde_json::Value) -> Result<Declared, String> {
    let model = v
        .get("model")
        .and_then(|m| m.as_str())
        .ok_or("missing \"model\"")?;
    if model.is_empty() {
        return Err("\"model\" is empty".into());
    }
    // Both spellings, because the same core serves a camelCase language and a
    // snake_case one and neither should have to translate.
    let weights = size_of(v.get("weightsBytes").or_else(|| v.get("weights_bytes")))
        .ok_or_else(|| format!("{model}: \"weightsBytes\" missing or not a whole byte count"))?;
    let kv = match v.get("kvBytes").or_else(|| v.get("kv_bytes")) {
        None | Some(serde_json::Value::Null) => 0,
        some => size_of(some)
            .ok_or_else(|| format!("{model}: \"kvBytes\" is not a whole byte count"))?,
    };
    Ok(Declared {
        model: model.to_string(),
        weights_bytes: weights,
        kv_bytes: kv,
    })
}