ramvamp-core 0.1.1

Runtime for streaming fine-grained MoE experts from NVMe: packed format, io_uring reads, LFU expert cache, CPU kernels, KV cache, generation loop
Documentation
//! Expert streaming and caching.
//!
//! - Common weights: read-only `mmap` (they are touched every token, so the
//!   page cache keeps them resident).
//! - Routed experts: explicit reads, never demand paging. On Linux with the
//!   `io-uring` feature (default), misses are fetched with io_uring +
//!   O_DIRECT into pre-allocated page-aligned slot buffers; the portable
//!   fallback uses positioned reads (`pread`) on a small thread pool.
//! - Cache: per-layer fixed slot arrays with LFU eviction over frequency
//!   counters indexed by expert id, sized `n_experts`, whose counts survive
//!   eviction (ghost history; EXP-005). No cross-layer prefetch: measured
//!   predictability is too low to pay for speculative reads.
//! - Prefill: **not** the cache. [`LayerSweep`] streams one layer's expert file
//!   front to back in large windows, borrowing the idle slot pool as scratch,
//!   and computes each expert against all of a chunk's routed rows as it
//!   arrives. A 512-token chunk reads every expert once per layer instead of
//!   `top_k` times per token; replaying the prompt into the decode cache was
//!   measured at +0.09 points and is deliberately not done (EXP-005).
//!
//! Concurrency invariant: a slot being filled by an in-flight read, or still
//! owned by queued compute, is never reassigned.
//!
//! # File access layer
//!
//! What exists today is the synchronous file-access layer beneath that
//! design:
//!
//! - [`MappedCommon`]: the read-only map of `common.bin`.
//! - [`ExpertReader`]: lazy per-layer file handles + positioned blob reads,
//!   returning [`ExpertView`]s over the gate/up/down slabs.
//! - [`LoadOptions`]: the load-time integrity dial.
//!
//! Integrity at load: `manifest.json` is the trust root — it carries the
//! digests but has no recorded digest of its own, so its gate is schema
//! validation ([`Manifest::validate`](crate::format::Manifest::validate)).
//! `common.bin` and `experts/layout.json` are hashed against the manifest
//! when the model opens (the ~1 GiB `common.bin` hash is an accepted
//! one-time cost); each `experts/layer_NN.bin` is **size-checked** on first
//! open, and hashed in full only under
//! [`LoadOptions::verify_layer_hashes`] — re-reading 16.35 GiB of expert
//! files through the page cache on every process start is exactly what
//! streaming them with O_DIRECT exists to avoid, and the thorough path is
//! `ramvamp-repack verify-install`.
//! [`LoadOptions::skip_hashes`] skips every SHA-256 check for fast dev
//! iteration; size checks always run.

mod cache;
mod common;
mod direct;
mod error;
mod expert;
mod slots;
mod stream;
mod sweep;
#[cfg(test)]
pub(crate) mod testutil;

use std::fs;
use std::path::Path;

pub use cache::{CacheError, CachePlan, CacheStats, LayerCache, MAX_SLOTS, STUCK_PROTECTED_PLANS};
pub use common::MappedCommon;
pub use direct::{DIO_ALIGN, DirectFault, DirectSupport};
pub use error::IoError;
pub use expert::{ExpertReader, ExpertSlab, ExpertView};
pub(crate) use slots::mib;
pub use slots::{
    Footprint, FootprintError, MAX_POOL_BYTES, RUNTIME_ANON_BYTES, SLOT_ALIGN, SlotError,
    SlotGuard, SlotPool, resident_ceiling,
};
pub use stream::{ExpertStream, StreamMode, StreamPhase, StreamStats};
pub use sweep::{
    DEFAULT_EXPERTS_PER_WINDOW, DEFAULT_WINDOWS_IN_FLIGHT, LayerSweep, MAX_WINDOWS_IN_FLIGHT,
    PrefillSession, SweepConfig, SweepError, SweepExpert, SweepPlan, SweepWindow,
};

use crate::format::{FormatError, Manifest, sha256_file};
use crate::kernels::quants::QuantFormat;

/// Knobs for opening an installed model.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct LoadOptions {
    /// Skip every SHA-256 integrity check (load-time metadata hashes and
    /// first-open layer-file hashes). Development flag only: size checks
    /// still run, but corruption goes undetected. Defaults to `false`.
    pub skip_hashes: bool,

    /// Hash every `experts/layer_NN.bin` in full when it is first opened.
    ///
    /// **Off by default, deliberately.** The layer files are the bulk of an
    /// install (16.35 GiB for Qwen3-30B-A3B), and hashing them is a
    /// buffered sequential read of all of it — which both costs seconds of
    /// startup and charges the whole model to the page cache, defeating the
    /// point of streaming experts with O_DIRECT. Every layer file is still
    /// **size-checked** on first open, unconditionally, and the thorough
    /// path remains `ramvamp-repack verify-install`.
    ///
    /// Ignored when [`LoadOptions::skip_hashes`] is set; that flag skips
    /// everything.
    pub verify_layer_hashes: bool,

    /// How `common.bin` is held in memory. See [`CommonMemory`].
    pub common_memory: CommonMemory,
}

/// How the common weights (`common.bin`: embeddings, lm_head, attention,
/// routers, norms; ~1 GiB for the v0 model) are held in memory.
///
/// [`CommonMemory::Resident`] is the default (EXP-031): on a machine short of
/// memory, `kswapd` took up to 801 MiB of the mapped weights mid-run and
/// decode fell to 1.43 tok/s, while resident runs never had to read a page
/// back, at the same speed, for +0.75 s of load. At 4K context it peaks at
/// 2,963 MiB, inside the 3 GB contract and within 2 MiB of the footprint
/// projection, which always assumed the whole file resident.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum CommonMemory {
    /// Memory-mapped read-only: file-backed pages the kernel may drop under
    /// memory pressure and fault back in from disk.
    Mapped,
    /// Read once into anonymous memory with `O_DIRECT`, so the bytes are
    /// not also in the page cache. Anonymous pages can only leave memory
    /// through swap, so in a cgroup with swap disabled the kernel cannot
    /// reclaim them at all: other processes' memory pressure can no longer
    /// evict the weights the runtime touches every token. Same size either
    /// way, so the footprint does not change; the load is one sequential
    /// read of the file.
    #[default]
    Resident,
}

/// Map a lowercase on-disk quant name (as the repacker writes it) to the
/// weight format the kernels compute. Returns `None` for anything that is
/// not a storable weight format — including `q8_k`, which is
/// activation-only and never on disk.
pub fn parse_quant_format(name: &str) -> Option<QuantFormat> {
    match name {
        "q4_k" => Some(QuantFormat::Q4_K),
        "q5_k" => Some(QuantFormat::Q5_K),
        "q6_k" => Some(QuantFormat::Q6_K),
        "q8_0" => Some(QuantFormat::Q8_0),
        _ => None,
    }
}

/// Check one manifest-listed file on disk: exact size always, streaming
/// SHA-256 unless [`LoadOptions::skip_hashes`] is set.
///
/// # Errors
///
/// [`IoError::Format`] wrapping the missing-entry, size-mismatch,
/// hash-mismatch, or I/O failure.
pub fn verify_named_file(
    dir: &Path,
    manifest: &Manifest,
    name: &str,
    options: LoadOptions,
) -> Result<(), IoError> {
    let entry = manifest
        .files
        .get(name)
        .ok_or_else(|| FormatError::MissingFileEntry(name.to_owned()))?;
    let path = dir.join(name);
    let metadata = fs::metadata(&path).map_err(|e| IoError::io(&path, e))?;
    if metadata.len() != entry.size {
        return Err(FormatError::SizeMismatch {
            name: name.to_owned(),
            expected: entry.size,
            actual: metadata.len(),
        }
        .into());
    }
    if !options.skip_hashes {
        let actual = sha256_file(&path)?;
        if !actual.eq_ignore_ascii_case(&entry.sha256) {
            return Err(FormatError::HashMismatch {
                name: name.to_owned(),
                expected: entry.sha256.clone(),
                actual,
            }
            .into());
        }
        tracing::debug!(file = name, bytes = entry.size, "file verified");
    }
    Ok(())
}

/// Narrow a metadata `u64` into `usize`, naming the value on failure.
pub(crate) fn to_usize(value: u64, what: &'static str) -> Result<usize, IoError> {
    usize::try_from(value).map_err(|_| IoError::TooLarge { what, value })
}