use anyhow::{bail, Context, Result};
use memmap2::MmapOptions;
use std::fs::File;
use std::path::Path;
#[derive(Debug)]
pub struct LayerView<'a> {
pub layer_idx: usize,
pub weights: &'a [u8],
pub rows: usize,
pub cols: usize,
}
pub struct StreamingLoader {
path: std::path::PathBuf,
num_layers: usize,
verbose: bool,
}
impl StreamingLoader {
pub fn new(path: &Path, num_layers: usize) -> Self {
StreamingLoader {
path: path.to_path_buf(),
num_layers,
verbose: false,
}
}
pub fn verbose(mut self) -> Self {
self.verbose = true;
self
}
pub fn should_stream(model_size_mb: f64, vram_free_mb: u64) -> bool {
model_size_mb > vram_free_mb as f64 * 0.8
}
pub fn for_each_layer<F>(&self, mut f: F) -> Result<()>
where
F: FnMut(LayerView<'_>) -> Result<()>,
{
let file = File::open(&self.path).with_context(|| format!("opening {:?}", self.path))?;
let mmap = unsafe { MmapOptions::new().map(&file) }.with_context(|| "mmap failed")?;
if mmap.len() < 8 {
bail!("file too small to be a valid model");
}
let magic = &mmap[..4];
if magic != b"GGUF" && magic != b"TPTF" {
bail!("unknown file format (magic={:?})", magic);
}
let header_bytes = 256usize;
let payload = mmap.len().saturating_sub(header_bytes);
let layer_bytes = payload / self.num_layers.max(1);
for i in 0..self.num_layers {
let start = header_bytes + i * layer_bytes;
let end = (start + layer_bytes).min(mmap.len());
let slice = &mmap[start..end];
if self.verbose {
eprintln!("[streaming] layer {}/{}", i + 1, self.num_layers);
}
f(LayerView {
layer_idx: i,
weights: slice,
rows: 4096,
cols: 11008,
})?;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn should_stream_threshold() {
assert!(StreamingLoader::should_stream(14336.0, 16384));
assert!(!StreamingLoader::should_stream(8192.0, 16384));
}
}