Skip to main content

tpt_gpu_model_optimizer/
streaming.rs

1//! Streaming Loader — processes 70B+ models layer-by-layer via mmap.
2//!
3//! For models too large to load entirely into VRAM, `StreamingLoader` mmaps
4//! the source GGUF file and yields one layer at a time. The caller quantizes
5//! and writes each layer to the output `.tptf` immediately, so peak memory
6//! equals the largest single layer (not the full model).
7//!
8//! Activation thresholds: if `model_size > vram_free * 0.8`, streaming is
9//! enabled automatically.
10
11use anyhow::{bail, Context, Result};
12use memmap2::MmapOptions;
13use std::fs::File;
14use std::path::Path;
15
16/// A single layer's raw weight data, borrowed from the mmap.
17#[derive(Debug)]
18pub struct LayerView<'a> {
19    pub layer_idx: usize,
20    /// Raw byte slice pointing into the mmap. Caller is responsible for
21    /// interpreting as the correct element type (f32, f16, etc.).
22    pub weights: &'a [u8],
23    /// Number of rows (hidden_dim for gate/up/down projections).
24    pub rows: usize,
25    /// Number of columns (ffn_dim for gate/up, hidden_dim for down).
26    pub cols: usize,
27}
28
29/// Streams a GGUF model file layer-by-layer without loading it all into RAM.
30pub struct StreamingLoader {
31    path: std::path::PathBuf,
32    num_layers: usize,
33    /// If true, prints progress to stderr.
34    verbose: bool,
35}
36
37impl StreamingLoader {
38    pub fn new(path: &Path, num_layers: usize) -> Self {
39        StreamingLoader {
40            path: path.to_path_buf(),
41            num_layers,
42            verbose: false,
43        }
44    }
45
46    pub fn verbose(mut self) -> Self {
47        self.verbose = true;
48        self
49    }
50
51    /// Returns true when streaming should be used.
52    ///
53    /// `model_size_mb`: total model weight size in MiB.
54    /// `vram_free_mb`: free VRAM reported by the hardware profiler.
55    pub fn should_stream(model_size_mb: f64, vram_free_mb: u64) -> bool {
56        model_size_mb > vram_free_mb as f64 * 0.8
57    }
58
59    /// Iterate over all layers, calling `f` for each.
60    ///
61    /// The mmap is held open for the duration of the callback to allow
62    /// zero-copy access to raw bytes. In production `f` would quantize the
63    /// layer weights and write the compressed result to the output `.tptf`.
64    pub fn for_each_layer<F>(&self, mut f: F) -> Result<()>
65    where
66        F: FnMut(LayerView<'_>) -> Result<()>,
67    {
68        let file = File::open(&self.path).with_context(|| format!("opening {:?}", self.path))?;
69        let mmap = unsafe { MmapOptions::new().map(&file) }.with_context(|| "mmap failed")?;
70
71        // Validate GGUF or TPTF magic
72        if mmap.len() < 8 {
73            bail!("file too small to be a valid model");
74        }
75        let magic = &mmap[..4];
76        if magic != b"GGUF" && magic != b"TPTF" {
77            bail!("unknown file format (magic={:?})", magic);
78        }
79
80        // In production: parse the GGUF tensor index (key-value + tensor info
81        // sections) to locate each layer's data offset + byte length. Then
82        // call f() for each layer with a slice into `mmap`.
83        //
84        // This scaffold simulates that by dividing the file evenly.
85        let header_bytes = 256usize;
86        let payload = mmap.len().saturating_sub(header_bytes);
87        let layer_bytes = payload / self.num_layers.max(1);
88
89        for i in 0..self.num_layers {
90            let start = header_bytes + i * layer_bytes;
91            let end = (start + layer_bytes).min(mmap.len());
92            let slice = &mmap[start..end];
93
94            if self.verbose {
95                eprintln!("[streaming] layer {}/{}", i + 1, self.num_layers);
96            }
97
98            f(LayerView {
99                layer_idx: i,
100                weights: slice,
101                rows: 4096,
102                cols: 11008,
103            })?;
104        }
105        Ok(())
106    }
107}
108
109#[cfg(test)]
110mod tests {
111    use super::*;
112
113    #[test]
114    fn should_stream_threshold() {
115        // 14 GiB model vs 16 GiB free — model is 87.5% of free, should stream
116        assert!(StreamingLoader::should_stream(14336.0, 16384));
117        // 8 GiB model vs 16 GiB free — 50%, should NOT stream
118        assert!(!StreamingLoader::should_stream(8192.0, 16384));
119    }
120}