tpt_gpu_model_optimizer/streaming.rs
1//! Streaming Loader — processes 70B+ models layer-by-layer via mmap.
2//!
3//! For models too large to load entirely into VRAM, `StreamingLoader` mmaps
4//! the source GGUF file and yields one layer at a time. The caller quantizes
5//! and writes each layer to the output `.tptf` immediately, so peak memory
6//! equals the largest single layer (not the full model).
7//!
8//! Activation thresholds: if `model_size > vram_free * 0.8`, streaming is
9//! enabled automatically.
10
11use anyhow::{bail, Context, Result};
12use memmap2::MmapOptions;
13use std::fs::File;
14use std::path::Path;
15
16/// A single layer's raw weight data, borrowed from the mmap.
17#[derive(Debug)]
18pub struct LayerView<'a> {
19 pub layer_idx: usize,
20 /// Raw byte slice pointing into the mmap. Caller is responsible for
21 /// interpreting as the correct element type (f32, f16, etc.).
22 pub weights: &'a [u8],
23 /// Number of rows (hidden_dim for gate/up/down projections).
24 pub rows: usize,
25 /// Number of columns (ffn_dim for gate/up, hidden_dim for down).
26 pub cols: usize,
27}
28
29/// Streams a GGUF model file layer-by-layer without loading it all into RAM.
30pub struct StreamingLoader {
31 path: std::path::PathBuf,
32 num_layers: usize,
33 /// If true, prints progress to stderr.
34 verbose: bool,
35}
36
37impl StreamingLoader {
38 pub fn new(path: &Path, num_layers: usize) -> Self {
39 StreamingLoader {
40 path: path.to_path_buf(),
41 num_layers,
42 verbose: false,
43 }
44 }
45
46 pub fn verbose(mut self) -> Self {
47 self.verbose = true;
48 self
49 }
50
51 /// Returns true when streaming should be used.
52 ///
53 /// `model_size_mb`: total model weight size in MiB.
54 /// `vram_free_mb`: free VRAM reported by the hardware profiler.
55 pub fn should_stream(model_size_mb: f64, vram_free_mb: u64) -> bool {
56 model_size_mb > vram_free_mb as f64 * 0.8
57 }
58
59 /// Iterate over all layers, calling `f` for each.
60 ///
61 /// The mmap is held open for the duration of the callback to allow
62 /// zero-copy access to raw bytes. In production `f` would quantize the
63 /// layer weights and write the compressed result to the output `.tptf`.
64 pub fn for_each_layer<F>(&self, mut f: F) -> Result<()>
65 where
66 F: FnMut(LayerView<'_>) -> Result<()>,
67 {
68 let file = File::open(&self.path).with_context(|| format!("opening {:?}", self.path))?;
69 let mmap = unsafe { MmapOptions::new().map(&file) }.with_context(|| "mmap failed")?;
70
71 // Validate GGUF or TPTF magic
72 if mmap.len() < 8 {
73 bail!("file too small to be a valid model");
74 }
75 let magic = &mmap[..4];
76 if magic != b"GGUF" && magic != b"TPTF" {
77 bail!("unknown file format (magic={:?})", magic);
78 }
79
80 // In production: parse the GGUF tensor index (key-value + tensor info
81 // sections) to locate each layer's data offset + byte length. Then
82 // call f() for each layer with a slice into `mmap`.
83 //
84 // This scaffold simulates that by dividing the file evenly.
85 let header_bytes = 256usize;
86 let payload = mmap.len().saturating_sub(header_bytes);
87 let layer_bytes = payload / self.num_layers.max(1);
88
89 for i in 0..self.num_layers {
90 let start = header_bytes + i * layer_bytes;
91 let end = (start + layer_bytes).min(mmap.len());
92 let slice = &mmap[start..end];
93
94 if self.verbose {
95 eprintln!("[streaming] layer {}/{}", i + 1, self.num_layers);
96 }
97
98 f(LayerView {
99 layer_idx: i,
100 weights: slice,
101 rows: 4096,
102 cols: 11008,
103 })?;
104 }
105 Ok(())
106 }
107}
108
109#[cfg(test)]
110mod tests {
111 use super::*;
112
113 #[test]
114 fn should_stream_threshold() {
115 // 14 GiB model vs 16 GiB free — model is 87.5% of free, should stream
116 assert!(StreamingLoader::should_stream(14336.0, 16384));
117 // 8 GiB model vs 16 GiB free — 50%, should NOT stream
118 assert!(!StreamingLoader::should_stream(8192.0, 16384));
119 }
120}