ramvamp-core 0.1.1

Runtime for streaming fine-grained MoE experts from NVMe: packed format, io_uring reads, LFU expert cache, CPU kernels, KV cache, generation loop
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
//! Synchronous expert blob reads.
//!
//! [`ExpertReader`] is the positioned-read path over the per-layer expert
//! files: one `pread` of `stride` bytes fetches exactly one expert blob
//! into a caller-owned buffer, and [`ExpertView`] carves the blob into its
//! gate/up/down projection slabs using the validated layout. Layer files
//! are opened lazily (handles cached per layer) and size-checked on first
//! open; the full SHA-256 runs only under
//! [`LoadOptions::verify_layer_hashes`] (and never under
//! [`LoadOptions::skip_hashes`]).
//!
//! This is the portable baseline the io_uring + O_DIRECT streamer will sit
//! beside: same blob geometry, same validation, different submission path.

use std::fs::File;
use std::os::unix::fs::FileExt;
use std::path::{Path, PathBuf};
use std::sync::OnceLock;

use crate::format::{ExpertsLayout, Manifest, Projection, ProjectionName, sha256_file};
use crate::kernels::quants::QuantFormat;

use super::{IoError, LoadOptions, parse_quant_format, to_usize};

/// One projection slab inside a fetched expert blob.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ExpertSlab<'a> {
    /// Packed quantized bytes of the slab.
    pub bytes: &'a [u8],
    /// Quantization format of those bytes.
    pub format: QuantFormat,
}

/// Byte range and format of one slab within a blob, resolved at
/// construction so per-read work is slicing only.
#[derive(Debug, Clone, Copy)]
struct SlabSpec {
    offset: usize,
    len: usize,
    format: QuantFormat,
}

/// One expert blob, fetched into a caller-owned buffer.
///
/// The three projection accessors return slices of that buffer at the
/// layout's offsets; the ranges were validated against the stride when the
/// reader was built, so slicing cannot fail.
#[derive(Debug)]
pub struct ExpertView<'a> {
    bytes: &'a [u8],
    /// Indexed as `[gate, up, down]`.
    slabs: [SlabSpec; 3],
}

impl<'a> ExpertView<'a> {
    /// SwiGLU gate projection slab.
    pub fn gate(&self) -> ExpertSlab<'a> {
        self.slab(ProjectionName::Gate)
    }

    /// Up projection slab.
    pub fn up(&self) -> ExpertSlab<'a> {
        self.slab(ProjectionName::Up)
    }

    /// Down projection slab.
    pub fn down(&self) -> ExpertSlab<'a> {
        self.slab(ProjectionName::Down)
    }

    /// Slab of the named projection.
    pub fn slab(&self, name: ProjectionName) -> ExpertSlab<'a> {
        let spec = self.slabs[name as usize];
        ExpertSlab {
            // In bounds by construction: projection end <= stride ==
            // bytes.len(), enforced by the layout validator and read_expert.
            bytes: &self.bytes[spec.offset..spec.offset + spec.len],
            format: spec.format,
        }
    }

    /// The whole blob, padding included.
    pub fn blob(&self) -> &'a [u8] {
        self.bytes
    }
}

/// Per-layer state: resolved geometry plus the lazily opened file handle.
#[derive(Debug)]
struct LayerFile {
    /// Absolute path of the layer file.
    path: PathBuf,
    /// Install-relative name, for error messages.
    name: String,
    stride: u64,
    stride_usize: usize,
    n_experts: u32,
    expected_size: u64,
    expected_sha256: String,
    slabs: [SlabSpec; 3],
    file: OnceLock<File>,
}

/// Lazy positioned-read access to every layer's expert file.
///
/// Construction validates the layout against the manifest and resolves the
/// gate/up/down slab geometry for every layer; all three projections must
/// be present with known quant formats. Files open on first use; the first
/// open size-checks the file always, and hashes it against the manifest only
/// under [`LoadOptions::verify_layer_hashes`]. Concurrent first reads of one
/// layer may both hash the file — benign, one handle wins.
#[derive(Debug)]
pub struct ExpertReader {
    layers: Vec<LayerFile>,
    /// Whether first-open hashes the whole layer file (the opt-in
    /// [`LoadOptions::verify_layer_hashes`], and never with
    /// [`LoadOptions::skip_hashes`]).
    hash_layers: bool,
}

impl ExpertReader {
    /// Build a reader over `<dir>`'s expert files.
    ///
    /// # Errors
    ///
    /// [`IoError::Format`] when the layout fails validation against the
    /// manifest; [`IoError::MissingProjection`] when a layer lacks gate,
    /// up, or down; [`IoError::UnknownQuant`] when a slab's quant name is
    /// not a weight format this build computes.
    pub fn new(
        dir: &Path,
        manifest: &Manifest,
        layout: &ExpertsLayout,
        options: LoadOptions,
    ) -> Result<Self, IoError> {
        layout.validate_against(manifest)?;
        let mut layers = Vec::with_capacity(layout.layers.len());
        for (index, layer) in layout.layers.iter().enumerate() {
            let layer_idx = index as u32;
            let find = |name: ProjectionName| -> Result<SlabSpec, IoError> {
                let projection = layer.projections.iter().find(|p| p.name == name).ok_or(
                    IoError::MissingProjection {
                        layer: layer_idx,
                        name,
                    },
                )?;
                slab_spec(layer_idx, projection)
            };
            let slabs = [
                find(ProjectionName::Gate)?,
                find(ProjectionName::Up)?,
                find(ProjectionName::Down)?,
            ];
            // Present and size-consistent per validate_against.
            let entry = &manifest.files[&layer.file];
            layers.push(LayerFile {
                path: dir.join(&layer.file),
                name: layer.file.clone(),
                stride: layer.stride,
                stride_usize: to_usize(layer.stride, "expert blob stride")?,
                n_experts: layer.n_experts,
                expected_size: entry.size,
                expected_sha256: entry.sha256.clone(),
                slabs,
                file: OnceLock::new(),
            });
        }
        Ok(Self {
            layers,
            hash_layers: options.verify_layer_hashes && !options.skip_hashes,
        })
    }

    /// Number of layers the reader serves.
    pub fn n_layers(&self) -> u32 {
        self.layers.len() as u32
    }

    /// Blob stride of one layer in bytes, or `None` past the last layer.
    pub fn stride(&self, layer: u32) -> Option<u64> {
        self.layers.get(layer as usize).map(|l| l.stride)
    }

    /// Read one expert blob into `buf` (resized to the layer's stride) and
    /// return the view over its projection slabs.
    ///
    /// # Errors
    ///
    /// [`IoError::LayerOutOfRange`] / [`IoError::ExpertOutOfRange`] on bad
    /// indices; [`IoError::Format`] when the file fails its first-open
    /// size or hash check; [`IoError::Io`] when the read itself fails.
    pub fn read_expert<'buf>(
        &self,
        layer: u32,
        expert: u32,
        buf: &'buf mut Vec<u8>,
    ) -> Result<ExpertView<'buf>, IoError> {
        let state = self
            .layers
            .get(layer as usize)
            .ok_or(IoError::LayerOutOfRange {
                layer,
                n_layers: self.n_layers(),
            })?;
        if expert >= state.n_experts {
            return Err(IoError::ExpertOutOfRange {
                layer,
                expert,
                n_experts: state.n_experts,
            });
        }
        let file = self.layer_file(state)?;
        buf.resize(state.stride_usize, 0);
        // In bounds: expert < n_experts and file size == n_experts * stride.
        let offset = u64::from(expert) * state.stride;
        file.read_exact_at(buf, offset)
            .map_err(|e| IoError::io(&state.path, e))?;
        Ok(ExpertView {
            bytes: buf,
            slabs: state.slabs,
        })
    }

    /// View the projection slabs of a blob some other path already fetched.
    ///
    /// [`read_expert`](Self::read_expert) owns both halves — the read and the
    /// slicing — which the streaming path cannot use: it reads into slot-pool
    /// buffers through io_uring or a positioned read, and then needs exactly
    /// the slicing. Same resolved geometry, no read.
    ///
    /// `bytes` must be at least the layer's blob stride; anything past it is
    /// ignored, so a slot buffer padded to its pitch is fine.
    ///
    /// # Errors
    ///
    /// [`IoError::LayerOutOfRange`] past the last layer;
    /// [`IoError::RangeOutOfBounds`] when `bytes` is shorter than the stride
    /// the slab offsets were validated against.
    pub fn view_over<'a>(&self, layer: u32, bytes: &'a [u8]) -> Result<ExpertView<'a>, IoError> {
        let state = self
            .layers
            .get(layer as usize)
            .ok_or(IoError::LayerOutOfRange {
                layer,
                n_layers: self.n_layers(),
            })?;
        let blob = bytes
            .get(..state.stride_usize)
            .ok_or_else(|| IoError::RangeOutOfBounds {
                what: format!("layer {layer} expert blob"),
                offset: 0,
                len: state.stride,
                available: bytes.len() as u64,
            })?;
        Ok(ExpertView {
            bytes: blob,
            slabs: state.slabs,
        })
    }

    /// The layer's cached file handle, opening and verifying on first use.
    fn layer_file<'a>(&self, state: &'a LayerFile) -> Result<&'a File, IoError> {
        if let Some(file) = state.file.get() {
            return Ok(file);
        }
        let file = File::open(&state.path).map_err(|e| IoError::io(&state.path, e))?;
        let size = file
            .metadata()
            .map_err(|e| IoError::io(&state.path, e))?
            .len();
        if size != state.expected_size {
            return Err(crate::format::FormatError::SizeMismatch {
                name: state.name.clone(),
                expected: state.expected_size,
                actual: size,
            }
            .into());
        }
        if self.hash_layers {
            let actual = sha256_file(&state.path)?;
            if !actual.eq_ignore_ascii_case(&state.expected_sha256) {
                return Err(crate::format::FormatError::HashMismatch {
                    name: state.name.clone(),
                    expected: state.expected_sha256.clone(),
                    actual,
                }
                .into());
            }
            tracing::debug!(
                file = state.name.as_str(),
                bytes = size,
                "layer file verified"
            );
        }
        Ok(state.file.get_or_init(|| file))
    }
}

/// Resolve one layout projection into a slab spec.
fn slab_spec(layer: u32, projection: &Projection) -> Result<SlabSpec, IoError> {
    let format = parse_quant_format(&projection.quant).ok_or_else(|| IoError::UnknownQuant {
        what: format!("layer {layer} {:?} projection", projection.name),
        quant: projection.quant.clone(),
    })?;
    Ok(SlabSpec {
        offset: to_usize(projection.offset_in_blob, "projection offset")?,
        len: to_usize(projection.len, "projection len")?,
        format,
    })
}

#[cfg(test)]
mod tests {
    use super::super::testutil::{self, build_install};
    use super::*;
    use crate::format::FormatError;

    fn reader(fx: &testutil::Fixture, options: LoadOptions) -> ExpertReader {
        ExpertReader::new(&fx.root, &fx.manifest, &fx.layout, options).unwrap()
    }

    /// Skip everything (the dev flag).
    const SKIP: LoadOptions = LoadOptions {
        skip_hashes: true,
        verify_layer_hashes: false,
        common_memory: crate::io::CommonMemory::Mapped,
    };

    /// Opt in to the full per-layer SHA-256.
    const VERIFY: LoadOptions = LoadOptions {
        skip_hashes: false,
        verify_layer_hashes: true,
        common_memory: crate::io::CommonMemory::Mapped,
    };

    #[test]
    fn reads_expert_and_slabs_match_layout_offsets() {
        let fx = build_install("expert-slabs");
        let reader = reader(&fx, LoadOptions::default());
        assert_eq!(reader.n_layers(), 2);

        let mut buf = Vec::new();
        for (layer, expert) in [(0u32, 0u32), (0, 3), (1, 2)] {
            let view = reader.read_expert(layer, expert, &mut buf).unwrap();
            let layer_layout = &fx.layout.layers[layer as usize];
            assert_eq!(view.blob().len() as u64, layer_layout.stride);

            let file_bytes = std::fs::read(fx.root.join(&layer_layout.file)).unwrap();
            let base = (u64::from(expert) * layer_layout.stride) as usize;
            for projection in &layer_layout.projections {
                let slab = view.slab(projection.name);
                let start = base + projection.offset_in_blob as usize;
                let end = start + projection.len as usize;
                assert_eq!(slab.bytes, &file_bytes[start..end], "{:?}", projection.name);
                assert_eq!(slab.format, parse_quant_format(&projection.quant).unwrap());
            }
        }
        // Named accessors agree with slab-by-name.
        let view = reader.read_expert(0, 1, &mut buf).unwrap();
        assert_eq!(view.gate(), view.slab(ProjectionName::Gate));
        assert_eq!(view.up(), view.slab(ProjectionName::Up));
        assert_eq!(view.down(), view.slab(ProjectionName::Down));
        assert_eq!(view.down().format, QuantFormat::Q6_K);
        assert_eq!(view.gate().format, QuantFormat::Q4_K);
    }

    #[test]
    fn rejects_out_of_range_indices() {
        let fx = build_install("expert-bounds");
        let reader = reader(&fx, SKIP);
        let mut buf = Vec::new();
        assert!(matches!(
            reader.read_expert(2, 0, &mut buf).unwrap_err(),
            IoError::LayerOutOfRange {
                layer: 2,
                n_layers: 2
            }
        ));
        assert!(matches!(
            reader.read_expert(0, 4, &mut buf).unwrap_err(),
            IoError::ExpertOutOfRange {
                layer: 0,
                expert: 4,
                n_experts: 4
            }
        ));
    }

    /// Layer hashing is opt-in: the default load reads a corrupt layer file
    /// without complaint, and only `verify_layer_hashes` catches it. That
    /// asymmetry is deliberate — see [`LoadOptions::verify_layer_hashes`].
    #[test]
    fn first_open_hash_check_is_opt_in() {
        let fx = build_install("expert-hash");
        let victim = fx.root.join(&fx.layout.layers[0].file);
        let mut data = std::fs::read(&victim).unwrap();
        data[5000] ^= 0xff;
        std::fs::write(&victim, data).unwrap();

        let strict = reader(&fx, VERIFY);
        let mut buf = Vec::new();
        assert!(matches!(
            strict.read_expert(0, 0, &mut buf).unwrap_err(),
            IoError::Format(FormatError::HashMismatch { .. })
        ));
        // The untouched layer still reads.
        strict.read_expert(1, 0, &mut buf).unwrap();

        // Neither the default nor the dev flag hashes layer files.
        for options in [LoadOptions::default(), SKIP] {
            reader(&fx, options).read_expert(0, 0, &mut buf).unwrap();
        }
    }

    #[test]
    fn size_check_runs_whatever_the_hash_policy() {
        let fx = build_install("expert-size");
        let victim = fx.root.join(&fx.layout.layers[1].file);
        let data = std::fs::read(&victim).unwrap();
        std::fs::write(&victim, &data[..data.len() - 4096]).unwrap();
        let mut buf = Vec::new();
        for options in [LoadOptions::default(), SKIP, VERIFY] {
            assert!(matches!(
                reader(&fx, options)
                    .read_expert(1, 0, &mut buf)
                    .unwrap_err(),
                IoError::Format(FormatError::SizeMismatch { .. })
            ));
        }
    }

    #[test]
    fn rejects_layout_missing_a_projection() {
        let fx = build_install("expert-missing-proj");
        let mut layout = fx.layout.clone();
        layout.layers[0]
            .projections
            .retain(|p| p.name != ProjectionName::Down);
        let err =
            ExpertReader::new(&fx.root, &fx.manifest, &layout, LoadOptions::default()).unwrap_err();
        assert!(matches!(
            err,
            IoError::MissingProjection {
                layer: 0,
                name: ProjectionName::Down
            }
        ));
    }

    #[test]
    fn rejects_unknown_slab_quant() {
        let fx = build_install("expert-bad-quant");
        let mut layout = fx.layout.clone();
        layout.layers[1].projections[0].quant = "f16".to_owned();
        let err =
            ExpertReader::new(&fx.root, &fx.manifest, &layout, LoadOptions::default()).unwrap_err();
        assert!(matches!(
            err,
            IoError::UnknownQuant { quant, .. } if quant == "f16"
        ));
    }
}