nodedb_vector/mmap_segment/
reader.rs

1// SPDX-License-Identifier: Apache-2.0
2
3//! Memory-mapped reader for the NDVS v2 vector segment format.
4
5use std::os::fd::AsRawFd;
6use std::path::{Path, PathBuf};
7use std::sync::Arc;
8use std::sync::atomic::Ordering;
9
10use nodedb_mem::{BudgetGuard, EngineId, MemoryGovernor};
11
12use super::format::{
13    FOOTER_SIZE, FORMAT_VERSION, HEADER_SIZE, MAGIC, VectorSegmentCodec, VectorSegmentDropPolicy,
14    observability, vec_pad,
15};
16use super::writer::write_segment;
17use crate::error::VectorError;
18
19/// Memory-mapped vector segment file (v2 NDVS format).
20///
21/// Exposes a `&[f32]` view of the vector data block and a `&[u64]` view of
22/// the surrogate ID block — both zero-copy slices into the mmap region.
23///
24/// Typically owned by a single Data Plane core, but safe to share across
25/// threads behind an [`Arc`]: see the `Send`/`Sync` safety note below.
26#[derive(Debug)]
27pub struct MmapVectorSegment {
28    path: PathBuf,
29    _fd: std::fs::File,
30    base: *const u8,
31    mmap_size: usize,
32    dim: usize,
33    count: usize,
34    /// Byte offset of the vector data block within the mmap.
35    vec_offset: usize,
36    /// Byte offset of the surrogate ID block within the mmap.
37    sid_offset: usize,
38    drop_policy: VectorSegmentDropPolicy,
39    madvise_state: Option<libc::c_int>,
40    /// RAII budget guard for the mmap region.  Held for the lifetime of the
41    /// mapping; released automatically on `Drop` alongside `munmap`.
42    _budget_guard: Option<BudgetGuard>,
43}
44
45// SAFETY: `MmapVectorSegment` holds a `*const u8` (`base`) into a read-only
46// `MAP_PRIVATE` mmap region. The region is immutable after construction,
47// process-global, and valid for the lifetime of the value. There is no
48// interior mutability, so concurrent reads from multiple threads are sound.
49unsafe impl Send for MmapVectorSegment {}
50unsafe impl Sync for MmapVectorSegment {}
51
52impl MmapVectorSegment {
53    // ── Constructors ──────────────────────────────────────────────────────────
54
55    /// Create a new segment file (surrogates default to 0) and open it.
56    pub fn create(path: &Path, dim: usize, vectors: &[&[f32]]) -> std::io::Result<Self> {
57        write_segment(path, dim, vectors, &[])?;
58        Self::open_with_policy(path, VectorSegmentDropPolicy::default())
59    }
60
61    /// Create a new segment file with explicit surrogate IDs and open it.
62    pub fn create_with_surrogates(
63        path: &Path,
64        dim: usize,
65        vectors: &[&[f32]],
66        surrogate_ids: &[u64],
67    ) -> std::io::Result<Self> {
68        write_segment(path, dim, vectors, surrogate_ids)?;
69        Self::open_with_policy(path, VectorSegmentDropPolicy::default())
70    }
71
72    /// Create a new segment with an explicit drop policy.
73    pub fn create_with_policy(
74        path: &Path,
75        dim: usize,
76        vectors: &[&[f32]],
77        policy: VectorSegmentDropPolicy,
78    ) -> std::io::Result<Self> {
79        write_segment(path, dim, vectors, &[])?;
80        Self::open_with_policy(path, policy)
81    }
82
83    /// Open an existing segment file and memory-map it.
84    pub fn open(path: &Path) -> std::io::Result<Self> {
85        Self::open_with_policy(path, VectorSegmentDropPolicy::default())
86    }
87
88    /// Open an existing segment with an explicit drop policy.
89    pub fn open_with_policy(path: &Path, policy: VectorSegmentDropPolicy) -> std::io::Result<Self> {
90        let fd = std::fs::OpenOptions::new().read(true).open(path)?;
91        let file_size = fd.metadata()?.len() as usize;
92
93        let min_size = HEADER_SIZE + FOOTER_SIZE;
94        if file_size < min_size {
95            return Err(std::io::Error::new(
96                std::io::ErrorKind::InvalidData,
97                format!("segment file too small: {file_size} < {min_size} bytes"),
98            ));
99        }
100
101        let base = unsafe {
102            libc::mmap(
103                std::ptr::null_mut(),
104                file_size,
105                libc::PROT_READ,
106                libc::MAP_PRIVATE,
107                fd.as_raw_fd(),
108                0,
109            )
110        };
111        if base == libc::MAP_FAILED {
112            return Err(std::io::Error::last_os_error());
113        }
114        let base = base as *const u8;
115
116        Self::validate_and_build(fd, base, file_size, path, policy, None).inspect_err(|_e| {
117            unsafe { libc::munmap(base as *mut libc::c_void, file_size) };
118        })
119    }
120
121    /// Open an existing segment with a memory governor.
122    ///
123    /// Reserves `file_size` bytes in the `EngineId::Vector` budget before
124    /// mapping the file.  Returns `VectorError::BudgetExhausted` if the
125    /// governor rejects the reservation.  The reservation is released
126    /// automatically when the segment is dropped (RAII via `BudgetGuard`).
127    pub fn open_with_governor(
128        path: &Path,
129        governor: &Arc<MemoryGovernor>,
130    ) -> Result<Self, VectorError> {
131        Self::open_with_governor_and_policy(path, governor, VectorSegmentDropPolicy::default())
132    }
133
134    /// Open an existing segment with a memory governor and explicit drop policy.
135    pub fn open_with_governor_and_policy(
136        path: &Path,
137        governor: &Arc<MemoryGovernor>,
138        policy: VectorSegmentDropPolicy,
139    ) -> Result<Self, VectorError> {
140        let fd = std::fs::OpenOptions::new().read(true).open(path)?;
141        let file_size = fd.metadata()?.len() as usize;
142
143        let budget_guard = governor.reserve(EngineId::Vector, file_size)?;
144
145        let min_size = HEADER_SIZE + FOOTER_SIZE;
146        if file_size < min_size {
147            // budget_guard dropped here → bytes returned to budget
148            return Err(std::io::Error::new(
149                std::io::ErrorKind::InvalidData,
150                format!("segment file too small: {file_size} < {min_size} bytes"),
151            )
152            .into());
153        }
154
155        let base = unsafe {
156            libc::mmap(
157                std::ptr::null_mut(),
158                file_size,
159                libc::PROT_READ,
160                libc::MAP_PRIVATE,
161                fd.as_raw_fd(),
162                0,
163            )
164        };
165        if base == libc::MAP_FAILED {
166            // budget_guard dropped here → bytes returned to budget
167            return Err(std::io::Error::last_os_error().into());
168        }
169        let base = base as *const u8;
170
171        Self::validate_and_build(fd, base, file_size, path, policy, Some(budget_guard))
172            .map_err(VectorError::from)
173            .inspect_err(|_| {
174                unsafe { libc::munmap(base as *mut libc::c_void, file_size) };
175            })
176    }
177
178    // ── Validation ────────────────────────────────────────────────────────────
179
180    fn validate_and_build(
181        fd: std::fs::File,
182        base: *const u8,
183        file_size: usize,
184        path: &Path,
185        policy: VectorSegmentDropPolicy,
186        budget_guard: Option<BudgetGuard>,
187    ) -> std::io::Result<Self> {
188        // Validate magic + format version.
189        let header = unsafe { std::slice::from_raw_parts(base, HEADER_SIZE) };
190        if &header[0..4] != MAGIC.as_slice() {
191            return Err(std::io::Error::new(
192                std::io::ErrorKind::InvalidData,
193                "invalid NDVS magic bytes",
194            ));
195        }
196        let fv = u16::from_le_bytes([header[4], header[5]]);
197        if fv != FORMAT_VERSION {
198            return Err(std::io::Error::new(
199                std::io::ErrorKind::InvalidData,
200                format!("unsupported segment format version {fv}; expected {FORMAT_VERSION}"),
201            ));
202        }
203
204        let dim = u32::from_le_bytes([header[8], header[9], header[10], header[11]]) as usize;
205        let count = u64::from_le_bytes([
206            header[12], header[13], header[14], header[15], header[16], header[17], header[18],
207            header[19],
208        ]) as usize;
209        let compression_byte = header[21];
210
211        // Codec dispatch — exhaustive match; non-None variants will be
212        // obvious when compression is wired in the future.
213        let codec = VectorSegmentCodec::from_u8(compression_byte)?;
214        match codec {
215            VectorSegmentCodec::None => { /* raw packed f32 — proceed */ }
216        }
217
218        if dim == 0 && count > 0 {
219            return Err(std::io::Error::new(
220                std::io::ErrorKind::InvalidData,
221                "segment has dim=0 with nonzero count",
222            ));
223        }
224
225        // Validate total file size with overflow-safe arithmetic.
226        let vec_bytes = dim
227            .checked_mul(count)
228            .and_then(|n| n.checked_mul(4))
229            .ok_or_else(|| {
230                std::io::Error::new(
231                    std::io::ErrorKind::InvalidData,
232                    format!("segment header overflow: dim={dim}, count={count}"),
233                )
234            })?;
235        let sid_bytes = count.checked_mul(8).ok_or_else(|| {
236            std::io::Error::new(
237                std::io::ErrorKind::InvalidData,
238                format!("surrogate block overflow: count={count}"),
239            )
240        })?;
241        let pad_bytes = vec_pad(vec_bytes);
242        let expected = HEADER_SIZE
243            .checked_add(vec_bytes)
244            .and_then(|n| n.checked_add(pad_bytes))
245            .and_then(|n| n.checked_add(sid_bytes))
246            .and_then(|n| n.checked_add(FOOTER_SIZE))
247            .ok_or_else(|| {
248                std::io::Error::new(
249                    std::io::ErrorKind::InvalidData,
250                    "total segment size overflow",
251                )
252            })?;
253        if file_size != expected {
254            return Err(std::io::Error::new(
255                std::io::ErrorKind::InvalidData,
256                format!("segment size mismatch: expected {expected} bytes, got {file_size}"),
257            ));
258        }
259
260        // Validate footer.
261        let footer_start = file_size - FOOTER_SIZE;
262        let footer = unsafe { std::slice::from_raw_parts(base.add(footer_start), FOOTER_SIZE) };
263
264        // Trailing magic — last 4 bytes of the file must be b"NDVS".
265        if &footer[42..46] != MAGIC.as_slice() {
266            return Err(std::io::Error::new(
267                std::io::ErrorKind::InvalidData,
268                "invalid NDVS trailing magic bytes",
269            ));
270        }
271
272        let footer_fv = u16::from_le_bytes([footer[0], footer[1]]);
273        if footer_fv != FORMAT_VERSION {
274            return Err(std::io::Error::new(
275                std::io::ErrorKind::InvalidData,
276                format!(
277                    "unsupported segment footer version {footer_fv}; expected {FORMAT_VERSION}"
278                ),
279            ));
280        }
281        let stored_footer_size =
282            u32::from_le_bytes([footer[38], footer[39], footer[40], footer[41]]) as usize;
283        if stored_footer_size != FOOTER_SIZE {
284            return Err(std::io::Error::new(
285                std::io::ErrorKind::InvalidData,
286                format!("footer_size field {stored_footer_size} != {FOOTER_SIZE}"),
287            ));
288        }
289
290        // CRC32C integrity check.
291        let body = unsafe { std::slice::from_raw_parts(base, footer_start) };
292        let computed = crc32c::crc32c(body);
293        let stored = u32::from_le_bytes([footer[34], footer[35], footer[36], footer[37]]);
294        if computed != stored {
295            return Err(std::io::Error::new(
296                std::io::ErrorKind::InvalidData,
297                format!("CRC32C mismatch: stored {stored:#010x}, computed {computed:#010x}"),
298            ));
299        }
300
301        let vec_offset = HEADER_SIZE;
302        let sid_offset = HEADER_SIZE + vec_bytes + pad_bytes;
303
304        // Advise MADV_RANDOM: HNSW traversal is non-sequential.
305        let mut madvise_state = None;
306        if vec_bytes + sid_bytes > 0 {
307            let rc =
308                unsafe { libc::madvise(base as *mut libc::c_void, file_size, libc::MADV_RANDOM) };
309            if rc == 0 {
310                madvise_state = Some(libc::MADV_RANDOM);
311                observability::RANDOM_COUNT.fetch_add(1, Ordering::Relaxed);
312            } else {
313                tracing::warn!(
314                    path = %path.display(),
315                    errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0),
316                    "madvise(MADV_RANDOM) failed on vector segment; continuing with kernel default",
317                );
318            }
319        }
320
321        Ok(Self {
322            path: path.to_path_buf(),
323            _fd: fd,
324            base,
325            mmap_size: file_size,
326            dim,
327            count,
328            vec_offset,
329            sid_offset,
330            drop_policy: policy,
331            madvise_state,
332            _budget_guard: budget_guard,
333        })
334    }
335
336    // ── Accessors ─────────────────────────────────────────────────────────────
337
338    /// The madvise hint set on this segment (if any).
339    pub fn madvise_state(&self) -> Option<libc::c_int> {
340        self.madvise_state
341    }
342
343    /// Get a vector by local index. Returns a slice into the mmap'd region.
344    #[inline]
345    pub fn get_vector(&self, id: u32) -> Option<&[f32]> {
346        let idx = id as usize;
347        if idx >= self.count {
348            return None;
349        }
350        let byte_len = self.dim.checked_mul(4)?;
351        let offset = self.vec_offset.checked_add(idx.checked_mul(byte_len)?)?;
352        let end = offset.checked_add(byte_len)?;
353        if end > self.sid_offset {
354            return None;
355        }
356        unsafe {
357            let ptr = self.base.add(offset) as *const f32;
358            Some(std::slice::from_raw_parts(ptr, self.dim))
359        }
360    }
361
362    /// Get the surrogate ID for a local index (0-based row in this segment).
363    #[inline]
364    pub fn get_surrogate_id(&self, id: u32) -> Option<u64> {
365        let idx = id as usize;
366        if idx >= self.count {
367            return None;
368        }
369        let offset = self.sid_offset.checked_add(idx.checked_mul(8)?)?;
370        let end = offset.checked_add(8)?;
371        let sid_end = self.mmap_size - FOOTER_SIZE;
372        if end > sid_end {
373            return None;
374        }
375        let bytes = unsafe { std::slice::from_raw_parts(self.base.add(offset), 8) };
376        Some(u64::from_le_bytes(bytes.try_into().expect("invariant: from_raw_parts constructed with len=8, so try_into::<[u8;8]> always succeeds")))
377    }
378
379    /// The full vector data block as a contiguous `&[f32]` of length `D × N`.
380    ///
381    /// Suitable for SIMD distance computation over all vectors.
382    #[inline]
383    pub fn all_vectors_flat(&self) -> &[f32] {
384        let float_count = self.dim * self.count;
385        unsafe {
386            let ptr = self.base.add(self.vec_offset) as *const f32;
387            std::slice::from_raw_parts(ptr, float_count)
388        }
389    }
390
391    /// The full surrogate ID block as a contiguous `&[u64]` of length `N`.
392    ///
393    /// Parallel to `all_vectors_flat`: row `i` in vectors ↔ `surrogate_ids[i]`.
394    #[inline]
395    pub fn all_surrogate_ids(&self) -> &[u64] {
396        unsafe {
397            let ptr = self.base.add(self.sid_offset) as *const u64;
398            std::slice::from_raw_parts(ptr, self.count)
399        }
400    }
401
402    /// Prefetch a vector's page into memory via `madvise(MADV_WILLNEED)`.
403    pub fn prefetch(&self, id: u32) {
404        let idx = id as usize;
405        if idx >= self.count {
406            return;
407        }
408        let byte_len = match self.dim.checked_mul(4) {
409            Some(v) => v,
410            None => return,
411        };
412        let Some(idx_bytes) = idx.checked_mul(byte_len) else {
413            return;
414        };
415        let Some(offset) = self.vec_offset.checked_add(idx_bytes) else {
416            return;
417        };
418        if offset
419            .checked_add(byte_len)
420            .is_none_or(|e| e > self.sid_offset)
421        {
422            return;
423        }
424        let page_start = offset & !(4095);
425        let len = (byte_len + 4095) & !(4095);
426        unsafe {
427            libc::madvise(
428                self.base.add(page_start) as *mut libc::c_void,
429                len,
430                libc::MADV_WILLNEED,
431            );
432        }
433    }
434
435    /// Prefetch a batch of vector IDs.
436    pub fn prefetch_batch(&self, ids: &[u32]) {
437        for &id in ids {
438            self.prefetch(id);
439        }
440    }
441
442    pub fn dim(&self) -> usize {
443        self.dim
444    }
445
446    pub fn count(&self) -> usize {
447        self.count
448    }
449
450    pub fn path(&self) -> &Path {
451        &self.path
452    }
453
454    pub fn mmap_bytes(&self) -> usize {
455        self.mmap_size
456    }
457
458    pub fn file_size(&self) -> usize {
459        self.mmap_size
460    }
461}
462
463impl Drop for MmapVectorSegment {
464    fn drop(&mut self) {
465        if !self.base.is_null() && self.mmap_size > 0 {
466            if self.drop_policy.dontneed_on_drop() {
467                let data_bytes = self.mmap_size.saturating_sub(HEADER_SIZE + FOOTER_SIZE);
468                if data_bytes > 0 {
469                    unsafe {
470                        libc::madvise(
471                            self.base as *mut libc::c_void,
472                            self.mmap_size,
473                            libc::MADV_DONTNEED,
474                        );
475                    }
476                    observability::DONTNEED_COUNT.fetch_add(1, Ordering::Relaxed);
477                }
478            }
479            unsafe {
480                libc::munmap(self.base as *mut libc::c_void, self.mmap_size);
481            }
482        }
483    }
484}
485
486#[cfg(test)]
487mod tests {
488    use super::*;
489
490    #[test]
491    fn create_and_read() {
492        let dir = tempfile::tempdir().unwrap();
493        let path = dir.path().join("test.vseg");
494
495        let v0 = vec![1.0f32, 2.0, 3.0];
496        let v1 = vec![4.0f32, 5.0, 6.0];
497        let v2 = vec![7.0f32, 8.0, 9.0];
498        let surrogates = vec![10u64, 20, 30];
499
500        let seg =
501            MmapVectorSegment::create_with_surrogates(&path, 3, &[&v0, &v1, &v2], &surrogates)
502                .unwrap();
503
504        assert_eq!(seg.dim(), 3);
505        assert_eq!(seg.count(), 3);
506        assert_eq!(seg.get_vector(0).unwrap(), &[1.0, 2.0, 3.0]);
507        assert_eq!(seg.get_vector(1).unwrap(), &[4.0, 5.0, 6.0]);
508        assert_eq!(seg.get_vector(2).unwrap(), &[7.0, 8.0, 9.0]);
509        assert!(seg.get_vector(3).is_none());
510        assert_eq!(seg.get_surrogate_id(0).unwrap(), 10);
511        assert_eq!(seg.get_surrogate_id(1).unwrap(), 20);
512        assert_eq!(seg.get_surrogate_id(2).unwrap(), 30);
513        assert!(seg.get_surrogate_id(3).is_none());
514    }
515
516    #[test]
517    fn flat_slices() {
518        let dir = tempfile::tempdir().unwrap();
519        let path = dir.path().join("flat.vseg");
520
521        let v0 = vec![1.0f32, 2.0, 3.0];
522        let v1 = vec![4.0f32, 5.0, 6.0];
523        let sids = vec![100u64, 200];
524
525        let seg = MmapVectorSegment::create_with_surrogates(&path, 3, &[&v0, &v1], &sids).unwrap();
526
527        assert_eq!(seg.all_vectors_flat(), &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]);
528        assert_eq!(seg.all_surrogate_ids(), &[100u64, 200]);
529    }
530
531    #[test]
532    fn reopen_roundtrip() {
533        let dir = tempfile::tempdir().unwrap();
534        let path = dir.path().join("reopen.vseg");
535
536        let vectors: Vec<Vec<f32>> = (0..100)
537            .map(|i| vec![i as f32, (i as f32).sin(), (i as f32).cos()])
538            .collect();
539        let refs: Vec<&[f32]> = vectors.iter().map(|v| v.as_slice()).collect();
540        let sids: Vec<u64> = (0u64..100).collect();
541
542        MmapVectorSegment::create_with_surrogates(&path, 3, &refs, &sids).unwrap();
543
544        let seg = MmapVectorSegment::open(&path).unwrap();
545        assert_eq!(seg.count(), 100);
546        for (i, v) in vectors.iter().enumerate() {
547            assert_eq!(seg.get_vector(i as u32).unwrap(), v.as_slice());
548            assert_eq!(seg.get_surrogate_id(i as u32).unwrap(), i as u64);
549        }
550    }
551
552    #[test]
553    fn no_surrogates_defaults_to_zero() {
554        let dir = tempfile::tempdir().unwrap();
555        let path = dir.path().join("nosid.vseg");
556
557        let v = vec![1.0f32, 2.0];
558        let seg = MmapVectorSegment::create(&path, 2, &[&v]).unwrap();
559        assert_eq!(seg.get_surrogate_id(0).unwrap(), 0);
560    }
561
562    #[test]
563    fn prefetch_does_not_crash() {
564        let dir = tempfile::tempdir().unwrap();
565        let path = dir.path().join("prefetch.vseg");
566
567        let v = vec![1.0f32; 768];
568        let seg = MmapVectorSegment::create(&path, 768, &[&v]).unwrap();
569        seg.prefetch(0);
570        seg.prefetch(999);
571    }
572
573    #[test]
574    fn empty_segment() {
575        let dir = tempfile::tempdir().unwrap();
576        let path = dir.path().join("empty.vseg");
577
578        let seg = MmapVectorSegment::create(&path, 3, &[]).unwrap();
579        assert_eq!(seg.count(), 0);
580        assert!(seg.get_vector(0).is_none());
581        assert_eq!(seg.all_vectors_flat().len(), 0);
582        assert_eq!(seg.all_surrogate_ids().len(), 0);
583    }
584
585    #[test]
586    fn footer_golden_layout() {
587        let dir = tempfile::tempdir().unwrap();
588        let path = dir.path().join("golden.vseg");
589        let v = vec![1.0f32, 2.0, 3.0];
590        write_segment(&path, 3, &[&v], &[42]).unwrap();
591        let data = std::fs::read(&path).unwrap();
592
593        // Footer starts at file_size - FOOTER_SIZE.
594        let footer_start = data.len() - FOOTER_SIZE;
595        let footer = &data[footer_start..];
596
597        // [0..2] format_version = FORMAT_VERSION.
598        let fv = u16::from_le_bytes([footer[0], footer[1]]);
599        assert_eq!(fv, FORMAT_VERSION);
600
601        // [34..38] checksum matches body CRC32C.
602        let body = &data[..footer_start];
603        let expected_crc = crc32c::crc32c(body);
604        let stored_crc = u32::from_le_bytes([footer[34], footer[35], footer[36], footer[37]]);
605        assert_eq!(stored_crc, expected_crc);
606
607        // [38..42] footer_size = FOOTER_SIZE (46).
608        let fs = u32::from_le_bytes([footer[38], footer[39], footer[40], footer[41]]) as usize;
609        assert_eq!(fs, FOOTER_SIZE);
610
611        // [42..46] trailing magic = b"NDVS".
612        assert_eq!(&footer[42..46], b"NDVS");
613    }
614
615    #[test]
616    fn trailing_magic_corruption_rejected() {
617        let dir = tempfile::tempdir().unwrap();
618        let path = dir.path().join("trailmagic.vseg");
619        let v = vec![1.0f32, 2.0, 3.0];
620        write_segment(&path, 3, &[&v], &[42]).unwrap();
621
622        let mut data = std::fs::read(&path).unwrap();
623        // Corrupt the last 4 bytes (trailing magic).
624        let last = data.len();
625        data[last - 4] = 0xde;
626        data[last - 3] = 0xad;
627        data[last - 2] = 0xbe;
628        data[last - 1] = 0xef;
629        std::fs::write(&path, &data).unwrap();
630
631        let result = MmapVectorSegment::open(&path);
632        assert!(result.is_err(), "expected trailing magic error");
633        let err = result.unwrap_err();
634        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
635        assert!(
636            err.to_string().contains("trailing magic"),
637            "expected trailing magic message, got: {err}"
638        );
639    }
640
641    #[test]
642    fn footer_version_mismatch_rejected() {
643        let dir = tempfile::tempdir().unwrap();
644        let path = dir.path().join("fvmismatch.vseg");
645        let v = vec![1.0f32, 2.0, 3.0];
646        write_segment(&path, 3, &[&v], &[42]).unwrap();
647
648        let mut data = std::fs::read(&path).unwrap();
649        // Corrupt the footer format version bytes to 99.
650        let footer_start = data.len() - FOOTER_SIZE;
651        let fv_bytes = 99u16.to_le_bytes();
652        data[footer_start] = fv_bytes[0];
653        data[footer_start + 1] = fv_bytes[1];
654        std::fs::write(&path, &data).unwrap();
655
656        let result = MmapVectorSegment::open(&path);
657        assert!(result.is_err(), "expected footer version mismatch error");
658        assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::InvalidData);
659    }
660
661    #[test]
662    fn crc_corruption_rejected() {
663        let dir = tempfile::tempdir().unwrap();
664        let path = dir.path().join("corrupt.vseg");
665
666        let v = vec![1.0f32, 2.0, 3.0];
667        write_segment(&path, 3, &[&v], &[42]).unwrap();
668
669        let mut data = std::fs::read(&path).unwrap();
670        data[HEADER_SIZE] ^= 0xff;
671        std::fs::write(&path, &data).unwrap();
672
673        let result = MmapVectorSegment::open(&path);
674        assert!(result.is_err(), "expected CRC error");
675        assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::InvalidData);
676    }
677
678    #[test]
679    fn bad_magic_rejected() {
680        let dir = tempfile::tempdir().unwrap();
681        let path = dir.path().join("badmagic.vseg");
682
683        let v = vec![1.0f32, 2.0];
684        write_segment(&path, 2, &[&v], &[]).unwrap();
685
686        let mut data = std::fs::read(&path).unwrap();
687        data[0] = b'X';
688        std::fs::write(&path, &data).unwrap();
689
690        let result = MmapVectorSegment::open(&path);
691        assert!(result.is_err());
692        assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::InvalidData);
693    }
694
695    #[test]
696    fn overflow_dim_count_rejected() {
697        let dir = tempfile::tempdir().unwrap();
698        let path = dir.path().join("overflow.vseg");
699
700        let dim: u32 = 0x40000001;
701        let count: u64 = 0x40000001;
702
703        let mut buf = Vec::new();
704        buf.extend_from_slice(&MAGIC);
705        buf.extend_from_slice(&FORMAT_VERSION.to_le_bytes());
706        buf.extend_from_slice(&0u16.to_le_bytes());
707        buf.extend_from_slice(&dim.to_le_bytes());
708        buf.extend_from_slice(&count.to_le_bytes());
709        buf.push(0u8); // dtype
710        buf.push(0u8); // codec None
711        buf.extend_from_slice(&[0u8; 6]); // reserved
712        std::fs::write(&path, &buf).unwrap();
713
714        let result = MmapVectorSegment::open(&path);
715        assert!(
716            result.is_err(),
717            "expected Err for overflow-inducing dim/count"
718        );
719    }
720
721    #[test]
722    fn zero_dim_with_nonzero_count_rejected() {
723        let dir = tempfile::tempdir().unwrap();
724        let path = dir.path().join("zerodim.vseg");
725
726        let dim: u32 = 0;
727        let count: u64 = 1000;
728
729        let mut buf = Vec::new();
730        buf.extend_from_slice(&MAGIC);
731        buf.extend_from_slice(&FORMAT_VERSION.to_le_bytes());
732        buf.extend_from_slice(&0u16.to_le_bytes());
733        buf.extend_from_slice(&dim.to_le_bytes());
734        buf.extend_from_slice(&count.to_le_bytes());
735        buf.push(0u8);
736        buf.push(0u8);
737        buf.extend_from_slice(&[0u8; 6]);
738        buf.extend_from_slice(&[0u8; 64]);
739        std::fs::write(&path, &buf).unwrap();
740
741        let result = MmapVectorSegment::open(&path);
742        assert!(result.is_err(), "expected Err for dim=0 with nonzero count");
743    }
744}
nodedb_vector/mmap_segment/reader.rs

nodedb_vector/mmap_segment/
reader.rs
