nodedb_vector/mmap_segment/
reader.rs

1// SPDX-License-Identifier: Apache-2.0
2
3//! Memory-mapped reader for the NDVS v2 vector segment format.
4
5use std::os::fd::AsRawFd;
6use std::path::{Path, PathBuf};
7use std::sync::atomic::Ordering;
8
9use nodedb_mem::{ReservationToken, ScopedMemory};
10
11use super::format::{
12    FOOTER_SIZE, FORMAT_VERSION, HEADER_SIZE, MAGIC, VectorSegmentCodec, VectorSegmentDropPolicy,
13    observability, vec_pad,
14};
15use super::writer::write_segment;
16use crate::error::VectorError;
17
18/// Memory-mapped vector segment file (v2 NDVS format).
19///
20/// Exposes a `&[f32]` view of the vector data block and a `&[u64]` view of
21/// the surrogate ID block — both zero-copy slices into the mmap region.
22///
23/// Typically owned by a single Data Plane core, but safe to share across
24/// threads behind an [`Arc`]: see the `Send`/`Sync` safety note below.
25#[derive(Debug)]
26pub struct MmapVectorSegment {
27    path: PathBuf,
28    _fd: std::fs::File,
29    base: *const u8,
30    mmap_size: usize,
31    dim: usize,
32    count: usize,
33    /// Byte offset of the vector data block within the mmap.
34    vec_offset: usize,
35    /// Byte offset of the surrogate ID block within the mmap.
36    sid_offset: usize,
37    drop_policy: VectorSegmentDropPolicy,
38    madvise_state: Option<libc::c_int>,
39    /// RAII reservation token for the mmap region.  Held for the lifetime
40    /// of the mapping; released automatically on `Drop` alongside `munmap`.
41    _reservation: ReservationToken,
42}
43
44// SAFETY: `MmapVectorSegment` holds a `*const u8` (`base`) into a read-only
45// `MAP_PRIVATE` mmap region. The region is immutable after construction,
46// process-global, and valid for the lifetime of the value. There is no
47// interior mutability, so concurrent reads from multiple threads are sound.
48unsafe impl Send for MmapVectorSegment {}
49unsafe impl Sync for MmapVectorSegment {}
50
51impl MmapVectorSegment {
52    // ── Constructors ──────────────────────────────────────────────────────────
53
54    /// Create the segment file `name` inside `dir` (surrogates default to 0)
55    /// and open it, tracked against `memory`.
56    pub fn create(
57        dir: &Path,
58        name: &str,
59        dim: usize,
60        vectors: &[&[f32]],
61        memory: &ScopedMemory,
62    ) -> Result<Self, VectorError> {
63        write_segment(dir, name, dim, vectors, &[])?;
64        Self::open_with_policy(&dir.join(name), memory, VectorSegmentDropPolicy::default())
65    }
66
67    /// Create the segment file `name` inside `dir` with explicit surrogate IDs
68    /// and open it, tracked against `memory`.
69    pub fn create_with_surrogates(
70        dir: &Path,
71        name: &str,
72        dim: usize,
73        vectors: &[&[f32]],
74        surrogate_ids: &[u64],
75        memory: &ScopedMemory,
76    ) -> Result<Self, VectorError> {
77        write_segment(dir, name, dim, vectors, surrogate_ids)?;
78        Self::open_with_policy(&dir.join(name), memory, VectorSegmentDropPolicy::default())
79    }
80
81    /// Create the segment file `name` inside `dir` with an explicit drop
82    /// policy, tracked against `memory`.
83    pub fn create_with_policy(
84        dir: &Path,
85        name: &str,
86        dim: usize,
87        vectors: &[&[f32]],
88        policy: VectorSegmentDropPolicy,
89        memory: &ScopedMemory,
90    ) -> Result<Self, VectorError> {
91        write_segment(dir, name, dim, vectors, &[])?;
92        Self::open_with_policy(&dir.join(name), memory, policy)
93    }
94
95    /// Open an existing segment file and memory-map it.
96    ///
97    /// Reserves `file_size` bytes against the bound database, tenant, and
98    /// engine budget before mapping the file.  Returns
99    /// `VectorError::BudgetExhausted` if the reservation is rejected.  The
100    /// reservation is released automatically when the segment is dropped
101    /// (RAII via `ReservationToken`).
102    pub fn open(path: &Path, memory: &ScopedMemory) -> Result<Self, VectorError> {
103        Self::open_with_policy(path, memory, VectorSegmentDropPolicy::default())
104    }
105
106    /// Open an existing segment with an explicit drop policy, tracked
107    /// against `memory`.
108    pub fn open_with_policy(
109        path: &Path,
110        memory: &ScopedMemory,
111        policy: VectorSegmentDropPolicy,
112    ) -> Result<Self, VectorError> {
113        let fd = std::fs::OpenOptions::new().read(true).open(path)?;
114        let file_size = fd.metadata()?.len() as usize;
115
116        let reservation = memory.reserve(file_size)?;
117
118        let min_size = HEADER_SIZE + FOOTER_SIZE;
119        if file_size < min_size {
120            // reservation dropped here → bytes returned to budget
121            return Err(std::io::Error::new(
122                std::io::ErrorKind::InvalidData,
123                format!("segment file too small: {file_size} < {min_size} bytes"),
124            )
125            .into());
126        }
127
128        let base = unsafe {
129            libc::mmap(
130                std::ptr::null_mut(),
131                file_size,
132                libc::PROT_READ,
133                libc::MAP_PRIVATE,
134                fd.as_raw_fd(),
135                0,
136            )
137        };
138        if base == libc::MAP_FAILED {
139            // reservation dropped here → bytes returned to budget
140            return Err(std::io::Error::last_os_error().into());
141        }
142        let base = base as *const u8;
143
144        Self::validate_and_build(fd, base, file_size, path, policy, reservation)
145            .map_err(VectorError::from)
146            .inspect_err(|_| {
147                unsafe { libc::munmap(base as *mut libc::c_void, file_size) };
148            })
149    }
150
151    // ── Validation ────────────────────────────────────────────────────────────
152
153    fn validate_and_build(
154        fd: std::fs::File,
155        base: *const u8,
156        file_size: usize,
157        path: &Path,
158        policy: VectorSegmentDropPolicy,
159        reservation: ReservationToken,
160    ) -> std::io::Result<Self> {
161        // Validate magic + format version.
162        let header = unsafe { std::slice::from_raw_parts(base, HEADER_SIZE) };
163        if &header[0..4] != MAGIC.as_slice() {
164            return Err(std::io::Error::new(
165                std::io::ErrorKind::InvalidData,
166                "invalid NDVS magic bytes",
167            ));
168        }
169        let fv = u16::from_le_bytes([header[4], header[5]]);
170        if fv != FORMAT_VERSION {
171            return Err(std::io::Error::new(
172                std::io::ErrorKind::InvalidData,
173                format!("unsupported segment format version {fv}; expected {FORMAT_VERSION}"),
174            ));
175        }
176
177        let dim = u32::from_le_bytes([header[8], header[9], header[10], header[11]]) as usize;
178        let count = u64::from_le_bytes([
179            header[12], header[13], header[14], header[15], header[16], header[17], header[18],
180            header[19],
181        ]) as usize;
182        let compression_byte = header[21];
183
184        // Codec dispatch — exhaustive match; non-None variants will be
185        // obvious when compression is wired in the future.
186        let codec = VectorSegmentCodec::from_u8(compression_byte)?;
187        match codec {
188            VectorSegmentCodec::None => { /* raw packed f32 — proceed */ }
189        }
190
191        if dim == 0 && count > 0 {
192            return Err(std::io::Error::new(
193                std::io::ErrorKind::InvalidData,
194                "segment has dim=0 with nonzero count",
195            ));
196        }
197
198        // Validate total file size with overflow-safe arithmetic.
199        let vec_bytes = dim
200            .checked_mul(count)
201            .and_then(|n| n.checked_mul(4))
202            .ok_or_else(|| {
203                std::io::Error::new(
204                    std::io::ErrorKind::InvalidData,
205                    format!("segment header overflow: dim={dim}, count={count}"),
206                )
207            })?;
208        let sid_bytes = count.checked_mul(8).ok_or_else(|| {
209            std::io::Error::new(
210                std::io::ErrorKind::InvalidData,
211                format!("surrogate block overflow: count={count}"),
212            )
213        })?;
214        let pad_bytes = vec_pad(vec_bytes);
215        let expected = HEADER_SIZE
216            .checked_add(vec_bytes)
217            .and_then(|n| n.checked_add(pad_bytes))
218            .and_then(|n| n.checked_add(sid_bytes))
219            .and_then(|n| n.checked_add(FOOTER_SIZE))
220            .ok_or_else(|| {
221                std::io::Error::new(
222                    std::io::ErrorKind::InvalidData,
223                    "total segment size overflow",
224                )
225            })?;
226        if file_size != expected {
227            return Err(std::io::Error::new(
228                std::io::ErrorKind::InvalidData,
229                format!("segment size mismatch: expected {expected} bytes, got {file_size}"),
230            ));
231        }
232
233        // Validate footer.
234        let footer_start = file_size - FOOTER_SIZE;
235        let footer = unsafe { std::slice::from_raw_parts(base.add(footer_start), FOOTER_SIZE) };
236
237        // Trailing magic — last 4 bytes of the file must be b"NDVS".
238        if &footer[42..46] != MAGIC.as_slice() {
239            return Err(std::io::Error::new(
240                std::io::ErrorKind::InvalidData,
241                "invalid NDVS trailing magic bytes",
242            ));
243        }
244
245        let footer_fv = u16::from_le_bytes([footer[0], footer[1]]);
246        if footer_fv != FORMAT_VERSION {
247            return Err(std::io::Error::new(
248                std::io::ErrorKind::InvalidData,
249                format!(
250                    "unsupported segment footer version {footer_fv}; expected {FORMAT_VERSION}"
251                ),
252            ));
253        }
254        let stored_footer_size =
255            u32::from_le_bytes([footer[38], footer[39], footer[40], footer[41]]) as usize;
256        if stored_footer_size != FOOTER_SIZE {
257            return Err(std::io::Error::new(
258                std::io::ErrorKind::InvalidData,
259                format!("footer_size field {stored_footer_size} != {FOOTER_SIZE}"),
260            ));
261        }
262
263        // CRC32C integrity check.
264        let body = unsafe { std::slice::from_raw_parts(base, footer_start) };
265        let computed = crc32c::crc32c(body);
266        let stored = u32::from_le_bytes([footer[34], footer[35], footer[36], footer[37]]);
267        if computed != stored {
268            return Err(std::io::Error::new(
269                std::io::ErrorKind::InvalidData,
270                format!("CRC32C mismatch: stored {stored:#010x}, computed {computed:#010x}"),
271            ));
272        }
273
274        let vec_offset = HEADER_SIZE;
275        let sid_offset = HEADER_SIZE + vec_bytes + pad_bytes;
276
277        // Advise MADV_RANDOM: HNSW traversal is non-sequential.
278        let mut madvise_state = None;
279        if vec_bytes + sid_bytes > 0 {
280            let rc =
281                unsafe { libc::madvise(base as *mut libc::c_void, file_size, libc::MADV_RANDOM) };
282            if rc == 0 {
283                madvise_state = Some(libc::MADV_RANDOM);
284                observability::RANDOM_COUNT.fetch_add(1, Ordering::Relaxed);
285            } else {
286                tracing::warn!(
287                    path = %path.display(),
288                    errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0),
289                    "madvise(MADV_RANDOM) failed on vector segment; continuing with kernel default",
290                );
291            }
292        }
293
294        Ok(Self {
295            path: path.to_path_buf(),
296            _fd: fd,
297            base,
298            mmap_size: file_size,
299            dim,
300            count,
301            vec_offset,
302            sid_offset,
303            drop_policy: policy,
304            madvise_state,
305            _reservation: reservation,
306        })
307    }
308
309    // ── Accessors ─────────────────────────────────────────────────────────────
310
311    /// The madvise hint set on this segment (if any).
312    pub fn madvise_state(&self) -> Option<libc::c_int> {
313        self.madvise_state
314    }
315
316    /// Get a vector by local index. Returns a slice into the mmap'd region.
317    #[inline]
318    pub fn get_vector(&self, id: u32) -> Option<&[f32]> {
319        let idx = id as usize;
320        if idx >= self.count {
321            return None;
322        }
323        let byte_len = self.dim.checked_mul(4)?;
324        let offset = self.vec_offset.checked_add(idx.checked_mul(byte_len)?)?;
325        let end = offset.checked_add(byte_len)?;
326        if end > self.sid_offset {
327            return None;
328        }
329        unsafe {
330            let ptr = self.base.add(offset) as *const f32;
331            Some(std::slice::from_raw_parts(ptr, self.dim))
332        }
333    }
334
335    /// Get the surrogate ID for a local index (0-based row in this segment).
336    #[inline]
337    pub fn get_surrogate_id(&self, id: u32) -> Option<u64> {
338        let idx = id as usize;
339        if idx >= self.count {
340            return None;
341        }
342        let offset = self.sid_offset.checked_add(idx.checked_mul(8)?)?;
343        let end = offset.checked_add(8)?;
344        let sid_end = self.mmap_size - FOOTER_SIZE;
345        if end > sid_end {
346            return None;
347        }
348        let bytes = unsafe { std::slice::from_raw_parts(self.base.add(offset), 8) };
349        Some(u64::from_le_bytes(bytes.try_into().expect("invariant: from_raw_parts constructed with len=8, so try_into::<[u8;8]> always succeeds")))
350    }
351
352    /// The full vector data block as a contiguous `&[f32]` of length `D × N`.
353    ///
354    /// Suitable for SIMD distance computation over all vectors.
355    #[inline]
356    pub fn all_vectors_flat(&self) -> &[f32] {
357        let float_count = self.dim * self.count;
358        unsafe {
359            let ptr = self.base.add(self.vec_offset) as *const f32;
360            std::slice::from_raw_parts(ptr, float_count)
361        }
362    }
363
364    /// The full surrogate ID block as a contiguous `&[u64]` of length `N`.
365    ///
366    /// Parallel to `all_vectors_flat`: row `i` in vectors ↔ `surrogate_ids[i]`.
367    #[inline]
368    pub fn all_surrogate_ids(&self) -> &[u64] {
369        unsafe {
370            let ptr = self.base.add(self.sid_offset) as *const u64;
371            std::slice::from_raw_parts(ptr, self.count)
372        }
373    }
374
375    /// Prefetch a vector's page into memory via `madvise(MADV_WILLNEED)`.
376    pub fn prefetch(&self, id: u32) {
377        let idx = id as usize;
378        if idx >= self.count {
379            return;
380        }
381        let byte_len = match self.dim.checked_mul(4) {
382            Some(v) => v,
383            None => return,
384        };
385        let Some(idx_bytes) = idx.checked_mul(byte_len) else {
386            return;
387        };
388        let Some(offset) = self.vec_offset.checked_add(idx_bytes) else {
389            return;
390        };
391        if offset
392            .checked_add(byte_len)
393            .is_none_or(|e| e > self.sid_offset)
394        {
395            return;
396        }
397        let page_start = offset & !(4095);
398        let len = (byte_len + 4095) & !(4095);
399        unsafe {
400            libc::madvise(
401                self.base.add(page_start) as *mut libc::c_void,
402                len,
403                libc::MADV_WILLNEED,
404            );
405        }
406    }
407
408    /// Prefetch a batch of vector IDs.
409    pub fn prefetch_batch(&self, ids: &[u32]) {
410        for &id in ids {
411            self.prefetch(id);
412        }
413    }
414
415    pub fn dim(&self) -> usize {
416        self.dim
417    }
418
419    pub fn count(&self) -> usize {
420        self.count
421    }
422
423    pub fn path(&self) -> &Path {
424        &self.path
425    }
426
427    pub fn mmap_bytes(&self) -> usize {
428        self.mmap_size
429    }
430
431    pub fn file_size(&self) -> usize {
432        self.mmap_size
433    }
434}
435
436impl Drop for MmapVectorSegment {
437    fn drop(&mut self) {
438        if !self.base.is_null() && self.mmap_size > 0 {
439            if self.drop_policy.dontneed_on_drop() {
440                let data_bytes = self.mmap_size.saturating_sub(HEADER_SIZE + FOOTER_SIZE);
441                if data_bytes > 0 {
442                    unsafe {
443                        libc::madvise(
444                            self.base as *mut libc::c_void,
445                            self.mmap_size,
446                            libc::MADV_DONTNEED,
447                        );
448                    }
449                    observability::DONTNEED_COUNT.fetch_add(1, Ordering::Relaxed);
450                }
451            }
452            unsafe {
453                libc::munmap(self.base as *mut libc::c_void, self.mmap_size);
454            }
455        }
456    }
457}
458
459#[cfg(test)]
460mod tests {
461    use super::*;
462    use crate::test_support::test_memory;
463
464    /// Unwrap the `std::io::Error` inside a `VectorError::SegmentIo`.
465    fn io_err(err: VectorError) -> std::io::Error {
466        match err {
467            VectorError::SegmentIo(e) => e,
468            other => panic!("expected VectorError::SegmentIo, got: {other:?}"),
469        }
470    }
471
472    #[test]
473    fn create_and_read() {
474        let dir = tempfile::tempdir().unwrap();
475
476        let v0 = vec![1.0f32, 2.0, 3.0];
477        let v1 = vec![4.0f32, 5.0, 6.0];
478        let v2 = vec![7.0f32, 8.0, 9.0];
479        let surrogates = vec![10u64, 20, 30];
480
481        let seg = MmapVectorSegment::create_with_surrogates(
482            dir.path(),
483            "test.vseg",
484            3,
485            &[&v0, &v1, &v2],
486            &surrogates,
487            &test_memory(),
488        )
489        .unwrap();
490
491        assert_eq!(seg.dim(), 3);
492        assert_eq!(seg.count(), 3);
493        assert_eq!(seg.get_vector(0).unwrap(), &[1.0, 2.0, 3.0]);
494        assert_eq!(seg.get_vector(1).unwrap(), &[4.0, 5.0, 6.0]);
495        assert_eq!(seg.get_vector(2).unwrap(), &[7.0, 8.0, 9.0]);
496        assert!(seg.get_vector(3).is_none());
497        assert_eq!(seg.get_surrogate_id(0).unwrap(), 10);
498        assert_eq!(seg.get_surrogate_id(1).unwrap(), 20);
499        assert_eq!(seg.get_surrogate_id(2).unwrap(), 30);
500        assert!(seg.get_surrogate_id(3).is_none());
501    }
502
503    #[test]
504    fn flat_slices() {
505        let dir = tempfile::tempdir().unwrap();
506
507        let v0 = vec![1.0f32, 2.0, 3.0];
508        let v1 = vec![4.0f32, 5.0, 6.0];
509        let sids = vec![100u64, 200];
510
511        let seg = MmapVectorSegment::create_with_surrogates(
512            dir.path(),
513            "flat.vseg",
514            3,
515            &[&v0, &v1],
516            &sids,
517            &test_memory(),
518        )
519        .unwrap();
520
521        assert_eq!(seg.all_vectors_flat(), &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]);
522        assert_eq!(seg.all_surrogate_ids(), &[100u64, 200]);
523    }
524
525    #[test]
526    fn reopen_roundtrip() {
527        let dir = tempfile::tempdir().unwrap();
528        let path = dir.path().join("reopen.vseg");
529
530        let vectors: Vec<Vec<f32>> = (0..100)
531            .map(|i| vec![i as f32, (i as f32).sin(), (i as f32).cos()])
532            .collect();
533        let refs: Vec<&[f32]> = vectors.iter().map(|v| v.as_slice()).collect();
534        let sids: Vec<u64> = (0u64..100).collect();
535
536        MmapVectorSegment::create_with_surrogates(
537            dir.path(),
538            "reopen.vseg",
539            3,
540            &refs,
541            &sids,
542            &test_memory(),
543        )
544        .unwrap();
545
546        let seg = MmapVectorSegment::open(&path, &test_memory()).unwrap();
547        assert_eq!(seg.count(), 100);
548        for (i, v) in vectors.iter().enumerate() {
549            assert_eq!(seg.get_vector(i as u32).unwrap(), v.as_slice());
550            assert_eq!(seg.get_surrogate_id(i as u32).unwrap(), i as u64);
551        }
552    }
553
554    #[test]
555    fn no_surrogates_defaults_to_zero() {
556        let dir = tempfile::tempdir().unwrap();
557
558        let v = vec![1.0f32, 2.0];
559        let seg =
560            MmapVectorSegment::create(dir.path(), "nosid.vseg", 2, &[&v], &test_memory()).unwrap();
561        assert_eq!(seg.get_surrogate_id(0).unwrap(), 0);
562    }
563
564    #[test]
565    fn prefetch_does_not_crash() {
566        let dir = tempfile::tempdir().unwrap();
567
568        let v = vec![1.0f32; 768];
569        let seg =
570            MmapVectorSegment::create(dir.path(), "prefetch.vseg", 768, &[&v], &test_memory())
571                .unwrap();
572        seg.prefetch(0);
573        seg.prefetch(999);
574    }
575
576    #[test]
577    fn empty_segment() {
578        let dir = tempfile::tempdir().unwrap();
579
580        let seg =
581            MmapVectorSegment::create(dir.path(), "empty.vseg", 3, &[], &test_memory()).unwrap();
582        assert_eq!(seg.count(), 0);
583        assert!(seg.get_vector(0).is_none());
584        assert_eq!(seg.all_vectors_flat().len(), 0);
585        assert_eq!(seg.all_surrogate_ids().len(), 0);
586    }
587
588    #[test]
589    fn footer_golden_layout() {
590        let dir = tempfile::tempdir().unwrap();
591        let path = dir.path().join("golden.vseg");
592        let v = vec![1.0f32, 2.0, 3.0];
593        write_segment(dir.path(), "golden.vseg", 3, &[&v], &[42]).unwrap();
594        let data = std::fs::read(&path).unwrap();
595
596        // Footer starts at file_size - FOOTER_SIZE.
597        let footer_start = data.len() - FOOTER_SIZE;
598        let footer = &data[footer_start..];
599
600        // [0..2] format_version = FORMAT_VERSION.
601        let fv = u16::from_le_bytes([footer[0], footer[1]]);
602        assert_eq!(fv, FORMAT_VERSION);
603
604        // [34..38] checksum matches body CRC32C.
605        let body = &data[..footer_start];
606        let expected_crc = crc32c::crc32c(body);
607        let stored_crc = u32::from_le_bytes([footer[34], footer[35], footer[36], footer[37]]);
608        assert_eq!(stored_crc, expected_crc);
609
610        // [38..42] footer_size = FOOTER_SIZE (46).
611        let fs = u32::from_le_bytes([footer[38], footer[39], footer[40], footer[41]]) as usize;
612        assert_eq!(fs, FOOTER_SIZE);
613
614        // [42..46] trailing magic = b"NDVS".
615        assert_eq!(&footer[42..46], b"NDVS");
616    }
617
618    #[test]
619    fn trailing_magic_corruption_rejected() {
620        let dir = tempfile::tempdir().unwrap();
621        let path = dir.path().join("trailmagic.vseg");
622        let v = vec![1.0f32, 2.0, 3.0];
623        write_segment(dir.path(), "trailmagic.vseg", 3, &[&v], &[42]).unwrap();
624
625        let mut data = std::fs::read(&path).unwrap();
626        // Corrupt the last 4 bytes (trailing magic).
627        let last = data.len();
628        data[last - 4] = 0xde;
629        data[last - 3] = 0xad;
630        data[last - 2] = 0xbe;
631        data[last - 1] = 0xef;
632        std::fs::write(&path, &data).unwrap();
633
634        let result = MmapVectorSegment::open(&path, &test_memory());
635        assert!(result.is_err(), "expected trailing magic error");
636        let err = result.unwrap_err();
637        let err_string = err.to_string();
638        assert_eq!(io_err(err).kind(), std::io::ErrorKind::InvalidData);
639        assert!(
640            err_string.contains("trailing magic"),
641            "expected trailing magic message, got: {err_string}"
642        );
643    }
644
645    #[test]
646    fn footer_version_mismatch_rejected() {
647        let dir = tempfile::tempdir().unwrap();
648        let path = dir.path().join("fvmismatch.vseg");
649        let v = vec![1.0f32, 2.0, 3.0];
650        write_segment(dir.path(), "fvmismatch.vseg", 3, &[&v], &[42]).unwrap();
651
652        let mut data = std::fs::read(&path).unwrap();
653        // Corrupt the footer format version bytes to 99.
654        let footer_start = data.len() - FOOTER_SIZE;
655        let fv_bytes = 99u16.to_le_bytes();
656        data[footer_start] = fv_bytes[0];
657        data[footer_start + 1] = fv_bytes[1];
658        std::fs::write(&path, &data).unwrap();
659
660        let result = MmapVectorSegment::open(&path, &test_memory());
661        assert!(result.is_err(), "expected footer version mismatch error");
662        assert_eq!(
663            io_err(result.unwrap_err()).kind(),
664            std::io::ErrorKind::InvalidData
665        );
666    }
667
668    #[test]
669    fn crc_corruption_rejected() {
670        let dir = tempfile::tempdir().unwrap();
671        let path = dir.path().join("corrupt.vseg");
672
673        let v = vec![1.0f32, 2.0, 3.0];
674        write_segment(dir.path(), "corrupt.vseg", 3, &[&v], &[42]).unwrap();
675
676        let mut data = std::fs::read(&path).unwrap();
677        data[HEADER_SIZE] ^= 0xff;
678        std::fs::write(&path, &data).unwrap();
679
680        let result = MmapVectorSegment::open(&path, &test_memory());
681        assert!(result.is_err(), "expected CRC error");
682        assert_eq!(
683            io_err(result.unwrap_err()).kind(),
684            std::io::ErrorKind::InvalidData
685        );
686    }
687
688    #[test]
689    fn bad_magic_rejected() {
690        let dir = tempfile::tempdir().unwrap();
691        let path = dir.path().join("badmagic.vseg");
692
693        let v = vec![1.0f32, 2.0];
694        write_segment(dir.path(), "badmagic.vseg", 2, &[&v], &[]).unwrap();
695
696        let mut data = std::fs::read(&path).unwrap();
697        data[0] = b'X';
698        std::fs::write(&path, &data).unwrap();
699
700        let result = MmapVectorSegment::open(&path, &test_memory());
701        assert!(result.is_err());
702        assert_eq!(
703            io_err(result.unwrap_err()).kind(),
704            std::io::ErrorKind::InvalidData
705        );
706    }
707
708    #[test]
709    fn overflow_dim_count_rejected() {
710        let dir = tempfile::tempdir().unwrap();
711        let path = dir.path().join("overflow.vseg");
712
713        let dim: u32 = 0x40000001;
714        let count: u64 = 0x40000001;
715
716        let mut buf = Vec::new();
717        buf.extend_from_slice(&MAGIC);
718        buf.extend_from_slice(&FORMAT_VERSION.to_le_bytes());
719        buf.extend_from_slice(&0u16.to_le_bytes());
720        buf.extend_from_slice(&dim.to_le_bytes());
721        buf.extend_from_slice(&count.to_le_bytes());
722        buf.push(0u8); // dtype
723        buf.push(0u8); // codec None
724        buf.extend_from_slice(&[0u8; 6]); // reserved
725        std::fs::write(&path, &buf).unwrap();
726
727        let result = MmapVectorSegment::open(&path, &test_memory());
728        assert!(
729            result.is_err(),
730            "expected Err for overflow-inducing dim/count"
731        );
732    }
733
734    #[test]
735    fn zero_dim_with_nonzero_count_rejected() {
736        let dir = tempfile::tempdir().unwrap();
737        let path = dir.path().join("zerodim.vseg");
738
739        let dim: u32 = 0;
740        let count: u64 = 1000;
741
742        let mut buf = Vec::new();
743        buf.extend_from_slice(&MAGIC);
744        buf.extend_from_slice(&FORMAT_VERSION.to_le_bytes());
745        buf.extend_from_slice(&0u16.to_le_bytes());
746        buf.extend_from_slice(&dim.to_le_bytes());
747        buf.extend_from_slice(&count.to_le_bytes());
748        buf.push(0u8);
749        buf.push(0u8);
750        buf.extend_from_slice(&[0u8; 6]);
751        buf.extend_from_slice(&[0u8; 64]);
752        std::fs::write(&path, &buf).unwrap();
753
754        let result = MmapVectorSegment::open(&path, &test_memory());
755        assert!(result.is_err(), "expected Err for dim=0 with nonzero count");
756    }
757}
nodedb_vector/mmap_segment/reader.rs

nodedb_vector/mmap_segment/
reader.rs
