laurus 0.10.0

Unified search library for lexical, vector, and semantic retrieval
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
//! Flat vector index reader implementation.

use std::collections::HashMap;
use std::sync::Arc;

use crate::error::{LaurusError, Result};
use crate::storage::Storage;
use crate::vector::core::distance::DistanceMetric;
use crate::vector::core::quantization::QuantizedVectorMeta;
use crate::vector::core::vector::Vector;
use crate::vector::index::format::{
    FieldInterner, QuantHeader, VectorSegmentHeader, record_prefix_size,
};
use crate::vector::index::quantized_io::quantized_record_payload_size;
use crate::vector::index::quantized_storage::QuantizedVectorPool;
use crate::vector::reader::{ValidationReport, VectorIndexMetadata, VectorStats};
use crate::vector::reader::{VectorIndexReader, VectorIterator};

use crate::maintenance::deletion::DeletionBitmap;
/// Storage for vectors (in-memory or on-demand).
use crate::vector::index::storage::VectorStorage;

/// Reader for flat (brute-force) vector indexes.
#[derive(Debug)]
pub struct FlatVectorIndexReader {
    vectors: VectorStorage,
    /// `(doc_id, field_id)` per record; ids index [`Self::field_dict`]
    /// (Issue #633 PR-B — interned, no per-record heap `String`).
    vector_ids: Vec<(u64, u16)>,
    /// Per-segment field-name dictionary (synthesized at load for
    /// v1/v2 segments, taken from the header for v3).
    field_dict: Arc<[Arc<str>]>,
    dimension: usize,
    distance_metric: DistanceMetric,
    deletion_bitmap: Option<Arc<DeletionBitmap>>,
    /// Pre-built per-field doc-id list (`field_name → Arc<[u64]>`). Built
    /// once at load so `doc_ids_for_field` returns a refcount-shared
    /// slice without re-cloning `vector_ids`. #405.
    vector_ids_by_field: std::collections::HashMap<String, Arc<[u64]>>,
    /// Stage 2 rerank sidecar pool (Issue #481, extended to Flat by
    /// #650 PR-2 / #932). `Some` only when the `.f32` sidecar exists and
    /// the loading mode is Eager; absence keeps Stage 1 behavior.
    rerank_storage: Option<Arc<crate::vector::index::rerank_storage::RerankStoragePool>>,
}

/// Group `vector_ids` by field name into refcount-shared slices.
fn build_vector_ids_by_field(
    vector_ids: &[(u64, u16)],
    field_dict: &[Arc<str>],
) -> std::collections::HashMap<String, Arc<[u64]>> {
    let mut by_field: Vec<Vec<u64>> = vec![Vec::new(); field_dict.len()];
    for &(doc_id, fid) in vector_ids {
        by_field[fid as usize].push(doc_id);
    }
    field_dict
        .iter()
        .zip(by_field)
        .map(|(field, ids)| (field.to_string(), Arc::<[u64]>::from(ids)))
        .collect()
}

impl FlatVectorIndexReader {
    /// Create a reader from serialized bytes.
    pub fn from_bytes(_data: &[u8]) -> Result<Self> {
        Err(LaurusError::InvalidOperation(
            "from_bytes is deprecated, use load() instead".to_string(),
        ))
    }

    /// Load a flat vector index from storage.
    ///
    /// # Arguments
    ///
    /// * `storage` - Shared storage backend (cloned into `OnDemand` for concurrent reads).
    /// * `path` - Base path/name for the index file (`.flat` extension is appended).
    /// * `distance_metric` - Distance metric used for similarity computations.
    ///
    /// # Returns
    ///
    /// A new `FlatIndexReader` instance.
    ///
    /// # Errors
    ///
    /// Returns [`LaurusError`] on I/O or format errors.
    pub fn load(
        storage: Arc<dyn Storage>,
        path: &str,
        distance_metric: DistanceMetric,
    ) -> Result<Self> {
        use crate::vector::index::alloc_bounds::checked_capacity;
        use std::io::{Read, Seek};

        // Open the index file
        let file_name = format!("{}.flat", path);
        let mut input = storage.open_input(&file_name)?;

        // Ground truth for bounding allocations sized from the unverified
        // header counts below (Issue #806). The `.flat` reader has no
        // pre-parse checksum, so every header count reaches its allocation
        // unverified.
        let file_size = input.size()?;

        // Read metadata
        let mut num_vectors_buf = [0u8; 4];
        input.read_exact(&mut num_vectors_buf)?;
        let num_vectors = u32::from_le_bytes(num_vectors_buf) as usize;

        let mut dimension_buf = [0u8; 4];
        input.read_exact(&mut dimension_buf)?;
        let dimension = u32::from_le_bytes(dimension_buf) as usize;

        // Read the Issue #481 Stage 1 vector segment header (LVS1).
        // Pre-Stage-1 segments are rejected with IncompatibleFormat.
        // Matched by reference so `header` (version + field dictionary,
        // Issue #633) stays alive for the record parse below.
        // Issue #921: pass the bytes physically left in the file so the
        // header's PQ codebook allocation is bounded before it reserves.
        let header_available =
            file_size.saturating_sub(input.stream_position().map_err(LaurusError::Io)?);
        let header = VectorSegmentHeader::read_from(&mut input, header_available)?;
        let params = match &header.quant {
            QuantHeader::Scalar8Bit(p) => *p,
            QuantHeader::ProductQuantization { .. } => {
                return Err(crate::error::LaurusError::NotImplemented(
                    "Product quantization (Issue #481 Stage 3) is HNSW-only; \
                     the Flat reader does not support PQ segments yet"
                        .to_string(),
                ));
            }
            #[cfg(feature = "pq-fastscan")]
            QuantHeader::ProductQuantizationFastScan { .. } => {
                return Err(crate::error::LaurusError::NotImplemented(
                    "PQ FastScan (#695) is HNSW-only; the Flat reader does not \
                     support PQ FastScan segments"
                        .to_string(),
                ));
            }
        };

        // Bytes left for the per-vector records section, captured once at its
        // start (Issue #806). Each record is at least the version-dependent
        // prefix (doc_id + field reference, Issue #633) + the fixed quantized
        // payload (dim int8 + 8 meta), so this stride also bounds the
        // per-record `dimension`-sized int8 read.
        let records_remaining =
            file_size.saturating_sub(input.stream_position().map_err(LaurusError::Io)?);
        let record_stride =
            record_prefix_size(header.version) + quantized_record_payload_size(dimension) as u64;

        // Interned field ids (Issue #633 PR-B): one shared dictionary per
        // segment instead of one heap `String` per record.
        let mut interner = FieldInterner::from_header(&header);

        let (vectors, vector_ids, field_dict) = match storage.loading_mode() {
            crate::storage::LoadingMode::Eager => {
                // Step 7 of #481 Stage 1: load vectors as int8 + meta
                // directly into a QuantizedVectorPool.
                checked_capacity(
                    num_vectors,
                    record_stride,
                    records_remaining,
                    "flat num_vectors",
                )?;
                let mut vector_ids = Vec::with_capacity(num_vectors);
                let mut records: Vec<(u64, String, Vec<u8>, QuantizedVectorMeta)> =
                    Vec::with_capacity(num_vectors);

                for _ in 0..num_vectors {
                    let mut doc_id_buf = [0u8; 8];
                    input.read_exact(&mut doc_id_buf)?;
                    let doc_id = u64::from_le_bytes(doc_id_buf);

                    // Field reference: interned id (v3 = dictionary id,
                    // v1/v2 = inline name interned on first appearance).
                    let fid = interner.read_record_field_id(
                        &header,
                        &mut input,
                        records_remaining,
                        "flat field_name_len",
                    )?;

                    // Read int8 + meta directly (no dequantize).
                    let mut int8 = vec![0u8; dimension];
                    input.read_exact(&mut int8)?;
                    let mut sum_q_buf = [0u8; 4];
                    let mut norm_q_buf = [0u8; 4];
                    input.read_exact(&mut sum_q_buf)?;
                    input.read_exact(&mut norm_q_buf)?;
                    let meta = QuantizedVectorMeta {
                        sum_q: u32::from_le_bytes(sum_q_buf),
                        norm_q: f32::from_le_bytes(norm_q_buf),
                    };

                    vector_ids.push((doc_id, fid));
                    // The pool's build input keeps the String shape; this
                    // clone is transient (the pool retains only per-field
                    // grouping keys), so nothing per-record is kept.
                    records.push((doc_id, interner.name(fid).to_string(), int8, meta));
                }
                let pool = QuantizedVectorPool::build(params, dimension, records);
                (
                    VectorStorage::OwnedQuantized(Arc::new(pool)),
                    vector_ids,
                    interner.into_dict(),
                )
            }
            crate::storage::LoadingMode::Lazy => {
                checked_capacity(
                    num_vectors,
                    record_stride,
                    records_remaining,
                    "flat num_vectors",
                )?;
                let mut offsets = HashMap::with_capacity(num_vectors);
                let mut vector_ids = Vec::with_capacity(num_vectors);

                // Seek to the start of the per-vector entries: Flat preamble
                // (count u32 + dim u32 = 8 bytes) + the parsed header's real
                // size (which includes the v3 field dictionary, Issue #633 —
                // reconstructing a fresh header here would omit it).
                let start_pos = 8u64 + header.serialized_size() as u64;
                input
                    .seek(std::io::SeekFrom::Start(start_pos))
                    .map_err(LaurusError::Io)?;

                let quant_payload_size = quantized_record_payload_size(dimension) as i64;

                for _ in 0..num_vectors {
                    let mut doc_id_buf = [0u8; 8];
                    input.read_exact(&mut doc_id_buf)?;
                    let doc_id = u64::from_le_bytes(doc_id_buf);

                    let fid = interner.read_record_field_id(
                        &header,
                        &mut input,
                        records_remaining,
                        "flat field_name_len",
                    )?;

                    // Offsets point at the payload start (right after the
                    // record prefix), so `VectorStorage::get` seeks straight
                    // to the int8 data without re-parsing the prefix.
                    let payload_offset = input.stream_position().map_err(LaurusError::Io)?;
                    offsets.insert((doc_id, fid), payload_offset);
                    vector_ids.push((doc_id, fid));

                    // Skip int8 payload + per-vector meta.
                    input
                        .seek(std::io::SeekFrom::Current(quant_payload_size))
                        .map_err(LaurusError::Io)?;
                }

                let field_dict = interner.into_dict();
                (
                    VectorStorage::OnDemand {
                        storage: storage.clone(),
                        file_name: file_name.clone(),
                        offsets: Arc::new(offsets),
                        field_dict: field_dict.clone(),
                        quant_params: Some(params),
                        cached_input: Arc::new(std::sync::RwLock::new(None)),
                    },
                    vector_ids,
                    field_dict,
                )
            }
        };

        let vector_ids_by_field = build_vector_ids_by_field(&vector_ids, &field_dict);

        // Stage 2 rerank sidecar (Issue #481, extended to Flat by #650
        // PR-2 / #932): loaded eagerly when present, mirroring HNSW. The
        // pool's positions pair with `vector_ids` (the record order the
        // writer also used for the sidecar payload) — an identity mapping.
        // Lazy mode skips the sidecar to honor its memory-savings promise.
        let rerank_storage = crate::vector::index::rerank_sidecar::load_rerank_sidecar(
            storage.as_ref(),
            &file_name,
            dimension,
            &vector_ids,
            &field_dict,
        )?;

        Ok(Self {
            vectors,
            vector_ids,
            field_dict,
            dimension,
            distance_metric,
            deletion_bitmap: None,
            vector_ids_by_field,
            rerank_storage,
        })
    }

    /// Borrow the optional Stage 2 rerank storage pool (#932).
    pub fn rerank_storage(
        &self,
    ) -> Option<&Arc<crate::vector::index::rerank_storage::RerankStoragePool>> {
        self.rerank_storage.as_ref()
    }

    pub fn set_deletion_bitmap(&mut self, bitmap: Arc<DeletionBitmap>) {
        self.deletion_bitmap = Some(bitmap);
    }

    /// Borrow the underlying [`VectorStorage`] so the Flat searcher
    /// can detect the [`VectorStorage::OwnedQuantized`] variant and
    /// switch to the int8 hot path (Issue #481 Stage 1 Step 7).
    pub fn vectors(&self) -> &VectorStorage {
        &self.vectors
    }

    /// Whether `doc_id` is logically deleted per the attached bitmap (if
    /// any). `pub(crate)` so the searcher's per-storage-mode fast paths
    /// (e.g. the quantized-pool scan in [`crate::vector::index::flat::searcher`],
    /// which reads straight from the quantized pool and bypasses
    /// [`VectorIndexReader::get_vector`]) can filter deleted docs without
    /// going through that slower, allocation-heavy accessor.
    pub(crate) fn is_deleted(&self, doc_id: u64) -> bool {
        if let Some(bitmap) = &self.deletion_bitmap {
            bitmap.is_deleted(doc_id)
        } else {
            false
        }
    }
}

impl VectorIndexReader for FlatVectorIndexReader {
    fn as_any(&self) -> &dyn std::any::Any {
        self
    }

    fn get_vector(&self, doc_id: u64, field_name: &str) -> Result<Option<Vector>> {
        if self.is_deleted(doc_id) {
            return Ok(None);
        }
        self.vectors.get(doc_id, field_name, self.dimension)
    }

    fn get_vectors_for_doc(&self, doc_id: u64) -> Result<Vec<(String, Vector)>> {
        let mut result = Vec::new();
        for &(id, fid) in &self.vector_ids {
            let field = &self.field_dict[fid as usize];
            if id == doc_id
                && !self.is_deleted(id)
                && let Some(vec) = self.vectors.get(id, field, self.dimension)?
            {
                result.push((field.to_string(), vec));
            }
        }
        Ok(result)
    }

    fn get_vectors(&self, doc_ids: &[(u64, String)]) -> Result<Vec<Option<Vector>>> {
        let mut result = Vec::with_capacity(doc_ids.len());
        for (id, field) in doc_ids {
            if self.is_deleted(*id) {
                result.push(None);
            } else {
                result.push(self.vectors.get(*id, field, self.dimension)?);
            }
        }
        Ok(result)
    }

    fn vector_ids(&self) -> Result<Vec<(u64, String)>> {
        // Rehydrated at the trait boundary (Issue #633 PR-B): internal
        // state is `(u64, u16)` + the shared dictionary.
        Ok(self
            .vector_ids
            .iter()
            .map(|&(id, fid)| (id, self.field_dict[fid as usize].to_string()))
            .collect())
    }

    fn doc_ids_for_field(&self, field_name: &str) -> Arc<[u64]> {
        // O(1) HashMap lookup + Arc clone (refcount bump). Default impl
        // would clone `vector_ids` (Vec<(u64, String)>) and filter
        // linearly per call. #405.
        self.vector_ids_by_field
            .get(field_name)
            .cloned()
            .unwrap_or_else(|| Vec::<u64>::new().into())
    }

    fn vector_count(&self) -> usize {
        self.vectors.len()
    }

    fn dimension(&self) -> usize {
        self.dimension
    }

    fn distance_metric(&self) -> DistanceMetric {
        self.distance_metric
    }

    fn stats(&self) -> VectorStats {
        let _memory_usage = match &self.vectors {
            VectorStorage::Owned(vectors) => vectors.len() * (8 + self.dimension * 4),
            VectorStorage::OwnedQuantized(pool) => pool.heap_size(),
            VectorStorage::OwnedPq(pool) => pool.data.len() + pool.codebook.len() * 4,
            #[cfg(feature = "pq-fastscan")]
            VectorStorage::OwnedPqFastScan(_) => {
                unreachable!("Flat reader rejects PQ FastScan at the segment header (HNSW-only)")
            }
            VectorStorage::OnDemand { offsets, .. } => {
                // Estimate memory for offsets map + ID list
                offsets.len() * (8 + 32 + 8) // Key + Valid + Offset roughly
            }
        };
        VectorStats {
            vector_count: self.vectors.len(),
            dimension: self.dimension,
            memory_usage: self.vectors.len() * (8 + self.dimension * 4),
            build_time_ms: 0,
        }
    }

    fn contains_vector(&self, doc_id: u64, field_name: &str) -> bool {
        self.vectors.contains(doc_id, field_name)
    }

    fn get_vector_range(
        &self,
        start_doc_id: u64,
        end_doc_id: u64,
    ) -> Result<Vec<(u64, String, Vector)>> {
        let mut result = Vec::new();
        for &(id, fid) in &self.vector_ids {
            let field = &self.field_dict[fid as usize];
            if id >= start_doc_id
                && id < end_doc_id
                && !self.is_deleted(id)
                && let Some(vec) = self.vectors.get(id, field, self.dimension)?
            {
                result.push((id, field.to_string(), vec));
            }
        }
        Ok(result)
    }

    fn get_vectors_by_field(&self, field_name: &str) -> Result<Vec<(u64, Vector)>> {
        // One dictionary resolve, then integer compares per record.
        let Some(target) =
            crate::vector::index::format::resolve_field_id(&self.field_dict, field_name)
        else {
            return Ok(Vec::new());
        };
        let mut result = Vec::new();
        for &(id, fid) in &self.vector_ids {
            if fid == target
                && !self.is_deleted(id)
                && let Some(vec) = self.vectors.get(id, field_name, self.dimension)?
            {
                result.push((id, vec));
            }
        }
        Ok(result)
    }

    fn field_names(&self) -> Result<Vec<String>> {
        Ok(self.field_dict.iter().map(|f| f.to_string()).collect())
    }

    fn vector_iterator(&self) -> Result<Box<dyn VectorIterator>> {
        Ok(Box::new(FlatVectorIterator {
            storage: self.vectors.clone(),
            keys: self.vector_ids.clone(),
            field_dict: self.field_dict.clone(),
            current: 0,
            dimension: self.dimension,
            deletion_bitmap: self.deletion_bitmap.clone(),
        }))
    }

    fn metadata(&self) -> Result<VectorIndexMetadata> {
        Ok(VectorIndexMetadata {
            index_type: "flat".to_string(),
            created_at: chrono::Utc::now(),
            modified_at: chrono::Utc::now(),
            version: "1".to_string(),
            build_config: serde_json::json!({}),
            custom_metadata: std::collections::HashMap::new(),
        })
    }

    fn validate(&self) -> Result<ValidationReport> {
        let mut errors = Vec::new();
        let mut warnings = Vec::new();

        if self.vector_ids.len() != self.vectors.len() {
            errors.push(format!(
                "Mismatch between vector_ids count ({}) and vectors count ({})",
                self.vector_ids.len(),
                self.vectors.len()
            ));
        }

        match &self.vectors {
            VectorStorage::Owned(map) => {
                for &(id, fid) in &self.vector_ids {
                    let field = &self.field_dict[fid as usize];
                    let (id, field) = (&id, &field.to_string());
                    if let Some(vector) = map.get(&(*id, field.clone())) {
                        if vector.dimension() != self.dimension {
                            errors.push(format!(
                                "Vector {}:{} has dimension {}, expected {}",
                                id,
                                field,
                                vector.dimension(),
                                self.dimension
                            ));
                        }
                        if !vector.is_valid() {
                            errors.push(format!(
                                "Vector {}:{} contains invalid values (NaN or infinity)",
                                id, field
                            ));
                        }
                    } else {
                        errors.push(format!(
                            "Vector {}:{} found in keys but missing in storage",
                            id, field
                        ));
                    }
                }
            }
            VectorStorage::OwnedQuantized(pool) => {
                for &(id, fid) in &self.vector_ids {
                    let field = &self.field_dict[fid as usize];
                    let id = &id;
                    if !pool.contains(*id, field) {
                        errors.push(format!(
                            "Vector {}:{} found in keys but missing in quantized pool",
                            id, field
                        ));
                    }
                }
                warnings.push(
                    "OwnedQuantized mode: dimension / NaN checks skipped (int8 storage \
                     guarantees finite values within [offset, offset + 255*scale])"
                        .to_string(),
                );
            }
            VectorStorage::OwnedPq(pool) => {
                for &(id, fid) in &self.vector_ids {
                    let field = &self.field_dict[fid as usize];
                    let id = &id;
                    if !pool.contains(*id, field) {
                        errors.push(format!(
                            "Vector {}:{} found in keys but missing in PQ pool",
                            id, field
                        ));
                    }
                }
                warnings.push(
                    "OwnedPq mode: dimension / NaN checks skipped (codes index into \
                     the trained codebook which is bounded by construction)"
                        .to_string(),
                );
            }
            #[cfg(feature = "pq-fastscan")]
            VectorStorage::OwnedPqFastScan(_) => {
                unreachable!("Flat reader rejects PQ FastScan at the segment header (HNSW-only)")
            }
            VectorStorage::OnDemand { offsets, .. } => {
                for &(id, fid) in &self.vector_ids {
                    let field = &self.field_dict[fid as usize];
                    let id = &id;
                    if !offsets.contains_key(&(*id, fid)) {
                        errors.push(format!(
                            "Vector {}:{} in ids but missing in storage",
                            id, field
                        ));
                    }
                }
                warnings.push("OnDemand mode: Deep vector validation skipped".to_string());
            }
        }

        Ok(ValidationReport {
            repair_suggestions: Vec::new(),
            is_valid: errors.is_empty(),
            errors,
            warnings,
        })
    }
}

/// Iterator for flat vector index.
struct FlatVectorIterator {
    storage: VectorStorage,
    keys: Vec<(u64, u16)>,
    field_dict: Arc<[Arc<str>]>,
    current: usize,
    dimension: usize,
    deletion_bitmap: Option<Arc<DeletionBitmap>>,
}

impl VectorIterator for FlatVectorIterator {
    fn next(&mut self) -> Result<Option<(u64, String, Vector)>> {
        // Use a loop instead of recursion to avoid stack overflow when
        // many consecutive entries are deleted.
        while self.current < self.keys.len() {
            let (doc_id, fid) = self.keys[self.current];
            let field = &self.field_dict[fid as usize];

            // Skip deleted entries
            if let Some(bitmap) = &self.deletion_bitmap
                && bitmap.is_deleted(doc_id)
            {
                self.current += 1;
                continue;
            }

            if let Some(vec) = self.storage.get(doc_id, field, self.dimension)? {
                self.current += 1;
                return Ok(Some((doc_id, field.to_string(), vec)));
            } else {
                return Err(LaurusError::internal(format!(
                    "Vector {}:{} found in keys but missing in storage",
                    doc_id, field
                )));
            }
        }

        Ok(None)
    }

    fn skip_to(&mut self, doc_id: u64, field_name: &str) -> Result<bool> {
        while self.current < self.keys.len() {
            let (id, fid) = self.keys[self.current];
            let field = &self.field_dict[fid as usize];
            if id > doc_id || (id == doc_id && field.as_ref() as &str >= field_name) {
                return Ok(true);
            }
            self.current += 1;
        }
        Ok(false)
    }

    fn position(&self) -> (u64, String) {
        if self.current < self.keys.len() {
            let (id, fid) = self.keys[self.current];
            (id, self.field_dict[fid as usize].to_string())
        } else {
            (u64::MAX, String::new())
        }
    }

    fn reset(&mut self) -> Result<()> {
        self.current = 0;
        Ok(())
    }
}

#[cfg(test)]
mod alloc_bound_tests {
    use super::*;
    use crate::storage::memory::{MemoryStorage, MemoryStorageConfig};
    use crate::vector::core::quantization::ScalarQuantParams;
    use std::io::Write;

    /// Build an in-memory storage holding `bytes` under `name`.
    fn storage_with(name: &str, bytes: Vec<u8>) -> Arc<dyn Storage> {
        let storage = MemoryStorage::new(MemoryStorageConfig::default());
        let mut out = storage.create_output(name).unwrap();
        out.write_all(&bytes).unwrap();
        out.flush_and_sync().unwrap();
        Arc::new(storage)
    }

    /// Serialized neutral LVS1 (Scalar8Bit) header bytes.
    fn neutral_header_bytes() -> Vec<u8> {
        let mut buf = Vec::new();
        VectorSegmentHeader::scalar_8bit(ScalarQuantParams {
            offset: 0.0,
            scale: 1.0,
        })
        .write_to(&mut buf)
        .unwrap();
        buf
    }

    #[test]
    fn load_rejects_oversized_num_vectors_without_aborting() {
        // A `.flat` segment whose `num_vectors` field is corrupted to a huge
        // value while the file holds no records must be rejected cleanly,
        // never drive a multi-GiB `Vec::with_capacity` that aborts the
        // process via `handle_alloc_error` (Issue #806).
        let mut bytes = Vec::new();
        bytes.extend_from_slice(&u32::MAX.to_le_bytes()); // num_vectors (corrupt)
        bytes.extend_from_slice(&4u32.to_le_bytes()); // dimension
        bytes.extend_from_slice(&neutral_header_bytes()); // LVS1 header, no records

        let storage = storage_with("corrupt.flat", bytes);
        let err = FlatVectorIndexReader::load(storage, "corrupt", DistanceMetric::Cosine)
            .expect_err("oversized num_vectors must be rejected as corruption");
        match err {
            LaurusError::Index(msg) => {
                assert!(msg.contains("num_vectors"), "got: {msg}");
                assert!(msg.contains("corrupted"), "got: {msg}");
            }
            other => panic!("expected Index error, got {other:?}"),
        }
    }
}