second-brain-core 0.13.2

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
use std::fs;
use std::io::ErrorKind;
use std::path::{Path, PathBuf};
use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};

use anyhow::{Context, Result, anyhow};
use usearch::{Index, IndexOptions, MetricKind, ScalarKind};

use crate::sealed::{self, SealError, SealKeys};

/// BGE-small output width; must match the FLOAT[384] the store schema fixes.
pub const EMBEDDING_DIM: usize = 384;

const INITIAL_CAPACITY: usize = 1024;

/// `ReadWrite` indexes are RAM-loaded via `load()` and mutable. `View`
/// indexes are mmapped read-only via usearch's `view()`, so opening
/// thousands of tenant indexes for search never pays the RAM-load cost.
/// `Loaded` indexes are read-only copies loaded into RAM, for sealed files,
/// which cannot be mmapped. `Shared` indexes search a writer's in-memory
/// index and see its changes as they are made.
#[derive(Clone, Copy, PartialEq, Eq)]
enum VectorMode {
    ReadWrite,
    View,
    Loaded,
    Shared,
}

pub struct VectorIndex {
    // Search, add and remove take the read side, which usearch runs
    // concurrently. Anything that reallocates or replaces the index
    // (`reserve`, a load, a rebuild's swap) takes the write side, because a
    // search beside it would read freed memory.
    index: Arc<RwLock<Index>>,
    path: PathBuf,
    mode: VectorMode,
    scalar_kind: ScalarKind,
    seal: Option<SealKeys>,
}

fn new_index(scalar_kind: ScalarKind) -> Result<Index> {
    let options = IndexOptions {
        dimensions: EMBEDDING_DIM,
        metric: MetricKind::Cos,
        quantization: scalar_kind,
        ..IndexOptions::default()
    };
    Index::new(&options).map_err(|e| anyhow!("creating usearch index: {e}"))
}

fn reserved_index(scalar_kind: ScalarKind) -> Result<Index> {
    let index = new_index(scalar_kind)?;
    index
        .reserve(INITIAL_CAPACITY)
        .map_err(|e| anyhow!("reserving usearch capacity: {e}"))?;
    Ok(index)
}

/// usearch reinterprets an index file's stored bytes at whatever width the
/// caller's options declare rather than refusing a mismatch, so opening an
/// I8-quantized file with F32 options would silently produce garbage
/// distances instead of an error. The on-disk header carries the true kind
/// (`Index::metadata` reads only that header), so it is checked against what
/// the caller asked for before `load`/`view` ever runs.
fn check_on_disk_kind(path: &Path, path_str: &str, expected: ScalarKind) -> Result<()> {
    check_kind(path, read_on_disk_kind(path, path_str)?, expected)
}

fn check_kind(path: &Path, actual: ScalarKind, expected: ScalarKind) -> Result<()> {
    if actual != expected {
        return Err(anyhow!(
            "vector index at {} was written as {:?} but is being opened as {:?}",
            path.display(),
            actual,
            expected
        ));
    }
    Ok(())
}

const KEY_TABLE_SLOTS_PER_BUCKET: usize = 64;
const KEY_TABLE_BUCKET_HEADER_BYTES: usize = 2 * size_of::<u64>();
/// A u64 key beside a u32 slot, padded to the key's alignment.
const KEY_TABLE_ENTRY_BYTES: usize =
    (size_of::<u64>() + size_of::<u32>()).next_multiple_of(align_of::<u64>());

/// Mirrors how usearch 2.26 sizes its key-to-slot hash table for
/// `capacity` keys.
fn key_table_bytes(capacity: usize) -> usize {
    if capacity == 0 {
        return 0;
    }
    let slots = (capacity * 3).div_ceil(2).next_power_of_two();
    let buckets = slots.div_ceil(KEY_TABLE_SLOTS_PER_BUCKET);
    buckets * (KEY_TABLE_SLOTS_PER_BUCKET * KEY_TABLE_ENTRY_BYTES + KEY_TABLE_BUCKET_HEADER_BYTES)
}

fn read_on_disk_kind(path: &Path, path_str: &str) -> Result<ScalarKind> {
    refuse_sealed(path)?;
    Ok(Index::metadata(path_str)
        .map_err(|e| anyhow!("reading usearch header at {}: {e}", path.display()))?
        .quantization)
}

fn refuse_sealed(path: &Path) -> Result<()> {
    if sealed::sealed_key_id(path)?.is_some() {
        return Err(anyhow::Error::new(SealError::KeyRequired).context(format!(
            "vector index at {} is sealed and was opened without keys",
            path.display()
        )));
    }
    Ok(())
}

fn file_name(path: &Path) -> Result<&str> {
    path.file_name()
        .and_then(|n| n.to_str())
        .with_context(|| format!("{} has no utf-8 file name", path.display()))
}

/// Opens a plaintext file as-is, which is how an index written before
/// sealing is adopted. The decrypted bytes live only until the load returns.
fn load_keyed(path: &Path, scalar_kind: ScalarKind, keys: &SealKeys) -> Result<Index> {
    let bytes = sealed::read_file(path, Some(keys), file_name(path)?)?;
    let actual = Index::metadata_from_buffer(&bytes)
        .map_err(|e| anyhow!("reading usearch header at {}: {e}", path.display()))?
        .quantization;
    check_kind(path, actual, scalar_kind)?;
    let index = new_index(scalar_kind)?;
    index
        .load_from_buffer(&bytes)
        .map_err(|e| anyhow!("loading usearch index from {}: {e}", path.display()))?;
    Ok(index)
}

/// Removes the temporary files a save killed mid-write leaves beside
/// `path`; the unkeyed ones hold plaintext vectors.
fn remove_stale_tmp_files(path: &Path) -> Result<()> {
    let Some(dir) = path.parent() else {
        return Ok(());
    };
    let name = file_name(path)?;
    let entries = match fs::read_dir(dir) {
        Ok(entries) => entries,
        Err(e) if e.kind() == ErrorKind::NotFound => return Ok(()),
        Err(e) => return Err(e).with_context(|| format!("listing {}", dir.display())),
    };
    for entry in entries {
        let entry = entry?;
        let Some(entry_name) = entry.file_name().to_str().map(str::to_owned) else {
            continue;
        };
        let Some(rest) = entry_name.strip_prefix(name) else {
            continue;
        };
        if rest == ".tmp" || sealed::is_tmp_name(rest) {
            fs::remove_file(entry.path())
                .with_context(|| format!("removing {}", entry.path().display()))?;
        }
    }
    Ok(())
}

impl VectorIndex {
    /// Errors for a sealed file, whose header only its key can read.
    pub fn on_disk_kind(path: &Path) -> Result<Option<ScalarKind>> {
        if !path.exists() {
            return Ok(None);
        }
        let path_str = path.to_str().context("vector index path is not utf-8")?;
        read_on_disk_kind(path, path_str).map(Some)
    }

    fn with_index(
        index: Index,
        path: &Path,
        mode: VectorMode,
        scalar_kind: ScalarKind,
        seal: Option<SealKeys>,
    ) -> Self {
        Self {
            index: Arc::new(RwLock::new(index)),
            path: path.to_path_buf(),
            mode,
            scalar_kind,
            seal,
        }
    }

    /// Every on-disk index that predates the scalar-kind option was written
    /// as `F32`, which is exactly what `Indexes::open` still passes.
    pub fn open_with_kind(path: &Path, scalar_kind: ScalarKind) -> Result<Self> {
        let path_str = path.to_str().context("vector index path is not utf-8")?;
        let index = if path.exists() {
            check_on_disk_kind(path, path_str, scalar_kind)?;
            let index = new_index(scalar_kind)?;
            index
                .load(path_str)
                .map_err(|e| anyhow!("loading usearch index from {path_str}: {e}"))?;
            index
        } else {
            reserved_index(scalar_kind)?
        };
        Ok(Self::with_index(
            index,
            path,
            VectorMode::ReadWrite,
            scalar_kind,
            None,
        ))
    }

    /// Same as `open_with_kind`, with the file read and saved sealed under
    /// `keys`. A plaintext file opens as-is and is sealed at the next save.
    pub fn open_with_kind_keyed(
        path: &Path,
        scalar_kind: ScalarKind,
        keys: &SealKeys,
    ) -> Result<Self> {
        remove_stale_tmp_files(path)?;
        let index = if path.exists() {
            load_keyed(path, scalar_kind, keys)?
        } else {
            reserved_index(scalar_kind)?
        };
        Ok(Self::with_index(
            index,
            path,
            VectorMode::ReadWrite,
            scalar_kind,
            Some(keys.clone()),
        ))
    }

    /// Mmaps the index file read-only via usearch's `view()`: pages are
    /// faulted in on search instead of the whole file loading up front, so a
    /// multi-tenant reader can hold thousands of these open at once. Errors
    /// rather than creating a file, because a reader that can create a
    /// tenant's index is a reader that can corrupt one.
    pub fn open_view_with_kind(path: &Path, scalar_kind: ScalarKind) -> Result<Self> {
        if !path.exists() {
            return Err(anyhow!(
                "vector index view: {} does not exist",
                path.display()
            ));
        }
        let path_str = path.to_str().context("vector index path is not utf-8")?;
        check_on_disk_kind(path, path_str, scalar_kind)?;
        let index = new_index(scalar_kind)?;
        index
            .view(path_str)
            .map_err(|e| anyhow!("viewing usearch index at {path_str}: {e}"))?;
        Ok(Self::with_index(
            index,
            path,
            VectorMode::View,
            scalar_kind,
            None,
        ))
    }

    /// A read-only view over nothing yet, for an index file a writer has not
    /// persisted. `reload_view` maps the file in once it exists.
    pub fn empty_view_with_kind(path: &Path, scalar_kind: ScalarKind) -> Result<Self> {
        Ok(Self::with_index(
            new_index(scalar_kind)?,
            path,
            VectorMode::View,
            scalar_kind,
            None,
        ))
    }

    /// The read-only counterpart of `open_view_with_kind` for a sealed file:
    /// loaded whole into RAM, since a sealed file cannot be mmapped.
    pub fn open_loaded_with_kind(
        path: &Path,
        scalar_kind: ScalarKind,
        keys: &SealKeys,
    ) -> Result<Self> {
        if !path.exists() {
            return Err(anyhow!("vector index: {} does not exist", path.display()));
        }
        Ok(Self::with_index(
            load_keyed(path, scalar_kind, keys)?,
            path,
            VectorMode::Loaded,
            scalar_kind,
            Some(keys.clone()),
        ))
    }

    /// The `Loaded` counterpart of `empty_view_with_kind`.
    pub fn empty_loaded_with_kind(
        path: &Path,
        scalar_kind: ScalarKind,
        keys: &SealKeys,
    ) -> Result<Self> {
        Ok(Self::with_index(
            new_index(scalar_kind)?,
            path,
            VectorMode::Loaded,
            scalar_kind,
            Some(keys.clone()),
        ))
    }

    /// A read-only handle on this index's memory: it sees every change made
    /// through this handle at once, with no save or reload between.
    pub fn shared_reader(&self) -> Self {
        Self {
            index: Arc::clone(&self.index),
            path: self.path.clone(),
            mode: VectorMode::Shared,
            scalar_kind: self.scalar_kind,
            seal: self.seal.clone(),
        }
    }

    /// An empty writable index that saves where this one does, for a
    /// rebuild to fill before `adopt` swaps it in.
    pub fn detached(&self) -> Result<Self> {
        self.require_writable()?;
        Ok(Self::with_index(
            reserved_index(self.scalar_kind)?,
            &self.path,
            VectorMode::ReadWrite,
            self.scalar_kind,
            self.seal.clone(),
        ))
    }

    /// True while a reader made by `shared_reader` still reads this index.
    pub fn has_shared_readers(&self) -> bool {
        Arc::strong_count(&self.index) > 1
    }

    /// Replaces this index's contents with `fresh`'s in one step, so every
    /// shared reader goes from the old contents to the new ones.
    pub fn adopt(&mut self, fresh: VectorIndex) -> Result<()> {
        self.require_writable()?;
        let index = Arc::try_unwrap(fresh.index)
            .map_err(|_| anyhow!("an index with shared readers cannot be adopted"))?
            .into_inner()
            .unwrap_or_else(PoisonError::into_inner);
        *self.write() = index;
        Ok(())
    }

    /// Files saved from now on are sealed under `keys`' current key.
    pub fn set_seal_keys(&mut self, keys: SealKeys) {
        self.seal = Some(keys);
    }

    pub fn scalar_kind(&self) -> ScalarKind {
        self.scalar_kind
    }

    fn read(&self) -> RwLockReadGuard<'_, Index> {
        self.index.read().unwrap_or_else(PoisonError::into_inner)
    }

    fn write(&self) -> RwLockWriteGuard<'_, Index> {
        self.index.write().unwrap_or_else(PoisonError::into_inner)
    }

    fn require_writable(&self) -> Result<()> {
        match self.mode {
            VectorMode::ReadWrite => Ok(()),
            VectorMode::View | VectorMode::Loaded | VectorMode::Shared => Err(anyhow!(
                "vector index at {} is read-only; mutation is not permitted",
                self.path.display()
            )),
        }
    }

    /// Replace-then-add so a replayed outbox row cannot duplicate a key.
    pub fn upsert(&mut self, key: u64, embedding: &[f32]) -> Result<()> {
        self.require_writable()?;
        self.ensure_capacity(1)?;
        let index = self.read();
        index
            .remove(key)
            .map_err(|e| anyhow!("removing key {key}: {e}"))?;
        index
            .add(key, embedding)
            .map_err(|e| anyhow!("adding key {key}: {e}"))?;
        Ok(())
    }

    pub fn remove(&mut self, key: u64) -> Result<()> {
        self.require_writable()?;
        self.read()
            .remove(key)
            .map_err(|e| anyhow!("removing key {key}: {e}"))?;
        Ok(())
    }

    /// Returns (key, cosine similarity) pairs, best first.
    pub fn search(&self, query: &[f32], k: usize) -> Result<Vec<(u64, f32)>> {
        let matches = self
            .read()
            .search(query, k)
            .map_err(|e| anyhow!("usearch search: {e}"))?;
        Ok(matches
            .keys
            .iter()
            .zip(matches.distances.iter())
            .map(|(key, distance)| (*key, 1.0 - *distance))
            .collect())
    }

    pub fn len(&self) -> usize {
        self.read().size()
    }

    /// A relatively accurate lower bound on usearch's own RAM usage. A
    /// `View` index reports far less than the same data `ReadWrite`-loaded,
    /// since its vectors and graph stay mmapped rather than resident.
    pub fn memory_usage(&self) -> usize {
        self.read().memory_usage()
    }

    /// Anonymous resident bytes. `memory_usage` counts the unused tails of
    /// usearch's lazily faulted arena mappings, which never become resident,
    /// and leaves out the key table and the per-slot vector pointers, both
    /// sized to `capacity`. A view's vectors and graph are the mapped file.
    /// A shared handle reports none: they are its writer's.
    pub(crate) fn anonymous_bytes(&self) -> u64 {
        if self.mode == VectorMode::Shared {
            return 0;
        }
        let index = self.read();
        let stats = index.memory_stats();
        let arenas = index
            .memory_usage()
            .saturating_sub(stats.graph_reserved)
            .saturating_sub(stats.vectors_reserved);
        let capacity = index.capacity();
        let vector_pointers = capacity * size_of::<*const u8>();
        (arenas + key_table_bytes(capacity) + vector_pointers) as u64
    }

    /// Writes to a sibling temporary file, fsyncs it, then renames it onto
    /// the live path. Rename is atomic on the same filesystem, so a `View`
    /// reader with the old file already mmapped never observes a
    /// half-written index: it keeps its existing mapping until
    /// `reload_view` re-views the (now new) file whole.
    pub fn save(&self) -> Result<()> {
        self.require_writable()?;
        match &self.seal {
            Some(keys) => self.save_sealed(keys),
            None => self.save_plain(),
        }
    }

    fn save_plain(&self) -> Result<()> {
        let tmp_path = self.path.with_extension("usearch.tmp");
        let tmp_str = tmp_path
            .to_str()
            .context("vector index tmp path is not utf-8")?;
        self.read()
            .save(tmp_str)
            .map_err(|e| anyhow!("saving usearch index to {tmp_str}: {e}"))?;
        let tmp_file = std::fs::File::open(&tmp_path)
            .with_context(|| format!("reopening {} to fsync", tmp_path.display()))?;
        tmp_file
            .sync_all()
            .with_context(|| format!("fsyncing {}", tmp_path.display()))?;
        drop(tmp_file);
        std::fs::rename(&tmp_path, &self.path).with_context(|| {
            format!("renaming {} to {}", tmp_path.display(), self.path.display())
        })?;
        Ok(())
    }

    fn save_sealed(&self, keys: &SealKeys) -> Result<()> {
        let bytes = {
            let index = self.read();
            let mut bytes = vec![0u8; index.serialized_length()];
            index
                .save_to_buffer(&mut bytes)
                .map_err(|e| anyhow!("serializing usearch index: {e}"))?;
            bytes
        };
        sealed::write_file(&self.path, Some(keys), file_name(&self.path)?, &bytes)
    }

    /// Re-reads the live file so a `View` or `Loaded` index picks up
    /// whatever a writer last saved. A no-op on `ReadWrite`, which already
    /// holds the authoritative in-memory copy, and on `Shared`, which reads
    /// that copy.
    pub fn reload_view(&mut self) -> Result<()> {
        match self.mode {
            VectorMode::View => {
                refuse_sealed(&self.path)?;
                let path_str = self
                    .path
                    .to_str()
                    .context("vector index path is not utf-8")?;
                self.write()
                    .view(path_str)
                    .map_err(|e| anyhow!("reloading usearch view at {path_str}: {e}"))?;
            }
            VectorMode::Loaded => {
                let keys = self
                    .seal
                    .as_ref()
                    .context("a loaded vector index holds its keys")?;
                let fresh = load_keyed(&self.path, self.scalar_kind, keys)?;
                *self.write() = fresh;
            }
            VectorMode::ReadWrite | VectorMode::Shared => {}
        }
        Ok(())
    }

    fn ensure_capacity(&self, extra: usize) -> Result<()> {
        let (needed, capacity) = {
            let index = self.read();
            (index.size() + extra, index.capacity())
        };
        if needed > capacity {
            self.write()
                .reserve((needed * 2).max(INITIAL_CAPACITY))
                .map_err(|e| anyhow!("growing usearch capacity: {e}"))?;
        }
        Ok(())
    }

    #[cfg(test)]
    fn contains(&self, key: u64) -> bool {
        self.read().contains(key)
    }
}
#[cfg(test)]
mod tests {
    use super::*;
    use tempfile::tempdir;

    fn unit_vector(hot: usize) -> Vec<f32> {
        let mut v = vec![0.0f32; EMBEDDING_DIM];
        v[hot] = 1.0;
        v
    }

    #[test]
    fn key_table_bytes_follow_the_capacity_usearch_reports() {
        assert_eq!(key_table_bytes(0), 0);
        assert_eq!(key_table_bytes(1), 64 * 16 + 16);
        let index = new_index(ScalarKind::F32).unwrap();
        index.reserve(INITIAL_CAPACITY).unwrap();
        assert!(index.capacity() >= INITIAL_CAPACITY);
        assert_eq!(
            key_table_bytes(index.capacity()),
            key_table_bytes(INITIAL_CAPACITY)
        );
    }

    #[test]
    fn anonymous_bytes_leave_out_the_unfaulted_arena_tail() {
        let dir = tempdir().unwrap();
        let mut idx =
            VectorIndex::open_with_kind(&dir.path().join("vectors.usearch"), ScalarKind::F32)
                .unwrap();
        idx.upsert(0, &unit_vector(0)).unwrap();
        assert!(
            idx.memory_usage() >= 4 << 20,
            "one vector maps a whole arena"
        );
        assert!(
            idx.anonymous_bytes() < 1 << 20,
            "only the touched head of the arena is resident: {}",
            idx.anonymous_bytes()
        );
        assert!(idx.anonymous_bytes() as usize >= EMBEDDING_DIM * 4);
    }

    #[test]
    fn f32_index_writes_and_reads_back() {
        let dir = tempdir().unwrap();
        let path = dir.path().join("vectors.usearch");

        let mut idx = VectorIndex::open_with_kind(&path, ScalarKind::F32).unwrap();
        assert_eq!(idx.scalar_kind(), ScalarKind::F32);
        for i in 0..5u64 {
            idx.upsert(i, &unit_vector(i as usize)).unwrap();
        }
        idx.save().unwrap();

        let reopened = VectorIndex::open_with_kind(&path, ScalarKind::F32).unwrap();
        assert_eq!(reopened.scalar_kind(), ScalarKind::F32);
        let hits = reopened.search(&unit_vector(2), 1).unwrap();
        assert_eq!(hits[0].0, 2);

        let viewed = VectorIndex::open_view_with_kind(&path, ScalarKind::F32).unwrap();
        let hits = viewed.search(&unit_vector(3), 1).unwrap();
        assert_eq!(hits[0].0, 3);
    }

    #[test]
    fn opening_f32_file_as_a_different_kind_errors_instead_of_corrupting() {
        let dir = tempdir().unwrap();
        let path = dir.path().join("vectors.usearch");

        let mut idx = VectorIndex::open_with_kind(&path, ScalarKind::F32).unwrap();
        idx.upsert(0, &unit_vector(0)).unwrap();
        idx.save().unwrap();

        let err = VectorIndex::open_with_kind(&path, ScalarKind::I8)
            .err()
            .expect("opening an F32 file as I8 must error");
        assert!(err.to_string().contains("F32"), "error was: {err}");

        let err = VectorIndex::open_view_with_kind(&path, ScalarKind::F16)
            .err()
            .expect("viewing an F32 file as F16 must error");
        assert!(err.to_string().contains("F32"), "error was: {err}");
    }

    #[test]
    fn i8_and_f16_kinds_round_trip() {
        for kind in [ScalarKind::I8, ScalarKind::F16] {
            let dir = tempdir().unwrap();
            let path = dir.path().join("vectors.usearch");

            let mut idx = VectorIndex::open_with_kind(&path, kind).unwrap();
            for i in 0..5u64 {
                idx.upsert(i, &unit_vector(i as usize)).unwrap();
            }
            idx.save().unwrap();

            let viewed = VectorIndex::open_view_with_kind(&path, kind).unwrap();
            let hits = viewed.search(&unit_vector(4), 1).unwrap();
            assert_eq!(hits[0].0, 4, "kind {kind:?} lost the nearest match");
        }
    }

    fn keys(byte: u8) -> SealKeys {
        SealKeys::single(
            crate::sealed::SealKey::from_bytes(crate::keys::KeyGuard::new([byte; 32])).unwrap(),
        )
    }

    fn spread_vector(key: u64) -> Vec<f32> {
        let mut v: Vec<f32> = (0..EMBEDDING_DIM)
            .map(|j| ((key as usize * 31 + j * 17) % 97) as f32 - 48.0)
            .collect();
        v[key as usize % EMBEDDING_DIM] += 500.0;
        v
    }

    #[test]
    fn a_shared_reader_searches_while_the_writer_adds_and_grows() {
        const ADDED: u64 = 5_000;
        let dir = tempdir().unwrap();
        let mut writer =
            VectorIndex::open_with_kind(&dir.path().join("vectors.usearch"), ScalarKind::F32)
                .unwrap();
        let initial_capacity = writer.read().capacity();
        let reader = writer.shared_reader();
        let stop = Arc::new(std::sync::atomic::AtomicBool::new(false));
        let (done_tx, done_rx) = std::sync::mpsc::channel();
        let searcher = {
            let stop = Arc::clone(&stop);
            std::thread::spawn(move || {
                let mut searches = 0u64;
                let mut outcome = Ok(());
                while !stop.load(std::sync::atomic::Ordering::Acquire) {
                    if let Err(e) = reader.search(&spread_vector(searches % ADDED), 10) {
                        outcome = Err(format!("{e:#}"));
                        break;
                    }
                    searches += 1;
                }
                let _ = done_tx.send((outcome, searches));
            })
        };

        for key in 0..ADDED {
            writer.upsert(key, &spread_vector(key)).unwrap();
        }
        stop.store(true, std::sync::atomic::Ordering::Release);
        let (outcome, searches) = done_rx
            .recv_timeout(std::time::Duration::from_secs(60))
            .expect("the searching thread did not stop within 60 s");
        searcher.join().unwrap();

        outcome.unwrap();
        assert!(searches > 0, "no search ran beside the adds");
        assert!(
            writer.read().capacity() >= 4 * initial_capacity,
            "the adds never grew the index more than once"
        );
        assert_eq!(writer.len(), ADDED as usize);
        for key in 0..ADDED {
            assert!(writer.contains(key), "key {key} was lost");
        }
    }

    fn random_vector(seed: u64) -> Vec<f32> {
        use rand::{Rng, SeedableRng};
        let mut rng = rand::rngs::StdRng::seed_from_u64(seed);
        (0..EMBEDDING_DIM)
            .map(|_| rng.random_range(-1.0f32..1.0))
            .collect()
    }

    #[test]
    fn shared_readers_search_while_the_writer_updates_and_removes() {
        const KEYS: u64 = 2_000;
        let dir = tempdir().unwrap();
        let mut writer =
            VectorIndex::open_with_kind(&dir.path().join("vectors.usearch"), ScalarKind::F32)
                .unwrap();
        for key in 0..KEYS {
            writer.upsert(key, &random_vector(key)).unwrap();
        }
        let stop = Arc::new(std::sync::atomic::AtomicBool::new(false));
        let (done_tx, done_rx) = std::sync::mpsc::channel();
        let searchers: Vec<_> = (0..2u64)
            .map(|offset| {
                let reader = writer.shared_reader();
                let stop = Arc::clone(&stop);
                let done_tx = done_tx.clone();
                std::thread::spawn(move || {
                    let mut searches = 0u64;
                    let mut outcome = Ok(());
                    while !stop.load(std::sync::atomic::Ordering::Acquire) {
                        let probe = random_vector((searches * 7 + offset) % (2 * KEYS));
                        if let Err(e) = reader.search(&probe, 10) {
                            outcome = Err(format!("{e:#}"));
                            break;
                        }
                        searches += 1;
                    }
                    let _ = done_tx.send((outcome, searches));
                })
            })
            .collect();
        drop(done_tx);

        for key in 0..KEYS {
            if key % 3 == 0 {
                writer.remove(key).unwrap();
            } else {
                writer.upsert(key, &random_vector(key + KEYS)).unwrap();
            }
        }
        stop.store(true, std::sync::atomic::Ordering::Release);
        for _ in 0..2 {
            let (outcome, searches) = done_rx
                .recv_timeout(std::time::Duration::from_secs(60))
                .expect("a searching thread did not stop within 60 s");
            outcome.unwrap();
            assert!(searches > 0, "no search ran beside the updates");
        }
        for searcher in searchers {
            searcher.join().unwrap();
        }

        for key in 0..KEYS {
            assert_eq!(writer.contains(key), key % 3 != 0, "key {key}");
        }
        for key in (1..KEYS).filter(|k| k % 3 != 0).step_by(97) {
            let hits = writer.search(&random_vector(key + KEYS), 10).unwrap();
            assert!(
                hits.iter().any(|(hit, _)| *hit == key),
                "the update of key {key} was lost: {hits:?}"
            );
        }
    }

    #[test]
    fn a_sealed_vector_file_round_trips_and_refuses_unkeyed_opens() {
        let dir = tempdir().unwrap();
        let path = dir.path().join("vectors.usearch");
        let mut idx = VectorIndex::open_with_kind_keyed(&path, ScalarKind::F32, &keys(1)).unwrap();
        for i in 0..5u64 {
            idx.upsert(i, &unit_vector(i as usize)).unwrap();
        }
        idx.save().unwrap();
        assert!(crate::sealed::sealed_key_id(&path).unwrap().is_some());

        let reopened = VectorIndex::open_with_kind_keyed(&path, ScalarKind::F32, &keys(1)).unwrap();
        assert_eq!(reopened.search(&unit_vector(2), 1).unwrap()[0].0, 2);
        let loaded = VectorIndex::open_loaded_with_kind(&path, ScalarKind::F32, &keys(1)).unwrap();
        assert_eq!(loaded.search(&unit_vector(3), 1).unwrap()[0].0, 3);

        for err in [
            VectorIndex::on_disk_kind(&path).err(),
            VectorIndex::open_with_kind(&path, ScalarKind::F32).err(),
            VectorIndex::open_view_with_kind(&path, ScalarKind::F32).err(),
        ] {
            let err = err.expect("a sealed file must not open without its key");
            assert_eq!(
                crate::sealed::seal_error(&err),
                Some(&SealError::KeyRequired),
                "error was: {err:#}"
            );
        }

        let err = VectorIndex::open_with_kind_keyed(&path, ScalarKind::I8, &keys(1))
            .err()
            .expect("opening a sealed F32 file as I8 must error");
        assert!(err.to_string().contains("F32"), "error was: {err}");
    }

    #[test]
    fn a_keyed_open_removes_the_temporary_files_a_killed_save_left() {
        let dir = tempdir().unwrap();
        let path = dir.path().join("vectors.usearch");
        let plain_tmp = dir.path().join("vectors.usearch.tmp");
        let sealed_tmp = dir.path().join("vectors.usearch.seal-tmp.0123456789abcdef");
        let other = dir.path().join("chunks.usearch.tmp");
        for p in [&plain_tmp, &sealed_tmp, &other] {
            fs::write(p, b"leftover").unwrap();
        }
        VectorIndex::open_with_kind_keyed(&path, ScalarKind::F32, &keys(1)).unwrap();
        assert!(!plain_tmp.exists());
        assert!(!sealed_tmp.exists());
        assert!(other.exists(), "another index's file was removed");
    }
}