Skip to main content

updating/
updating.rs

1// Copyright 2026 Elias Benali (@ebenali) and TheCleaners.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Keeping a scann-core index up to date from Rust: upsert (insert and
16//! update), delete, rebalance, and save over an existing index.
17//!
18//! `ScannIndex` identifies points by row index. Deleting moves the last
19//! row into each freed slot, and `delete` returns those moves, so an
20//! application-side id table (here, `docids`) stays in step.
21//!
22//! Run through CMake (`cmake --build <build> --target scann_core_rust`
23//! builds it), or directly:
24//!   SCANN_CORE_BUILD_ENV=<build>/rust/scann_core_rust_build.env \
25//!     cargo run --release --example updating
26
27use scann_core::{
28    AhOptions, ConfigBuilder, DistanceMeasure, ReorderOptions, ScannError, ScannIndex,
29    SearchOptions, TreeOptions,
30};
31
32const DIM: usize = 32;
33const LEAVES: u32 = 60;
34
35/// A small deterministic generator (xorshift), uniform in [-1, 1).
36struct Rng(u64);
37
38impl Rng {
39    fn new(seed: u64) -> Self {
40        Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1)
41    }
42    fn uniform(&mut self) -> f32 {
43        self.0 ^= self.0 << 13;
44        self.0 ^= self.0 >> 7;
45        self.0 ^= self.0 << 17;
46        (self.0 >> 11) as f32 / (1u64 << 53) as f32 * 2.0 - 1.0
47    }
48}
49
50/// `n` unit-norm points around 40 fixed centers, row-major n x DIM.
51fn make_data(n: usize, seed: u64) -> Vec<f32> {
52    let mut centers_rng = Rng::new(42);
53    let centers: Vec<f32> = (0..40 * DIM).map(|_| centers_rng.uniform()).collect();
54    let mut rng = Rng::new(seed);
55    let mut data = Vec::with_capacity(n * DIM);
56    for _ in 0..n {
57        let c = (rng.0 % 40) as usize;
58        let row: Vec<f32> =
59            centers[c * DIM..][..DIM].iter().map(|x| x + 0.2 * rng.uniform()).collect();
60        let norm = row.iter().map(|x| x * x).sum::<f32>().sqrt();
61        data.extend(row.iter().map(|x| x / norm));
62    }
63    data
64}
65
66/// The docid of each query's nearest stored point, searching every leaf and
67/// reordering 1000 candidates: effectively exact, so a stored vector always
68/// finds itself.
69fn nearest(index: &ScannIndex, docids: &[String], queries: &[f32]) -> Vec<String> {
70    let options =
71        SearchOptions::k(1).leaves_to_search(LEAVES as usize).pre_reorder_num_neighbors(1000);
72    let found = index.search_batched(queries, options).expect("search");
73    found.iter().map(|nn| docids[nn.indices[0] as usize].clone()).collect()
74}
75
76fn print_health(when: &str, index: &ScannIndex) -> Result<(), ScannError> {
77    let s = index.health_stats()?;
78    println!(
79        "{when:24} {:5} points, imbalance {:.3}, quantization error {:.4}",
80        index.len(),
81        s.partition_avg_relative_positive_imbalance,
82        s.avg_quantization_error
83    );
84    Ok(())
85}
86
87fn main() -> Result<(), Box<dyn std::error::Error>> {
88    let initial = make_data(3000, 1);
89    // The application's ids, one per row: row i of the index is docids[i].
90    let mut docids: Vec<String> = (0..3000).map(|i| format!("doc-{i}")).collect();
91
92    let builder = ConfigBuilder::new(10, DistanceMeasure::DotProduct, DIM)
93        .tree(TreeOptions::new(LEAVES, 10).random_init(false))
94        .score_ah(AhOptions::new(2).anisotropic_quantization_threshold(0.2))
95        .reorder(ReorderOptions::new(100));
96    let mut index = builder.build_index(&initial)?;
97    print_health("built:", &index)?;
98
99    // --- Insert ----------------------------------------------------------
100    // upsert: `None` adds a row, `Some(i)` replaces row i. The result is
101    // each row's index; new rows are appended. batch_size > 1 assigns the
102    // batch to leaves on the index's thread pool. An upsert (or delete) can
103    // trigger incremental maintenance and, when that asks for it, a full
104    // retrain, as in Python.
105    let added = make_data(1000, 2);
106    let added_docids: Vec<String> = (0..1000).map(|i| format!("new-{i}")).collect();
107    let new_rows = index.upsert(&[None; 1000], &added, 256)?;
108    assert_eq!(new_rows, (3000..4000).collect::<Vec<u32>>());
109    docids.extend(added_docids.iter().cloned());
110    assert_eq!(nearest(&index, &docids, &added), added_docids);
111    print_health("after inserting 1000:", &index)?;
112
113    // --- Update ----------------------------------------------------------
114    // Replace row 0's vector; it keeps its index (and so its docid).
115    let moved = make_data(1, 3);
116    index.upsert(&[Some(0)], &moved, 1)?;
117    assert_eq!(nearest(&index, &docids, &moved), ["doc-0"]);
118    assert_ne!(nearest(&index, &docids, &initial[..DIM]), ["doc-0"]);
119    println!("updated doc-0: found at its new vector, not at its old one");
120
121    // An index listed twice in one upsert is rejected before anything
122    // changes, like a repeated docid in Python.
123    let err = index.upsert(&[Some(1), Some(1)], &make_data(2, 4), 1).unwrap_err();
124    assert!(matches!(err, ScannError::InvalidArgument(_)));
125    println!("repeated index rejected: {err}");
126    assert_eq!(index.len(), 4000);
127
128    // --- Delete ----------------------------------------------------------
129    // delete takes indices as they are before the call and returns the
130    // (old index, new index) of every row that moved into a freed slot.
131    // Apply the moves to the id table, then drop its tail.
132    let doomed: Vec<u32> = (1..=500).collect();
133    let moves = index.delete(&doomed)?;
134    for &(from, to) in &moves {
135        docids[to as usize] = docids[from as usize].clone();
136    }
137    docids.truncate(index.len());
138    assert_eq!(index.len(), 3500);
139    println!("deleted {} points; {} others moved to new indices", doomed.len(), moves.len());
140    // The added points still find themselves, under their own docids.
141    assert_eq!(nearest(&index, &docids, &added), added_docids);
142    print_health("after deleting 500:", &index)?;
143
144    // --- Rebalance -------------------------------------------------------
145    // Retrains the tree and AH codebooks on the current data (a full
146    // retrain, as slow as a build) and recomputes the health stats. Indices
147    // don't change.
148    index.rebalance(None)?;
149    print_health("after rebalance:", &index)?;
150    assert_eq!(nearest(&index, &docids, &moved), ["doc-0"]);
151
152    // --- Save over an existing index, reload -----------------------------
153    // serialize replaces an index already in the directory: files are
154    // staged, then renamed into place with the manifest last, so an
155    // interrupted save leaves the old index, the new one, or a directory
156    // that `load` rejects; never a mix. `true`: relative paths, movable.
157    let dir = std::env::temp_dir().join(format!("scann-core-rust-updating-{}", std::process::id()));
158    std::fs::create_dir_all(&dir)?;
159    index.serialize(&dir, true)?;
160    index.add(&make_data(1, 5))?;
161    docids.push("late-arrival".to_string());
162    index.serialize(&dir, true)?;
163
164    let reloaded = ScannIndex::load(&dir)?;
165    assert_eq!(reloaded.len(), docids.len());
166    let queries = make_data(50, 6);
167    let a = index.search_batched(&queries, SearchOptions::default())?;
168    let b = reloaded.search_batched(&queries, SearchOptions::default())?;
169    assert_eq!(a, b);
170    println!("re-saved and reloaded {} points: identical results", reloaded.len());
171    // The ids are the application's to store (Python's scann_docids.pkl is
172    // written only by the Python wrapper).
173    std::fs::remove_dir_all(&dir)?;
174    Ok(())
175}