Struct Collection

Source

pub struct Collection { /* private fields */ }

Implementations§

Source §

impl Collection

Source

pub fn name(&self) -> &str

Source

pub fn config(&self) -> &CollectionConfig

Examples found in repository ?

examples/search_bench.rs (line 25)

21fn main() {
22    let args: Vec<String> = std::env::args().skip(1).collect();
23    let db = Database::open(&args[0]).expect("index");
24    let c = db.collections().next().expect("a collection");
25    let dim = c.config().dim;
26    let queries = read(&args[1], f32::from_le_bytes);
27    let truth = read(&args[2], u32::from_le_bytes);
28    let efs: Vec<usize> = args
29        .get(3)
30        .map_or("10,20,40,80,160", String::as_str)
31        .split(',')
32        .map(|e| e.parse().unwrap())
33        .collect();
34    let n = queries.len() / dim;
35
36    println!(
37        "{:>5}  {:>8}  {:>9}  {:>9}  {:>10}",
38        "ef", "recall", "mean µs", "p50 µs", "distances"
39    );
40    for ef in efs {
41        let options = SearchOptions::default().ef(ef);
42        let (mut times, mut found, mut distances) = (Vec::with_capacity(n), 0, 0);
43        for (q, t) in queries.chunks_exact(dim).zip(truth.chunks_exact(10)) {
44            let start = Instant::now();
45            let report = c.explain(q, 10, &options).unwrap();
46            times.push(start.elapsed().as_secs_f64() * 1e6);
47            distances += report.distance_computations;
48            found += report
49                .hits
50                .iter()
51                .filter(|h| t.contains(&h.id.parse::<u32>().unwrap()))
52                .count();
53        }
54        let mean = times.iter().sum::<f64>() / n as f64;
55        times.sort_by(f64::total_cmp);
56        println!(
57            "{ef:>5}  {:>8.4}  {mean:>9.1}  {:>9.1}  {:>10}",
58            found as f64 / (n * 10) as f64,
59            times[n / 2],
60            distances / n
61        );
62    }
63}

Source

pub fn len(&self) -> usize

Number of live records.

Source

pub fn is_empty(&self) -> bool

Source

pub fn contains(&self, id: &str) -> bool

Source

pub fn upsert( &mut self, id: &str, vector: &[f32], metadata: Option<Value>, ) -> Result<()>

Inserts a record, replacing any record with the same id.

Examples found in repository ?

examples/bench.rs (line 74)

55fn main() -> recern_vector::Result<()> {
56    let args: Vec<String> = std::env::args().skip(1).collect();
57    let arg = |i: usize| {
58        args.get(i)
59            .map(|a| a.parse::<f64>().expect("numeric argument"))
60    };
61    let records = arg(0).map_or(50_000, |v| v as usize);
62    let dim = arg(1).map_or(128, |v| v as usize);
63    let noise = arg(2).map_or(1.0, |v| v as f32);
64    let path = std::env::temp_dir().join("recern-vector-bench.rvec");
65    let _ = std::fs::remove_file(&path);
66
67    let mut data = Clusters::new(100, dim, noise, 0x5EED);
68    let vectors: Vec<Vec<f32>> = (0..records).map(|_| data.sample()).collect();
69
70    let mut db = Database::create(&path)?;
71    let collection = db.create_collection("bench", CollectionConfig::new(dim, Metric::Cosine))?;
72    let start = Instant::now();
73    for (i, v) in vectors.iter().enumerate() {
74        collection.upsert(&i.to_string(), v, None)?;
75    }
76    let build = start.elapsed();
77
78    let start = Instant::now();
79    db.save()?;
80    let save = start.elapsed();
81    let start = Instant::now();
82    let db = Database::open(&path)?;
83    let open = start.elapsed();
84    let collection = db.collection("bench")?;
85    let stats = collection.stats();
86
87    println!("records {records} · dim {dim} · noise {noise} · cosine · m=16 ef_construction=200");
88    println!(
89        "build   {:.2} s ({:.0} inserts/s)",
90        build.as_secs_f64(),
91        records as f64 / build.as_secs_f64()
92    );
93    println!(
94        "file    {:.1} MB · save {:.0} ms · open {:.0} ms",
95        std::fs::metadata(&path)?.len() as f64 / 1_048_576.0,
96        save.as_secs_f64() * 1e3,
97        open.as_secs_f64() * 1e3
98    );
99    println!(
100        "graph   layers {:?} · avg degree L0 {:.1} · unreachable {}",
101        stats.nodes_per_layer, stats.avg_degree_layer0, stats.unreachable
102    );
103
104    let report = collection.estimate_recall(&RecallOptions {
105        sample: 200,
106        k: 10,
107        ef_values: vec![10, 16, 32, 64, 128, 256],
108        seed: 7,
109    })?;
110    println!();
111    println!(
112        "exact scan p50 {:.3} ms",
113        report.exact_p50.as_secs_f64() * 1e3
114    );
115    println!(
116        "{:>6}  {:>9}  {:>10}  {:>10}  {:>9}",
117        "ef", "recall@10", "p50 ms", "p95 ms", "speedup"
118    );
119    for p in &report.points {
120        println!(
121            "{:>6}  {:>9.3}  {:>10.3}  {:>10.3}  {:>8.0}x",
122            p.ef,
123            p.recall,
124            p.p50.as_secs_f64() * 1e3,
125            p.p95.as_secs_f64() * 1e3,
126            report.exact_p50.as_secs_f64() / p.p50.as_secs_f64()
127        );
128    }
129
130    let _ = std::fs::remove_file(&path);
131    Ok(())
132}

Source

pub fn upsert_many<I, S, V>(&mut self, records: I) -> Result<usize>
where I: IntoIterator<Item = (S, V, Option<Value>)>, S: Into<String>, V: AsRef<[f32]>,

Inserts many records, replacing records with the same ids, and links them into the index using all available cores. The batch is atomic: if any vector is invalid, the collection is left unchanged. Returns the number of records written.

Source

pub fn upsert_many_with_threads<I, S, V>( &mut self, records: I, threads: usize, ) -> Result<usize>
where I: IntoIterator<Item = (S, V, Option<Value>)>, S: Into<String>, V: AsRef<[f32]>,

Collection::upsert_many with an explicit thread count. With one thread the graph is identical to upserting the records one by one; with more, link order (and so the exact graph) depends on scheduling.

Source

pub fn delete(&mut self, id: &str) -> bool

Removes a record. Returns whether it existed.

The node stays in the graph as a waypoint until Collection::compact.

Source

pub fn get(&self, id: &str) -> Option<Record>

Source

pub fn search( &self, query: &[f32], k: usize, options: &SearchOptions, ) -> Result<Vec<SearchHit>>

Source

pub fn explain( &self, query: &[f32], k: usize, options: &SearchOptions, ) -> Result<SearchReport>

Runs a search and reports how it was executed.

Examples found in repository ?

examples/search_bench.rs (line 45)

21fn main() {
22    let args: Vec<String> = std::env::args().skip(1).collect();
23    let db = Database::open(&args[0]).expect("index");
24    let c = db.collections().next().expect("a collection");
25    let dim = c.config().dim;
26    let queries = read(&args[1], f32::from_le_bytes);
27    let truth = read(&args[2], u32::from_le_bytes);
28    let efs: Vec<usize> = args
29        .get(3)
30        .map_or("10,20,40,80,160", String::as_str)
31        .split(',')
32        .map(|e| e.parse().unwrap())
33        .collect();
34    let n = queries.len() / dim;
35
36    println!(
37        "{:>5}  {:>8}  {:>9}  {:>9}  {:>10}",
38        "ef", "recall", "mean µs", "p50 µs", "distances"
39    );
40    for ef in efs {
41        let options = SearchOptions::default().ef(ef);
42        let (mut times, mut found, mut distances) = (Vec::with_capacity(n), 0, 0);
43        for (q, t) in queries.chunks_exact(dim).zip(truth.chunks_exact(10)) {
44            let start = Instant::now();
45            let report = c.explain(q, 10, &options).unwrap();
46            times.push(start.elapsed().as_secs_f64() * 1e6);
47            distances += report.distance_computations;
48            found += report
49                .hits
50                .iter()
51                .filter(|h| t.contains(&h.id.parse::<u32>().unwrap()))
52                .count();
53        }
54        let mean = times.iter().sum::<f64>() / n as f64;
55        times.sort_by(f64::total_cmp);
56        println!(
57            "{ef:>5}  {:>8.4}  {mean:>9.1}  {:>9.1}  {:>10}",
58            found as f64 / (n * 10) as f64,
59            times[n / 2],
60            distances / n
61        );
62    }
63}

Source

pub fn stats(&self) -> CollectionStats

Examples found in repository ?

examples/bench.rs (line 85)

55fn main() -> recern_vector::Result<()> {
56    let args: Vec<String> = std::env::args().skip(1).collect();
57    let arg = |i: usize| {
58        args.get(i)
59            .map(|a| a.parse::<f64>().expect("numeric argument"))
60    };
61    let records = arg(0).map_or(50_000, |v| v as usize);
62    let dim = arg(1).map_or(128, |v| v as usize);
63    let noise = arg(2).map_or(1.0, |v| v as f32);
64    let path = std::env::temp_dir().join("recern-vector-bench.rvec");
65    let _ = std::fs::remove_file(&path);
66
67    let mut data = Clusters::new(100, dim, noise, 0x5EED);
68    let vectors: Vec<Vec<f32>> = (0..records).map(|_| data.sample()).collect();
69
70    let mut db = Database::create(&path)?;
71    let collection = db.create_collection("bench", CollectionConfig::new(dim, Metric::Cosine))?;
72    let start = Instant::now();
73    for (i, v) in vectors.iter().enumerate() {
74        collection.upsert(&i.to_string(), v, None)?;
75    }
76    let build = start.elapsed();
77
78    let start = Instant::now();
79    db.save()?;
80    let save = start.elapsed();
81    let start = Instant::now();
82    let db = Database::open(&path)?;
83    let open = start.elapsed();
84    let collection = db.collection("bench")?;
85    let stats = collection.stats();
86
87    println!("records {records} · dim {dim} · noise {noise} · cosine · m=16 ef_construction=200");
88    println!(
89        "build   {:.2} s ({:.0} inserts/s)",
90        build.as_secs_f64(),
91        records as f64 / build.as_secs_f64()
92    );
93    println!(
94        "file    {:.1} MB · save {:.0} ms · open {:.0} ms",
95        std::fs::metadata(&path)?.len() as f64 / 1_048_576.0,
96        save.as_secs_f64() * 1e3,
97        open.as_secs_f64() * 1e3
98    );
99    println!(
100        "graph   layers {:?} · avg degree L0 {:.1} · unreachable {}",
101        stats.nodes_per_layer, stats.avg_degree_layer0, stats.unreachable
102    );
103
104    let report = collection.estimate_recall(&RecallOptions {
105        sample: 200,
106        k: 10,
107        ef_values: vec![10, 16, 32, 64, 128, 256],
108        seed: 7,
109    })?;
110    println!();
111    println!(
112        "exact scan p50 {:.3} ms",
113        report.exact_p50.as_secs_f64() * 1e3
114    );
115    println!(
116        "{:>6}  {:>9}  {:>10}  {:>10}  {:>9}",
117        "ef", "recall@10", "p50 ms", "p95 ms", "speedup"
118    );
119    for p in &report.points {
120        println!(
121            "{:>6}  {:>9.3}  {:>10.3}  {:>10.3}  {:>8.0}x",
122            p.ef,
123            p.recall,
124            p.p50.as_secs_f64() * 1e3,
125            p.p95.as_secs_f64() * 1e3,
126            report.exact_p50.as_secs_f64() / p.p50.as_secs_f64()
127        );
128    }
129
130    let _ = std::fs::remove_file(&path);
131    Ok(())
132}

Source

pub fn estimate_recall(&self, options: &RecallOptions) -> Result<RecallReport>

Measures how many of the exact nearest neighbors the index finds.

Stored vectors are sampled as queries; each query’s own record is excluded from both the exact and the approximate results.

Examples found in repository ?

examples/bench.rs (lines 104-109)

55fn main() -> recern_vector::Result<()> {
56    let args: Vec<String> = std::env::args().skip(1).collect();
57    let arg = |i: usize| {
58        args.get(i)
59            .map(|a| a.parse::<f64>().expect("numeric argument"))
60    };
61    let records = arg(0).map_or(50_000, |v| v as usize);
62    let dim = arg(1).map_or(128, |v| v as usize);
63    let noise = arg(2).map_or(1.0, |v| v as f32);
64    let path = std::env::temp_dir().join("recern-vector-bench.rvec");
65    let _ = std::fs::remove_file(&path);
66
67    let mut data = Clusters::new(100, dim, noise, 0x5EED);
68    let vectors: Vec<Vec<f32>> = (0..records).map(|_| data.sample()).collect();
69
70    let mut db = Database::create(&path)?;
71    let collection = db.create_collection("bench", CollectionConfig::new(dim, Metric::Cosine))?;
72    let start = Instant::now();
73    for (i, v) in vectors.iter().enumerate() {
74        collection.upsert(&i.to_string(), v, None)?;
75    }
76    let build = start.elapsed();
77
78    let start = Instant::now();
79    db.save()?;
80    let save = start.elapsed();
81    let start = Instant::now();
82    let db = Database::open(&path)?;
83    let open = start.elapsed();
84    let collection = db.collection("bench")?;
85    let stats = collection.stats();
86
87    println!("records {records} · dim {dim} · noise {noise} · cosine · m=16 ef_construction=200");
88    println!(
89        "build   {:.2} s ({:.0} inserts/s)",
90        build.as_secs_f64(),
91        records as f64 / build.as_secs_f64()
92    );
93    println!(
94        "file    {:.1} MB · save {:.0} ms · open {:.0} ms",
95        std::fs::metadata(&path)?.len() as f64 / 1_048_576.0,
96        save.as_secs_f64() * 1e3,
97        open.as_secs_f64() * 1e3
98    );
99    println!(
100        "graph   layers {:?} · avg degree L0 {:.1} · unreachable {}",
101        stats.nodes_per_layer, stats.avg_degree_layer0, stats.unreachable
102    );
103
104    let report = collection.estimate_recall(&RecallOptions {
105        sample: 200,
106        k: 10,
107        ef_values: vec![10, 16, 32, 64, 128, 256],
108        seed: 7,
109    })?;
110    println!();
111    println!(
112        "exact scan p50 {:.3} ms",
113        report.exact_p50.as_secs_f64() * 1e3
114    );
115    println!(
116        "{:>6}  {:>9}  {:>10}  {:>10}  {:>9}",
117        "ef", "recall@10", "p50 ms", "p95 ms", "speedup"
118    );
119    for p in &report.points {
120        println!(
121            "{:>6}  {:>9.3}  {:>10.3}  {:>10.3}  {:>8.0}x",
122            p.ef,
123            p.recall,
124            p.p50.as_secs_f64() * 1e3,
125            p.p95.as_secs_f64() * 1e3,
126            report.exact_p50.as_secs_f64() / p.p50.as_secs_f64()
127        );
128    }
129
130    let _ = std::fs::remove_file(&path);
131    Ok(())
132}