Struct Database

Source

pub struct Database { /* private fields */ }

Expand description

A database file and the collections it holds.

The whole database is loaded into memory on open. Changes stay in memory until Database::save, which atomically replaces the file.

Implementations§

Source §

impl Database

Source

pub fn create(path: impl AsRef<Path>) -> Result<Self>

Creates a new, empty database file. Fails if the file exists.

Examples found in repository ?

examples/bench.rs (line 70)

55fn main() -> recern_vector::Result<()> {
56    let args: Vec<String> = std::env::args().skip(1).collect();
57    let arg = |i: usize| {
58        args.get(i)
59            .map(|a| a.parse::<f64>().expect("numeric argument"))
60    };
61    let records = arg(0).map_or(50_000, |v| v as usize);
62    let dim = arg(1).map_or(128, |v| v as usize);
63    let noise = arg(2).map_or(1.0, |v| v as f32);
64    let path = std::env::temp_dir().join("recern-vector-bench.rvec");
65    let _ = std::fs::remove_file(&path);
66
67    let mut data = Clusters::new(100, dim, noise, 0x5EED);
68    let vectors: Vec<Vec<f32>> = (0..records).map(|_| data.sample()).collect();
69
70    let mut db = Database::create(&path)?;
71    let collection = db.create_collection("bench", CollectionConfig::new(dim, Metric::Cosine))?;
72    let start = Instant::now();
73    for (i, v) in vectors.iter().enumerate() {
74        collection.upsert(&i.to_string(), v, None)?;
75    }
76    let build = start.elapsed();
77
78    let start = Instant::now();
79    db.save()?;
80    let save = start.elapsed();
81    let start = Instant::now();
82    let db = Database::open(&path)?;
83    let open = start.elapsed();
84    let collection = db.collection("bench")?;
85    let stats = collection.stats();
86
87    println!("records {records} · dim {dim} · noise {noise} · cosine · m=16 ef_construction=200");
88    println!(
89        "build   {:.2} s ({:.0} inserts/s)",
90        build.as_secs_f64(),
91        records as f64 / build.as_secs_f64()
92    );
93    println!(
94        "file    {:.1} MB · save {:.0} ms · open {:.0} ms",
95        std::fs::metadata(&path)?.len() as f64 / 1_048_576.0,
96        save.as_secs_f64() * 1e3,
97        open.as_secs_f64() * 1e3
98    );
99    println!(
100        "graph   layers {:?} · avg degree L0 {:.1} · unreachable {}",
101        stats.nodes_per_layer, stats.avg_degree_layer0, stats.unreachable
102    );
103
104    let report = collection.estimate_recall(&RecallOptions {
105        sample: 200,
106        k: 10,
107        ef_values: vec![10, 16, 32, 64, 128, 256],
108        seed: 7,
109    })?;
110    println!();
111    println!(
112        "exact scan p50 {:.3} ms",
113        report.exact_p50.as_secs_f64() * 1e3
114    );
115    println!(
116        "{:>6}  {:>9}  {:>10}  {:>10}  {:>9}",
117        "ef", "recall@10", "p50 ms", "p95 ms", "speedup"
118    );
119    for p in &report.points {
120        println!(
121            "{:>6}  {:>9.3}  {:>10.3}  {:>10.3}  {:>8.0}x",
122            p.ef,
123            p.recall,
124            p.p50.as_secs_f64() * 1e3,
125            p.p95.as_secs_f64() * 1e3,
126            report.exact_p50.as_secs_f64() / p.p50.as_secs_f64()
127        );
128    }
129
130    let _ = std::fs::remove_file(&path);
131    Ok(())
132}

Source

pub fn open(path: impl AsRef<Path>) -> Result<Self>

Examples found in repository ?

examples/search_bench.rs (line 23)

21fn main() {
22    let args: Vec<String> = std::env::args().skip(1).collect();
23    let db = Database::open(&args[0]).expect("index");
24    let c = db.collections().next().expect("a collection");
25    let dim = c.config().dim;
26    let queries = read(&args[1], f32::from_le_bytes);
27    let truth = read(&args[2], u32::from_le_bytes);
28    let efs: Vec<usize> = args
29        .get(3)
30        .map_or("10,20,40,80,160", String::as_str)
31        .split(',')
32        .map(|e| e.parse().unwrap())
33        .collect();
34    let n = queries.len() / dim;
35
36    println!(
37        "{:>5}  {:>8}  {:>9}  {:>9}  {:>10}",
38        "ef", "recall", "mean µs", "p50 µs", "distances"
39    );
40    for ef in efs {
41        let options = SearchOptions::default().ef(ef);
42        let (mut times, mut found, mut distances) = (Vec::with_capacity(n), 0, 0);
43        for (q, t) in queries.chunks_exact(dim).zip(truth.chunks_exact(10)) {
44            let start = Instant::now();
45            let report = c.explain(q, 10, &options).unwrap();
46            times.push(start.elapsed().as_secs_f64() * 1e6);
47            distances += report.distance_computations;
48            found += report
49                .hits
50                .iter()
51                .filter(|h| t.contains(&h.id.parse::<u32>().unwrap()))
52                .count();
53        }
54        let mean = times.iter().sum::<f64>() / n as f64;
55        times.sort_by(f64::total_cmp);
56        println!(
57            "{ef:>5}  {:>8.4}  {mean:>9.1}  {:>9.1}  {:>10}",
58            found as f64 / (n * 10) as f64,
59            times[n / 2],
60            distances / n
61        );
62    }
63}

More examples

Hide additional examples

examples/bench.rs (line 82)

55fn main() -> recern_vector::Result<()> {
56    let args: Vec<String> = std::env::args().skip(1).collect();
57    let arg = |i: usize| {
58        args.get(i)
59            .map(|a| a.parse::<f64>().expect("numeric argument"))
60    };
61    let records = arg(0).map_or(50_000, |v| v as usize);
62    let dim = arg(1).map_or(128, |v| v as usize);
63    let noise = arg(2).map_or(1.0, |v| v as f32);
64    let path = std::env::temp_dir().join("recern-vector-bench.rvec");
65    let _ = std::fs::remove_file(&path);
66
67    let mut data = Clusters::new(100, dim, noise, 0x5EED);
68    let vectors: Vec<Vec<f32>> = (0..records).map(|_| data.sample()).collect();
69
70    let mut db = Database::create(&path)?;
71    let collection = db.create_collection("bench", CollectionConfig::new(dim, Metric::Cosine))?;
72    let start = Instant::now();
73    for (i, v) in vectors.iter().enumerate() {
74        collection.upsert(&i.to_string(), v, None)?;
75    }
76    let build = start.elapsed();
77
78    let start = Instant::now();
79    db.save()?;
80    let save = start.elapsed();
81    let start = Instant::now();
82    let db = Database::open(&path)?;
83    let open = start.elapsed();
84    let collection = db.collection("bench")?;
85    let stats = collection.stats();
86
87    println!("records {records} · dim {dim} · noise {noise} · cosine · m=16 ef_construction=200");
88    println!(
89        "build   {:.2} s ({:.0} inserts/s)",
90        build.as_secs_f64(),
91        records as f64 / build.as_secs_f64()
92    );
93    println!(
94        "file    {:.1} MB · save {:.0} ms · open {:.0} ms",
95        std::fs::metadata(&path)?.len() as f64 / 1_048_576.0,
96        save.as_secs_f64() * 1e3,
97        open.as_secs_f64() * 1e3
98    );
99    println!(
100        "graph   layers {:?} · avg degree L0 {:.1} · unreachable {}",
101        stats.nodes_per_layer, stats.avg_degree_layer0, stats.unreachable
102    );
103
104    let report = collection.estimate_recall(&RecallOptions {
105        sample: 200,
106        k: 10,
107        ef_values: vec![10, 16, 32, 64, 128, 256],
108        seed: 7,
109    })?;
110    println!();
111    println!(
112        "exact scan p50 {:.3} ms",
113        report.exact_p50.as_secs_f64() * 1e3
114    );
115    println!(
116        "{:>6}  {:>9}  {:>10}  {:>10}  {:>9}",
117        "ef", "recall@10", "p50 ms", "p95 ms", "speedup"
118    );
119    for p in &report.points {
120        println!(
121            "{:>6}  {:>9.3}  {:>10.3}  {:>10.3}  {:>8.0}x",
122            p.ef,
123            p.recall,
124            p.p50.as_secs_f64() * 1e3,
125            p.p95.as_secs_f64() * 1e3,
126            report.exact_p50.as_secs_f64() / p.p50.as_secs_f64()
127        );
128    }
129
130    let _ = std::fs::remove_file(&path);
131    Ok(())
132}

Source

pub fn open_or_create(path: impl AsRef<Path>) -> Result<Self>

Source

pub fn path(&self) -> &Path

Source

pub fn create_collection( &mut self, name: &str, config: CollectionConfig, ) -> Result<&mut Collection>

Examples found in repository ?

examples/bench.rs (line 71)

55fn main() -> recern_vector::Result<()> {
56    let args: Vec<String> = std::env::args().skip(1).collect();
57    let arg = |i: usize| {
58        args.get(i)
59            .map(|a| a.parse::<f64>().expect("numeric argument"))
60    };
61    let records = arg(0).map_or(50_000, |v| v as usize);
62    let dim = arg(1).map_or(128, |v| v as usize);
63    let noise = arg(2).map_or(1.0, |v| v as f32);
64    let path = std::env::temp_dir().join("recern-vector-bench.rvec");
65    let _ = std::fs::remove_file(&path);
66
67    let mut data = Clusters::new(100, dim, noise, 0x5EED);
68    let vectors: Vec<Vec<f32>> = (0..records).map(|_| data.sample()).collect();
69
70    let mut db = Database::create(&path)?;
71    let collection = db.create_collection("bench", CollectionConfig::new(dim, Metric::Cosine))?;
72    let start = Instant::now();
73    for (i, v) in vectors.iter().enumerate() {
74        collection.upsert(&i.to_string(), v, None)?;
75    }
76    let build = start.elapsed();
77
78    let start = Instant::now();
79    db.save()?;
80    let save = start.elapsed();
81    let start = Instant::now();
82    let db = Database::open(&path)?;
83    let open = start.elapsed();
84    let collection = db.collection("bench")?;
85    let stats = collection.stats();
86
87    println!("records {records} · dim {dim} · noise {noise} · cosine · m=16 ef_construction=200");
88    println!(
89        "build   {:.2} s ({:.0} inserts/s)",
90        build.as_secs_f64(),
91        records as f64 / build.as_secs_f64()
92    );
93    println!(
94        "file    {:.1} MB · save {:.0} ms · open {:.0} ms",
95        std::fs::metadata(&path)?.len() as f64 / 1_048_576.0,
96        save.as_secs_f64() * 1e3,
97        open.as_secs_f64() * 1e3
98    );
99    println!(
100        "graph   layers {:?} · avg degree L0 {:.1} · unreachable {}",
101        stats.nodes_per_layer, stats.avg_degree_layer0, stats.unreachable
102    );
103
104    let report = collection.estimate_recall(&RecallOptions {
105        sample: 200,
106        k: 10,
107        ef_values: vec![10, 16, 32, 64, 128, 256],
108        seed: 7,
109    })?;
110    println!();
111    println!(
112        "exact scan p50 {:.3} ms",
113        report.exact_p50.as_secs_f64() * 1e3
114    );
115    println!(
116        "{:>6}  {:>9}  {:>10}  {:>10}  {:>9}",
117        "ef", "recall@10", "p50 ms", "p95 ms", "speedup"
118    );
119    for p in &report.points {
120        println!(
121            "{:>6}  {:>9.3}  {:>10.3}  {:>10.3}  {:>8.0}x",
122            p.ef,
123            p.recall,
124            p.p50.as_secs_f64() * 1e3,
125            p.p95.as_secs_f64() * 1e3,
126            report.exact_p50.as_secs_f64() / p.p50.as_secs_f64()
127        );
128    }
129
130    let _ = std::fs::remove_file(&path);
131    Ok(())
132}

Source

pub fn collection(&self, name: &str) -> Result<&Collection>

Examples found in repository ?

examples/bench.rs (line 84)

55fn main() -> recern_vector::Result<()> {
56    let args: Vec<String> = std::env::args().skip(1).collect();
57    let arg = |i: usize| {
58        args.get(i)
59            .map(|a| a.parse::<f64>().expect("numeric argument"))
60    };
61    let records = arg(0).map_or(50_000, |v| v as usize);
62    let dim = arg(1).map_or(128, |v| v as usize);
63    let noise = arg(2).map_or(1.0, |v| v as f32);
64    let path = std::env::temp_dir().join("recern-vector-bench.rvec");
65    let _ = std::fs::remove_file(&path);
66
67    let mut data = Clusters::new(100, dim, noise, 0x5EED);
68    let vectors: Vec<Vec<f32>> = (0..records).map(|_| data.sample()).collect();
69
70    let mut db = Database::create(&path)?;
71    let collection = db.create_collection("bench", CollectionConfig::new(dim, Metric::Cosine))?;
72    let start = Instant::now();
73    for (i, v) in vectors.iter().enumerate() {
74        collection.upsert(&i.to_string(), v, None)?;
75    }
76    let build = start.elapsed();
77
78    let start = Instant::now();
79    db.save()?;
80    let save = start.elapsed();
81    let start = Instant::now();
82    let db = Database::open(&path)?;
83    let open = start.elapsed();
84    let collection = db.collection("bench")?;
85    let stats = collection.stats();
86
87    println!("records {records} · dim {dim} · noise {noise} · cosine · m=16 ef_construction=200");
88    println!(
89        "build   {:.2} s ({:.0} inserts/s)",
90        build.as_secs_f64(),
91        records as f64 / build.as_secs_f64()
92    );
93    println!(
94        "file    {:.1} MB · save {:.0} ms · open {:.0} ms",
95        std::fs::metadata(&path)?.len() as f64 / 1_048_576.0,
96        save.as_secs_f64() * 1e3,
97        open.as_secs_f64() * 1e3
98    );
99    println!(
100        "graph   layers {:?} · avg degree L0 {:.1} · unreachable {}",
101        stats.nodes_per_layer, stats.avg_degree_layer0, stats.unreachable
102    );
103
104    let report = collection.estimate_recall(&RecallOptions {
105        sample: 200,
106        k: 10,
107        ef_values: vec![10, 16, 32, 64, 128, 256],
108        seed: 7,
109    })?;
110    println!();
111    println!(
112        "exact scan p50 {:.3} ms",
113        report.exact_p50.as_secs_f64() * 1e3
114    );
115    println!(
116        "{:>6}  {:>9}  {:>10}  {:>10}  {:>9}",
117        "ef", "recall@10", "p50 ms", "p95 ms", "speedup"
118    );
119    for p in &report.points {
120        println!(
121            "{:>6}  {:>9.3}  {:>10.3}  {:>10.3}  {:>8.0}x",
122            p.ef,
123            p.recall,
124            p.p50.as_secs_f64() * 1e3,
125            p.p95.as_secs_f64() * 1e3,
126            report.exact_p50.as_secs_f64() / p.p50.as_secs_f64()
127        );
128    }
129
130    let _ = std::fs::remove_file(&path);
131    Ok(())
132}

Source

pub fn collection_mut(&mut self, name: &str) -> Result<&mut Collection>

Source

pub fn drop_collection(&mut self, name: &str) -> Result<()>

Source

pub fn collections(&self) -> impl Iterator<Item = &Collection>

Examples found in repository ?

examples/search_bench.rs (line 24)

21fn main() {
22    let args: Vec<String> = std::env::args().skip(1).collect();
23    let db = Database::open(&args[0]).expect("index");
24    let c = db.collections().next().expect("a collection");
25    let dim = c.config().dim;
26    let queries = read(&args[1], f32::from_le_bytes);
27    let truth = read(&args[2], u32::from_le_bytes);
28    let efs: Vec<usize> = args
29        .get(3)
30        .map_or("10,20,40,80,160", String::as_str)
31        .split(',')
32        .map(|e| e.parse().unwrap())
33        .collect();
34    let n = queries.len() / dim;
35
36    println!(
37        "{:>5}  {:>8}  {:>9}  {:>9}  {:>10}",
38        "ef", "recall", "mean µs", "p50 µs", "distances"
39    );
40    for ef in efs {
41        let options = SearchOptions::default().ef(ef);
42        let (mut times, mut found, mut distances) = (Vec::with_capacity(n), 0, 0);
43        for (q, t) in queries.chunks_exact(dim).zip(truth.chunks_exact(10)) {
44            let start = Instant::now();
45            let report = c.explain(q, 10, &options).unwrap();
46            times.push(start.elapsed().as_secs_f64() * 1e6);
47            distances += report.distance_computations;
48            found += report
49                .hits
50                .iter()
51                .filter(|h| t.contains(&h.id.parse::<u32>().unwrap()))
52                .count();
53        }
54        let mean = times.iter().sum::<f64>() / n as f64;
55        times.sort_by(f64::total_cmp);
56        println!(
57            "{ef:>5}  {:>8.4}  {mean:>9.1}  {:>9.1}  {:>10}",
58            found as f64 / (n * 10) as f64,
59            times[n / 2],
60            distances / n
61        );
62    }
63}

Source

pub fn save(&self) -> Result<()>

Writes the database to a temporary file, syncs it, and renames it over the original, so a crash leaves either the old or the new version.

Examples found in repository ?

examples/bench.rs (line 79)

55fn main() -> recern_vector::Result<()> {
56    let args: Vec<String> = std::env::args().skip(1).collect();
57    let arg = |i: usize| {
58        args.get(i)
59            .map(|a| a.parse::<f64>().expect("numeric argument"))
60    };
61    let records = arg(0).map_or(50_000, |v| v as usize);
62    let dim = arg(1).map_or(128, |v| v as usize);
63    let noise = arg(2).map_or(1.0, |v| v as f32);
64    let path = std::env::temp_dir().join("recern-vector-bench.rvec");
65    let _ = std::fs::remove_file(&path);
66
67    let mut data = Clusters::new(100, dim, noise, 0x5EED);
68    let vectors: Vec<Vec<f32>> = (0..records).map(|_| data.sample()).collect();
69
70    let mut db = Database::create(&path)?;
71    let collection = db.create_collection("bench", CollectionConfig::new(dim, Metric::Cosine))?;
72    let start = Instant::now();
73    for (i, v) in vectors.iter().enumerate() {
74        collection.upsert(&i.to_string(), v, None)?;
75    }
76    let build = start.elapsed();
77
78    let start = Instant::now();
79    db.save()?;
80    let save = start.elapsed();
81    let start = Instant::now();
82    let db = Database::open(&path)?;
83    let open = start.elapsed();
84    let collection = db.collection("bench")?;
85    let stats = collection.stats();
86
87    println!("records {records} · dim {dim} · noise {noise} · cosine · m=16 ef_construction=200");
88    println!(
89        "build   {:.2} s ({:.0} inserts/s)",
90        build.as_secs_f64(),
91        records as f64 / build.as_secs_f64()
92    );
93    println!(
94        "file    {:.1} MB · save {:.0} ms · open {:.0} ms",
95        std::fs::metadata(&path)?.len() as f64 / 1_048_576.0,
96        save.as_secs_f64() * 1e3,
97        open.as_secs_f64() * 1e3
98    );
99    println!(
100        "graph   layers {:?} · avg degree L0 {:.1} · unreachable {}",
101        stats.nodes_per_layer, stats.avg_degree_layer0, stats.unreachable
102    );
103
104    let report = collection.estimate_recall(&RecallOptions {
105        sample: 200,
106        k: 10,
107        ef_values: vec![10, 16, 32, 64, 128, 256],
108        seed: 7,
109    })?;
110    println!();
111    println!(
112        "exact scan p50 {:.3} ms",
113        report.exact_p50.as_secs_f64() * 1e3
114    );
115    println!(
116        "{:>6}  {:>9}  {:>10}  {:>10}  {:>9}",
117        "ef", "recall@10", "p50 ms", "p95 ms", "speedup"
118    );
119    for p in &report.points {
120        println!(
121            "{:>6}  {:>9.3}  {:>10.3}  {:>10.3}  {:>8.0}x",
122            p.ef,
123            p.recall,
124            p.p50.as_secs_f64() * 1e3,
125            p.p95.as_secs_f64() * 1e3,
126            report.exact_p50.as_secs_f64() / p.p50.as_secs_f64()
127        );
128    }
129
130    let _ = std::fs::remove_file(&path);
131    Ok(())
132}