Struct Database 

pub struct Database { /* private fields */ }

An in-memory database backed by a snapshot and a checksummed .wal journal. Mutations become durable at save(). Use checkpoint() before copying a standalone file. Concurrent handles can read; stale writers fail on save.

Implementations

impl Database

pub fn create(path: impl AsRef<Path>) -> Result<Self>

Examples found in repository

examples/quickstart.rs (line 14)

fn main() -> recern_vector::Result<()> {
    let path = std::env::temp_dir().join("recern-vector-quickstart.rvec");
    let _ = std::fs::remove_file(&path);
    let _ = std::fs::remove_file(format!("{}.wal", path.display()));

    let mut db = Database::create(&path)?;
    let docs = db.create_collection("docs", CollectionConfig::new(32, Metric::Cosine))?;

    // Batch insert: (id, vector, metadata). Built on all cores, atomic.
    let records: Vec<_> = (0..10_000)
        .map(|i| {
            let vector: Vec<f32> = (0..32).map(|d| pseudo_random(i * 32 + d)).collect();
            let lang = ["en", "de", "fr"][i % 3];
            (
                format!("doc-{i}"),
                vector,
                Some(json!({"lang": lang, "year": 2015 + i % 10})),
            )
        })
        .collect();
    let query = records[42].1.clone();
    docs.upsert_many(records)?;
    println!("{} records", docs.len());

    // Search with a filter; explain() returns the hits and how they were found.
    let filter = Filter::from_json(&json!({"lang": "en", "year": {"$gte": 2020}}))?;
    // The same filter, built in code:
    let _same = Filter::And(vec![
        Filter::eq("lang", "en"),
        Filter::between("year", Some(2020.0), None),
    ]);
    let report = docs.explain(&query, 3, &SearchOptions::default().filter(filter))?;
    for hit in &report.hits {
        println!(
            "  {:<10} distance={:.4} {}",
            hit.id,
            hit.distance,
            hit.metadata.as_ref().unwrap()
        );
    }
    println!(
        "strategy={} visited={} selectivity={:.1}% time={:?}",
        report.strategy,
        report.visited,
        report.filter_selectivity.unwrap_or(1.0) * 100.0,
        report.elapsed
    );

    // Index health and recall against exact search.
    let stats = docs.stats();
    println!(
        "layers={:?} unreachable={}",
        stats.nodes_per_layer, stats.unreachable
    );
    let recall = docs.estimate_recall(&RecallOptions {
        ef_values: vec![16, 64],
        ..Default::default()
    })?;
    for point in &recall.points {
        println!(
            "  ef={:<3} recall@{}={:.3}",
            point.ef, recall.k, point.recall
        );
    }

    db.save()?;
    let reopened = Database::open(&path)?;
    println!("reopened: {} records", reopened.collection("docs")?.len());
    Ok(())
}

examples/bench.rs (line 70)

fn main() -> recern_vector::Result<()> {
    let args: Vec<String> = std::env::args().skip(1).collect();
    let arg = |i: usize| {
        args.get(i)
            .map(|a| a.parse::<f64>().expect("numeric argument"))
    };
    let records = arg(0).map_or(50_000, |v| v as usize);
    let dim = arg(1).map_or(128, |v| v as usize);
    let noise = arg(2).map_or(1.0, |v| v as f32);
    let path = std::env::temp_dir().join("recern-vector-bench.rvec");
    let _ = std::fs::remove_file(&path);

    let mut data = Clusters::new(100, dim, noise, 0x5EED);
    let vectors: Vec<Vec<f32>> = (0..records).map(|_| data.sample()).collect();

    let mut db = Database::create(&path)?;
    let collection = db.create_collection("bench", CollectionConfig::new(dim, Metric::Cosine))?;
    let start = Instant::now();
    for (i, v) in vectors.iter().enumerate() {
        collection.upsert(&i.to_string(), v, None)?;
    }
    let build = start.elapsed();

    let start = Instant::now();
    db.save()?;
    let save = start.elapsed();
    let start = Instant::now();
    let db = Database::open(&path)?;
    let open = start.elapsed();
    let collection = db.collection("bench")?;
    let stats = collection.stats();

    println!("records {records} · dim {dim} · noise {noise} · cosine · m=16 ef_construction=200");
    println!(
        "build   {:.2} s ({:.0} inserts/s)",
        build.as_secs_f64(),
        records as f64 / build.as_secs_f64()
    );
    println!(
        "file    {:.1} MB · save {:.0} ms · open {:.0} ms",
        std::fs::metadata(&path)?.len() as f64 / 1_048_576.0,
        save.as_secs_f64() * 1e3,
        open.as_secs_f64() * 1e3
    );
    println!(
        "graph   layers {:?} · avg degree L0 {:.1} · unreachable {}",
        stats.nodes_per_layer, stats.avg_degree_layer0, stats.unreachable
    );

    let report = collection.estimate_recall(&RecallOptions {
        sample: 200,
        k: 10,
        ef_values: vec![10, 16, 32, 64, 128, 256],
        seed: 7,
    })?;
    println!();
    println!(
        "exact scan p50 {:.3} ms",
        report.exact_p50.as_secs_f64() * 1e3
    );
    println!(
        "{:>6}  {:>9}  {:>10}  {:>10}  {:>9}",
        "ef", "recall@10", "p50 ms", "p95 ms", "speedup"
    );
    for p in &report.points {
        println!(
            "{:>6}  {:>9.3}  {:>10.3}  {:>10.3}  {:>8.0}x",
            p.ef,
            p.recall,
            p.p50.as_secs_f64() * 1e3,
            p.p95.as_secs_f64() * 1e3,
            report.exact_p50.as_secs_f64() / p.p50.as_secs_f64()
        );
    }

    let _ = std::fs::remove_file(&path);
    Ok(())
}

pub fn open(path: impl AsRef<Path>) -> Result<Self>

Examples found in repository

examples/search_bench.rs (line 25)

fn main() {
    let args: Vec<String> = std::env::args().skip(1).collect();
    let db = Database::open(&args[0]).expect("index");
    let c = db.collections().next().expect("a collection");
    let dim = c.config().dim;
    let queries = read(&args[1], f32::from_le_bytes);
    let truth = read(&args[2], u32::from_le_bytes);
    let efs: Vec<usize> = args
        .get(3)
        .map_or("10,20,40,80,160", String::as_str)
        .split(',')
        .map(|e| e.parse().unwrap())
        .collect();
    let n = queries.len() / dim;

    println!(
        "{:>5}  {:>8}  {:>9}  {:>9}  {:>10}",
        "ef", "recall", "mean µs", "p50 µs", "distances"
    );
    for ef in efs {
        let options = SearchOptions::default().ef(ef);
        let (mut times, mut found, mut distances) = (Vec::with_capacity(n), 0, 0);
        for (q, t) in queries
            .chunks_exact(dim)
            .zip(truth.as_chunks::<10>().0.iter())
        {
            let start = Instant::now();
            let report = c.explain(q, 10, &options).unwrap();
            times.push(start.elapsed().as_secs_f64() * 1e6);
            distances += report.distance_computations;
            found += report
                .hits
                .iter()
                .filter(|h| t.contains(&h.id.parse::<u32>().unwrap()))
                .count();
        }
        let mean = times.iter().sum::<f64>() / n as f64;
        times.sort_by(f64::total_cmp);
        println!(
            "{ef:>5}  {:>8.4}  {mean:>9.1}  {:>9.1}  {:>10}",
            found as f64 / (n * 10) as f64,
            times[n / 2],
            distances / n
        );
    }
}

examples/quickstart.rs (line 75)

fn main() -> recern_vector::Result<()> {
    let path = std::env::temp_dir().join("recern-vector-quickstart.rvec");
    let _ = std::fs::remove_file(&path);
    let _ = std::fs::remove_file(format!("{}.wal", path.display()));

    let mut db = Database::create(&path)?;
    let docs = db.create_collection("docs", CollectionConfig::new(32, Metric::Cosine))?;

    // Batch insert: (id, vector, metadata). Built on all cores, atomic.
    let records: Vec<_> = (0..10_000)
        .map(|i| {
            let vector: Vec<f32> = (0..32).map(|d| pseudo_random(i * 32 + d)).collect();
            let lang = ["en", "de", "fr"][i % 3];
            (
                format!("doc-{i}"),
                vector,
                Some(json!({"lang": lang, "year": 2015 + i % 10})),
            )
        })
        .collect();
    let query = records[42].1.clone();
    docs.upsert_many(records)?;
    println!("{} records", docs.len());

    // Search with a filter; explain() returns the hits and how they were found.
    let filter = Filter::from_json(&json!({"lang": "en", "year": {"$gte": 2020}}))?;
    // The same filter, built in code:
    let _same = Filter::And(vec![
        Filter::eq("lang", "en"),
        Filter::between("year", Some(2020.0), None),
    ]);
    let report = docs.explain(&query, 3, &SearchOptions::default().filter(filter))?;
    for hit in &report.hits {
        println!(
            "  {:<10} distance={:.4} {}",
            hit.id,
            hit.distance,
            hit.metadata.as_ref().unwrap()
        );
    }
    println!(
        "strategy={} visited={} selectivity={:.1}% time={:?}",
        report.strategy,
        report.visited,
        report.filter_selectivity.unwrap_or(1.0) * 100.0,
        report.elapsed
    );

    // Index health and recall against exact search.
    let stats = docs.stats();
    println!(
        "layers={:?} unreachable={}",
        stats.nodes_per_layer, stats.unreachable
    );
    let recall = docs.estimate_recall(&RecallOptions {
        ef_values: vec![16, 64],
        ..Default::default()
    })?;
    for point in &recall.points {
        println!(
            "  ef={:<3} recall@{}={:.3}",
            point.ef, recall.k, point.recall
        );
    }

    db.save()?;
    let reopened = Database::open(&path)?;
    println!("reopened: {} records", reopened.collection("docs")?.len());
    Ok(())
}

examples/bench.rs (line 82)

fn main() -> recern_vector::Result<()> {
    let args: Vec<String> = std::env::args().skip(1).collect();
    let arg = |i: usize| {
        args.get(i)
            .map(|a| a.parse::<f64>().expect("numeric argument"))
    };
    let records = arg(0).map_or(50_000, |v| v as usize);
    let dim = arg(1).map_or(128, |v| v as usize);
    let noise = arg(2).map_or(1.0, |v| v as f32);
    let path = std::env::temp_dir().join("recern-vector-bench.rvec");
    let _ = std::fs::remove_file(&path);

    let mut data = Clusters::new(100, dim, noise, 0x5EED);
    let vectors: Vec<Vec<f32>> = (0..records).map(|_| data.sample()).collect();

    let mut db = Database::create(&path)?;
    let collection = db.create_collection("bench", CollectionConfig::new(dim, Metric::Cosine))?;
    let start = Instant::now();
    for (i, v) in vectors.iter().enumerate() {
        collection.upsert(&i.to_string(), v, None)?;
    }
    let build = start.elapsed();

    let start = Instant::now();
    db.save()?;
    let save = start.elapsed();
    let start = Instant::now();
    let db = Database::open(&path)?;
    let open = start.elapsed();
    let collection = db.collection("bench")?;
    let stats = collection.stats();

    println!("records {records} · dim {dim} · noise {noise} · cosine · m=16 ef_construction=200");
    println!(
        "build   {:.2} s ({:.0} inserts/s)",
        build.as_secs_f64(),
        records as f64 / build.as_secs_f64()
    );
    println!(
        "file    {:.1} MB · save {:.0} ms · open {:.0} ms",
        std::fs::metadata(&path)?.len() as f64 / 1_048_576.0,
        save.as_secs_f64() * 1e3,
        open.as_secs_f64() * 1e3
    );
    println!(
        "graph   layers {:?} · avg degree L0 {:.1} · unreachable {}",
        stats.nodes_per_layer, stats.avg_degree_layer0, stats.unreachable
    );

    let report = collection.estimate_recall(&RecallOptions {
        sample: 200,
        k: 10,
        ef_values: vec![10, 16, 32, 64, 128, 256],
        seed: 7,
    })?;
    println!();
    println!(
        "exact scan p50 {:.3} ms",
        report.exact_p50.as_secs_f64() * 1e3
    );
    println!(
        "{:>6}  {:>9}  {:>10}  {:>10}  {:>9}",
        "ef", "recall@10", "p50 ms", "p95 ms", "speedup"
    );
    for p in &report.points {
        println!(
            "{:>6}  {:>9.3}  {:>10.3}  {:>10.3}  {:>8.0}x",
            p.ef,
            p.recall,
            p.p50.as_secs_f64() * 1e3,
            p.p95.as_secs_f64() * 1e3,
            report.exact_p50.as_secs_f64() / p.p50.as_secs_f64()
        );
    }

    let _ = std::fs::remove_file(&path);
    Ok(())
}

pub fn open_read_only(path: impl AsRef<Path>) -> Result<Self>

Loads a consistent snapshot including committed WAL records, without creating sidecars or allowing mutations through this handle.

pub fn open_or_create(path: impl AsRef<Path>) -> Result<Self>

pub fn format_version(&self) -> u32

Format of this handle’s last durable snapshot (including WAL).

pub fn is_read_only(&self) -> bool

pub fn path(&self) -> &Path

pub fn create_collection( &mut self, name: &str, config: CollectionConfig, ) -> Result<&mut Collection>

Examples found in repository

examples/quickstart.rs (line 15)

fn main() -> recern_vector::Result<()> {
    let path = std::env::temp_dir().join("recern-vector-quickstart.rvec");
    let _ = std::fs::remove_file(&path);
    let _ = std::fs::remove_file(format!("{}.wal", path.display()));

    let mut db = Database::create(&path)?;
    let docs = db.create_collection("docs", CollectionConfig::new(32, Metric::Cosine))?;

    // Batch insert: (id, vector, metadata). Built on all cores, atomic.
    let records: Vec<_> = (0..10_000)
        .map(|i| {
            let vector: Vec<f32> = (0..32).map(|d| pseudo_random(i * 32 + d)).collect();
            let lang = ["en", "de", "fr"][i % 3];
            (
                format!("doc-{i}"),
                vector,
                Some(json!({"lang": lang, "year": 2015 + i % 10})),
            )
        })
        .collect();
    let query = records[42].1.clone();
    docs.upsert_many(records)?;
    println!("{} records", docs.len());

    // Search with a filter; explain() returns the hits and how they were found.
    let filter = Filter::from_json(&json!({"lang": "en", "year": {"$gte": 2020}}))?;
    // The same filter, built in code:
    let _same = Filter::And(vec![
        Filter::eq("lang", "en"),
        Filter::between("year", Some(2020.0), None),
    ]);
    let report = docs.explain(&query, 3, &SearchOptions::default().filter(filter))?;
    for hit in &report.hits {
        println!(
            "  {:<10} distance={:.4} {}",
            hit.id,
            hit.distance,
            hit.metadata.as_ref().unwrap()
        );
    }
    println!(
        "strategy={} visited={} selectivity={:.1}% time={:?}",
        report.strategy,
        report.visited,
        report.filter_selectivity.unwrap_or(1.0) * 100.0,
        report.elapsed
    );

    // Index health and recall against exact search.
    let stats = docs.stats();
    println!(
        "layers={:?} unreachable={}",
        stats.nodes_per_layer, stats.unreachable
    );
    let recall = docs.estimate_recall(&RecallOptions {
        ef_values: vec![16, 64],
        ..Default::default()
    })?;
    for point in &recall.points {
        println!(
            "  ef={:<3} recall@{}={:.3}",
            point.ef, recall.k, point.recall
        );
    }

    db.save()?;
    let reopened = Database::open(&path)?;
    println!("reopened: {} records", reopened.collection("docs")?.len());
    Ok(())
}

examples/bench.rs (line 71)

fn main() -> recern_vector::Result<()> {
    let args: Vec<String> = std::env::args().skip(1).collect();
    let arg = |i: usize| {
        args.get(i)
            .map(|a| a.parse::<f64>().expect("numeric argument"))
    };
    let records = arg(0).map_or(50_000, |v| v as usize);
    let dim = arg(1).map_or(128, |v| v as usize);
    let noise = arg(2).map_or(1.0, |v| v as f32);
    let path = std::env::temp_dir().join("recern-vector-bench.rvec");
    let _ = std::fs::remove_file(&path);

    let mut data = Clusters::new(100, dim, noise, 0x5EED);
    let vectors: Vec<Vec<f32>> = (0..records).map(|_| data.sample()).collect();

    let mut db = Database::create(&path)?;
    let collection = db.create_collection("bench", CollectionConfig::new(dim, Metric::Cosine))?;
    let start = Instant::now();
    for (i, v) in vectors.iter().enumerate() {
        collection.upsert(&i.to_string(), v, None)?;
    }
    let build = start.elapsed();

    let start = Instant::now();
    db.save()?;
    let save = start.elapsed();
    let start = Instant::now();
    let db = Database::open(&path)?;
    let open = start.elapsed();
    let collection = db.collection("bench")?;
    let stats = collection.stats();

    println!("records {records} · dim {dim} · noise {noise} · cosine · m=16 ef_construction=200");
    println!(
        "build   {:.2} s ({:.0} inserts/s)",
        build.as_secs_f64(),
        records as f64 / build.as_secs_f64()
    );
    println!(
        "file    {:.1} MB · save {:.0} ms · open {:.0} ms",
        std::fs::metadata(&path)?.len() as f64 / 1_048_576.0,
        save.as_secs_f64() * 1e3,
        open.as_secs_f64() * 1e3
    );
    println!(
        "graph   layers {:?} · avg degree L0 {:.1} · unreachable {}",
        stats.nodes_per_layer, stats.avg_degree_layer0, stats.unreachable
    );

    let report = collection.estimate_recall(&RecallOptions {
        sample: 200,
        k: 10,
        ef_values: vec![10, 16, 32, 64, 128, 256],
        seed: 7,
    })?;
    println!();
    println!(
        "exact scan p50 {:.3} ms",
        report.exact_p50.as_secs_f64() * 1e3
    );
    println!(
        "{:>6}  {:>9}  {:>10}  {:>10}  {:>9}",
        "ef", "recall@10", "p50 ms", "p95 ms", "speedup"
    );
    for p in &report.points {
        println!(
            "{:>6}  {:>9.3}  {:>10.3}  {:>10.3}  {:>8.0}x",
            p.ef,
            p.recall,
            p.p50.as_secs_f64() * 1e3,
            p.p95.as_secs_f64() * 1e3,
            report.exact_p50.as_secs_f64() / p.p50.as_secs_f64()
        );
    }

    let _ = std::fs::remove_file(&path);
    Ok(())
}

pub fn collection(&self, name: &str) -> Result<&Collection>

Examples found in repository

examples/quickstart.rs (line 76)

fn main() -> recern_vector::Result<()> {
    let path = std::env::temp_dir().join("recern-vector-quickstart.rvec");
    let _ = std::fs::remove_file(&path);
    let _ = std::fs::remove_file(format!("{}.wal", path.display()));

    let mut db = Database::create(&path)?;
    let docs = db.create_collection("docs", CollectionConfig::new(32, Metric::Cosine))?;

    // Batch insert: (id, vector, metadata). Built on all cores, atomic.
    let records: Vec<_> = (0..10_000)
        .map(|i| {
            let vector: Vec<f32> = (0..32).map(|d| pseudo_random(i * 32 + d)).collect();
            let lang = ["en", "de", "fr"][i % 3];
            (
                format!("doc-{i}"),
                vector,
                Some(json!({"lang": lang, "year": 2015 + i % 10})),
            )
        })
        .collect();
    let query = records[42].1.clone();
    docs.upsert_many(records)?;
    println!("{} records", docs.len());

    // Search with a filter; explain() returns the hits and how they were found.
    let filter = Filter::from_json(&json!({"lang": "en", "year": {"$gte": 2020}}))?;
    // The same filter, built in code:
    let _same = Filter::And(vec![
        Filter::eq("lang", "en"),
        Filter::between("year", Some(2020.0), None),
    ]);
    let report = docs.explain(&query, 3, &SearchOptions::default().filter(filter))?;
    for hit in &report.hits {
        println!(
            "  {:<10} distance={:.4} {}",
            hit.id,
            hit.distance,
            hit.metadata.as_ref().unwrap()
        );
    }
    println!(
        "strategy={} visited={} selectivity={:.1}% time={:?}",
        report.strategy,
        report.visited,
        report.filter_selectivity.unwrap_or(1.0) * 100.0,
        report.elapsed
    );

    // Index health and recall against exact search.
    let stats = docs.stats();
    println!(
        "layers={:?} unreachable={}",
        stats.nodes_per_layer, stats.unreachable
    );
    let recall = docs.estimate_recall(&RecallOptions {
        ef_values: vec![16, 64],
        ..Default::default()
    })?;
    for point in &recall.points {
        println!(
            "  ef={:<3} recall@{}={:.3}",
            point.ef, recall.k, point.recall
        );
    }

    db.save()?;
    let reopened = Database::open(&path)?;
    println!("reopened: {} records", reopened.collection("docs")?.len());
    Ok(())
}

examples/bench.rs (line 84)

fn main() -> recern_vector::Result<()> {
    let args: Vec<String> = std::env::args().skip(1).collect();
    let arg = |i: usize| {
        args.get(i)
            .map(|a| a.parse::<f64>().expect("numeric argument"))
    };
    let records = arg(0).map_or(50_000, |v| v as usize);
    let dim = arg(1).map_or(128, |v| v as usize);
    let noise = arg(2).map_or(1.0, |v| v as f32);
    let path = std::env::temp_dir().join("recern-vector-bench.rvec");
    let _ = std::fs::remove_file(&path);

    let mut data = Clusters::new(100, dim, noise, 0x5EED);
    let vectors: Vec<Vec<f32>> = (0..records).map(|_| data.sample()).collect();

    let mut db = Database::create(&path)?;
    let collection = db.create_collection("bench", CollectionConfig::new(dim, Metric::Cosine))?;
    let start = Instant::now();
    for (i, v) in vectors.iter().enumerate() {
        collection.upsert(&i.to_string(), v, None)?;
    }
    let build = start.elapsed();

    let start = Instant::now();
    db.save()?;
    let save = start.elapsed();
    let start = Instant::now();
    let db = Database::open(&path)?;
    let open = start.elapsed();
    let collection = db.collection("bench")?;
    let stats = collection.stats();

    println!("records {records} · dim {dim} · noise {noise} · cosine · m=16 ef_construction=200");
    println!(
        "build   {:.2} s ({:.0} inserts/s)",
        build.as_secs_f64(),
        records as f64 / build.as_secs_f64()
    );
    println!(
        "file    {:.1} MB · save {:.0} ms · open {:.0} ms",
        std::fs::metadata(&path)?.len() as f64 / 1_048_576.0,
        save.as_secs_f64() * 1e3,
        open.as_secs_f64() * 1e3
    );
    println!(
        "graph   layers {:?} · avg degree L0 {:.1} · unreachable {}",
        stats.nodes_per_layer, stats.avg_degree_layer0, stats.unreachable
    );

    let report = collection.estimate_recall(&RecallOptions {
        sample: 200,
        k: 10,
        ef_values: vec![10, 16, 32, 64, 128, 256],
        seed: 7,
    })?;
    println!();
    println!(
        "exact scan p50 {:.3} ms",
        report.exact_p50.as_secs_f64() * 1e3
    );
    println!(
        "{:>6}  {:>9}  {:>10}  {:>10}  {:>9}",
        "ef", "recall@10", "p50 ms", "p95 ms", "speedup"
    );
    for p in &report.points {
        println!(
            "{:>6}  {:>9.3}  {:>10.3}  {:>10.3}  {:>8.0}x",
            p.ef,
            p.recall,
            p.p50.as_secs_f64() * 1e3,
            p.p95.as_secs_f64() * 1e3,
            report.exact_p50.as_secs_f64() / p.p50.as_secs_f64()
        );
    }

    let _ = std::fs::remove_file(&path);
    Ok(())
}

pub fn collection_mut(&mut self, name: &str) -> Result<&mut Collection>

pub fn drop_collection(&mut self, name: &str) -> Result<()>

pub fn collections(&self) -> impl Iterator<Item = &Collection>

Examples found in repository

examples/search_bench.rs (line 26)

fn main() {
    let args: Vec<String> = std::env::args().skip(1).collect();
    let db = Database::open(&args[0]).expect("index");
    let c = db.collections().next().expect("a collection");
    let dim = c.config().dim;
    let queries = read(&args[1], f32::from_le_bytes);
    let truth = read(&args[2], u32::from_le_bytes);
    let efs: Vec<usize> = args
        .get(3)
        .map_or("10,20,40,80,160", String::as_str)
        .split(',')
        .map(|e| e.parse().unwrap())
        .collect();
    let n = queries.len() / dim;

    println!(
        "{:>5}  {:>8}  {:>9}  {:>9}  {:>10}",
        "ef", "recall", "mean µs", "p50 µs", "distances"
    );
    for ef in efs {
        let options = SearchOptions::default().ef(ef);
        let (mut times, mut found, mut distances) = (Vec::with_capacity(n), 0, 0);
        for (q, t) in queries
            .chunks_exact(dim)
            .zip(truth.as_chunks::<10>().0.iter())
        {
            let start = Instant::now();
            let report = c.explain(q, 10, &options).unwrap();
            times.push(start.elapsed().as_secs_f64() * 1e6);
            distances += report.distance_computations;
            found += report
                .hits
                .iter()
                .filter(|h| t.contains(&h.id.parse::<u32>().unwrap()))
                .count();
        }
        let mean = times.iter().sum::<f64>() / n as f64;
        times.sort_by(f64::total_cmp);
        println!(
            "{ef:>5}  {:>8.4}  {mean:>9.1}  {:>9.1}  {:>10}",
            found as f64 / (n * 10) as f64,
            times[n / 2],
            distances / n
        );
    }
}

pub fn save(&self) -> Result<()>

Appends changed pages and fsyncs a commit. The base snapshot is unchanged except when migrating a format-1 database. No-op saves do not grow WAL.

Examples found in repository

examples/quickstart.rs (line 74)

fn main() -> recern_vector::Result<()> {
    let path = std::env::temp_dir().join("recern-vector-quickstart.rvec");
    let _ = std::fs::remove_file(&path);
    let _ = std::fs::remove_file(format!("{}.wal", path.display()));

    let mut db = Database::create(&path)?;
    let docs = db.create_collection("docs", CollectionConfig::new(32, Metric::Cosine))?;

    // Batch insert: (id, vector, metadata). Built on all cores, atomic.
    let records: Vec<_> = (0..10_000)
        .map(|i| {
            let vector: Vec<f32> = (0..32).map(|d| pseudo_random(i * 32 + d)).collect();
            let lang = ["en", "de", "fr"][i % 3];
            (
                format!("doc-{i}"),
                vector,
                Some(json!({"lang": lang, "year": 2015 + i % 10})),
            )
        })
        .collect();
    let query = records[42].1.clone();
    docs.upsert_many(records)?;
    println!("{} records", docs.len());

    // Search with a filter; explain() returns the hits and how they were found.
    let filter = Filter::from_json(&json!({"lang": "en", "year": {"$gte": 2020}}))?;
    // The same filter, built in code:
    let _same = Filter::And(vec![
        Filter::eq("lang", "en"),
        Filter::between("year", Some(2020.0), None),
    ]);
    let report = docs.explain(&query, 3, &SearchOptions::default().filter(filter))?;
    for hit in &report.hits {
        println!(
            "  {:<10} distance={:.4} {}",
            hit.id,
            hit.distance,
            hit.metadata.as_ref().unwrap()
        );
    }
    println!(
        "strategy={} visited={} selectivity={:.1}% time={:?}",
        report.strategy,
        report.visited,
        report.filter_selectivity.unwrap_or(1.0) * 100.0,
        report.elapsed
    );

    // Index health and recall against exact search.
    let stats = docs.stats();
    println!(
        "layers={:?} unreachable={}",
        stats.nodes_per_layer, stats.unreachable
    );
    let recall = docs.estimate_recall(&RecallOptions {
        ef_values: vec![16, 64],
        ..Default::default()
    })?;
    for point in &recall.points {
        println!(
            "  ef={:<3} recall@{}={:.3}",
            point.ef, recall.k, point.recall
        );
    }

    db.save()?;
    let reopened = Database::open(&path)?;
    println!("reopened: {} records", reopened.collection("docs")?.len());
    Ok(())
}

examples/bench.rs (line 79)

fn main() -> recern_vector::Result<()> {
    let args: Vec<String> = std::env::args().skip(1).collect();
    let arg = |i: usize| {
        args.get(i)
            .map(|a| a.parse::<f64>().expect("numeric argument"))
    };
    let records = arg(0).map_or(50_000, |v| v as usize);
    let dim = arg(1).map_or(128, |v| v as usize);
    let noise = arg(2).map_or(1.0, |v| v as f32);
    let path = std::env::temp_dir().join("recern-vector-bench.rvec");
    let _ = std::fs::remove_file(&path);

    let mut data = Clusters::new(100, dim, noise, 0x5EED);
    let vectors: Vec<Vec<f32>> = (0..records).map(|_| data.sample()).collect();

    let mut db = Database::create(&path)?;
    let collection = db.create_collection("bench", CollectionConfig::new(dim, Metric::Cosine))?;
    let start = Instant::now();
    for (i, v) in vectors.iter().enumerate() {
        collection.upsert(&i.to_string(), v, None)?;
    }
    let build = start.elapsed();

    let start = Instant::now();
    db.save()?;
    let save = start.elapsed();
    let start = Instant::now();
    let db = Database::open(&path)?;
    let open = start.elapsed();
    let collection = db.collection("bench")?;
    let stats = collection.stats();

    println!("records {records} · dim {dim} · noise {noise} · cosine · m=16 ef_construction=200");
    println!(
        "build   {:.2} s ({:.0} inserts/s)",
        build.as_secs_f64(),
        records as f64 / build.as_secs_f64()
    );
    println!(
        "file    {:.1} MB · save {:.0} ms · open {:.0} ms",
        std::fs::metadata(&path)?.len() as f64 / 1_048_576.0,
        save.as_secs_f64() * 1e3,
        open.as_secs_f64() * 1e3
    );
    println!(
        "graph   layers {:?} · avg degree L0 {:.1} · unreachable {}",
        stats.nodes_per_layer, stats.avg_degree_layer0, stats.unreachable
    );

    let report = collection.estimate_recall(&RecallOptions {
        sample: 200,
        k: 10,
        ef_values: vec![10, 16, 32, 64, 128, 256],
        seed: 7,
    })?;
    println!();
    println!(
        "exact scan p50 {:.3} ms",
        report.exact_p50.as_secs_f64() * 1e3
    );
    println!(
        "{:>6}  {:>9}  {:>10}  {:>10}  {:>9}",
        "ef", "recall@10", "p50 ms", "p95 ms", "speedup"
    );
    for p in &report.points {
        println!(
            "{:>6}  {:>9.3}  {:>10.3}  {:>10.3}  {:>8.0}x",
            p.ef,
            p.recall,
            p.p50.as_secs_f64() * 1e3,
            p.p95.as_secs_f64() * 1e3,
            report.exact_p50.as_secs_f64() / p.p50.as_secs_f64()
        );
    }

    let _ = std::fs::remove_file(&path);
    Ok(())
}

pub fn checkpoint(&self) -> Result<()>

Atomically merges committed and pending changes into the base snapshot, then resets WAL. A crash between those steps is safe to recover.