Skip to main content

Collection

git_vdb::adapter

Struct Collection 

Source 
pub struct Collection { /* private fields */ }
Expand description

A named collection view backed by the Git commit/ref adapter.

Implementations§

Source§

impl Collection

Source

pub fn root(&self) -> Result<ObjectId>

Returns the deterministic root resolved by this collection view.

Source

pub fn at(&self, revision: impl AsRef<str>) -> Result<Self>

Creates a read-only historical view at a root, commit, or revision.

Examples found in repository?
examples/lancedb_git_vdb_runner.rs (line 165)
28fn main() -> Result<(), Box<dyn std::error::Error>> {
29    let mut args = env::args_os().skip(1);
30    let input = args.next().ok_or("usage: runner INPUT.json OUTPUT.json")?;
31    let output = args.next().ok_or("usage: runner INPUT.json OUTPUT.json")?;
32    if args.next().is_some() {
33        return Err("usage: runner INPUT.json OUTPUT.json".into());
34    }
35    let spec: RunSpec = serde_json::from_slice(&fs::read(input)?)?;
36    if spec.schema_version != 1 {
37        return Err(format!("unsupported harness schema version {}", spec.schema_version).into());
38    }
39    let vectors = read_vectors(&spec.points_path, spec.point_count, spec.dimension)?;
40    let queries = read_vectors(&spec.queries_path, spec.query_count, spec.dimension)?;
41    let points = make_points(&vectors);
42    let maximum_k = *spec.k.iter().max().ok_or("k must not be empty")?;
43    let config = CollectionConfig {
44        dimension: spec.dimension,
45        ..CollectionConfig::default()
46    };
47
48    let setup_started = Instant::now();
49    let core_dir = tempfile::TempDir::new()?;
50    let engine = SnapshotEngine::init(core_dir.path())?;
51    let setup_us = micros(setup_started);
52    let build_started = Instant::now();
53    let snapshot = engine.build(config.clone(), points.clone())?;
54    let build_us = micros(build_started);
55    let baseline_on_disk_bytes = directory_bytes(core_dir.path())?;
56
57    let (exact_query_us, exact_results, exact_vectors_scored) =
58        query_all(&snapshot, &queries, maximum_k, true, None)?;
59    let (approximate_query_us, approximate_results, approximate_vectors_scored) =
60        query_all(&snapshot, &queries, maximum_k, false, None)?;
61    let snapshot_throughput = json!({
62        "exact": query_snapshot_throughput(
63            &snapshot,
64            &queries,
65            maximum_k,
66            true,
67            &spec.concurrency,
68        )?,
69        "approximate": query_snapshot_throughput(
70            &snapshot,
71            &queries,
72            maximum_k,
73            false,
74            &spec.concurrency,
75        )?,
76    });
77
78    let mut filtered = Map::new();
79    for selectivity in &spec.filter_selectivities {
80        let filter = selectivity_filter(*selectivity);
81        let (exact_us, exact, exact_scored) =
82            query_all(&snapshot, &queries, maximum_k, true, Some(filter.clone()))?;
83        let (approximate_us, approximate, approximate_scored) =
84            query_all(&snapshot, &queries, maximum_k, false, Some(filter))?;
85        filtered.insert(
86            selectivity.to_string(),
87            json!({
88                "exact_query_us": exact_us,
89                "approximate_query_us": approximate_us,
90                "exact_results": exact,
91                "approximate_results": approximate,
92                "exact_vectors_scored": exact_scored,
93                "approximate_vectors_scored": approximate_scored,
94            }),
95        );
96    }
97
98    let mut mutations = Map::new();
99    for fraction in &spec.mutation_fractions {
100        let count = fraction_count(spec.point_count, *fraction);
101        let changed = changed_points(&points[..count]);
102        let upsert_started = Instant::now();
103        let upserted = engine.apply(
104            snapshot.root(),
105            changed.into_iter().map(SnapshotMutation::upsert).collect(),
106        )?;
107        let upsert_us = micros(upsert_started);
108        let delete_started = Instant::now();
109        let deleted = engine.apply(
110            snapshot.root(),
111            vec![SnapshotMutation::delete_ids(
112                (0..count).map(|id| PointId::from(id as u64)),
113            )],
114        )?;
115        let delete_us = micros(delete_started);
116        mutations.insert(
117            fraction.to_string(),
118            json!({
119                "points": count,
120                "upsert_us": upsert_us,
121                "delete_us": delete_us,
122                "upsert_root": upserted.root(),
123                "delete_root": deleted.root(),
124            }),
125        );
126    }
127
128    let adapter_setup_started = Instant::now();
129    let adapter_dir = tempfile::TempDir::new()?;
130    let database = Database::init_bare(adapter_dir.path())?;
131    let collection = database.create_collection("benchmark", config)?;
132    let adapter_setup_us = micros(adapter_setup_started);
133    let adapter_build_started = Instant::now();
134    let adapter_root = collection.upsert(points)?.root;
135    let adapter_build_us = micros(adapter_build_started);
136    if adapter_root != snapshot.root() {
137        return Err("snapshot-core and named-adapter roots differ".into());
138    }
139    query_collection_all(&collection, &queries[..1], maximum_k, true, None)?;
140    query_collection_all(&collection, &queries[..1], maximum_k, false, None)?;
141    let (adapter_exact_query_us, adapter_exact_results, adapter_exact_vectors_scored) =
142        query_collection_all(&collection, &queries, maximum_k, true, None)?;
143    let (
144        adapter_approximate_query_us,
145        adapter_approximate_results,
146        adapter_approximate_vectors_scored,
147    ) = query_collection_all(&collection, &queries, maximum_k, false, None)?;
148    let adapter_throughput = json!({
149        "exact": query_collection_throughput(
150            &collection,
151            &queries,
152            maximum_k,
153            true,
154            &spec.concurrency,
155        )?,
156        "approximate": query_collection_throughput(
157            &collection,
158            &queries,
159            maximum_k,
160            false,
161            &spec.concurrency,
162        )?,
163    });
164    let historical_started = Instant::now();
165    let historical = collection.at(&adapter_root)?;
166    let historical_count = historical.count(None)?.count;
167    let historical_read_us = micros(historical_started);
168
169    let report = json!({
170        "schema_version": 1,
171        "engine": "git-vdb",
172        "case_name": spec.case_name,
173        "point_count": spec.point_count,
174        "dimension": spec.dimension,
175        "query_count": spec.query_count,
176        "k": spec.k,
177        "root": snapshot.root(),
178        "setup_us": setup_us,
179        "snapshot_core": {
180            "build_us": build_us,
181            "exact_query_us": exact_query_us,
182            "approximate_query_us": approximate_query_us,
183            "exact_results": exact_results,
184            "approximate_results": approximate_results,
185            "exact_vectors_scored": exact_vectors_scored,
186            "approximate_vectors_scored": approximate_vectors_scored,
187            "throughput": snapshot_throughput,
188            "filtered": filtered,
189            "mutations": mutations,
190            "on_disk_bytes": baseline_on_disk_bytes,
191        },
192        "named_adapter": {
193            "setup_us": adapter_setup_us,
194            "build_us": adapter_build_us,
195            "exact_query_us": adapter_exact_query_us,
196            "approximate_query_us": adapter_approximate_query_us,
197            "exact_results": adapter_exact_results,
198            "approximate_results": adapter_approximate_results,
199            "exact_vectors_scored": adapter_exact_vectors_scored,
200            "approximate_vectors_scored": adapter_approximate_vectors_scored,
201            "throughput": adapter_throughput,
202            "historical_read_us": historical_read_us,
203            "historical_count": historical_count,
204            "on_disk_bytes": directory_bytes(adapter_dir.path())?,
205        }
206    });
207    fs::write(output, serde_json::to_vec_pretty(&report)?)?;
208    Ok(())
209}
Source

pub fn info(&self) -> Result<CollectionInfo>

Returns metadata for the root resolved by this collection view.

Source

pub fn upsert(&self, points: Vec<Point>) -> Result<WriteResult>

Adds or replaces a non-empty point batch at the current collection root.

The ref is advanced with compare-and-swap semantics. Use Collection::upsert_expect to supply an explicit expected root.

Examples found in repository?
examples/benchmark.rs (line 72)
32fn main() -> git_vdb::Result<()> {
33    let args: Vec<_> = env::args().collect();
34    let count = argument(&args, 1, 1_000);
35    let dimension = argument(&args, 2, 768);
36    let query_count = argument(&args, 3, 100);
37    let clusters = 32.min(count.max(1));
38    let mut rng = SplitMix64(SEED);
39    let centers: Vec<Vec<f32>> = (0..clusters)
40        .map(|_| (0..dimension).map(|_| rng.signed()).collect())
41        .collect();
42    let mut points = Vec::with_capacity(count);
43    for id in 0..count {
44        let center = &centers[id % clusters];
45        let vector = center
46            .iter()
47            .map(|component| component + rng.signed() * 0.08)
48            .collect();
49        points.push(Point {
50            id: (id as u64).into(),
51            vector,
52            payload: Default::default(),
53        });
54    }
55    let queries: Vec<Vec<f32>> = (0..query_count)
56        .map(|index| {
57            centers[index % clusters]
58                .iter()
59                .map(|component| component + rng.signed() * 0.04)
60                .collect()
61        })
62        .collect();
63
64    let temp = tempfile::TempDir::new().expect("temporary benchmark repository");
65    let db = Database::init_bare(temp.path())?;
66    let config = CollectionConfig {
67        dimension,
68        ..CollectionConfig::default()
69    };
70    let collection = db.create_collection("benchmark", config.clone())?;
71    let started = Instant::now();
72    let root = collection.upsert(points)?.root;
73    let build_ms = started.elapsed().as_millis();
74
75    let exact_started = Instant::now();
76    let mut exact = Vec::new();
77    for vector in &queries {
78        exact.push(collection.query(Query {
79            vector: vector.clone(),
80            limit: 10,
81            params: QueryParams {
82                exact: Some(true),
83                ..QueryParams::default()
84            },
85            ..Query::default()
86        })?);
87    }
88    let exact_query_ms = exact_started.elapsed().as_millis();
89
90    let approximate_started = Instant::now();
91    let mut approximate = Vec::new();
92    for vector in &queries {
93        approximate.push(collection.query(Query {
94            vector: vector.clone(),
95            limit: 10,
96            params: QueryParams {
97                exact: Some(false),
98                ..QueryParams::default()
99            },
100            ..Query::default()
101        })?);
102    }
103    let approximate_query_ms = approximate_started.elapsed().as_millis();
104
105    let recall = |k: usize| -> f64 {
106        exact
107            .iter()
108            .zip(&approximate)
109            .map(|(oracle, result)| {
110                let wanted: BTreeSet<_> = oracle.points.iter().take(k).map(|p| &p.id).collect();
111                result
112                    .points
113                    .iter()
114                    .take(k)
115                    .filter(|p| wanted.contains(&p.id))
116                    .count() as f64
117                    / k as f64
118            })
119            .sum::<f64>()
120            / query_count.max(1) as f64
121    };
122    let mut fractions: Vec<_> = approximate
123        .iter()
124        .map(|result| result.stats.vectors_scored as f64 / count.max(1) as f64)
125        .collect();
126    fractions.sort_by(f64::total_cmp);
127    let median_scored_fraction = fractions.get(fractions.len() / 2).copied().unwrap_or(0.0);
128    let count_objects = Command::new("git")
129        .arg("--git-dir")
130        .arg(temp.path())
131        .args(["count-objects", "-v"])
132        .output()
133        .expect("git count-objects");
134    let count_objects = String::from_utf8(count_objects.stdout).expect("UTF-8 Git output");
135    let metric = |name: &str| {
136        count_objects
137            .lines()
138            .find_map(|line| line.strip_prefix(&format!("{name}: ")))
139            .and_then(|value| value.parse().ok())
140            .unwrap_or(0)
141    };
142    let revision = Command::new("git")
143        .args(["rev-parse", "--short=12", "HEAD"])
144        .output()
145        .ok()
146        .filter(|output| output.status.success())
147        .and_then(|output| String::from_utf8(output.stdout).ok())
148        .map(|value| value.trim().to_owned())
149        .unwrap_or_else(|| "uncommitted".into());
150    println!(
151        "{}",
152        serde_json::to_string_pretty(&Report {
153            seed: SEED,
154            points: count,
155            dimension,
156            queries: query_count,
157            clusters,
158            index: config.index,
159            root: root.0,
160            build_ms,
161            exact_query_ms,
162            approximate_query_ms,
163            recall_at_1: recall(1),
164            recall_at_5: recall(5),
165            recall_at_10: recall(10),
166            median_scored_fraction,
167            loose_objects: metric("count"),
168            loose_kib: metric("size"),
169            revision,
170            target: format!("{}-{}", env::consts::OS, env::consts::ARCH),
171        })?
172    );
173    Ok(())
174}
More examples
Hide additional examples
examples/lancedb_git_vdb_runner.rs (line 134)
28fn main() -> Result<(), Box<dyn std::error::Error>> {
29    let mut args = env::args_os().skip(1);
30    let input = args.next().ok_or("usage: runner INPUT.json OUTPUT.json")?;
31    let output = args.next().ok_or("usage: runner INPUT.json OUTPUT.json")?;
32    if args.next().is_some() {
33        return Err("usage: runner INPUT.json OUTPUT.json".into());
34    }
35    let spec: RunSpec = serde_json::from_slice(&fs::read(input)?)?;
36    if spec.schema_version != 1 {
37        return Err(format!("unsupported harness schema version {}", spec.schema_version).into());
38    }
39    let vectors = read_vectors(&spec.points_path, spec.point_count, spec.dimension)?;
40    let queries = read_vectors(&spec.queries_path, spec.query_count, spec.dimension)?;
41    let points = make_points(&vectors);
42    let maximum_k = *spec.k.iter().max().ok_or("k must not be empty")?;
43    let config = CollectionConfig {
44        dimension: spec.dimension,
45        ..CollectionConfig::default()
46    };
47
48    let setup_started = Instant::now();
49    let core_dir = tempfile::TempDir::new()?;
50    let engine = SnapshotEngine::init(core_dir.path())?;
51    let setup_us = micros(setup_started);
52    let build_started = Instant::now();
53    let snapshot = engine.build(config.clone(), points.clone())?;
54    let build_us = micros(build_started);
55    let baseline_on_disk_bytes = directory_bytes(core_dir.path())?;
56
57    let (exact_query_us, exact_results, exact_vectors_scored) =
58        query_all(&snapshot, &queries, maximum_k, true, None)?;
59    let (approximate_query_us, approximate_results, approximate_vectors_scored) =
60        query_all(&snapshot, &queries, maximum_k, false, None)?;
61    let snapshot_throughput = json!({
62        "exact": query_snapshot_throughput(
63            &snapshot,
64            &queries,
65            maximum_k,
66            true,
67            &spec.concurrency,
68        )?,
69        "approximate": query_snapshot_throughput(
70            &snapshot,
71            &queries,
72            maximum_k,
73            false,
74            &spec.concurrency,
75        )?,
76    });
77
78    let mut filtered = Map::new();
79    for selectivity in &spec.filter_selectivities {
80        let filter = selectivity_filter(*selectivity);
81        let (exact_us, exact, exact_scored) =
82            query_all(&snapshot, &queries, maximum_k, true, Some(filter.clone()))?;
83        let (approximate_us, approximate, approximate_scored) =
84            query_all(&snapshot, &queries, maximum_k, false, Some(filter))?;
85        filtered.insert(
86            selectivity.to_string(),
87            json!({
88                "exact_query_us": exact_us,
89                "approximate_query_us": approximate_us,
90                "exact_results": exact,
91                "approximate_results": approximate,
92                "exact_vectors_scored": exact_scored,
93                "approximate_vectors_scored": approximate_scored,
94            }),
95        );
96    }
97
98    let mut mutations = Map::new();
99    for fraction in &spec.mutation_fractions {
100        let count = fraction_count(spec.point_count, *fraction);
101        let changed = changed_points(&points[..count]);
102        let upsert_started = Instant::now();
103        let upserted = engine.apply(
104            snapshot.root(),
105            changed.into_iter().map(SnapshotMutation::upsert).collect(),
106        )?;
107        let upsert_us = micros(upsert_started);
108        let delete_started = Instant::now();
109        let deleted = engine.apply(
110            snapshot.root(),
111            vec![SnapshotMutation::delete_ids(
112                (0..count).map(|id| PointId::from(id as u64)),
113            )],
114        )?;
115        let delete_us = micros(delete_started);
116        mutations.insert(
117            fraction.to_string(),
118            json!({
119                "points": count,
120                "upsert_us": upsert_us,
121                "delete_us": delete_us,
122                "upsert_root": upserted.root(),
123                "delete_root": deleted.root(),
124            }),
125        );
126    }
127
128    let adapter_setup_started = Instant::now();
129    let adapter_dir = tempfile::TempDir::new()?;
130    let database = Database::init_bare(adapter_dir.path())?;
131    let collection = database.create_collection("benchmark", config)?;
132    let adapter_setup_us = micros(adapter_setup_started);
133    let adapter_build_started = Instant::now();
134    let adapter_root = collection.upsert(points)?.root;
135    let adapter_build_us = micros(adapter_build_started);
136    if adapter_root != snapshot.root() {
137        return Err("snapshot-core and named-adapter roots differ".into());
138    }
139    query_collection_all(&collection, &queries[..1], maximum_k, true, None)?;
140    query_collection_all(&collection, &queries[..1], maximum_k, false, None)?;
141    let (adapter_exact_query_us, adapter_exact_results, adapter_exact_vectors_scored) =
142        query_collection_all(&collection, &queries, maximum_k, true, None)?;
143    let (
144        adapter_approximate_query_us,
145        adapter_approximate_results,
146        adapter_approximate_vectors_scored,
147    ) = query_collection_all(&collection, &queries, maximum_k, false, None)?;
148    let adapter_throughput = json!({
149        "exact": query_collection_throughput(
150            &collection,
151            &queries,
152            maximum_k,
153            true,
154            &spec.concurrency,
155        )?,
156        "approximate": query_collection_throughput(
157            &collection,
158            &queries,
159            maximum_k,
160            false,
161            &spec.concurrency,
162        )?,
163    });
164    let historical_started = Instant::now();
165    let historical = collection.at(&adapter_root)?;
166    let historical_count = historical.count(None)?.count;
167    let historical_read_us = micros(historical_started);
168
169    let report = json!({
170        "schema_version": 1,
171        "engine": "git-vdb",
172        "case_name": spec.case_name,
173        "point_count": spec.point_count,
174        "dimension": spec.dimension,
175        "query_count": spec.query_count,
176        "k": spec.k,
177        "root": snapshot.root(),
178        "setup_us": setup_us,
179        "snapshot_core": {
180            "build_us": build_us,
181            "exact_query_us": exact_query_us,
182            "approximate_query_us": approximate_query_us,
183            "exact_results": exact_results,
184            "approximate_results": approximate_results,
185            "exact_vectors_scored": exact_vectors_scored,
186            "approximate_vectors_scored": approximate_vectors_scored,
187            "throughput": snapshot_throughput,
188            "filtered": filtered,
189            "mutations": mutations,
190            "on_disk_bytes": baseline_on_disk_bytes,
191        },
192        "named_adapter": {
193            "setup_us": adapter_setup_us,
194            "build_us": adapter_build_us,
195            "exact_query_us": adapter_exact_query_us,
196            "approximate_query_us": adapter_approximate_query_us,
197            "exact_results": adapter_exact_results,
198            "approximate_results": adapter_approximate_results,
199            "exact_vectors_scored": adapter_exact_vectors_scored,
200            "approximate_vectors_scored": adapter_approximate_vectors_scored,
201            "throughput": adapter_throughput,
202            "historical_read_us": historical_read_us,
203            "historical_count": historical_count,
204            "on_disk_bytes": directory_bytes(adapter_dir.path())?,
205        }
206    });
207    fs::write(output, serde_json::to_vec_pretty(&report)?)?;
208    Ok(())
209}
Source

pub fn upsert_expect( &self, points: Vec<Point>, expected_root: Option<ObjectId>, ) -> Result<WriteResult>

Adds or replaces points only if the current root matches expected_root.

All immutable objects are written before the collection ref is advanced. A stale precondition returns Error::StaleRoot without advancing it.

Source

pub fn apply(&self, mutations: Vec<SnapshotMutation>) -> Result<MutationResult>

Applies an ordered non-empty batch of upserts and deletions atomically.

Every mutation observes the result of the preceding mutation. Immutable objects are built first and the collection ref advances exactly once.

Source

pub fn restore(&self, revision: impl AsRef<str>) -> Result<WriteResult>

Restores a historical root as a new commit at the collection tip.

History is preserved: this never rewinds or deletes existing commits.

Source

pub fn delete(&self, selector: DeleteSelector) -> Result<WriteResult>

Deletes the union of selected IDs and filter matches.

The selector must contain at least one ID or a filter.

Source

pub fn delete_expect( &self, selector: DeleteSelector, expected_root: Option<ObjectId>, ) -> Result<WriteResult>

Deletes selected points only when the current root matches a precondition.

Source

pub fn get(&self, request: GetRequest) -> Result<GetResult>

Retrieves canonically ordered records without similarity scoring.

Source

pub fn count(&self, filter: Option<Filter>) -> Result<CountResult>

Counts all points or those matching a filter.

Examples found in repository?
examples/lancedb_git_vdb_runner.rs (line 166)
28fn main() -> Result<(), Box<dyn std::error::Error>> {
29    let mut args = env::args_os().skip(1);
30    let input = args.next().ok_or("usage: runner INPUT.json OUTPUT.json")?;
31    let output = args.next().ok_or("usage: runner INPUT.json OUTPUT.json")?;
32    if args.next().is_some() {
33        return Err("usage: runner INPUT.json OUTPUT.json".into());
34    }
35    let spec: RunSpec = serde_json::from_slice(&fs::read(input)?)?;
36    if spec.schema_version != 1 {
37        return Err(format!("unsupported harness schema version {}", spec.schema_version).into());
38    }
39    let vectors = read_vectors(&spec.points_path, spec.point_count, spec.dimension)?;
40    let queries = read_vectors(&spec.queries_path, spec.query_count, spec.dimension)?;
41    let points = make_points(&vectors);
42    let maximum_k = *spec.k.iter().max().ok_or("k must not be empty")?;
43    let config = CollectionConfig {
44        dimension: spec.dimension,
45        ..CollectionConfig::default()
46    };
47
48    let setup_started = Instant::now();
49    let core_dir = tempfile::TempDir::new()?;
50    let engine = SnapshotEngine::init(core_dir.path())?;
51    let setup_us = micros(setup_started);
52    let build_started = Instant::now();
53    let snapshot = engine.build(config.clone(), points.clone())?;
54    let build_us = micros(build_started);
55    let baseline_on_disk_bytes = directory_bytes(core_dir.path())?;
56
57    let (exact_query_us, exact_results, exact_vectors_scored) =
58        query_all(&snapshot, &queries, maximum_k, true, None)?;
59    let (approximate_query_us, approximate_results, approximate_vectors_scored) =
60        query_all(&snapshot, &queries, maximum_k, false, None)?;
61    let snapshot_throughput = json!({
62        "exact": query_snapshot_throughput(
63            &snapshot,
64            &queries,
65            maximum_k,
66            true,
67            &spec.concurrency,
68        )?,
69        "approximate": query_snapshot_throughput(
70            &snapshot,
71            &queries,
72            maximum_k,
73            false,
74            &spec.concurrency,
75        )?,
76    });
77
78    let mut filtered = Map::new();
79    for selectivity in &spec.filter_selectivities {
80        let filter = selectivity_filter(*selectivity);
81        let (exact_us, exact, exact_scored) =
82            query_all(&snapshot, &queries, maximum_k, true, Some(filter.clone()))?;
83        let (approximate_us, approximate, approximate_scored) =
84            query_all(&snapshot, &queries, maximum_k, false, Some(filter))?;
85        filtered.insert(
86            selectivity.to_string(),
87            json!({
88                "exact_query_us": exact_us,
89                "approximate_query_us": approximate_us,
90                "exact_results": exact,
91                "approximate_results": approximate,
92                "exact_vectors_scored": exact_scored,
93                "approximate_vectors_scored": approximate_scored,
94            }),
95        );
96    }
97
98    let mut mutations = Map::new();
99    for fraction in &spec.mutation_fractions {
100        let count = fraction_count(spec.point_count, *fraction);
101        let changed = changed_points(&points[..count]);
102        let upsert_started = Instant::now();
103        let upserted = engine.apply(
104            snapshot.root(),
105            changed.into_iter().map(SnapshotMutation::upsert).collect(),
106        )?;
107        let upsert_us = micros(upsert_started);
108        let delete_started = Instant::now();
109        let deleted = engine.apply(
110            snapshot.root(),
111            vec![SnapshotMutation::delete_ids(
112                (0..count).map(|id| PointId::from(id as u64)),
113            )],
114        )?;
115        let delete_us = micros(delete_started);
116        mutations.insert(
117            fraction.to_string(),
118            json!({
119                "points": count,
120                "upsert_us": upsert_us,
121                "delete_us": delete_us,
122                "upsert_root": upserted.root(),
123                "delete_root": deleted.root(),
124            }),
125        );
126    }
127
128    let adapter_setup_started = Instant::now();
129    let adapter_dir = tempfile::TempDir::new()?;
130    let database = Database::init_bare(adapter_dir.path())?;
131    let collection = database.create_collection("benchmark", config)?;
132    let adapter_setup_us = micros(adapter_setup_started);
133    let adapter_build_started = Instant::now();
134    let adapter_root = collection.upsert(points)?.root;
135    let adapter_build_us = micros(adapter_build_started);
136    if adapter_root != snapshot.root() {
137        return Err("snapshot-core and named-adapter roots differ".into());
138    }
139    query_collection_all(&collection, &queries[..1], maximum_k, true, None)?;
140    query_collection_all(&collection, &queries[..1], maximum_k, false, None)?;
141    let (adapter_exact_query_us, adapter_exact_results, adapter_exact_vectors_scored) =
142        query_collection_all(&collection, &queries, maximum_k, true, None)?;
143    let (
144        adapter_approximate_query_us,
145        adapter_approximate_results,
146        adapter_approximate_vectors_scored,
147    ) = query_collection_all(&collection, &queries, maximum_k, false, None)?;
148    let adapter_throughput = json!({
149        "exact": query_collection_throughput(
150            &collection,
151            &queries,
152            maximum_k,
153            true,
154            &spec.concurrency,
155        )?,
156        "approximate": query_collection_throughput(
157            &collection,
158            &queries,
159            maximum_k,
160            false,
161            &spec.concurrency,
162        )?,
163    });
164    let historical_started = Instant::now();
165    let historical = collection.at(&adapter_root)?;
166    let historical_count = historical.count(None)?.count;
167    let historical_read_us = micros(historical_started);
168
169    let report = json!({
170        "schema_version": 1,
171        "engine": "git-vdb",
172        "case_name": spec.case_name,
173        "point_count": spec.point_count,
174        "dimension": spec.dimension,
175        "query_count": spec.query_count,
176        "k": spec.k,
177        "root": snapshot.root(),
178        "setup_us": setup_us,
179        "snapshot_core": {
180            "build_us": build_us,
181            "exact_query_us": exact_query_us,
182            "approximate_query_us": approximate_query_us,
183            "exact_results": exact_results,
184            "approximate_results": approximate_results,
185            "exact_vectors_scored": exact_vectors_scored,
186            "approximate_vectors_scored": approximate_vectors_scored,
187            "throughput": snapshot_throughput,
188            "filtered": filtered,
189            "mutations": mutations,
190            "on_disk_bytes": baseline_on_disk_bytes,
191        },
192        "named_adapter": {
193            "setup_us": adapter_setup_us,
194            "build_us": adapter_build_us,
195            "exact_query_us": adapter_exact_query_us,
196            "approximate_query_us": adapter_approximate_query_us,
197            "exact_results": adapter_exact_results,
198            "approximate_results": adapter_approximate_results,
199            "exact_vectors_scored": adapter_exact_vectors_scored,
200            "approximate_vectors_scored": adapter_approximate_vectors_scored,
201            "throughput": adapter_throughput,
202            "historical_read_us": historical_read_us,
203            "historical_count": historical_count,
204            "on_disk_bytes": directory_bytes(adapter_dir.path())?,
205        }
206    });
207    fs::write(output, serde_json::to_vec_pretty(&report)?)?;
208    Ok(())
209}
Source

pub fn query(&self, query: Query) -> Result<QueryResult>

Executes an exact or deterministic approximate vector query.

The returned result identifies the root actually read. Query caches are scoped to that immutable root and are replaced when the collection moves.

Examples found in repository?
examples/filtering.rs (lines 11-15)
4fn main() -> git_vdb::Result<()> {
5    let temporary = tempfile::TempDir::new()?;
6    let docs = open(temporary.path().join("vectors.git"))?.collection("docs");
7    docs.upsert([
8        Point::new("guide", [1.0, 0.0]).with_metadata(json!({"kind": "docs"}))?,
9        Point::new("code", [0.9, 0.1]).with_metadata(json!({"kind": "source"}))?,
10    ])?;
11    let result = docs.advanced()?.query(
12        Query::new([1.0, 0.0], 5)
13            .with_filter(Filter::must([Condition::matches("kind", "docs")]))
14            .with_payload(),
15    )?;
16    assert_eq!(result.points[0].id.to_string(), "guide");
17    Ok(())
18}
More examples
Hide additional examples
examples/lancedb_git_vdb_runner.rs (lines 304-313)
292fn query_collection_all(
293    collection: &git_vdb::Collection,
294    queries: &[Vec<f32>],
295    limit: usize,
296    exact: bool,
297    filter: Option<Filter>,
298) -> git_vdb::Result<QueryBatch> {
299    let mut durations = Vec::with_capacity(queries.len());
300    let mut results = Vec::with_capacity(queries.len());
301    let mut vectors_scored = Vec::with_capacity(queries.len());
302    for vector in queries {
303        let started = Instant::now();
304        let result = collection.query(Query {
305            vector: vector.clone(),
306            limit,
307            filter: filter.clone(),
308            params: QueryParams {
309                exact: Some(exact),
310                ..QueryParams::default()
311            },
312            ..Query::default()
313        })?;
314        durations.push(micros(started));
315        vectors_scored.push(result.stats.vectors_scored);
316        results.push(Value::Array(
317            result
318                .points
319                .into_iter()
320                .map(|point| json!({"id": point.id, "score": point.score}))
321                .collect(),
322        ));
323    }
324    Ok((durations, results, vectors_scored))
325}
326
327fn query_snapshot_throughput(
328    snapshot: &git_vdb::Snapshot,
329    queries: &[Vec<f32>],
330    limit: usize,
331    exact: bool,
332    concurrencies: &[usize],
333) -> git_vdb::Result<Map<String, Value>> {
334    let mut measurements = Map::new();
335    for &workers in concurrencies {
336        if workers == 0 {
337            return Err(git_vdb::Error::Invalid(
338                "benchmark concurrency must be positive".into(),
339            ));
340        }
341        let started = Instant::now();
342        let outcome: git_vdb::Result<()> = thread::scope(|scope| {
343            let mut handles = Vec::with_capacity(workers);
344            for worker in 0..workers {
345                let snapshot = snapshot.clone();
346                handles.push(scope.spawn(move || -> git_vdb::Result<()> {
347                    for vector in queries.iter().skip(worker).step_by(workers) {
348                        snapshot.query(Query {
349                            vector: vector.clone(),
350                            limit,
351                            params: QueryParams {
352                                exact: Some(exact),
353                                ..QueryParams::default()
354                            },
355                            ..Query::default()
356                        })?;
357                    }
358                    Ok(())
359                }));
360            }
361            join_query_workers(handles)
362        });
363        outcome?;
364        measurements.insert(
365            workers.to_string(),
366            throughput_measurement(queries.len(), micros(started)),
367        );
368    }
369    Ok(measurements)
370}
371
372fn query_collection_throughput(
373    collection: &git_vdb::Collection,
374    queries: &[Vec<f32>],
375    limit: usize,
376    exact: bool,
377    concurrencies: &[usize],
378) -> git_vdb::Result<Map<String, Value>> {
379    let mut measurements = Map::new();
380    for &workers in concurrencies {
381        if workers == 0 {
382            return Err(git_vdb::Error::Invalid(
383                "benchmark concurrency must be positive".into(),
384            ));
385        }
386        let started = Instant::now();
387        let outcome: git_vdb::Result<()> = thread::scope(|scope| {
388            let mut handles = Vec::with_capacity(workers);
389            for worker in 0..workers {
390                let collection = collection.clone();
391                handles.push(scope.spawn(move || -> git_vdb::Result<()> {
392                    for vector in queries.iter().skip(worker).step_by(workers) {
393                        collection.query(Query {
394                            vector: vector.clone(),
395                            limit,
396                            params: QueryParams {
397                                exact: Some(exact),
398                                ..QueryParams::default()
399                            },
400                            ..Query::default()
401                        })?;
402                    }
403                    Ok(())
404                }));
405            }
406            join_query_workers(handles)
407        });
408        outcome?;
409        measurements.insert(
410            workers.to_string(),
411            throughput_measurement(queries.len(), micros(started)),
412        );
413    }
414    Ok(measurements)
415}
examples/benchmark.rs (lines 78-86)
32fn main() -> git_vdb::Result<()> {
33    let args: Vec<_> = env::args().collect();
34    let count = argument(&args, 1, 1_000);
35    let dimension = argument(&args, 2, 768);
36    let query_count = argument(&args, 3, 100);
37    let clusters = 32.min(count.max(1));
38    let mut rng = SplitMix64(SEED);
39    let centers: Vec<Vec<f32>> = (0..clusters)
40        .map(|_| (0..dimension).map(|_| rng.signed()).collect())
41        .collect();
42    let mut points = Vec::with_capacity(count);
43    for id in 0..count {
44        let center = &centers[id % clusters];
45        let vector = center
46            .iter()
47            .map(|component| component + rng.signed() * 0.08)
48            .collect();
49        points.push(Point {
50            id: (id as u64).into(),
51            vector,
52            payload: Default::default(),
53        });
54    }
55    let queries: Vec<Vec<f32>> = (0..query_count)
56        .map(|index| {
57            centers[index % clusters]
58                .iter()
59                .map(|component| component + rng.signed() * 0.04)
60                .collect()
61        })
62        .collect();
63
64    let temp = tempfile::TempDir::new().expect("temporary benchmark repository");
65    let db = Database::init_bare(temp.path())?;
66    let config = CollectionConfig {
67        dimension,
68        ..CollectionConfig::default()
69    };
70    let collection = db.create_collection("benchmark", config.clone())?;
71    let started = Instant::now();
72    let root = collection.upsert(points)?.root;
73    let build_ms = started.elapsed().as_millis();
74
75    let exact_started = Instant::now();
76    let mut exact = Vec::new();
77    for vector in &queries {
78        exact.push(collection.query(Query {
79            vector: vector.clone(),
80            limit: 10,
81            params: QueryParams {
82                exact: Some(true),
83                ..QueryParams::default()
84            },
85            ..Query::default()
86        })?);
87    }
88    let exact_query_ms = exact_started.elapsed().as_millis();
89
90    let approximate_started = Instant::now();
91    let mut approximate = Vec::new();
92    for vector in &queries {
93        approximate.push(collection.query(Query {
94            vector: vector.clone(),
95            limit: 10,
96            params: QueryParams {
97                exact: Some(false),
98                ..QueryParams::default()
99            },
100            ..Query::default()
101        })?);
102    }
103    let approximate_query_ms = approximate_started.elapsed().as_millis();
104
105    let recall = |k: usize| -> f64 {
106        exact
107            .iter()
108            .zip(&approximate)
109            .map(|(oracle, result)| {
110                let wanted: BTreeSet<_> = oracle.points.iter().take(k).map(|p| &p.id).collect();
111                result
112                    .points
113                    .iter()
114                    .take(k)
115                    .filter(|p| wanted.contains(&p.id))
116                    .count() as f64
117                    / k as f64
118            })
119            .sum::<f64>()
120            / query_count.max(1) as f64
121    };
122    let mut fractions: Vec<_> = approximate
123        .iter()
124        .map(|result| result.stats.vectors_scored as f64 / count.max(1) as f64)
125        .collect();
126    fractions.sort_by(f64::total_cmp);
127    let median_scored_fraction = fractions.get(fractions.len() / 2).copied().unwrap_or(0.0);
128    let count_objects = Command::new("git")
129        .arg("--git-dir")
130        .arg(temp.path())
131        .args(["count-objects", "-v"])
132        .output()
133        .expect("git count-objects");
134    let count_objects = String::from_utf8(count_objects.stdout).expect("UTF-8 Git output");
135    let metric = |name: &str| {
136        count_objects
137            .lines()
138            .find_map(|line| line.strip_prefix(&format!("{name}: ")))
139            .and_then(|value| value.parse().ok())
140            .unwrap_or(0)
141    };
142    let revision = Command::new("git")
143        .args(["rev-parse", "--short=12", "HEAD"])
144        .output()
145        .ok()
146        .filter(|output| output.status.success())
147        .and_then(|output| String::from_utf8(output.stdout).ok())
148        .map(|value| value.trim().to_owned())
149        .unwrap_or_else(|| "uncommitted".into());
150    println!(
151        "{}",
152        serde_json::to_string_pretty(&Report {
153            seed: SEED,
154            points: count,
155            dimension,
156            queries: query_count,
157            clusters,
158            index: config.index,
159            root: root.0,
160            build_ms,
161            exact_query_ms,
162            approximate_query_ms,
163            recall_at_1: recall(1),
164            recall_at_5: recall(5),
165            recall_at_10: recall(10),
166            median_scored_fraction,
167            loose_objects: metric("count"),
168            loose_kib: metric("size"),
169            revision,
170            target: format!("{}-{}", env::consts::OS, env::consts::ARCH),
171        })?
172    );
173    Ok(())
174}
Source

pub fn history(&self, limit: usize) -> Result<Vec<HistoryEntry>>

Returns at most limit collection commits, newest first.

Examples found in repository?
examples/history.rs (line 8)
3fn main() -> git_vdb::Result<()> {
4    let temporary = tempfile::TempDir::new()?;
5    let docs = open(temporary.path().join("vectors.git"))?.collection("docs");
6    docs.upsert([Point::new("v1", [1.0, 0.0])])?;
7    docs.upsert([Point::new("v2", [0.0, 1.0])])?;
8    assert_eq!(docs.advanced()?.history(10)?.len(), 3);
9    Ok(())
10}
Source

pub fn diff( &self, left_revision: impl AsRef<str>, right_revision: impl AsRef<str>, ) -> Result<DiffResult>

Compares logical points and structural sharing between two revisions.

Source

pub fn validate(&self, full: bool) -> Result<ValidationReport>

Validates the resolved root without modifying objects or refs.

Full validation recomputes every approximate-index bucket.

Trait Implementations§

Source§

impl Clone for Collection

Source§

fn clone(&self) -> Self

Returns a duplicate of the value. Read more
1.0.0 (const: unstable) · Source§

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source. Read more
Source§

impl Debug for Collection

Source§

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter. Read more

Auto Trait Implementations§

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<ST, DT> CastableFrom<ST, Initialized, Initialized> for DT
where ST: ?Sized, DT: ?Sized,

Source§

impl<ST, DT> CastableFrom<ST, Uninit, Uninit> for DT
where ST: ?Sized, DT: ?Sized,

Source§

impl<T> CloneToUninit for T
where T: Clone,

Source§

unsafe fn clone_to_uninit(&self, dest: *mut u8)

🔬This is a nightly-only experimental API. (clone_to_uninit)
Performs copy-assignment from self to dest. Read more
Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> IntoEither for T

Source§

fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ

Converts self into a Left variant of Either<Self, Self> if into_left is true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more
Source§

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
where F: FnOnce(&Self) -> bool,

Converts self into a Left variant of Either<Self, Self> if into_left(&self) returns true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more
Source§

impl<T> Pointable for T

Source§

const ALIGN: usize

The alignment of pointer.
Source§

type Init = T

The type for initializers.
Source§

unsafe fn init(init: <T as Pointable>::Init) -> usize

Initializes a with the given initializer. Read more
Source§

unsafe fn deref<'a>(ptr: usize) -> &'a T

Dereferences the given pointer. Read more
Source§

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

Mutably dereferences the given pointer. Read more
Source§

unsafe fn drop(ptr: usize)

Drops the object pointed to by the given pointer. Read more
Source§

impl<T> Read<Exclusive, BecauseExclusive> for T
where T: ?Sized,

Source§

impl<T> Same for T

Source§

type Output = T

Should always be Self
Source§

impl<T> ToOwned for T
where T: Clone,

Source§

type Owned = T

The resulting type after obtaining ownership.
Source§

fn to_owned(&self) -> T

Creates owned data from borrowed data, usually by cloning. Read more
Source§

fn clone_into(&self, target: &mut T)

Uses borrowed data to replace owned data, usually by cloning. Read more
Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = !

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, !>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.
Source§

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

Source§

fn vzip(self) -> V