vicinity 0.11.1

Approximate nearest-neighbor search
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138

# vicinity -- task runner

ann_broad_features := "hnsw,nsw,vamana,diskann,ivf_pq,ivf_avq,ivf_rabitq,emg,nsg,pipnn,sng,finger,fresh_graph,filtered_graph,curator,range_filtered,rp_quant,binary_index,sq4,sq8,lsh,rptree,kdtree,balltree,kmeans_tree"
ann_broad_algos := "--algo hnsw --algo nsw --algo vamana --algo diskann --algo ivfpq --algo ivf_avq --algo ivf_rabitq --algo emg --algo nsg --algo dual_branch --algo deg --algo pipnn --algo sng --algo finger --algo fresh_graph --algo filtered_graph --algo curator --algo range_filtered --algo rp_quant --algo binary_index --algo sq4 --algo sq4u --algo sq8u --algo symphony_qg --algo symphony_qg_vr --algo adsampling --algo lsh --algo hnsw_prt --algo brute --algo kdtree --algo balltree --algo rptree --algo rp_forest --algo kmeans_tree"
python_lint_paths := "pyvicinity tests/test_python.py tests/test_download_ann_benchmarks.py tests/test_generate_ann_smoke_data.py tests/test_generate_multiscale_data.py tests/test_generate_sample_data.py tests/test_generate_sparse_mips_smoke_data.py tests/test_summarize_ann_results.py examples/python scripts/generate_ann_smoke_data.py scripts/generate_sample_data.py scripts/generate_sparse_mips_smoke_data.py scripts/download_ann_benchmarks.py scripts/summarize_ann_results.py"
python_test_paths := "tests/test_python.py tests/test_download_ann_benchmarks.py tests/test_generate_ann_smoke_data.py tests/test_generate_multiscale_data.py tests/test_generate_sample_data.py tests/test_generate_sparse_mips_smoke_data.py tests/test_summarize_ann_results.py"

default:
    @just --list

# ─── Standard ann-benchmarks suite ───────────────────────────────────────────

# Download an ann-benchmarks dataset (e.g. just download glove-25-angular)
download dataset:
    uv run scripts/download_ann_benchmarks.py {{dataset}}

# List available ann-benchmarks datasets
download-list:
    uv run scripts/download_ann_benchmarks.py --list

# Run ann-benchmark on a dataset with all compiled algorithms
ann dataset="data/ann-benchmarks/glove-25-angular":
    cargo run --example ann_benchmark --release --features hnsw,nsw -- {{dataset}}

# Run ann-benchmark with JSON output (for plotting)
ann-json dataset="data/ann-benchmarks/glove-25-angular":
    cargo run --example ann_benchmark --release --features hnsw,nsw -- {{dataset}} --json --fresh

# Resume ann-benchmark JSON output, skipping completed rows in the result file
ann-json-resume dataset="data/ann-benchmarks/glove-25-angular":
    cargo run --example ann_benchmark --release --features hnsw,nsw -- {{dataset}} --json --resume

# Run the broad dense-vector sweep documented in docs/benchmark-results.md
ann-broad dataset="data/ann-benchmarks/glove-25-angular":
    cargo run --example ann_benchmark --release --features "{{ann_broad_features}}" -- {{dataset}} {{ann_broad_algos}} --pq-training-sample-size 100000 --pq-kmeans-max-iter 20

# Run the broad dense-vector sweep with JSONL output
ann-broad-json dataset="data/ann-benchmarks/glove-25-angular":
    cargo run --example ann_benchmark --release --features "{{ann_broad_features}}" -- {{dataset}} {{ann_broad_algos}} --pq-training-sample-size 100000 --pq-kmeans-max-iter 20 --json --fresh

# Resume the broad dense-vector JSONL sweep
ann-broad-json-resume dataset="data/ann-benchmarks/glove-25-angular":
    cargo run --example ann_benchmark --release --features "{{ann_broad_features}}" -- {{dataset}} {{ann_broad_algos}} --pq-training-sample-size 100000 --pq-kmeans-max-iter 20 --json --resume

# Full standard benchmark pipeline: download + run + plot
bench-standard:
    @echo "Downloading datasets (if needed)..."
    just download glove-25-angular
    just download sift-128-euclidean
    @echo "Running benchmarks..."
    mkdir -p data/ann-benchmarks/results
    just ann-json data/ann-benchmarks/glove-25-angular > data/ann-benchmarks/results/glove-25.jsonl
    just ann-json data/ann-benchmarks/sift-128-euclidean > data/ann-benchmarks/results/sift-128.jsonl
    @echo "Done. Results in data/ann-benchmarks/results/"

# ─── Rigorous benchmark (multi-run, CI, LID-stratified) ─────────────────────

# Generate synthetic multiscale data (S/M/L/B/T/P)
gen scale:
    uvx --with numpy python scripts/generate_multiscale_data.py --scale {{scale}}

gen-all:
    uvx --with numpy python scripts/generate_multiscale_data.py --scale all

# Run rigorous benchmark at a scale
rigorous scale:
    cargo run --example 04_rigorous_benchmark --release -- --scale {{scale}}

# Full rigorous suite
rigorous-all:
    just gen S && just rigorous S
    just gen M && just rigorous M
    just gen L && just rigorous L
    just plot

# Generate plots and report from rigorous benchmark data
plot:
    uvx --with numpy --with matplotlib python scripts/plot_pareto.py

# ─── Criterion microbenchmarks ───────────────────────────────────────────────

# Run all Criterion benchmarks
criterion:
    cargo bench

# Run a specific Criterion benchmark (e.g. just criterion-one hnsw)
criterion-one name:
    cargo bench --bench {{name}}

# ─── Development ─────────────────────────────────────────────────────────────

# Check all features compile
check:
    cargo check --all-features

# Run tests
test:
    cargo test

# Clippy (default features)
lint:
    cargo clippy --features hnsw -- -D warnings

# Format check
fmt:
    cargo fmt -- --check

# Full QA: fmt + lint + test
qa: fmt lint test

# ─── Python bindings (pyvicinity) ────────────────────────────────────────────

# Build + install the Python wheel into the local venv
py-build:
    .venv/bin/maturin develop --release

# Run the Python test suite (rebuilds wheel first)
py-test: py-build
    .venv/bin/python -m pytest {{python_test_paths}} -v

# Lint the Python sources with ruff
py-lint:
    .venv/bin/python -m ruff check {{python_lint_paths}}

# Check Python formatting with ruff
py-fmt:
    .venv/bin/python -m ruff format --check {{python_lint_paths}}

# Type-check a sample caller against the .pyi stubs
py-typecheck: py-build
    .venv/bin/python -m mypy --strict pyvicinity

# Verify the hand-written .pyi matches the compiled module (voyager pattern)
py-stubtest: py-build
    .venv/bin/python -m mypy.stubtest pyvicinity._core

# Full Python QA: build + lint + format + typecheck + stubtest + test
py-qa: py-build py-lint py-fmt py-typecheck py-stubtest py-test