1use scann_core::{
25 AhOptions, ConfigBuilder, DistanceMeasure, Neighbors, ReorderOptions, ScannIndex,
26 SearchOptions, TreeOptions,
27};
28use std::path::Path;
29use std::time::Instant;
30
31struct Npy {
34 rows: usize,
35 cols: usize,
36 bytes: Vec<u8>,
37}
38
39impl Npy {
40 fn load(path: &Path) -> Npy {
41 let bytes = std::fs::read(path).unwrap_or_else(|e| panic!("{}: {e}", path.display()));
42 assert_eq!(&bytes[..6], b"\x93NUMPY", "{}: not a .npy file", path.display());
43 let (header_len, start) = match bytes[6] {
44 1 => (u16::from_le_bytes([bytes[8], bytes[9]]) as usize, 10),
45 _ => (u32::from_le_bytes(bytes[8..12].try_into().unwrap()) as usize, 12),
46 };
47 let header = std::str::from_utf8(&bytes[start..start + header_len]).unwrap();
48 assert!(header.contains("'<f4'") || header.contains("'<u4'"), "{header}");
49 assert!(header.contains("'fortran_order': False"), "{header}");
50 let shape = &header[header.find("'shape': (").unwrap() + 10..];
51 let dims: Vec<usize> = shape[..shape.find(')').unwrap()]
52 .split(',')
53 .filter_map(|s| s.trim().parse().ok())
54 .collect();
55 Npy { rows: dims[0], cols: dims[1], bytes: bytes[start + header_len..].to_vec() }
56 }
57
58 fn f32s(&self) -> Vec<f32> {
59 self.bytes.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect()
60 }
61
62 fn u32s(&self) -> Vec<u32> {
63 self.bytes.as_chunks::<4>().0.iter().map(|b| u32::from_le_bytes(*b)).collect()
64 }
65}
66
67fn recall(found: &[Neighbors], truth: &[u32], truth_cols: usize) -> f64 {
69 let hits: usize = found
70 .iter()
71 .enumerate()
72 .map(|(q, nn)| {
73 let t = &truth[q * truth_cols..][..10];
74 nn.indices.iter().filter(|i| t.contains(i)).count()
75 })
76 .sum();
77 hits as f64 / (found.len() * 10) as f64
78}
79
80fn evaluate(name: &str, index: &mut ScannIndex, queries: &[f32], truth: &[u32], cols: usize) {
81 index.set_num_threads(std::thread::available_parallelism().map_or(1, |n| n.get())).unwrap();
82 let n_queries = queries.len() / index.dimensionality();
83 let mut best = f64::INFINITY;
84 let mut found = Vec::new();
85 for _ in 0..3 {
86 let start = Instant::now();
88 found = index.search_batched_parallel(queries, SearchOptions::default(), 256).unwrap();
89 best = best.min(start.elapsed().as_secs_f64());
90 }
91 println!(
92 "{name:26} recall@10 {:.4} {:8.0} QPS",
93 recall(&found, truth, cols),
94 n_queries as f64 / best
95 );
96}
97
98fn main() {
99 let dir = std::env::args().nth(1).expect("usage: tutorial_glove <data dir>");
100 let dir = Path::new(&dir);
101 let train = Npy::load(&dir.join("glove_train.npy"));
102 let test = Npy::load(&dir.join("glove_test.npy"));
103 let truth = Npy::load(&dir.join("glove_neighbors.npy"));
104 println!("dataset {} x {}, {} queries", train.rows, train.cols, test.rows);
105 let (dataset, queries, truth_ids) = (train.f32s(), test.f32s(), truth.u32s());
106
107 let start = Instant::now();
109 let mut built = ConfigBuilder::new(10, DistanceMeasure::DotProduct, train.cols)
110 .tree(TreeOptions::new(2000, 100).training_sample_size(250_000))
111 .score_ah(AhOptions::new(2).anisotropic_quantization_threshold(0.2))
112 .reorder(ReorderOptions::new(100))
113 .build_index(&dataset)
114 .unwrap();
115 println!("built in {:.1} s", start.elapsed().as_secs_f64());
116 evaluate("built in Rust", &mut built, &queries, &truth_ids, truth.cols);
117
118 let start = Instant::now();
120 let mut loaded = ScannIndex::load(dir.join("glove-index")).unwrap();
121 println!("loaded in {:.1} s", start.elapsed().as_secs_f64());
122 evaluate("built in Python, loaded", &mut loaded, &queries, &truth_ids, truth.cols);
123
124 let dim = loaded.dimensionality();
127 let index = &loaded;
128 for threads in [1, 8, 32, 64] {
129 let start = Instant::now();
130 std::thread::scope(|s| {
131 for t in 0..threads {
132 let queries = &queries;
133 s.spawn(move || {
134 for q in queries.chunks(dim).skip(t).step_by(threads) {
136 index.search(q, SearchOptions::default()).unwrap();
137 }
138 });
139 }
140 });
141 let qps = (queries.len() / dim) as f64 / start.elapsed().as_secs_f64();
142 println!("concurrent search() from {threads:2} threads: {qps:8.0} QPS");
143 }
144}