diskann_benchmark_core/
lib.rs

1/*
2 * Copyright (c) Microsoft Corporation.
3 * Licensed under the MIT license.
4 */
5
6//! # Shareable Infrastructure for Benchmarking Vector Indexing
7//!
8//! The purpose of this crate is to create abstractions and implementations for benchmarking
9//! DiskANN vector indexing operations. We try to facilitate infrastructure that can be
10//! shared across a range of [`diskann::provider::DataProvider`]s with stable APIs to enable
11//!
12//! * A tight benchmarking loop for developers performing performance optimization.
13//! * Creating standalone binaries for CI benchmarking jobs.
14//! * Shared infrastructure to facilitate developing new providers.
15//!
16//! # Algorithms
17//!
18//! - [`build`]: Tools for running parallelized index builds.
19//!   - [`build::graph`]: Built-in utilities for working with [`diskann::graph::DiskANNIndex`].
20//!
21//! - [`search`]: Tools for running parallelized search operations.
22//!   - [`search::graph`]: Built-in utilities for working with [`diskann::graph::DiskANNIndex`].
23//!
24//! - [`streaming`]: Tools for running streaming workloads consisting of inserts, deletes,
25//!   replaces, searches, etc.
26//!   - [`streaming::runbooks`]: Built-in [`streaming::Executor`]s for dynamic operations.
27//!     - [`streaming::runbooks::bigann`]: BigANN style runbook support.
28//!   - [`streaming::graph`]: Built-in utilities for working with [`diskann::graph::DiskANNIndex`].
29//!
30//! # Tools
31//!
32//! - [`recall`]: KNN-Recall and other accuracy measures.
33//! - [`tokio`]: Quickly create new [`tokio::runtime::Runtime]`s.
34//!
35//! # Error Handling
36//!
37//! Index benchmark operations typically live high in a program's call stack and need to
38//! support a wide variety of index implementations and thus error types. To that end,
39//! [`anyhow::Error`] is typically used at API boundaries. While this does hide the ways
40//! in which method can fail, the [`anyhow::Error`] type balances generality and fidelity.
41
42mod internal;
43pub(crate) mod utils;
44
45// Public Utility Modules
46pub mod recall;
47pub mod tokio;
48
49// Algorithms
50pub mod build;
51pub mod search;
52pub mod streaming;
53
54/// # Notes on Testing
55///
56/// Some components in thsi framework (e.g. runbook parsing), have UX tests to report errors
57/// encountered during parsing. The necessary input files and expected outputs are checked
58/// in to the repo in the `tests` directory.
59///
60/// If error message change, the expected results can generally be regenerated by running
61/// the test suite with the environment variable
62/// ```text
63/// DISKANN_BENCHMARK_TEST=true
64/// ```
65/// set.
66#[cfg(test)]
67mod ux {
68    const ENV_VAR: &str = "DISKANN_BENCHMARK_TEST";
69
70    /// Check if we should overwrite expected files.
71    pub(crate) fn should_overwrite() -> bool {
72        match std::env::var(ENV_VAR) {
73            Ok(v) if v == "overwrite" => true,
74            Ok(v) => panic!(
75                "Unknown value for {}: \"{}\". Expected \"overwrite\"",
76                ENV_VAR, v
77            ),
78            Err(std::env::VarError::NotPresent) => false,
79            Err(std::env::VarError::NotUnicode(_)) => {
80                panic!("Value for {} is not unicode", ENV_VAR)
81            }
82        }
83    }
84
85    pub(crate) fn help() -> String {
86        format!("{}=overwrite", ENV_VAR)
87    }
88
89    pub(crate) fn test_dir() -> std::path::PathBuf {
90        let manifest: &std::path::Path = env!("CARGO_MANIFEST_DIR").as_ref();
91        manifest.join("tests")
92        // let test_data_path: PathBuf = format!("{}/tests/{}", manifest_dir, TEST_DATA_DIR).into();
93    }
94}
diskann_benchmark_core/lib.rs

diskann_benchmark_core/
lib.rs
