lance_index/
optimize.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3
4/// Options for optimizing all indices.
5#[non_exhaustive]
6#[derive(Debug, Clone, Default)]
7pub struct OptimizeOptions {
8    /// Number of delta indices to merge for one column. Default: 1.
9    ///
10    /// If `num_indices_to_merge` is None, lance will create a new delta index if no partition is split, otherwise it will merge all delta indices.
11    /// If `num_indices_to_merge` is Some(N), the delta updates and latest N indices
12    /// will be merged into one single index.
13    ///
14    /// It is up to the caller to decide how many indices to merge / keep. Callers can
15    /// find out how many indices are there by calling [`Dataset::index_statistics`].
16    ///
17    /// A common usage pattern will be that, the caller can keep a large snapshot of the index of the base version,
18    /// and accumulate a few delta indices, then merge them into the snapshot.
19    pub num_indices_to_merge: Option<usize>,
20
21    /// the index names to optimize. If None, all indices will be optimized.
22    pub index_names: Option<Vec<String>>,
23
24    /// whether to retrain the whole index. Default: false.
25    ///
26    /// If true, the index will be retrained based on the current data,
27    /// `num_indices_to_merge` will be ignored, and all indices will be merged into one.
28    /// If false, the index will be optimized by merging `num_indices_to_merge` indices.
29    ///
30    /// This is useful when the data distribution has changed significantly,
31    /// and we want to retrain the index to improve the search quality.
32    /// This would be faster than re-create the index from scratch.
33    ///
34    /// NOTE: this option is only supported for v3 vector indices.
35    pub retrain: bool,
36}
37
38impl OptimizeOptions {
39    pub fn new() -> Self {
40        Self::default()
41    }
42
43    pub fn merge(num: usize) -> Self {
44        Self {
45            num_indices_to_merge: Some(num),
46            index_names: None,
47            ..Default::default()
48        }
49    }
50
51    pub fn append() -> Self {
52        Self {
53            num_indices_to_merge: Some(0),
54            index_names: None,
55            ..Default::default()
56        }
57    }
58
59    pub fn retrain() -> Self {
60        Self {
61            num_indices_to_merge: None,
62            index_names: None,
63            retrain: true,
64        }
65    }
66
67    pub fn num_indices_to_merge(mut self, num: Option<usize>) -> Self {
68        self.num_indices_to_merge = num;
69        self
70    }
71
72    pub fn index_names(mut self, names: Vec<String>) -> Self {
73        self.index_names = Some(names);
74        self
75    }
76}
lance_index/optimize.rs

lance_index/
optimize.rs
