lance_index/
lib.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3#![cfg_attr(coverage, feature(coverage_attribute))]
4
5//! Lance secondary index library
6//!
7//! <section class="warning">
8//! This is internal crate used by <a href="https://github.com/lance-format/lance">the lance project</a>.
9//! <br/>
10//! API stability is not guaranteed.
11//! </section>
12
13use crate::frag_reuse::FRAG_REUSE_INDEX_NAME;
14use crate::mem_wal::MEM_WAL_INDEX_NAME;
15use serde::{Deserialize, Serialize};
16
17pub mod frag_reuse;
18pub mod mem_wal;
19pub mod metrics;
20pub mod optimize;
21pub mod prefilter;
22pub mod progress;
23pub mod registry;
24pub mod scalar;
25pub mod traits;
26pub mod vector;
27
28pub use crate::traits::*;
29
30// Re-export core traits from lance-index-core
31pub use lance_index_core::{Index, IndexParams, IndexType};
32
33pub const INDEX_FILE_NAME: &str = "index.idx";
34/// The name of the auxiliary index file.
35///
36/// This file is used to store additional information about the index, to improve performance.
37/// - For 'IVF_HNSW' index, it stores the partitioned PQ Storage.
38pub const INDEX_AUXILIARY_FILE_NAME: &str = "auxiliary.idx";
39pub const INDEX_METADATA_SCHEMA_KEY: &str = "lance:index";
40
41/// Default version for vector index metadata.
42///
43/// Most vector indices should use this version unless they need to bump for a
44/// format change.
45pub const VECTOR_INDEX_VERSION: u32 = 1;
46/// Version for IVF_RQ indices.
47pub const IVF_RQ_INDEX_VERSION: u32 = 2;
48
49/// The factor of threshold to trigger split / join for vector index.
50///
51/// If the number of rows in the single partition is greater than `MAX_PARTITION_SIZE_FACTOR * target_partition_size`,
52/// the partition will be split.
53/// If the number of rows in the single partition is less than `MIN_PARTITION_SIZE_PERCENT *target_partition_size / 100`,
54/// the partition will be joined.
55pub const MAX_PARTITION_SIZE_FACTOR: usize = 4;
56pub const MIN_PARTITION_SIZE_PERCENT: usize = 25;
57
58pub mod pb {
59    #![allow(clippy::use_self)]
60    include!(concat!(env!("OUT_DIR"), "/lance.index.pb.rs"));
61}
62
63pub mod pbold {
64    #![allow(clippy::use_self)]
65    include!(concat!(env!("OUT_DIR"), "/lance.table.rs"));
66}
67
68/// Protobuf headers for serialized index cache entries (FTS posting lists,
69/// scalar indices, and IVF vector partitions).
70pub mod cache_pb {
71    #![allow(clippy::use_self)]
72    include!(concat!(env!("OUT_DIR"), "/lance.index.cache.rs"));
73}
74
75#[derive(Serialize, Deserialize, Debug)]
76pub struct IndexMetadata {
77    #[serde(rename = "type")]
78    pub index_type: String,
79    pub distance_type: String,
80}
81
82pub fn is_system_index(index_meta: &lance_table::format::IndexMetadata) -> bool {
83    index_meta.name == FRAG_REUSE_INDEX_NAME || index_meta.name == MEM_WAL_INDEX_NAME
84}
85
86pub fn infer_system_index_type(
87    index_meta: &lance_table::format::IndexMetadata,
88) -> Option<IndexType> {
89    if index_meta.name == FRAG_REUSE_INDEX_NAME {
90        Some(IndexType::FragmentReuse)
91    } else if index_meta.name == MEM_WAL_INDEX_NAME {
92        Some(IndexType::MemWal)
93    } else {
94        None
95    }
96}
97
98#[cfg(test)]
99mod tests {
100    use super::*;
101
102    #[test]
103    fn test_ivf_rq_has_dedicated_index_version() {
104        assert!(IndexType::IvfRq.version() > IndexType::IvfPq.version());
105        assert_eq!(IndexType::IvfRq.version() as u32, IVF_RQ_INDEX_VERSION);
106    }
107
108    #[test]
109    fn test_max_vector_version_tracks_highest_supported() {
110        assert_eq!(IndexType::max_vector_version(), IVF_RQ_INDEX_VERSION);
111    }
112
113    #[test]
114    fn test_ivf_rq_target_partition_size() {
115        assert_eq!(IndexType::IvfRq.target_partition_size(), 4096);
116    }
117
118    #[test]
119    fn test_index_type_try_from_i32_covers_all_variants() {
120        let all = [
121            IndexType::Scalar,
122            IndexType::BTree,
123            IndexType::Bitmap,
124            IndexType::LabelList,
125            IndexType::Inverted,
126            IndexType::NGram,
127            IndexType::FragmentReuse,
128            IndexType::MemWal,
129            IndexType::ZoneMap,
130            IndexType::BloomFilter,
131            IndexType::RTree,
132            IndexType::Fm,
133            IndexType::Vector,
134            IndexType::IvfFlat,
135            IndexType::IvfSq,
136            IndexType::IvfPq,
137            IndexType::IvfHnswSq,
138            IndexType::IvfHnswPq,
139            IndexType::IvfHnswFlat,
140            IndexType::IvfRq,
141        ];
142
143        for index_type in all {
144            assert_eq!(
145                IndexType::try_from(index_type as i32).unwrap(),
146                index_type,
147                "IndexType::try_from(i32) should support {:?}",
148                index_type
149            );
150        }
151    }
152
153    #[test]
154    fn test_index_type_try_from_str_covers_all_parseable_variants() {
155        let cases = [
156            ("BTree", IndexType::BTree),
157            ("BTREE", IndexType::BTree),
158            ("Bitmap", IndexType::Bitmap),
159            ("BITMAP", IndexType::Bitmap),
160            ("LabelList", IndexType::LabelList),
161            ("LABELLIST", IndexType::LabelList),
162            ("Inverted", IndexType::Inverted),
163            ("INVERTED", IndexType::Inverted),
164            ("NGram", IndexType::NGram),
165            ("NGRAM", IndexType::NGram),
166            ("ZoneMap", IndexType::ZoneMap),
167            ("ZONEMAP", IndexType::ZoneMap),
168            ("BloomFilter", IndexType::BloomFilter),
169            ("BLOOMFILTER", IndexType::BloomFilter),
170            ("BLOOM_FILTER", IndexType::BloomFilter),
171            ("RTree", IndexType::RTree),
172            ("RTREE", IndexType::RTree),
173            ("R_TREE", IndexType::RTree),
174            ("Fm", IndexType::Fm),
175            ("FM", IndexType::Fm),
176            ("Vector", IndexType::Vector),
177            ("VECTOR", IndexType::Vector),
178            ("IVF_FLAT", IndexType::IvfFlat),
179            ("IVF_SQ", IndexType::IvfSq),
180            ("IVF_PQ", IndexType::IvfPq),
181            ("IVF_RQ", IndexType::IvfRq),
182            ("IVF_HNSW_FLAT", IndexType::IvfHnswFlat),
183            ("IVF_HNSW_SQ", IndexType::IvfHnswSq),
184            ("IVF_HNSW_PQ", IndexType::IvfHnswPq),
185            ("FragmentReuse", IndexType::FragmentReuse),
186            ("MemWal", IndexType::MemWal),
187        ];
188
189        for (text, expected) in cases {
190            assert_eq!(
191                IndexType::try_from(text).unwrap(),
192                expected,
193                "IndexType::try_from(&str) should support '{text}'"
194            );
195        }
196    }
197}
lance_index/lib.rs

lance_index/
lib.rs
