lance_table/format/
index.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3
4//! Metadata for index
5
6use deepsize::DeepSizeOf;
7use roaring::RoaringBitmap;
8use snafu::location;
9use uuid::Uuid;
10
11use super::pb;
12use lance_core::{Error, Result};
13/// Index metadata
14#[derive(Debug, Clone, PartialEq)]
15pub struct Index {
16    /// Unique ID across all dataset versions.
17    pub uuid: Uuid,
18
19    /// Fields to build the index.
20    pub fields: Vec<i32>,
21
22    /// Human readable index name
23    pub name: String,
24
25    /// The latest version of the dataset this index covers
26    pub dataset_version: u64,
27
28    /// The fragment ids this index covers.
29    ///
30    /// If this is None, then this is unknown.
31    pub fragment_bitmap: Option<RoaringBitmap>,
32
33    /// Metadata specific to the index type
34    ///
35    /// This is an Option because older versions of Lance may not have this defined.  However, it should always
36    /// be present in newer versions.
37    pub index_details: Option<prost_types::Any>,
38
39    /// The index version.
40    pub index_version: i32,
41}
42
43impl DeepSizeOf for Index {
44    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
45        self.uuid.as_bytes().deep_size_of_children(context)
46            + self.fields.deep_size_of_children(context)
47            + self.name.deep_size_of_children(context)
48            + self.dataset_version.deep_size_of_children(context)
49            + self
50                .fragment_bitmap
51                .as_ref()
52                .map(|fragment_bitmap| fragment_bitmap.serialized_size())
53                .unwrap_or(0)
54    }
55}
56
57impl TryFrom<pb::IndexMetadata> for Index {
58    type Error = Error;
59
60    fn try_from(proto: pb::IndexMetadata) -> Result<Self> {
61        let fragment_bitmap = if proto.fragment_bitmap.is_empty() {
62            None
63        } else {
64            Some(RoaringBitmap::deserialize_from(
65                &mut proto.fragment_bitmap.as_slice(),
66            )?)
67        };
68
69        Ok(Self {
70            uuid: proto.uuid.as_ref().map(Uuid::try_from).ok_or_else(|| {
71                Error::io(
72                    "uuid field does not exist in Index metadata".to_string(),
73                    location!(),
74                )
75            })??,
76            name: proto.name,
77            fields: proto.fields,
78            dataset_version: proto.dataset_version,
79            fragment_bitmap,
80            index_details: proto.index_details,
81            index_version: proto.index_version.unwrap_or_default(),
82        })
83    }
84}
85
86impl From<&Index> for pb::IndexMetadata {
87    fn from(idx: &Index) -> Self {
88        let mut fragment_bitmap = Vec::new();
89        if let Some(bitmap) = &idx.fragment_bitmap {
90            if let Err(e) = bitmap.serialize_into(&mut fragment_bitmap) {
91                // In theory, this should never error. But if we do, just
92                // recover gracefully.
93                log::error!("Failed to serialize fragment bitmap: {}", e);
94                fragment_bitmap.clear();
95            }
96        }
97
98        Self {
99            uuid: Some((&idx.uuid).into()),
100            name: idx.name.clone(),
101            fields: idx.fields.clone(),
102            dataset_version: idx.dataset_version,
103            fragment_bitmap,
104            index_details: idx.index_details.clone(),
105            index_version: Some(idx.index_version),
106        }
107    }
108}
lance_table/format/index.rs

lance_table/format/
index.rs
