lance_table/
feature_flags.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3
4//! Feature flags
5
6use snafu::location;
7
8use crate::format::Manifest;
9use lance_core::{Error, Result};
10
11/// Fragments may contain deletion files, which record the tombstones of
12/// soft-deleted rows.
13pub const FLAG_DELETION_FILES: u64 = 1;
14/// Row ids are stable after moves, but not updates. Fragments contain an index
15/// mapping row ids to row addresses.
16pub const FLAG_MOVE_STABLE_ROW_IDS: u64 = 2;
17/// Files are written with the new v2 format (this flag is no longer used)
18pub const FLAG_USE_V2_FORMAT_DEPRECATED: u64 = 4;
19/// Table config is present
20pub const FLAG_TABLE_CONFIG: u64 = 8;
21/// The first bit that is unknown as a feature flag
22pub const FLAG_UNKNOWN: u64 = 16;
23
24/// Set the reader and writer feature flags in the manifest based on the contents of the manifest.
25pub fn apply_feature_flags(manifest: &mut Manifest, enable_stable_row_id: bool) -> Result<()> {
26    // Reset flags
27    manifest.reader_feature_flags = 0;
28    manifest.writer_feature_flags = 0;
29
30    let has_deletion_files = manifest
31        .fragments
32        .iter()
33        .any(|frag| frag.deletion_file.is_some());
34    if has_deletion_files {
35        // Both readers and writers need to be able to read deletion files
36        manifest.reader_feature_flags |= FLAG_DELETION_FILES;
37        manifest.writer_feature_flags |= FLAG_DELETION_FILES;
38    }
39
40    // If any fragment has row ids, they must all have row ids.
41    let has_row_ids = manifest
42        .fragments
43        .iter()
44        .any(|frag| frag.row_id_meta.is_some());
45    if has_row_ids || enable_stable_row_id {
46        if !manifest
47            .fragments
48            .iter()
49            .all(|frag| frag.row_id_meta.is_some())
50        {
51            return Err(Error::invalid_input(
52                "All fragments must have row ids",
53                location!(),
54            ));
55        }
56        manifest.reader_feature_flags |= FLAG_MOVE_STABLE_ROW_IDS;
57        manifest.writer_feature_flags |= FLAG_MOVE_STABLE_ROW_IDS;
58    }
59
60    // Test whether any table metadata has been set
61    if !manifest.config.is_empty() {
62        manifest.writer_feature_flags |= FLAG_TABLE_CONFIG;
63    }
64
65    Ok(())
66}
67
68pub fn can_read_dataset(reader_flags: u64) -> bool {
69    reader_flags < FLAG_UNKNOWN
70}
71
72pub fn can_write_dataset(writer_flags: u64) -> bool {
73    writer_flags < FLAG_UNKNOWN
74}
75
76pub fn has_deprecated_v2_feature_flag(writer_flags: u64) -> bool {
77    writer_flags & FLAG_USE_V2_FORMAT_DEPRECATED != 0
78}
79
80#[cfg(test)]
81mod tests {
82    use super::*;
83
84    #[test]
85    fn test_read_check() {
86        assert!(can_read_dataset(0));
87        assert!(can_read_dataset(super::FLAG_DELETION_FILES));
88        assert!(can_read_dataset(super::FLAG_MOVE_STABLE_ROW_IDS));
89        assert!(can_read_dataset(super::FLAG_USE_V2_FORMAT_DEPRECATED));
90        assert!(can_read_dataset(
91            super::FLAG_DELETION_FILES
92                | super::FLAG_MOVE_STABLE_ROW_IDS
93                | super::FLAG_USE_V2_FORMAT_DEPRECATED
94        ));
95        assert!(!can_read_dataset(super::FLAG_UNKNOWN));
96    }
97
98    #[test]
99    fn test_write_check() {
100        assert!(can_write_dataset(0));
101        assert!(can_write_dataset(super::FLAG_DELETION_FILES));
102        assert!(can_write_dataset(super::FLAG_MOVE_STABLE_ROW_IDS));
103        assert!(can_write_dataset(super::FLAG_USE_V2_FORMAT_DEPRECATED));
104        assert!(can_write_dataset(super::FLAG_TABLE_CONFIG));
105        assert!(can_write_dataset(
106            super::FLAG_DELETION_FILES
107                | super::FLAG_MOVE_STABLE_ROW_IDS
108                | super::FLAG_USE_V2_FORMAT_DEPRECATED
109                | super::FLAG_TABLE_CONFIG
110        ));
111        assert!(!can_write_dataset(super::FLAG_UNKNOWN));
112    }
113}
lance_table/feature_flags.rs

lance_table/
feature_flags.rs
