lance_table/
feature_flags.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3
4//! Feature flags
5
6use crate::format::Manifest;
7use lance_core::{Error, Result};
8
9/// Fragments may contain deletion files, which record the tombstones of
10/// soft-deleted rows.
11pub const FLAG_DELETION_FILES: u64 = 1;
12/// Row ids are stable for both moves and updates. Fragments contain an index
13/// mapping row ids to row addresses.
14pub const FLAG_STABLE_ROW_IDS: u64 = 2;
15/// Files are written with the new v2 format (this flag is no longer used)
16pub const FLAG_USE_V2_FORMAT_DEPRECATED: u64 = 4;
17/// Table config is present
18pub const FLAG_TABLE_CONFIG: u64 = 8;
19/// Dataset uses multiple base paths (for shallow clones or multi-base datasets)
20pub const FLAG_BASE_PATHS: u64 = 16;
21/// Disable writing transaction file under _transaction/, this flag is set when we only want to write inline transaction in manifest
22pub const FLAG_DISABLE_TRANSACTION_FILE: u64 = 32;
23/// Fragments contain data overlay files, which supply new values for a subset of
24/// cells without rewriting base data files. A reader that does not understand
25/// overlays must refuse the dataset, since ignoring an overlay would silently
26/// return stale base values.
27///
28/// Data overlay files are not yet a released feature: in release builds this flag
29/// is treated as unknown (so a release reader/writer refuses an overlay dataset)
30/// unless [`ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV`] is set, which lets benchmarks opt in.
31/// Debug builds always understand it so tests exercise the path.
32pub const FLAG_UNSTABLE_DATA_OVERLAY_FILES: u64 = 64;
33/// The first bit that is unknown as a feature flag
34pub const FLAG_UNKNOWN: u64 = 128;
35
36/// Environment variable that opts a release build into reading and writing data
37/// overlay files before the feature is generally released.
38pub const ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV: &str = "LANCE_ENABLE_UNSTABLE_DATA_OVERLAY_FILES";
39
40/// Set the reader and writer feature flags in the manifest based on the contents of the manifest.
41pub fn apply_feature_flags(
42    manifest: &mut Manifest,
43    enable_stable_row_id: bool,
44    disable_transaction_file: bool,
45) -> Result<()> {
46    // Reset flags
47    manifest.reader_feature_flags = 0;
48    manifest.writer_feature_flags = 0;
49
50    let has_deletion_files = manifest
51        .fragments
52        .iter()
53        .any(|frag| frag.deletion_file.is_some());
54    if has_deletion_files {
55        // Both readers and writers need to be able to read deletion files
56        manifest.reader_feature_flags |= FLAG_DELETION_FILES;
57        manifest.writer_feature_flags |= FLAG_DELETION_FILES;
58    }
59
60    // If any fragment has row ids, they must all have row ids.
61    let has_row_ids = manifest
62        .fragments
63        .iter()
64        .any(|frag| frag.row_id_meta.is_some());
65    if has_row_ids || enable_stable_row_id {
66        if !manifest
67            .fragments
68            .iter()
69            .all(|frag| frag.row_id_meta.is_some())
70        {
71            return Err(Error::invalid_input("All fragments must have row ids"));
72        }
73        manifest.reader_feature_flags |= FLAG_STABLE_ROW_IDS;
74        manifest.writer_feature_flags |= FLAG_STABLE_ROW_IDS;
75    }
76
77    // Test whether any table metadata has been set
78    if !manifest.config.is_empty() {
79        manifest.writer_feature_flags |= FLAG_TABLE_CONFIG;
80    }
81
82    // Check if this dataset uses multiple base paths (for shallow clones or multi-base datasets)
83    if !manifest.base_paths.is_empty() {
84        manifest.reader_feature_flags |= FLAG_BASE_PATHS;
85        manifest.writer_feature_flags |= FLAG_BASE_PATHS;
86    }
87
88    // Overlay files change cell values on read, so a reader that ignores them
89    // would return stale base values. Both readers and writers must understand
90    // them.
91    let has_overlays = manifest
92        .fragments
93        .iter()
94        .any(|frag| !frag.overlays.is_empty());
95    if has_overlays {
96        manifest.reader_feature_flags |= FLAG_UNSTABLE_DATA_OVERLAY_FILES;
97        manifest.writer_feature_flags |= FLAG_UNSTABLE_DATA_OVERLAY_FILES;
98    }
99
100    if disable_transaction_file {
101        manifest.writer_feature_flags |= FLAG_DISABLE_TRANSACTION_FILE;
102    }
103    Ok(())
104}
105
106/// Whether this build understands data overlay files: always in debug builds,
107/// and in release builds only when [`ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV`] is set.
108fn data_overlay_files_enabled() -> bool {
109    cfg!(debug_assertions) || std::env::var_os(ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV).is_some()
110}
111
112/// Clear `flag` from `flags` when its gating feature is not enabled in this
113/// build; leave it set otherwise. One call per unstable flag, so support for
114/// several unstable features chains cleanly.
115fn mark_supported(flags: &mut u64, flag: u64, feature_enabled: bool) {
116    if !feature_enabled {
117        *flags &= !flag;
118    }
119}
120
121/// The feature-flag bits this build understands, given whether overlay support
122/// is enabled. Split out from [`supported_flags`] so the policy is testable
123/// without toggling the build profile or environment.
124fn supported_flags_when(overlay_enabled: bool) -> u64 {
125    let mut supported = FLAG_UNKNOWN - 1;
126    mark_supported(
127        &mut supported,
128        FLAG_UNSTABLE_DATA_OVERLAY_FILES,
129        overlay_enabled,
130    );
131    supported
132}
133
134fn supported_flags() -> u64 {
135    supported_flags_when(data_overlay_files_enabled())
136}
137
138pub fn can_read_dataset(reader_flags: u64) -> bool {
139    reader_flags & !supported_flags() == 0
140}
141
142pub fn can_write_dataset(writer_flags: u64) -> bool {
143    writer_flags & !supported_flags() == 0
144}
145
146pub fn has_deprecated_v2_feature_flag(writer_flags: u64) -> bool {
147    writer_flags & FLAG_USE_V2_FORMAT_DEPRECATED != 0
148}
149
150#[cfg(test)]
151mod tests {
152    use super::*;
153    use crate::format::BasePath;
154
155    #[test]
156    fn test_read_check() {
157        assert!(can_read_dataset(0));
158        assert!(can_read_dataset(super::FLAG_DELETION_FILES));
159        assert!(can_read_dataset(super::FLAG_STABLE_ROW_IDS));
160        assert!(can_read_dataset(super::FLAG_USE_V2_FORMAT_DEPRECATED));
161        assert!(can_read_dataset(super::FLAG_TABLE_CONFIG));
162        assert!(can_read_dataset(super::FLAG_BASE_PATHS));
163        assert!(can_read_dataset(super::FLAG_DISABLE_TRANSACTION_FILE));
164        // Overlay support is gated on the build profile / env opt-in, so the
165        // flag is readable exactly when overlays are enabled (see
166        // test_data_overlay_flag_release_gating for the full policy).
167        assert_eq!(
168            can_read_dataset(super::FLAG_UNSTABLE_DATA_OVERLAY_FILES),
169            data_overlay_files_enabled()
170        );
171        assert!(can_read_dataset(
172            super::FLAG_DELETION_FILES
173                | super::FLAG_STABLE_ROW_IDS
174                | super::FLAG_USE_V2_FORMAT_DEPRECATED
175        ));
176        assert!(!can_read_dataset(super::FLAG_UNKNOWN));
177    }
178
179    #[test]
180    fn test_data_overlay_flag_release_gating() {
181        // Release default (overlays disabled): the overlay flag is treated as
182        // unknown so the dataset is refused, while other known flags still pass.
183        let supported = supported_flags_when(false);
184        assert_eq!(supported & FLAG_UNSTABLE_DATA_OVERLAY_FILES, 0);
185        assert_eq!(FLAG_DELETION_FILES & !supported, 0);
186        assert_ne!(FLAG_UNSTABLE_DATA_OVERLAY_FILES & !supported, 0);
187        // Enabled (debug or env opt-in): the overlay flag is understood.
188        let supported = supported_flags_when(true);
189        assert_eq!(FLAG_UNSTABLE_DATA_OVERLAY_FILES & !supported, 0);
190    }
191
192    #[test]
193    fn test_apply_feature_flags_sets_overlay_flag() {
194        use crate::format::overlay::{DataOverlayFile, OverlayCoverage};
195        use crate::format::{DataFile, DataStorageFormat, Fragment};
196        use arrow_schema::{Field as ArrowField, Schema as ArrowSchema};
197        use lance_core::datatypes::Schema;
198        use roaring::RoaringBitmap;
199        use std::collections::HashMap;
200        use std::sync::Arc;
201
202        let arrow_schema = ArrowSchema::new(vec![ArrowField::new(
203            "id",
204            arrow_schema::DataType::Int64,
205            false,
206        )]);
207        let schema = Schema::try_from(&arrow_schema).unwrap();
208        let mut fragment = Fragment::new(0);
209        fragment.overlays = vec![DataOverlayFile {
210            data_file: DataFile::new_legacy_from_fields("o.lance", vec![0], None),
211            coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
212            committed_version: 1,
213        }];
214        let mut manifest = Manifest::new(
215            schema,
216            Arc::new(vec![fragment]),
217            DataStorageFormat::default(),
218            HashMap::new(),
219        );
220        apply_feature_flags(&mut manifest, false, false).unwrap();
221        assert_ne!(
222            manifest.reader_feature_flags & FLAG_UNSTABLE_DATA_OVERLAY_FILES,
223            0
224        );
225        assert_ne!(
226            manifest.writer_feature_flags & FLAG_UNSTABLE_DATA_OVERLAY_FILES,
227            0
228        );
229    }
230
231    #[test]
232    fn test_write_check() {
233        assert!(can_write_dataset(0));
234        assert!(can_write_dataset(super::FLAG_DELETION_FILES));
235        assert!(can_write_dataset(super::FLAG_STABLE_ROW_IDS));
236        assert!(can_write_dataset(super::FLAG_USE_V2_FORMAT_DEPRECATED));
237        assert!(can_write_dataset(super::FLAG_TABLE_CONFIG));
238        assert!(can_write_dataset(super::FLAG_BASE_PATHS));
239        assert!(can_write_dataset(super::FLAG_DISABLE_TRANSACTION_FILE));
240        // Overlay support is gated on the build profile / env opt-in, so the
241        // flag is writable exactly when overlays are enabled (see
242        // test_data_overlay_flag_release_gating for the full policy).
243        assert_eq!(
244            can_write_dataset(super::FLAG_UNSTABLE_DATA_OVERLAY_FILES),
245            data_overlay_files_enabled()
246        );
247        assert!(can_write_dataset(
248            super::FLAG_DELETION_FILES
249                | super::FLAG_STABLE_ROW_IDS
250                | super::FLAG_USE_V2_FORMAT_DEPRECATED
251                | super::FLAG_TABLE_CONFIG
252                | super::FLAG_BASE_PATHS
253        ));
254        assert!(!can_write_dataset(super::FLAG_UNKNOWN));
255    }
256
257    #[test]
258    fn test_base_paths_feature_flags() {
259        use crate::format::{DataStorageFormat, Manifest};
260        use arrow_schema::{Field as ArrowField, Schema as ArrowSchema};
261        use lance_core::datatypes::Schema;
262        use std::collections::HashMap;
263        use std::sync::Arc;
264        // Create a basic schema for testing
265        let arrow_schema = ArrowSchema::new(vec![ArrowField::new(
266            "test_field",
267            arrow_schema::DataType::Int64,
268            false,
269        )]);
270        let schema = Schema::try_from(&arrow_schema).unwrap();
271        // Test 1: Normal dataset (no base_paths) should not have FLAG_BASE_PATHS
272        let mut normal_manifest = Manifest::new(
273            schema.clone(),
274            Arc::new(vec![]),
275            DataStorageFormat::default(),
276            HashMap::new(), // Empty base_paths
277        );
278        apply_feature_flags(&mut normal_manifest, false, false).unwrap();
279        assert_eq!(normal_manifest.reader_feature_flags & FLAG_BASE_PATHS, 0);
280        assert_eq!(normal_manifest.writer_feature_flags & FLAG_BASE_PATHS, 0);
281        // Test 2: Dataset with base_paths (shallow clone or multi-base) should have FLAG_BASE_PATHS
282        let mut base_paths: HashMap<u32, BasePath> = HashMap::new();
283        base_paths.insert(
284            1,
285            BasePath::new(
286                1,
287                "file:///path/to/original".to_string(),
288                Some("test_ref".to_string()),
289                true,
290            ),
291        );
292        let mut multi_base_manifest = Manifest::new(
293            schema,
294            Arc::new(vec![]),
295            DataStorageFormat::default(),
296            base_paths,
297        );
298        apply_feature_flags(&mut multi_base_manifest, false, false).unwrap();
299        assert_ne!(
300            multi_base_manifest.reader_feature_flags & FLAG_BASE_PATHS,
301            0
302        );
303        assert_ne!(
304            multi_base_manifest.writer_feature_flags & FLAG_BASE_PATHS,
305            0
306        );
307    }
308}
lance_table/feature_flags.rs

lance_table/
feature_flags.rs
