lance-encoding 10.0.0

Encoders and decoders for the Lance file format
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Lance Authors

use lance_core::Result;

use crate::repdef::MiniBlockRepDefBudget;

/// Runs automatic sparse planning only after the dense mini-block budget makes it useful.
pub(super) fn select_automatic_sparse<T>(
    requested_encoding: Option<&str>,
    dense_budget: &MiniBlockRepDefBudget,
    candidate: impl FnOnce() -> Result<Option<T>>,
) -> Result<Option<T>> {
    if requested_encoding.is_some()
        || !matches!(
            dense_budget,
            MiniBlockRepDefBudget::RequiresPageSplit(_)
                | MiniBlockRepDefBudget::SingleRowOverBudget(_)
        )
    {
        return Ok(None);
    }
    candidate()
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::constants::{
        STRUCTURAL_ENCODING_FULLZIP, STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_SPARSE,
    };

    #[test]
    fn within_budget_does_not_construct_sparse_candidate() {
        let selected =
            select_automatic_sparse::<()>(None, &MiniBlockRepDefBudget::WithinBudget, || {
                panic!("within-budget pages must not construct sparse candidates")
            })
            .unwrap();
        assert!(selected.is_none());
    }

    #[test]
    fn over_budget_selects_only_eligible_candidates() {
        let split = MiniBlockRepDefBudget::RequiresPageSplit(Vec::new());
        let selected = select_automatic_sparse(None, &split, || Ok(Some(42))).unwrap();
        assert_eq!(selected, Some(42));

        let ineligible = select_automatic_sparse::<()>(None, &split, || Ok(None)).unwrap();
        assert!(ineligible.is_none());

        let unsplittable = MiniBlockRepDefBudget::SingleRowOverBudget(70_000);
        let selected = select_automatic_sparse(None, &unsplittable, || Ok(Some(7))).unwrap();
        assert_eq!(selected, Some(7));
    }

    #[test]
    fn explicit_modes_and_lance_2_2_do_not_auto_select() {
        let split = MiniBlockRepDefBudget::RequiresPageSplit(Vec::new());
        for requested in [
            STRUCTURAL_ENCODING_MINIBLOCK,
            STRUCTURAL_ENCODING_FULLZIP,
            STRUCTURAL_ENCODING_SPARSE,
        ] {
            let selected = select_automatic_sparse::<()>(Some(requested), &split, || {
                panic!("explicit modes must not invoke automatic sparse planning")
            })
            .unwrap();
            assert!(selected.is_none());
        }
    }
}