lance_encoding/
constants.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3
4//! Constants for Lance encoding metadata keys
5//!
6//! These constants define the metadata keys used in Arrow field metadata
7//! to configure various encoding behaviors in Lance.
8
9// Compression-related metadata keys
10/// Metadata key for specifying compression scheme (e.g., "lz4", "zstd", "none")
11pub const COMPRESSION_META_KEY: &str = "lance-encoding:compression";
12/// Metadata key for specifying compression level (applies to schemes that support levels)
13pub const COMPRESSION_LEVEL_META_KEY: &str = "lance-encoding:compression-level";
14/// Metadata key for specifying RLE (Run-Length Encoding) threshold
15pub const RLE_THRESHOLD_META_KEY: &str = "lance-encoding:rle-threshold";
16/// Metadata key for specifying minichunk size
17pub const MINICHUNK_SIZE_META_KEY: &str = "lance-encoding:minichunk-size";
18
19// Dictionary encoding metadata keys
20/// Metadata key for specifying dictionary encoding threshold divisor
21/// Set to a large value to discourage dictionary encoding
22/// Set to a small value to encourage dictionary encoding
23pub const DICT_DIVISOR_META_KEY: &str = "lance-encoding:dict-divisor";
24/// Metadata key for dictionary encoding size ratio threshold (0.0-1.0]
25/// If estimated_dict_size/raw_size < ratio, use dictionary encoding.
26/// Example: 0.8 means use dict if encoded size < 80% of raw size
27/// Default: 0.8
28pub const DICT_SIZE_RATIO_META_KEY: &str = "lance-encoding:dict-size-ratio";
29/// Metadata key for the largest dictionary, in bytes of variable-width values, a page
30/// may be encoded with. A page's dictionary is read whole the first time the page is
31/// touched, so a dictionary of large values (images, documents) turns every point read
32/// into a read of the whole dictionary even when it shrinks the page.
33/// Default: 8 MiB
34pub const DICT_MAX_SIZE_META_KEY: &str = "lance-encoding:dict-max-size";
35/// Metadata key for selecting general compression scheme for dictionary values
36/// Valid values: "lz4", "zstd", "none"
37pub const DICT_VALUES_COMPRESSION_META_KEY: &str = "lance-encoding:dict-values-compression";
38/// Metadata key for selecting compression level for dictionary values
39/// Applies to schemes that support levels (e.g. zstd)
40pub const DICT_VALUES_COMPRESSION_LEVEL_META_KEY: &str =
41    "lance-encoding:dict-values-compression-level";
42
43/// Environment variable for selecting general compression scheme for dictionary values
44pub const DICT_VALUES_COMPRESSION_ENV_VAR: &str = "LANCE_ENCODING_DICT_VALUES_COMPRESSION";
45/// Environment variable for selecting compression level for dictionary values
46pub const DICT_VALUES_COMPRESSION_LEVEL_ENV_VAR: &str =
47    "LANCE_ENCODING_DICT_VALUES_COMPRESSION_LEVEL";
48
49// NOTE: BLOB_META_KEY is defined in lance-core to avoid circular dependency
50
51// Packed struct encoding metadata keys
52/// Legacy metadata key for packed struct encoding (deprecated)
53pub const PACKED_STRUCT_LEGACY_META_KEY: &str = "packed";
54/// Metadata key for packed struct encoding
55pub const PACKED_STRUCT_META_KEY: &str = "lance-encoding:packed";
56
57// Structural encoding metadata keys
58/// Metadata key for specifying structural encoding type
59pub const STRUCTURAL_ENCODING_META_KEY: &str = "lance-encoding:structural-encoding";
60/// Value for miniblock structural encoding
61pub const STRUCTURAL_ENCODING_MINIBLOCK: &str = "miniblock";
62/// Value for fullzip structural encoding
63pub const STRUCTURAL_ENCODING_FULLZIP: &str = "fullzip";
64/// Value for sparse structural encoding
65pub const STRUCTURAL_ENCODING_SPARSE: &str = "sparse";
66
67// Byte stream split metadata keys
68/// Metadata key for byte stream split encoding configuration
69pub const BSS_META_KEY: &str = "lance-encoding:bss";
70/// Default BSS mode
71pub const DEFAULT_BSS_MODE: &str = "auto";
lance_encoding/constants.rs

lance_encoding/
constants.rs
