// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Lance Authors
syntax = "proto3";
package lance.table;
import "google/protobuf/any.proto";
import "google/protobuf/timestamp.proto";
import "file.proto";
import public "fragment_metadata.proto";
/*
Format:
+----------------------------------------+
| Encoded Column 0, Chunk 0 |
...
| Encoded Column M, Chunk N - 1 |
| Encoded Column M, Chunk N |
| Indices ... |
| Chunk Position (M x N x 8) |
| Manifest (Optional) |
| Metadata |
| i64: metadata position |
| MAJOR_VERSION | MINOR_VERSION | "LANC" |
+----------------------------------------+
*/
// UUID type. encoded as 16 bytes.
message UUID {
bytes uuid = 1;
}
// Manifest is a global section shared between all the files.
message Manifest {
// All fields of the dataset, including the nested fields.
repeated lance.file.Field fields = 1;
// Schema metadata.
map<string, bytes> schema_metadata = 5;
// Fragments of the dataset.
repeated DataFragment fragments = 2;
/* Tree tables set this, leave fragments empty, and set bit 12 on both flag
* fields. Old readers skip unknown fields and would otherwise see an empty table.
*/
FragmentTreeMetadata fragment_tree = 22;
// Snapshot version number.
uint64 version = 3;
/* The file position of the version auxiliary data.
* * It is not inheritable between versions.
* * It is not loaded by default during query.
*/
uint64 version_aux_data = 4;
message WriterVersion {
// The name of the library that created this file.
string library = 1;
/* The version of the library that created this file. Because we cannot assume
* that the library is semantically versioned, this is a string. However, if it
* is semantically versioned, it should be a valid semver string without any 'v'
* prefix. For example: `2.0.0`, `2.0.0-rc.1`.
*
* For forward compatibility with older readers, when writing new manifests this
* field should contain only the core version (major.minor.patch) without any
* prerelease or build metadata. The prerelease/build info should be stored in
* the separate prerelease and build_metadata fields instead.
*/
string version = 2;
/* Optional semver prerelease identifier.
*
* This field stores the prerelease portion of a semantic version separately
* from the core version number. For example, if the full version is "2.0.0-rc.1",
* the version field would contain "2.0.0" and prerelease would contain "rc.1".
*
* This separation ensures forward compatibility: older readers can parse the
* clean version field without errors, while newer readers can reconstruct the
* full semantic version by combining version, prerelease, and build_metadata.
*
* If absent, the version field is used as-is.
*/
optional string prerelease = 3;
/* Optional semver build metadata.
*
* This field stores the build metadata portion of a semantic version separately
* from the core version number. For example, if the full version is
* "2.0.0-rc.1+build.123", the version field would contain "2.0.0", prerelease
* would contain "rc.1", and build_metadata would contain "build.123".
*
* If absent, no build metadata is present.
*/
optional string build_metadata = 4;
}
/* The version of the writer that created this file.
*
* This information may be used to detect whether the file may have known bugs
* associated with that writer.
*/
WriterVersion writer_version = 13;
// If present, the file position of the index metadata.
optional uint64 index_section = 6;
// Version creation Timestamp, UTC timezone
google.protobuf.Timestamp timestamp = 7;
// Optional version tag
string tag = 8;
/* Feature flags for readers.
*
* A bitmap of flags that indicate which features are required to be able to
* read the table. If a reader does not recognize a flag that is set, it
* should not attempt to read the dataset.
*
* Known flags:
* * 1 << 0: deletion files are present
* * 1 << 1: row ids are stable and stored as part of the fragment metadata.
* * 1 << 2: use v2 format (deprecated)
* * 1 << 3: table config is present
* * 1 << 4: dataset uses multiple base paths
* * 1 << 5: transaction file writes are disabled
* * 1 << 6: data overlay files are present (see DataOverlayFile). Readers that do
* not understand overlays must refuse the dataset, since ignoring an overlay
* would silently return stale base values.
* * 1 << 7: some index declares covering columns. Without 1 << 13,
* the carried columns form the trailing suffix of IndexMetadata.fields in
* emission order (see IndexMetadata). Readers
* that do not understand covering columns must refuse the dataset, since
* selecting an index by membership of fields would answer a query on a
* merely-carried column with an index keyed on a different column. Writers
* must refuse it too: one that treats every entry of fields as keyed would
* maintain the index against the wrong dependency set.
* * 1 << 8: the dataset may reference recognized V2 data files with different
* exact versions. Readers and writers must use each DataFile's version
* instead of treating data_format.version as a snapshot-wide identity.
* This bit is paired in the reader and writer feature words and is one-way.
* * 1 << 9: the table uses stable row ids and carries a fragment reuse index.
* * 1 << 12: fragment records live in fragment_tree. The fragments list
* must be empty. Readers that do not know this bit must refuse the dataset.
* * 1 << 13: IndexMetadata.fields and covering_fields are independent
* declarations. This bit requires 1 << 7 in both the reader and writer
* flags, and the two bits are retained together: whenever this bit is set,
* 1 << 7 is also set, including manifests with no covering entries and
* manifests that drop the last covering index. Fields contains only the columns the index is keyed on, while
* covering_fields contains the columns whose values it carries. A field may
* occur in both lists. Implementations that only support the legacy suffix
* contract must refuse the dataset. Setting this bit reinterprets every
* IndexMetadata in the index section, so a writer must normalize every
* legacy-form entry in the same manifest -- or set the bit only when no
* entry declares covering fields -- and manifests derived from it must keep
* the bit set until a reverse conversion is likewise atomic.
*/
uint64 reader_feature_flags = 9;
/* Feature flags for writers.
*
* A bitmap of flags that indicate which features must be used when writing to the
* dataset. If a writer does not recognize a flag that is set, it should not attempt to
* write to the dataset.
*
* The flag identities are the same as for reader_feature_flags, but the values of
* reader_feature_flags and writer_feature_flags are not required to be identical.
*/
uint64 writer_feature_flags = 10;
/* Highest fragment id ever allocated or reserved in this lineage.
* Absent only before the first allocation or reservation. Never decreases.
* u32::MAX means the fragment-id space is exhausted.
*/
optional uint32 max_fragment_id = 11;
/* Path to the transaction file, relative to `{root}/_transactions`. The file at that
* location contains a wire-format serialized Transaction message representing the
* transaction that created this version.
*
* This string field "transaction_file" may be empty if no transaction file was written.
*
* The path format is "{read_version}-{uuid}.txn" where {read_version} is the version of
* the table the transaction read from (serialized to decimal with no padding digits),
* and {uuid} is a hyphen-separated UUID.
*/
string transaction_file = 12;
/* The file position of the transaction content. None if transaction is empty
* This transaction content begins with the transaction content length as u32
* If the transaction proto message has a length of `len`, the message ends at `len` + 4
*/
optional uint64 transaction_section = 21;
/* The next unused row id. If zero, then the table does not have any rows.
*
* This is only used if the "stable_row_ids" feature flag is set.
*/
uint64 next_row_id = 14;
message DataStorageFormat {
// The format of the data files (e.g. "lance")
string file_format = 1;
/* The default format version used when writing data files. Write operations
* that do not select an exact version use this value. This setting does not
* constrain index or other file types. The version syntax is defined by
* file_format and is not required to follow semver.
*
* Unless the mixed data-file-version capability is set, every referenced
* data file must use this version. When the capability is set, referenced
* data files may use other recognized exact V2 versions, and each DataFile's
* version is authoritative for decoding.
*/
string version = 2;
}
/* The data storage format
*
* This specifies what format is used to store the data files.
*/
DataStorageFormat data_format = 15;
/* Table config.
*
* Keys with the prefix "lance." are reserved for the Lance library. Other
* libraries may wish to similarly prefix their configuration keys
* appropriately.
*/
map<string, string> config = 16;
/* Metadata associated with the table.
*
* This is a key-value map that can be used to store arbitrary metadata
* associated with the table.
*
* This is different than configuration, which is used to tell libraries how
* to read, write, or manage the table.
*
* This is different than schema metadata, which is used to describe the
* data itself and is attached to the output schema of scans.
*/
map<string, string> table_metadata = 19;
// Field number 17 (`blob_dataset_version`) was used for a secondary blob dataset.
reserved 17;
reserved "blob_dataset_version";
/* The base paths of data files.
*
* This is used to determine the base path of a data file. In common cases data file paths are under current dataset base path.
* But for shallow cloning, importing file and other multi-tier storage cases, the actual data files could be outside of the current dataset.
* This field is used with the `base_id` in `lance.file.File` and `lance.file.DeletionFile`.
*
* For example, if we have a dataset with base path `s3://bucket/dataset`, we have a DataFile with base_id 0, we get the actual data file path by:
* base_paths[id = 0] + /data/ + file.path
* the key(a.k.a index) starts from 0, increased by 1 for each new base path.
*/
repeated BasePath base_paths = 18;
// The branch of the dataset. None means main branch.
optional string branch = 20;
} // Manifest
// external dataset base path
message BasePath {
uint32 id = 1;
/* This is an alias name of the base path, it is optional.
* When we use shallow clone and the target version is a tag, the tag name will be set here.
*/
optional string name = 2;
/* Flag indicating whether this path is a dataset root path or file directory:
* - true: Path is a dataset root (actual files under subdirectories like `data`, '_deletions')
* - false: Path is a direct file directory (scenario like importing files)
*/
bool is_dataset_root = 3;
// Note: This absolute path will be directly used by Path:parse(),
string path = 4;
}
/* Auxiliary Data attached to a version.
* Only load on-demand.
*/
message VersionAuxData {
// key-value metadata.
map<string, bytes> metadata = 3;
}
// Metadata describing an index.
message IndexMetadata {
// Unique ID of an index. It is unique across all the dataset versions.
UUID uuid = 1;
/* The columns the index is keyed on, in key order. These refer to file.Field.id.
*
* When the manifest has feature flag 1 << 13, this list contains only key
* fields. `covering_fields` independently declares the values the index
* carries.
*
* Without feature flag 1 << 13, the legacy contract applies: the carried fields
* form the trailing suffix of this list in emission order.
*/
repeated int32 fields = 2;
// Index name. Must be unique within one dataset version.
string name = 3;
// The version of the dataset this index was built from.
uint64 dataset_version = 4;
/* A bitmap of the included fragment ids.
*
* This may by used to determine how much of the dataset is covered by the
* index. This information can be retrieved from the dataset by looking at
* the dataset at `dataset_version`. However, since the old version may be
* deleted while the index is still in use, this information is also stored
* in the index.
*
* The bitmap is stored as a 32-bit Roaring bitmap.
*/
bytes fragment_bitmap = 5;
/* Details, specific to the index type, which are needed to load / interpret the index
*
* Indices should avoid putting large amounts of information in this field, as it will
* bloat the manifest.
*
* Indexes are plugins, and so the format of the details message is flexible and not fully
* defined by the table format. However, there are some conventions that should be followed:
*
* - When Lance APIs refer to indexes they will use the type URL of the index details as the
* identifier for the index type. If a user provides a simple string identifier like
* "btree" then it will be converted to "/lance.table.BTreeIndexDetails"
* - Type URLs comparisons are case-insensitive. Thereform an index must have a unique type
* URL ignoring case.
*/
google.protobuf.Any index_details = 6;
// The minimum lance version that this index is compatible with.
optional int32 index_version = 7;
/* Timestamp when the index was created (UTC timestamp in milliseconds since epoch)
*
* This field is optional for backward compatibility. For existing indices created before
* this field was added, this will be None/null.
*/
optional uint64 created_at = 8;
/* The base path index of the data file. Used when the file is imported or referred from another dataset.
* Lance use it as key of the base_paths field in Manifest to determine the actual base path of the data file.
*/
optional uint32 base_id = 9;
/* List of files and their sizes for this index segment.
* This enables skipping HEAD calls when opening indices and allows reporting
* of index sizes without extra IO.
* If this is empty, the index files sizes are unknown.
*/
repeated IndexFile files = 10;
/* The fields whose values this index co-locates alongside its own data, so a
* query projecting only those columns can be answered from the index without
* a take against the base table.
*
* When the manifest has feature flag 1 << 13, this declaration is independent
* of `fields` and has no positional relationship to it. A field may occur in
* both lists when the index carries a value it is also keyed on. Consumers
* must use the union of `fields` and `covering_fields` as the index dependency
* set.
*
* Without feature flag 1 << 13, the legacy contract applies: the carried columns
* form the trailing suffix of `fields` in emission order. A column the
* index is both keyed on and carries appears twice, once as a key and once in
* the carried suffix.
*
* Empty for an index that carries no extra columns, which is every index
* written before this field existed.
*
* This declaration is not authoritative for what the segment can actually
* serve. The segment's own storage schema is: a reader must confirm the
* storage carries a column before answering a query from it, and fall back to
* a take against the base table otherwise. A declaration naming columns the
* storage does not hold is a legal state, not corruption -- a maintenance
* operation that cannot carry the values through a rebuild is permitted to
* withdraw the payload while leaving this declaration in place.
*/
repeated int32 covering_fields = 11;
}
// Metadata about a single file within an index segment.
message IndexFile {
// Path relative to the index directory (e.g., "index.idx", "auxiliary.idx")
string path = 1;
// Size of the file in bytes
uint64 size_bytes = 2;
}
// Index Section, containing a list of index metadata for one dataset version.
message IndexSection {
repeated IndexMetadata indices = 1;
}
// VectorIndexDetails and HnswParameters (formerly HnswIndexDetails) moved to index.proto
message FragmentReuseIndexDetails {
oneof content {
// if < 200KB, store the content inline, otherwise store the InlineContent bytes in external file
InlineContent inline = 1;
ExternalFile external = 2;
}
message InlineContent {
// Existing compaction records, with the original field number and contents.
repeated Version legacy_versions = 1;
// New mappings; requires index_version = 1.
repeated Transition transitions = 2;
}
// One transition from source fragments to destination fragments.
message Transition {
// Source fragments in scan order; counts include deleted rows.
repeated FragmentDigest sources = 1;
// Destination fragments in label order; counts are recorded at creation.
repeated FragmentDigest destinations = 2;
/* Exactly one mapping is required.
* New mapping types do not require an index_version increase if they preserve
* backward compatibility and older readers can handle them safely, including
* by falling back to a scan. Otherwise, index_version must increase.
* Other field numbers may be used for ordinary metadata.
*/
oneof mapping {
/* Concatenate surviving rows in source fragment order, preserving row order
* within each fragment, then split into destination fragments.
*/
OrderedCompaction ordered_compaction = 3;
/* Process source fragments sequentially, distributing their rows across
* destination fragments by label in source row order.
*/
StablePartition stable_partition = 4;
}
}
message OrderedCompaction {
// RoaringTreemap of surviving source row addresses; missing rows are deleted.
bytes changed_row_addrs = 1;
}
message StablePartition {
// UUID in _fri/<map_id>/stable_partition.lance; unchanged when the FRI UUID changes.
string map_id = 1;
// Size of stable_partition.lance in bytes.
uint64 map_size_bytes = 2;
// Unset: use the dataset base. Set: look up this ID in Manifest.base_paths.
optional uint32 base_id = 3;
}
message FragmentDigest {
uint64 id = 1;
uint64 physical_rows = 2;
uint64 num_deleted_rows = 3;
}
// A summarized version of the RewriteGroup information in a Rewrite transaction
message Group {
/* A roaring treemap of the changed row addresses.
* When combined with the old fragment IDs and new fragment IDs,
* it can recover the full mapping of old row addresses to either new row addresses or deleted.
* this mapping can then be used to remap indexes or satisfy index queries for the new unindexed fragments.
*/
bytes changed_row_addrs = 1;
repeated FragmentDigest old_fragments = 2;
repeated FragmentDigest new_fragments = 3;
}
message Version {
// The dataset_version at the time the index adds this version entry
uint64 dataset_version = 1;
repeated Group groups = 3;
}
}
/* ============================================================================
* MemWAL Index Types
* ============================================================================
*/
/* Lifecycle status of a WAL shard. Drives drop-table two-phase commit:
* a SEALED shard refuses new writer claims (reversible) until the drop
* commits (the shard dir is deleted) or rolls back (status -> ACTIVE).
*/
enum ShardStatus {
// Normal: the shard accepts writer claims.
ACTIVE = 0;
// A drop is in flight: claims are refused. Reversible to ACTIVE.
SEALED = 1;
}
/* Shard manifest containing epoch-based fencing and WAL state.
* Each shard has exactly one active writer at any time.
*/
message ShardManifest {
// Shard identifier (UUID v4).
UUID shard_id = 11;
/* Manifest version number.
* Matches the version encoded in the filename.
*/
uint64 version = 1;
/* Shard spec ID this shard was created with.
* Set at shard creation and immutable thereafter.
* A value of 0 indicates a manually-created shard not governed by any spec.
*/
uint32 shard_spec_id = 10;
/* Computed shard field values as raw Arrow scalar bytes, keyed by shard
* field id. The byte encoding follows Arrow's little-endian convention:
* int32 is 4 LE bytes, utf8 is raw UTF-8 bytes, etc. The receiver looks
* up the result_type from the ShardingSpec to interpret each value.
*/
repeated ShardFieldEntry shard_field_entries = 14;
/* Writer fencing token - monotonically increasing.
* A writer must increment this when claiming the shard.
*/
uint64 writer_epoch = 2;
/* The most recent WAL entry position that has been flushed to a MemTable.
* During recovery, replay starts from replay_after_wal_entry_position + 1.
* WAL positions are 1-based, so the default value 0 unambiguously means
* "no flush has ever stamped this shard" and recovery replays from 1.
*/
uint64 replay_after_wal_entry_position = 3;
/* The most recent WAL entry position observed at the time the manifest was
* updated. WAL positions are 1-based; default 0 means no entry has been
* written yet. This is a hint, not authoritative - recovery must list
* files to find actual state.
*/
uint64 wal_entry_position_last_seen = 4;
// Generation to assign to the next SSTable (incremented after each MemTable flush).
uint64 current_generation = 6;
/* Field 7 removed: compaction progress lives in
* MemWalIndexDetails.compacted_sstables.
*/
// List of SSTables created by flushing MemTables and their directory paths.
repeated SsTable sstables = 8;
/* Lifecycle status. Default ACTIVE; SEALED marks an in-flight drop
* (drop-table 2PC). A SEALED manifest refuses claims at claim_epoch.
*/
ShardStatus status = 15;
}
// A shard field value stored as raw Arrow scalar bytes.
message ShardFieldEntry {
// Shard field id (matches ShardingField.field_id in the ShardingSpec).
string field_id = 1;
/* Raw Arrow scalar value bytes in little-endian encoding.
* The data type is determined by the result_type of the matching ShardingField.
*/
bytes value = 2;
}
// An SSTable: the immutable result of flushing a MemTable, stored as a Lance dataset.
message SsTable {
// Generation number identifying this SSTable.
uint64 generation = 1;
// Directory name relative to the shard directory.
string path = 2;
/* Payload size of the rows, as the writer's MemTable accounted for them: the
sum over its buffers of the window each one holds.
An estimate of the payload, not a bound on what reading the SSTable costs.
It excludes the per-array structure a reader materializes, so a consumer
budgeting memory from it must add its own headroom.
Absent on an SSTable written before this field existed, so a reader must
not read a missing value as zero. The same applies to the two below. */
optional uint64 in_memory_bytes = 3;
/* Rows held, counting the older duplicates of a primary key that the
generation's deletion vector masks. A scan that applies the deletion vector
yields fewer. */
optional uint64 physical_rows = 4;
/* Total payload size of the primary-key columns, summed over every row in
`physical_rows`. Not a per-row size: a variable-length key differs from row
to row, and a consumer wanting a mean divides by the row count.
Accounted the same way as `in_memory_bytes`, so the same caveat applies --
it is not the size of any encoded form of the key, such as a comparable
byte-ordered encoding a reader might build for sorting or deduplication.
Absent also on a table with no primary key. */
optional uint64 primary_key_bytes = 5;
}
// A pointer to the latest SSTable compacted for a shard.
message CompactedSsTable {
// Shard identifier (UUID v4).
UUID shard_id = 1;
// Generation of the latest SSTable compacted into the base table for this shard.
uint64 generation = 2;
}
/* Tracks which compacted SSTable generation a base table index has been rebuilt to cover.
* Used to determine whether to read from SSTable indexes or base table.
*/
message IndexCatchupProgress {
// Name of the base table index (must match an entry in maintained_indexes).
string index_name = 1;
/* Per-shard progress: the generation up to which this index covers.
*
* An absent shard means *unknown*: this index has recorded no catch-up for
* that shard, so its SSTables must be retained and a repair scheduled.
*/
repeated CompactedSsTable caught_up_generations = 2;
}
/* Index details for MemWAL Index, stored in IndexMetadata.index_details.
* This is the centralized structure for all MemWAL metadata:
* - Configuration (sharding specs, indexes to maintain)
* - SSTable compaction progress
* - Shard state snapshots
*
* Writers read this index to get configuration before writing.
* Readers may use shard snapshots in this index as a point-in-time
* optimization. Readers that need the latest shard set should list shard
* directories in storage and read each shard's latest manifest.
* A background process updates the index periodically to keep shard snapshots current.
*
* Shard snapshots are stored as a Lance file with one row per shard.
* The schema records shard discovery fields. Full mutable shard state remains
* authoritative in the shard manifest files.
* shard_id: utf8
* shard_spec_id: uint32
* shard_field_{field_id}: typed per the matching ShardingField.result_type
*/
message MemWalIndexDetails {
// Snapshot timestamp (Unix timestamp in milliseconds).
int64 snapshot_ts_millis = 1;
/* Number of shards in the snapshot.
* Used to determine storage format without reading the snapshot data.
*/
uint32 num_shards = 2;
/* Inline shard snapshots for small shard counts.
* When num_shards <= threshold (implementation-defined, e.g., 100),
* snapshots are stored inline as serialized bytes.
* Format: Lance file bytes with the shard snapshot schema.
*/
optional bytes inline_snapshots = 3;
/* Sharding specs defining how to derive shard identifiers.
* This configuration determines how rows are partitioned into shards.
*/
repeated ShardingSpec sharding_specs = 7;
/* Indexes from the base table to maintain in MemTables.
* These are index names referencing indexes defined on the base table.
* The primary key btree index is always maintained implicitly and
* should not be listed here.
*
* For vector indexes, MemTables inherit quantization parameters (PQ codebook,
* SQ params) from the base table index to ensure distance comparability.
*/
repeated string maintained_indexes = 8;
/* Latest SSTable compacted into the base table for each shard.
* This is updated atomically with merge-insert data commits, enabling
* conflict resolution when multiple compactors operate concurrently.
*
* Note: This is separate from shard snapshots because:
* 1. compacted_sstables is updated by compactors (atomic with data commit)
* 2. shard snapshots are updated by background index builder
*/
repeated CompactedSsTable compacted_sstables = 9;
/* Per-index catchup progress tracking.
* When data is compacted into the base table, base table indexes are rebuilt
* asynchronously. This field tracks which generation each index covers.
*
* For indexed queries, if an index's caught_up_generation < compacted_generation,
* readers should use SSTable indexes for the gap instead of
* scanning unindexed data in the base table.
*
* An index absent from this list has recorded no catch-up, so the SSTables it
* would need stay live until a repair records it. Only the dedicated WAL
* index-repair path may add entries here;
* ordinary index operations have their entry removed automatically when they
* change an index, since they do not report what the new index covers.
*/
repeated IndexCatchupProgress index_catchup = 10;
/* Default ShardWriter configuration values for this MemWAL index.
*
* A free-form string map persisted so that every writer — across
* processes and restarts — starts from the same default writer
* configuration. These are defaults only: an individual writer may
* still override any value at runtime in its own ShardWriterConfig
* (which is not persisted).
*/
map<string, string> writer_config_defaults = 11;
}
// Sharding spec definition.
message ShardingSpec {
/* Unique identifier for this spec within the index.
* IDs are never reused.
*/
uint32 spec_id = 1;
// Sharding field definitions that determine how to compute shard identifiers.
repeated ShardingField fields = 2;
}
// Sharding field definition.
message ShardingField {
// Unique string identifier for this shard field.
string field_id = 1;
// Field IDs referencing source columns in the schema.
repeated int32 source_ids = 2;
/* Well-known shard transform name (e.g., "identity", "year", "bucket").
* Mutually exclusive with expression.
*/
optional string transform = 3;
/* DataFusion SQL expression for custom logic.
* Mutually exclusive with transform.
*/
optional string expression = 4;
// Output type of the shard value (Arrow type name).
string result_type = 5;
// Transform parameters (e.g., num_buckets for bucket transform).
map<string, string> parameters = 6;
}