// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Lance Authors
/* Protobuf headers for serialized index cache entries.
*
* These messages describe the *cache* serialization format, not the on-disk
* Lance format spec, so they live with the library (lance-index) rather than in
* the top-level `protos/` spec folder.
*
* Field numbers and enum values are append-only across all messages here: never
* renumber or reuse them. A change the proto cannot express transparently
* (adding/removing/reordering the IPC/raw sections that follow a header) must
* bump the relevant codec's `CURRENT_VERSION` instead.
*/
syntax = "proto3";
package lance.index.cache;
/* ---------------------------------------------------------------------------
* Full-text search (FTS) posting lists
* ---------------------------------------------------------------------------
*/
// Header for a serialized `CompressedPostingList` cache entry.
message CompressedPostingHeader {
float max_score = 1;
uint32 length = 2;
PostingTailCodec posting_tail_codec = 3;
PositionStorage position_storage = 4;
// Only meaningful when position_storage == POSITION_STORAGE_SHARED.
PositionStreamCodec position_stream_codec = 5;
/* Number of documents in each compressed posting block. Older cache entries
* omit this field and decode as the legacy 128-doc block size.
*/
uint32 block_size = 6;
// Whether an impact IPC section follows the posting/position sections.
bool has_impacts = 7;
}
/* Header for a serialized `PlainPostingList` cache entry. Followed by an Arrow
* IPC section of (row_ids: UInt64, frequencies: Float32), then — when
* position_storage == POSITION_STORAGE_LEGACY — an IPC section of the per-doc
* position list. Plain postings never carry a shared position stream.
*/
message PlainPostingHeader {
/* Absent when the posting has no precomputed block-max score (the in-memory
* `max_score` is `None`); present otherwise.
*/
optional float max_score = 1;
// POSITION_STORAGE_NONE or POSITION_STORAGE_LEGACY only.
PositionStorage position_storage = 2;
}
/* Header for a serialized standalone `Positions` cache entry. Followed by the
* position sections framed per `position_storage`, which is never
* POSITION_STORAGE_NONE for a standalone entry.
*/
message PositionsHeader {
PositionStorage position_storage = 1;
// Only meaningful when position_storage == POSITION_STORAGE_SHARED.
PositionStreamCodec position_stream_codec = 2;
}
/* Header for a serialized `PostingListGroup`: a member count followed by that
* many `PostingList` bodies written inline. Each member body is
* self-delimiting, so members need no length prefixes, and writing them inline
* keeps their Arrow IPC sections 64-byte aligned within the group entry.
*/
message PostingListGroupHeader {
uint32 count = 1;
}
// Tail-block encoding of a compressed posting list.
enum PostingTailCodec {
POSTING_TAIL_CODEC_FIXED32 = 0;
POSTING_TAIL_CODEC_VARINT_DELTA = 1;
}
// Encoding of a shared position stream's byte buffer.
enum PositionStreamCodec {
POSITION_STREAM_CODEC_VARINT_DOC_DELTA = 0;
POSITION_STREAM_CODEC_PACKED_DELTA = 1;
}
/* Which (if any) positions accompany the posting list, and how they are framed
* in the sections after the header.
*/
enum PositionStorage {
POSITION_STORAGE_NONE = 0;
// Legacy per-doc positions as a single Arrow IPC section.
POSITION_STORAGE_LEGACY = 1;
/* Shared stream: an Arrow IPC section of block offsets, then a raw blob of
* the (codec-encoded) position bytes.
*/
POSITION_STORAGE_SHARED = 2;
}
/* ---------------------------------------------------------------------------
* Scalar indices
* ---------------------------------------------------------------------------
*/
/* Header for a serialized `BTreeIndexState` cache entry, followed by a single
* Arrow IPC section holding the page-lookup batch.
*/
message BTreeIndexHeader {
uint64 batch_size = 1;
/* Whether an explicit page-range -> file mapping is present. Distinguishes a
* non-range-partitioned index (false) from a range-partitioned one whose map
* happens to be empty (true with no entries).
*/
bool has_ranges_to_files = 2;
repeated RangeToFile ranges_to_files = 3;
}
/* One entry of a `BTreeIndexState` page-range -> file mapping. The range is
* inclusive on both ends (a `RangeInclusive<u32>`).
*/
message RangeToFile {
uint32 start = 1;
uint32 end = 2;
uint32 page_offset = 3;
string path = 4;
}
/* ---------------------------------------------------------------------------
* Vector indices (IVF partitions)
* ---------------------------------------------------------------------------
*/
/* Headers for serialized IVF partition cache entries (`PartitionEntry<S, Q>`).
*
* Each header is followed by 64-byte-aligned Arrow IPC sections in a fixed,
* version-keyed order (sub-index, then any quantizer-specific arrays, then the
* quantizer storage batches).
*/
// Distance metric a quantizer's storage was built for.
enum DistanceType {
DISTANCE_TYPE_L2 = 0;
DISTANCE_TYPE_COSINE = 1;
DISTANCE_TYPE_DOT = 2;
DISTANCE_TYPE_HAMMING = 3;
}
// Rotation applied by a RabitQ quantizer.
enum RotationType {
ROTATION_TYPE_MATRIX = 0;
ROTATION_TYPE_FAST = 1;
}
// Estimator a RabitQ quantizer uses at query time.
enum RabitQueryEstimator {
RABIT_QUERY_ESTIMATOR_RESIDUAL_QUERY = 0;
RABIT_QUERY_ESTIMATOR_RAW_QUERY = 1;
}
// Product quantizer. Sections: sub-index IPC, codebook IPC, storage IPC.
message PqPartitionHeader {
DistanceType distance_type = 1;
uint32 nbits = 2;
uint64 num_sub_vectors = 3;
uint64 dimension = 4;
bool transposed = 5;
}
// Flat (float) and flat-binary quantizers. Sections: sub-index IPC, storage IPC.
message FlatPartitionHeader {
DistanceType distance_type = 1;
uint64 dim = 2;
}
// Scalar quantizer. Sections: sub-index IPC, storage IPC (possibly multi-batch).
message SqPartitionHeader {
DistanceType distance_type = 1;
uint32 num_bits = 2;
uint64 dim = 3;
double bounds_start = 4;
double bounds_end = 5;
}
/* Header for a serialized IVF index state (`IvfIndexState<Q>`), followed by
* three raw blobs: the IVF model protobuf, the quantizer's extra-metadata
* buffer (may be empty), and the auxiliary IVF model protobuf.
*/
message IvfStateHeader {
reserved 8;
reserved "cache_key_prefix";
string index_file_path = 1;
string uuid = 2;
string distance_type = 3;
repeated string sub_index_metadata = 4;
string sub_index_type = 5;
string quantization_type = 6;
/* Per-quantizer `Q::Metadata` as JSON. Kept as a string because the metadata
* type is generic over the quantizer; the proto envelope still provides
* additive evolution for the surrounding fields.
*/
string quantizer_metadata_json = 7;
uint64 index_file_size = 9;
uint64 aux_file_size = 10;
}
/* RabitQ quantizer. Sections: sub-index IPC, rotate-matrix IPC (Matrix rotation
* only), storage IPC.
*/
message RabitPartitionHeader {
DistanceType distance_type = 1;
uint32 num_bits = 2;
uint32 code_dim = 3;
RotationType rotation_type = 4;
/* Fast-rotation sign vector; present only when rotation_type ==
* ROTATION_TYPE_FAST (the Matrix case stores its rotation as an IPC section).
*/
optional bytes fast_rotation_signs = 5;
// Estimator the RabitQ storage uses at query time (residual vs raw query).
RabitQueryEstimator query_estimator = 6;
}