// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Lance Authors
syntax = "proto3";
package lance.table;
// NOTE: Do *NOT* add new index details here. Add them to the index.proto file instead.
// This file is in the lance.table package namespace while the index.proto file is in the
// lance.index package namespace.
//
// These are only here for forward compatibility. Older versions of Lance expect btree indexes
// to have lance.table in the package namespace.
//
// If you need to modify these messages (e.g. to add new fields to btree or bitmap) then
// it is ok to modify them here.
// Currently many of these are empty messages because all needed details are either hard-coded (e.g.
// filenames) or stored in the index itself. However, we may want to add more details in the
// future, in particular we can add details that may be useful for planning queries (e.g. don't
// force us to load the index until we know we can make use of it)
message BTreeIndexDetails {}
message BitmapIndexDetails {}
message LabelListIndexDetails {}
message NGramIndexDetails {}
message ZoneMapIndexDetails {}
message InvertedIndexDetails {
message CodeTokenizerConfig {
// Split one lexical identifier into subwords, e.g. getUserName ->
// get/user/name.
bool split_identifiers = 1;
// Split identifier subwords across letter/number boundaries, e.g.
// HTML2JSON -> html/2/json. An absent value uses the code tokenizer default;
// a present value records the explicit index-time choice.
optional bool split_on_numerics = 2;
// Keep the complete lexical identifier in addition to subwords, e.g.
// user_name plus user/name. An absent value uses the code tokenizer default;
// a present value records the explicit index-time choice.
optional bool preserve_original = 3;
// Index operator tokens such as "::", "->", and "!=". Operators are not
// indexed by default because they are often high-frequency noise.
bool index_operators = 4;
}
// Lexical tokenizer used after document-level text extraction. This is an
// implementation component such as "simple", "icu", "ngram", or "code".
// Input-time analyzer profiles are expanded into this field and the concrete
// options below before these details are persisted.
// Marking this field as optional as old versions of the index store blank details and we
// need to make sure we have a proper optional field to detect this.
optional string base_tokenizer = 1;
string language = 2;
bool with_position = 3;
optional uint32 max_token_length = 4;
bool lower_case = 5;
bool stem = 6;
bool remove_stop_words = 7;
bool ascii_folding = 8;
uint32 min_ngram_length = 9;
uint32 max_ngram_length = 10;
bool prefix_only = 11;
// Number of documents per compressed posting block. An absent value means
// the index predates this field and must use the legacy block size of 128.
// A present value records the block size used by the index; 256 is valid
// with format versions 3 and 4.
optional uint32 block_size = 12;
// Options for base_tokenizer = "code". Presence records the code tokenizer
// configuration used to build the index; absence means there is no
// code-specific configuration to apply.
CodeTokenizerConfig code_config = 13;
}