// Quickwit
// Copyright (C) 2021 Quickwit Inc.
//
// Quickwit is offered under the AGPL v3.0 and as commercial software.
// For commercial licensing, contact us at hello@quickwit.io.
//
// AGPL:
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
syntax = "proto3";
package quickwit;
service SearchService {
// Root search API.
// This RPC identifies the set of splits on which the query should run on,
// and dispatch the several calls to `LeafSearch`.
//
// It is also in charge of merging back the results.
rpc RootSearch(SearchRequest) returns (SearchResponse);
// Perform a leaf search on a given set of splits.
//
// It is like a regular search except that:
// - the node should perform the search locally instead of dispatching
// it to other nodes.
// - it should be applied on the given subset of splits
// - Hit content is not fetched, and we instead return so called `PartialHit`.
rpc LeafSearch(LeafSearchRequest) returns (LeafSearchResponse);
/// Fetches the documents contents from the document store.
/// This methods takes `PartialHit`s and returns `Hit`s.
rpc FetchDocs(FetchDocsRequest) returns (FetchDocsResponse);
// Perform a leaf stream on a given set of splits.
rpc LeafSearchStream(LeafSearchStreamRequest) returns (stream LeafSearchStreamResponse);
}
// -- Search -------------------
message SearchRequest {
// Index ID
string index_id = 1;
// Query
string query = 2;
// Fields to search on
repeated string search_fields = 3;
// Time filter
optional int64 start_timestamp = 4;
optional int64 end_timestamp = 5;
// Maximum number of hits to return.
uint64 max_hits = 6;
// First hit to return. Together with max_hits, this parameter
// can be used for pagination.
//
// E.g.
// The results with rank [start_offset..start_offset + max_hits) are returned.
uint64 start_offset = 7;
// deprecated tag field
reserved 8;
// Sort order
optional SortOrder sort_order = 9;
// Sort by fast field. If unset sort by docid
optional string sort_by_field = 10;
// json serialized aggregation_request
optional string aggregation_request = 11;
}
enum SortOrder {
/// Ascending order.
ASC = 0;
/// Descending order.
DESC = 1; //< This will be the default value;
}
message SearchResponse {
// Number of hits matching the query.
uint64 num_hits = 1;
// Matched hits
repeated Hit hits = 2;
// Elapsed time to perform the request. This time is measured
// server-side and expressed in microseconds.
uint64 elapsed_time_micros = 3;
// The searcherrors that occured formatted as string.
repeated string errors = 4;
// Serialized aggregation response
optional string aggregation = 5;
}
message SplitSearchError {
// The searcherror that occured formatted as string.
string error = 1;
// Split id that failed.
string split_id = 2;
// Flag to indicate if the error can be considered a retryable error
bool retryable_error = 3;
}
message LeafSearchRequest {
// Search request. This is a perfect copy of the original search request,
// that was sent to root apart from the start_offset & max_hits params.
SearchRequest search_request = 1;
// Index split ids to apply the query on.
// This ids are resolved from the index_uri defined in the search_request.
repeated SplitIdAndFooterOffsets split_offsets = 4;
// `DocMapper` as json serialized trait.
string doc_mapper = 5;
// Index URI. The index URI defines the location of the storage that contains the
// split files.
string index_uri = 6;
}
message SplitIdAndFooterOffsets {
// Index split id to apply the query on.
// This id is resolved from the index_uri defined in the search_request.
string split_id = 1;
// The offset of the start of footer in the split bundle. The footer contains the file bundle metadata and the hotcache.
uint64 split_footer_start = 2;
// The offset of the end of the footer in split bundle. The footer contains the file bundle metada and the hotcache.
uint64 split_footer_end = 3;
}
/// Hits returned by a FetchDocRequest.
///
/// The json that is joined is the raw tantivy json doc.
/// It is very different from a quickwit json doc.
///
/// For instance:
/// - it may contain a _source and a _dynamic field.
/// - since tantivy has no notion of cardinality,
/// all fields is are arrays.
/// - since tantivy has no notion of object, the object is
/// flattened by concatenating the path to the root.
///
/// See `quickwit_search::convert_leaf_hit`
message LeafHit {
// The actual content of the hit/
string leaf_json = 1;
// The partial hit (ie: the sorting field + the document address)
PartialHit partial_hit = 2;
}
message Hit {
// The actual content of the hit/
string json = 1;
// The partial hit (ie: the sorting field + the document address)
PartialHit partial_hit = 2;
}
// A partial hit, is a hit for which we have not fetch the content yet.
// Instead, it holds a document_uri which is enough information to
// go and fetch the actual document data, by performing a `get_doc(...)`
// request.
message PartialHit {
// Sorting field value. (e.g. timestamp)
//
// Quickwit only computes top-K of this sorting field.
// If the user requested for a bottom-K of a given fast field, then quickwit simply
// emits an decreasing mapping of this fast field.
//
// In case of a tie, quickwit uses the increasing order of
// - the split_id,
// - the segment_ord,
// - the doc id.
uint64 sorting_field_value = 1;
string split_id = 2;
// (segment_ord, doc) form a tantivy DocAddress, which is sufficient to identify a document
// within a split
uint32 segment_ord = 3;
// The DocId identifies a unique document at the scale of a tantivy segment.
uint32 doc_id = 4;
}
message LeafSearchResponse {
// Total number of documents matched by the query.
uint64 num_hits = 1;
// List of the best top-K candidates for the given leaf query.
repeated PartialHit partial_hits = 2;
// The list of splits that failed. LeafSearchResponse can be an aggregation of results, so there may be multiple.
repeated SplitSearchError failed_splits = 3;
// Total number of splits the leaf(s) were in charge of.
// num_attempted_splits = num_successful_splits + num_failed_splits.
uint64 num_attempted_splits = 4;
// json serialized intermediate aggregation_result.
optional string intermediate_aggregation_result = 5;
}
message FetchDocsRequest {
// Request fetching the content of a given list of partial_hits.
repeated PartialHit partial_hits = 1;
// Index ID
string index_id = 2;
// Split footer offsets. They are required for fetch docs to
// fetch the document content in two reads, when the footer is not
// cached.
repeated SplitIdAndFooterOffsets split_offsets = 3;
// Index URI. The index URI defines the location of the storage that contains the
// split files.
string index_uri = 4;
}
message FetchDocsResponse {
// List of complete hits.
repeated LeafHit hits = 1;
}
// -- Stream -------------------
enum OutputFormat {
/// Comma Separated Values format (https://datatracker.ietf.org/doc/html/rfc4180).
/// The delimiter is `,`.
CSV = 0; //< This will be the default value
/// Format data by row in ClickHouse binary format.
/// https://clickhouse.tech/docs/en/interfaces/formats/#rowbinary
CLICK_HOUSE_ROW_BINARY = 1;
}
message SearchStreamRequest {
// Index ID
string index_id = 1;
// Query
string query = 2;
// Fields to search on
repeated string search_fields = 3;
// The time filter is interpreted as a semi-open interval. [start, end)
optional int64 start_timestamp = 4;
optional int64 end_timestamp = 5;
// Name of the fast field to extract
string fast_field = 6;
// The output format
OutputFormat output_format = 7;
reserved 8; // deprecated field: tags
// The field by which we want to partition
optional string partition_by_field = 9;
}
message LeafSearchStreamRequest {
// Stream request. This is a perfect copy of the original stream request,
// that was sent to root.
SearchStreamRequest request = 1;
// Index split ids to apply the query on.
// This ids are resolved from the index_uri defined in the stream request.
repeated SplitIdAndFooterOffsets split_offsets = 2;
// `DocMapper` as json serialized trait.
string doc_mapper = 5;
// Index URI. The index URI defines the location of the storage that contains the
// split files.
string index_uri = 6;
}
message LeafSearchStreamResponse {
// Row of data serialized in bytes.
bytes data = 1;
// Split id.
string split_id = 2;
}