syntax = "proto3";
package arc_vector;
import "json_with_int.proto";
import "collections.proto";
enum WriteOrderingType {
Weak = 0; // Write operations may be reordered, works faster, default
Medium = 1; // Write operations go through dynamically selected leader, may be inconsistent for a short period of time in case of leader change
Strong = 2; // Write operations go through the permanent leader, consistent, but may be unavailable if leader is down
}
message WriteOrdering {
WriteOrderingType type = 1; // Write ordering guarantees
}
enum ReadConsistencyType {
All = 0; // Send request to all nodes and return points which are present on all of them
Majority = 1; // Send requests to all nodes and return points which are present on majority of them
Quorum = 2; // Send requests to half + 1 nodes, return points which are present on all of them
}
message ReadConsistency {
oneof value {
ReadConsistencyType type = 1; // Common read consistency configurations
uint64 factor = 2; // Send request to a specified number of nodes, and return points which are present on all of them
}
}
// ---------------------------------------------
// ------------- Point Id Requests -------------
// ---------------------------------------------
message PointId {
oneof point_id_options {
uint64 num = 1; // Numerical ID of the point
string uuid = 2; // UUID
}
}
message Vector {
repeated float data = 1;
}
// ---------------------------------------------
// ---------------- RPC Requests ---------------
// ---------------------------------------------
message UpsertPoints {
string collection_name = 1; // name of the collection
optional bool wait = 2; // Wait until the changes have been applied?
repeated PointStruct points = 3;
optional WriteOrdering ordering = 4; // Write ordering guarantees
}
message DeletePoints {
string collection_name = 1; // name of the collection
optional bool wait = 2; // Wait until the changes have been applied?
PointsSelector points = 3; // Affected points
optional WriteOrdering ordering = 4; // Write ordering guarantees
}
message GetPoints {
string collection_name = 1; // name of the collection
repeated PointId ids = 2; // List of points to retrieve
reserved 3; // deprecated "with_vector" field
WithPayloadSelector with_payload = 4; // Options for specifying which payload to include or not
optional WithVectorsSelector with_vectors = 5; // Options for specifying which vectors to include into response
optional ReadConsistency read_consistency = 6; // Options for specifying read consistency guarantees
}
message UpdatePointVectors {
string collection_name = 1; // name of the collection
optional bool wait = 2; // Wait until the changes have been applied?
repeated PointVectors points = 3; // List of points and vectors to update
optional WriteOrdering ordering = 4; // Write ordering guarantees
}
message PointVectors {
PointId id = 1; // ID to update vectors for
Vectors vectors = 2; // Named vectors to update, leave others intact
}
message DeletePointVectors {
string collection_name = 1; // name of the collection
optional bool wait = 2; // Wait until the changes have been applied?
PointsSelector points_selector = 3; // Affected points
VectorsSelector vectors = 4; // List of vector names to delete
optional WriteOrdering ordering = 5; // Write ordering guarantees
}
message SetPayloadPoints {
string collection_name = 1; // name of the collection
optional bool wait = 2; // Wait until the changes have been applied?
map<string, Value> payload = 3; // New payload values
reserved 4; // List of point to modify, deprecated
optional PointsSelector points_selector = 5; // Affected points
optional WriteOrdering ordering = 6; // Write ordering guarantees
}
message DeletePayloadPoints {
string collection_name = 1; // name of the collection
optional bool wait = 2; // Wait until the changes have been applied?
repeated string keys = 3; // List of keys to delete
reserved 4; // Affected points, deprecated
optional PointsSelector points_selector = 5; // Affected points
optional WriteOrdering ordering = 6; // Write ordering guarantees
}
message ClearPayloadPoints {
string collection_name = 1; // name of the collection
optional bool wait = 2; // Wait until the changes have been applied?
PointsSelector points = 3; // Affected points
optional WriteOrdering ordering = 4; // Write ordering guarantees
}
enum FieldType {
FieldTypeKeyword = 0;
FieldTypeInteger = 1;
FieldTypeFloat = 2;
FieldTypeGeo = 3;
FieldTypeText = 4;
FieldTypeBool = 5;
}
message CreateFieldIndexCollection {
string collection_name = 1; // name of the collection
optional bool wait = 2; // Wait until the changes have been applied?
string field_name = 3; // Field name to index
optional FieldType field_type = 4; // Field type.
optional PayloadIndexParams field_index_params = 5; // Payload index params.
optional WriteOrdering ordering = 6; // Write ordering guarantees
}
message DeleteFieldIndexCollection {
string collection_name = 1; // name of the collection
optional bool wait = 2; // Wait until the changes have been applied?
string field_name = 3; // Field name to delete
optional WriteOrdering ordering = 4; // Write ordering guarantees
}
message PayloadIncludeSelector {
repeated string fields = 1; // List of payload keys to include into result
}
message PayloadExcludeSelector {
repeated string fields = 1; // List of payload keys to exclude from the result
}
message WithPayloadSelector {
oneof selector_options {
bool enable = 1; // If `true` - return all payload, if `false` - none
PayloadIncludeSelector include = 2;
PayloadExcludeSelector exclude = 3;
}
}
message NamedVectors {
map<string, Vector> vectors = 1;
}
message Vectors {
oneof vectors_options {
Vector vector = 1;
NamedVectors vectors = 2;
}
}
message VectorsSelector {
repeated string names = 1; // List of vectors to include into result
}
message WithVectorsSelector {
oneof selector_options {
bool enable = 1; // If `true` - return all vectors, if `false` - none
VectorsSelector include = 2; // List of payload keys to include into result
}
}
message QuantizationSearchParams {
/*
If set to true, search will ignore quantized vector data
*/
optional bool ignore = 1;
/*
If true, use original vectors to re-score top-k results. Default is true.
*/
optional bool rescore = 2;
/*
Oversampling factor for quantization.
Defines how many extra vectors should be pre-selected using quantized index,
and then re-scored using original vectors.
For example, if `oversampling` is 2.4 and `limit` is 100, then 240 vectors will be pre-selected using quantized index,
and then top-100 will be returned after re-scoring.
*/
optional double oversampling = 3;
}
message SearchParams {
/*
Params relevant to HNSW index. Size of the beam in a beam-search.
Larger the value - more accurate the result, more time required for search.
*/
optional uint64 hnsw_ef = 1;
/*
Search without approximation. If set to true, search may run long but with exact results.
*/
optional bool exact = 2;
/*
If set to true, search will ignore quantized vector data
*/
optional QuantizationSearchParams quantization = 3;
}
message SearchPoints {
string collection_name = 1; // name of the collection
repeated float vector = 2; // vector
Filter filter = 3; // Filter conditions - return only those points that satisfy the specified conditions
uint64 limit = 4; // Max number of result
reserved 5; // deprecated "with_vector" field
WithPayloadSelector with_payload = 6; // Options for specifying which payload to include or not
SearchParams params = 7; // Search config
optional float score_threshold = 8; // If provided - cut off results with worse scores
optional uint64 offset = 9; // Offset of the result
optional string vector_name = 10; // Which vector to use for search, if not specified - use default vector
optional WithVectorsSelector with_vectors = 11; // Options for specifying which vectors to include into response
optional ReadConsistency read_consistency = 12; // Options for specifying read consistency guarantees
}
message SearchBatchPoints {
string collection_name = 1; // Name of the collection
repeated SearchPoints search_points = 2;
optional ReadConsistency read_consistency = 3; // Options for specifying read consistency guarantees
}
message WithLookup {
string collection = 1; // Name of the collection to use for points lookup
optional WithPayloadSelector with_payload = 2; // Options for specifying which payload to include (or not)
optional WithVectorsSelector with_vectors = 3; // Options for specifying which vectors to include (or not)
}
message SearchPointGroups {
string collection_name = 1; // Name of the collection
repeated float vector = 2; // Vector to compare against
Filter filter = 3; // Filter conditions - return only those points that satisfy the specified conditions
uint32 limit = 4; // Max number of result
WithPayloadSelector with_payload = 5; // Options for specifying which payload to include or not
SearchParams params = 6; // Search config
optional float score_threshold = 7; // If provided - cut off results with worse scores
optional string vector_name = 8; // Which vector to use for search, if not specified - use default vector
optional WithVectorsSelector with_vectors = 9; // Options for specifying which vectors to include into response
string group_by = 10; // Payload field to group by, must be a string or number field. If there are multiple values for the field, all of them will be used. One point can be in multiple groups.
uint32 group_size = 11; // Maximum amount of points to return per group
optional ReadConsistency read_consistency = 12; // Options for specifying read consistency guarantees
optional WithLookup with_lookup = 13; // Options for specifying how to use the group id to lookup points in another collection
}
message ScrollPoints {
string collection_name = 1;
Filter filter = 2; // Filter conditions - return only those points that satisfy the specified conditions
optional PointId offset = 3; // Start with this ID
optional uint32 limit = 4; // Max number of result
reserved 5; // deprecated "with_vector" field
WithPayloadSelector with_payload = 6; // Options for specifying which payload to include or not
optional WithVectorsSelector with_vectors = 7; // Options for specifying which vectors to include into response
optional ReadConsistency read_consistency = 8; // Options for specifying read consistency guarantees
}
message LookupLocation {
string collection_name = 1;
optional string vector_name = 2; // Which vector to use for search, if not specified - use default vector
}
message RecommendPoints {
string collection_name = 1; // name of the collection
repeated PointId positive = 2; // Look for vectors closest to those
repeated PointId negative = 3; // Try to avoid vectors like this
Filter filter = 4; // Filter conditions - return only those points that satisfy the specified conditions
uint64 limit = 5; // Max number of result
reserved 6; // deprecated "with_vector" field
WithPayloadSelector with_payload = 7; // Options for specifying which payload to include or not
SearchParams params = 8; // Search config
optional float score_threshold = 9; // If provided - cut off results with worse scores
optional uint64 offset = 10; // Offset of the result
optional string using = 11; // Define which vector to use for recommendation, if not specified - default vector
optional WithVectorsSelector with_vectors = 12; // Options for specifying which vectors to include into response
optional LookupLocation lookup_from = 13; // Name of the collection to use for points lookup, if not specified - use current collection
optional ReadConsistency read_consistency = 14; // Options for specifying read consistency guarantees
}
message RecommendBatchPoints {
string collection_name = 1; // Name of the collection
repeated RecommendPoints recommend_points = 2;
optional ReadConsistency read_consistency = 3; // Options for specifying read consistency guarantees
}
message RecommendPointGroups {
string collection_name = 1; // Name of the collection
repeated PointId positive = 2; // Look for vectors closest to those
repeated PointId negative = 3; // Try to avoid vectors like this
Filter filter = 4; // Filter conditions - return only those points that satisfy the specified conditions
uint32 limit = 5; // Max number of groups in result
WithPayloadSelector with_payload = 6; // Options for specifying which payload to include or not
SearchParams params = 7; // Search config
optional float score_threshold = 8; // If provided - cut off results with worse scores
optional string using = 9; // Define which vector to use for recommendation, if not specified - default vector
optional WithVectorsSelector with_vectors = 10; // Options for specifying which vectors to include into response
optional LookupLocation lookup_from = 11; // Name of the collection to use for points lookup, if not specified - use current collection
string group_by = 12; // Payload field to group by, must be a string or number field. If there are multiple values for the field, all of them will be used. One point can be in multiple groups.
uint32 group_size = 13; // Maximum amount of points to return per group
optional ReadConsistency read_consistency = 14; // Options for specifying read consistency guarantees
optional WithLookup with_lookup = 15; // Options for specifying how to use the group id to lookup points in another collection
}
message CountPoints {
string collection_name = 1; // name of the collection
Filter filter = 2; // Filter conditions - return only those points that satisfy the specified conditions
optional bool exact = 3; // If `true` - return exact count, if `false` - return approximate count
}
// ---------------------------------------------
// ---------------- RPC Response ---------------
// ---------------------------------------------
message PointsOperationResponse {
UpdateResult result = 1;
double time = 2; // Time spent to process
}
message UpdateResult {
uint64 operation_id = 1; // Number of operation
UpdateStatus status = 2; // Operation status
}
enum UpdateStatus {
UnknownUpdateStatus = 0;
Acknowledged = 1; // Update is received, but not processed yet
Completed = 2; // Update is applied and ready for search
}
message ScoredPoint {
PointId id = 1; // Point id
map<string, Value> payload = 2; // Payload
float score = 3; // Similarity score
reserved 4; // deprecated "vector" field
uint64 version = 5; // Last update operation applied to this point
optional Vectors vectors = 6; // Vectors to search
}
message GroupId {
oneof kind {
// Represents a double value.
uint64 unsigned_value = 1;
// Represents an integer value
int64 integer_value = 2;
// Represents a string value.
string string_value = 3;
}
}
message PointGroup {
GroupId id = 1; // Group id
repeated ScoredPoint hits = 2; // Points in the group
RetrievedPoint lookup = 3; // Point(s) from the lookup collection that matches the group id
}
message GroupsResult {
repeated PointGroup groups = 1; // Groups
}
message SearchResponse {
repeated ScoredPoint result = 1;
double time = 2; // Time spent to process
}
message BatchResult {
repeated ScoredPoint result = 1;
}
message SearchBatchResponse {
repeated BatchResult result = 1;
double time = 2; // Time spent to process
}
message SearchGroupsResponse {
GroupsResult result = 1;
double time = 2; // Time spent to process
}
message CountResponse {
CountResult result = 1;
double time = 2; // Time spent to process
}
message ScrollResponse {
optional PointId next_page_offset = 1; // Use this offset for the next query
repeated RetrievedPoint result = 2;
double time = 3; // Time spent to process
}
message CountResult {
uint64 count = 1;
}
message RetrievedPoint {
PointId id = 1;
map<string, Value> payload = 2;
reserved 3; // deprecated "vector" field
optional Vectors vectors = 4;
}
message GetResponse {
repeated RetrievedPoint result = 1;
double time = 2; // Time spent to process
}
message RecommendResponse {
repeated ScoredPoint result = 1;
double time = 2; // Time spent to process
}
message RecommendBatchResponse {
repeated BatchResult result = 1;
double time = 2; // Time spent to process
}
message RecommendGroupsResponse {
GroupsResult result = 1;
double time = 2; // Time spent to process
}
// ---------------------------------------------
// ------------- Filter Conditions -------------
// ---------------------------------------------
message Filter {
repeated Condition should = 1; // At least one of those conditions should match
repeated Condition must = 2; // All conditions must match
repeated Condition must_not = 3; // All conditions must NOT match
}
message Condition {
oneof condition_one_of {
FieldCondition field = 1;
IsEmptyCondition is_empty = 2;
HasIdCondition has_id = 3;
Filter filter = 4;
IsNullCondition is_null = 5;
NestedCondition nested = 6;
}
}
message IsEmptyCondition {
string key = 1;
}
message IsNullCondition {
string key = 1;
}
message HasIdCondition {
repeated PointId has_id = 1;
}
message NestedCondition {
string key = 1; // Path to nested object
Filter filter = 2; // Filter condition
}
message FieldCondition {
string key = 1;
Match match = 2; // Check if point has field with a given value
Range range = 3; // Check if points value lies in a given range
GeoBoundingBox geo_bounding_box = 4; // Check if points geolocation lies in a given area
GeoRadius geo_radius = 5; // Check if geo point is within a given radius
ValuesCount values_count = 6; // Check number of values for a specific field
// GeoPolygon geo_polygon = 7; // Check if geo point is within a given polygon
}
message Match {
oneof match_value {
string keyword = 1; // Match string keyword
int64 integer = 2; // Match integer
bool boolean = 3; // Match boolean
string text = 4; // Match text
RepeatedStrings keywords = 5; // Match multiple keywords
RepeatedIntegers integers = 6; // Match multiple integers
RepeatedIntegers except_integers = 7; // Match any other value except those integers
RepeatedStrings except_keywords = 8; // Match any other value except those keywords
}
}
message RepeatedStrings {
repeated string strings = 1;
}
message RepeatedIntegers {
repeated int64 integers = 1;
}
message Range {
optional double lt = 1;
optional double gt = 2;
optional double gte = 3;
optional double lte = 4;
}
message GeoBoundingBox {
GeoPoint top_left = 1; // north-west corner
GeoPoint bottom_right = 2; // south-east corner
}
message GeoRadius {
GeoPoint center = 1; // Center of the circle
float radius = 2; // In meters
}
message GeoPolygon {
// Ordered list of coordinates representing the vertices of a polygon.
// The minimum size is 4, and the first coordinate and the last coordinate
// should be the same to form a closed polygon.
repeated GeoPoint points = 1;
}
message ValuesCount {
optional uint64 lt = 1;
optional uint64 gt = 2;
optional uint64 gte = 3;
optional uint64 lte = 4;
}
// ---------------------------------------------
// -------------- Points Selector --------------
// ---------------------------------------------
message PointsSelector {
oneof points_selector_one_of {
PointsIdsList points = 1;
Filter filter = 2;
}
}
message PointsIdsList {
repeated PointId ids = 1;
}
// ---------------------------------------------
// ------------------- Point -------------------
// ---------------------------------------------
message PointStruct {
PointId id = 1;
reserved 2; // deprecated "vector" field
map<string, Value> payload = 3;
optional Vectors vectors = 4;
}
message GeoPoint {
double lon = 1;
double lat = 2;
}