// Copyright (c) Sean Lawlor
//
// This source code is licensed under both the MIT license found in the
// LICENSE-MIT file in the root directory of this source tree.
// ============================================================
// Protobuf protocol specification. This is the top-level "meta" API
// and inner APIs will be included here
// ============================================================
syntax = "proto3";
// import "auth.proto";
// import "node.proto";
// import "control.proto";
package meta;
// Handshake: identifies the connecting node
message Hello {
string node_id = 1;
// Retained at its original field number for safe protobuf field evolution.
// Since protocol v2 this field must be empty; the challenge-bound proof is
// carried in HelloAuth.
bytes auth_tag = 2;
// Inclusive wire-version range understood by this peer. Both are nonzero;
// a connection proceeds only when the ranges overlap.
uint32 min_protocol_version = 3;
uint32 max_protocol_version = 4;
// Feature bitset. Required bits are checked before either side installs the
// connection, so mixed binaries fail closed rather than mis-acknowledging
// mutations or replication.
uint64 capabilities = 5;
// Fresh 256-bit challenge generated independently for every physical
// connection. Both peers' complete Hellos are bound into each HelloAuth.
bytes challenge = 6;
}
// Second handshake flight. With a configured cluster secret, auth_tag is an
// HMAC-SHA256 over the ordered dialer/listener Hello transcript and the
// proving role. Without a secret it is empty, but the flight still makes
// handshake completion explicit before either peer publishes the connection.
message HelloAuth {
bytes auth_tag = 1;
}
// Phase 7: operation priority for traffic isolation. Wire values are
// stable; PRIORITY_DEFAULT = 0 so an absent/unknown field degrades to
// normal priority rather than misrouting.
enum Priority {
PRIORITY_DEFAULT = 0;
PRIORITY_HIGH = 1;
PRIORITY_BACKGROUND = 2;
}
// A single replicated write, sent from a fragment's coordinator to the
// other replica holders. Delivery is at-least-once; receivers merge with
// last-writer-wins on (timestamp_ms, node_id) and do NOT re-forward.
// `seq`/`boot_id` are propagation metadata.
message ReplicateOp {
string key = 1;
bytes value = 2;
uint64 timestamp_ms = 3;
string node_id = 4;
uint64 seq = 5;
// Random per process start. Carried as origin metadata; replay safety is
// provided by the value's LWW version rather than transport deduplication.
uint64 boot_id = 6;
// Optional key expiry (wall-clock ms). 0/absent = no expiry.
uint64 expires_at_ms = 7;
// True if this is a deletion tombstone.
bool is_tombstone = 8;
}
// A client-routed write: "you own this key's fragment — apply it as
// coordinator and replicate to the other holders." One network hop.
message ForwardedPut {
string key = 1;
bytes value = 2;
uint64 timestamp_ms = 3;
string node_id = 4;
uint64 seq = 5;
uint64 boot_id = 6;
// Optional key expiry (wall-clock ms). 0/absent = no expiry.
uint64 expires_at_ms = 7;
// True if this is a deletion tombstone.
bool is_tombstone = 8;
// Phase 7: priority the receiving node should apply this op with.
Priority priority = 9;
// Phase 8: re-forward count. A node that receives a ForwardedPut for a
// fragment it no longer serves (stale placement maps during
// migration) re-forwards it to the current owner instead of
// dropping it; hops bounds the chain.
uint32 hops = 10;
// Correlates the exact apply outcome back to the originating node.
uint64 req_id = 11;
}
// A single-key routed read. The origin tries configured holders in order; a
// non-serving holder may relay it under a fresh request id while placement
// maps converge. GetResponse has no unavailable value, so routing/actor
// unavailability produces no false not-found reply and the origin times out.
message ForwardedGet {
string key = 1;
uint64 req_id = 2;
// Phase 7: priority the receiving node should serve this read with.
Priority priority = 3;
// Phase 8: re-forward count, as with ForwardedPut. A non-serving node
// relays the read to the current owner and relays the response back.
uint32 hops = 4;
}
message GetResponse {
uint64 req_id = 1;
bool found = 2;
bytes value = 3;
uint64 timestamp_ms = 4;
string node_id = 5;
// Optional key expiry (wall-clock ms). 0/absent = no expiry.
uint64 expires_at_ms = 6;
// True if this is a deletion tombstone.
bool is_tombstone = 7;
// Phase 7: true if the read was shed due to backpressure (distinct
// from not-found).
bool shed = 8;
}
// Retired protocol-v1/v2 full-state frames. They remain generated for source
// compatibility, but a protocol-v3 connection closes if either is received.
message SyncRequest {
repeated uint32 fragments = 1;
}
// Retired peer full-state response; see SyncRequest above.
message SyncResponse {
repeated SyncEntry entries = 1;
}
message SyncEntry {
string key = 1;
bytes value = 2;
uint64 timestamp_ms = 3;
string node_id = 4;
// Optional key expiry (wall-clock ms). 0/absent = no expiry.
uint64 expires_at_ms = 5;
// True if this is a deletion tombstone.
bool is_tombstone = 6;
}
// Top-level wire envelope, length-delimited on the wire
// Phase 5: async sender/receiver replication. The sender batches ops per
// (fragment, peer) and ships them on a timer. Tracked batches receive an ACK
// reporting the achieved durability after apply and any requested flush attempt,
// or a NACK on execution failure; reconciliation batches are untracked.
// Session ids distinguish pairings
// across reconnects; stale batches (old session) are discarded.
message SessionStart {
uint64 session_id = 1;
uint32 fragment = 2;
}
message ReplicateBatch {
uint64 session_id = 1;
uint64 batch_seq = 2;
uint32 fragment = 3;
repeated ReplicateOp ops = 4;
// Minimum fence the sender requires before this tracked batch may retire.
// Ignored for untracked reconciliation batches.
ReplicationAckLevel required_ack_level = 5;
}
message BatchAck {
uint64 session_id = 1;
uint64 batch_seq = 2;
uint32 fragment = 3;
// Distinguishes volatile in-memory application from a successful flush to
// the receiver's configured KvStore.
ReplicationAckLevel level = 4;
}
enum ReplicationAckLevel {
REPLICATION_ACK_LEVEL_UNKNOWN = 0;
REPLICATION_ACK_LEVEL_APPLIED = 1;
REPLICATION_ACK_LEVEL_STORE_COMMITTED = 2;
REPLICATION_ACK_LEVEL_STABLE_COMMITTED = 3;
}
// A receiver could not account for or persist a batch. The sender retains
// and retries the exact idempotent operations instead of treating them as
// delivered.
message BatchNack {
uint64 session_id = 1;
uint64 batch_seq = 2;
uint32 fragment = 3;
string error = 4;
}
// Incremental anti-entropy: exchange (timestamp, tombstone) summaries,
// then only the divergent keys.
message KeySummary {
string key = 1;
uint64 timestamp_ms = 2;
string node_id = 3;
bool is_tombstone = 4;
}
message ReconcileRequest {
uint32 fragment = 1;
// Correlates the request to the active SessionStart on this connection.
uint64 session_id = 2;
// Monotonic within a sender session. Responses from older rounds are stale.
uint64 round_id = 3;
// V3 byte-bounded summary pagination. Page zero starts a pass; later pages
// echo its cursor, observed revision, and pinned TTL time. Revision drift is
// allowed during the bounded pass and forces a fresh verification pass.
uint64 page_seq = 4;
string after_key = 5;
uint64 snapshot_revision = 6;
uint64 snapshot_time_ms = 7;
}
message ReconcileResponse {
uint32 fragment = 1;
repeated KeySummary summaries = 2;
uint64 session_id = 3;
uint64 round_id = 4;
// V3 page correlation and continuation metadata.
uint64 page_seq = 5;
string next_after_key = 6;
uint64 snapshot_revision = 7;
uint64 snapshot_time_ms = 8;
bool done = 9;
// The responder could not continue this pass; discard its partial result
// and begin again with a fresh nonzero round id.
bool restart = 10;
}
message ReconcileFetch {
uint32 fragment = 1;
repeated string keys = 2;
uint64 session_id = 3;
uint64 round_id = 4;
// Identifies the requested page within the round.
uint64 chunk_seq = 5;
}
message ReconcileData {
uint32 fragment = 1;
repeated SyncEntry entries = 2;
uint64 session_id = 3;
uint64 round_id = 4;
uint64 chunk_seq = 5;
// V3 page correlation within one bounded fetch chunk. Large value sets are
// streamed across exact-byte-bounded pages.
uint64 page_seq = 6;
bool done = 7;
}
// Phase 6: multi-key fan-out. One message per remote fragment/owner attempt.
message BatchPut {
repeated ForwardedPut puts = 1;
// Phase 7: priority applied to every put in the batch.
Priority priority = 2;
// Correlates one ordered result per `puts` entry.
uint64 req_id = 3;
}
// Exact result of one remotely forwarded mutation. Results in a
// MutationResponse have the same order and cardinality as the request.
enum MutationStatus {
MUTATION_STATUS_UNKNOWN = 0;
MUTATION_STATUS_APPLIED = 1;
MUTATION_STATUS_ALREADY_PRESENT = 2;
MUTATION_STATUS_SUPERSEDED = 3;
MUTATION_STATUS_EXPIRED = 4;
MUTATION_STATUS_BACKPRESSURE = 5;
MUTATION_STATUS_STORAGE_ERROR = 6;
MUTATION_STATUS_ACTOR_ERROR = 7;
MUTATION_STATUS_UNAVAILABLE = 8;
}
message MutationResult {
string key = 1;
MutationStatus status = 2;
string error = 3;
uint64 winner_timestamp_ms = 4;
string winner_node_id = 5;
bool winner_is_tombstone = 6;
uint32 request_index = 7;
}
message MutationResponse {
uint64 req_id = 1;
repeated MutationResult results = 2;
}
message BatchGet {
uint64 req_id = 1;
repeated string keys = 2;
// Phase 7: priority applied to every get in the batch.
Priority priority = 3;
}
message BatchGetResponse {
uint64 req_id = 1;
message KeyValue {
string key = 1;
bool found = 2;
bytes value = 3;
uint64 timestamp_ms = 4;
string node_id = 5;
uint64 expires_at_ms = 6;
bool is_tombstone = 7;
// Phase 7: true if the read was shed due to backpressure (distinct
// from not-found).
bool shed = 8;
}
// With CAPABILITY_BYTE_BOUNDED_BATCHES, a non-empty request returns either
// an empty unavailable response or a non-empty ordered prefix. The caller
// resumes an omitted suffix under a fresh request id.
repeated KeyValue results = 2;
}
message Envelope {
oneof msg {
Hello hello = 1;
// Phase 5: ReplicateOp (2) retired; batches carry Repeated ReplicateOp.
SyncRequest sync_request = 3;
SyncResponse sync_response = 4;
ForwardedPut forwarded_put = 5;
ForwardedGet forwarded_get = 6;
GetResponse get_response = 7;
SessionStart session_start = 8;
ReplicateBatch replicate_batch = 9;
BatchAck batch_ack = 10;
ReconcileRequest reconcile_request = 11;
ReconcileResponse reconcile_response = 12;
ReconcileFetch reconcile_fetch = 13;
ReconcileData reconcile_data = 14;
BatchPut batch_put = 15;
BatchGet batch_get = 16;
BatchGetResponse batch_get_response = 17;
MutationResponse mutation_response = 18;
BatchNack batch_nack = 19;
HelloAuth hello_auth = 20;
ScanRequest scan_request = 21;
ScanResponse scan_response = 22;
}
}
// Distributed scan: coordinator -> fragment holder. Mirrors ForwardedGet:
// one holder per fragment, failover across replica_order, hop-bounded
// re-forwarding during placement migration. The spec is data, never code.
message ScanRequest {
uint64 req_id = 1;
uint32 fragment = 2;
// Matcher descriptor: 0=all, 1=key prefix.
uint32 matcher_kind = 3;
// Matcher argument: prefix bytes when matcher_kind=1.
bytes matcher_arg = 4;
// Transform descriptor: 0=identity, 1=key_only, 2=prefix_bytes.
uint32 transform_kind = 5;
// Transform argument: N when transform_kind=2.
uint32 transform_arg = 6;
// Per-fragment match bound. 0 = server default. This bound is also the
// wire-size safety: the response is a single envelope, so limit *
// avg value size must stay comfortably under the 16 MiB frame cap.
uint32 limit = 7;
// Skip-first-N pagination cursor (deterministic key order).
uint64 offset = 8;
// Always PRIORITY_BACKGROUND for scans.
Priority priority = 9;
// Re-forward bound, as with ForwardedGet.
uint32 hops = 10;
}
// Holder -> coordinator. A single response per request; the per-fragment
// limit keeps it under the frame cap.
message ScanResponse {
uint64 req_id = 1;
repeated ScanMatch matches = 2;
// Non-empty = this holder could not serve the scan (not serving, bad
// spec, shed). The coordinator fails over to the next replica holder.
string error = 3;
}
message ScanMatch {
string key = 1;
// Transformed value. Empty when the transform is key_only.
bytes value = 2;
}