use crate::codes::QUERY_OP_VERSION;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct KeyMatch {
pub field: String,
pub value: String,
}
impl KeyMatch {
pub fn new(field: impl Into<String>, value: impl Into<String>) -> Self {
Self {
field: field.into(),
value: value.into(),
}
}
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[cfg_attr(feature = "builders", derive(bon::Builder))]
pub struct Query {
#[cfg_attr(feature = "builders", builder(into))]
pub index: String,
#[cfg_attr(feature = "builders", builder(default))]
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub by_key: Vec<KeyMatch>,
#[cfg_attr(feature = "builders", builder(into))]
#[serde(default, skip_serializing_if = "Option::is_none")]
pub message_type: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub time_range: Option<(u64, u64)>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub filter: Option<Filter>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub vector: Option<VectorQuery>,
#[cfg_attr(feature = "builders", builder(default))]
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub order: Vec<Sort>,
#[cfg_attr(feature = "builders", builder(default = 50))]
pub limit: usize,
#[cfg_attr(feature = "builders", builder(default))]
#[serde(default)]
pub offset: usize,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub aggregate: Option<Aggregate>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub having: Option<Filter>,
#[cfg_attr(feature = "builders", builder(default))]
#[serde(default, skip_serializing_if = "is_false")]
pub distinct: bool,
#[cfg_attr(feature = "builders", builder(default))]
#[serde(default)]
pub select: Select,
#[cfg_attr(feature = "builders", builder(into))]
#[serde(default, skip_serializing_if = "Option::is_none")]
pub fork: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub raw_sql: Option<RawSql>,
#[cfg_attr(feature = "builders", builder(default))]
#[serde(default, skip_serializing_if = "Consistency::is_eventual")]
pub consistency: Consistency,
}
fn is_false(value: &bool) -> bool {
!*value
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum Consistency {
#[default]
Eventual,
ReadYourWrites,
Strong,
}
impl Consistency {
pub fn is_eventual(&self) -> bool {
matches!(self, Consistency::Eventual)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ConsistencyGate {
pub applied: u64,
pub required: u64,
}
impl ConsistencyGate {
pub fn new(applied: u64, required: u64) -> Self {
Self { applied, required }
}
pub fn is_caught_up(&self) -> bool {
self.applied >= self.required
}
pub fn check(&self, level: Consistency, what: impl Into<String>) -> Result<(), QueryError> {
if level.is_eventual() || self.is_caught_up() {
return Ok(());
}
Err(QueryError::Stale {
what: what.into(),
applied: self.applied,
required: self.required,
})
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Filter {
All(Vec<Filter>),
Any(Vec<Filter>),
Not(Box<Filter>),
Pred(Predicate),
}
impl Filter {
pub fn all(filters: impl IntoIterator<Item = Filter>) -> Self {
Filter::All(filters.into_iter().collect())
}
pub fn any(filters: impl IntoIterator<Item = Filter>) -> Self {
Filter::Any(filters.into_iter().collect())
}
pub fn negate(filter: Filter) -> Self {
Filter::Not(Box::new(filter))
}
pub fn pred(field: impl Into<String>, op: CmpOp, value: impl Into<Value>) -> Self {
Filter::Pred(Predicate {
field: field.into(),
op,
value: value.into(),
})
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct Predicate {
pub field: String,
pub op: CmpOp,
pub value: Value,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct RawSql {
pub sql: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub params: Vec<Value>,
}
#[derive(
Clone,
Copy,
Debug,
PartialEq,
Eq,
Serialize,
Deserialize,
strum::Display,
strum::EnumString,
strum::VariantArray,
)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
pub enum CmpOp {
Eq,
Ne,
Lt,
Lte,
Gt,
Gte,
In,
Contains,
Prefix,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct Sort {
pub field: String,
#[serde(default)]
pub dir: Dir,
}
#[derive(
Clone,
Copy,
Debug,
Default,
PartialEq,
Eq,
Serialize,
Deserialize,
strum::Display,
strum::EnumString,
strum::VariantArray,
)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
pub enum Dir {
#[default]
Asc,
Desc,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct VectorQuery {
pub field: String,
pub embedding: Vec<f32>,
pub top_k: usize,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct Aggregate {
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub group_by: Vec<String>,
pub funcs: Vec<AggCall>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub window: Option<Window>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AggCall {
pub func: AggFunc,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub field: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub arg: Option<f64>,
pub alias: String,
}
#[derive(
Clone,
Copy,
Debug,
PartialEq,
Eq,
Serialize,
Deserialize,
strum::Display,
strum::EnumString,
strum::VariantArray,
)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
pub enum AggFunc {
Count,
CountDistinct,
Sum,
Avg,
Min,
Max,
Percentile,
StdDev,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Window {
pub field: String,
pub every_micros: u64,
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Select {
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub fields: Vec<String>,
#[serde(default)]
pub payload: bool,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum Value {
Str(String),
Int(i64),
Uint(u64),
Float(f64),
Bool(bool),
Null,
List(Vec<Value>),
}
impl From<&str> for Value {
fn from(value: &str) -> Self {
Self::Str(value.to_owned())
}
}
impl From<String> for Value {
fn from(value: String) -> Self {
Self::Str(value)
}
}
impl From<&String> for Value {
fn from(value: &String) -> Self {
Self::Str(value.clone())
}
}
impl From<i64> for Value {
fn from(value: i64) -> Self {
Self::Int(value)
}
}
impl From<u64> for Value {
fn from(value: u64) -> Self {
Self::Uint(value)
}
}
impl From<i32> for Value {
fn from(value: i32) -> Self {
Self::Int(value as i64)
}
}
impl From<u32> for Value {
fn from(value: u32) -> Self {
Self::Int(value as i64)
}
}
impl From<f64> for Value {
fn from(value: f64) -> Self {
Self::Float(value)
}
}
impl From<f32> for Value {
fn from(value: f32) -> Self {
Self::Float(value as f64)
}
}
impl From<bool> for Value {
fn from(value: bool) -> Self {
Self::Bool(value)
}
}
impl<T: Into<Value>> From<Vec<T>> for Value {
fn from(values: Vec<T>) -> Self {
Self::List(values.into_iter().map(Into::into).collect())
}
}
impl Value {
pub fn from_input(input: &str) -> Self {
match input {
"null" => return Value::Null,
"true" => return Value::Bool(true),
"false" => return Value::Bool(false),
_ => {}
}
if let Ok(int) = input.parse::<i64>() {
return Value::Int(int);
}
if let Ok(uint) = input.parse::<u64>() {
return Value::Uint(uint);
}
if !input.is_empty()
&& input
.bytes()
.all(|b| b.is_ascii_digit() || b == b'.' || b == b'-' || b == b'+')
&& let Ok(float) = input.parse::<f64>()
{
return Value::Float(float);
}
Value::Str(input.to_owned())
}
}
impl std::fmt::Display for Value {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Value::Str(value) => f.write_str(value),
Value::Int(value) => write!(f, "{value}"),
Value::Uint(value) => write!(f, "{value}"),
Value::Float(value) => write!(f, "{value}"),
Value::Bool(value) => write!(f, "{value}"),
Value::Null => f.write_str("null"),
Value::List(values) => {
f.write_str("[")?;
for (index, value) in values.iter().enumerate() {
if index > 0 {
f.write_str(", ")?;
}
write!(f, "{value}")?;
}
f.write_str("]")
}
}
}
}
impl std::str::FromStr for Value {
type Err = std::convert::Infallible;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(Value::from_input(s))
}
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct QueryResult {
pub rows: Vec<Row>,
#[serde(default)]
pub page: Page,
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Page {
pub offset: usize,
pub limit: usize,
pub total: usize,
pub has_more: bool,
}
impl Page {
pub fn total_pages(&self) -> usize {
if self.limit == 0 {
0
} else {
self.total.div_ceil(self.limit)
}
}
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct Row {
pub headers: BTreeMap<String, String>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub metadata: BTreeMap<String, String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub partition: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub offset: Option<u64>,
#[serde(
default,
skip_serializing_if = "Option::is_none",
with = "crate::encoding::opt_bin_bytes"
)]
pub payload: Option<Vec<u8>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub score: Option<f32>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[non_exhaustive]
pub struct QueryEnvelope {
pub v: u32,
pub query: Query,
}
impl QueryEnvelope {
pub fn new(query: Query) -> Self {
Self {
v: QUERY_OP_VERSION,
query,
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[non_exhaustive]
pub enum QueryReply {
Ok(QueryResult),
Err(QueryError),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, thiserror::Error)]
#[non_exhaustive]
pub enum QueryError {
#[error("query not supported: {0}")]
Unsupported(String),
#[error("index not found: {0}")]
IndexNotFound(String),
#[error("fork not found: {0}")]
ForkNotFound(String),
#[error("backend error: {0}")]
Backend(String),
#[error("result too large: {what} {size} exceeds cap {cap}")]
TooLarge {
what: String,
size: usize,
cap: usize,
},
#[error("unsupported envelope version (expected {expected}, got {got})")]
Version { expected: u32, got: u32 },
#[error("stale read: {what} applied {applied}, required {required}")]
Stale {
what: String,
applied: u64,
required: u64,
},
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn given_dsl_enums_when_displayed_then_should_be_snake_case() {
assert_eq!(CmpOp::Gte.to_string(), "gte");
assert_eq!(CmpOp::Prefix.to_string(), "prefix");
assert_eq!("ne".parse::<CmpOp>().expect("ne parses"), CmpOp::Ne);
assert_eq!(Dir::Desc.to_string(), "desc");
assert_eq!(AggFunc::Count.to_string(), "count");
}
#[test]
fn given_a_consistency_gate_when_checked_then_should_fail_not_downgrade() {
assert!(
ConsistencyGate::new(0, 100)
.check(Consistency::Eventual, "orders")
.is_ok()
);
assert!(
ConsistencyGate::new(100, 100)
.check(Consistency::ReadYourWrites, "orders")
.is_ok()
);
let stale = ConsistencyGate::new(41, 57)
.check(Consistency::Strong, "orders")
.expect_err("a lagging projector must fail, never downgrade");
assert!(matches!(
stale,
QueryError::Stale {
applied: 41,
required: 57,
..
}
));
}
#[test]
fn given_a_page_when_computing_total_pages_then_should_divide_by_limit() {
let page = Page {
offset: 0,
limit: 3,
total: 10,
has_more: true,
};
assert_eq!(page.total_pages(), 4);
assert_eq!(Page::default().total_pages(), 0);
}
}
#[cfg(all(test, feature = "codecs"))]
mod serde_tests {
use super::*;
#[cfg(feature = "builders")]
use crate::codes::QUERY_OP_VERSION;
use crate::framing::{decode_named, encode_named};
#[test]
fn given_dsl_enums_when_serialized_then_serde_should_match_display() {
assert_eq!(
serde_json::to_string(&CmpOp::Lte).expect("CmpOp serializes"),
"\"lte\""
);
assert_eq!(
serde_json::from_str::<CmpOp>("\"in\"").expect("CmpOp deserializes"),
CmpOp::In
);
assert_eq!(
serde_json::to_string(&Dir::Asc).expect("Dir serializes"),
"\"asc\""
);
}
#[test]
#[cfg(feature = "builders")]
fn given_a_query_when_round_tripped_through_the_envelope_then_should_be_unchanged() {
let query = Query::builder()
.index("orders")
.by_key(vec![KeyMatch::new("customer_id", "abc")])
.filter(Filter::pred("status", CmpOp::Eq, "paid"))
.order(vec![Sort {
field: "ts".to_owned(),
dir: Dir::Desc,
}])
.limit(20)
.build();
let request = QueryEnvelope::new(query);
let json = serde_json::to_string(&request).expect("the request serializes");
let back: QueryEnvelope = serde_json::from_str(&json).expect("the request deserializes");
assert_eq!(back.v, QUERY_OP_VERSION);
assert_eq!(back.query.index, "orders");
assert_eq!(back.query.limit, 20);
assert_eq!(back.query.by_key, vec![KeyMatch::new("customer_id", "abc")]);
let Some(Filter::Pred(predicate)) = &back.query.filter else {
panic!("expected a single predicate filter");
};
assert_eq!(predicate.value, Value::Str("paid".to_owned()));
assert_eq!(back.query.order[0].dir, Dir::Desc);
}
#[test]
#[cfg(feature = "builders")]
fn given_each_consistency_level_when_round_tripped_then_should_preserve_it_and_skip_eventual() {
for level in [
Consistency::Eventual,
Consistency::ReadYourWrites,
Consistency::Strong,
] {
let query = Query::builder().index("orders").consistency(level).build();
let bytes = encode_named(&QueryEnvelope::new(query)).expect("serializes");
let back: QueryEnvelope = decode_named(&bytes).expect("deserializes");
assert_eq!(back.query.consistency, level);
}
let default = Query::builder().index("orders").build();
assert_eq!(default.consistency, Consistency::Eventual);
let json = serde_json::to_string(&default).expect("json");
assert!(
!json.contains("consistency"),
"default Eventual must be omitted: {json}"
);
}
#[test]
fn given_a_stale_reply_when_round_tripped_then_should_preserve_the_offsets() {
let reply = QueryReply::Err(QueryError::Stale {
what: "orders".to_owned(),
applied: 41,
required: 57,
});
let bytes = encode_named(&reply).expect("serializes");
let back: QueryReply = decode_named(&bytes).expect("deserializes");
let QueryReply::Err(QueryError::Stale {
what,
applied,
required,
}) = back
else {
panic!("expected a Stale error");
};
assert_eq!((what.as_str(), applied, required), ("orders", 41, 57));
}
#[test]
#[cfg(feature = "builders")]
fn given_a_vector_query_when_round_tripped_then_should_preserve_the_embedding() {
let query = Query::builder()
.index("mem:conv-1")
.vector(VectorQuery {
field: "embedding".to_owned(),
embedding: vec![0.1, 0.2, 0.3],
top_k: 5,
})
.build();
let json = serde_json::to_string(&query).expect("the query serializes");
let back: Query = serde_json::from_str(&json).expect("the query deserializes");
let vector = back.vector.expect("the vector survives the round-trip");
assert_eq!(vector.embedding, vec![0.1, 0.2, 0.3]);
assert_eq!(vector.top_k, 5);
}
#[test]
fn given_a_reply_with_a_payload_row_when_round_tripped_then_should_preserve_the_bytes() {
let mut headers = BTreeMap::new();
headers.insert("order_id".to_owned(), "123".to_owned());
let reply = QueryReply::Ok(QueryResult {
rows: vec![Row {
headers,
metadata: BTreeMap::from([("agdx.ct".to_owned(), "1".to_owned())]),
partition: Some(2),
offset: Some(17),
payload: Some(b"{\"total\":42}".to_vec()),
score: None,
}],
page: Page {
offset: 0,
limit: 50,
total: 1,
has_more: false,
},
});
let bytes = encode_named(&reply).expect("the reply serializes");
let back: QueryReply = decode_named(&bytes).expect("the reply deserializes");
let QueryReply::Ok(result) = back else {
panic!("the reply should decode as Ok");
};
assert_eq!(result.rows[0].headers["order_id"], "123");
assert_eq!(
result.rows[0].payload.as_deref(),
Some(b"{\"total\":42}".as_ref())
);
assert_eq!(result.page.total, 1);
assert!(!result.page.has_more);
}
}