use crate::actions::gsi::IndexWrite;
use crate::errors::{DynoxideError, Result};
use crate::storage::TableMetadata;
use crate::storage_backend::{IndexWriteOp, StorageBackend, VectorItemRow};
use crate::types::{
AttributeDefinition, AttributeValue, Item, ScalarAttributeType, VectorIndex, VectorValueError,
};
use std::collections::HashMap;
pub fn parse_vector_defs(meta: &TableMetadata) -> Result<Vec<VectorIndex>> {
if meta.vector_index_definitions.is_some() {
crate::bench_counters::record(&crate::bench_counters::INDEX_DEFS_PARSES);
}
match meta.vector_index_definitions.as_ref() {
Some(json) => serde_json::from_str(json)
.map_err(|e| DynoxideError::InternalServerError(format!("Bad vector index JSON: {e}"))),
None => Ok(Vec::new()),
}
}
pub(crate) fn hash_attr(vix: &VectorIndex) -> Option<&str> {
vix.search_schema.as_ref().and_then(|schema| {
schema
.iter()
.find(|e| e.search_schema_element_type == "HASH")
.map(|e| e.attribute_name.as_str())
})
}
pub(crate) fn parse_attr_defs(meta: &TableMetadata) -> Result<Vec<AttributeDefinition>> {
serde_json::from_str(&meta.attribute_definitions).map_err(|e| {
DynoxideError::InternalServerError(format!("Bad attribute definitions JSON: {e}"))
})
}
pub(crate) fn scalar_type_str(t: &ScalarAttributeType) -> &'static str {
match t {
ScalarAttributeType::S => "S",
ScalarAttributeType::N => "N",
ScalarAttributeType::B => "B",
}
}
fn mismatched_attr_def<'a>(
attr_defs: &'a [AttributeDefinition],
attr: &str,
val: &AttributeValue,
) -> Option<&'a AttributeDefinition> {
let def = attr_defs.iter().find(|d| d.attribute_name == attr)?;
let type_matches = match def.attribute_type {
ScalarAttributeType::S => matches!(val, AttributeValue::S(_)),
ScalarAttributeType::N => matches!(val, AttributeValue::N(_)),
ScalarAttributeType::B => matches!(val, AttributeValue::B(_)),
};
if type_matches { None } else { Some(def) }
}
fn vector_value_message(err: &VectorValueError, attr: &str, dims: u32, index: &str) -> String {
match err {
VectorValueError::NotAList => format!(
"One or more parameter values were invalid. Invalid type for parameter {attr}, \
Expected: 32-bit floating point number list IndexName: {index}"
),
VectorValueError::WrongDimensions { actual } => format!(
"One or more parameter values were invalid. Invalid size for parameter {attr}, \
Expected: {dims}, Actual: {actual} IndexName: {index}"
),
VectorValueError::ElementNotANumber { position, actual } => format!(
"One or more parameter values were invalid. Invalid type for parameter \
{attr}[{position}], Expected: 32-bit floating point number, Actual: {actual}. \
IndexName: {index}"
),
VectorValueError::ElementOutOfRange { position, value } => format!(
"One or more parameter values were invalid. Invalid value for parameter \
{attr}[{position}], Value: {value} is outside valid range \
[-3.4028235E38, 3.4028235E38]. IndexName: {index}"
),
}
}
pub fn validate_vector_write_attributes(item: &Item, meta: &TableMetadata) -> Result<()> {
run_vector_write_validation(item, meta, None)
}
pub fn validate_updated_vector_attributes(
before: &Item,
after: &Item,
meta: &TableMetadata,
) -> Result<()> {
run_vector_write_validation(after, meta, Some(before))
}
fn run_vector_write_validation(
item: &Item,
meta: &TableMetadata,
before: Option<&Item>,
) -> Result<()> {
let mut vixs = parse_vector_defs(meta)?;
if vixs.is_empty() {
return Ok(());
}
let attr_defs = parse_attr_defs(meta)?;
vixs.sort_by(|a, b| a.index_name.cmp(&b.index_name));
let changed = |attr: &str, val: &AttributeValue| -> bool {
match before {
Some(b) => b.get(attr) != Some(val),
None => true,
}
};
for vix in &vixs {
let vector_attr = vix.vector_attribute.attribute_name.as_str();
if let Some(val) = item.get(vector_attr)
&& changed(vector_attr, val)
&& let Err(e) = crate::types::check_vector_f32_values(val, vix.dimensions)
{
return Err(DynoxideError::ValidationException(vector_value_message(
&e,
vector_attr,
vix.dimensions,
&vix.index_name,
)));
}
let Some(schema) = vix.search_schema.as_ref() else {
continue;
};
for elem in schema {
let attr = elem.attribute_name.as_str();
let Some(val) = item.get(attr) else {
continue;
};
if !changed(attr, val) {
continue;
}
if let Some(def) = mismatched_attr_def(&attr_defs, attr, val) {
return Err(DynoxideError::ValidationException(format!(
"One or more parameter values were invalid. Attribute '{attr}' type \
mismatch. Expected: {}, Actual: {}. IndexName: {}",
scalar_type_str(&def.attribute_type),
val.type_name(),
vix.index_name
)));
}
if elem.search_schema_element_type != "HASH" {
continue;
}
let empty_kind = match val {
AttributeValue::S(s) if s.is_empty() => Some("string"),
AttributeValue::B(b) if b.is_empty() => Some("binary"),
_ => None,
};
if let Some(kind) = empty_kind {
let msg = if before.is_some() {
format!(
"One or more parameter values are not valid. The update expression \
attempted to update a secondary index key to a value that is not \
supported. The AttributeValue for a key attribute cannot contain an \
empty {kind} value."
)
} else {
format!(
"One or more parameter values are not valid. A value specified for a \
secondary index key is not supported. The AttributeValue for a key \
attribute cannot contain an empty {kind} value. IndexName: {}, \
IndexKey: {attr}",
vix.index_name
)
};
return Err(DynoxideError::KeyEmptyValueValidation(msg));
}
}
}
Ok(())
}
pub(crate) struct DerivedVectorRow {
pub row: VectorItemRow,
pub projected: Item,
}
pub(crate) fn derive_vector_row(
item: &Item,
vix: &VectorIndex,
pk_attr: &str,
sk_attr: Option<&str>,
attr_defs: &[AttributeDefinition],
table_pk: &str,
table_sk: &str,
) -> Result<Option<DerivedVectorRow>> {
let Some(value) = item.get(&vix.vector_attribute.attribute_name) else {
return Ok(None);
};
let Some(values) = crate::types::vector_f32_values(value, vix.dimensions) else {
return Ok(None);
};
let hash_value = match hash_attr(vix) {
Some(attr) => {
let Some(hash_val) = item.get(attr) else {
return Ok(None);
};
if mismatched_attr_def(attr_defs, attr, hash_val).is_some() {
return Ok(None);
}
if matches!(hash_val, AttributeValue::S(s) if s.is_empty())
|| matches!(hash_val, AttributeValue::B(b) if b.is_empty())
{
return Ok(None);
}
match hash_val.to_key_string() {
Some(s) => s,
None => return Ok(None),
}
}
None => String::new(),
};
let mut filter_map = serde_json::Map::new();
if let Some(ref schema) = vix.search_schema {
for elem in schema {
if elem.search_schema_element_type == "INLINE_FILTER"
&& let Some(v) = item.get(&elem.attribute_name)
{
if mismatched_attr_def(attr_defs, &elem.attribute_name, v).is_some() {
return Ok(None);
}
filter_map.insert(
elem.attribute_name.clone(),
serde_json::to_value(v)
.map_err(|e| DynoxideError::InternalServerError(e.to_string()))?,
);
}
}
}
let filter_json = serde_json::to_string(&serde_json::Value::Object(filter_map))
.map_err(|e| DynoxideError::InternalServerError(e.to_string()))?;
let f32_value = AttributeValue::L(
values
.iter()
.map(|v| AttributeValue::N(crate::types::f32_number_string(*v)))
.collect(),
);
let projected = build_vector_index_item(item, vix, pk_attr, sk_attr, &f32_value);
let item_json = serde_json::to_string(&projected)
.map_err(|e| DynoxideError::InternalServerError(e.to_string()))?;
let vector_json = serde_json::to_string(&values)
.map_err(|e| DynoxideError::InternalServerError(e.to_string()))?;
let entry_bytes = vector_entry_bytes(&projected, vix) as i64;
Ok(Some(DerivedVectorRow {
row: VectorItemRow {
table_pk: table_pk.to_string(),
table_sk: table_sk.to_string(),
hash_value,
vector_json,
filter_json,
item_json,
entry_bytes,
},
projected,
}))
}
pub fn vector_index_row(
item: &Item,
vix: &VectorIndex,
pk_attr: &str,
sk_attr: Option<&str>,
attr_defs: &[AttributeDefinition],
table_pk: &str,
table_sk: &str,
) -> Result<Option<VectorItemRow>> {
Ok(
derive_vector_row(item, vix, pk_attr, sk_attr, attr_defs, table_pk, table_sk)?
.map(|derived| derived.row),
)
}
fn build_vector_index_item(
item: &Item,
vix: &VectorIndex,
pk_attr: &str,
sk_attr: Option<&str>,
f32_value: &AttributeValue,
) -> Item {
use crate::types::ProjectionType;
let vector_attr = vix.vector_attribute.attribute_name.as_str();
let projection_type = vix.projection.projection_type.clone().unwrap_or_default();
match projection_type {
ProjectionType::ALL => {
let mut projected = item.clone();
projected.insert(vector_attr.to_string(), f32_value.clone());
projected
}
ProjectionType::KEYS_ONLY | ProjectionType::INCLUDE => {
let mut projected = Item::new();
if let Some(v) = item.get(pk_attr) {
projected.insert(pk_attr.to_string(), v.clone());
}
if let Some(sk) = sk_attr
&& let Some(v) = item.get(sk)
{
projected.insert(sk.to_string(), v.clone());
}
if let Some(ref schema) = vix.search_schema {
for elem in schema {
if let Some(v) = item.get(&elem.attribute_name) {
projected.insert(elem.attribute_name.clone(), v.clone());
}
}
}
if projection_type == ProjectionType::INCLUDE
&& let Some(ref attrs) = vix.projection.non_key_attributes
{
for attr in attrs {
if attr == vector_attr {
projected.insert(attr.clone(), f32_value.clone());
} else if let Some(v) = item.get(attr) {
projected.insert(attr.clone(), v.clone());
}
}
}
projected
}
}
}
pub(crate) fn vector_entry_bytes(projected: &Item, vix: &VectorIndex) -> usize {
let vector_attr = vix.vector_attribute.attribute_name.as_str();
let rest: usize = projected
.iter()
.filter(|(name, _)| name.as_str() != vector_attr)
.map(|(name, value)| name.len() + value.size())
.sum();
4 * vix.dimensions as usize + vector_attr.len() + rest
}
fn vector_rows_agree(a: &DerivedVectorRow, b: &DerivedVectorRow) -> bool {
let VectorItemRow {
table_pk: _,
table_sk: _,
hash_value,
vector_json,
filter_json,
item_json: _,
entry_bytes: _,
} = &a.row;
hash_value == &b.row.hash_value
&& vector_json == &b.row.vector_json
&& filter_json == &b.row.filter_json
&& super::index_capacity::unchanged(&a.projected, &b.projected)
}
fn vector_write_bytes(
old: Option<&DerivedVectorRow>,
new: Option<&DerivedVectorRow>,
vix: &VectorIndex,
) -> Option<f64> {
match (old, new) {
(None, None) => None,
(None, Some(entry)) | (Some(entry), None) => Some(crate::types::vector_request_bytes(
vector_entry_bytes(&entry.projected, vix),
)),
(Some(before), Some(after)) if vector_rows_agree(before, after) => None,
(Some(before), Some(after)) => Some(crate::types::vector_request_bytes(
vector_entry_bytes(&before.projected, vix)
.max(vector_entry_bytes(&after.projected, vix)),
)),
}
}
pub async fn maintain_vector_indexes_after_write<S: StorageBackend>(
storage: &S,
meta: &TableMetadata,
target: &IndexWrite<'_>,
item: &Item,
) -> Result<HashMap<String, f64>> {
let vixs = parse_vector_defs(meta)?;
if vixs.is_empty() {
return Ok(HashMap::new());
}
let attr_defs = parse_attr_defs(meta)?;
maintain_vector_indexes_after_write_with_defs(storage, &vixs, &attr_defs, target, item, false)
.await
}
#[allow(clippy::too_many_arguments)]
pub async fn maintain_vector_indexes_after_write_with_defs<S: StorageBackend>(
storage: &S,
vixs: &[VectorIndex],
attr_defs: &[AttributeDefinition],
target: &IndexWrite<'_>,
item: &Item,
skip_deletes: bool,
) -> Result<HashMap<String, f64>> {
if vixs.is_empty() {
return Ok(HashMap::new());
}
let want_capacity = crate::types::vector_capacity_wanted(target.capacity_mode);
let mut vector_bytes: HashMap<String, f64> = HashMap::new();
let mut ops: Vec<IndexWriteOp> = Vec::new();
for vix in vixs {
if !skip_deletes {
ops.push(IndexWriteOp::DeleteVector {
table_name: target.table_name.to_string(),
index_name: vix.index_name.clone(),
table_pk: target.pk.to_string(),
table_sk: target.sk.to_string(),
});
}
let derived = derive_vector_row(
item,
vix,
target.pk_attr,
target.sk_attr,
attr_defs,
target.pk,
target.sk,
)?;
if want_capacity {
let previous = match target.old_item {
Some(old) => derive_vector_row(
old,
vix,
target.pk_attr,
target.sk_attr,
attr_defs,
target.pk,
target.sk,
)?,
None => None,
};
if let Some(bytes) = vector_write_bytes(previous.as_ref(), derived.as_ref(), vix) {
vector_bytes.insert(vix.index_name.clone(), bytes);
}
}
if let Some(derived) = derived {
ops.push(IndexWriteOp::InsertVector {
table_name: target.table_name.to_string(),
index_name: vix.index_name.clone(),
row: Box::new(derived.row),
});
}
}
storage.apply_index_writes(&ops).await?;
Ok(vector_bytes)
}
pub async fn maintain_vector_indexes_after_delete<S: StorageBackend>(
storage: &S,
meta: &TableMetadata,
target: &IndexWrite<'_>,
) -> Result<HashMap<String, f64>> {
let vixs = parse_vector_defs(meta)?;
if vixs.is_empty() {
return Ok(HashMap::new());
}
let attr_defs = parse_attr_defs(meta)?;
maintain_vector_indexes_after_delete_with_defs(storage, &vixs, &attr_defs, target).await
}
pub async fn maintain_vector_indexes_after_delete_with_defs<S: StorageBackend>(
storage: &S,
vixs: &[VectorIndex],
attr_defs: &[AttributeDefinition],
target: &IndexWrite<'_>,
) -> Result<HashMap<String, f64>> {
if vixs.is_empty() {
return Ok(HashMap::new());
}
let want_capacity = crate::types::vector_capacity_wanted(target.capacity_mode);
let mut vector_bytes: HashMap<String, f64> = HashMap::new();
let mut ops: Vec<IndexWriteOp> = Vec::new();
for vix in vixs {
ops.push(IndexWriteOp::DeleteVector {
table_name: target.table_name.to_string(),
index_name: vix.index_name.clone(),
table_pk: target.pk.to_string(),
table_sk: target.sk.to_string(),
});
if want_capacity
&& let Some(old) = target.old_item
&& let Some(derived) = derive_vector_row(
old,
vix,
target.pk_attr,
target.sk_attr,
attr_defs,
target.pk,
target.sk,
)?
&& let Some(bytes) = vector_write_bytes(Some(&derived), None, vix)
{
vector_bytes.insert(vix.index_name.clone(), bytes);
}
}
storage.apply_index_writes(&ops).await?;
Ok(vector_bytes)
}