use std::str;
use simsimd::SpatialSimilarity;
use sonic_rs::{JsonContainerTrait, JsonValueTrait};
use crate::{
error::{Error, Result},
meta::{decode_hex_u64, u64_to_hex_16},
search::meta::{
DistanceMetric, IndexFieldType, IndexOnDataType, VectorAlgorithm, VectorFieldMetadata,
VectorType,
},
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub enum SearchSubkeyType {
IndexMeta = 0,
Prefixes = 1,
FieldMeta = 2,
Field = 3,
FieldAlias = 4,
}
impl SearchSubkeyType {
#[inline]
pub const fn from_u8(val: u8) -> Option<Self> {
match val {
0 => Some(Self::IndexMeta),
1 => Some(Self::Prefixes),
2 => Some(Self::FieldMeta),
3 => Some(Self::Field),
4 => Some(Self::FieldAlias),
_ => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub enum HnswLevelType {
Node = 1,
Edge = 2,
}
impl HnswLevelType {
#[inline]
pub const fn from_u8(val: u8) -> Option<Self> {
match val {
1 => Some(Self::Node),
2 => Some(Self::Edge),
_ => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SearchKey<'a> {
pub ns: &'a str,
pub index: &'a str,
pub field: Option<&'a str>,
}
impl<'a> SearchKey<'a> {
#[inline]
pub const fn new(ns: &'a str, index: &'a str) -> Self {
Self {
ns,
index,
field: None,
}
}
#[inline]
pub const fn with_field(ns: &'a str, index: &'a str, field: &'a str) -> Self {
Self {
ns,
index,
field: Some(field),
}
}
#[inline]
pub fn put_namespace(dst: &mut Vec<u8>, ns: &str) {
dst.push(ns.len() as u8);
dst.extend_from_slice(ns.as_bytes());
}
#[inline]
pub fn put_type(dst: &mut Vec<u8>, subkey_type: SearchSubkeyType) {
dst.push(subkey_type as u8);
}
#[inline]
pub fn put_sized_string(dst: &mut Vec<u8>, s: &str) {
dst.extend_from_slice(&(s.len() as u32).to_be_bytes());
dst.extend_from_slice(s.as_bytes());
}
#[inline]
pub fn get_sized_string<'b>(input: &mut &'b [u8]) -> Option<&'b str> {
if input.len() < 4 {
return None;
}
let len = u32::from_be_bytes([input[0], input[1], input[2], input[3]]) as usize;
*input = &input[4..];
if input.len() < len {
return None;
}
let str_bytes = &input[..len];
*input = &input[len..];
str::from_utf8(str_bytes).ok()
}
#[inline]
pub fn put_hnsw_level_prefix(dst: &mut Vec<u8>, ns: &str, index: &str, field: &str, level: u16) {
Self::put_namespace(dst, ns);
Self::put_type(dst, SearchSubkeyType::Field);
Self::put_sized_string(dst, index);
Self::put_sized_string(dst, field);
dst.extend_from_slice(&level.to_be_bytes());
}
#[inline]
pub fn put_hnsw_level_node_prefix(
dst: &mut Vec<u8>,
ns: &str,
index: &str,
field: &str,
level: u16,
) {
Self::put_hnsw_level_prefix(dst, ns, index, field, level);
dst.push(HnswLevelType::Node as u8);
}
#[inline]
pub fn put_hnsw_level_edge_prefix(
dst: &mut Vec<u8>,
ns: &str,
index: &str,
field: &str,
level: u16,
) {
Self::put_hnsw_level_prefix(dst, ns, index, field, level);
dst.push(HnswLevelType::Edge as u8);
}
#[inline]
pub fn construct_index_meta(&self) -> Vec<u8> {
let mut dst = Vec::with_capacity(1 + self.ns.len() + 1 + 4 + self.index.len());
Self::put_namespace(&mut dst, self.ns);
Self::put_type(&mut dst, SearchSubkeyType::IndexMeta);
Self::put_sized_string(&mut dst, self.index);
dst
}
#[inline]
pub fn construct_index_prefixes(&self) -> Vec<u8> {
let mut dst = Vec::with_capacity(1 + self.ns.len() + 1 + 4 + self.index.len());
Self::put_namespace(&mut dst, self.ns);
Self::put_type(&mut dst, SearchSubkeyType::Prefixes);
Self::put_sized_string(&mut dst, self.index);
dst
}
#[inline]
pub fn construct_field_meta(&self) -> Vec<u8> {
let field_name = self.field.unwrap_or("");
let mut dst =
Vec::with_capacity(1 + self.ns.len() + 1 + 4 + self.index.len() + 4 + field_name.len());
Self::put_namespace(&mut dst, self.ns);
Self::put_type(&mut dst, SearchSubkeyType::FieldMeta);
Self::put_sized_string(&mut dst, self.index);
Self::put_sized_string(&mut dst, field_name);
dst
}
#[inline]
pub fn construct_all_field_meta_begin(&self) -> Vec<u8> {
let mut dst = Vec::with_capacity(1 + self.ns.len() + 1 + 4 + self.index.len() + 4);
Self::put_namespace(&mut dst, self.ns);
Self::put_type(&mut dst, SearchSubkeyType::FieldMeta);
Self::put_sized_string(&mut dst, self.index);
dst.extend_from_slice(&0u32.to_be_bytes());
dst
}
#[inline]
pub fn construct_all_field_meta_end(&self) -> Vec<u8> {
let mut dst = Vec::with_capacity(1 + self.ns.len() + 1 + 4 + self.index.len() + 4);
Self::put_namespace(&mut dst, self.ns);
Self::put_type(&mut dst, SearchSubkeyType::FieldMeta);
Self::put_sized_string(&mut dst, self.index);
dst.extend_from_slice(&u32::MAX.to_be_bytes());
dst
}
#[inline]
pub fn construct_all_field_data_begin(&self) -> Vec<u8> {
let mut dst = Vec::with_capacity(1 + self.ns.len() + 1 + 4 + self.index.len() + 4);
Self::put_namespace(&mut dst, self.ns);
Self::put_type(&mut dst, SearchSubkeyType::Field);
Self::put_sized_string(&mut dst, self.index);
dst.extend_from_slice(&0u32.to_be_bytes());
dst
}
#[inline]
pub fn construct_all_field_data_end(&self) -> Vec<u8> {
let mut dst = Vec::with_capacity(1 + self.ns.len() + 1 + 4 + self.index.len() + 4);
Self::put_namespace(&mut dst, self.ns);
Self::put_type(&mut dst, SearchSubkeyType::Field);
Self::put_sized_string(&mut dst, self.index);
dst.extend_from_slice(&u32::MAX.to_be_bytes());
dst
}
#[inline]
pub fn construct_tag_field_data(&self, tag: &str, key: &str) -> Vec<u8> {
let field_name = self.field.unwrap_or("");
let mut dst = Vec::with_capacity(
1 + self.ns.len()
+ 1
+ 4
+ self.index.len()
+ 4
+ field_name.len()
+ 4
+ tag.len()
+ 4
+ key.len(),
);
Self::put_namespace(&mut dst, self.ns);
Self::put_type(&mut dst, SearchSubkeyType::Field);
Self::put_sized_string(&mut dst, self.index);
Self::put_sized_string(&mut dst, field_name);
Self::put_sized_string(&mut dst, tag);
Self::put_sized_string(&mut dst, key);
dst
}
#[inline]
pub fn construct_numeric_field_data(&self, num: f64, key: &str) -> Vec<u8> {
let field_name = self.field.unwrap_or("");
let mut dst = Vec::with_capacity(
1 + self.ns.len() + 1 + 4 + self.index.len() + 4 + field_name.len() + 8 + 4 + key.len(),
);
Self::put_namespace(&mut dst, self.ns);
Self::put_type(&mut dst, SearchSubkeyType::Field);
Self::put_sized_string(&mut dst, self.index);
Self::put_sized_string(&mut dst, field_name);
dst.extend_from_slice(&encode_sortable_f64_u64(num).to_be_bytes());
Self::put_sized_string(&mut dst, key);
dst
}
#[inline]
pub fn construct_hnsw_level_node_prefix(&self, level: u16) -> Vec<u8> {
let field_name = self.field.unwrap_or("");
let mut dst = Vec::with_capacity(
1 + self.ns.len() + 1 + 4 + self.index.len() + 4 + field_name.len() + 2 + 1,
);
Self::put_hnsw_level_node_prefix(&mut dst, self.ns, self.index, field_name, level);
dst
}
#[inline]
pub fn construct_hnsw_node(&self, level: u16, key: &str) -> Vec<u8> {
let field_name = self.field.unwrap_or("");
let mut dst = Vec::with_capacity(
1 + self.ns.len() + 1 + 4 + self.index.len() + 4 + field_name.len() + 2 + 1 + 4 + key.len(),
);
Self::put_hnsw_level_node_prefix(&mut dst, self.ns, self.index, field_name, level);
Self::put_sized_string(&mut dst, key);
dst
}
#[inline]
pub fn construct_hnsw_edge_with_single_end(&self, level: u16, key: &str) -> Vec<u8> {
let field_name = self.field.unwrap_or("");
let mut dst = Vec::with_capacity(
1 + self.ns.len() + 1 + 4 + self.index.len() + 4 + field_name.len() + 2 + 1 + 4 + key.len(),
);
Self::put_hnsw_level_edge_prefix(&mut dst, self.ns, self.index, field_name, level);
Self::put_sized_string(&mut dst, key);
dst
}
#[inline]
pub fn construct_hnsw_edge(&self, level: u16, key1: &str, key2: &str) -> Vec<u8> {
let field_name = self.field.unwrap_or("");
let mut dst = Vec::with_capacity(
1 + self.ns.len()
+ 1
+ 4
+ self.index.len()
+ 4
+ field_name.len()
+ 2
+ 1
+ 4
+ key1.len()
+ 4
+ key2.len(),
);
Self::put_hnsw_level_edge_prefix(&mut dst, self.ns, self.index, field_name, level);
Self::put_sized_string(&mut dst, key1);
Self::put_sized_string(&mut dst, key2);
dst
}
}
#[inline]
pub fn encode_index_meta(on_data_type: IndexOnDataType) -> Vec<u8> {
vec![0u8, on_data_type as u8]
}
#[inline]
pub fn decode_index_meta(slice: &[u8]) -> Result<IndexOnDataType> {
if slice.len() < 2 {
return Err(Error::invalid_data(
"insufficient length while decoding metadata",
));
}
match slice[1] {
2 => Ok(IndexOnDataType::Hash),
10 => Ok(IndexOnDataType::Json),
other => Err(Error::invalid_data(format!(
"unknown on_data_type: {other}"
))),
}
}
pub fn encode_index_prefixes(prefixes: &[&str]) -> Vec<u8> {
let total_cap: usize = prefixes.iter().map(|p| 4 + p.len()).sum();
let mut dst = Vec::with_capacity(total_cap);
for prefix in prefixes {
dst.extend_from_slice(&(prefix.len() as u32).to_be_bytes());
dst.extend_from_slice(prefix.as_bytes());
}
dst
}
pub fn decode_index_prefixes(mut slice: &[u8]) -> Result<Vec<String>> {
let mut prefixes = Vec::new();
while slice.len() >= 4 {
let len = u32::from_be_bytes([slice[0], slice[1], slice[2], slice[3]]) as usize;
slice = &slice[4..];
if slice.len() < len {
return Err(Error::invalid_data(
"insufficient length while decoding index prefixes",
));
}
let prefix_str = str::from_utf8(&slice[..len])
.map_err(|_| Error::invalid_data("invalid utf-8 string in index prefixes"))?;
prefixes.push(prefix_str.to_string());
slice = &slice[len..];
}
Ok(prefixes)
}
#[inline]
pub fn encode_tag_field_meta(separator: char, case_sensitive: bool, noindex: bool) -> Vec<u8> {
let flag = (noindex as u8) | ((IndexFieldType::Tag as u8) << 1);
vec![flag, separator as u8, case_sensitive as u8]
}
#[inline]
pub fn decode_tag_field_meta(slice: &[u8]) -> Result<(char, bool, bool)> {
if slice.len() < 3 {
return Err(Error::invalid_data(
"insufficient length while decoding tag field metadata",
));
}
let flag = slice[0];
let noindex = (flag & 1) != 0;
let separator = slice[1] as char;
let case_sensitive = slice[2] != 0;
Ok((separator, case_sensitive, noindex))
}
#[inline]
pub fn encode_numeric_field_meta(noindex: bool) -> Vec<u8> {
let flag = (noindex as u8) | ((IndexFieldType::Numeric as u8) << 1);
vec![flag]
}
#[inline]
pub fn decode_numeric_field_meta(slice: &[u8]) -> Result<bool> {
if slice.is_empty() {
return Err(Error::invalid_data(
"insufficient length while decoding numeric field metadata",
));
}
let flag = slice[0];
let noindex = (flag & 1) != 0;
Ok(noindex)
}
pub const HNSW_VECTOR_FIELD_META_LEN: usize = 29;
pub fn encode_hnsw_vector_field_meta(meta: &VectorFieldMetadata, noindex: bool) -> Vec<u8> {
let flag = (noindex as u8) | ((IndexFieldType::Vector as u8) << 1);
let mut dst = Vec::with_capacity(HNSW_VECTOR_FIELD_META_LEN);
dst.push(flag);
dst.push(meta.vector_type as u8);
dst.extend_from_slice(&(meta.dim as u16).to_be_bytes());
dst.push(meta.distance_metric as u8);
dst.extend_from_slice(&(meta.initial_cap as u32).to_be_bytes());
dst.extend_from_slice(&(meta.m as u16).to_be_bytes());
dst.extend_from_slice(&(meta.ef_construction as u32).to_be_bytes());
dst.extend_from_slice(&(meta.ef_runtime as u32).to_be_bytes());
dst.extend_from_slice(&encode_sortable_f64_u64(meta.epsilon).to_be_bytes());
dst.extend_from_slice(&meta.num_levels.to_be_bytes());
dst
}
pub fn decode_hnsw_vector_field_meta(slice: &[u8]) -> Result<(VectorFieldMetadata, bool)> {
if slice.len() < HNSW_VECTOR_FIELD_META_LEN {
return Err(Error::invalid_data(
"insufficient length while decoding hnsw vector field metadata",
));
}
let flag = slice[0];
let noindex = (flag & 1) != 0;
let vector_type = match slice[1] {
1 => VectorType::Float64,
2 => VectorType::Float32,
_ => VectorType::Float64,
};
let dim = u16::from_be_bytes([slice[2], slice[3]]) as usize;
let distance_metric = match slice[4] {
0 => DistanceMetric::L2,
1 => DistanceMetric::IP,
2 => DistanceMetric::Cosine,
_ => DistanceMetric::Cosine,
};
let initial_cap = u32::from_be_bytes([slice[5], slice[6], slice[7], slice[8]]) as usize;
let m = u16::from_be_bytes([slice[9], slice[10]]) as usize;
let ef_construction = u32::from_be_bytes([slice[11], slice[12], slice[13], slice[14]]) as usize;
let ef_runtime = u32::from_be_bytes([slice[15], slice[16], slice[17], slice[18]]) as usize;
let epsilon_u64 = u64::from_be_bytes([
slice[19], slice[20], slice[21], slice[22], slice[23], slice[24], slice[25], slice[26],
]);
let epsilon = decode_sortable_f64_u64(epsilon_u64);
let num_levels = u16::from_be_bytes([slice[27], slice[28]]);
let meta = VectorFieldMetadata {
vector_type,
dim,
distance_metric,
algorithm: VectorAlgorithm::Hnsw,
initial_cap,
m,
ef_construction,
ef_runtime,
epsilon,
num_levels,
};
Ok((meta, noindex))
}
pub fn encode_hnsw_node_meta(num_neighbours: u16, vector: &[f64]) -> Vec<u8> {
let mut dst = Vec::with_capacity(4 + vector.len() * 8);
dst.extend_from_slice(&num_neighbours.to_be_bytes());
dst.extend_from_slice(&(vector.len() as u16).to_be_bytes());
for &element in vector {
dst.extend_from_slice(&encode_sortable_f64_u64(element).to_be_bytes());
}
dst
}
pub fn decode_hnsw_node_meta(slice: &[u8]) -> Result<(u16, Vec<f64>)> {
if slice.len() < 4 {
return Err(Error::invalid_data(
"insufficient length while decoding hnsw node metadata",
));
}
let num_neighbours = u16::from_be_bytes([slice[0], slice[1]]);
let dim = u16::from_be_bytes([slice[2], slice[3]]) as usize;
if slice.len() != 4 + dim * 8 {
return Err(Error::invalid_data(
"length is too short or too long to be parsed as a vector",
));
}
let mut vec = Vec::with_capacity(dim);
for chunk in slice[4..].as_chunks::<8>().0 {
let u = u64::from_be_bytes(*chunk);
vec.push(decode_sortable_f64_u64(u));
}
Ok((num_neighbours, vec))
}
pub use crate::meta::{decode_sortable_f64_u64, encode_sortable_f64_u64};
#[inline]
pub fn encode_sortable_f64(val: f64) -> String {
let encoded = encode_sortable_f64_u64(val);
let bytes = u64_to_hex_16(encoded);
unsafe { str::from_utf8_unchecked(&bytes) }.to_string()
}
#[inline]
pub fn decode_sortable_f64(hex_str: &str) -> Option<f64> {
let sortable = decode_hex_u64(hex_str.as_bytes())?;
Some(decode_sortable_f64_u64(sortable))
}
#[inline]
pub fn encode_sortable_i64(val: i64) -> String {
let unsigned = (val as u64) ^ (1 << 63);
let bytes = u64_to_hex_16(unsigned);
unsafe { str::from_utf8_unchecked(&bytes) }.to_string()
}
#[inline]
pub fn decode_sortable_i64(hex_str: &str) -> Option<i64> {
let unsigned = decode_hex_u64(hex_str.as_bytes())?;
Some((unsigned ^ (1 << 63)) as i64)
}
#[inline]
pub fn compute_sq8_distance(q: &[i8], v: &[i8], metric: DistanceMetric) -> Result<f64> {
if q.len() != v.len() {
let len1 = q.len();
let len2 = v.len();
return Err(Error::invalid_data(format!(
"sq8 vector dimension mismatch: {len1} vs {len2}"
)));
}
if q.is_empty() {
return Err(Error::invalid_data("empty vector is invalid"));
}
match metric {
DistanceMetric::L2 => {
if let Some(sq) = <i8 as SpatialSimilarity>::sqeuclidean(q, v) {
Ok(sq.max(0.0).sqrt())
} else {
let mut sum = 0u64;
for (&a, &b) in q.iter().zip(v.iter()) {
let diff = (a as i32) - (b as i32);
sum += (diff * diff) as u64;
}
Ok((sum as f64).sqrt())
}
}
DistanceMetric::IP => {
if let Some(dot) = <i8 as SpatialSimilarity>::dot(q, v) {
Ok(-dot)
} else {
let mut dot = 0i64;
for (&a, &b) in q.iter().zip(v.iter()) {
dot += (a as i64) * (b as i64);
}
Ok(-(dot as f64))
}
}
DistanceMetric::Cosine => {
if let Some(cos) = <i8 as SpatialSimilarity>::cosine(q, v) {
Ok(cos)
} else {
let mut dot = 0i64;
let mut sum_sq1 = 0i64;
let mut sum_sq2 = 0i64;
for (&a, &b) in q.iter().zip(v.iter()) {
let a = a as i64;
let b = b as i64;
dot += a * b;
sum_sq1 += a * a;
sum_sq2 += b * b;
}
let denom = ((sum_sq1 * sum_sq2) as f64).sqrt();
if denom == 0.0 || !denom.is_finite() {
return Ok(1.0);
}
let cos = ((dot as f64) / denom).clamp(-1.0, 1.0);
Ok(1.0 - cos)
}
}
}
}
#[inline]
pub fn compute_vector_distance(v1: &[f64], v2: &[f64], metric: DistanceMetric) -> Result<f64> {
if v1.len() != v2.len() {
let len1 = v1.len();
let len2 = v2.len();
return Err(Error::invalid_data(format!(
"vector dimension mismatch: {len1} vs {len2}"
)));
}
if v1.is_empty() {
return Err(Error::invalid_data("empty vector is invalid"));
}
match metric {
DistanceMetric::L2 => {
if let Some(sq) = <f64 as SpatialSimilarity>::sqeuclidean(v1, v2) {
Ok(sq.max(0.0).sqrt())
} else {
let (c1, r1) = v1.as_chunks::<8>();
let (c2, r2) = v2.as_chunks::<8>();
let mut acc0 = 0.0;
let mut acc1 = 0.0;
let mut acc2 = 0.0;
let mut acc3 = 0.0;
let mut acc4 = 0.0;
let mut acc5 = 0.0;
let mut acc6 = 0.0;
let mut acc7 = 0.0;
for (a, b) in c1.iter().zip(c2.iter()) {
let d0 = a[0] - b[0];
let d1 = a[1] - b[1];
let d2 = a[2] - b[2];
let d3 = a[3] - b[3];
let d4 = a[4] - b[4];
let d5 = a[5] - b[5];
let d6 = a[6] - b[6];
let d7 = a[7] - b[7];
acc0 += d0 * d0;
acc1 += d1 * d1;
acc2 += d2 * d2;
acc3 += d3 * d3;
acc4 += d4 * d4;
acc5 += d5 * d5;
acc6 += d6 * d6;
acc7 += d7 * d7;
}
let mut sum = ((acc0 + acc1) + (acc2 + acc3)) + ((acc4 + acc5) + (acc6 + acc7));
for (&a, &b) in r1.iter().zip(r2.iter()) {
let diff = a - b;
sum += diff * diff;
}
Ok(sum.sqrt())
}
}
DistanceMetric::IP => {
if let Some(dot) = <f64 as SpatialSimilarity>::dot(v1, v2) {
Ok(-dot)
} else {
let (c1, r1) = v1.as_chunks::<8>();
let (c2, r2) = v2.as_chunks::<8>();
let mut dot0 = 0.0;
let mut dot1 = 0.0;
let mut dot2 = 0.0;
let mut dot3 = 0.0;
let mut dot4 = 0.0;
let mut dot5 = 0.0;
let mut dot6 = 0.0;
let mut dot7 = 0.0;
for (a, b) in c1.iter().zip(c2.iter()) {
dot0 += a[0] * b[0];
dot1 += a[1] * b[1];
dot2 += a[2] * b[2];
dot3 += a[3] * b[3];
dot4 += a[4] * b[4];
dot5 += a[5] * b[5];
dot6 += a[6] * b[6];
dot7 += a[7] * b[7];
}
let mut dot = ((dot0 + dot1) + (dot2 + dot3)) + ((dot4 + dot5) + (dot6 + dot7));
for (&a, &b) in r1.iter().zip(r2.iter()) {
dot += a * b;
}
Ok(-dot)
}
}
DistanceMetric::Cosine => {
if let Some(cos) = <f64 as SpatialSimilarity>::cosine(v1, v2) {
Ok(cos)
} else {
let (c1, r1) = v1.as_chunks::<8>();
let (c2, r2) = v2.as_chunks::<8>();
let mut dot0 = 0.0;
let mut dot1 = 0.0;
let mut dot2 = 0.0;
let mut dot3 = 0.0;
let mut n1_0 = 0.0;
let mut n1_1 = 0.0;
let mut n1_2 = 0.0;
let mut n1_3 = 0.0;
let mut n2_0 = 0.0;
let mut n2_1 = 0.0;
let mut n2_2 = 0.0;
let mut n2_3 = 0.0;
for (a, b) in c1.iter().zip(c2.iter()) {
dot0 += a[0] * b[0] + a[1] * b[1];
dot1 += a[2] * b[2] + a[3] * b[3];
dot2 += a[4] * b[4] + a[5] * b[5];
dot3 += a[6] * b[6] + a[7] * b[7];
n1_0 += a[0] * a[0] + a[1] * a[1];
n1_1 += a[2] * a[2] + a[3] * a[3];
n1_2 += a[4] * a[4] + a[5] * a[5];
n1_3 += a[6] * a[6] + a[7] * a[7];
n2_0 += b[0] * b[0] + b[1] * b[1];
n2_1 += b[2] * b[2] + b[3] * b[3];
n2_2 += b[4] * b[4] + b[5] * b[5];
n2_3 += b[6] * b[6] + b[7] * b[7];
}
let mut dot = (dot0 + dot1) + (dot2 + dot3);
let mut sum_sq1 = (n1_0 + n1_1) + (n1_2 + n1_3);
let mut sum_sq2 = (n2_0 + n2_1) + (n2_2 + n2_3);
for (&a, &b) in r1.iter().zip(r2.iter()) {
dot += a * b;
sum_sq1 += a * a;
sum_sq2 += b * b;
}
let denom = (sum_sq1 * sum_sq2).sqrt();
if denom == 0.0 || !denom.is_finite() {
return Ok(1.0);
}
let cos = (dot / denom).clamp(-1.0, 1.0);
Ok(1.0 - cos)
}
}
}
}
pub fn parse_vector_from_slice(bytes: &[u8], vector_type: VectorType) -> Result<Vec<f64>> {
let mut vec = Vec::new();
parse_vector_from_slice_into(bytes, vector_type, &mut vec)?;
Ok(vec)
}
pub fn parse_vector_from_slice_into(
bytes: &[u8],
vector_type: VectorType,
out: &mut Vec<f64>,
) -> Result<()> {
if bytes.is_empty() {
return Err(Error::invalid_data("empty vector byte format"));
}
if bytes.starts_with(b"[")
&& let Ok(json_v) = sonic_rs::from_slice::<sonic_rs::Value>(bytes)
&& let Some(arr) = json_v.as_array()
{
out.clear();
out.reserve(arr.len());
for item in arr {
if let Some(n) = item.as_f64() {
out.push(n);
}
}
if !out.is_empty() {
return Ok(());
}
}
let elem_size = vector_type.byte_size();
if bytes.len().is_multiple_of(elem_size) {
match vector_type {
VectorType::Float64 => {
let count = bytes.len() / 8;
out.clear();
out.reserve(count);
for chunk in bytes.as_chunks::<8>().0 {
out.push(f64::from_le_bytes(*chunk));
}
return Ok(());
}
VectorType::Float32 => {
let count = bytes.len() / 4;
out.clear();
out.reserve(count);
for chunk in bytes.as_chunks::<4>().0 {
out.push(f32::from_le_bytes(*chunk) as f64);
}
return Ok(());
}
}
}
if let Ok(s) = str::from_utf8(bytes)
&& s.contains(',')
{
let clean = s.trim().trim_start_matches('[').trim_end_matches(']');
out.clear();
let mut valid = true;
for part in clean.split(',') {
let p = part.trim();
if p.is_empty() {
continue;
}
if let Ok(num) = p.parse::<f64>() {
out.push(num);
} else {
valid = false;
break;
}
}
if valid && !out.is_empty() {
return Ok(());
}
}
Err(Error::invalid_data("invalid vector byte format or length"))
}