use crate::error::{MnemoError, Result};
use serde_json::{Map, Value};
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct EmbedderIdentity {
pub name: String,
pub dimensions: usize,
pub normalize: bool,
pub query_convention: String,
}
impl EmbedderIdentity {
pub const MISMATCH_MSG: &'static str = "embedder identity mismatch";
pub(crate) fn describe(&self) -> String {
let mut s = format!(
"(name={:?}, dimensions={}, normalize={}",
self.name, self.dimensions, self.normalize
);
if !self.query_convention.is_empty() {
s.push_str(&format!(", query_convention={:?}", self.query_convention));
}
s.push(')');
s
}
}
pub trait Embedder: Send + Sync {
fn identity(&self) -> EmbedderIdentity;
fn embed(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>>;
fn embed_query(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
self.embed(texts)
}
fn query_keywords(&self, text: &str) -> Option<String> {
let _ = text;
None
}
}
impl<E: Embedder + ?Sized> Embedder for Box<E> {
fn identity(&self) -> EmbedderIdentity {
(**self).identity()
}
fn embed(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
(**self).embed(texts)
}
fn embed_query(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
(**self).embed_query(texts)
}
fn query_keywords(&self, text: &str) -> Option<String> {
(**self).query_keywords(text)
}
}
impl<E: Embedder + ?Sized> Embedder for std::sync::Arc<E> {
fn identity(&self) -> EmbedderIdentity {
(**self).identity()
}
fn embed(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
(**self).embed(texts)
}
fn embed_query(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
(**self).embed_query(texts)
}
fn query_keywords(&self, text: &str) -> Option<String> {
(**self).query_keywords(text)
}
}
pub(crate) const QUERY_CONVENTION_KEY: &str = "query_convention";
fn convention_from(obj: &Map<String, Value>) -> Option<String> {
match obj.get(QUERY_CONVENTION_KEY) {
None => Some(String::new()),
Some(v) => v.as_str().map(str::to_string),
}
}
pub(crate) fn is_zero_vector(v: &[f32]) -> bool {
!v.is_empty() && v.iter().all(|x| *x == 0.0)
}
pub(crate) const MANIFEST_EMBEDDER_KEY: &str = "embedder";
pub(crate) const LEGACY_FLAT_NAME: &str = "embedder_name";
pub(crate) const LEGACY_FLAT_DIMENSIONS: &str = "embedder_dimensions";
pub(crate) const LEGACY_FLAT_NORMALIZE: &str = "embedder_normalize";
pub(crate) fn read_identity_from_metadata(meta: &Map<String, Value>) -> Option<EmbedderIdentity> {
if let Some(obj) = meta.get(MANIFEST_EMBEDDER_KEY).and_then(Value::as_object) {
let name = obj.get("name").and_then(Value::as_str)?;
let dims = obj.get("dimensions").and_then(Value::as_u64)?;
let norm = obj.get("normalize").and_then(Value::as_bool)?;
return Some(EmbedderIdentity {
name: name.to_string(),
dimensions: dims as usize,
normalize: norm,
query_convention: convention_from(obj)?,
});
}
let name = meta.get(LEGACY_FLAT_NAME).and_then(Value::as_str)?;
let dims = meta.get(LEGACY_FLAT_DIMENSIONS).and_then(Value::as_u64)?;
let norm = meta.get(LEGACY_FLAT_NORMALIZE).and_then(Value::as_bool)?;
Some(EmbedderIdentity {
name: name.to_string(),
dimensions: dims as usize,
normalize: norm,
query_convention: String::new(),
})
}
pub(crate) fn write_identity_into_metadata(
meta: &mut Map<String, Value>,
identity: &EmbedderIdentity,
) {
let mut obj = Map::new();
obj.insert("name".into(), Value::String(identity.name.clone()));
obj.insert(
"dimensions".into(),
Value::Number(serde_json::Number::from(identity.dimensions as u64)),
);
obj.insert("normalize".into(), Value::Bool(identity.normalize));
if !identity.query_convention.is_empty() {
obj.insert(
QUERY_CONVENTION_KEY.into(),
Value::String(identity.query_convention.clone()),
);
}
meta.insert(MANIFEST_EMBEDDER_KEY.into(), Value::Object(obj));
meta.remove(LEGACY_FLAT_NAME);
meta.remove(LEGACY_FLAT_DIMENSIONS);
meta.remove(LEGACY_FLAT_NORMALIZE);
}
pub(crate) fn identity_mismatch_err(
stored: &EmbedderIdentity,
given: &EmbedderIdentity,
) -> MnemoError {
MnemoError::EmbedderMismatch {
stored: Box::new(stored.clone()),
given: Box::new(given.clone()),
}
}
pub(crate) const IDENTITY_NAMESPACE: &str = "_mnemo.embedder";
pub(crate) const IDENTITY_KEY: &str = "identity";
pub(crate) const RESERVED_CACHE_PREFIX: &str = "_mnemo.";
pub(crate) fn is_reserved_namespace(namespace: &str) -> bool {
namespace.starts_with(RESERVED_CACHE_PREFIX)
}
pub(crate) fn identity_to_json(identity: &EmbedderIdentity) -> Vec<u8> {
let mut v = serde_json::json!({
"name": identity.name,
"dimensions": identity.dimensions as u64,
"normalize": identity.normalize,
});
if !identity.query_convention.is_empty() {
v[QUERY_CONVENTION_KEY] = Value::String(identity.query_convention.clone());
}
serde_json::to_vec(&v).expect("a JSON object of strings, a number and a bool serializes")
}
pub(crate) fn identity_from_json(bytes: &[u8]) -> Option<EmbedderIdentity> {
let v: Value = serde_json::from_slice(bytes).ok()?;
let obj = v.as_object()?;
Some(EmbedderIdentity {
name: obj.get("name")?.as_str()?.to_string(),
dimensions: obj.get("dimensions")?.as_u64()? as usize,
normalize: obj.get("normalize")?.as_bool()?,
query_convention: convention_from(obj)?,
})
}
pub(crate) fn identity_conflict_err(
manifest: &EmbedderIdentity,
record: &EmbedderIdentity,
) -> MnemoError {
MnemoError::Invalid(format!(
"{}: this file's manifest records embedder {}, but its identity record holds {}; \
fix the manifest (`mnemo update`) to match the embedder its vectors came from",
EmbedderIdentity::MISMATCH_MSG,
manifest.describe(),
record.describe(),
))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn is_zero_vector_detects_all_zero() {
assert!(is_zero_vector(&[0.0, 0.0, 0.0]));
assert!(!is_zero_vector(&[0.0, 0.0, 1e-9]));
assert!(!is_zero_vector(&[]));
}
#[test]
fn read_identity_prefers_nested_over_legacy() {
let json = serde_json::json!({
"area": "onboarding",
"embedder": { "name": "nested", "dimensions": 384, "normalize": true },
"embedder_name": "legacy",
"embedder_dimensions": 999,
"embedder_normalize": false,
});
let obj = json.as_object().unwrap().clone();
let id = read_identity_from_metadata(&obj).expect("nested identity read");
assert_eq!(id.name, "nested");
assert_eq!(id.dimensions, 384);
assert!(id.normalize);
}
#[test]
fn read_identity_falls_back_to_legacy_flat() {
let json = serde_json::json!({
"embedder_name": "legacy",
"embedder_dimensions": 128,
"embedder_normalize": false,
});
let obj = json.as_object().unwrap().clone();
let id = read_identity_from_metadata(&obj).expect("legacy identity read");
assert_eq!(id.name, "legacy");
assert_eq!(id.dimensions, 128);
assert!(!id.normalize);
}
#[test]
fn read_identity_none_when_absent_or_malformed() {
assert!(read_identity_from_metadata(&Map::new()).is_none());
let json = serde_json::json!({ "embedder": { "name": "x", "dimensions": 4 } });
let obj = json.as_object().unwrap().clone();
assert!(read_identity_from_metadata(&obj).is_none());
}
#[test]
fn write_identity_canonicalizes_and_drops_legacy() {
let json = serde_json::json!({
"embedder_name": "old",
"embedder_dimensions": 128,
"embedder_normalize": false,
"unrelated": "keep-me",
});
let mut obj = json.as_object().unwrap().clone();
write_identity_into_metadata(
&mut obj,
&EmbedderIdentity {
name: "new".into(),
dimensions: 384,
normalize: true,
..Default::default()
},
);
assert!(!obj.contains_key(LEGACY_FLAT_NAME));
assert!(!obj.contains_key(LEGACY_FLAT_DIMENSIONS));
assert!(!obj.contains_key(LEGACY_FLAT_NORMALIZE));
assert_eq!(obj["unrelated"], serde_json::json!("keep-me"));
let nested = obj["embedder"].as_object().unwrap();
assert_eq!(nested["name"], serde_json::json!("new"));
assert_eq!(nested["dimensions"], serde_json::json!(384));
assert_eq!(nested["normalize"], serde_json::json!(true));
}
#[test]
fn mismatch_err_includes_both_identities() {
let want = EmbedderIdentity {
name: "A".into(),
dimensions: 4,
normalize: true,
..Default::default()
};
let saw = EmbedderIdentity {
name: "B".into(),
dimensions: 4,
normalize: false,
..Default::default()
};
let msg = format!("{}", identity_mismatch_err(&want, &saw));
assert!(msg.contains(EmbedderIdentity::MISMATCH_MSG));
assert!(msg.contains("name=\"A\""));
assert!(msg.contains("name=\"B\""));
}
}