Skip to main content

mnemo_engine/
embed.rs

1//! Embedder trait + text-side convenience paths (Phase 3, PR 10a).
2//!
3//! Mnemo stores vectors; **embedders** are the thing that turns text
4//! into those vectors. This module adds a small trait that any Rust
5//! embedder (hand-written, feature-gated `fastembed`, or a bound Python
6//! callable) can implement, plus the two convenience paths every
7//! downstream caller actually wants:
8//!
9//! - [`Mnemo::remember_text`] — embed a string and `remember` the result.
10//! - [`RecallRequest::from_text`] — embed a query string and build a
11//!   `RecallRequest`.
12//!
13//! Both validate the embedder's dimensionality against the database's
14//! configured width ([`MnemoError::DimensionMismatch`]) and compare the
15//! embedder's identity (name, dimensions, normalize) to the manifest
16//! (`EmbedderIdentity::MISMATCH_MSG`). An identity mismatch is a loud
17//! error — mixing two embedders in one file silently corrupts recall
18//! quality, so we refuse rather than warn.
19//!
20//! The identity check lives on the handle via [`Mnemo::set_embedder`]:
21//! one call registers the embedder, performs the identity check once,
22//! and upserts the manifest's `metadata.embedder = { name, dimensions,
23//! normalize }` nested object on first registration. Subsequent
24//! `remember_text`/`from_text` calls on the handle are a plain
25//! delegation.
26//!
27//! **Manifest schema (canonical, docs-matching):** the embedder
28//! identity lives at `metadata.embedder.{name,dimensions,normalize}`
29//! on any onboarding/manifest memory — the shape `test/scripts/seed.json`
30//! writes and `mnemo/README.md` documents. For back-compat with the
31//! very first PR 10a dev builds that wrote flat
32//! `embedder_name`/`embedder_dimensions`/`embedder_normalize` keys,
33//! those are still accepted on read as a legacy alias; writes are
34//! always the nested form.
35
36use crate::error::{MnemoError, Result};
37use serde_json::{Map, Value};
38
39/// Describe an embedder's observable identity. Used both in-memory on
40/// an open [`Mnemo`] handle and persisted at `metadata.embedder` on the
41/// scaffold manifest so that reopening with a different embedder is
42/// caught at `set_embedder` time instead of silently polluting the
43/// index.
44#[derive(Clone, Debug, PartialEq, Eq)]
45pub struct EmbedderIdentity {
46    /// Model identifier the caller chose. Convention: a short, stable
47    /// slug — `"all-MiniLM-L6-v2"`, `"text-embedding-3-small"`.
48    pub name: String,
49    /// Vector width this embedder emits. Validated against
50    /// [`Mnemo::dimensions`] on registration.
51    pub dimensions: usize,
52    /// Whether the embedder returns unit-normalized vectors. Caller's
53    /// claim — mnemo doesn't verify it, but records it on the manifest
54    /// so downstream readers can tell whether a dot product is cosine
55    /// equivalent.
56    pub normalize: bool,
57}
58
59impl EmbedderIdentity {
60    /// Message rendered inside [`MnemoError::Invalid`] on an identity
61    /// mismatch. Public so tests + binding layers can assert on it.
62    pub const MISMATCH_MSG: &'static str = "embedder identity mismatch";
63}
64
65/// Minimal trait any embedder implementation must satisfy. Returns
66/// vectors in the same order as the input `texts`.
67///
68/// Implementations are expected to be cheap to clone / `Send + Sync`
69/// when they wrap thread-safe state; the engine holds an `Arc<dyn
70/// Embedder>` on the handle via [`Mnemo::set_embedder`]. See
71/// [`Mnemo::remember_text`] for the full user-facing contract.
72pub trait Embedder: Send + Sync {
73    /// This embedder's observable identity (name, dims, normalize).
74    /// Must be stable for the lifetime of the embedder — mnemo caches
75    /// it on `set_embedder` and does not re-query per call.
76    fn identity(&self) -> EmbedderIdentity;
77
78    /// Embed `texts`, returning one vector per input in the same order.
79    /// Caller-level errors (network failure for a hosted embedder, OOM
80    /// in a local model) should bubble up as [`MnemoError::Invalid`]
81    /// with a short message — the specific crypto / page errors in
82    /// [`MnemoError`] are reserved for the engine's own concerns.
83    fn embed(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>>;
84}
85
86/// Returns `true` iff every component of `v` is exactly `0.0`. This is
87/// the predicate `remember` uses to refuse all-zero vectors (which
88/// score 0 on every cosine query and pollute recall).
89pub(crate) fn is_zero_vector(v: &[f32]) -> bool {
90    !v.is_empty() && v.iter().all(|x| *x == 0.0)
91}
92
93/// Canonical nested metadata key: `metadata.embedder = { name,
94/// dimensions, normalize }`. This is the shape `test/scripts/seed.json`
95/// writes and `mnemo/README.md` documents.
96pub(crate) const MANIFEST_EMBEDDER_KEY: &str = "embedder";
97
98/// Legacy flat keys written by the very first PR 10a dev builds
99/// (never released). Still accepted on read so a locally-bootstrapped
100/// repo doesn't trip the new check; writes are always the nested form
101/// above.
102pub(crate) const LEGACY_FLAT_NAME: &str = "embedder_name";
103pub(crate) const LEGACY_FLAT_DIMENSIONS: &str = "embedder_dimensions";
104pub(crate) const LEGACY_FLAT_NORMALIZE: &str = "embedder_normalize";
105
106/// Read the recorded embedder identity off a memory's metadata map.
107/// Returns `None` if neither the canonical nested form nor the legacy
108/// flat form is present (or either is present but malformed — missing
109/// a required field, wrong type).
110///
111/// Nested form (canonical) is tried first; legacy flat form is only
112/// consulted when nothing nested is found. This keeps a seed.json-
113/// shaped file hitting the canonical path even if a bugged earlier
114/// `set_embedder` call left stale flat keys alongside.
115pub(crate) fn read_identity_from_metadata(meta: &Map<String, Value>) -> Option<EmbedderIdentity> {
116    // Canonical nested form.
117    if let Some(obj) = meta.get(MANIFEST_EMBEDDER_KEY).and_then(Value::as_object) {
118        let name = obj.get("name").and_then(Value::as_str)?;
119        let dims = obj.get("dimensions").and_then(Value::as_u64)?;
120        let norm = obj.get("normalize").and_then(Value::as_bool)?;
121        return Some(EmbedderIdentity {
122            name: name.to_string(),
123            dimensions: dims as usize,
124            normalize: norm,
125        });
126    }
127    // Legacy flat fallback.
128    let name = meta.get(LEGACY_FLAT_NAME).and_then(Value::as_str)?;
129    let dims = meta.get(LEGACY_FLAT_DIMENSIONS).and_then(Value::as_u64)?;
130    let norm = meta.get(LEGACY_FLAT_NORMALIZE).and_then(Value::as_bool)?;
131    Some(EmbedderIdentity {
132        name: name.to_string(),
133        dimensions: dims as usize,
134        normalize: norm,
135    })
136}
137
138/// Write (or overwrite) the embedder identity on a memory's metadata
139/// map, in the canonical nested form. Also removes any legacy flat
140/// keys from the same map so the upsert doesn't leave duplicates
141/// behind.
142pub(crate) fn write_identity_into_metadata(
143    meta: &mut Map<String, Value>,
144    identity: &EmbedderIdentity,
145) {
146    let mut obj = Map::new();
147    obj.insert("name".into(), Value::String(identity.name.clone()));
148    obj.insert(
149        "dimensions".into(),
150        Value::Number(serde_json::Number::from(identity.dimensions as u64)),
151    );
152    obj.insert("normalize".into(), Value::Bool(identity.normalize));
153    meta.insert(MANIFEST_EMBEDDER_KEY.into(), Value::Object(obj));
154    // Drop legacy flat keys if present so the manifest canonicalizes
155    // on the next flush. Harmless no-op when they're absent.
156    meta.remove(LEGACY_FLAT_NAME);
157    meta.remove(LEGACY_FLAT_DIMENSIONS);
158    meta.remove(LEGACY_FLAT_NORMALIZE);
159}
160
161/// The error for an embedder that isn't the one the file records
162/// ([`MnemoError::EmbedderMismatch`]). Its message starts with
163/// [`EmbedderIdentity::MISMATCH_MSG`], so callers that grep the string
164/// (Python's `RuntimeError`) keep working.
165pub(crate) fn identity_mismatch_err(
166    stored: &EmbedderIdentity,
167    given: &EmbedderIdentity,
168) -> MnemoError {
169    MnemoError::EmbedderMismatch {
170        stored: stored.clone(),
171        given: given.clone(),
172    }
173}
174
175/// EMBED-ID-1: the reserved result-cache namespace that holds a file's
176/// embedder identity. Every file (manifest or not) records the first
177/// embedder attached to it here. The public cache API refuses
178/// [`RESERVED_CACHE_PREFIX`] namespaces and leaves them out of its stats
179/// and purges; compaction, the rekeyed exports and the import copy the
180/// entry like any other cache entry, so the identity follows the file.
181pub(crate) const IDENTITY_NAMESPACE: &str = "_mnemo.embedder";
182/// The key of the identity entry in [`IDENTITY_NAMESPACE`].
183pub(crate) const IDENTITY_KEY: &str = "identity";
184/// Cache namespaces with this prefix belong to the engine.
185pub(crate) const RESERVED_CACHE_PREFIX: &str = "_mnemo.";
186
187/// Whether `namespace` is one of the engine's own cache namespaces.
188pub(crate) fn is_reserved_namespace(namespace: &str) -> bool {
189    namespace.starts_with(RESERVED_CACHE_PREFIX)
190}
191
192/// The identity as the JSON the identity entry stores.
193pub(crate) fn identity_to_json(identity: &EmbedderIdentity) -> Vec<u8> {
194    serde_json::to_vec(&serde_json::json!({
195        "name": identity.name,
196        "dimensions": identity.dimensions as u64,
197        "normalize": identity.normalize,
198    }))
199    .expect("a JSON object of a string, a number and a bool serializes")
200}
201
202/// The identity from the identity entry's JSON, or `None` if it's
203/// malformed.
204pub(crate) fn identity_from_json(bytes: &[u8]) -> Option<EmbedderIdentity> {
205    let v: Value = serde_json::from_slice(bytes).ok()?;
206    let obj = v.as_object()?;
207    Some(EmbedderIdentity {
208        name: obj.get("name")?.as_str()?.to_string(),
209        dimensions: obj.get("dimensions")?.as_u64()? as usize,
210        normalize: obj.get("normalize")?.as_bool()?,
211    })
212}
213
214/// The error for a file whose manifest and identity entry disagree (a
215/// file changed by hand, or by two builds that saw different embedders).
216/// It names both; neither embedder is attached.
217pub(crate) fn identity_conflict_err(
218    manifest: &EmbedderIdentity,
219    record: &EmbedderIdentity,
220) -> MnemoError {
221    MnemoError::Invalid(format!(
222        "{}: this file's manifest records embedder (name={:?}, dimensions={}, normalize={}), \
223         but its identity record holds (name={:?}, dimensions={}, normalize={}); \
224         fix the manifest (`mnemo update`) to match the embedder its vectors came from",
225        EmbedderIdentity::MISMATCH_MSG,
226        manifest.name,
227        manifest.dimensions,
228        manifest.normalize,
229        record.name,
230        record.dimensions,
231        record.normalize,
232    ))
233}
234
235#[cfg(test)]
236mod tests {
237    use super::*;
238
239    #[test]
240    fn is_zero_vector_detects_all_zero() {
241        assert!(is_zero_vector(&[0.0, 0.0, 0.0]));
242        assert!(!is_zero_vector(&[0.0, 0.0, 1e-9]));
243        assert!(!is_zero_vector(&[]));
244    }
245
246    #[test]
247    fn read_identity_prefers_nested_over_legacy() {
248        // Canonical nested form.
249        let json = serde_json::json!({
250            "area": "onboarding",
251            "embedder": { "name": "nested", "dimensions": 384, "normalize": true },
252            // Garbage legacy keys — must be ignored while the nested
253            // form is present.
254            "embedder_name": "legacy",
255            "embedder_dimensions": 999,
256            "embedder_normalize": false,
257        });
258        let obj = json.as_object().unwrap().clone();
259        let id = read_identity_from_metadata(&obj).expect("nested identity read");
260        assert_eq!(id.name, "nested");
261        assert_eq!(id.dimensions, 384);
262        assert!(id.normalize);
263    }
264
265    #[test]
266    fn read_identity_falls_back_to_legacy_flat() {
267        let json = serde_json::json!({
268            "embedder_name": "legacy",
269            "embedder_dimensions": 128,
270            "embedder_normalize": false,
271        });
272        let obj = json.as_object().unwrap().clone();
273        let id = read_identity_from_metadata(&obj).expect("legacy identity read");
274        assert_eq!(id.name, "legacy");
275        assert_eq!(id.dimensions, 128);
276        assert!(!id.normalize);
277    }
278
279    #[test]
280    fn read_identity_none_when_absent_or_malformed() {
281        // Empty.
282        assert!(read_identity_from_metadata(&Map::new()).is_none());
283        // Nested missing a field.
284        let json = serde_json::json!({ "embedder": { "name": "x", "dimensions": 4 } });
285        let obj = json.as_object().unwrap().clone();
286        assert!(read_identity_from_metadata(&obj).is_none());
287    }
288
289    #[test]
290    fn write_identity_canonicalizes_and_drops_legacy() {
291        let json = serde_json::json!({
292            "embedder_name": "old",
293            "embedder_dimensions": 128,
294            "embedder_normalize": false,
295            "unrelated": "keep-me",
296        });
297        let mut obj = json.as_object().unwrap().clone();
298        write_identity_into_metadata(
299            &mut obj,
300            &EmbedderIdentity {
301                name: "new".into(),
302                dimensions: 384,
303                normalize: true,
304            },
305        );
306        // Legacy keys gone.
307        assert!(!obj.contains_key(LEGACY_FLAT_NAME));
308        assert!(!obj.contains_key(LEGACY_FLAT_DIMENSIONS));
309        assert!(!obj.contains_key(LEGACY_FLAT_NORMALIZE));
310        // Unrelated metadata preserved.
311        assert_eq!(obj["unrelated"], serde_json::json!("keep-me"));
312        // Nested identity written correctly.
313        let nested = obj["embedder"].as_object().unwrap();
314        assert_eq!(nested["name"], serde_json::json!("new"));
315        assert_eq!(nested["dimensions"], serde_json::json!(384));
316        assert_eq!(nested["normalize"], serde_json::json!(true));
317    }
318
319    #[test]
320    fn mismatch_err_includes_both_identities() {
321        let want = EmbedderIdentity {
322            name: "A".into(),
323            dimensions: 4,
324            normalize: true,
325        };
326        let saw = EmbedderIdentity {
327            name: "B".into(),
328            dimensions: 4,
329            normalize: false,
330        };
331        let msg = format!("{}", identity_mismatch_err(&want, &saw));
332        assert!(msg.contains(EmbedderIdentity::MISMATCH_MSG));
333        assert!(msg.contains("name=\"A\""));
334        assert!(msg.contains("name=\"B\""));
335    }
336}