Skip to main content

mnemo_engine/
embed.rs

1//! Embedder trait + text-side convenience paths (Phase 3, PR 10a).
2//!
3//! Mnemo stores vectors; **embedders** are the thing that turns text
4//! into those vectors. This module adds a small trait that any Rust
5//! embedder (hand-written, feature-gated `fastembed`, or a bound Python
6//! callable) can implement, plus the two convenience paths every
7//! downstream caller actually wants:
8//!
9//! - [`Mnemo::remember_text`] — embed a string and `remember` the result.
10//! - [`RecallRequest::from_text`] — embed a query string and build a
11//!   `RecallRequest`.
12//!
13//! Both validate the embedder's dimensionality against the database's
14//! configured width ([`MnemoError::DimensionMismatch`]) and compare the
15//! embedder's identity (name, dimensions, normalize) to the manifest
16//! (`EmbedderIdentity::MISMATCH_MSG`). An identity mismatch is a loud
17//! error — mixing two embedders in one file silently corrupts recall
18//! quality, so we refuse rather than warn.
19//!
20//! The identity check lives on the handle via [`Mnemo::set_embedder`]:
21//! one call registers the embedder, performs the identity check once,
22//! and upserts the manifest's `metadata.embedder = { name, dimensions,
23//! normalize }` nested object on first registration. Subsequent
24//! `remember_text`/`from_text` calls on the handle are a plain
25//! delegation.
26//!
27//! **Manifest schema (canonical, docs-matching):** the embedder
28//! identity lives at `metadata.embedder.{name,dimensions,normalize}`
29//! on any onboarding/manifest memory — the shape `test/scripts/seed.json`
30//! writes and `mnemo/README.md` documents. For back-compat with the
31//! very first PR 10a dev builds that wrote flat
32//! `embedder_name`/`embedder_dimensions`/`embedder_normalize` keys,
33//! those are still accepted on read as a legacy alias; writes are
34//! always the nested form.
35
36use crate::error::{MnemoError, Result};
37use serde_json::{Map, Value};
38
39/// Describe an embedder's observable identity. Used both in-memory on
40/// an open [`Mnemo`] handle and persisted at `metadata.embedder` on the
41/// scaffold manifest so that reopening with a different embedder is
42/// caught at `set_embedder` time instead of silently polluting the
43/// index.
44///
45/// Build one with a struct literal ending in `..Default::default()`, so a
46/// field added later (as `query_convention` was in v0.15) doesn't break it.
47#[derive(Clone, Debug, Default, PartialEq, Eq)]
48pub struct EmbedderIdentity {
49    /// Model identifier the caller chose. Convention: a short, stable
50    /// slug — `"all-MiniLM-L6-v2"`, `"text-embedding-3-small"`.
51    pub name: String,
52    /// Vector width this embedder emits. Validated against
53    /// [`Mnemo::dimensions`] on registration.
54    pub dimensions: usize,
55    /// Whether the embedder returns unit-normalized vectors. Caller's
56    /// claim — mnemo doesn't verify it, but records it on the manifest
57    /// so downstream readers can tell whether a dot product is cosine
58    /// equivalent.
59    pub normalize: bool,
60    /// 12.3 step 4 (v0.15): how [`Embedder::embed_query`] differs from
61    /// [`Embedder::embed`]. Empty (the default) when queries are embedded
62    /// like documents; otherwise a short, stable name for the convention,
63    /// such as `"bge-query-instruction"` for a model that prefixes queries
64    /// with an instruction. It's part of the identity: a file embedded one
65    /// way refuses an embedder that queries another way
66    /// ([`MnemoError::EmbedderMismatch`]); `mnemo reembed` moves a file to a
67    /// new identity. Stored only when it isn't empty, so a symmetric
68    /// embedder records what it did before v0.15. Builds before v0.15
69    /// ignore it.
70    pub query_convention: String,
71}
72
73impl EmbedderIdentity {
74    /// Message rendered inside [`MnemoError::Invalid`] on an identity
75    /// mismatch. Public so tests + binding layers can assert on it.
76    pub const MISMATCH_MSG: &'static str = "embedder identity mismatch";
77
78    /// The identity as `(name=…, dimensions=…, normalize=…)`, with
79    /// `query_convention=…` when it isn't empty.
80    pub(crate) fn describe(&self) -> String {
81        let mut s = format!(
82            "(name={:?}, dimensions={}, normalize={}",
83            self.name, self.dimensions, self.normalize
84        );
85        if !self.query_convention.is_empty() {
86            s.push_str(&format!(", query_convention={:?}", self.query_convention));
87        }
88        s.push(')');
89        s
90    }
91}
92
93/// Minimal trait any embedder implementation must satisfy. Returns
94/// vectors in the same order as the input `texts`.
95///
96/// Implementations are expected to be cheap to clone / `Send + Sync`
97/// when they wrap thread-safe state; the engine holds an `Arc<dyn
98/// Embedder>` on the handle via [`Mnemo::set_embedder`]. See
99/// [`Mnemo::remember_text`] for the full user-facing contract.
100pub trait Embedder: Send + Sync {
101    /// This embedder's observable identity (name, dims, normalize).
102    /// Must be stable for the lifetime of the embedder — mnemo caches
103    /// it on `set_embedder` and does not re-query per call.
104    fn identity(&self) -> EmbedderIdentity;
105
106    /// Embed `texts`, returning one vector per input in the same order.
107    /// Caller-level errors (network failure for a hosted embedder, OOM
108    /// in a local model) should bubble up as [`MnemoError::Invalid`]
109    /// with a short message — the specific crypto / page errors in
110    /// [`MnemoError`] are reserved for the engine's own concerns.
111    ///
112    /// This is the **document side**: everything stored is embedded with it.
113    fn embed(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>>;
114
115    /// 12.3 step 4 (v0.15): embed `texts` as **queries**. Every recall by
116    /// text uses it ([`crate::RecallRequest::from_text`], the CLI's
117    /// `recall --text`, the MCP `recall` tool, Python's `recall_text`). It
118    /// defaults to [`Embedder::embed`]. A model that embeds queries
119    /// differently (an instruction prefix, a separate query encoder)
120    /// overrides it and names the difference in
121    /// [`EmbedderIdentity::query_convention`].
122    fn embed_query(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
123        self.embed(texts)
124    }
125
126    /// 12.3 step 5 (v0.15): the keywords a recall by text scores with BM25
127    /// in place of its text, when the caller gave none. `None` (the
128    /// default) scores the text itself, at
129    /// [`crate::RecallRequest::text_keyword_weight`]. `Some(k)` scores `k`
130    /// as explicit keywords, at [`crate::RecallRequest::keyword_weight`];
131    /// `Some("")` makes the recall vector-only.
132    ///
133    /// The handle's registered embedder ([`crate::Mnemo::set_embedder`]) is
134    /// asked on every recall that has a text and no keywords, on a v9 file.
135    /// Keywords the caller gives always win. The built-in embedder answers
136    /// with the identifiers in the question (identifier-aware recall; see
137    /// `mnemo_engine::builtin`).
138    fn query_keywords(&self, text: &str) -> Option<String> {
139        let _ = text;
140        None
141    }
142}
143
144/// A boxed embedder is an embedder, so a caller can choose one at run time
145/// (the CLI's `--embed-cmd` or the built-in) and register it.
146impl<E: Embedder + ?Sized> Embedder for Box<E> {
147    fn identity(&self) -> EmbedderIdentity {
148        (**self).identity()
149    }
150    fn embed(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
151        (**self).embed(texts)
152    }
153    fn embed_query(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
154        (**self).embed_query(texts)
155    }
156    fn query_keywords(&self, text: &str) -> Option<String> {
157        (**self).query_keywords(text)
158    }
159}
160
161/// A shared embedder is an embedder, so one loaded model can serve several
162/// handles.
163impl<E: Embedder + ?Sized> Embedder for std::sync::Arc<E> {
164    fn identity(&self) -> EmbedderIdentity {
165        (**self).identity()
166    }
167    fn embed(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
168        (**self).embed(texts)
169    }
170    fn embed_query(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
171        (**self).embed_query(texts)
172    }
173    fn query_keywords(&self, text: &str) -> Option<String> {
174        (**self).query_keywords(text)
175    }
176}
177
178/// The JSON key of [`EmbedderIdentity::query_convention`] in both stored
179/// identities, written only when it isn't empty.
180pub(crate) const QUERY_CONVENTION_KEY: &str = "query_convention";
181
182/// The query convention from a stored identity object: empty when absent.
183/// A present key that isn't a string makes the identity malformed.
184fn convention_from(obj: &Map<String, Value>) -> Option<String> {
185    match obj.get(QUERY_CONVENTION_KEY) {
186        None => Some(String::new()),
187        Some(v) => v.as_str().map(str::to_string),
188    }
189}
190
191/// Returns `true` iff every component of `v` is exactly `0.0`. This is
192/// the predicate `remember` uses to refuse all-zero vectors (which
193/// score 0 on every cosine query and pollute recall).
194pub(crate) fn is_zero_vector(v: &[f32]) -> bool {
195    !v.is_empty() && v.iter().all(|x| *x == 0.0)
196}
197
198/// Canonical nested metadata key: `metadata.embedder = { name,
199/// dimensions, normalize }`. This is the shape `test/scripts/seed.json`
200/// writes and `mnemo/README.md` documents.
201pub(crate) const MANIFEST_EMBEDDER_KEY: &str = "embedder";
202
203/// Legacy flat keys written by the very first PR 10a dev builds
204/// (never released). Still accepted on read so a locally-bootstrapped
205/// repo doesn't trip the new check; writes are always the nested form
206/// above.
207pub(crate) const LEGACY_FLAT_NAME: &str = "embedder_name";
208pub(crate) const LEGACY_FLAT_DIMENSIONS: &str = "embedder_dimensions";
209pub(crate) const LEGACY_FLAT_NORMALIZE: &str = "embedder_normalize";
210
211/// Read the recorded embedder identity off a memory's metadata map.
212/// Returns `None` if neither the canonical nested form nor the legacy
213/// flat form is present (or either is present but malformed — missing
214/// a required field, wrong type).
215///
216/// Nested form (canonical) is tried first; legacy flat form is only
217/// consulted when nothing nested is found. This keeps a seed.json-
218/// shaped file hitting the canonical path even if a bugged earlier
219/// `set_embedder` call left stale flat keys alongside.
220pub(crate) fn read_identity_from_metadata(meta: &Map<String, Value>) -> Option<EmbedderIdentity> {
221    // Canonical nested form.
222    if let Some(obj) = meta.get(MANIFEST_EMBEDDER_KEY).and_then(Value::as_object) {
223        let name = obj.get("name").and_then(Value::as_str)?;
224        let dims = obj.get("dimensions").and_then(Value::as_u64)?;
225        let norm = obj.get("normalize").and_then(Value::as_bool)?;
226        return Some(EmbedderIdentity {
227            name: name.to_string(),
228            dimensions: dims as usize,
229            normalize: norm,
230            query_convention: convention_from(obj)?,
231        });
232    }
233    // Legacy flat fallback.
234    let name = meta.get(LEGACY_FLAT_NAME).and_then(Value::as_str)?;
235    let dims = meta.get(LEGACY_FLAT_DIMENSIONS).and_then(Value::as_u64)?;
236    let norm = meta.get(LEGACY_FLAT_NORMALIZE).and_then(Value::as_bool)?;
237    Some(EmbedderIdentity {
238        name: name.to_string(),
239        dimensions: dims as usize,
240        normalize: norm,
241        query_convention: String::new(),
242    })
243}
244
245/// Write (or overwrite) the embedder identity on a memory's metadata
246/// map, in the canonical nested form. Also removes any legacy flat
247/// keys from the same map so the upsert doesn't leave duplicates
248/// behind.
249pub(crate) fn write_identity_into_metadata(
250    meta: &mut Map<String, Value>,
251    identity: &EmbedderIdentity,
252) {
253    let mut obj = Map::new();
254    obj.insert("name".into(), Value::String(identity.name.clone()));
255    obj.insert(
256        "dimensions".into(),
257        Value::Number(serde_json::Number::from(identity.dimensions as u64)),
258    );
259    obj.insert("normalize".into(), Value::Bool(identity.normalize));
260    if !identity.query_convention.is_empty() {
261        obj.insert(
262            QUERY_CONVENTION_KEY.into(),
263            Value::String(identity.query_convention.clone()),
264        );
265    }
266    meta.insert(MANIFEST_EMBEDDER_KEY.into(), Value::Object(obj));
267    // Drop legacy flat keys if present so the manifest canonicalizes
268    // on the next flush. Harmless no-op when they're absent.
269    meta.remove(LEGACY_FLAT_NAME);
270    meta.remove(LEGACY_FLAT_DIMENSIONS);
271    meta.remove(LEGACY_FLAT_NORMALIZE);
272}
273
274/// The error for an embedder that isn't the one the file records
275/// ([`MnemoError::EmbedderMismatch`]). Its message starts with
276/// [`EmbedderIdentity::MISMATCH_MSG`], so callers that grep the string
277/// (Python's `RuntimeError`) keep working.
278pub(crate) fn identity_mismatch_err(
279    stored: &EmbedderIdentity,
280    given: &EmbedderIdentity,
281) -> MnemoError {
282    MnemoError::EmbedderMismatch {
283        stored: Box::new(stored.clone()),
284        given: Box::new(given.clone()),
285    }
286}
287
288/// EMBED-ID-1: the reserved result-cache namespace that holds a file's
289/// embedder identity. Every file (manifest or not) records the first
290/// embedder attached to it here. The public cache API refuses
291/// [`RESERVED_CACHE_PREFIX`] namespaces and leaves them out of its stats
292/// and purges; compaction, the rekeyed exports and the import copy the
293/// entry like any other cache entry, so the identity follows the file.
294pub(crate) const IDENTITY_NAMESPACE: &str = "_mnemo.embedder";
295/// The key of the identity entry in [`IDENTITY_NAMESPACE`].
296pub(crate) const IDENTITY_KEY: &str = "identity";
297/// Cache namespaces with this prefix belong to the engine.
298pub(crate) const RESERVED_CACHE_PREFIX: &str = "_mnemo.";
299
300/// Whether `namespace` is one of the engine's own cache namespaces.
301pub(crate) fn is_reserved_namespace(namespace: &str) -> bool {
302    namespace.starts_with(RESERVED_CACHE_PREFIX)
303}
304
305/// The identity as the JSON the identity entry stores.
306pub(crate) fn identity_to_json(identity: &EmbedderIdentity) -> Vec<u8> {
307    let mut v = serde_json::json!({
308        "name": identity.name,
309        "dimensions": identity.dimensions as u64,
310        "normalize": identity.normalize,
311    });
312    if !identity.query_convention.is_empty() {
313        v[QUERY_CONVENTION_KEY] = Value::String(identity.query_convention.clone());
314    }
315    serde_json::to_vec(&v).expect("a JSON object of strings, a number and a bool serializes")
316}
317
318/// The identity from the identity entry's JSON, or `None` if it's
319/// malformed.
320pub(crate) fn identity_from_json(bytes: &[u8]) -> Option<EmbedderIdentity> {
321    let v: Value = serde_json::from_slice(bytes).ok()?;
322    let obj = v.as_object()?;
323    Some(EmbedderIdentity {
324        name: obj.get("name")?.as_str()?.to_string(),
325        dimensions: obj.get("dimensions")?.as_u64()? as usize,
326        normalize: obj.get("normalize")?.as_bool()?,
327        query_convention: convention_from(obj)?,
328    })
329}
330
331/// The error for a file whose manifest and identity entry disagree (a
332/// file changed by hand, or by two builds that saw different embedders).
333/// It names both; neither embedder is attached.
334pub(crate) fn identity_conflict_err(
335    manifest: &EmbedderIdentity,
336    record: &EmbedderIdentity,
337) -> MnemoError {
338    MnemoError::Invalid(format!(
339        "{}: this file's manifest records embedder {}, but its identity record holds {}; \
340         fix the manifest (`mnemo update`) to match the embedder its vectors came from",
341        EmbedderIdentity::MISMATCH_MSG,
342        manifest.describe(),
343        record.describe(),
344    ))
345}
346
347#[cfg(test)]
348mod tests {
349    use super::*;
350
351    #[test]
352    fn is_zero_vector_detects_all_zero() {
353        assert!(is_zero_vector(&[0.0, 0.0, 0.0]));
354        assert!(!is_zero_vector(&[0.0, 0.0, 1e-9]));
355        assert!(!is_zero_vector(&[]));
356    }
357
358    #[test]
359    fn read_identity_prefers_nested_over_legacy() {
360        // Canonical nested form.
361        let json = serde_json::json!({
362            "area": "onboarding",
363            "embedder": { "name": "nested", "dimensions": 384, "normalize": true },
364            // Garbage legacy keys — must be ignored while the nested
365            // form is present.
366            "embedder_name": "legacy",
367            "embedder_dimensions": 999,
368            "embedder_normalize": false,
369        });
370        let obj = json.as_object().unwrap().clone();
371        let id = read_identity_from_metadata(&obj).expect("nested identity read");
372        assert_eq!(id.name, "nested");
373        assert_eq!(id.dimensions, 384);
374        assert!(id.normalize);
375    }
376
377    #[test]
378    fn read_identity_falls_back_to_legacy_flat() {
379        let json = serde_json::json!({
380            "embedder_name": "legacy",
381            "embedder_dimensions": 128,
382            "embedder_normalize": false,
383        });
384        let obj = json.as_object().unwrap().clone();
385        let id = read_identity_from_metadata(&obj).expect("legacy identity read");
386        assert_eq!(id.name, "legacy");
387        assert_eq!(id.dimensions, 128);
388        assert!(!id.normalize);
389    }
390
391    #[test]
392    fn read_identity_none_when_absent_or_malformed() {
393        // Empty.
394        assert!(read_identity_from_metadata(&Map::new()).is_none());
395        // Nested missing a field.
396        let json = serde_json::json!({ "embedder": { "name": "x", "dimensions": 4 } });
397        let obj = json.as_object().unwrap().clone();
398        assert!(read_identity_from_metadata(&obj).is_none());
399    }
400
401    #[test]
402    fn write_identity_canonicalizes_and_drops_legacy() {
403        let json = serde_json::json!({
404            "embedder_name": "old",
405            "embedder_dimensions": 128,
406            "embedder_normalize": false,
407            "unrelated": "keep-me",
408        });
409        let mut obj = json.as_object().unwrap().clone();
410        write_identity_into_metadata(
411            &mut obj,
412            &EmbedderIdentity {
413                name: "new".into(),
414                dimensions: 384,
415                normalize: true,
416                ..Default::default()
417            },
418        );
419        // Legacy keys gone.
420        assert!(!obj.contains_key(LEGACY_FLAT_NAME));
421        assert!(!obj.contains_key(LEGACY_FLAT_DIMENSIONS));
422        assert!(!obj.contains_key(LEGACY_FLAT_NORMALIZE));
423        // Unrelated metadata preserved.
424        assert_eq!(obj["unrelated"], serde_json::json!("keep-me"));
425        // Nested identity written correctly.
426        let nested = obj["embedder"].as_object().unwrap();
427        assert_eq!(nested["name"], serde_json::json!("new"));
428        assert_eq!(nested["dimensions"], serde_json::json!(384));
429        assert_eq!(nested["normalize"], serde_json::json!(true));
430    }
431
432    #[test]
433    fn mismatch_err_includes_both_identities() {
434        let want = EmbedderIdentity {
435            name: "A".into(),
436            dimensions: 4,
437            normalize: true,
438            ..Default::default()
439        };
440        let saw = EmbedderIdentity {
441            name: "B".into(),
442            dimensions: 4,
443            normalize: false,
444            ..Default::default()
445        };
446        let msg = format!("{}", identity_mismatch_err(&want, &saw));
447        assert!(msg.contains(EmbedderIdentity::MISMATCH_MSG));
448        assert!(msg.contains("name=\"A\""));
449        assert!(msg.contains("name=\"B\""));
450    }
451}