mnemo_engine/embed.rs
1//! Embedder trait + text-side convenience paths (Phase 3, PR 10a).
2//!
3//! Mnemo stores vectors; **embedders** are the thing that turns text
4//! into those vectors. This module adds a small trait that any Rust
5//! embedder (hand-written, feature-gated `fastembed`, or a bound Python
6//! callable) can implement, plus the two convenience paths every
7//! downstream caller actually wants:
8//!
9//! - [`Mnemo::remember_text`] — embed a string and `remember` the result.
10//! - [`RecallRequest::from_text`] — embed a query string and build a
11//! `RecallRequest`.
12//!
13//! Both validate the embedder's dimensionality against the database's
14//! configured width ([`MnemoError::DimensionMismatch`]) and compare the
15//! embedder's identity (name, dimensions, normalize) to the manifest
16//! (`EmbedderIdentity::MISMATCH_MSG`). An identity mismatch is a loud
17//! error — mixing two embedders in one file silently corrupts recall
18//! quality, so we refuse rather than warn.
19//!
20//! The identity check lives on the handle via [`Mnemo::set_embedder`]:
21//! one call registers the embedder, performs the identity check once,
22//! and upserts the manifest's `metadata.embedder = { name, dimensions,
23//! normalize }` nested object on first registration. Subsequent
24//! `remember_text`/`from_text` calls on the handle are a plain
25//! delegation.
26//!
27//! **Manifest schema (canonical, docs-matching):** the embedder
28//! identity lives at `metadata.embedder.{name,dimensions,normalize}`
29//! on any onboarding/manifest memory — the shape `test/scripts/seed.json`
30//! writes and `mnemo/README.md` documents. For back-compat with the
31//! very first PR 10a dev builds that wrote flat
32//! `embedder_name`/`embedder_dimensions`/`embedder_normalize` keys,
33//! those are still accepted on read as a legacy alias; writes are
34//! always the nested form.
35
36use crate::error::{MnemoError, Result};
37use serde_json::{Map, Value};
38
39/// Describe an embedder's observable identity. Used both in-memory on
40/// an open [`Mnemo`] handle and persisted at `metadata.embedder` on the
41/// scaffold manifest so that reopening with a different embedder is
42/// caught at `set_embedder` time instead of silently polluting the
43/// index.
44///
45/// Build one with a struct literal ending in `..Default::default()`, so a
46/// field added later (as `query_convention` was in v0.15) doesn't break it.
47#[derive(Clone, Debug, Default, PartialEq, Eq)]
48pub struct EmbedderIdentity {
49 /// Model identifier the caller chose. Convention: a short, stable
50 /// slug — `"all-MiniLM-L6-v2"`, `"text-embedding-3-small"`.
51 pub name: String,
52 /// Vector width this embedder emits. Validated against
53 /// [`Mnemo::dimensions`] on registration.
54 pub dimensions: usize,
55 /// Whether the embedder returns unit-normalized vectors. Caller's
56 /// claim — mnemo doesn't verify it, but records it on the manifest
57 /// so downstream readers can tell whether a dot product is cosine
58 /// equivalent.
59 pub normalize: bool,
60 /// 12.3 step 4 (v0.15): how [`Embedder::embed_query`] differs from
61 /// [`Embedder::embed`]. Empty (the default) when queries are embedded
62 /// like documents; otherwise a short, stable name for the convention,
63 /// such as `"bge-query-instruction"` for a model that prefixes queries
64 /// with an instruction. It's part of the identity: a file embedded one
65 /// way refuses an embedder that queries another way
66 /// ([`MnemoError::EmbedderMismatch`]); `mnemo reembed` moves a file to a
67 /// new identity. Stored only when it isn't empty, so a symmetric
68 /// embedder records what it did before v0.15. Builds before v0.15
69 /// ignore it.
70 pub query_convention: String,
71}
72
73impl EmbedderIdentity {
74 /// Message rendered inside [`MnemoError::Invalid`] on an identity
75 /// mismatch. Public so tests + binding layers can assert on it.
76 pub const MISMATCH_MSG: &'static str = "embedder identity mismatch";
77
78 /// The identity as `(name=…, dimensions=…, normalize=…)`, with
79 /// `query_convention=…` when it isn't empty.
80 pub(crate) fn describe(&self) -> String {
81 let mut s = format!(
82 "(name={:?}, dimensions={}, normalize={}",
83 self.name, self.dimensions, self.normalize
84 );
85 if !self.query_convention.is_empty() {
86 s.push_str(&format!(", query_convention={:?}", self.query_convention));
87 }
88 s.push(')');
89 s
90 }
91}
92
93/// Minimal trait any embedder implementation must satisfy. Returns
94/// vectors in the same order as the input `texts`.
95///
96/// Implementations are expected to be cheap to clone / `Send + Sync`
97/// when they wrap thread-safe state; the engine holds an `Arc<dyn
98/// Embedder>` on the handle via [`Mnemo::set_embedder`]. See
99/// [`Mnemo::remember_text`] for the full user-facing contract.
100pub trait Embedder: Send + Sync {
101 /// This embedder's observable identity (name, dims, normalize).
102 /// Must be stable for the lifetime of the embedder — mnemo caches
103 /// it on `set_embedder` and does not re-query per call.
104 fn identity(&self) -> EmbedderIdentity;
105
106 /// Embed `texts`, returning one vector per input in the same order.
107 /// Caller-level errors (network failure for a hosted embedder, OOM
108 /// in a local model) should bubble up as [`MnemoError::Invalid`]
109 /// with a short message — the specific crypto / page errors in
110 /// [`MnemoError`] are reserved for the engine's own concerns.
111 ///
112 /// This is the **document side**: everything stored is embedded with it.
113 fn embed(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>>;
114
115 /// 12.3 step 4 (v0.15): embed `texts` as **queries**. Every recall by
116 /// text uses it ([`crate::RecallRequest::from_text`], the CLI's
117 /// `recall --text`, the MCP `recall` tool, Python's `recall_text`). It
118 /// defaults to [`Embedder::embed`]. A model that embeds queries
119 /// differently (an instruction prefix, a separate query encoder)
120 /// overrides it and names the difference in
121 /// [`EmbedderIdentity::query_convention`].
122 fn embed_query(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
123 self.embed(texts)
124 }
125
126 /// 12.3 step 5 (v0.15): the keywords a recall by text scores with BM25
127 /// in place of its text, when the caller gave none. `None` (the
128 /// default) scores the text itself, at
129 /// [`crate::RecallRequest::text_keyword_weight`]. `Some(k)` scores `k`
130 /// as explicit keywords, at [`crate::RecallRequest::keyword_weight`];
131 /// `Some("")` makes the recall vector-only.
132 ///
133 /// The handle's registered embedder ([`crate::Mnemo::set_embedder`]) is
134 /// asked on every recall that has a text and no keywords, on a v9 file.
135 /// Keywords the caller gives always win. The built-in embedder answers
136 /// with the identifiers in the question (identifier-aware recall; see
137 /// `mnemo_engine::builtin`).
138 fn query_keywords(&self, text: &str) -> Option<String> {
139 let _ = text;
140 None
141 }
142}
143
144/// A boxed embedder is an embedder, so a caller can choose one at run time
145/// (the CLI's `--embed-cmd` or the built-in) and register it.
146impl<E: Embedder + ?Sized> Embedder for Box<E> {
147 fn identity(&self) -> EmbedderIdentity {
148 (**self).identity()
149 }
150 fn embed(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
151 (**self).embed(texts)
152 }
153 fn embed_query(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
154 (**self).embed_query(texts)
155 }
156 fn query_keywords(&self, text: &str) -> Option<String> {
157 (**self).query_keywords(text)
158 }
159}
160
161/// A shared embedder is an embedder, so one loaded model can serve several
162/// handles.
163impl<E: Embedder + ?Sized> Embedder for std::sync::Arc<E> {
164 fn identity(&self) -> EmbedderIdentity {
165 (**self).identity()
166 }
167 fn embed(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
168 (**self).embed(texts)
169 }
170 fn embed_query(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
171 (**self).embed_query(texts)
172 }
173 fn query_keywords(&self, text: &str) -> Option<String> {
174 (**self).query_keywords(text)
175 }
176}
177
178/// The JSON key of [`EmbedderIdentity::query_convention`] in both stored
179/// identities, written only when it isn't empty.
180pub(crate) const QUERY_CONVENTION_KEY: &str = "query_convention";
181
182/// The query convention from a stored identity object: empty when absent.
183/// A present key that isn't a string makes the identity malformed.
184fn convention_from(obj: &Map<String, Value>) -> Option<String> {
185 match obj.get(QUERY_CONVENTION_KEY) {
186 None => Some(String::new()),
187 Some(v) => v.as_str().map(str::to_string),
188 }
189}
190
191/// Returns `true` iff every component of `v` is exactly `0.0`. This is
192/// the predicate `remember` uses to refuse all-zero vectors (which
193/// score 0 on every cosine query and pollute recall).
194pub(crate) fn is_zero_vector(v: &[f32]) -> bool {
195 !v.is_empty() && v.iter().all(|x| *x == 0.0)
196}
197
198/// Canonical nested metadata key: `metadata.embedder = { name,
199/// dimensions, normalize }`. This is the shape `test/scripts/seed.json`
200/// writes and `mnemo/README.md` documents.
201pub(crate) const MANIFEST_EMBEDDER_KEY: &str = "embedder";
202
203/// Legacy flat keys written by the very first PR 10a dev builds
204/// (never released). Still accepted on read so a locally-bootstrapped
205/// repo doesn't trip the new check; writes are always the nested form
206/// above.
207pub(crate) const LEGACY_FLAT_NAME: &str = "embedder_name";
208pub(crate) const LEGACY_FLAT_DIMENSIONS: &str = "embedder_dimensions";
209pub(crate) const LEGACY_FLAT_NORMALIZE: &str = "embedder_normalize";
210
211/// Read the recorded embedder identity off a memory's metadata map.
212/// Returns `None` if neither the canonical nested form nor the legacy
213/// flat form is present (or either is present but malformed — missing
214/// a required field, wrong type).
215///
216/// Nested form (canonical) is tried first; legacy flat form is only
217/// consulted when nothing nested is found. This keeps a seed.json-
218/// shaped file hitting the canonical path even if a bugged earlier
219/// `set_embedder` call left stale flat keys alongside.
220pub(crate) fn read_identity_from_metadata(meta: &Map<String, Value>) -> Option<EmbedderIdentity> {
221 // Canonical nested form.
222 if let Some(obj) = meta.get(MANIFEST_EMBEDDER_KEY).and_then(Value::as_object) {
223 let name = obj.get("name").and_then(Value::as_str)?;
224 let dims = obj.get("dimensions").and_then(Value::as_u64)?;
225 let norm = obj.get("normalize").and_then(Value::as_bool)?;
226 return Some(EmbedderIdentity {
227 name: name.to_string(),
228 dimensions: dims as usize,
229 normalize: norm,
230 query_convention: convention_from(obj)?,
231 });
232 }
233 // Legacy flat fallback.
234 let name = meta.get(LEGACY_FLAT_NAME).and_then(Value::as_str)?;
235 let dims = meta.get(LEGACY_FLAT_DIMENSIONS).and_then(Value::as_u64)?;
236 let norm = meta.get(LEGACY_FLAT_NORMALIZE).and_then(Value::as_bool)?;
237 Some(EmbedderIdentity {
238 name: name.to_string(),
239 dimensions: dims as usize,
240 normalize: norm,
241 query_convention: String::new(),
242 })
243}
244
245/// Write (or overwrite) the embedder identity on a memory's metadata
246/// map, in the canonical nested form. Also removes any legacy flat
247/// keys from the same map so the upsert doesn't leave duplicates
248/// behind.
249pub(crate) fn write_identity_into_metadata(
250 meta: &mut Map<String, Value>,
251 identity: &EmbedderIdentity,
252) {
253 let mut obj = Map::new();
254 obj.insert("name".into(), Value::String(identity.name.clone()));
255 obj.insert(
256 "dimensions".into(),
257 Value::Number(serde_json::Number::from(identity.dimensions as u64)),
258 );
259 obj.insert("normalize".into(), Value::Bool(identity.normalize));
260 if !identity.query_convention.is_empty() {
261 obj.insert(
262 QUERY_CONVENTION_KEY.into(),
263 Value::String(identity.query_convention.clone()),
264 );
265 }
266 meta.insert(MANIFEST_EMBEDDER_KEY.into(), Value::Object(obj));
267 // Drop legacy flat keys if present so the manifest canonicalizes
268 // on the next flush. Harmless no-op when they're absent.
269 meta.remove(LEGACY_FLAT_NAME);
270 meta.remove(LEGACY_FLAT_DIMENSIONS);
271 meta.remove(LEGACY_FLAT_NORMALIZE);
272}
273
274/// The error for an embedder that isn't the one the file records
275/// ([`MnemoError::EmbedderMismatch`]). Its message starts with
276/// [`EmbedderIdentity::MISMATCH_MSG`], so callers that grep the string
277/// (Python's `RuntimeError`) keep working.
278pub(crate) fn identity_mismatch_err(
279 stored: &EmbedderIdentity,
280 given: &EmbedderIdentity,
281) -> MnemoError {
282 MnemoError::EmbedderMismatch {
283 stored: Box::new(stored.clone()),
284 given: Box::new(given.clone()),
285 }
286}
287
288/// EMBED-ID-1: the reserved result-cache namespace that holds a file's
289/// embedder identity. Every file (manifest or not) records the first
290/// embedder attached to it here. The public cache API refuses
291/// [`RESERVED_CACHE_PREFIX`] namespaces and leaves them out of its stats
292/// and purges; compaction, the rekeyed exports and the import copy the
293/// entry like any other cache entry, so the identity follows the file.
294pub(crate) const IDENTITY_NAMESPACE: &str = "_mnemo.embedder";
295/// The key of the identity entry in [`IDENTITY_NAMESPACE`].
296pub(crate) const IDENTITY_KEY: &str = "identity";
297/// Cache namespaces with this prefix belong to the engine.
298pub(crate) const RESERVED_CACHE_PREFIX: &str = "_mnemo.";
299
300/// Whether `namespace` is one of the engine's own cache namespaces.
301pub(crate) fn is_reserved_namespace(namespace: &str) -> bool {
302 namespace.starts_with(RESERVED_CACHE_PREFIX)
303}
304
305/// The identity as the JSON the identity entry stores.
306pub(crate) fn identity_to_json(identity: &EmbedderIdentity) -> Vec<u8> {
307 let mut v = serde_json::json!({
308 "name": identity.name,
309 "dimensions": identity.dimensions as u64,
310 "normalize": identity.normalize,
311 });
312 if !identity.query_convention.is_empty() {
313 v[QUERY_CONVENTION_KEY] = Value::String(identity.query_convention.clone());
314 }
315 serde_json::to_vec(&v).expect("a JSON object of strings, a number and a bool serializes")
316}
317
318/// The identity from the identity entry's JSON, or `None` if it's
319/// malformed.
320pub(crate) fn identity_from_json(bytes: &[u8]) -> Option<EmbedderIdentity> {
321 let v: Value = serde_json::from_slice(bytes).ok()?;
322 let obj = v.as_object()?;
323 Some(EmbedderIdentity {
324 name: obj.get("name")?.as_str()?.to_string(),
325 dimensions: obj.get("dimensions")?.as_u64()? as usize,
326 normalize: obj.get("normalize")?.as_bool()?,
327 query_convention: convention_from(obj)?,
328 })
329}
330
331/// The error for a file whose manifest and identity entry disagree (a
332/// file changed by hand, or by two builds that saw different embedders).
333/// It names both; neither embedder is attached.
334pub(crate) fn identity_conflict_err(
335 manifest: &EmbedderIdentity,
336 record: &EmbedderIdentity,
337) -> MnemoError {
338 MnemoError::Invalid(format!(
339 "{}: this file's manifest records embedder {}, but its identity record holds {}; \
340 fix the manifest (`mnemo update`) to match the embedder its vectors came from",
341 EmbedderIdentity::MISMATCH_MSG,
342 manifest.describe(),
343 record.describe(),
344 ))
345}
346
347#[cfg(test)]
348mod tests {
349 use super::*;
350
351 #[test]
352 fn is_zero_vector_detects_all_zero() {
353 assert!(is_zero_vector(&[0.0, 0.0, 0.0]));
354 assert!(!is_zero_vector(&[0.0, 0.0, 1e-9]));
355 assert!(!is_zero_vector(&[]));
356 }
357
358 #[test]
359 fn read_identity_prefers_nested_over_legacy() {
360 // Canonical nested form.
361 let json = serde_json::json!({
362 "area": "onboarding",
363 "embedder": { "name": "nested", "dimensions": 384, "normalize": true },
364 // Garbage legacy keys — must be ignored while the nested
365 // form is present.
366 "embedder_name": "legacy",
367 "embedder_dimensions": 999,
368 "embedder_normalize": false,
369 });
370 let obj = json.as_object().unwrap().clone();
371 let id = read_identity_from_metadata(&obj).expect("nested identity read");
372 assert_eq!(id.name, "nested");
373 assert_eq!(id.dimensions, 384);
374 assert!(id.normalize);
375 }
376
377 #[test]
378 fn read_identity_falls_back_to_legacy_flat() {
379 let json = serde_json::json!({
380 "embedder_name": "legacy",
381 "embedder_dimensions": 128,
382 "embedder_normalize": false,
383 });
384 let obj = json.as_object().unwrap().clone();
385 let id = read_identity_from_metadata(&obj).expect("legacy identity read");
386 assert_eq!(id.name, "legacy");
387 assert_eq!(id.dimensions, 128);
388 assert!(!id.normalize);
389 }
390
391 #[test]
392 fn read_identity_none_when_absent_or_malformed() {
393 // Empty.
394 assert!(read_identity_from_metadata(&Map::new()).is_none());
395 // Nested missing a field.
396 let json = serde_json::json!({ "embedder": { "name": "x", "dimensions": 4 } });
397 let obj = json.as_object().unwrap().clone();
398 assert!(read_identity_from_metadata(&obj).is_none());
399 }
400
401 #[test]
402 fn write_identity_canonicalizes_and_drops_legacy() {
403 let json = serde_json::json!({
404 "embedder_name": "old",
405 "embedder_dimensions": 128,
406 "embedder_normalize": false,
407 "unrelated": "keep-me",
408 });
409 let mut obj = json.as_object().unwrap().clone();
410 write_identity_into_metadata(
411 &mut obj,
412 &EmbedderIdentity {
413 name: "new".into(),
414 dimensions: 384,
415 normalize: true,
416 ..Default::default()
417 },
418 );
419 // Legacy keys gone.
420 assert!(!obj.contains_key(LEGACY_FLAT_NAME));
421 assert!(!obj.contains_key(LEGACY_FLAT_DIMENSIONS));
422 assert!(!obj.contains_key(LEGACY_FLAT_NORMALIZE));
423 // Unrelated metadata preserved.
424 assert_eq!(obj["unrelated"], serde_json::json!("keep-me"));
425 // Nested identity written correctly.
426 let nested = obj["embedder"].as_object().unwrap();
427 assert_eq!(nested["name"], serde_json::json!("new"));
428 assert_eq!(nested["dimensions"], serde_json::json!(384));
429 assert_eq!(nested["normalize"], serde_json::json!(true));
430 }
431
432 #[test]
433 fn mismatch_err_includes_both_identities() {
434 let want = EmbedderIdentity {
435 name: "A".into(),
436 dimensions: 4,
437 normalize: true,
438 ..Default::default()
439 };
440 let saw = EmbedderIdentity {
441 name: "B".into(),
442 dimensions: 4,
443 normalize: false,
444 ..Default::default()
445 };
446 let msg = format!("{}", identity_mismatch_err(&want, &saw));
447 assert!(msg.contains(EmbedderIdentity::MISMATCH_MSG));
448 assert!(msg.contains("name=\"A\""));
449 assert!(msg.contains("name=\"B\""));
450 }
451}