mnemo-engine 0.16.1

Encrypted, single-file, portable agent memory engine with multi-signal recall, sessions, snapshots, and an IVF+PQ index
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
//! 12.3 step 5 (v0.15): the built-in embedder, behind the `embed-candle`
//! feature (on in the release CLI binaries and wheels, off for the library
//! and the mobile packages).
//!
//! **The model** is [`MODEL`] (`Snowflake/snowflake-arctic-embed-xs`,
//! Apache-2.0) at the pinned [`REVISION`]: a 6-layer BERT with 384
//! dimensions, CLS pooling, up to 512 tokens, unit-normalised. Queries are
//! embedded with the model's own instruction ([`QUERY_PREFIX`]), recorded
//! in the identity as the [`QUERY_CONVENTION`]. It runs on candle, in pure
//! Rust on the CPU.
//!
//! **The files** ([`FILES`]: `config.json`, `tokenizer.json`,
//! `model.safetensors`, 91 MB together) aren't in the binary. They're
//! fetched once, over HTTPS, into the platform's cache directory
//! ([`model_dir`]), either on first use ([`BuiltinEmbedder::load_or_install`],
//! which reports the size first) or ahead of time with [`install`] (`mnemo
//! embedder install`). Each file's SHA-256 is in the binary and is checked
//! before the model is loaded, every time, so a changed or truncated file is
//! refused, never used.
//!
//! **A model directory of your own** ([`BuiltinEmbedder::from_dir`], the
//! CLI's `--model-dir`) holds the same three files. When they're the pinned
//! files, it's the built-in embedder. Otherwise it's loaded the same way
//! (BERT, CLS pooling, the arctic query prefix) under an identity that
//! records each file's SHA-256, so a file embedded with it can't be read
//! with another model by mistake.
//!
//! **Identifier-aware recall** (built-in identity only): a recall by text
//! scores no keywords for the question's plain words, since arctic-xs's
//! vector does better alone; the identifiers in it (`snapshot_list`,
//! `--features`, `RecallRequest::from_text`, `store.rs`, …) become explicit
//! keywords instead ([`crate::Embedder::query_keywords`]). Keywords the
//! caller gives still win.
//!
//! **Environment:** `MNEMO_CACHE_DIR` moves the cache (the model goes under
//! `<dir>/models/`). `MNEMO_MODEL_URL` replaces `https://huggingface.co` as
//! the download's base, for a mirror; the hashes are checked all the same.

use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::sync::Mutex;

use candle_core::{DType, Device, Tensor};
use candle_nn::VarBuilder;
use candle_transformers::models::bert::{BertModel, Config};
use sha2::{Digest, Sha256};
use tokenizers::{PaddingParams, Tokenizer, TruncationParams};

use crate::embed::{Embedder, EmbedderIdentity};
use crate::error::{MnemoError, Result};

/// The model's Hugging Face repository.
pub const MODEL: &str = "Snowflake/snowflake-arctic-embed-xs";
/// The pinned revision (a commit of [`MODEL`]'s repository).
pub const REVISION: &str = "d8c86521100d3556476a063fc2342036d45c106f";
/// The vector width.
pub const DIMENSIONS: usize = 384;
/// The longest input, in tokens; longer text is truncated.
pub const MAX_TOKENS: usize = 512;
/// The instruction arctic-xs puts before a query (not before a document).
pub const QUERY_PREFIX: &str = "Represent this sentence for searching relevant passages: ";
/// The identity's name for that query side
/// ([`EmbedderIdentity::query_convention`]).
pub const QUERY_CONVENTION: &str = "arctic-query-prefix";
/// Where the files come from unless `MNEMO_MODEL_URL` says otherwise.
const DEFAULT_BASE_URL: &str = "https://huggingface.co";
/// Texts per forward pass.
const BATCH: usize = 8;

/// One of the model's files, as pinned.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ModelFile {
    /// The file's name, in the repository and in the model directory.
    pub name: &'static str,
    /// Its SHA-256, in lowercase hex.
    pub sha256: &'static str,
    /// Its size in bytes.
    pub size: u64,
}

/// The model's three files at [`REVISION`].
pub const FILES: [ModelFile; 3] = [
    ModelFile {
        name: "config.json",
        sha256: "d7d071046ab952af96b7abad788db7ab3fc997b465e1b9914ff39707092254ec",
        size: 737,
    },
    ModelFile {
        name: "tokenizer.json",
        sha256: "91f1def9b9391fdabe028cd3f3fcc4efd34e5d1f08c3bf2de513ebb5911a1854",
        size: 711_649,
    },
    ModelFile {
        name: "model.safetensors",
        sha256: "ee789e0b1d6ecbbd5ce37b474af556cc1a1319cee4417d9e3b11f82e90300706",
        size: 90_272_656,
    },
];

/// The download's size in bytes (all of [`FILES`]).
pub fn download_size() -> u64 {
    FILES.iter().map(|f| f.size).sum()
}

/// `bytes` as megabytes for a person, such as `91.0 MB`.
pub fn human_size(bytes: u64) -> String {
    format!("{:.1} MB", bytes as f64 / 1_000_000.0)
}

/// The built-in embedder's identity: [`MODEL`]`@`[`REVISION`], 384
/// dimensions, normalised, with the arctic query convention. A file
/// embedded with it records this identity.
pub fn identity() -> EmbedderIdentity {
    EmbedderIdentity {
        name: format!("{MODEL}@{REVISION}"),
        dimensions: DIMENSIONS,
        normalize: true,
        query_convention: QUERY_CONVENTION.to_string(),
    }
}

/// The cache directory: `MNEMO_CACHE_DIR` if set, otherwise the platform's
/// (`~/Library/Caches/mnemo` on macOS, `%LOCALAPPDATA%\mnemo\cache` on
/// Windows, `$XDG_CACHE_HOME/mnemo` or `~/.cache/mnemo` elsewhere).
pub fn cache_dir() -> Result<PathBuf> {
    if let Some(d) = std::env::var_os("MNEMO_CACHE_DIR").filter(|d| !d.is_empty()) {
        return Ok(PathBuf::from(d));
    }
    let home = || {
        std::env::var_os("HOME")
            .filter(|h| !h.is_empty())
            .map(PathBuf::from)
    };
    let dir = if cfg!(target_os = "macos") {
        home().map(|h| h.join("Library").join("Caches").join("mnemo"))
    } else if cfg!(windows) {
        std::env::var_os("LOCALAPPDATA")
            .filter(|d| !d.is_empty())
            .map(|d| PathBuf::from(d).join("mnemo").join("cache"))
    } else {
        std::env::var_os("XDG_CACHE_HOME")
            .map(PathBuf::from)
            .filter(|d| d.is_absolute())
            .map(|d| d.join("mnemo"))
            .or_else(|| home().map(|h| h.join(".cache").join("mnemo")))
    };
    dir.ok_or_else(|| {
        MnemoError::Invalid(
            "the built-in embedder has no cache directory (no HOME); set MNEMO_CACHE_DIR".into(),
        )
    })
}

/// Where the pinned model's files are kept:
/// `<cache>/models/snowflake-arctic-embed-xs/<revision>/`.
pub fn model_dir() -> Result<PathBuf> {
    Ok(cache_dir()?
        .join("models")
        .join("snowflake-arctic-embed-xs")
        .join(REVISION))
}

/// Whether every file is in [`model_dir`] at its pinned size: the cheap
/// check. The hashes are checked at every load, and by [`verify_installed`].
pub fn is_installed() -> bool {
    model_dir().is_ok_and(|d| {
        FILES.iter().all(|f| {
            std::fs::metadata(d.join(f.name)).is_ok_and(|m| m.is_file() && m.len() == f.size)
        })
    })
}

/// Whether `dir`'s copy of `f` is there at its pinned size and SHA-256.
fn file_matches(dir: &Path, f: &ModelFile) -> Result<bool> {
    let p = dir.join(f.name);
    match std::fs::metadata(&p) {
        Ok(m) if m.is_file() && m.len() == f.size => {}
        _ => return Ok(false),
    }
    let mut file = std::fs::File::open(&p).map_err(|e| {
        MnemoError::Invalid(format!(
            "the built-in embedder can't read {}: {e}",
            p.display()
        ))
    })?;
    let mut hasher = Sha256::new();
    std::io::copy(&mut file, &mut hasher).map_err(|e| {
        MnemoError::Invalid(format!(
            "the built-in embedder can't read {}: {e}",
            p.display()
        ))
    })?;
    Ok(hex(&hasher.finalize()) == f.sha256)
}

/// Check every file in [`model_dir`] against its pinned size and SHA-256
/// (`mnemo embedder status`). The error names the first file that's missing
/// or doesn't match, and the fix.
pub fn verify_installed() -> Result<()> {
    let dir = model_dir()?;
    for f in FILES {
        if !dir.join(f.name).exists() {
            return Err(MnemoError::Invalid(format!(
                "{} isn't in {} (`mnemo embedder install` fetches it)",
                f.name,
                dir.display()
            )));
        }
        if !file_matches(&dir, &f)? {
            return Err(MnemoError::Invalid(format!(
                "{} in {} doesn't match its pinned SHA-256 (`mnemo embedder install` replaces \
                 it)",
                f.name,
                dir.display()
            )));
        }
    }
    Ok(())
}

/// Fetch whatever [`model_dir`] is missing or doesn't match, checking each
/// file's size and SHA-256 before it takes its place, and return the
/// directory. A file there already is hashed, and replaced if it doesn't
/// match. `notice` hears, before any download, what's fetched, its size, and
/// where to. Nothing is fetched when every file matches.
pub fn install(notice: &mut dyn FnMut(&str)) -> Result<PathBuf> {
    install_with(notice, true)
}

/// [`install`], checking the files already there by size only when
/// `hash_present` is off (the first-use path: the load that follows hashes
/// every file anyway).
fn install_with(notice: &mut dyn FnMut(&str), hash_present: bool) -> Result<PathBuf> {
    let dir = model_dir()?;
    let mut missing: Vec<&ModelFile> = Vec::new();
    for f in &FILES {
        let ok = if hash_present {
            file_matches(&dir, f)?
        } else {
            std::fs::metadata(dir.join(f.name)).is_ok_and(|m| m.len() == f.size)
        };
        if !ok {
            missing.push(f);
        }
    }
    if missing.is_empty() {
        return Ok(dir);
    }
    let bytes: u64 = missing.iter().map(|f| f.size).sum();
    notice(&format!(
        "mnemo: downloading the built-in embedder ({MODEL}, {}) into {}",
        human_size(bytes),
        dir.display()
    ));
    std::fs::create_dir_all(&dir).map_err(|e| {
        MnemoError::Invalid(format!(
            "the built-in embedder can't create {}: {e}",
            dir.display()
        ))
    })?;
    remove_stale_parts(&dir);
    let base = std::env::var("MNEMO_MODEL_URL")
        .ok()
        .filter(|b| !b.is_empty())
        .unwrap_or_else(|| DEFAULT_BASE_URL.to_string());
    let base = base.trim_end_matches('/');
    for f in missing {
        let url = format!("{base}/{MODEL}/resolve/{REVISION}/{}", f.name);
        fetch(&url, &dir.join(f.name), f.sha256, f.size)?;
    }
    Ok(dir)
}

/// Remove the temporary files (`<name>.<pid>.part`) that downloads which
/// never finished left in `dir`: a process that isn't running any more
/// (Unix), or a file more than an hour old. Another process's download in
/// progress keeps its file.
fn remove_stale_parts(dir: &Path) {
    let Ok(entries) = std::fs::read_dir(dir) else {
        return;
    };
    for e in entries.flatten() {
        let name = e.file_name().to_string_lossy().to_string();
        let Some(stem) = name.strip_suffix(".part") else {
            continue;
        };
        let Some(pid) = stem
            .rsplit_once('.')
            .and_then(|(file, pid)| FILES.iter().any(|f| f.name == file).then_some(pid))
            .and_then(|pid| pid.parse::<u32>().ok())
        else {
            continue;
        };
        let old = e
            .metadata()
            .and_then(|m| m.modified())
            .ok()
            .and_then(|t| t.elapsed().ok())
            .is_some_and(|age| age > std::time::Duration::from_secs(3600));
        let gone = pid != std::process::id() && !crate::shared::pid_alive(pid);
        if gone || old {
            let _ = std::fs::remove_file(e.path());
        }
    }
}

/// Download `url` to `dest`, through a temporary file beside it that
/// replaces `dest` only once its size and SHA-256 match.
pub(crate) fn fetch(url: &str, dest: &Path, sha256: &str, size: u64) -> Result<()> {
    let fail = |what: String| MnemoError::Invalid(format!("the built-in embedder's {what}"));
    let name = dest
        .file_name()
        .and_then(|n| n.to_str())
        .unwrap_or("model file");
    let agent: ureq::Agent = ureq::Agent::config_builder()
        .timeout_connect(Some(std::time::Duration::from_secs(15)))
        .timeout_per_read(Some(std::time::Duration::from_secs(60)))
        .user_agent(concat!("mnemo/", env!("CARGO_PKG_VERSION")))
        .build()
        .into();
    let mut resp = agent
        .get(url)
        .call()
        .map_err(|e| fail(format!("download of {name} failed ({url}): {e}")))?;
    let tmp = dest.with_file_name(format!("{name}.{}.part", std::process::id()));
    let result = (|| {
        // `create_new`: never through a link, or over another writer's file.
        let mut out = std::fs::OpenOptions::new()
            .write(true)
            .create_new(true)
            .open(&tmp)
            .map_err(|e| fail(format!("download can't write {}: {e}", tmp.display())))?;
        // One byte over the size is enough to tell a too-long body.
        let mut body = resp.body_mut().with_config().limit(size + 1).reader();
        let mut hasher = Sha256::new();
        let mut got: u64 = 0;
        let mut buf = vec![0u8; 1 << 16];
        loop {
            let n = match body.read(&mut buf) {
                Ok(0) => break,
                Ok(n) => n,
                Err(e) => {
                    // ureq reports the limit as an error: the body is too long.
                    if got + buf.len() as u64 > size {
                        return Err(fail(format!(
                            "{name} is larger than the {size} bytes pinned"
                        )));
                    }
                    return Err(fail(format!("download of {name} failed: {e}")));
                }
            };
            got += n as u64;
            if got > size {
                return Err(fail(format!(
                    "{name} is larger than the {size} bytes pinned"
                )));
            }
            hasher.update(&buf[..n]);
            out.write_all(&buf[..n])
                .map_err(|e| fail(format!("download can't write {}: {e}", tmp.display())))?;
        }
        if got != size {
            return Err(fail(format!(
                "{name} is {got} bytes, not the {size} pinned (an interrupted download?)"
            )));
        }
        let digest = hex(&hasher.finalize());
        if digest != sha256 {
            return Err(fail(format!(
                "{name} doesn't match its pinned SHA-256 (got {digest}); nothing was installed"
            )));
        }
        out.sync_all()
            .map_err(|e| fail(format!("download can't write {}: {e}", tmp.display())))?;
        drop(out);
        std::fs::rename(&tmp, dest)
            .map_err(|e| fail(format!("download can't move {name} into place: {e}")))
    })();
    if result.is_err() {
        let _ = std::fs::remove_file(&tmp);
    }
    result
}

fn hex(bytes: &[u8]) -> String {
    bytes.iter().map(|b| format!("{b:02x}")).collect()
}

/// The built-in embedder: arctic-xs on candle, as an [`Embedder`].
///
/// Load it with [`BuiltinEmbedder::load_or_install`] (fetching the files on
/// first use), [`BuiltinEmbedder::load`] (the cached files only) or
/// [`BuiltinEmbedder::from_dir`], then register it with
/// [`crate::Mnemo::set_embedder`].
pub struct BuiltinEmbedder {
    model: BertModel,
    tokenizer: Mutex<Tokenizer>,
    identity: EmbedderIdentity,
    /// Whether the files are the pinned ones (the built-in identity), which
    /// turns on identifier-aware recall.
    builtin: bool,
}

impl std::fmt::Debug for BuiltinEmbedder {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("BuiltinEmbedder")
            .field("identity", &self.identity)
            .field("builtin", &self.builtin)
            .finish_non_exhaustive()
    }
}

impl BuiltinEmbedder {
    /// Load the model from [`model_dir`], fetching it first if it isn't
    /// there ([`install`]; `notice` hears the size before the download).
    pub fn load_or_install(notice: &mut dyn FnMut(&str)) -> Result<Self> {
        // Sizes only here: the load hashes every file.
        let dir = install_with(notice, false)?;
        Self::load_pinned(&dir)
    }

    /// Load the model from [`model_dir`] without fetching anything. Fails
    /// if a file is missing or doesn't match its pinned hash.
    pub fn load() -> Result<Self> {
        let dir = model_dir()?;
        if !is_installed() {
            return Err(MnemoError::Invalid(format!(
                "the built-in embedder isn't installed in {} (run `mnemo embedder install`)",
                dir.display()
            )));
        }
        Self::load_pinned(&dir)
    }

    /// Load the three files from a directory of your own. When they're the
    /// pinned files this is the built-in embedder; otherwise its identity
    /// is named by the files' SHA-256s
    /// (`local:config.json=<sha>,tokenizer.json=<sha>,model.safetensors=<sha>`)
    /// and recall keeps the default keyword weight.
    pub fn from_dir(dir: &Path) -> Result<Self> {
        let files = read_files(dir)?;
        let hashes: Vec<String> = files.iter().map(|b| hex(&Sha256::digest(b))).collect();
        let pinned = FILES.iter().zip(&hashes).all(|(f, h)| f.sha256 == h);
        let mut e = Self::build(files)?;
        if !pinned {
            let named: Vec<String> = FILES
                .iter()
                .zip(&hashes)
                .map(|(f, h)| format!("{}={h}", f.name))
                .collect();
            e.identity.name = format!("local:{}", named.join(","));
            e.builtin = false;
        }
        Ok(e)
    }

    /// Whether this is the pinned model (the built-in identity).
    pub fn is_builtin(&self) -> bool {
        self.builtin
    }

    fn load_pinned(dir: &Path) -> Result<Self> {
        let files = read_files(dir)?;
        for (f, bytes) in FILES.iter().zip(&files) {
            let got = hex(&Sha256::digest(bytes));
            if got != f.sha256 {
                return Err(MnemoError::Invalid(format!(
                    "the built-in embedder's {} in {} doesn't match its pinned SHA-256 \
                     (got {got}); `mnemo embedder install` replaces it",
                    f.name,
                    dir.display()
                )));
            }
        }
        Self::build(files)
    }

    /// The model from its three files' bytes, in [`FILES`] order.
    fn build(files: [Vec<u8>; 3]) -> Result<Self> {
        let err = |what: &str, e: String| {
            MnemoError::Invalid(format!("the built-in embedder can't load its {what}: {e}"))
        };
        let [config, tokenizer, weights] = files;
        let config: Config =
            serde_json::from_slice(&config).map_err(|e| err("config.json", e.to_string()))?;
        let device = Device::Cpu;
        // The weights are read into memory, not mapped, so no `unsafe`.
        let vb = VarBuilder::from_buffered_safetensors(weights, DType::F32, &device)
            .map_err(|e| err("model.safetensors", e.to_string()))?;
        let model =
            BertModel::load(vb, &config).map_err(|e| err("model.safetensors", e.to_string()))?;
        let mut tok =
            Tokenizer::from_bytes(&tokenizer).map_err(|e| err("tokenizer.json", e.to_string()))?;
        tok.with_truncation(Some(TruncationParams {
            max_length: MAX_TOKENS,
            ..Default::default()
        }))
        .map_err(|e| err("tokenizer.json", e.to_string()))?;
        tok.with_padding(Some(PaddingParams::default()));
        let mut identity = identity();
        identity.dimensions = config.hidden_size;
        Ok(BuiltinEmbedder {
            model,
            tokenizer: Mutex::new(tok),
            identity,
            builtin: true,
        })
    }

    /// Embed `texts` in batches: padded within a batch, masked, CLS-pooled
    /// and normalised.
    fn vectors(&self, texts: &[String]) -> Result<Vec<Vec<f32>>> {
        let fail = |e: String| MnemoError::Invalid(format!("the built-in embedder failed: {e}"));
        let mut out = Vec::with_capacity(texts.len());
        for chunk in texts.chunks(BATCH) {
            let enc = self
                .tokenizer
                .lock()
                .map_err(|_| fail("its tokenizer lock is poisoned".into()))?
                .encode_batch(chunk.to_vec(), true)
                .map_err(|e| fail(e.to_string()))?;
            let pooled = (|| -> candle_core::Result<Tensor> {
                let ids: Vec<Tensor> = enc
                    .iter()
                    .map(|e| Tensor::new(e.get_ids(), &Device::Cpu))
                    .collect::<candle_core::Result<_>>()?;
                let mask: Vec<Tensor> = enc
                    .iter()
                    .map(|e| Tensor::new(e.get_attention_mask(), &Device::Cpu))
                    .collect::<candle_core::Result<_>>()?;
                let ids = Tensor::stack(&ids, 0)?;
                let mask = Tensor::stack(&mask, 0)?;
                let types = ids.zeros_like()?;
                let hidden = self.model.forward(&ids, &types, Some(&mask))?;
                // CLS pooling: the first token's vector.
                let cls = hidden.narrow(1, 0, 1)?.squeeze(1)?;
                let norm = cls.sqr()?.sum_keepdim(1)?.sqrt()?;
                cls.broadcast_div(&norm)
            })()
            .map_err(|e| fail(e.to_string()))?;
            out.extend(pooled.to_vec2::<f32>().map_err(|e| fail(e.to_string()))?);
        }
        Ok(out)
    }
}

/// The three files' bytes from `dir`, in [`FILES`] order.
fn read_files(dir: &Path) -> Result<[Vec<u8>; 3]> {
    let read = |f: &ModelFile| {
        std::fs::read(dir.join(f.name)).map_err(|e| {
            MnemoError::Invalid(format!(
                "the built-in embedder can't read {}: {e}",
                dir.join(f.name).display()
            ))
        })
    };
    Ok([read(&FILES[0])?, read(&FILES[1])?, read(&FILES[2])?])
}

impl Embedder for BuiltinEmbedder {
    fn identity(&self) -> EmbedderIdentity {
        self.identity.clone()
    }

    fn embed(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
        self.vectors(&texts.iter().map(|t| t.to_string()).collect::<Vec<_>>())
    }

    fn embed_query(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
        self.vectors(
            &texts
                .iter()
                .map(|t| format!("{QUERY_PREFIX}{t}"))
                .collect::<Vec<_>>(),
        )
    }

    fn query_keywords(&self, text: &str) -> Option<String> {
        // Identifier-aware recall, for the pinned model only: the question's
        // identifiers as explicit keywords, or none (vector-only).
        self.builtin
            .then(|| crate::ident::identifiers(text).join(" "))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::io::BufRead;
    use std::net::TcpListener;

    /// Serve `body` once over plain HTTP on a local port; returns the URL.
    fn serve_once(body: Vec<u8>) -> String {
        let l = TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = l.local_addr().unwrap();
        std::thread::spawn(move || {
            let (mut s, _) = l.accept().unwrap();
            let mut r = std::io::BufReader::new(s.try_clone().unwrap());
            let mut line = String::new();
            while r.read_line(&mut line).unwrap() > 0 && line != "\r\n" {
                line.clear();
            }
            write!(
                s,
                "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
                body.len()
            )
            .unwrap();
            s.write_all(&body).unwrap();
        });
        format!("http://{addr}/f")
    }

    #[test]
    fn a_download_takes_its_place_only_when_it_matches() {
        let dir = tempfile::tempdir().unwrap();
        let dest = dir.path().join("config.json");
        let body = b"{\"hidden_size\": 384}".to_vec();
        let sha = hex(&Sha256::digest(&body));
        fetch(&serve_once(body.clone()), &dest, &sha, body.len() as u64).unwrap();
        assert_eq!(std::fs::read(&dest).unwrap(), body);

        // The wrong bytes: refused, nothing left behind, the old file kept.
        let bad = b"{\"hidden_size\": 768}".to_vec();
        let e = fetch(
            &serve_once(bad.clone()),
            &dest,
            &"0".repeat(64),
            bad.len() as u64,
        )
        .unwrap_err()
        .to_string();
        assert!(e.contains("doesn't match its pinned SHA-256"), "{e}");
        assert_eq!(std::fs::read(&dest).unwrap(), body);
        assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1);

        // Too short and too long.
        let e = fetch(
            &serve_once(body.clone()),
            &dest,
            &sha,
            body.len() as u64 + 5,
        )
        .unwrap_err()
        .to_string();
        assert!(e.contains("not the"), "{e}");
        let e = fetch(&serve_once(body.clone()), &dest, &sha, 4)
            .unwrap_err()
            .to_string();
        assert!(e.contains("larger than"), "{e}");
        assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1);
    }

    #[test]
    fn an_unreachable_server_is_an_error_not_a_hang() {
        let l = TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = l.local_addr().unwrap();
        drop(l);
        let dir = tempfile::tempdir().unwrap();
        let e = fetch(
            &format!("http://{addr}/f"),
            &dir.path().join("x"),
            &"0".repeat(64),
            1,
        )
        .unwrap_err()
        .to_string();
        assert!(e.contains("download of x failed"), "{e}");
    }

    #[test]
    fn only_a_finished_downloads_leftovers_are_removed() {
        let dir = tempfile::tempdir().unwrap();
        let touch = |name: &str| {
            let p = dir.path().join(name);
            std::fs::write(&p, b"x").unwrap();
            p
        };
        let dead = touch("config.json.999999999.part");
        let mine = touch(&format!("tokenizer.json.{}.part", std::process::id()));
        let other = touch("notes.123.part");
        let model = touch("model.safetensors");
        #[cfg(unix)]
        let live = touch("model.safetensors.1.part"); // pid 1 is always running
        remove_stale_parts(dir.path());
        #[cfg(unix)]
        assert!(!dead.exists(), "a dead process's file stays");
        #[cfg(not(unix))]
        let _ = dead;
        assert!(mine.exists() && other.exists() && model.exists());
        #[cfg(unix)]
        assert!(live.exists(), "a running process's download was removed");
    }

    #[test]
    fn the_pins() {
        assert_eq!(download_size(), 90_985_042);
        assert_eq!(human_size(download_size()), "91.0 MB");
        let id = identity();
        assert_eq!(
            id.name,
            "Snowflake/snowflake-arctic-embed-xs@d8c86521100d3556476a063fc2342036d45c106f"
        );
        assert_eq!((id.dimensions, id.normalize), (384, true));
        assert_eq!(id.query_convention, "arctic-query-prefix");
        for f in FILES {
            assert_eq!(f.sha256.len(), 64);
        }
    }
}