fathomdb_embedder/
lib.rs

1//! **FathomDB embedder** — the built-in embedder implementations behind the
2//! `fathomdb-embedder-api` trait.
3//!
4//! An internal workspace crate. **To use FathomDB, depend on the `fathomdb`
5//! facade crate** and opt into the default embedder at open; you do not need to
6//! name this crate. To write your OWN embedder, implement the trait in
7//! `fathomdb-embedder-api` — which pulls in no model runtime — rather than
8//! depending on this one.
9//!
10//! The default embedder is `bge-small-en-v1.5` (384-dim) running on a pure-Rust
11//! `candle-transformers` BERT in process: no Python, no sidecar. It is gated
12//! behind the `default-embedder` cargo feature so a consumer who never uses it
13//! pays neither the dependency nor the binary-size cost, and it is **opt-in per
14//! engine** at open — a fresh engine has no embedder configured.
15//!
16//! On first use the loader downloads and sha256-verifies the weights into the
17//! platform cache; that is the crate's only network access, and it happens only
18//! when the feature is on and the embedder is opted into.
19
20use std::path::PathBuf;
21
22use fathomdb_embedder_api::{Embedder, EmbedderError, EmbedderIdentity, Vector};
23
24#[cfg(feature = "default-embedder")]
25pub mod loader;
26
27/// Structured event surfaced through `OpenReport.embedder_events`
28/// (`dev/design/embedder.md` §7).
29///
30/// Defined unconditionally at the crate root so the engine can reference
31/// it regardless of the `default-embedder` feature; the loader (under
32/// `default-embedder`) emits these variants and re-exports the enum for
33/// ergonomic in-module use.
34#[derive(Debug, Clone, PartialEq, Eq)]
35pub enum EmbedderEvent {
36    /// A file was fetched from the network and written to the cache.
37    DefaultEmbedderDownload {
38        file: String,
39        url: String,
40        bytes: u64,
41        sha256: String,
42        cache_path: PathBuf,
43        duration_ms: u64,
44    },
45    /// A file was found in the cache and verified by sha256. No network.
46    DefaultEmbedderCacheHit { file: String, sha256: String, cache_path: PathBuf },
47    /// EU-5a2 — emitted at the commit that materializes the per-workspace
48    /// mean vector into `_fathomdb_embedder_profiles.mean_vec`. `dim`
49    /// matches the default embedder identity's dimension; `doc_count` is
50    /// the number of pre-pin rows the same transaction's re-quantize
51    /// pass updated (per `dev/design/embedder.md` §0.5, §7).
52    ///
53    /// EU-5a2's only live identity is NoopEmbedder, which does NOT
54    /// request mean-centering, so this event is dormant until EU-5b
55    /// flips the default identity. Defined now so EU-5b is a no-op
56    /// addition to this enum.
57    MeanVecPinned { dim: u32, doc_count: u64 },
58    /// 0.7.2 PR-2b — emitted after the transaction that REFRESHES an
59    /// already-pinned `mean_vec` is durable. `dim` is the embedder
60    /// identity dimension; `doc_count` is the number of rows the
61    /// re-quantize pass re-centered; `trigger` records what drove the
62    /// refresh. As of 0.7.2 PR-2bc the only trigger is the explicit
63    /// `doctor recompute-mean` verb (`Manual`); the automatic in-ingest
64    /// drift detector was carved out and deferred to 0.8.x. See
65    /// `dev/design/embedder.md` §0.3/§0.5 and
66    /// `dev/design/embedder-decision.md` §3.4.
67    MeanVecRecomputed { dim: u32, doc_count: u64, trigger: MeanRecomputeTrigger },
68}
69
70/// 0.7.2 PR-2b — what drove a [`EmbedderEvent::MeanVecRecomputed`].
71///
72/// As of 0.7.2 PR-2bc the only variant is `Manual` (the explicit
73/// `doctor recompute-mean` CLI verb). The `DriftAuto` variant for the
74/// automatic in-ingest drift detector was REMOVED when that path was carved
75/// out and deferred to 0.8.x (see
76/// `dev/plans/prompts/0.8.x-auto-mean-drift-DEFERRED.md`); the enum is kept
77/// (rather than collapsed to a unit) so reviving the auto path in 0.8.x is a
78/// pure additive re-introduction of a variant + tag.
79#[derive(Debug, Clone, Copy, PartialEq, Eq)]
80pub enum MeanRecomputeTrigger {
81    /// Fired explicitly by the `doctor recompute-mean` CLI verb.
82    Manual,
83}
84
85impl MeanRecomputeTrigger {
86    /// Stable lowercase tag used in machine-readable surfaces (CLI/py/napi).
87    #[must_use]
88    pub fn as_str(&self) -> &'static str {
89        match self {
90            MeanRecomputeTrigger::Manual => "manual",
91        }
92    }
93}
94
95// 0.8.12 — shared device-request parser for the Candle backends. Compiled
96// whenever EITHER the embedder or reranker Candle path is on, so the embedder's
97// `FATHOMDB_EMBED_DEVICE` and the reranker's `FATHOMDB_RERANK_DEVICE` resolve
98// through one grammar (no duplicate parse logic) even though they sit behind
99// independent features.
100#[cfg(any(feature = "default-embedder", feature = "default-reranker", feature = "onnx-embedder"))]
101mod device;
102
103#[cfg(feature = "default-embedder")]
104mod candle_bge;
105#[cfg(feature = "default-embedder")]
106mod nomic;
107
108// 0.8.16 Slice 10 (ADR-0.8.16-onnx-embedder-backend) — cross-vendor ONNX
109// Runtime BGE-small embedder. Behind its own NON-default `onnx-embedder`
110// feature so the thin default build pulls in zero ONNX code/deps; injected
111// by the caller via `EmbedderChoice::Caller` (zero engine change).
112#[cfg(feature = "onnx-embedder")]
113mod ort_bge;
114
115// 0.8.2 Slice E1: the default CPU cross-encoder reranker (TinyBERT-L-2).
116// Lives behind its own `default-reranker` feature so the default build pulls
117// in zero ML code. The engine's `default-reranker` feature forwards to this.
118#[cfg(feature = "default-reranker")]
119mod candle_reranker;
120
121#[cfg(feature = "default-embedder")]
122pub use candle_bge::{CandleBgeEmbedder, Pooling, DEFAULT_EMBEDDER_DIM, DEFAULT_EMBEDDER_NAME};
123#[cfg(feature = "default-embedder")]
124pub use nomic::{NomicEmbedder, NOMIC_DIM};
125
126#[cfg(feature = "onnx-embedder")]
127pub use ort_bge::{OrtBgeEmbedder, OrtPooling, ORT_BGE_EMBEDDER_DIM, ORT_BGE_EMBEDDER_NAME};
128
129#[cfg(all(feature = "default-reranker", any(test, feature = "loader-test-hooks")))]
130pub use candle_reranker::RERANKER_REVISION;
131#[cfg(feature = "default-reranker")]
132pub use candle_reranker::{CandleTinyBertReranker, RerankerLoadError, DEFAULT_RERANKER_NAME};
133
134#[derive(Clone, Debug)]
135pub struct NoopEmbedder {
136    identity: EmbedderIdentity,
137}
138
139impl Default for NoopEmbedder {
140    fn default() -> Self {
141        Self { identity: EmbedderIdentity::new("fathomdb-noop", "0.6.0-scaffold", 384) }
142    }
143}
144
145impl Embedder for NoopEmbedder {
146    fn identity(&self) -> EmbedderIdentity {
147        self.identity.clone()
148    }
149
150    fn embed(&self, _input: &str) -> Result<Vector, EmbedderError> {
151        let mut vector = vec![0.0_f32; self.identity.dimension as usize];
152        if let Some(first) = vector.first_mut() {
153            *first = 1.0;
154        }
155        Ok(vector)
156    }
157}
fathomdb_embedder/lib.rs

fathomdb_embedder/
lib.rs
