Skip to main content

lapis_lattice/
lib.rs

1//! Embedded SQLite + FTS5 lattice for a Markdown vault.
2//!
3//! Files on disk are the write source of truth. This crate only writes
4//! `<vault>/.lapis/lattice.sqlite` by default; [`Engine::open_in`] can keep the
5//! index in a caller-chosen directory instead. Search is FTS5 BM25; vectors are
6//! not in 0.2.
7//!
8//! The database is stamped `producer = "lapis-lattice"` at creation and the
9//! stamp is asserted on open, so a path pointed at a foreign SQLite file (an
10//! agent-memory store, say) fails loudly instead of quietly mixing corpora.
11
12mod analytics;
13mod embed;
14mod graph;
15mod index;
16mod kinds;
17mod location;
18mod refresh;
19mod scope;
20mod sqlite;
21
22#[cfg(test)]
23mod search_ids;
24
25use std::path::{Path, PathBuf};
26
27use rusqlite::Connection;
28use serde::{Deserialize, Serialize};
29
30pub use analytics::{ANALYTICS_QUERIES, Analytics};
31pub use embed::Embedder;
32pub use graph::{GraphEdge, GraphNode, GraphSnapshot, Neighbor, SNAPSHOT_NODE_CAP};
33pub use kinds::{HTML, MARKDOWN, YAML, kind_for};
34pub use refresh::RefreshReport;
35pub use scope::Scope;
36
37/// Written into `meta.producer` at creation; asserted on open.
38pub const PRODUCER: &str = "lapis-lattice";
39
40#[derive(Debug, thiserror::Error)]
41pub enum Error {
42    #[error("{0}")]
43    Usage(String),
44    #[error("{0}")]
45    Io(#[from] std::io::Error),
46    #[error("sqlite: {0}")]
47    Sqlite(#[from] rusqlite::Error),
48}
49
50pub type Result<T> = std::result::Result<T, Error>;
51
52/// Open an index over a Markdown vault: `<vault>/.lapis/lattice.sqlite` by
53/// default, or in a caller-chosen directory with [`Engine::open_in`].
54pub struct Engine {
55    vault: PathBuf,
56    /// The SQLite file, wherever the default or [`Engine::open_in`] put it.
57    db: PathBuf,
58    conn: Connection,
59    embedder: Option<std::sync::Arc<dyn Embedder>>,
60}
61
62#[derive(Debug, Clone, Serialize, Deserialize)]
63pub struct Hit {
64    pub path: String,
65    /// `markdown` | `html` | `yaml`, as recorded at index time rather than
66    /// re-derived from the extension by every consumer.
67    pub kind: String,
68    pub title: String,
69    #[serde(skip_serializing_if = "Option::is_none")]
70    pub heading: Option<String>,
71    #[serde(skip_serializing_if = "Option::is_none")]
72    pub snippet: Option<String>,
73    pub rank: u32,
74    pub score: f64,
75    pub domain: Option<String>,
76    pub doc_type: Option<String>,
77    #[serde(default, skip_serializing_if = "Vec::is_empty")]
78    pub tags: Vec<String>,
79    #[serde(default, skip_serializing_if = "Option::is_none")]
80    pub chunk_id: Option<i64>,
81    #[serde(default, skip_serializing_if = "Option::is_none")]
82    pub chunk_index: Option<i64>,
83    /// Post-retrieve judge annotation. Never written to sqlite.
84    #[serde(default, skip_serializing_if = "Option::is_none")]
85    pub jev: Option<serde_json::Value>,
86}
87
88/// Retrieval arm selection. The embedded engine is FTS-only in 0.2, so
89/// `Hybrid` runs BM25 alone and reports that in [`SearchResult::modalities`];
90/// `Vector` is refused rather than silently downgraded.
91#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
92#[serde(rename_all = "lowercase")]
93pub enum Mode {
94    #[default]
95    Hybrid,
96    Bm25,
97    Vector,
98}
99
100/// Everything a surface can ask of search. Flags must not silently no-op.
101#[derive(Debug, Clone, Default)]
102pub struct SearchParams {
103    pub query: String,
104    pub limit: u32,
105    pub offset: u32,
106    pub domain: Option<String>,
107    /// Collapse to the best-scoring chunk per document.
108    pub per_doc: bool,
109    pub mode: Mode,
110    /// HTTP lattice only; the embedded index has no MMR arm.
111    pub mmr: bool,
112    /// HTTP lattice only; the embedded walk already skips `_archives/`.
113    pub include_archives: bool,
114    /// `none` skips the vector arm for this query (no embedder round-trip).
115    pub embedder: Option<String>,
116}
117
118#[derive(Debug, Clone, Serialize)]
119pub struct SearchResult {
120    pub hits: Vec<Hit>,
121    /// Arms that actually ran. FTS-only today, so `["bm25"]`.
122    pub modalities: Vec<String>,
123}
124
125/// Filters for the documents table (`lapis list`).
126#[derive(Debug, Clone, Default)]
127pub struct ListParams {
128    pub domain: Option<String>,
129    pub doc_type: Option<String>,
130    pub status: Option<String>,
131    pub tag: Option<String>,
132    pub prefix: Option<String>,
133    pub limit: u32,
134    pub offset: u32,
135}
136
137/// One row of the documents table. Wire casing belongs to the caller.
138#[derive(Debug, Clone, Serialize)]
139pub struct Document {
140    pub path: String,
141    pub kind: String,
142    pub title: Option<String>,
143    pub domain: Option<String>,
144    pub doc_type: Option<String>,
145    pub status: Option<String>,
146    pub priority: Option<String>,
147    pub tags: Vec<String>,
148    /// File mtime in milliseconds.
149    pub updated_at: Option<u64>,
150    /// Content fingerprint, so an agent can plan `--if-hash` writes from one list call.
151    pub hash: Option<String>,
152}
153
154/// Link-graph state. `built` is false before the first reindex, so "no edges
155/// yet" is distinguishable from "a vault with no links".
156#[derive(Debug, Clone, Serialize)]
157pub struct Graph {
158    pub built: bool,
159    pub edges: u64,
160    pub dangling_links: u64,
161}
162
163#[derive(Debug, Clone, Serialize)]
164pub struct Health {
165    pub status: String,
166    pub documents_indexed: u64,
167    /// Indexed paths missing on disk, new walkable files, or mtime drift.
168    #[serde(skip_serializing_if = "is_zero")]
169    pub stale_documents: u64,
170    pub db_path: String,
171    pub embedder: &'static str,
172    pub embed_model: Option<String>,
173    pub embed_dim: Option<u32>,
174    /// Chunks in the index, and how many of them carry a vector.
175    pub chunks: u64,
176    pub vectors: u64,
177    pub graph: Graph,
178}
179
180fn is_zero(n: &u64) -> bool {
181    *n == 0
182}
183
184/// The embedding space recorded in an index, read without a provider round-trip.
185#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
186pub struct StoredEmbedder {
187    pub provider: String,
188    pub model: String,
189    pub dim: u32,
190}
191
192#[derive(Debug, Clone, Serialize)]
193pub struct IndexReport {
194    pub documents: u64,
195    pub chunks: u64,
196    pub edges: u64,
197    /// Chunks given a vector this run. Zero without an embedder.
198    pub embedded: u64,
199    /// Why the semantic arm produced nothing. Indexing still succeeded: files
200    /// are the source of truth and BM25 still answers.
201    pub embed_error: Option<String>,
202}
203
204impl Engine {
205    /// Open with no semantic arm: FTS-only, which is the first-run default.
206    pub fn open(vault: impl AsRef<Path>) -> Result<Self> {
207        Self::open_with(vault, None)
208    }
209
210    /// Open with an optional [`Embedder`]. Transport lives in the caller. The
211    /// index lives in `<vault>/.lapis/`, exactly as before.
212    pub fn open_with(
213        vault: impl AsRef<Path>,
214        embedder: Option<std::sync::Arc<dyn Embedder>>,
215    ) -> Result<Self> {
216        Self::open_at(vault.as_ref(), None, embedder)
217    }
218    /// Open with the index in a caller-chosen `index_dir`, not the vault's
219    /// `.lapis/`: build, search, reindex and every helper work the same, and the
220    /// vault tree gains no file. The directory is created if needed, and when
221    /// this crate creates it, it is `0700` on unix (an existing directory keeps
222    /// its mode). The default [`Engine::open`] and [`Engine::open_with`] do not
223    /// change: `<vault>/.lapis/` is created as before.
224    ///
225    /// Confinement is enforced, not advisory, and happens before any write so
226    /// a refused call creates nothing: the requested directory is resolved
227    /// down to its deepest existing ancestor (symlinks followed, the same as
228    /// a reader would), `..` is refused, and a directory inside the vault —
229    /// equal to the vault counts — is refused. The caller should still
230    /// confine `index_dir` to its own app-data directory: an index anywhere
231    /// else outside the vault is accepted.
232    pub fn open_in(
233        vault: impl AsRef<Path>,
234        index_dir: impl AsRef<Path>,
235        embedder: Option<std::sync::Arc<dyn Embedder>>,
236    ) -> Result<Self> {
237        Self::open_at(vault.as_ref(), Some(index_dir.as_ref()), embedder)
238    }
239    fn open_at(
240        vault: &Path,
241        index_dir: Option<&Path>,
242        embedder: Option<std::sync::Arc<dyn Embedder>>,
243    ) -> Result<Self> {
244        let vault = vault.canonicalize().unwrap_or_else(|_| vault.to_path_buf());
245        if !vault.is_dir() {
246            return Err(Error::Usage(format!("vault is not a directory: {}", vault.display())));
247        }
248        let dir = match index_dir {
249            // Confinement is enforced before anything is written: reads only.
250            // The default `.lapis/` path behaves exactly as before.
251            Some(d) => location::confined_index_dir(&vault, d)?,
252            None => vault.join(location::DEFAULT_DIR),
253        };
254        if index_dir.is_some() {
255            location::create_dir_private(&dir)?;
256        } else {
257            std::fs::create_dir_all(&dir)?;
258        }
259        let db = location::db_file(&dir);
260        // Before the connection exists: sqlite-vec registers as an auto-extension,
261        // so `vec0` is present on this connection and every later one without a
262        // shared object to find and without enabling `load_extension`.
263        sqlite::register_vec();
264        let conn = Connection::open(&db)?;
265        sqlite::migrate(&conn)?;
266        sqlite::assert_producer(&conn, &db)?;
267        // A change of model or dimension wipes stored vectors: keeping rows from
268        // a different embedding space would compare incomparable numbers.
269        embed::reconcile_space(&conn, embedder.as_deref())?;
270        Ok(Self { vault, db, conn, embedder })
271    }
272
273    /// The SQLite file this engine holds, wherever the default or
274    /// [`Engine::open_in`] put it: `<vault>/.lapis/lattice.sqlite` by
275    /// default, and the confinement-resolved directory for `open_in`.
276    pub fn db_path(&self) -> PathBuf {
277        self.db.clone()
278    }
279
280    /// Full-vault reindex, then embed anything missing a vector.
281    pub fn reindex(&mut self) -> Result<IndexReport> {
282        self.reindex_with_progress(&mut |_, _| {})
283    }
284
285    /// [`Engine::reindex`], calling `progress(files indexed, files total)` during the
286    /// walk. Embedding missing vectors follows the last file and is not counted.
287    pub fn reindex_with_progress(&mut self, progress: &mut dyn FnMut(u64, u64)) -> Result<IndexReport> {
288        let mut r = index::reindex(&self.conn, &self.vault, progress)?;
289        self.embed_into(&mut r, None);
290        Ok(r)
291    }
292
293    /// Bring the index up to date with the vault: added, updated and removed
294    /// notes since the last build, compared by mtime and size, with [`Scope`]
295    /// and the same chunking as a full reindex. A second call with nothing
296    /// changed writes nothing, so this is safe on every app launch.
297    pub fn refresh(&mut self) -> Result<RefreshReport> {
298        let mut r = refresh::refresh(&self.conn, &self.vault)?;
299        let mut arm = IndexReport { documents: 0, chunks: 0, edges: 0, embedded: 0, embed_error: None };
300        self.embed_into(&mut arm, None);
301        r.embedded = arm.embedded;
302        r.embed_error = arm.embed_error;
303        Ok(r)
304    }
305    /// Reindex exactly one vault-relative path, in place. A removed file is
306    /// dropped from the index. Costs one file read, not a vault walk, and
307    /// embeds only that path's chunks.
308    pub fn reindex_path(&mut self, rel: &str) -> Result<IndexReport> {
309        let rel = rel.trim_start_matches("./").replace('\\', "/");
310        let mut r = index::reindex_path(&self.conn, &self.vault, &rel)?;
311        self.embed_into(&mut r, Some(&rel));
312        Ok(r)
313    }
314
315    /// Best-effort embedding. A dead embedder must not fail a write — the note
316    /// is already on disk and BM25 already finds it — so the error is carried
317    /// in the report instead of returned.
318    fn embed_into(&mut self, r: &mut IndexReport, only: Option<&str>) {
319        let pending = match self.embedder.clone() {
320            Some(e) => embed::embed_missing(&self.conn, e.as_ref(), 32, only),
321            None => Ok(0),
322        };
323        match pending {
324            Ok(n) => r.embedded = n,
325            Err(e) => r.embed_error = Some(e.to_string()),
326        }
327    }
328
329    /// Convenience wrapper kept for the 0.1 API.
330    pub fn search(&self, query: &str, limit: u32) -> Result<Vec<Hit>> {
331        Ok(self.search_with(&SearchParams { query: query.to_string(), limit, ..Default::default() })?.hits)
332    }
333
334    pub fn search_with(&self, p: &SearchParams) -> Result<SearchResult> {
335        let mut p = p.clone();
336        if p.embedder.as_deref() == Some("none") {
337            p.mode = Mode::Bm25;
338        }
339        sqlite::search(&self.conn, &p, self.embedder.as_deref())
340    }
341
342    /// Embed any chunk that has no vector yet. No-op without an embedder.
343    /// An embedder error surfaces here rather than storing a placeholder.
344    pub fn embed_pending(&mut self) -> Result<u64> {
345        match self.embedder.clone() {
346            Some(e) => embed::embed_missing(&self.conn, e.as_ref(), 32, None),
347            None => Ok(0),
348        }
349    }
350
351    /// The embedding space already recorded in a vault's index, without
352    /// opening a full engine and without contacting any provider. This is what
353    /// lets `auto` resolve once at init/doctor and stay resolved: ordinary
354    /// commands read the answer instead of probing a daemon on every search.
355    pub fn stored_embedder(vault: impl AsRef<Path>) -> Option<StoredEmbedder> {
356        Self::stored_embedder_in(vault.as_ref().join(location::DEFAULT_DIR))
357    }
358    /// [`Engine::stored_embedder`] for an index kept in a caller-chosen
359    /// directory, the one [`Engine::open_in`] opened.
360    pub fn stored_embedder_in(index_dir: impl AsRef<Path>) -> Option<StoredEmbedder> {
361        let db = location::db_file(index_dir.as_ref());
362        let conn = Connection::open_with_flags(&db, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY).ok()?;
363        let provider = sqlite::meta_get(&conn, "embedder")?;
364        if provider == "none" {
365            return None;
366        }
367        Some(StoredEmbedder {
368            model: sqlite::meta_get(&conn, "embed_model")?,
369            dim: sqlite::meta_get(&conn, "embed_dim")?.parse().ok()?,
370            provider,
371        })
372    }
373
374    /// Which arm is live: `none` | `ollama` | `onnx`.
375    pub fn embedder_name(&self) -> &'static str {
376        self.embedder.as_ref().map(|e| e.provider()).unwrap_or("none")
377    }
378
379    /// `lapis list`: the documents table, filtered and paged.
380    pub fn documents(&self, p: &ListParams) -> Result<Vec<Document>> {
381        sqlite::documents(&self.conn, p)
382    }
383
384    /// Whole-vault graph for the canvas: one node per indexed document plus one
385    /// per dangling link target, one edge per wikilink, degree = in + out.
386    ///
387    /// This is the **global** graph. [`Engine::neighbors`] is hop-1 and the
388    /// desktop's hop-ring walk is a local view; neither is this.
389    pub fn graph_snapshot(&self) -> Result<GraphSnapshot> {
390        graph::snapshot(&self.conn, &self.vault.display().to_string(), SNAPSHOT_NODE_CAP)
391    }
392
393    pub fn neighbors(&self, path: &str, direction: &str) -> Result<Vec<Neighbor>> {
394        graph::neighbors(&self.conn, path, direction)
395    }
396
397    pub fn health(&self) -> Result<Health> {
398        let mut h = sqlite::health(&self.conn, &self.db_path())?;
399        h.stale_documents = index::stale_count(&self.conn, &self.vault)?;
400        Ok(h)
401    }
402
403    pub fn analytics(&self, query: &str) -> Result<Analytics> {
404        analytics::run(&self.conn, query)
405    }
406}
407
408#[cfg(test)]
409mod tests;