1mod analytics;
13mod embed;
14mod graph;
15mod index;
16mod kinds;
17mod location;
18mod refresh;
19mod scope;
20mod sqlite;
21
22#[cfg(test)]
23mod search_ids;
24
25use std::path::{Path, PathBuf};
26
27use rusqlite::Connection;
28use serde::{Deserialize, Serialize};
29
30pub use analytics::{ANALYTICS_QUERIES, Analytics};
31pub use embed::Embedder;
32pub use graph::{GraphEdge, GraphNode, GraphSnapshot, Neighbor, SNAPSHOT_NODE_CAP};
33pub use kinds::{HTML, MARKDOWN, YAML, kind_for};
34pub use refresh::RefreshReport;
35pub use scope::Scope;
36
37pub const PRODUCER: &str = "lapis-lattice";
39
40#[derive(Debug, thiserror::Error)]
41pub enum Error {
42 #[error("{0}")]
43 Usage(String),
44 #[error("{0}")]
45 Io(#[from] std::io::Error),
46 #[error("sqlite: {0}")]
47 Sqlite(#[from] rusqlite::Error),
48}
49
50pub type Result<T> = std::result::Result<T, Error>;
51
52pub struct Engine {
55 vault: PathBuf,
56 db: PathBuf,
58 conn: Connection,
59 embedder: Option<std::sync::Arc<dyn Embedder>>,
60}
61
62#[derive(Debug, Clone, Serialize, Deserialize)]
63pub struct Hit {
64 pub path: String,
65 pub kind: String,
68 pub title: String,
69 #[serde(skip_serializing_if = "Option::is_none")]
70 pub heading: Option<String>,
71 #[serde(skip_serializing_if = "Option::is_none")]
72 pub snippet: Option<String>,
73 pub rank: u32,
74 pub score: f64,
75 pub domain: Option<String>,
76 pub doc_type: Option<String>,
77 #[serde(default, skip_serializing_if = "Vec::is_empty")]
78 pub tags: Vec<String>,
79 #[serde(default, skip_serializing_if = "Option::is_none")]
80 pub chunk_id: Option<i64>,
81 #[serde(default, skip_serializing_if = "Option::is_none")]
82 pub chunk_index: Option<i64>,
83 #[serde(default, skip_serializing_if = "Option::is_none")]
85 pub jev: Option<serde_json::Value>,
86}
87
88#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
92#[serde(rename_all = "lowercase")]
93pub enum Mode {
94 #[default]
95 Hybrid,
96 Bm25,
97 Vector,
98}
99
100#[derive(Debug, Clone, Default)]
102pub struct SearchParams {
103 pub query: String,
104 pub limit: u32,
105 pub offset: u32,
106 pub domain: Option<String>,
107 pub per_doc: bool,
109 pub mode: Mode,
110 pub mmr: bool,
112 pub include_archives: bool,
114 pub embedder: Option<String>,
116}
117
118#[derive(Debug, Clone, Serialize)]
119pub struct SearchResult {
120 pub hits: Vec<Hit>,
121 pub modalities: Vec<String>,
123}
124
125#[derive(Debug, Clone, Default)]
127pub struct ListParams {
128 pub domain: Option<String>,
129 pub doc_type: Option<String>,
130 pub status: Option<String>,
131 pub tag: Option<String>,
132 pub prefix: Option<String>,
133 pub limit: u32,
134 pub offset: u32,
135}
136
137#[derive(Debug, Clone, Serialize)]
139pub struct Document {
140 pub path: String,
141 pub kind: String,
142 pub title: Option<String>,
143 pub domain: Option<String>,
144 pub doc_type: Option<String>,
145 pub status: Option<String>,
146 pub priority: Option<String>,
147 pub tags: Vec<String>,
148 pub updated_at: Option<u64>,
150 pub hash: Option<String>,
152}
153
154#[derive(Debug, Clone, Serialize)]
157pub struct Graph {
158 pub built: bool,
159 pub edges: u64,
160 pub dangling_links: u64,
161}
162
163#[derive(Debug, Clone, Serialize)]
164pub struct Health {
165 pub status: String,
166 pub documents_indexed: u64,
167 #[serde(skip_serializing_if = "is_zero")]
169 pub stale_documents: u64,
170 pub db_path: String,
171 pub embedder: &'static str,
172 pub embed_model: Option<String>,
173 pub embed_dim: Option<u32>,
174 pub chunks: u64,
176 pub vectors: u64,
177 pub graph: Graph,
178}
179
180fn is_zero(n: &u64) -> bool {
181 *n == 0
182}
183
184#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
186pub struct StoredEmbedder {
187 pub provider: String,
188 pub model: String,
189 pub dim: u32,
190}
191
192#[derive(Debug, Clone, Serialize)]
193pub struct IndexReport {
194 pub documents: u64,
195 pub chunks: u64,
196 pub edges: u64,
197 pub embedded: u64,
199 pub embed_error: Option<String>,
202}
203
204impl Engine {
205 pub fn open(vault: impl AsRef<Path>) -> Result<Self> {
207 Self::open_with(vault, None)
208 }
209
210 pub fn open_with(
213 vault: impl AsRef<Path>,
214 embedder: Option<std::sync::Arc<dyn Embedder>>,
215 ) -> Result<Self> {
216 Self::open_at(vault.as_ref(), None, embedder)
217 }
218 pub fn open_in(
233 vault: impl AsRef<Path>,
234 index_dir: impl AsRef<Path>,
235 embedder: Option<std::sync::Arc<dyn Embedder>>,
236 ) -> Result<Self> {
237 Self::open_at(vault.as_ref(), Some(index_dir.as_ref()), embedder)
238 }
239 fn open_at(
240 vault: &Path,
241 index_dir: Option<&Path>,
242 embedder: Option<std::sync::Arc<dyn Embedder>>,
243 ) -> Result<Self> {
244 let vault = vault.canonicalize().unwrap_or_else(|_| vault.to_path_buf());
245 if !vault.is_dir() {
246 return Err(Error::Usage(format!("vault is not a directory: {}", vault.display())));
247 }
248 let dir = match index_dir {
249 Some(d) => location::confined_index_dir(&vault, d)?,
252 None => vault.join(location::DEFAULT_DIR),
253 };
254 if index_dir.is_some() {
255 location::create_dir_private(&dir)?;
256 } else {
257 std::fs::create_dir_all(&dir)?;
258 }
259 let db = location::db_file(&dir);
260 sqlite::register_vec();
264 let conn = Connection::open(&db)?;
265 sqlite::migrate(&conn)?;
266 sqlite::assert_producer(&conn, &db)?;
267 embed::reconcile_space(&conn, embedder.as_deref())?;
270 Ok(Self { vault, db, conn, embedder })
271 }
272
273 pub fn db_path(&self) -> PathBuf {
277 self.db.clone()
278 }
279
280 pub fn reindex(&mut self) -> Result<IndexReport> {
282 self.reindex_with_progress(&mut |_, _| {})
283 }
284
285 pub fn reindex_with_progress(&mut self, progress: &mut dyn FnMut(u64, u64)) -> Result<IndexReport> {
288 let mut r = index::reindex(&self.conn, &self.vault, progress)?;
289 self.embed_into(&mut r, None);
290 Ok(r)
291 }
292
293 pub fn refresh(&mut self) -> Result<RefreshReport> {
298 let mut r = refresh::refresh(&self.conn, &self.vault)?;
299 let mut arm = IndexReport { documents: 0, chunks: 0, edges: 0, embedded: 0, embed_error: None };
300 self.embed_into(&mut arm, None);
301 r.embedded = arm.embedded;
302 r.embed_error = arm.embed_error;
303 Ok(r)
304 }
305 pub fn reindex_path(&mut self, rel: &str) -> Result<IndexReport> {
309 let rel = rel.trim_start_matches("./").replace('\\', "/");
310 let mut r = index::reindex_path(&self.conn, &self.vault, &rel)?;
311 self.embed_into(&mut r, Some(&rel));
312 Ok(r)
313 }
314
315 fn embed_into(&mut self, r: &mut IndexReport, only: Option<&str>) {
319 let pending = match self.embedder.clone() {
320 Some(e) => embed::embed_missing(&self.conn, e.as_ref(), 32, only),
321 None => Ok(0),
322 };
323 match pending {
324 Ok(n) => r.embedded = n,
325 Err(e) => r.embed_error = Some(e.to_string()),
326 }
327 }
328
329 pub fn search(&self, query: &str, limit: u32) -> Result<Vec<Hit>> {
331 Ok(self.search_with(&SearchParams { query: query.to_string(), limit, ..Default::default() })?.hits)
332 }
333
334 pub fn search_with(&self, p: &SearchParams) -> Result<SearchResult> {
335 let mut p = p.clone();
336 if p.embedder.as_deref() == Some("none") {
337 p.mode = Mode::Bm25;
338 }
339 sqlite::search(&self.conn, &p, self.embedder.as_deref())
340 }
341
342 pub fn embed_pending(&mut self) -> Result<u64> {
345 match self.embedder.clone() {
346 Some(e) => embed::embed_missing(&self.conn, e.as_ref(), 32, None),
347 None => Ok(0),
348 }
349 }
350
351 pub fn stored_embedder(vault: impl AsRef<Path>) -> Option<StoredEmbedder> {
356 Self::stored_embedder_in(vault.as_ref().join(location::DEFAULT_DIR))
357 }
358 pub fn stored_embedder_in(index_dir: impl AsRef<Path>) -> Option<StoredEmbedder> {
361 let db = location::db_file(index_dir.as_ref());
362 let conn = Connection::open_with_flags(&db, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY).ok()?;
363 let provider = sqlite::meta_get(&conn, "embedder")?;
364 if provider == "none" {
365 return None;
366 }
367 Some(StoredEmbedder {
368 model: sqlite::meta_get(&conn, "embed_model")?,
369 dim: sqlite::meta_get(&conn, "embed_dim")?.parse().ok()?,
370 provider,
371 })
372 }
373
374 pub fn embedder_name(&self) -> &'static str {
376 self.embedder.as_ref().map(|e| e.provider()).unwrap_or("none")
377 }
378
379 pub fn documents(&self, p: &ListParams) -> Result<Vec<Document>> {
381 sqlite::documents(&self.conn, p)
382 }
383
384 pub fn graph_snapshot(&self) -> Result<GraphSnapshot> {
390 graph::snapshot(&self.conn, &self.vault.display().to_string(), SNAPSHOT_NODE_CAP)
391 }
392
393 pub fn neighbors(&self, path: &str, direction: &str) -> Result<Vec<Neighbor>> {
394 graph::neighbors(&self.conn, path, direction)
395 }
396
397 pub fn health(&self) -> Result<Health> {
398 let mut h = sqlite::health(&self.conn, &self.db_path())?;
399 h.stale_documents = index::stale_count(&self.conn, &self.vault)?;
400 Ok(h)
401 }
402
403 pub fn analytics(&self, query: &str) -> Result<Analytics> {
404 analytics::run(&self.conn, query)
405 }
406}
407
408#[cfg(test)]
409mod tests;